diff --git a/.github/workflows/hive-consensus-tests.yml b/.github/workflows/hive-consensus-tests.yml index e507dd4c96d..65575337757 100644 --- a/.github/workflows/hive-consensus-tests.yml +++ b/.github/workflows/hive-consensus-tests.yml @@ -69,7 +69,7 @@ jobs: # Output the variables echo "skip_docker_build=$skip_docker_build" >> $GITHUB_OUTPUT - echo "wait_for_docker=$wait_for_docker" >> $GITHUB_OUTPUT + echo "skip_wait_for_docker=$skip_wait_for_docker" >> $GITHUB_OUTPUT - name: Trigger Docker Build Action with Cleaned Ref if: steps.check_conditions.outputs.skip_docker_build != 'true' diff --git a/.github/workflows/nethermind-tests.yml b/.github/workflows/nethermind-tests.yml index afee5450960..3b9fba7070e 100644 --- a/.github/workflows/nethermind-tests.yml +++ b/.github/workflows/nethermind-tests.yml @@ -62,6 +62,7 @@ jobs: - Nethermind.Network.Dns.Test - Nethermind.Network.Enr.Test - Nethermind.Network.Test + - Nethermind.Optimism.Test - Nethermind.Overseer.Test - Nethermind.Runner.Test - Nethermind.Serialization.Ssz.Test diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 9773b4b600f..dd82250f539 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -23,9 +23,7 @@ on: required: true default: release type: choice - options: - - release - - debug + options: [release, debug] jobs: publish-docker: @@ -46,14 +44,16 @@ jobs: - name: Build and push image to Docker Hub (staging) run: | branch=$(echo "${{ github.ref }}" | sed -e "s/refs\/heads\///g") - original_tag=${{ github.event.inputs.tag || '$branch' }} - tag=$(echo "$original_tag" | sed 's/\//-/g') # replace '/' with '-' in tag name + tag=$(echo "${{ github.event.inputs.tag || '$branch' }}" | sed 's/\//-/g') # replace '/' with '-' + image_name=nethermindeth/${{ github.event.inputs.image-name || 'nethermind' }} build_timestamp=$(date '+%s') + echo "Building image with tag $tag" docker buildx build --platform=linux/amd64,linux/arm64 \ -f ${{ github.event.inputs.dockerfile || 'Dockerfile' }} \ - -t "nethermindeth/${{ github.event.inputs.image-name || 'nethermind' }}:$tag" \ + -t "$image_name:$tag" \ + ${{ endsWith(github.ref, '/master') && github.event_name == 'push' && '-t $image_name:master-${GITHUB_SHA:0:7}' || '' }} \ --build-arg BUILD_CONFIG=${{ github.event.inputs.build-config || 'release' }} \ --build-arg BUILD_TIMESTAMP=$build_timestamp \ --build-arg CI=$CI \ diff --git a/.github/workflows/rpc-comparison.yml b/.github/workflows/rpc-comparison.yml index a9332ca13f7..4d7a65a87be 100644 --- a/.github/workflows/rpc-comparison.yml +++ b/.github/workflows/rpc-comparison.yml @@ -49,6 +49,8 @@ jobs: verify_correctness_of_setup: name: Verify if inputs are correct runs-on: ubuntu-latest + outputs: + custom_machine_type: ${{ steps.compute_machine_type.outputs.custom_machine_type }} steps: - name: Verify if inputs are correctly applied run: | @@ -65,6 +67,21 @@ jobs: IS_PERFORMANCE_CHECK: ${{ inputs.is_performance_check }} BRANCH_TO_COMPARE: ${{ inputs.branch_to_compare }} COMPARE_WITH: ${{ inputs.compare_with }} + + - name: Compute Machine Type + id: compute_machine_type + run: | + convert_to_paprika="${{ github.event.inputs.convert_to_paprika }}" + is_performance_check="${{ github.event.inputs.is_performance_check }}" + machine_type="" + if [[ "$convert_to_paprika" == 'true' && "$is_performance_check" == 'true' ]]; then + machine_type="g7-premium-16" + elif [[ "$convert_to_paprika" == 'true' ]]; then + machine_type="g6-standard-8" + elif [[ "$is_performance_check" == 'true' ]]; then + machine_type="g7-premium-8" + fi + echo "custom_machine_type=$machine_type" >> $GITHUB_OUTPUT create_main_node: name: Create node from current branch @@ -79,10 +96,10 @@ jobs: "default_dockerfile_build_type": "release", "ssh_keys": "", "allowed_ips": "${{ inputs.allowed_ips }}", - "custom_machine_type": "${{ (inputs.convert_to_paprika == 'true' && inputs.is_performance_check == 'true' && 'g7-premium-16') || (inputs.convert_to_paprika == 'true' && 'g6-standard-8') || (inputs.is_performance_check == 'true' && 'g7-premium-8') || '' }}" + "custom_machine_type": "${{ needs.verify_correctness_of_setup.outputs.custom_machine_type }}" } non_validator_mode: true - additional_nethermind_flags: Pruning.Mode=None JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug,Admin] + additional_nethermind_flags: JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug,Admin] JsonRpc.Timeout=3600000 log=INFO nethermind_repo_ref: ${{ github.ref }} custom_run_id: ${{ github.run_id }} network: "${{ inputs.network || 'mainnet' }}" @@ -102,10 +119,10 @@ jobs: "default_dockerfile_build_type": "release", "ssh_keys": "", "allowed_ips": "${{ inputs.allowed_ips }}", - "custom_machine_type": "${{ (inputs.convert_to_paprika == 'true' && inputs.is_performance_check == 'true' && 'g7-premium-16') || (inputs.convert_to_paprika == 'true' && 'g6-standard-8') || (inputs.is_performance_check == 'true' && 'g7-premium-8') || '' }}" + "custom_machine_type": "${{ needs.verify_correctness_of_setup.outputs.custom_machine_type }}" } non_validator_mode: true - additional_nethermind_flags: Pruning.Mode=None JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug,Admin] + additional_nethermind_flags: JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug,Admin] JsonRpc.Timeout=3600000 log=INFO nethermind_repo_ref: ${{ inputs.branch_to_compare }} custom_run_id: ${{ github.run_id }} network: "${{ inputs.network || 'mainnet' }}" @@ -302,6 +319,11 @@ jobs: check_jsonrpc_responding "$url" check_chain_head "$url" fi + + # Extra wait - nodes need to process a few new blocks - nice to have at least 128 of them after StateHealing + # Adding (128 - 32) * 12 seconds (-32 because we always keep 32 blocks to be processed after healing) + echo "Waiting for (128 - 32) blocks to be synced" + sleep 1152 compare: name: Compare JSON-RPC responses between clients and versions @@ -311,7 +333,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install flood - run: pip install git+https://github.com/kamilchodola/flood.git + run: pip install --force-reinstall --no-deps git+https://github.com/kamilchodola/flood.git - name: Prepare Comparison Flags id: prep_comparison @@ -393,21 +415,21 @@ jobs: flood "$TEST" ${compare_to_other_branch_params} --rates 10 50 100 500 1000 --output "$TEST_perf_result" --duration 30 --deep-check | tee -a "$TEST_perf_result.txt" done else - echo "flood all ${compare_to_other_branch_params} --output equality_result_other --equality | tee output_other_branch.txt" - flood all ${compare_to_other_branch_params} --output equality_result_other --equality | tee output_other_branch.txt + echo "flood all ${compare_to_other_branch_params} --equality | tee output_other_branch.txt" + flood all ${compare_to_other_branch_params} --equality | tee output_other_branch.txt fi - name: Compare to INFURA Endpoint if: env.compare_to_infura == 'true' && inputs.is_performance_check != true run: | - echo "flood all ${compare_to_infura_params} --output equality_result_infura --equality | tee output_infura.txt" - flood all ${compare_to_infura_params} --output equality_result_infura --equality | tee output_infura.txt + echo "flood all ${compare_to_infura_params} --equality | tee output_infura.txt" + flood all ${compare_to_infura_params} --equality | tee output_infura.txt - name: Compare to Nethermind Archive Endpoint if: env.compare_to_archive == 'true' && inputs.is_performance_check != true run: | - echo "flood all ${compare_to_archive_params} --output equality_result_archive --equality | tee output_archive.txt" - flood all ${compare_to_archive_params} --output equality_result_archive --equality | tee output_archive.txt + echo "flood all ${compare_to_archive_params} --equality | tee output_archive.txt" + flood all ${compare_to_archive_params} --equality | tee output_archive.txt - name: Generate report run: | diff --git a/.github/workflows/run-a-single-node-from-branch.yml b/.github/workflows/run-a-single-node-from-branch.yml index 654dfc9f09c..e11cd1c8267 100644 --- a/.github/workflows/run-a-single-node-from-branch.yml +++ b/.github/workflows/run-a-single-node-from-branch.yml @@ -28,9 +28,10 @@ on: options: - lighthouse - lodestar - #- nimbus - prysm - teku + - nimbus + - nimbus_ws cl_custom_image: description: "In case of need to run non-default cl image (different than actually supported by Sedge) put it in there" default: "" diff --git a/.github/workflows/sync-testnets.yml b/.github/workflows/sync-testnets.yml index 0a8b99b9773..3a5b0bdfcdf 100644 --- a/.github/workflows/sync-testnets.yml +++ b/.github/workflows/sync-testnets.yml @@ -20,17 +20,17 @@ jobs: include: - network: "holesky" checkpoint-sync-url: "https://holesky.beaconstate.ethstaker.cc/" - cl-client: "lighthouse:sigp/lighthouse:latest" + cl-client: "lodestar:chainsafe/lodestar:latest" el-client: "nethermind:nethermindeth/nethermind:master" - agent: ubuntu-latest + agent: sync-agent-80gb - network: "chiado" checkpoint-sync-url: "http://139.144.26.89:4000/" - cl-client: "lighthouse:sigp/lighthouse:latest" + cl-client: "lodestar:chainsafe/lodestar:latest" el-client: "nethermind:nethermindeth/nethermind:master" agent: sync-agent-80gb - network: "sepolia" checkpoint-sync-url: "https://beaconstate-sepolia.chainsafe.io" - cl-client: "lighthouse:sigp/lighthouse:latest" + cl-client: "lodestar:chainsafe/lodestar:latest" el-client: "nethermind:nethermindeth/nethermind:master" agent: sync-agent-160gb name: "Run sync of ${{ matrix.network }} testnet" @@ -41,15 +41,6 @@ jobs: with: clean: true - - name: Configure settings - id: settings - run: | - echo "BUILD_TIMESTAMP=$(date '+%s')" >> $GITHUB_OUTPUT - echo "COMMIT_HASH=$(git describe --always --exclude=* --abbrev=40)" >> $GITHUB_OUTPUT - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Installing requirements run: | sudo apt-get update @@ -82,6 +73,7 @@ jobs: --el-extra-flag Sync.NonValidatorNode=true --el-extra-flag Sync.DownloadBodiesInFastSync=false \ --el-extra-flag Sync.DownloadReceiptsInFastSync=false \ --el-extra-flag JsonRpc.EnabledModules=[Eth,Subscribe,Trace,TxPool,Web3,Personal,Proof,Net,Parity,Health,Rpc,Debug] \ + --el-extra-flag Sync.SnapSync=true \ --checkpoint-sync-url=${{ matrix.checkpoint-sync-url }} echo 'Running sedge...' ./build/sedge run -p $GITHUB_WORKSPACE/sedge @@ -90,29 +82,31 @@ jobs: id: wait timeout-minutes: 180 run: | - set +e - declare -A bad_logs bad_logs["Corrupt"]=1 bad_logs["Exception"]=1 - + declare -A good_logs good_logs["Synced Chain Head"]=0 good_logs["Processed"]=0 - + declare -A required_count required_count["Synced Chain Head"]=20 - required_count["Processed"]=20 - + required_count["Processed"]=20 + counter=0 found_bad_log=false - - echo "Starting Docker logs monitoring..." + docker logs -f sedge-execution-client | while read -r line; do echo "$line" - - if [[ $found_bad_log == true ]]; then - ((counter++)) + + if [[ "$line" == *"All done"* ]]; then + echo "Unexpected termination detected: $line" + exit 1 + fi + + if [ "$found_bad_log" = true ]; then + counter=$((counter + 1)) if [ $counter -ge 100 ]; then echo "Exiting after capturing extra logs due to error." exit 1 @@ -120,21 +114,21 @@ jobs: continue fi fi - + for bad_log in "${!bad_logs[@]}"; do if [[ "$line" == *"$bad_log"* ]]; then echo "Error: $bad_log found in Docker logs." - $found_bad_log=true - continue 2 + found_bad_log=true + break fi done - + for good_log in "${!good_logs[@]}"; do if [[ "$line" == *"$good_log"* ]]; then - ((good_logs["$good_log"]++)) + good_logs["$good_log"]=$((good_logs["$good_log"]+1)) fi done - + # Check if all good logs have reached the required count all_reached_required_count=true for good_log in "${!good_logs[@]}"; do @@ -143,7 +137,7 @@ jobs: break fi done - + if $all_reached_required_count; then echo "All required logs found." exit 0 diff --git a/.github/workflows/update-docs.yml b/.github/workflows/update-docs.yml index c4d8b8ad235..b6ceb554788 100644 --- a/.github/workflows/update-docs.yml +++ b/.github/workflows/update-docs.yml @@ -39,7 +39,9 @@ jobs: mv d/docs/interacting/json-rpc-ns/eth_subscribe.md n/docgen/eth_subscribe.md mv d/docs/interacting/json-rpc-ns/eth_unsubscribe.md n/docgen/eth_unsubscribe.md mv d/docs/monitoring/metrics/metrics.md n/docgen/metrics.md - cd n/docgen && ./DocGen && cd ../.. + cd n/docgen + ./DocGen --config --jsonrpc --metrics + cd ../.. mv n/docgen/configuration.md d/docs/fundamentals/configuration.md rm -f d/docs/interacting/json-rpc-ns/*.md mv n/docgen/eth_subscribe.md d/docs/interacting/json-rpc-ns/eth_subscribe.md diff --git a/Dockerfile b/Dockerfile index 23fc81ca4ed..ccdea679109 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited # SPDX-License-Identifier: LGPL-3.0-only -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-jammy AS build +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-noble AS build ARG BUILD_CONFIG=release ARG BUILD_TIMESTAMP @@ -9,7 +9,6 @@ ARG CI ARG COMMIT_HASH ARG TARGETARCH -COPY .git .git COPY src/Nethermind src/Nethermind RUN arch=$([ "$TARGETARCH" = "amd64" ] && echo "x64" || echo "$TARGETARCH") && \ @@ -19,7 +18,7 @@ RUN arch=$([ "$TARGETARCH" = "amd64" ] && echo "x64" || echo "$TARGETARCH") && \ # A temporary symlink to support the old executable name RUN ln -s -r /publish/nethermind /publish/Nethermind.Runner -FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:8.0-jammy +FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:8.0-noble WORKDIR /nethermind diff --git a/Dockerfile.chiseled b/Dockerfile.chiseled index 2a7185f1855..936c78877dc 100644 --- a/Dockerfile.chiseled +++ b/Dockerfile.chiseled @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited # SPDX-License-Identifier: LGPL-3.0-only -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-jammy AS build +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-noble AS build ARG BUILD_CONFIG=release ARG BUILD_TIMESTAMP @@ -9,7 +9,6 @@ ARG CI ARG COMMIT_HASH ARG TARGETARCH -COPY .git .git COPY src/Nethermind src/Nethermind RUN arch=$([ "$TARGETARCH" = "amd64" ] && echo "x64" || echo "$TARGETARCH") && \ @@ -22,7 +21,7 @@ RUN cd /publish && \ mkdir logs && \ mkdir nethermind_db -FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:8.0-jammy-chiseled +FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:8.0-noble-chiseled WORKDIR /nethermind diff --git a/Dockerfile.diag b/Dockerfile.diag index 6fca05c60ff..768e8fa9c2e 100644 --- a/Dockerfile.diag +++ b/Dockerfile.diag @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited # SPDX-License-Identifier: LGPL-3.0-only -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-jammy AS build +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-noble AS build ARG BUILD_CONFIG=release ARG BUILD_TIMESTAMP @@ -9,7 +9,6 @@ ARG CI ARG COMMIT_HASH ARG TARGETARCH -COPY .git .git COPY src/Nethermind src/Nethermind RUN arch=$([ "$TARGETARCH" = "amd64" ] && echo "x64" || echo "$TARGETARCH") && \ @@ -23,7 +22,7 @@ RUN dotnet tool install -g dotnet-dump && \ dotnet tool install -g dotnet-trace && \ dotnet tool install -g JetBrains.dotTrace.GlobalTools -FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:8.0-jammy +FROM --platform=$TARGETPLATFORM mcr.microsoft.com/dotnet/aspnet:8.0-noble WORKDIR /nethermind diff --git a/src/Nethermind/Chains/base-mainnet.json b/src/Nethermind/Chains/base-mainnet.json index c4de3a61e80..c0853d933aa 100644 --- a/src/Nethermind/Chains/base-mainnet.json +++ b/src/Nethermind/Chains/base-mainnet.json @@ -6,6 +6,8 @@ "params": { "regolithTimestamp": "0x0", "bedrockBlockNumber": "0x0", + "canyonTimestamp": "0x65a01e91", + "ecotoneTimestamp": "0x65f23e01", "l1FeeRecipient": "0x420000000000000000000000000000000000001A", "l1BlockAddress": "0x4200000000000000000000000000000000000015" } @@ -47,11 +49,16 @@ "eip3529Transition": "0x0", "eip3541Transition": "0x0", - "eip1153TransitionTimestamp": "0x65F23E01", - "eip4788TransitionTimestamp": "0x65F23E01", - "eip4844TransitionTimestamp": "0x65F23E01", - "eip5656TransitionTimestamp": "0x65F23E01", - "eip6780TransitionTimestamp": "0x65F23E01", + "eip4895TransitionTimestamp": "0x65a01e91", + "eip3651TransitionTimestamp": "0x65a01e91", + "eip3855TransitionTimestamp": "0x65a01e91", + "eip3860TransitionTimestamp": "0x65a01e91", + + "eip1153TransitionTimestamp": "0x65f23e01", + "eip4788TransitionTimestamp": "0x65f23e01", + "eip4844TransitionTimestamp": "0x65f23e01", + "eip5656TransitionTimestamp": "0x65f23e01", + "eip6780TransitionTimestamp": "0x65f23e01", "terminalTotalDifficulty": "0" }, @@ -71,6 +78,16 @@ "baseFeePerGas": "0x3b9aca00", "gasLimit": "0x1c9c380" }, + "nodes": [ + "enode://ca2774c3c401325850b2477fd7d0f27911efbf79b1e8b335066516e2bd8c4c9e0ba9696a94b1cb030a88eac582305ff55e905e64fb77fe0edcd70a4e5296d3ec@34.65.175.185:30305", + "enode://dd751a9ef8912be1bfa7a5e34e2c3785cc5253110bd929f385e07ba7ac19929fb0e0c5d93f77827291f4da02b2232240fbc47ea7ce04c46e333e452f8656b667@34.65.107.0:30305", + "enode://c5d289b56a77b6a2342ca29956dfd07aadf45364dde8ab20d1dc4efd4d1bc6b4655d902501daea308f4d8950737a4e93a4dfedd17b49cd5760ffd127837ca965@34.65.202.239:30305", + "enode://87a32fd13bd596b2ffca97020e31aef4ddcc1bbd4b95bb633d16c1329f654f34049ed240a36b449fda5e5225d70fe40bc667f53c304b71f8e68fc9d448690b51@3.231.138.188:30301", + "enode://ca21ea8f176adb2e229ce2d700830c844af0ea941a1d8152a9513b966fe525e809c3a6c73a2c18a12b74ed6ec4380edf91662778fe0b79f6a591236e49e176f9@184.72.129.189:30301", + "enode://acf4507a211ba7c1e52cdf4eef62cdc3c32e7c9c47998954f7ba024026f9a6b2150cd3f0b734d9c78e507ab70d59ba61dfe5c45e1078c7ad0775fb251d7735a2@3.220.145.177:30301", + "enode://8a5a5006159bf079d06a04e5eceab2a1ce6e0f721875b2a9c96905336219dbe14203d38f70f3754686a6324f786c2f9852d8c0dd3adac2d080f4db35efc678c5@3.231.11.52:30301", + "enode://cdadbe835308ad3557f9a1de8db411da1a260a98f8421d62da90e71da66e55e98aaa8e90aa7ce01b408a54e4bd2253d701218081ded3dbe5efbbc7b41d7cef79@54.198.153.150:30301" + ], "accounts": { "0000000000000000000000000000000000000001": { "builtin": { diff --git a/src/Nethermind/Chains/base-sepolia.json b/src/Nethermind/Chains/base-sepolia.json index e095bb3dd38..7faf0bda489 100644 --- a/src/Nethermind/Chains/base-sepolia.json +++ b/src/Nethermind/Chains/base-sepolia.json @@ -7,6 +7,7 @@ "regolithTimestamp": "0x0", "bedrockBlockNumber": "0x0", "canyonTimestamp": "0x6553a790", + "ecotoneTimestamp": "0x65d62c10", "l1FeeRecipient": "0x420000000000000000000000000000000000001A", "l1BlockAddress": "0x4200000000000000000000000000000000000015", "canyonBaseFeeChangeDenominator": "250", @@ -80,6 +81,13 @@ "baseFeePerGas": "0x3b9aca00", "gasLimit": "0x17d7840" }, + "nodes": [ + "enode://2bd2e657bb3c8efffb8ff6db9071d9eb7be70d7c6d7d980ff80fc93b2629675c5f750bc0a5ef27cd788c2e491b8795a7e9a4a6e72178c14acc6753c0e5d77ae4@34.65.205.244:30305", + "enode://db8e1cab24624cc62fc35dbb9e481b88a9ef0116114cd6e41034c55b5b4f18755983819252333509bd8e25f6b12aadd6465710cd2e956558faf17672cce7551f@34.65.173.88:30305", + "enode://bfda2e0110cfd0f4c9f7aa5bf5ec66e6bd18f71a2db028d36b8bf8b0d6fdb03125c1606a6017b31311d96a36f5ef7e1ad11604d7a166745e6075a715dfa67f8a@34.65.229.245:30305", + "enode://548f715f3fc388a7c917ba644a2f16270f1ede48a5d88a4d14ea287cc916068363f3092e39936f1a3e7885198bef0e5af951f1d7b1041ce8ba4010917777e71f@18.210.176.114:30301", + "enode://6f10052847a966a725c9f4adf6716f9141155b99a0fb487fea3f51498f4c2a2cb8d534e680ee678f9447db85b93ff7c74562762c3714783a7233ac448603b25f@107.21.251.55:30301" + ], "accounts": { "0000000000000000000000000000000000000001": { "balance": "0x1", diff --git a/src/Nethermind/Chains/chiado.json b/src/Nethermind/Chains/chiado.json index e2cef850df4..5a61c9ef01e 100644 --- a/src/Nethermind/Chains/chiado.json +++ b/src/Nethermind/Chains/chiado.json @@ -98,11 +98,12 @@ "gasLimit": "0x989680" }, "nodes": [ - "enode://7dd44af6138120f328bb031eb56e00985c149319d4f1e33275b30be7fddadd8ccd9f7b9c3b35a16136a61e85b2b2d1de073f30ec1d0ddf576a33be8ff48d88d0@139.144.26.89:30303", - "enode://317b9cee65ccf1d747b00e604242bfa3ae367beee8f149e28c5b2b88820f855ea7b5a75eb5327cfc3d8ca97adbf71538468290a46592ed7009f3fb394ec752f1@139.144.26.115:30303", - "enode://b77ae97906155ebbb83fd32c87ab0aa57372a24abbd8aa4bae679f048b726de4a195709f613be4981e44b24640bc89e4824427d94e9a37afc148da8250c8ab2d@139.144.26.101:30303", - "enode://69f8abfa3b0221161f8c19014b90857a18742554af27af73fd779c486728750a0ff11b873975f104fc5276a3a7c3b5b68cb3c26c815e9f78462901895d652124@139.144.26.85:30303", - "enode://ac7fc76f9b2ab343fb2d091365a7f46d17018e525cbedfbf24b247c76657e934ef4df61cc2f6dad6bfcf722425e03e1a8a6e4e4b52743acc2319cb8ebf27d742@170.187.154.239:30303", + "enode://3c9849b809dc34c914fcdf507ff59931942bdf5cd11510782a7d5695eacd622a281ae0b46aa53b8b893e1308f94867001d0fb0b52c854f96e7ecf43490f5b7bb@139.144.26.89:30303", + "enode://1556022f95f2910ed795b80df68466284b5a7de112cb00d5f4843b486392e7e790c9d27cf358bf1e3ceff3089df36dcadae593eda9730565a7221e40a96b8cd4@139.144.26.115:30303", + "enode://b39e929805542fb141ca6946931d06fdbbbd9ea8202eb1e72d7e7484877658b5baf0d8b0a6eb86a7f34b6f5c4b15b080c807c851692baef96c6a5aeca2cbf29a@139.144.26.101:30303", + "enode://5f7074a38a84e7ed7cbb207b9b1a54b4c537c5d06ebc955c892528f95927eb63a3373e344302cb1bcc2242451899c276a21e360a4347674cb97e0b9c251c2704@139.144.26.85:30303", + "enode://9ff64d021a83c72d68e7a3cefac5ea0661071cfdeed065782418ff2b0fccaace1072644118ed9fe6a304f451a2e75e2d4c69b502b3486ce16b4c50afc347cbff@170.187.154.239:30303", + "enode://4504f03b4773251188e80d2c36186de4c2dd0e1e83aadaa1164cdae2ebc510d47a3dba6c80972ea18a71177ab3aa9883e081f5a350e8979cb7127e63bb6b81ea@139.144.173.54:30303", "enode://712144ac396fd2298b3e2559e2930d7f3a36fded3addd66955224958f1845634067717ab9522757ed2948f480fc52add5676487c8378e9011a7e2c0ac2f36cc3@3.71.132.231:30303", "enode://595160631241ea41b187b85716f9f9572a266daa940d74edbe3b83477264ce284d69208e61cf50e91641b1b4f9a03fa8e60eb73d435a84cf4616b1c969bc2512@3.69.35.13:30303", "enode://5abc2f73f81ea6b94f1e1b1e376731fc662ecd7863c4c7bc83ec307042542a64feab5af7985d52b3b1432acf3cb82460b327d0b6b70cb732afb1e5a16d6b1e58@35.206.174.92:30303", diff --git a/src/Nethermind/Chains/op-mainnet.json b/src/Nethermind/Chains/op-mainnet.json index 09cd4bdf480..2c04e2cc3c1 100644 --- a/src/Nethermind/Chains/op-mainnet.json +++ b/src/Nethermind/Chains/op-mainnet.json @@ -6,6 +6,8 @@ "params": { "regolithTimestamp": "0x0", "bedrockBlockNumber": "0x645C277", + "canyonTimestamp": "0x65a01e91", + "ecotoneTimestamp": "0x65f23e01", "l1FeeRecipient": "0x420000000000000000000000000000000000001A", "l1BlockAddress": "0x4200000000000000000000000000000000000015" } @@ -52,11 +54,16 @@ "eip3529Transition": "0x645C277", "eip3541Transition": "0x645C277", - "eip1153TransitionTimestamp": "0x65F23E01", - "eip4788TransitionTimestamp": "0x65F23E01", - "eip4844TransitionTimestamp": "0x65F23E01", - "eip5656TransitionTimestamp": "0x65F23E01", - "eip6780TransitionTimestamp": "0x65F23E01", + "eip4895TransitionTimestamp": "0x65a01e91", + "eip3651TransitionTimestamp": "0x65a01e91", + "eip3855TransitionTimestamp": "0x65a01e91", + "eip3860TransitionTimestamp": "0x65a01e91", + + "eip1153TransitionTimestamp": "0x65f23e01", + "eip4788TransitionTimestamp": "0x65f23e01", + "eip4844TransitionTimestamp": "0x65f23e01", + "eip5656TransitionTimestamp": "0x65f23e01", + "eip6780TransitionTimestamp": "0x65f23e01", "terminalTotalDifficulty": "0" }, diff --git a/src/Nethermind/Chains/op-sepolia.json b/src/Nethermind/Chains/op-sepolia.json index 1068ceb6c0e..f90ba4e5a25 100644 --- a/src/Nethermind/Chains/op-sepolia.json +++ b/src/Nethermind/Chains/op-sepolia.json @@ -7,6 +7,8 @@ "regolithTimestamp": "0x0", "bedrockBlockNumber": "0x0", "canyonTimestamp": "0x6553a790", + "ecotoneTimestamp": "0x65D62C10", + "fjordTimestamp": "0x66575100", "l1FeeRecipient": "0x420000000000000000000000000000000000001A", "l1BlockAddress": "0x4200000000000000000000000000000000000015", "canyonBaseFeeChangeDenominator": "250", @@ -50,18 +52,16 @@ "eip3198Transition": "0x0", "eip3529Transition": "0x0", "eip3541Transition": "0x0", - "eip4895TransitionTimestamp": "0x6553a790", "eip3651TransitionTimestamp": "0x6553a790", "eip3855TransitionTimestamp": "0x6553a790", "eip3860TransitionTimestamp": "0x6553a790", - "eip1153TransitionTimestamp": "0x65D62C10", "eip4788TransitionTimestamp": "0x65D62C10", "eip4844TransitionTimestamp": "0x65D62C10", "eip5656TransitionTimestamp": "0x65D62C10", "eip6780TransitionTimestamp": "0x65D62C10", - + "eip7212TransitionTimestamp": "0x66575100", "terminalTotalDifficulty": "0" }, "genesis": { @@ -82,8 +82,8 @@ }, "nodes": [ "enode://2bd2e657bb3c8efffb8ff6db9071d9eb7be70d7c6d7d980ff80fc93b2629675c5f750bc0a5ef27cd788c2e491b8795a7e9a4a6e72178c14acc6753c0e5d77ae4@34.65.205.244:30305", - "enode://db8e1cab24624cc62fc35dbb9e481b88a9ef0116114cd6e41034c55b5b4f18755983819252333509bd8e25f6b12aadd6465710cd2e956558faf17672cce7551f@34.65.173.88:30305", - "enode://bfda2e0110cfd0f4c9f7aa5bf5ec66e6bd18f71a2db028d36b8bf8b0d6fdb03125c1606a6017b31311d96a36f5ef7e1ad11604d7a166745e6075a715dfa67f8a@34.65.229.245:30305", + "enode://db8e1cab24624cc62fc35dbb9e481b88a9ef0116114cd6e41034c55b5b4f18755983819252333509bd8e25f6b12aadd6465710cd2e956558faf17672cce7551f@34.65.173.88:30305", + "enode://bfda2e0110cfd0f4c9f7aa5bf5ec66e6bd18f71a2db028d36b8bf8b0d6fdb03125c1606a6017b31311d96a36f5ef7e1ad11604d7a166745e6075a715dfa67f8a@34.65.229.245:30305", "enode://548f715f3fc388a7c917ba644a2f16270f1ede48a5d88a4d14ea287cc916068363f3092e39936f1a3e7885198bef0e5af951f1d7b1041ce8ba4010917777e71f@18.210.176.114:30301", "enode://6f10052847a966a725c9f4adf6716f9141155b99a0fb487fea3f51498f4c2a2cb8d534e680ee678f9447db85b93ff7c74562762c3714783a7233ac448603b25f@107.21.251.55:30301" ], @@ -12574,4 +12574,4 @@ "balance": "0x0" } } -} +} \ No newline at end of file diff --git a/src/Nethermind/Directory.Build.props b/src/Nethermind/Directory.Build.props index 5a068779c77..b1ac734da65 100644 --- a/src/Nethermind/Directory.Build.props +++ b/src/Nethermind/Directory.Build.props @@ -13,8 +13,8 @@ $([System.DateTimeOffset]::UtcNow.ToUnixTimeSeconds()) Demerzel Solutions Limited Nethermind - $(Commit.Substring(0, 8)) - 1.26.0 + $(Commit) + 1.27.0 unstable @@ -23,10 +23,6 @@ <_Parameter1>BuildTimestamp <_Parameter2>$(BuildTimestamp) - - <_Parameter1>Commit - <_Parameter2>$(Commit) - diff --git a/src/Nethermind/Directory.Packages.props b/src/Nethermind/Directory.Packages.props index 1f3ac180c03..5d784e7530d 100644 --- a/src/Nethermind/Directory.Packages.props +++ b/src/Nethermind/Directory.Packages.props @@ -3,62 +3,63 @@ true - - - - - - + + + + + + + - + - - + + - + - - - - - - - - + + + + + + + + + - + - + - - + + - + - - + + - - + @@ -66,14 +67,14 @@ - + - + - - + + \ No newline at end of file diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/BerlinBlockChainTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/BerlinBlockChainTests.cs new file mode 100644 index 00000000000..bc1ba343cf8 --- /dev/null +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/BerlinBlockChainTests.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Ethereum.Test.Base; +using NUnit.Framework; + +namespace Ethereum.Blockchain.Pyspec.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class BerlinBlockChainTests : BlockchainTestBase +{ + [TestCaseSource(nameof(LoadTests))] + public async Task Test(BlockchainTest test) => await RunTest(test); + + private static IEnumerable LoadTests() + { + TestsSourceLoader loader = new(new LoadPyspecTestsStrategy(), "fixtures/blockchain_tests/berlin"); + return loader.LoadTests().Cast(); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/BerlinStateTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/BerlinStateTests.cs new file mode 100644 index 00000000000..34346dc7e0f --- /dev/null +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/BerlinStateTests.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; +using Ethereum.Test.Base; +using FluentAssertions; +using NUnit.Framework; + +namespace Ethereum.Blockchain.Pyspec.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class BerlinStateTests : GeneralStateTestBase +{ + [TestCaseSource(nameof(LoadTests))] + public void Test(GeneralStateTest test) => RunTest(test).Pass.Should().BeTrue(); + + private static IEnumerable LoadTests() + { + TestsSourceLoader loader = new(new LoadPyspecTestsStrategy(), $"fixtures/state_tests/berlin"); + return loader.LoadTests().Cast(); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ByzantiumBlockChainTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ByzantiumBlockChainTests.cs new file mode 100644 index 00000000000..08c90d0e5a8 --- /dev/null +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ByzantiumBlockChainTests.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Ethereum.Test.Base; +using NUnit.Framework; + +namespace Ethereum.Blockchain.Pyspec.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class ByzantiumBlockChainTests : BlockchainTestBase +{ + [TestCaseSource(nameof(LoadTests))] + public async Task Test(BlockchainTest test) => await RunTest(test); + + private static IEnumerable LoadTests() + { + TestsSourceLoader loader = new(new LoadPyspecTestsStrategy(), "fixtures/blockchain_tests/byzantium"); + return loader.LoadTests().Cast(); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ByzantiumStateTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ByzantiumStateTests.cs new file mode 100644 index 00000000000..6ef336fcd70 --- /dev/null +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ByzantiumStateTests.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; +using Ethereum.Test.Base; +using FluentAssertions; +using NUnit.Framework; + +namespace Ethereum.Blockchain.Pyspec.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class ByzantiumStateTests : GeneralStateTestBase +{ + [TestCaseSource(nameof(LoadTests))] + public void Test(GeneralStateTest test) => RunTest(test).Pass.Should().BeTrue(); + + private static IEnumerable LoadTests() + { + TestsSourceLoader loader = new(new LoadPyspecTestsStrategy(), $"fixtures/state_tests/byzantium"); + return loader.LoadTests().Cast(); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/CancunStateTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/CancunStateTests.cs index 5a0082d4b28..737487c2740 100644 --- a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/CancunStateTests.cs +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/CancunStateTests.cs @@ -13,8 +13,6 @@ namespace Ethereum.Blockchain.Pyspec.Test; [Parallelizable(ParallelScope.All)] public class CancunStateTests : GeneralStateTestBase { - - [Explicit("We are failing some of those tests")] [TestCaseSource(nameof(LoadTests))] public void Test(GeneralStateTest test) => RunTest(test).Pass.Should().BeTrue(); diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/FrontierBlockChainTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/FrontierBlockChainTests.cs new file mode 100644 index 00000000000..c289ecc91bc --- /dev/null +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/FrontierBlockChainTests.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Ethereum.Test.Base; +using NUnit.Framework; + +namespace Ethereum.Blockchain.Pyspec.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class FrontierBlockChainTests : BlockchainTestBase +{ + [TestCaseSource(nameof(LoadTests))] + public async Task Test(BlockchainTest test) => await RunTest(test); + + private static IEnumerable LoadTests() + { + TestsSourceLoader loader = new(new LoadPyspecTestsStrategy(), "fixtures/blockchain_tests/frontier"); + return loader.LoadTests().Cast(); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/FrontierStateTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/FrontierStateTests.cs new file mode 100644 index 00000000000..89526a552bf --- /dev/null +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/FrontierStateTests.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; +using Ethereum.Test.Base; +using FluentAssertions; +using NUnit.Framework; + +namespace Ethereum.Blockchain.Pyspec.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class FrontierStateTests : GeneralStateTestBase +{ + [TestCaseSource(nameof(LoadTests))] + public void Test(GeneralStateTest test) => RunTest(test).Pass.Should().BeTrue(); + + private static IEnumerable LoadTests() + { + TestsSourceLoader loader = new(new LoadPyspecTestsStrategy(), $"fixtures/state_tests/frontier"); + return loader.LoadTests().Cast(); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/HomesteadBlockChainTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/HomesteadBlockChainTests.cs new file mode 100644 index 00000000000..e73cc62f28f --- /dev/null +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/HomesteadBlockChainTests.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Ethereum.Test.Base; +using NUnit.Framework; + +namespace Ethereum.Blockchain.Pyspec.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class HomesteadBlockChainTests : BlockchainTestBase +{ + [TestCaseSource(nameof(LoadTests))] + public async Task Test(BlockchainTest test) => await RunTest(test); + + private static IEnumerable LoadTests() + { + TestsSourceLoader loader = new(new LoadPyspecTestsStrategy(), "fixtures/blockchain_tests/homestead"); + return loader.LoadTests().Cast(); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/HomesteadStateTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/HomesteadStateTests.cs new file mode 100644 index 00000000000..48bee00d4ff --- /dev/null +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/HomesteadStateTests.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; +using Ethereum.Test.Base; +using FluentAssertions; +using NUnit.Framework; + +namespace Ethereum.Blockchain.Pyspec.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class HomesteadStateTests : GeneralStateTestBase +{ + [TestCaseSource(nameof(LoadTests))] + public void Test(GeneralStateTest test) => RunTest(test).Pass.Should().BeTrue(); + + private static IEnumerable LoadTests() + { + TestsSourceLoader loader = new(new LoadPyspecTestsStrategy(), $"fixtures/state_tests/homestead"); + return loader.LoadTests().Cast(); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/IstanbulBlockChainTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/IstanbulBlockChainTests.cs new file mode 100644 index 00000000000..5a239c1c93e --- /dev/null +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/IstanbulBlockChainTests.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Ethereum.Test.Base; +using NUnit.Framework; + +namespace Ethereum.Blockchain.Pyspec.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class IstanbulBlockChainTests : BlockchainTestBase +{ + [TestCaseSource(nameof(LoadTests))] + public async Task Test(BlockchainTest test) => await RunTest(test); + + private static IEnumerable LoadTests() + { + TestsSourceLoader loader = new(new LoadPyspecTestsStrategy(), "fixtures/blockchain_tests/istanbul"); + return loader.LoadTests().Cast(); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/IstanbulStateTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/IstanbulStateTests.cs new file mode 100644 index 00000000000..41274723230 --- /dev/null +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/IstanbulStateTests.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; +using Ethereum.Test.Base; +using FluentAssertions; +using NUnit.Framework; + +namespace Ethereum.Blockchain.Pyspec.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class IstanbulStateTests : GeneralStateTestBase +{ + [TestCaseSource(nameof(LoadTests))] + public void Test(GeneralStateTest test) => RunTest(test).Pass.Should().BeTrue(); + + private static IEnumerable LoadTests() + { + TestsSourceLoader loader = new(new LoadPyspecTestsStrategy(), $"fixtures/state_tests/istanbul"); + return loader.LoadTests().Cast(); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ParisBlockChainTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ParisBlockChainTests.cs new file mode 100644 index 00000000000..a9ff1201768 --- /dev/null +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ParisBlockChainTests.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Ethereum.Test.Base; +using NUnit.Framework; + +namespace Ethereum.Blockchain.Pyspec.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class ParisBlockChainTests : BlockchainTestBase +{ + [TestCaseSource(nameof(LoadTests))] + public async Task Test(BlockchainTest test) => await RunTest(test); + + private static IEnumerable LoadTests() + { + TestsSourceLoader loader = new(new LoadPyspecTestsStrategy(), "fixtures/blockchain_tests/paris"); + return loader.LoadTests().Cast(); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ShanghaiTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ShanghaiBlockChainTests.cs similarity index 92% rename from src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ShanghaiTests.cs rename to src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ShanghaiBlockChainTests.cs index 0c3928f6233..8a9aded47ca 100644 --- a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ShanghaiTests.cs +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ShanghaiBlockChainTests.cs @@ -11,7 +11,7 @@ namespace Ethereum.Blockchain.Pyspec.Test; [TestFixture] [Parallelizable(ParallelScope.All)] -public class ShanghaiTests : BlockchainTestBase +public class ShanghaiBlockChainTests : BlockchainTestBase { [TestCaseSource(nameof(LoadTests))] public async Task Test(BlockchainTest test) => await RunTest(test); diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ShanghaiStateTests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ShanghaiStateTests.cs new file mode 100644 index 00000000000..efab53db059 --- /dev/null +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/ShanghaiStateTests.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; +using System.Linq; +using Ethereum.Test.Base; +using FluentAssertions; +using NUnit.Framework; + +namespace Ethereum.Blockchain.Pyspec.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class ShanghaiStateTests : GeneralStateTestBase +{ + [TestCaseSource(nameof(LoadTests))] + public void Test(GeneralStateTest test) => RunTest(test).Pass.Should().BeTrue(); + + private static IEnumerable LoadTests() + { + TestsSourceLoader loader = new(new LoadPyspecTestsStrategy(), $"fixtures/state_tests/shanghai"); + return loader.LoadTests().Cast(); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Test/MetaTests.cs b/src/Nethermind/Ethereum.Blockchain.Test/MetaTests.cs index 4d6b6752274..6797ee8c60d 100644 --- a/src/Nethermind/Ethereum.Blockchain.Test/MetaTests.cs +++ b/src/Nethermind/Ethereum.Blockchain.Test/MetaTests.cs @@ -37,7 +37,7 @@ public void All_categories_are_tested() { string expectedTypeName = ExpectedTypeName(directory).Replace("-", ""); Type type = types.SingleOrDefault(t => string.Equals(t.Name, expectedTypeName, StringComparison.InvariantCultureIgnoreCase)); - if (type == null && !excludesDirectories.Contains(directory)) + if (type is null && !excludesDirectories.Contains(directory)) { if (new DirectoryInfo(directory).GetFiles().Any(f => f.Name.Contains(".resources."))) { diff --git a/src/Nethermind/Ethereum.Rlp.Test/RlpTests.cs b/src/Nethermind/Ethereum.Rlp.Test/RlpTests.cs index 2a1369a1ed6..6185329ed45 100644 --- a/src/Nethermind/Ethereum.Rlp.Test/RlpTests.cs +++ b/src/Nethermind/Ethereum.Rlp.Test/RlpTests.cs @@ -185,7 +185,7 @@ public void PerfTest() stopwatch.Stop(); Console.WriteLine($"2nd: {stopwatch.ElapsedMilliseconds}"); - if (block == null || perfBlock == null || block.Number != perfBlock.Number) + if (block is null || perfBlock is null || block.Number != perfBlock.Number) { throw new Exception(); } diff --git a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs index ad4d7078b2c..cba811e90e4 100644 --- a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs +++ b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs @@ -148,15 +148,18 @@ protected async Task RunTest(BlockchainTest test, Stopwatch? IHeaderValidator headerValidator = new HeaderValidator(blockTree, Sealer, specProvider, _logManager); IUnclesValidator unclesValidator = new UnclesValidator(blockTree, headerValidator, _logManager); IBlockValidator blockValidator = new BlockValidator(txValidator, headerValidator, unclesValidator, specProvider, _logManager); + CodeInfoRepository codeInfoRepository = new(); IVirtualMachine virtualMachine = new VirtualMachine( blockhashProvider, specProvider, + codeInfoRepository, _logManager); TransactionProcessor? txProcessor = new( specProvider, stateProvider, virtualMachine, + codeInfoRepository, _logManager); IBlockProcessor blockProcessor = new BlockProcessor( @@ -167,7 +170,6 @@ protected async Task RunTest(BlockchainTest test, Stopwatch? stateProvider), stateProvider, receiptStorage, - NullWitnessCollector.Instance, new BlockhashStore(blockTree, specProvider, stateProvider), txProcessor, _logManager); @@ -340,13 +342,13 @@ private void InitializeTestState(BlockchainTest test, IWorldState stateProvider, private List RunAssertions(BlockchainTest test, Block headBlock, IWorldState stateProvider) { - if (test.PostStateRoot != null) + if (test.PostStateRoot is not null) { return test.PostStateRoot != stateProvider.StateRoot ? new List { "state root mismatch" } : Enumerable.Empty().ToList(); } TestBlockHeaderJson testHeaderJson = (test.Blocks? - .Where(b => b.BlockHeader != null) + .Where(b => b.BlockHeader is not null) .SingleOrDefault(b => new Hash256(b.BlockHeader.Hash) == headBlock.Hash)?.BlockHeader) ?? test.GenesisBlockHeader; BlockHeader testHeader = JsonToEthereumTest.Convert(testHeaderJson); List differences = new(); diff --git a/src/Nethermind/Ethereum.Test.Base/FileTestsSource.cs b/src/Nethermind/Ethereum.Test.Base/FileTestsSource.cs index 71fa118309e..2e0baece45e 100644 --- a/src/Nethermind/Ethereum.Test.Base/FileTestsSource.cs +++ b/src/Nethermind/Ethereum.Test.Base/FileTestsSource.cs @@ -29,7 +29,7 @@ public IEnumerable LoadGeneralStateTests() return Enumerable.Empty(); } - if (_wildcard != null && !_fileName.Contains(_wildcard)) + if (_wildcard is not null && !_fileName.Contains(_wildcard)) { return Enumerable.Empty(); } @@ -52,7 +52,7 @@ public IEnumerable LoadBlockchainTests() return Enumerable.Empty(); } - if (_wildcard != null && !_fileName.Contains(_wildcard)) + if (_wildcard is not null && !_fileName.Contains(_wildcard)) { return Enumerable.Empty(); } diff --git a/src/Nethermind/Ethereum.Test.Base/GeneralStateTest.cs b/src/Nethermind/Ethereum.Test.Base/GeneralStateTest.cs index d3190aa83bf..09158536941 100644 --- a/src/Nethermind/Ethereum.Test.Base/GeneralStateTest.cs +++ b/src/Nethermind/Ethereum.Test.Base/GeneralStateTest.cs @@ -33,6 +33,7 @@ public class GeneralStateTest : IEthereumTest public Hash256? CurrentRandom { get; set; } public Hash256? CurrentBeaconRoot { get; set; } public Hash256? CurrentWithdrawalsRoot { get; set; } + public ulong? CurrentExcessBlobGas { get; set; } public UInt256? ParentBlobGasUsed { get; set; } public UInt256? ParentExcessBlobGas { get; set; } diff --git a/src/Nethermind/Ethereum.Test.Base/GeneralStateTestEnvJson.cs b/src/Nethermind/Ethereum.Test.Base/GeneralStateTestEnvJson.cs index f94f6015a02..a82748257d9 100644 --- a/src/Nethermind/Ethereum.Test.Base/GeneralStateTestEnvJson.cs +++ b/src/Nethermind/Ethereum.Test.Base/GeneralStateTestEnvJson.cs @@ -19,6 +19,7 @@ public class GeneralStateTestEnvJson public Hash256? CurrentRandom { get; set; } public Hash256? CurrentBeaconRoot { get; set; } public Hash256? CurrentWithdrawalsRoot { get; set; } + public ulong? CurrentExcessBlobGas { get; set; } public UInt256? ParentBlobGasUsed { get; set; } public UInt256? ParentExcessBlobGas { get; set; } } diff --git a/src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs b/src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs index 35b1533d69b..9965d252324 100644 --- a/src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs +++ b/src/Nethermind/Ethereum.Test.Base/GeneralTestBase.cs @@ -4,11 +4,14 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using Nethermind.Blockchain; +using Nethermind.Consensus.Ethash; using Nethermind.Consensus.Validators; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Core.Specs; +using Nethermind.Core.Test.Builders; using Nethermind.Crypto; using Nethermind.Db; using Nethermind.Int256; @@ -30,6 +33,7 @@ public abstract class GeneralStateTestBase private static ILogger _logger = new(new ConsoleAsyncLogger(LogLevel.Info)); private static ILogManager _logManager = LimboLogs.Instance; private static UInt256 _defaultBaseFeeForStateTest = 0xA; + private readonly TxValidator _txValidator = new(MainnetSpecProvider.Instance.ChainId); [SetUp] public void Setup() @@ -67,15 +71,18 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer) TrieStore trieStore = new(stateDb, _logManager); WorldState stateProvider = new(trieStore, codeDb, _logManager); IBlockhashProvider blockhashProvider = new TestBlockhashProvider(); + CodeInfoRepository codeInfoRepository = new(); IVirtualMachine virtualMachine = new VirtualMachine( blockhashProvider, specProvider, + codeInfoRepository, _logManager); TransactionProcessor transactionProcessor = new( specProvider, stateProvider, virtualMachine, + codeInfoRepository, _logManager); InitializeTestState(test, stateProvider, specProvider); @@ -96,13 +103,16 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer) header.MixHash = test.CurrentRandom; header.WithdrawalsRoot = test.CurrentWithdrawalsRoot; header.ParentBeaconBlockRoot = test.CurrentBeaconRoot; - header.ExcessBlobGas = 0; + header.ExcessBlobGas = test.CurrentExcessBlobGas ?? (test.Fork is Cancun ? 0ul : null); + header.BlobGasUsed = BlobGasCalculator.CalculateBlobGas(test.Transaction); header.RequestsRoot = test.RequestsRoot; Stopwatch stopwatch = Stopwatch.StartNew(); - TxValidator? txValidator = new((MainnetSpecProvider.Instance.ChainId)); IReleaseSpec? spec = specProvider.GetSpec((ForkActivation)test.CurrentNumber); - if (test.Transaction.ChainId == null) + + if (spec is Cancun) KzgPolynomialCommitments.InitializeAsync(); + + if (test.Transaction.ChainId is null) test.Transaction.ChainId = MainnetSpecProvider.Instance.ChainId; if (test.ParentBlobGasUsed is not null && test.ParentExcessBlobGas is not null) { @@ -122,7 +132,11 @@ protected EthereumTestResult RunTest(GeneralStateTest test, ITxTracer txTracer) }; header.ExcessBlobGas = BlobGasCalculator.CalculateExcessBlobGas(parent, spec); } - bool isValid = txValidator.IsWellFormed(test.Transaction, spec); + + Block block = Build.A.Block.WithTransactions(test.Transaction).WithHeader(header).TestObject; + + bool isValid = _txValidator.IsWellFormed(test.Transaction, spec) && IsValidBlock(block, specProvider); + if (isValid) transactionProcessor.Execute(test.Transaction, new BlockExecutionContext(header), txTracer); stopwatch.Stop(); @@ -168,6 +182,22 @@ private static void InitializeTestState(GeneralStateTest test, WorldState stateP stateProvider.Reset(); } + private bool IsValidBlock(Block block, ISpecProvider specProvider) + { + IBlockTree blockTree = Build.A.BlockTree() + .WithSpecProvider(specProvider) + .WithoutSettingHead + .TestObject; + + var difficultyCalculator = new EthashDifficultyCalculator(specProvider); + var sealer = new EthashSealValidator(_logManager, difficultyCalculator, new CryptoRandom(), new Ethash(_logManager), Timestamper.Default); + IHeaderValidator headerValidator = new HeaderValidator(blockTree, sealer, specProvider, _logManager); + IUnclesValidator unclesValidator = new UnclesValidator(blockTree, headerValidator, _logManager); + IBlockValidator blockValidator = new BlockValidator(_txValidator, headerValidator, unclesValidator, specProvider, _logManager); + + return blockValidator.ValidateOrphanedBlock(block, out _); + } + private List RunAssertions(GeneralStateTest test, IWorldState stateProvider) { List differences = new(); diff --git a/src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs b/src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs index f745b72618d..5e33bde1488 100644 --- a/src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs +++ b/src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs @@ -85,7 +85,7 @@ private static ForkActivation TransitionForkActivation(string transitionInfo) public static BlockHeader Convert(TestBlockHeaderJson? headerJson) { - if (headerJson == null) + if (headerJson is null) { throw new InvalidDataException("Header JSON was null when constructing test."); } @@ -151,7 +151,7 @@ public static Transaction Convert(PostStateJson postStateJson, TransactionJson t else transaction.AccessList = null; - if (transactionJson.MaxFeePerGas != null) + if (transactionJson.MaxFeePerGas is not null) transaction.Type = TxType.EIP1559; if (transaction.BlobVersionedHashes?.Length > 0) @@ -202,7 +202,7 @@ private static AccountState Convert(AccountStateJson accountStateJson) public static IEnumerable Convert(string name, GeneralStateTestJson testJson) { - if (testJson.LoadFailure != null) + if (testJson.LoadFailure is not null) { return Enumerable.Repeat(new GeneralStateTest { Name = name, LoadFailure = testJson.LoadFailure }, 1); } @@ -234,6 +234,7 @@ public static IEnumerable Convert(string name, GeneralStateTes test.CurrentRandom = testJson.Env.CurrentRandom; test.CurrentBeaconRoot = testJson.Env.CurrentBeaconRoot; test.CurrentWithdrawalsRoot = testJson.Env.CurrentWithdrawalsRoot; + test.CurrentExcessBlobGas = testJson.Env.CurrentExcessBlobGas; test.ParentBlobGasUsed = testJson.Env.ParentBlobGasUsed; test.ParentExcessBlobGas = testJson.Env.ParentExcessBlobGas; test.PostReceiptsRoot = stateJson.Logs; @@ -251,7 +252,7 @@ public static IEnumerable Convert(string name, GeneralStateTes public static BlockchainTest Convert(string name, BlockchainTestJson testJson) { - if (testJson.LoadFailure != null) + if (testJson.LoadFailure is not null) { return new BlockchainTest { Name = name, LoadFailure = testJson.LoadFailure }; } @@ -262,13 +263,13 @@ public static BlockchainTest Convert(string name, BlockchainTestJson testJson) test.NetworkAfterTransition = testJson.EthereumNetworkAfterTransition; test.TransitionForkActivation = testJson.TransitionForkActivation; test.LastBlockHash = new Hash256(testJson.LastBlockHash); - test.GenesisRlp = testJson.GenesisRlp == null ? null : new Rlp(Bytes.FromHexString(testJson.GenesisRlp)); + test.GenesisRlp = testJson.GenesisRlp is null ? null : new Rlp(Bytes.FromHexString(testJson.GenesisRlp)); test.GenesisBlockHeader = testJson.GenesisBlockHeader; test.Blocks = testJson.Blocks; test.Pre = testJson.Pre.ToDictionary(p => new Address(p.Key), p => Convert(p.Value)); HalfBlockchainTestJson half = testJson as HalfBlockchainTestJson; - if (half != null) + if (half is not null) { test.PostStateRoot = half.PostState; } diff --git a/src/Nethermind/Ethereum.Test.Base/TestLoader.cs b/src/Nethermind/Ethereum.Test.Base/TestLoader.cs index a3670fe5cba..4a910f73123 100644 --- a/src/Nethermind/Ethereum.Test.Base/TestLoader.cs +++ b/src/Nethermind/Ethereum.Test.Base/TestLoader.cs @@ -58,7 +58,7 @@ public static IEnumerable LoadFromFile( Assembly assembly = typeof(TTest).Assembly; string[] resourceNames = assembly.GetManifestResourceNames(); string resourceName = resourceNames.SingleOrDefault(r => r.Contains(testFileName)); - if (resourceName == null) + if (resourceName is null) { throw new ArgumentException($"Cannot find test resource: {testFileName}"); } diff --git a/src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs b/src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs index 9d005329538..23682737ab7 100644 --- a/src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs +++ b/src/Nethermind/Ethereum.Transaction.Test/TransactionTests.cs @@ -57,7 +57,7 @@ private static IEnumerable LoadTests(string testSet) { TransactionJson transactionJson = byName.Value.Transaction; TransactionTest test; - if (transactionJson != null) + if (transactionJson is not null) { test = new ValidTransactionTest(byDir.Key, byName.Key, byName.Value.Rlp); ValidTransactionTest validTest = (ValidTransactionTest)test; @@ -165,7 +165,7 @@ private void RunTest(TransactionTest test, IReleaseSpec spec) } catch (Exception) { - if (validTest == null) + if (validTest is null) { return; } @@ -177,7 +177,7 @@ private void RunTest(TransactionTest test, IReleaseSpec spec) TxValidator validator = new(useChainId ? BlockchainIds.Mainnet : 0UL); - if (validTest != null) + if (validTest is not null) { Assert.That(transaction.Value, Is.EqualTo(validTest.Value), "value"); Assert.That(transaction.Data.AsArray(), Is.EqualTo(validTest.Data), "data"); diff --git a/src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.TestAccountAbstractionRpcBlockchain.cs b/src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.TestAccountAbstractionRpcBlockchain.cs index 0f2e685dc5c..9403f6b893f 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.TestAccountAbstractionRpcBlockchain.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.TestAccountAbstractionRpcBlockchain.cs @@ -122,19 +122,18 @@ protected override IBlockProducer CreateTestBlockProducer(TxPoolTxSource txPoolT UserOperationTxSource = new(UserOperationTxBuilder, UserOperationPool, UserOperationSimulator, SpecProvider, State, Signer, LogManager.GetClassLogger()); - PostMergeBlockProducer CreatePostMergeBlockProducer(IBlockProductionTrigger blockProductionTrigger, - ITxSource? txSource = null) + PostMergeBlockProducer CreatePostMergeBlockProducer(ITxSource? txSource = null) { var blockProducerEnv = blockProducerEnvFactory.Create(txSource); return new PostMergeBlockProducerFactory(SpecProvider, SealEngine, Timestamper, blocksConfig, LogManager, GasLimitCalculator).Create( - blockProducerEnv, blockProductionTrigger); + blockProducerEnv); } - IBlockProducer blockProducer = - CreatePostMergeBlockProducer(BlockProductionTrigger, UserOperationTxSource); + IBlockProducer blockProducer = CreatePostMergeBlockProducer(UserOperationTxSource); - blockProducer.BlockProduced += OnBlockProduced; + BlockProducerRunner = new StandardBlockProducerRunner(BlockProductionTrigger, BlockTree, blockProducer); + BlockProducerRunner.BlockProduced += OnBlockProduced; return blockProducer; } @@ -190,7 +189,6 @@ protected override BlockProcessor CreateBlockProcessor() new BlockProcessor.BlockValidationTransactionsExecutor(TxProcessor, State), State, ReceiptStorage, - NullWitnessCollector.Instance, new BlockhashStore(BlockTree, SpecProvider, State), TxProcessor, LogManager); diff --git a/src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.cs b/src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.cs index a08ba710e10..6862acc8a0b 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction.Test/AccountAbstractionRpcModuleTests.cs @@ -18,6 +18,7 @@ using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; using Nethermind.Crypto; +using Nethermind.Facade.Eth; using Nethermind.Int256; using Nethermind.JsonRpc.Data; using Nethermind.JsonRpc.Test.Modules; diff --git a/src/Nethermind/Nethermind.AccountAbstraction.Test/Network/UserOperationsMessageSerializerTests.cs b/src/Nethermind/Nethermind.AccountAbstraction.Test/Network/UserOperationsMessageSerializerTests.cs index 26ffda723f3..3532209a60c 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction.Test/Network/UserOperationsMessageSerializerTests.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction.Test/Network/UserOperationsMessageSerializerTests.cs @@ -16,10 +16,10 @@ namespace Nethermind.AccountAbstraction.Test.Network [TestFixture, Parallelizable(ParallelScope.All)] public class UserOperationsMessageSerializerTests { - [SetUp] - public void Setup() + [OneTimeSetUp] + public void OneTimeSetUp() { - Rlp.RegisterDecoders(typeof(UserOperationDecoder).Assembly); + Rlp.RegisterDecoders(typeof(UserOperationDecoder).Assembly, true); } [Test] diff --git a/src/Nethermind/Nethermind.AccountAbstraction/AccountAbstractionPlugin.cs b/src/Nethermind/Nethermind.AccountAbstraction/AccountAbstractionPlugin.cs index 1a1198df67d..327b7715335 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction/AccountAbstractionPlugin.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction/AccountAbstractionPlugin.cs @@ -123,7 +123,7 @@ private UserOperationSimulator UserOperationSimulator(Address entryPoint) ReadOnlyTxProcessingEnvFactory readOnlyTxProcessingEnvFactory = new( getFromApi.WorldStateManager!, - getFromApi.BlockTree, + getFromApi.BlockTree!, getFromApi.SpecProvider, getFromApi.LogManager); @@ -348,7 +348,7 @@ public ValueTask DisposeAsync() return ValueTask.CompletedTask; } - public Task InitBlockProducer(IBlockProducerFactory consensusPlugin, IBlockProductionTrigger blockProductionTrigger, ITxSource? additionalTxSource) + public IBlockProducer InitBlockProducer(IBlockProducerFactory consensusPlugin, ITxSource? additionalTxSource) { if (!Enabled) throw new InvalidOperationException("Account Abstraction plugin is disabled"); @@ -378,9 +378,7 @@ public Task InitBlockProducer(IBlockProducerFactory consensusPlu $"Account Abstraction Plugin: Miner ({_nethermindApi.EngineSigner!.Address}) Ether balance adequate - {minerBalance / 1.Ether()} Ether"); } - IManualBlockProductionTrigger trigger = new BuildBlocksWhenRequested(); - - return consensusPlugin.InitBlockProducer(trigger, UserOperationTxSource); + return consensusPlugin.InitBlockProducer(UserOperationTxSource); } public bool MevPluginEnabled => _nethermindApi.Config().Enabled; diff --git a/src/Nethermind/Nethermind.AccountAbstraction/AccountAbstractionRpcModule.cs b/src/Nethermind/Nethermind.AccountAbstraction/AccountAbstractionRpcModule.cs index c25d80ff850..e7629f3d75a 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction/AccountAbstractionRpcModule.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction/AccountAbstractionRpcModule.cs @@ -19,7 +19,7 @@ public class AccountAbstractionRpcModule : IAccountAbstractionRpcModule static AccountAbstractionRpcModule() { - Rlp.RegisterDecoders(typeof(UserOperationDecoder).Assembly); + Rlp.RegisterDecoders(typeof(UserOperationDecoder).Assembly, true); } public AccountAbstractionRpcModule(IDictionary userOperationPool, Address[] supportedEntryPoints) diff --git a/src/Nethermind/Nethermind.AccountAbstraction/Broadcaster/PeerInfo.cs b/src/Nethermind/Nethermind.AccountAbstraction/Broadcaster/PeerInfo.cs index 487e62bbc08..41cb3d53438 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction/Broadcaster/PeerInfo.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction/Broadcaster/PeerInfo.cs @@ -12,7 +12,7 @@ public class PeerInfo : IUserOperationPoolPeer { private IUserOperationPoolPeer Peer { get; } - private LruKeyCache NotifiedUserOperations { get; } = new(MemoryAllowance.MemPoolSize, "notifiedUserOperations"); + private LruKeyCacheLowObject NotifiedUserOperations { get; } = new(MemoryAllowance.MemPoolSize, "notifiedUserOperations"); public PeerInfo(IUserOperationPoolPeer peer) { diff --git a/src/Nethermind/Nethermind.AccountAbstraction/Executor/AABlockProducerTransactionsExecutorFactory.cs b/src/Nethermind/Nethermind.AccountAbstraction/Executor/AABlockProducerTransactionsExecutorFactory.cs index 8dd11b9d59c..42f344bc7fd 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction/Executor/AABlockProducerTransactionsExecutorFactory.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction/Executor/AABlockProducerTransactionsExecutorFactory.cs @@ -6,6 +6,7 @@ using Nethermind.Consensus.Producers; using Nethermind.Core; using Nethermind.Core.Specs; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; namespace Nethermind.AccountAbstraction.Executor @@ -25,10 +26,10 @@ public AABlockProducerTransactionsExecutorFactory(ISpecProvider specProvider, IL _entryPointAddresses = entryPointAddresses; } - public IBlockProcessor.IBlockTransactionsExecutor Create(ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv) + public IBlockProcessor.IBlockTransactionsExecutor Create(IReadOnlyTxProcessingScope readOnlyTxProcessingEnv) => new AABlockProducerTransactionsExecutor( readOnlyTxProcessingEnv.TransactionProcessor, - readOnlyTxProcessingEnv.StateProvider, + readOnlyTxProcessingEnv.WorldState, _specProvider, _logManager, _signer, diff --git a/src/Nethermind/Nethermind.AccountAbstraction/Executor/IUserOperationSimulator.cs b/src/Nethermind/Nethermind.AccountAbstraction/Executor/IUserOperationSimulator.cs index 6781e276de8..d4af7269707 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction/Executor/IUserOperationSimulator.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction/Executor/IUserOperationSimulator.cs @@ -18,7 +18,7 @@ ResultWrapper Simulate(UserOperation userOperation, UInt256? timestamp = null, CancellationToken cancellationToken = default); - BlockchainBridge.CallOutput EstimateGas(BlockHeader header, Transaction tx, + CallOutput EstimateGas(BlockHeader header, Transaction tx, CancellationToken cancellationToken); } } diff --git a/src/Nethermind/Nethermind.AccountAbstraction/Executor/UserOperationSimulator.cs b/src/Nethermind/Nethermind.AccountAbstraction/Executor/UserOperationSimulator.cs index e2bd4f8a0b4..f8835cabfb1 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction/Executor/UserOperationSimulator.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction/Executor/UserOperationSimulator.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using Nethermind.Abi; using Nethermind.AccountAbstraction.Data; @@ -86,14 +87,14 @@ public ResultWrapper Simulate(UserOperation userOperation, } IEip1559Spec specFor1559 = _specProvider.GetSpecFor1559(parent.Number + 1); - ReadOnlyTxProcessingEnv txProcessingEnv = _readOnlyTxProcessingEnvFactory.Create(); - ITransactionProcessor transactionProcessor = txProcessingEnv.Build(_stateProvider.StateRoot); + IReadOnlyTxProcessorSource processorSource = _readOnlyTxProcessingEnvFactory.Create(); + using IReadOnlyTxProcessingScope scope = processorSource.Build(_stateProvider.StateRoot); // wrap userOp into a tx calling the simulateWallet function off-chain from zero-address (look at EntryPoint.sol for more context) Transaction simulateValidationTransaction = BuildSimulateValidationTransaction(userOperation, parent, specFor1559); - UserOperationSimulationResult simulationResult = SimulateValidation(simulateValidationTransaction, userOperation, parent, transactionProcessor); + UserOperationSimulationResult simulationResult = SimulateValidation(simulateValidationTransaction, userOperation, parent, scope.TransactionProcessor); if (!simulationResult.Success) return ResultWrapper.Fail(simulationResult.Error ?? "unknown simulation failure"); @@ -185,24 +186,24 @@ private Transaction BuildSimulateValidationTransaction( } [Todo("Refactor once BlockchainBridge is separated")] - public BlockchainBridge.CallOutput EstimateGas(BlockHeader header, Transaction tx, CancellationToken cancellationToken) + public CallOutput EstimateGas(BlockHeader header, Transaction tx, CancellationToken cancellationToken) { - ReadOnlyTxProcessingEnv txProcessingEnv = _readOnlyTxProcessingEnvFactory.Create(); - using IReadOnlyTransactionProcessor transactionProcessor = txProcessingEnv.Build(header.StateRoot!); + IReadOnlyTxProcessorSource txProcessorSource = _readOnlyTxProcessingEnvFactory.Create(); + using IReadOnlyTxProcessingScope scope = txProcessorSource.Build(header.StateRoot!); EstimateGasTracer estimateGasTracer = new(); (bool Success, string Error) tryCallResult = TryCallAndRestore( - transactionProcessor, + scope.TransactionProcessor, header, Math.Max(header.Timestamp + 1, _timestamper.UnixTime.Seconds), tx, true, estimateGasTracer.WithCancellation(cancellationToken)); - GasEstimator gasEstimator = new(transactionProcessor, _stateProvider, _specProvider, _blocksConfig); + GasEstimator gasEstimator = new(scope.TransactionProcessor, _stateProvider, _specProvider, _blocksConfig); long estimate = gasEstimator.Estimate(tx, header, estimateGasTracer, GasEstimator.DefaultErrorMargin, cancellationToken); - return new BlockchainBridge.CallOutput + return new CallOutput { Error = tryCallResult.Success ? estimateGasTracer.Error : tryCallResult.Error, GasSpent = estimate, diff --git a/src/Nethermind/Nethermind.AccountAbstraction/Source/UserOperationTxSource.cs b/src/Nethermind/Nethermind.AccountAbstraction/Source/UserOperationTxSource.cs index ec6def2cd1e..4dac4b3fe47 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction/Source/UserOperationTxSource.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction/Source/UserOperationTxSource.cs @@ -172,7 +172,7 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi // TODO: Remove logging, just for testing _logger.Info($"Constructed tx from {userOperationsToInclude!.Count} userOperations: {userOperationTransaction.Hash}"); - BlockchainBridge.CallOutput callOutput = _userOperationSimulators[entryPoint].EstimateGas(parent, userOperationTransaction, CancellationToken.None); + CallOutput callOutput = _userOperationSimulators[entryPoint].EstimateGas(parent, userOperationTransaction, CancellationToken.None); FailedOp? failedOp = txBuilder.DecodeEntryPointOutputError(callOutput.OutputData); if (failedOp is not null) { diff --git a/src/Nethermind/Nethermind.Api/Extensions/IConsensusPlugin.cs b/src/Nethermind/Nethermind.Api/Extensions/IConsensusPlugin.cs index 6b140a99d39..9a0be89152e 100644 --- a/src/Nethermind/Nethermind.Api/Extensions/IConsensusPlugin.cs +++ b/src/Nethermind/Nethermind.Api/Extensions/IConsensusPlugin.cs @@ -15,15 +15,9 @@ public interface IConsensusPlugin : INethermindPlugin, IBlockProducerFactory { string SealEngineType { get; } - /// - /// Default block production trigger for this consensus plugin. - /// - /// - /// Needed when this plugin is used in combination with other plugin that affects block production like MEV plugin. - /// - IBlockProductionTrigger DefaultBlockProductionTrigger { get; } - INethermindApi CreateApi(IConfigProvider configProvider, IJsonSerializer jsonSerializer, ILogManager logManager, ChainSpec chainSpec) => new NethermindApi(configProvider, jsonSerializer, logManager, chainSpec); + + IBlockProducerRunner CreateBlockProducerRunner(); } } diff --git a/src/Nethermind/Nethermind.Api/Extensions/IConsensusWrapperPlugin.cs b/src/Nethermind/Nethermind.Api/Extensions/IConsensusWrapperPlugin.cs index 33eb6c84a09..b81722632bc 100644 --- a/src/Nethermind/Nethermind.Api/Extensions/IConsensusWrapperPlugin.cs +++ b/src/Nethermind/Nethermind.Api/Extensions/IConsensusWrapperPlugin.cs @@ -3,14 +3,15 @@ using System.Threading.Tasks; using Nethermind.Consensus; -using Nethermind.Consensus.Producers; using Nethermind.Consensus.Transactions; namespace Nethermind.Api.Extensions { public interface IConsensusWrapperPlugin : INethermindPlugin { - Task InitBlockProducer(IBlockProducerFactory baseBlockProducerFactory, IBlockProductionTrigger blockProductionTrigger, ITxSource? txSource); + IBlockProducer InitBlockProducer(IBlockProducerFactory baseBlockProducerFactory, ITxSource? txSource); + + IBlockProducerRunner InitBlockProducerRunner(IBlockProducerRunner baseRunner) => baseRunner; /// /// Priorities for ordering multiple plugin. Only used to determine the wrapping order of block production. diff --git a/src/Nethermind/Nethermind.Api/Extensions/INethermindPlugin.cs b/src/Nethermind/Nethermind.Api/Extensions/INethermindPlugin.cs new file mode 100644 index 00000000000..ad1c8c28027 --- /dev/null +++ b/src/Nethermind/Nethermind.Api/Extensions/INethermindPlugin.cs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading.Tasks; + +namespace Nethermind.Api.Extensions; + +public interface INethermindPlugin : IAsyncDisposable +{ + string Name { get; } + + string Description { get; } + + string Author { get; } + + void InitRlpDecoders(INethermindApi api) { } + + Task Init(INethermindApi nethermindApi) => Task.CompletedTask; + + Task InitNetworkProtocol() => Task.CompletedTask; + + Task InitRpcModules() => Task.CompletedTask; + + bool MustInitialize => false; +} diff --git a/src/Nethermind/Nethermind.Api/Extensions/IPlugin.cs b/src/Nethermind/Nethermind.Api/Extensions/IPlugin.cs deleted file mode 100644 index 95c9e13ad81..00000000000 --- a/src/Nethermind/Nethermind.Api/Extensions/IPlugin.cs +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using System.Threading.Tasks; - -namespace Nethermind.Api.Extensions -{ - public interface INethermindPlugin : IAsyncDisposable - { - string Name { get; } - - string Description { get; } - - string Author { get; } - - Task Init(INethermindApi nethermindApi); - - Task InitNetworkProtocol(); - - Task InitRpcModules(); - - bool MustInitialize { get => false; } - } -} diff --git a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs index 7dd4d5b01de..496f107e7ea 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs @@ -38,6 +38,7 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory IBlockProcessingQueue? BlockProcessingQueue { get; set; } IBlockProcessor? MainBlockProcessor { get; set; } IBlockProducer? BlockProducer { get; set; } + IBlockProducerRunner? BlockProducerRunner { get; set; } IBlockValidator? BlockValidator { get; set; } IEnode? Enode { get; set; } IFilterStore? FilterStore { get; set; } @@ -70,8 +71,6 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory ITxPool? TxPool { get; set; } ITxPoolInfoProvider? TxPoolInfoProvider { get; set; } CompositeTxGossipPolicy TxGossipPolicy { get; } - IWitnessCollector? WitnessCollector { get; set; } - IWitnessRepository? WitnessRepository { get; set; } IHealthHintService? HealthHintService { get; set; } IRpcCapabilitiesProvider? RpcCapabilitiesProvider { get; set; } ITransactionComparerProvider? TransactionComparerProvider { get; set; } diff --git a/src/Nethermind/Nethermind.Api/NethermindApi.cs b/src/Nethermind/Nethermind.Api/NethermindApi.cs index 97e16f3041b..e07aa13ba9e 100644 --- a/src/Nethermind/Nethermind.Api/NethermindApi.cs +++ b/src/Nethermind/Nethermind.Api/NethermindApi.cs @@ -31,6 +31,7 @@ using Nethermind.Evm.TransactionProcessing; using Nethermind.Facade; using Nethermind.Facade.Eth; +using Nethermind.Facade.Simulate; using Nethermind.Grpc; using Nethermind.JsonRpc; using Nethermind.JsonRpc.Modules; @@ -81,11 +82,20 @@ public IBlockchainBridge CreateBlockchainBridge() SpecProvider, LogManager); + SimulateReadOnlyBlocksProcessingEnvFactory simulateReadOnlyBlocksProcessingEnvFactory = + new SimulateReadOnlyBlocksProcessingEnvFactory( + WorldStateManager!, + readOnlyTree, + DbProvider!, + SpecProvider!, + LogManager); + IMiningConfig miningConfig = ConfigProvider.GetConfig(); IBlocksConfig blocksConfig = ConfigProvider.GetConfig(); return new BlockchainBridge( readOnlyTxProcessingEnv, + simulateReadOnlyBlocksProcessingEnvFactory, TxPool, ReceiptFinder, FilterStore, @@ -106,6 +116,7 @@ public IBlockchainBridge CreateBlockchainBridge() public IBlockProcessingQueue? BlockProcessingQueue { get; set; } public IBlockProcessor? MainBlockProcessor { get; set; } public IBlockProducer? BlockProducer { get; set; } + public IBlockProducerRunner? BlockProducerRunner { get; set; } public IBlockTree? BlockTree { get; set; } public IBlockValidator? BlockValidator { get; set; } public IBloomStorage? BloomStorage { get; set; } @@ -146,8 +157,6 @@ public IBlockchainBridge CreateBlockchainBridge() public IProtocolsManager? ProtocolsManager { get; set; } public IProtocolValidator? ProtocolValidator { get; set; } public IReceiptStorage? ReceiptStorage { get; set; } - public IWitnessCollector? WitnessCollector { get; set; } - public IWitnessRepository? WitnessRepository { get; set; } public IReceiptFinder? ReceiptFinder { get; set; } public IReceiptMonitor? ReceiptMonitor { get; set; } public IRewardCalculatorSource? RewardCalculatorSource { get; set; } = NoBlockRewards.Instance; diff --git a/src/Nethermind/Nethermind.AuRa.Test/AuRaBlockProducerTests.cs b/src/Nethermind/Nethermind.AuRa.Test/AuRaBlockProducerTests.cs index b8494907bf7..b2fe5479336 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/AuRaBlockProducerTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/AuRaBlockProducerTests.cs @@ -43,6 +43,7 @@ private class Context public IAuRaStepCalculator AuRaStepCalculator { get; } public Address NodeAddress { get; } public AuRaBlockProducer AuRaBlockProducer { get; private set; } + public IBlockProducerRunner BlockProducerRunner { get; set; } public TimeSpan StepDelay { get; } public Context() @@ -97,7 +98,6 @@ public void InitProducer(IAuraConfig auraConfig) AuRaBlockProducer = new AuRaBlockProducer( TransactionSource, BlockchainProcessor, - onlyWhenNotProcessing, StateProvider, Sealer, BlockTree, @@ -110,7 +110,12 @@ public void InitProducer(IAuraConfig auraConfig) LimboLogs.Instance, blocksConfig); - ProducedBlockSuggester suggester = new(BlockTree, AuRaBlockProducer); + BlockProducerRunner = new StandardBlockProducerRunner( + onlyWhenNotProcessing, + BlockTree, + AuRaBlockProducer); + + ProducedBlockSuggester suggester = new(BlockTree, BlockProducerRunner); } } @@ -223,7 +228,7 @@ private async Task StartStop(Context context, bool processingQueueEm processedEvent.Set(); }); - await context.AuRaBlockProducer.Start(); + context.BlockProducerRunner.Start(); await processedEvent.WaitOneAsync(context.StepDelay * stepDelayMultiplier * 5, CancellationToken.None); context.BlockTree.ClearReceivedCalls(); await Task.Delay(context.StepDelay); @@ -248,7 +253,7 @@ private async Task StartStop(Context context, bool processingQueueEm } finally { - await context.AuRaBlockProducer.StopAsync(); + await context.BlockProducerRunner.StopAsync(); } return new TestResult(q => context.BlockTree.Received(q).SuggestBlock(Arg.Any(), Arg.Any())); diff --git a/src/Nethermind/Nethermind.AuRa.Test/AuRaPluginTests.cs b/src/Nethermind/Nethermind.AuRa.Test/AuRaPluginTests.cs index 009035305dd..03c84f9b2b8 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/AuRaPluginTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/AuRaPluginTests.cs @@ -6,6 +6,7 @@ using Nethermind.Api; using Nethermind.Config; using Nethermind.Consensus.AuRa; +using Nethermind.Consensus.AuRa.InitializationSteps; using Nethermind.Logging; using Nethermind.Serialization.Json; using Nethermind.Specs.ChainSpecStyle; @@ -19,7 +20,7 @@ public class AuRaPluginTests public void Init_when_not_AuRa_doesnt_trow() { AuRaPlugin auRaPlugin = new(); - Action init = () => auRaPlugin.Init(new NethermindApi(new ConfigProvider(), new EthereumJsonSerializer(), new TestLogManager(), new ChainSpec())); + Action init = () => auRaPlugin.Init(new AuRaNethermindApi(new ConfigProvider(), new EthereumJsonSerializer(), new TestLogManager(), new ChainSpec())); init.Should().NotThrow(); } diff --git a/src/Nethermind/Nethermind.AuRa.Test/Contract/AuRaContractGasLimitOverrideTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Contract/AuRaContractGasLimitOverrideTests.cs index d9e559f47dd..24b29330e4c 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Contract/AuRaContractGasLimitOverrideTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Contract/AuRaContractGasLimitOverrideTests.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using FluentAssertions; using Nethermind.Abi; +using Nethermind.Blockchain; using Nethermind.Consensus; using Nethermind.Consensus.AuRa; using Nethermind.Consensus.AuRa.Contracts; @@ -83,7 +84,7 @@ protected override BlockProcessor CreateBlockProcessor() BlockGasLimitContract gasLimitContract = new(AbiEncoder.Instance, blockGasLimitContractTransition.Value, blockGasLimitContractTransition.Key, new ReadOnlyTxProcessingEnv( WorldStateManager, - BlockTree, SpecProvider, LimboLogs.Instance)); + BlockTree.AsReadOnly(), SpecProvider, LimboLogs.Instance)); GasLimitOverrideCache = new AuRaContractGasLimitOverride.Cache(); GasLimitCalculator = new AuRaContractGasLimitOverride(new[] { gasLimitContract }, GasLimitOverrideCache, false, new FollowOtherMiners(SpecProvider), LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.AuRa.Test/Contract/TxPriorityContractTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Contract/TxPriorityContractTests.cs index 276671dab2a..1caa20bf3a9 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Contract/TxPriorityContractTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Contract/TxPriorityContractTests.cs @@ -10,6 +10,7 @@ using System.Threading.Tasks; using FluentAssertions; using Nethermind.Abi; +using Nethermind.Blockchain; using Nethermind.Blockchain.Data; using Nethermind.Consensus.AuRa.Contracts; using Nethermind.Consensus.AuRa.Contracts.DataStore; @@ -254,7 +255,7 @@ protected override TxPoolTxSource CreateTxPoolTxSource() TxPoolTxSource txPoolTxSource = base.CreateTxPoolTxSource(); TxPriorityContract = new TxPriorityContract(AbiEncoder.Instance, TestItem.AddressA, - new ReadOnlyTxProcessingEnv(WorldStateManager, BlockTree, SpecProvider, LimboLogs.Instance)); + new ReadOnlyTxProcessingEnv(WorldStateManager, BlockTree.AsReadOnly(), SpecProvider, LimboLogs.Instance)); Priorities = new DictionaryContractDataStore( new TxPriorityContract.DestinationSortedListContractDataStoreCollection(), diff --git a/src/Nethermind/Nethermind.AuRa.Test/Contract/ValidatorContractTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Contract/ValidatorContractTests.cs index 8b7ed8d3901..ae1a5040a4f 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Contract/ValidatorContractTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Contract/ValidatorContractTests.cs @@ -6,6 +6,7 @@ using Nethermind.Abi; using Nethermind.Consensus; using Nethermind.Consensus.AuRa.Contracts; +using Nethermind.Consensus.Processing; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Test; @@ -26,7 +27,7 @@ public class ValidatorContractTests { private Block _block; private readonly Address _contractAddress = Address.FromNumber(long.MaxValue); - private IReadOnlyTransactionProcessor _transactionProcessor; + private ITransactionProcessor _transactionProcessor; private IReadOnlyTxProcessorSource _readOnlyTxProcessorSource; private IWorldState _stateProvider; @@ -34,16 +35,13 @@ public class ValidatorContractTests public void SetUp() { _block = new Block(Build.A.BlockHeader.TestObject, new BlockBody()); - _transactionProcessor = Substitute.For(); - _readOnlyTxProcessorSource = Substitute.For(); - _readOnlyTxProcessorSource.Build(TestItem.KeccakA).Returns(_transactionProcessor); + _transactionProcessor = Substitute.For(); _stateProvider = Substitute.For(); _stateProvider.StateRoot.Returns(TestItem.KeccakA); + _readOnlyTxProcessorSource = Substitute.For(); + _readOnlyTxProcessorSource.Build(TestItem.KeccakA).Returns(new ReadOnlyTxProcessingScope(_transactionProcessor, _stateProvider, Keccak.EmptyTreeHash)); } - [TearDown] - public void TearDown() => _transactionProcessor?.Dispose(); - [Test] public void constructor_throws_ArgumentNullException_on_null_contractAddress() { diff --git a/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxCertifierFilterTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxCertifierFilterTests.cs index c65d2efc805..f6403fa1184 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxCertifierFilterTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxCertifierFilterTests.cs @@ -6,6 +6,7 @@ using FluentAssertions; using Nethermind.Abi; using Nethermind.AuRa.Test.Contract; +using Nethermind.Blockchain; using Nethermind.Consensus.AuRa; using Nethermind.Consensus.AuRa.Contracts; using Nethermind.Consensus.AuRa.Transactions; @@ -137,7 +138,7 @@ protected override BlockProcessor CreateBlockProcessor() AbiEncoder abiEncoder = AbiEncoder.Instance; ReadOnlyTransactionProcessorSource = new ReadOnlyTxProcessingEnv( WorldStateManager, - BlockTree, SpecProvider, + BlockTree.AsReadOnly(), SpecProvider, LimboLogs.Instance); RegisterContract = new RegisterContract(abiEncoder, ChainSpec.Parameters.Registrar, ReadOnlyTransactionProcessorSource); CertifierContract = new CertifierContract( diff --git a/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxPermissionFilterTest.cs b/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxPermissionFilterTest.cs index 87706013ea2..7bae62ac4df 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxPermissionFilterTest.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Transactions/TxPermissionFilterTest.cs @@ -8,6 +8,7 @@ using FluentAssertions; using Nethermind.Abi; using Nethermind.AuRa.Test.Contract; +using Nethermind.Blockchain; using Nethermind.Consensus.AuRa; using Nethermind.Consensus.AuRa.Contracts; using Nethermind.Consensus.AuRa.Transactions; @@ -277,12 +278,12 @@ protected override BlockProcessor CreateBlockProcessor() IReadOnlyTrieStore trieStore = new TrieStore(DbProvider.StateDb, LimboLogs.Instance).AsReadOnly(); IReadOnlyTxProcessorSource txProcessorSource = new ReadOnlyTxProcessingEnv( WorldStateManager, - BlockTree, + BlockTree.AsReadOnly(), SpecProvider, LimboLogs.Instance); VersionedTransactionPermissionContract transactionPermissionContract = new(AbiEncoder.Instance, _contractAddress, 1, - new ReadOnlyTxProcessingEnv(WorldStateManager, BlockTree, SpecProvider, LimboLogs.Instance), TransactionPermissionContractVersions, LimboLogs.Instance, SpecProvider); + new ReadOnlyTxProcessingEnv(WorldStateManager, BlockTree.AsReadOnly(), SpecProvider, LimboLogs.Instance), TransactionPermissionContractVersions, LimboLogs.Instance, SpecProvider); TxPermissionFilterCache = new PermissionBasedTxFilter.Cache(); PermissionBasedTxFilter = new PermissionBasedTxFilter(transactionPermissionContract, TxPermissionFilterCache, LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs b/src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs index 91dad8e1fc5..3902182d6e8 100644 --- a/src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs +++ b/src/Nethermind/Nethermind.AuRa.Test/Validators/ContractBasedValidatorTests.cs @@ -32,6 +32,7 @@ using Nethermind.Evm; using Nethermind.Core.Specs; using System.Text.Json; +using Nethermind.Consensus.Processing; namespace Nethermind.AuRa.Test.Validators; @@ -43,7 +44,7 @@ public class ContractBasedValidatorTests private AuRaParameters.Validator _validator; private Block _block; private BlockHeader _parentHeader; - private IReadOnlyTransactionProcessor _transactionProcessor; + private ITransactionProcessor _transactionProcessor; private IAuRaBlockFinalizationManager _blockFinalizationManager; private static readonly Address _contractAddress = Address.FromNumber(1000); private (Address Sender, byte[] TransactionData) _getValidatorsData = (Address.Zero, new byte[] { 0, 1, 2 }); @@ -74,11 +75,12 @@ public void SetUp() }; _block = new Block(Build.A.BlockHeader.WithNumber(1).WithAura(1, Array.Empty()).TestObject, new BlockBody()); - _transactionProcessor = Substitute.For(); - _transactionProcessor.IsContractDeployed(_contractAddress).Returns(true); - _readOnlyTxProcessorSource = Substitute.For(); - _readOnlyTxProcessorSource.Build(Arg.Any()).Returns(_transactionProcessor); + _transactionProcessor = Substitute.For(); _stateProvider.StateRoot.Returns(TestItem.KeccakA); + _stateProvider.IsContract(_contractAddress).Returns(true); + + _readOnlyTxProcessorSource = Substitute.For(); + _readOnlyTxProcessorSource.Build(Arg.Any()).Returns(new ReadOnlyTxProcessingScope(_transactionProcessor, _stateProvider, Keccak.EmptyTreeHash)); _blockTree.Head.Returns(_block); _abiEncoder @@ -96,7 +98,6 @@ public void SetUp() public void TearDown() { _blockFinalizationManager?.Dispose(); - _transactionProcessor?.Dispose(); } [Test] diff --git a/src/Nethermind/Nethermind.Benchmark/Evm/JumpDestinationsBenchmark.cs b/src/Nethermind/Nethermind.Benchmark/Evm/JumpDestinationsBenchmark.cs index b77cc017fa4..8ee151d0343 100644 --- a/src/Nethermind/Nethermind.Benchmark/Evm/JumpDestinationsBenchmark.cs +++ b/src/Nethermind/Nethermind.Benchmark/Evm/JumpDestinationsBenchmark.cs @@ -29,7 +29,7 @@ public void Setup() [Benchmark] public bool Current() { - return _codeInfo.ValidateJump(0, false); + return _codeInfo.ValidateJump(0); } } } diff --git a/src/Nethermind/Nethermind.Blockchain.Test.Runner/BlockchainTestsBugHunter.cs b/src/Nethermind/Nethermind.Blockchain.Test.Runner/BlockchainTestsBugHunter.cs index e3a38660ba0..ce6593f4751 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test.Runner/BlockchainTestsBugHunter.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test.Runner/BlockchainTestsBugHunter.cs @@ -32,7 +32,7 @@ public async Task> RunTestsAsync() Setup(); Console.Write($"{test,-120} "); - if (test.LoadFailure != null) + if (test.LoadFailure is not null) { WriteRed(test.LoadFailure); testResults.Add(new EthereumTestResult(test.Name, test.LoadFailure)); diff --git a/src/Nethermind/Nethermind.Blockchain.Test.Runner/PerfTest.cs b/src/Nethermind/Nethermind.Blockchain.Test.Runner/PerfTest.cs index c94b78867b1..a25bfb6038a 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test.Runner/PerfTest.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test.Runner/PerfTest.cs @@ -28,7 +28,7 @@ public IEnumerable RunTests() bool isNewLine = true; foreach (GeneralStateTest test in tests) { - if (test.LoadFailure != null) + if (test.LoadFailure is not null) { continue; } diff --git a/src/Nethermind/Nethermind.Blockchain.Test.Runner/StateTestsBugHunter.cs b/src/Nethermind/Nethermind.Blockchain.Test.Runner/StateTestsBugHunter.cs index d98679e02b9..9dd9cf56c7a 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test.Runner/StateTestsBugHunter.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test.Runner/StateTestsBugHunter.cs @@ -32,7 +32,7 @@ public IEnumerable RunTests() Setup(LimboLogs.Instance); Console.Write($"{test,-120} "); - if (test.LoadFailure != null) + if (test.LoadFailure is not null) { WriteRed(test.LoadFailure); testResults.Add(new EthereumTestResult(test.Name, test.ForkName, test.LoadFailure)); diff --git a/src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs index 0cc724cfccc..8e8172f1b0a 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs @@ -49,7 +49,6 @@ public void Prepared_block_contains_author_field() new BlockProcessor.BlockValidationTransactionsExecutor(transactionProcessor, stateProvider), stateProvider, NullReceiptStorage.Instance, - NullWitnessCollector.Instance, Substitute.For(), transactionProcessor, LimboLogs.Instance); @@ -65,39 +64,6 @@ public void Prepared_block_contains_author_field() Assert.That(processedBlocks[0].Author, Is.EqualTo(block.Author), "author"); } - [Test, Timeout(Timeout.MaxTestTime)] - public void Can_store_a_witness() - { - IDb stateDb = new MemDb(); - IDb codeDb = new MemDb(); - TrieStore trieStore = new TrieStore(stateDb, LimboLogs.Instance); - - IWorldState stateProvider = new WorldState(trieStore, codeDb, LimboLogs.Instance); - ITransactionProcessor transactionProcessor = Substitute.For(); - IWitnessCollector witnessCollector = Substitute.For(); - BlockProcessor processor = new( - HoleskySpecProvider.Instance, - TestBlockValidator.AlwaysValid, - NoBlockRewards.Instance, - new BlockProcessor.BlockValidationTransactionsExecutor(transactionProcessor, stateProvider), - stateProvider, - NullReceiptStorage.Instance, - witnessCollector, - Substitute.For(), - transactionProcessor, - LimboLogs.Instance); - - BlockHeader header = Build.A.BlockHeader.WithAuthor(TestItem.AddressD).TestObject; - Block block = Build.A.Block.WithHeader(header).TestObject; - _ = processor.Process( - Keccak.EmptyTreeHash, - new List { block }, - ProcessingOptions.None, - NullBlockTracer.Instance); - - witnessCollector.Received(1).Persist(block.Hash!); - } - [Test, Timeout(Timeout.MaxTestTime)] public void Recovers_state_on_cancel() { @@ -113,7 +79,6 @@ public void Recovers_state_on_cancel() new BlockProcessor.BlockValidationTransactionsExecutor(transactionProcessor, stateProvider), stateProvider, NullReceiptStorage.Instance, - NullWitnessCollector.Instance, Substitute.For(), transactionProcessor, LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Consensus/ClefSignerTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Consensus/ClefSignerTests.cs new file mode 100644 index 00000000000..b4ce5520b71 --- /dev/null +++ b/src/Nethermind/Nethermind.Blockchain.Test/Consensus/ClefSignerTests.cs @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading.Tasks; +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Test.Builders; +using Nethermind.JsonRpc; +using Nethermind.JsonRpc.Client; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.Blockchain.Test.Consensus +{ + [TestFixture] + public class ClefSignerTests + { + [Test] + public async Task Sign_SigningHash_RequestHasCorrectParameters() + { + IJsonRpcClient client = Substitute.For(); + client.Post("account_list").Returns(Task.FromResult([TestItem.AddressA!.ToString()])); + Task postMethod = client.Post("account_signData", "text/plain", Arg.Any(), Keccak.Zero); + var returnValue = (new byte[65]).ToHexString(); + postMethod.Returns(returnValue); + ClefSigner sut = await ClefSigner.Create(client); + + var result = sut.Sign(Keccak.Zero); + + Assert.That(new Signature(returnValue).Bytes, Is.EqualTo(result.Bytes)); + } + + [Test] + public async Task Sign_SigningCliqueHeader_PassingCorrectClefParametersForRequest() + { + IJsonRpcClient client = Substitute.For(); + client.Post("account_list").Returns(Task.FromResult([TestItem.AddressA!.ToString()])); + Task postMethod = client.Post(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); + var returnValue = (new byte[65]).ToHexString(); + postMethod.Returns(returnValue); + BlockHeader blockHeader = Build.A.BlockHeader.TestObject; + ClefSigner sut = await ClefSigner.Create(client); + + sut.Sign(blockHeader); + + await client.Received().Post("account_signData", "application/x-clique-header", Arg.Any(), Arg.Any()); + } + + + [TestCase(0, 27)] + [TestCase(1, 28)] + public async Task Sign_RecoveryIdIsSetToCliqueValues_RecoveryIdIsAdjusted(byte recId, byte expected) + { + IJsonRpcClient client = Substitute.For(); + client.Post("account_list").Returns(Task.FromResult([TestItem.AddressA!.ToString()])); + Task postMethod = client.Post("account_signData", "application/x-clique-header", Arg.Any(), Arg.Any()); + var returnValue = (new byte[65]); + returnValue[64] = recId; + postMethod.Returns(returnValue.ToHexString()); + BlockHeader blockHeader = Build.A.BlockHeader.TestObject; + ClefSigner sut = await ClefSigner.Create(client); + + var result = sut.Sign(blockHeader); + + Assert.That(result.V, Is.EqualTo(expected)); + } + + [Test] + public async Task Create_SignerAddressSpecified_CorrectAddressIsSet() + { + IJsonRpcClient client = Substitute.For(); + client.Post("account_list").Returns(Task.FromResult([TestItem.AddressA!.ToString(), TestItem.AddressB!.ToString()])); + + ClefSigner sut = await ClefSigner.Create(client, TestItem.AddressB); + + Assert.That(sut.Address, Is.EqualTo(TestItem.AddressB)); + } + + [Test] + public void Create_SignerAddressDoesNotExists_ThrowInvalidOperationException() + { + IJsonRpcClient client = Substitute.For(); + client.Post("account_list").Returns(Task.FromResult([TestItem.AddressA!.ToString(), TestItem.AddressB!.ToString()])); + + Assert.That(async () => await ClefSigner.Create(client, TestItem.AddressC), Throws.InstanceOf()); + } + + [Test] + public async Task SetSigner_TryingToASigner_ThrowInvalidOperationException() + { + IJsonRpcClient client = Substitute.For(); + client.Post("account_list").Returns(Task.FromResult([TestItem.AddressA!.ToString()])); + ClefSigner sut = await ClefSigner.Create(client); + + Assert.That(() => sut.SetSigner(Build.A.PrivateKey.TestObject), Throws.InstanceOf()); + } + } +} diff --git a/src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs index d17d8d06f79..9367267797b 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs @@ -14,8 +14,8 @@ public class KnownChainSizesTests public void Update_known_chain_sizes() { // Pruning size have to be updated frequently - ChainSizes.CreateChainSizeInfo(BlockchainIds.Mainnet).PruningSize.Should().BeLessThan(220.GB()); - ChainSizes.CreateChainSizeInfo(BlockchainIds.Sepolia).PruningSize.Should().BeLessThan(20.GB()); + ChainSizes.CreateChainSizeInfo(BlockchainIds.Mainnet).PruningSize.Should().BeLessThan(240.GB()); + ChainSizes.CreateChainSizeInfo(BlockchainIds.Sepolia).PruningSize.Should().BeLessThan(24.GB()); ChainSizes.CreateChainSizeInfo(BlockchainIds.Chiado).PruningSize.Should().Be(null); ChainSizes.CreateChainSizeInfo(BlockchainIds.Gnosis).PruningSize.Should().Be(null); diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Nethermind.Blockchain.Test.csproj b/src/Nethermind/Nethermind.Blockchain.Test/Nethermind.Blockchain.Test.csproj index a4aa0b53879..caa031ac836 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Nethermind.Blockchain.Test.csproj +++ b/src/Nethermind/Nethermind.Blockchain.Test/Nethermind.Blockchain.Test.csproj @@ -26,6 +26,7 @@ + diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.IsProducingBlocks.cs b/src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.IsProducingBlocks.cs index 49e67f7c50b..3735ff1bc1b 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.IsProducingBlocks.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.IsProducingBlocks.cs @@ -42,12 +42,13 @@ public async Task DevBlockProducer_IsProducingBlocks_returns_expected_results() testRpc.BlockchainProcessor, testRpc.State, testRpc.BlockTree, - Substitute.For(), testRpc.Timestamper, testRpc.SpecProvider, new BlocksConfig(), LimboLogs.Instance); - await AssertIsProducingBlocks(blockProducer); + StandardBlockProducerRunner runner = new StandardBlockProducerRunner( + Substitute.For(), testRpc.BlockTree, blockProducer); + await AssertIsProducingBlocks(runner); } [Test, Timeout(Timeout.MaxTestTime)] @@ -61,12 +62,13 @@ public async Task TestBlockProducer_IsProducingBlocks_returns_expected_results() testRpc.State, Substitute.For(), testRpc.BlockTree, - Substitute.For(), testRpc.Timestamper, testRpc.SpecProvider, LimboLogs.Instance, blocksConfig); - await AssertIsProducingBlocks(blockProducer); + StandardBlockProducerRunner runner = new StandardBlockProducerRunner( + Substitute.For(), testRpc.BlockTree, blockProducer); + await AssertIsProducingBlocks(runner); } [Test, Timeout(Timeout.MaxTestTime)] @@ -79,14 +81,15 @@ public async Task MinedBlockProducer_IsProducingBlocks_returns_expected_results( testRpc.BlockchainProcessor, Substitute.For(), testRpc.BlockTree, - Substitute.For(), testRpc.State, Substitute.For(), testRpc.Timestamper, testRpc.SpecProvider, LimboLogs.Instance, blocksConfig); - await AssertIsProducingBlocks(blockProducer); + StandardBlockProducerRunner runner = new StandardBlockProducerRunner( + Substitute.For(), testRpc.BlockTree, blockProducer); + await AssertIsProducingBlocks(runner); } [Test, Timeout(Timeout.MaxTestTime)] @@ -97,7 +100,6 @@ public async Task AuraTestBlockProducer_IsProducingBlocks_returns_expected_resul AuRaBlockProducer blockProducer = new( Substitute.For(), Substitute.For(), - Substitute.For(), Substitute.For(), Substitute.For(), Substitute.For(), @@ -109,7 +111,9 @@ public async Task AuraTestBlockProducer_IsProducingBlocks_returns_expected_resul Substitute.For(), LimboLogs.Instance, Substitute.For()); - await AssertIsProducingBlocks(blockProducer); + StandardBlockProducerRunner runner = new StandardBlockProducerRunner( + Substitute.For(), Substitute.For(), blockProducer); + await AssertIsProducingBlocks(runner); } [Test, Timeout(Timeout.MaxTestTime)] @@ -120,7 +124,6 @@ public async Task CliqueBlockProducer_IsProducingBlocks_returns_expected_results Substitute.For(), testRpc.BlockchainProcessor, testRpc.State, - testRpc.BlockTree, testRpc.Timestamper, Substitute.For(), Substitute.For(), @@ -129,7 +132,17 @@ public async Task CliqueBlockProducer_IsProducingBlocks_returns_expected_results Substitute.For(), new CliqueConfig(), LimboLogs.Instance); - await AssertIsProducingBlocks(blockProducer); + + CliqueBlockProducerRunner runner = new CliqueBlockProducerRunner( + testRpc.BlockTree, + testRpc.Timestamper, + Substitute.For(), + Substitute.For(), + blockProducer, + new CliqueConfig(), + LimboLogs.Instance); + + await AssertIsProducingBlocks(runner); } private async Task CreateTestRpc() @@ -143,10 +156,10 @@ private async Task CreateTestRpc() return testRpc; } - private async Task AssertIsProducingBlocks(IBlockProducer blockProducer) + private async Task AssertIsProducingBlocks(IBlockProducerRunner blockProducer) { Assert.That(blockProducer.IsProducingBlocks(null), Is.EqualTo(false)); - await blockProducer.Start(); + blockProducer.Start(); Assert.That(blockProducer.IsProducingBlocks(null), Is.EqualTo(true)); Thread.Sleep(5000); Assert.That(blockProducer.IsProducingBlocks(1), Is.EqualTo(false)); diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.cs index 4b8b6ef14b7..e4edbe07823 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Threading.Tasks; using FluentAssertions; using Nethermind.Config; using Nethermind.Consensus; @@ -30,7 +29,6 @@ public ProducerUnderTest( IBlockchainProcessor processor, ISealer sealer, IBlockTree blockTree, - IBlockProductionTrigger blockProductionTrigger, IWorldState stateProvider, IGasLimitCalculator gasLimitCalculator, ITimestamper timestamper, @@ -41,7 +39,6 @@ public ProducerUnderTest( processor, sealer, blockTree, - blockProductionTrigger, stateProvider, gasLimitCalculator, timestamper, @@ -52,16 +49,10 @@ public ProducerUnderTest( { } - public override Task Start() { return Task.CompletedTask; } - - public override Task StopAsync() => Task.CompletedTask; - public Block Prepare() => PrepareBlock(Build.A.BlockHeader.TestObject); public Block Prepare(BlockHeader header) => PrepareBlock(header); - protected override bool IsRunning() => true; - private class TimestampDifficultyCalculator : IDifficultyCalculator { public UInt256 Calculate(BlockHeader header, BlockHeader parent) => header.Timestamp; @@ -78,7 +69,6 @@ public void Time_passing_does_not_break_the_block() Substitute.For(), NullSealEngine.Instance, Build.A.BlockTree().TestObject, - Substitute.For(), Substitute.For(), Substitute.For(), timestamper, @@ -101,7 +91,6 @@ public void Parent_timestamp_is_used_consistently() Substitute.For(), NullSealEngine.Instance, Build.A.BlockTree().TestObject, - Substitute.For(), Substitute.For(), Substitute.For(), timestamper, diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs index 4d492270c74..ae107c08af5 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Producers/DevBlockproducerTests.cs @@ -6,6 +6,7 @@ using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Receipts; using Nethermind.Config; +using Nethermind.Consensus; using Nethermind.Consensus.Processing; using Nethermind.Consensus.Producers; using Nethermind.Consensus.Rewards; @@ -57,14 +58,17 @@ public void Test() LimboLogs.Instance); StateReader stateReader = new(trieStore, dbProvider.GetDb(DbNames.State), LimboLogs.Instance); BlockhashProvider blockhashProvider = new(blockTree, specProvider, stateProvider, LimboLogs.Instance); + CodeInfoRepository codeInfoRepository = new(); VirtualMachine virtualMachine = new( blockhashProvider, specProvider, + codeInfoRepository, LimboLogs.Instance); TransactionProcessor txProcessor = new( specProvider, stateProvider, virtualMachine, + codeInfoRepository, LimboLogs.Instance); BlockProcessor blockProcessor = new( specProvider, @@ -73,7 +77,6 @@ public void Test() new BlockProcessor.BlockValidationTransactionsExecutor(txProcessor, stateProvider), stateProvider, NullReceiptStorage.Instance, - NullWitnessCollector.Instance, new BlockhashStore(blockTree, specProvider, stateProvider), txProcessor, LimboLogs.Instance); @@ -91,15 +94,16 @@ public void Test() blockchainProcessor, stateProvider, blockTree, - trigger, timestamper, specProvider, new BlocksConfig(), LimboLogs.Instance); + StandardBlockProducerRunner blockProducerRunner = new StandardBlockProducerRunner(trigger, blockTree, devBlockProducer); + blockchainProcessor.Start(); - devBlockProducer.Start(); - ProducedBlockSuggester _ = new ProducedBlockSuggester(blockTree, devBlockProducer); + blockProducerRunner.Start(); + ProducedBlockSuggester _ = new ProducedBlockSuggester(blockTree, blockProducerRunner); AutoResetEvent autoResetEvent = new(false); diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Proofs/ReceiptTrieTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Proofs/ReceiptTrieTests.cs index 1fbaa51be02..6ee424c6b38 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Proofs/ReceiptTrieTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Proofs/ReceiptTrieTests.cs @@ -19,12 +19,14 @@ namespace Nethermind.Blockchain.Test.Proofs { public class ReceiptTrieTests { + private static readonly IRlpStreamDecoder _decoder = Rlp.GetStreamDecoder()!; + [Test, Timeout(Timeout.MaxTestTime)] public void Can_calculate_root_no_eip_658() { TxReceipt receipt = Build.A.Receipt.WithAllFieldsFilled.TestObject; Hash256 rootHash = ReceiptTrie.CalculateRoot(MainnetSpecProvider.Instance.GetSpec((1, null)), - new[] { receipt }, ReceiptMessageDecoder.Instance); + [receipt], _decoder); Assert.That(rootHash.ToString(), Is.EqualTo("0xe51a2d9f986d68628990c9d65e45c36128ec7bb697bd426b0bb4d18a3f3321be")); } @@ -35,7 +37,7 @@ public void Can_calculate_root() TxReceipt receipt = Build.A.Receipt.WithAllFieldsFilled.TestObject; Hash256 rootHash = ReceiptTrie.CalculateRoot( MainnetSpecProvider.Instance.GetSpec((MainnetSpecProvider.MuirGlacierBlockNumber, null)), - new[] { receipt }, ReceiptMessageDecoder.Instance); + [receipt], _decoder); Assert.That(rootHash.ToString(), Is.EqualTo("0x2e6d89c5b539e72409f2e587730643986c2ef33db5e817a4223aa1bb996476d5")); } @@ -46,7 +48,7 @@ public void Can_collect_proof_with_branch() TxReceipt receipt1 = Build.A.Receipt.WithAllFieldsFilled.TestObject; TxReceipt receipt2 = Build.A.Receipt.WithAllFieldsFilled.TestObject; ReceiptTrie trie = new(MainnetSpecProvider.Instance.GetSpec((ForkActivation)1), - new[] { receipt1, receipt2 }, ReceiptMessageDecoder.Instance, true); + [receipt1, receipt2], _decoder, true); byte[][] proof = trie.BuildProof(0); Assert.That(proof.Length, Is.EqualTo(2)); @@ -54,11 +56,11 @@ public void Can_collect_proof_with_branch() VerifyProof(proof, trie.RootHash); } - private static void VerifyProof(byte[][] proof, Hash256 receiptRoot) + private void VerifyProof(byte[][] proof, Hash256 receiptRoot) { TrieNode node = new(NodeType.Unknown, proof.Last()); node.ResolveNode(Substitute.For(), TreePath.Empty); - TxReceipt receipt = new ReceiptMessageDecoder().Decode(node.Value.AsRlpStream()); + TxReceipt receipt = _decoder.Decode(node.Value.AsRlpStream()); Assert.NotNull(receipt.Bloom); for (int i = proof.Length; i > 0; i--) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Receipts/ReceiptsIteratorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Receipts/ReceiptsIteratorTests.cs index e56498db73a..da658279ddc 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Receipts/ReceiptsIteratorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Receipts/ReceiptsIteratorTests.cs @@ -112,7 +112,7 @@ private ReceiptsIterator CreateIterator(TxReceipt[] receipts, Block block) false ); - ReceiptsIterator iterator = new ReceiptsIterator(span, blockDb, () => recovery.CreateRecoveryContext(new ReceiptRecoveryBlock(block)), ReceiptArrayStorageDecoder.GetRefDecoder(span)); + ReceiptsIterator iterator = new ReceiptsIterator(span, blockDb, () => recovery.CreateRecoveryContext(new ReceiptRecoveryBlock(block)), _decoder.GetRefDecoder(span)); return iterator; } } diff --git a/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs index 912cfd4f100..63e4bc13cdc 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs @@ -19,7 +19,6 @@ using Nethermind.Logging; using Nethermind.Specs; using Nethermind.State; -using Nethermind.State.Witnesses; using Nethermind.Trie.Pruning; using Nethermind.TxPool; using NUnit.Framework; @@ -58,14 +57,17 @@ public void Setup() LimboLogs.Instance, transactionComparerProvider.GetDefaultComparer()); BlockhashProvider blockhashProvider = new(_blockTree, specProvider, stateProvider, LimboLogs.Instance); + CodeInfoRepository codeInfoRepository = new(); VirtualMachine virtualMachine = new( blockhashProvider, specProvider, + codeInfoRepository, LimboLogs.Instance); TransactionProcessor transactionProcessor = new( specProvider, stateProvider, virtualMachine, + codeInfoRepository, LimboLogs.Instance); BlockProcessor blockProcessor = new( @@ -75,7 +77,6 @@ public void Setup() new BlockProcessor.BlockValidationTransactionsExecutor(transactionProcessor, stateProvider), stateProvider, NullReceiptStorage.Instance, - new WitnessCollector(memDbProvider.StateDb, LimboLogs.Instance), new BlockhashStore(_blockTree, MainnetSpecProvider.Instance, stateProvider), transactionProcessor, LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs index bb730fa4b20..6b833e60048 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs @@ -195,7 +195,7 @@ public void Proper_transactions_selected(ProperTransactionsSelectedTestCase test MemDb stateDb = new(); MemDb codeDb = new(); TrieStore trieStore = new(stateDb, LimboLogs.Instance); - WorldState stateProvider = new(trieStore, codeDb, LimboLogs.Instance); + IWorldState stateProvider = new WorldState(trieStore, codeDb, LimboLogs.Instance); StateReader _ = new(new TrieStore(stateDb, LimboLogs.Instance), codeDb, LimboLogs.Instance); ISpecProvider specProvider = Substitute.For(); diff --git a/src/Nethermind/Nethermind.Blockchain.Test/TransactionsExecutorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/TransactionsExecutorTests.cs index f43d92ead95..8799490191a 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/TransactionsExecutorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/TransactionsExecutorTests.cs @@ -262,7 +262,7 @@ public void Proper_transactions_selected(TransactionSelectorTests.ProperTransact MemDb stateDb = new(); MemDb codeDb = new(); TrieStore trieStore = new(stateDb, LimboLogs.Instance); - WorldState stateProvider = new(trieStore, codeDb, LimboLogs.Instance); + IWorldState stateProvider = new WorldState(trieStore, codeDb, LimboLogs.Instance); ISpecProvider specProvider = Substitute.For(); IReleaseSpec spec = testCase.ReleaseSpec; diff --git a/src/Nethermind/Nethermind.Blockchain/BlockTree.cs b/src/Nethermind/Nethermind.Blockchain/BlockTree.cs index d2a96800259..c7338fa2aec 100644 --- a/src/Nethermind/Nethermind.Blockchain/BlockTree.cs +++ b/src/Nethermind/Nethermind.Blockchain/BlockTree.cs @@ -628,32 +628,6 @@ as it does not require the step of resolving number -> hash */ return result; } - public BlockHeader? FindLowestCommonAncestor(BlockHeader firstDescendant, BlockHeader secondDescendant, - long maxSearchDepth) - { - if (firstDescendant.Number > secondDescendant.Number) - { - firstDescendant = GetAncestorAtNumber(firstDescendant, secondDescendant.Number); - } - else if (secondDescendant.Number > firstDescendant.Number) - { - secondDescendant = GetAncestorAtNumber(secondDescendant, firstDescendant.Number); - } - - long currentSearchDepth = 0; - while ( - firstDescendant is not null - && secondDescendant is not null - && firstDescendant.Hash != secondDescendant.Hash) - { - if (currentSearchDepth++ >= maxSearchDepth) return null; - firstDescendant = this.FindParentHeader(firstDescendant, BlockTreeLookupOptions.TotalDifficultyNotNeeded); - secondDescendant = this.FindParentHeader(secondDescendant, BlockTreeLookupOptions.TotalDifficultyNotNeeded); - } - - return firstDescendant; - } - private BlockHeader? GetAncestorAtNumber(BlockHeader header, long number) { BlockHeader? result = header; @@ -827,22 +801,17 @@ private void DeleteBlocks(Hash256 deletePointer) return childHash; } - public bool IsMainChain(BlockHeader blockHeader) - { - ChainLevelInfo? chainLevelInfo = LoadLevel(blockHeader.Number); - bool isMain = chainLevelInfo is not null && chainLevelInfo.MainChainBlock?.BlockHash.Equals(blockHeader.Hash) == true; - return isMain; - } + public bool IsMainChain(BlockHeader blockHeader) => + LoadLevel(blockHeader.Number)?.MainChainBlock?.BlockHash.Equals(blockHeader.Hash) == true; - public bool IsMainChain(Hash256 blockHash) + public bool IsMainChain(Hash256 blockHash, bool throwOnMissingHash = true) { BlockHeader? header = FindHeader(blockHash, BlockTreeLookupOptions.TotalDifficultyNotNeeded); - if (header is null) - { - throw new InvalidOperationException($"Not able to retrieve block number for an unknown block {blockHash}"); - } - - return IsMainChain(header); + return header is not null + ? IsMainChain(header) + : throwOnMissingHash + ? throw new InvalidOperationException($"Not able to retrieve block number for an unknown block {blockHash}") + : false; } public BlockHeader? FindBestSuggestedHeader() => BestSuggestedHeader; @@ -1279,7 +1248,7 @@ private ChainLevelInfo UpdateOrCreateLevel(long number, Hash256 hash, BlockInfo /// private bool ShouldCache(long number) { - return number == 0L || Head is null || number >= Head.Number - HeaderStore.CacheSize; + return number == 0L || Head is null || number >= Head.Number - BlockStore.CacheSize; } public ChainLevelInfo? FindLevel(long number) diff --git a/src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs b/src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs new file mode 100644 index 00000000000..c16834f63a8 --- /dev/null +++ b/src/Nethermind/Nethermind.Blockchain/BlockTreeOverlay.cs @@ -0,0 +1,251 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Blockchain.Visitors; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; + +namespace Nethermind.Blockchain; + +public class BlockTreeOverlay : IBlockTree +{ + private readonly IBlockTree _baseTree; + private readonly IBlockTree _overlayTree; + + public BlockTreeOverlay(IReadOnlyBlockTree baseTree, IBlockTree overlayTree) + { + _baseTree = baseTree ?? throw new ArgumentNullException(nameof(baseTree)); + _overlayTree = overlayTree ?? throw new ArgumentNullException(nameof(overlayTree)); + _overlayTree.UpdateMainChain(new[] { _baseTree.Head }, true, true); + } + + public ulong NetworkId => _baseTree.NetworkId; + public ulong ChainId => _baseTree.ChainId; + public BlockHeader? Genesis => _baseTree.Genesis; + public BlockHeader? BestSuggestedHeader => _overlayTree.BestSuggestedHeader ?? _baseTree.BestSuggestedHeader; + public Block? BestSuggestedBody => _overlayTree.BestSuggestedBody ?? _baseTree.BestSuggestedBody; + public BlockHeader? BestSuggestedBeaconHeader => _overlayTree.BestSuggestedBeaconHeader ?? _baseTree.BestSuggestedBeaconHeader; + public BlockHeader? LowestInsertedHeader => _overlayTree.LowestInsertedHeader ?? _baseTree.LowestInsertedHeader; + + public long? LowestInsertedBodyNumber + { + get => _overlayTree.LowestInsertedBodyNumber ?? _baseTree.LowestInsertedBodyNumber; + set => _overlayTree.LowestInsertedBodyNumber = value; + } + + public BlockHeader? LowestInsertedBeaconHeader + { + get => _overlayTree.LowestInsertedBeaconHeader ?? _baseTree.LowestInsertedBeaconHeader; + set => _overlayTree.LowestInsertedBeaconHeader = value; + } + + public long BestKnownNumber => Math.Max(_overlayTree.BestKnownNumber, _baseTree.BestKnownNumber); + public long BestKnownBeaconNumber => Math.Max(_overlayTree.BestKnownBeaconNumber, _baseTree.BestKnownBeaconNumber); + public Hash256 HeadHash => _overlayTree.HeadHash ?? _baseTree.HeadHash; + public Hash256 GenesisHash => _baseTree.GenesisHash; + public Hash256? PendingHash => _overlayTree.PendingHash ?? _baseTree.PendingHash; + public Hash256? FinalizedHash => _overlayTree.FinalizedHash ?? _baseTree.FinalizedHash; + public Hash256? SafeHash => _overlayTree.SafeHash ?? _baseTree.SafeHash; + public Block? Head => _overlayTree.Head ?? _baseTree.Head; + public long? BestPersistedState { get => _overlayTree.BestPersistedState; set => _overlayTree.BestPersistedState = value; } + + public AddBlockResult Insert(BlockHeader header, BlockTreeInsertHeaderOptions headerOptions = BlockTreeInsertHeaderOptions.None) => + _overlayTree.Insert(header, headerOptions); + + public AddBlockResult Insert(Block block, + BlockTreeInsertBlockOptions insertBlockOptions = BlockTreeInsertBlockOptions.None, + BlockTreeInsertHeaderOptions insertHeaderOptions = BlockTreeInsertHeaderOptions.None, + WriteFlags bodiesWriteFlags = WriteFlags.None) => + _overlayTree.Insert(block, insertBlockOptions, insertHeaderOptions, bodiesWriteFlags); + + public void UpdateHeadBlock(Hash256 blockHash) => + _overlayTree.UpdateHeadBlock(blockHash); + + public AddBlockResult SuggestBlock(Block block, + BlockTreeSuggestOptions options = BlockTreeSuggestOptions.ShouldProcess) => + _overlayTree.SuggestBlock(block, options); + + public ValueTask SuggestBlockAsync(Block block, + BlockTreeSuggestOptions options = BlockTreeSuggestOptions.ShouldProcess) => + _overlayTree.SuggestBlockAsync(block, options); + + public AddBlockResult SuggestHeader(BlockHeader header) => _overlayTree.SuggestHeader(header); + + public bool IsKnownBlock(long number, Hash256 blockHash) => _overlayTree.IsKnownBlock(number, blockHash) || _baseTree.IsKnownBlock(number, blockHash); + + public bool IsKnownBeaconBlock(long number, Hash256 blockHash) => _overlayTree.IsKnownBeaconBlock(number, blockHash) || _baseTree.IsKnownBeaconBlock(number, blockHash); + + public bool WasProcessed(long number, Hash256 blockHash) => _overlayTree.WasProcessed(number, blockHash) || _baseTree.WasProcessed(number, blockHash); + + public void UpdateMainChain(IReadOnlyList blocks, bool wereProcessed, bool forceHeadBlock = false) => + _overlayTree.UpdateMainChain(blocks, wereProcessed, forceHeadBlock); + + public void MarkChainAsProcessed(IReadOnlyList blocks) => _overlayTree.MarkChainAsProcessed(blocks); + + public bool CanAcceptNewBlocks => _overlayTree.CanAcceptNewBlocks; + + public Task Accept(IBlockTreeVisitor blockTreeVisitor, CancellationToken cancellationToken) => _overlayTree.Accept(blockTreeVisitor, cancellationToken); + + public (BlockInfo? Info, ChainLevelInfo? Level) GetInfo(long number, Hash256 blockHash) + { + (BlockInfo Info, ChainLevelInfo Level) overlayInfo = _overlayTree.GetInfo(number, blockHash); + return overlayInfo.Info is not null || overlayInfo.Level is not null ? overlayInfo : _baseTree.GetInfo(number, blockHash); + } + + public ChainLevelInfo? FindLevel(long number) => _overlayTree.FindLevel(number) ?? _baseTree.FindLevel(number); + + public BlockInfo FindCanonicalBlockInfo(long blockNumber) => _overlayTree.FindCanonicalBlockInfo(blockNumber) ?? _baseTree.FindCanonicalBlockInfo(blockNumber); + + public Hash256 FindHash(long blockNumber) => _overlayTree.FindHash(blockNumber) ?? _baseTree.FindHash(blockNumber); + + public IOwnedReadOnlyList FindHeaders(Hash256 hash, int numberOfBlocks, int skip, bool reverse) + { + IOwnedReadOnlyList overlayHeaders = _overlayTree.FindHeaders(hash, numberOfBlocks, skip, reverse); + return overlayHeaders.Count > 0 ? overlayHeaders : _baseTree.FindHeaders(hash, numberOfBlocks, skip, reverse); + } + + public void DeleteInvalidBlock(Block invalidBlock) => + _overlayTree.DeleteInvalidBlock(invalidBlock); + + public void ForkChoiceUpdated(Hash256? finalizedBlockHash, Hash256? safeBlockBlockHash) => + _overlayTree.ForkChoiceUpdated(finalizedBlockHash, safeBlockBlockHash); + + public event EventHandler? NewBestSuggestedBlock + { + add + { + if (value is not null) + { + _baseTree.NewBestSuggestedBlock += value; + _overlayTree.NewBestSuggestedBlock += value; + } + } + remove + { + if (value is not null) + { + _baseTree.NewBestSuggestedBlock -= value; + _overlayTree.NewBestSuggestedBlock -= value; + } + } + } + + public event EventHandler? NewSuggestedBlock + { + add + { + if (value is not null) + { + _baseTree.NewSuggestedBlock += value; + _overlayTree.NewSuggestedBlock += value; + } + } + remove + { + if (value is not null) + { + _baseTree.NewSuggestedBlock -= value; + _overlayTree.NewSuggestedBlock -= value; + } + } + } + + public event EventHandler? BlockAddedToMain + { + add + { + if (value is not null) + { + _baseTree.BlockAddedToMain += value; + _overlayTree.BlockAddedToMain += value; + } + } + remove + { + if (value is not null) + { + _baseTree.BlockAddedToMain -= value; + _overlayTree.BlockAddedToMain -= value; + } + } + } + + public event EventHandler? NewHeadBlock + { + add + { + if (value is not null) + { + _baseTree.NewHeadBlock += value; + _overlayTree.NewHeadBlock += value; + } + } + remove + { + if (value is not null) + { + _baseTree.NewHeadBlock -= value; + _overlayTree.NewHeadBlock -= value; + } + } + } + + public event EventHandler? OnUpdateMainChain + { + add + { + if (value is not null) + { + _baseTree.OnUpdateMainChain += value; + _overlayTree.OnUpdateMainChain += value; + } + } + remove + { + if (value is not null) + { + _baseTree.OnUpdateMainChain -= value; + _overlayTree.OnUpdateMainChain -= value; + } + } + } + + public int DeleteChainSlice(in long startNumber, long? endNumber = null, bool force = false) => + _overlayTree.DeleteChainSlice(startNumber, endNumber, force); + + public bool IsBetterThanHead(BlockHeader? header) => _overlayTree.IsBetterThanHead(header) || _baseTree.IsBetterThanHead(header); + + public void UpdateBeaconMainChain(BlockInfo[]? blockInfos, long clearBeaconMainChainStartPoint) => + _overlayTree.UpdateBeaconMainChain(blockInfos, clearBeaconMainChainStartPoint); + + public void RecalculateTreeLevels() => _overlayTree.RecalculateTreeLevels(); + + public Block? FindBlock(Hash256 blockHash, BlockTreeLookupOptions options, long? blockNumber = null) => + _overlayTree.FindBlock(blockHash, options, blockNumber) ?? _baseTree.FindBlock(blockHash, options, blockNumber); + + public Block? FindBlock(long blockNumber, BlockTreeLookupOptions options) => + _overlayTree.FindBlock(blockNumber, options) ?? _baseTree.FindBlock(blockNumber, options); + + public BlockHeader? FindHeader(Hash256 blockHash, BlockTreeLookupOptions options, long? blockNumber = null) => + _overlayTree.FindHeader(blockHash, options, blockNumber) ?? _baseTree.FindHeader(blockHash, options, blockNumber); + + public BlockHeader? FindHeader(long blockNumber, BlockTreeLookupOptions options) => + _overlayTree.FindHeader(blockNumber, options) ?? _baseTree.FindHeader(blockNumber, options); + + public Hash256? FindBlockHash(long blockNumber) => + _overlayTree.FindBlockHash(blockNumber) ?? _baseTree.FindBlockHash(blockNumber); + + public bool IsMainChain(BlockHeader blockHeader) => + _baseTree.IsMainChain(blockHeader) || _overlayTree.IsMainChain(blockHeader); + + public bool IsMainChain(Hash256 blockHash, bool throwOnMissingHash = true) => + _baseTree.IsMainChain(blockHash, false) || _overlayTree.IsMainChain(blockHash, throwOnMissingHash); + + public BlockHeader FindBestSuggestedHeader() => + _overlayTree.FindBestSuggestedHeader() ?? _baseTree.FindBestSuggestedHeader(); +} diff --git a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs index 29fcdcf5d33..20aee80ea04 100644 --- a/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Blocks/BlockStore.cs @@ -18,7 +18,7 @@ public class BlockStore : IBlockStore { private readonly IDb _blockDb; private readonly BlockDecoder _blockDecoder = new(); - private const int CacheSize = 128 + 32; + public const int CacheSize = 128 + 32; private readonly LruCache _blockCache = new(CacheSize, CacheSize, "blocks"); diff --git a/src/Nethermind/Nethermind.Blockchain/Contracts/Contract.ConstantContract.cs b/src/Nethermind/Nethermind.Blockchain/Contracts/Contract.ConstantContract.cs index 2d2255b0045..d8b3f164d50 100644 --- a/src/Nethermind/Nethermind.Blockchain/Contracts/Contract.ConstantContract.cs +++ b/src/Nethermind/Nethermind.Blockchain/Contracts/Contract.ConstantContract.cs @@ -58,7 +58,7 @@ protected ConstantContractBase(Contract contract) protected Transaction GenerateTransaction(CallInfo callInfo) => _contract.GenerateTransaction(callInfo.ContractAddress, callInfo.FunctionName, callInfo.Sender, DefaultConstantContractGasLimit, callInfo.ParentHeader, callInfo.Arguments); - protected byte[] CallCore(CallInfo callInfo, IReadOnlyTransactionProcessor readOnlyTransactionProcessor, Transaction transaction) => + protected byte[] CallCore(CallInfo callInfo, ITransactionProcessor readOnlyTransactionProcessor, Transaction transaction) => _contract.CallCore(readOnlyTransactionProcessor, callInfo.ParentHeader, callInfo.FunctionName, transaction, true); protected object[] DecodeReturnData(string functionName, byte[] data) => _contract.DecodeReturnData(functionName, data); @@ -85,17 +85,17 @@ public override object[] Call(CallInfo callInfo) lock (_readOnlyTxProcessorSource) { - using var readOnlyTransactionProcessor = _readOnlyTxProcessorSource.Build(GetState(callInfo.ParentHeader)); - return CallRaw(callInfo, readOnlyTransactionProcessor); + using var scope = _readOnlyTxProcessorSource.Build(GetState(callInfo.ParentHeader)); + return CallRaw(callInfo, scope); } } - protected virtual object[] CallRaw(CallInfo callInfo, IReadOnlyTransactionProcessor readOnlyTransactionProcessor) + protected virtual object[] CallRaw(CallInfo callInfo, IReadOnlyTxProcessingScope scope) { var transaction = GenerateTransaction(callInfo); - if (_contract.ContractAddress is not null && readOnlyTransactionProcessor.IsContractDeployed(_contract.ContractAddress)) + if (_contract.ContractAddress is not null && scope.WorldState.IsContract(_contract.ContractAddress)) { - var result = CallCore(callInfo, readOnlyTransactionProcessor, transaction); + var result = CallCore(callInfo, scope.TransactionProcessor, transaction); return callInfo.Result = _contract.DecodeReturnData(callInfo.FunctionName, result); } else if (callInfo.MissingContractResult is not null) diff --git a/src/Nethermind/Nethermind.Blockchain/Find/IBlockFinder.cs b/src/Nethermind/Nethermind.Blockchain/Find/IBlockFinder.cs index 69299ab7dda..e65b57f8c49 100644 --- a/src/Nethermind/Nethermind.Blockchain/Find/IBlockFinder.cs +++ b/src/Nethermind/Nethermind.Blockchain/Find/IBlockFinder.cs @@ -43,8 +43,9 @@ public interface IBlockFinder /// Checks if the block is currently in the canonical chain /// /// Hash of the block to check + /// If should throw when hash is not found /// True if part of the canonical chain, otherwise False - bool IsMainChain(Hash256 blockHash); + bool IsMainChain(Hash256 blockHash, bool throwOnMissingHash = true); public Block? FindBlock(Hash256 blockHash, long? blockNumber = null) => FindBlock(blockHash, BlockTreeLookupOptions.None, blockNumber); diff --git a/src/Nethermind/Nethermind.Blockchain/FullPruning/FullPruner.cs b/src/Nethermind/Nethermind.Blockchain/FullPruning/FullPruner.cs index 33e86022f79..e951e03226a 100644 --- a/src/Nethermind/Nethermind.Blockchain/FullPruning/FullPruner.cs +++ b/src/Nethermind/Nethermind.Blockchain/FullPruning/FullPruner.cs @@ -254,6 +254,16 @@ private async Task CopyTrie(IPruningContext pruning, Hash256 stateRoot, Cancella targetNodeStorage.Scheme = INodeStorage.KeyScheme.HalfPath; } + if (originalKeyScheme == INodeStorage.KeyScheme.Hash && targetNodeStorage.Scheme == INodeStorage.KeyScheme.HalfPath) + { + if (_logger.IsInfo) _logger.Info($"Converting from Hash key scheme to HalfPath key scheme"); + + if (_pruningConfig.FullPruningMemoryBudgetMb < 8000) + { + if (_logger.IsWarn) _logger.Warn($"Full pruning memory budget is less than 8 GB. Full pruning from Hash to HalfPath require more memory budget for efficient copy. Consider increasing full pruning memory budget to at least 8GB."); + } + } + VisitingOptions visitingOptions = new() { MaxDegreeOfParallelism = _pruningConfig.FullPruningMaxDegreeOfParallelism, diff --git a/src/Nethermind/Nethermind.Blockchain/IBlockTree.cs b/src/Nethermind/Nethermind.Blockchain/IBlockTree.cs index 1bfbfaa6f87..006f2de938b 100644 --- a/src/Nethermind/Nethermind.Blockchain/IBlockTree.cs +++ b/src/Nethermind/Nethermind.Blockchain/IBlockTree.cs @@ -153,8 +153,6 @@ AddBlockResult Insert(Block block, BlockTreeInsertBlockOptions insertBlockOption IOwnedReadOnlyList FindHeaders(Hash256 hash, int numberOfBlocks, int skip, bool reverse); - BlockHeader FindLowestCommonAncestor(BlockHeader firstDescendant, BlockHeader secondDescendant, long maxSearchDepth); - void DeleteInvalidBlock(Block invalidBlock); void ForkChoiceUpdated(Hash256? finalizedBlockHash, Hash256? safeBlockBlockHash); diff --git a/src/Nethermind/Nethermind.Blockchain/Processing/WitnessPruner.cs b/src/Nethermind/Nethermind.Blockchain/Processing/WitnessPruner.cs deleted file mode 100644 index c2863c3e354..00000000000 --- a/src/Nethermind/Nethermind.Blockchain/Processing/WitnessPruner.cs +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using Nethermind.Blockchain; -using Nethermind.Core; -using Nethermind.Logging; -using Nethermind.State; - -namespace Nethermind.Synchronization.Witness -{ - public static class WitnessCollectorExtensions - { - public static IWitnessRepository WithPruning( - this IWitnessRepository repository, - IBlockTree blockTree, - ILogManager logManager, - int followDistance = 16) - { - new WitnessPruner(blockTree, repository, logManager, followDistance).Start(); - return repository; - } - } - - public class WitnessPruner - { - private readonly IBlockTree _blockTree; - private readonly IWitnessRepository _witnessRepository; - private readonly int _followDistance; - private readonly ILogger _logger; - - public WitnessPruner(IBlockTree blockTree, IWitnessRepository witnessRepository, ILogManager logManager, int followDistance = 16) - { - _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); - _witnessRepository = witnessRepository ?? throw new ArgumentNullException(nameof(witnessRepository)); - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _followDistance = followDistance; - } - - public void Start() - { - _blockTree.NewHeadBlock += OnNewHeadBlock; - } - - private void OnNewHeadBlock(object? sender, BlockEventArgs e) - { - long toPrune = e.Block.Number - _followDistance; - if (toPrune > 0) - { - var level = _blockTree.FindLevel(toPrune); - if (level is not null) - { - if (_logger.IsTrace) _logger.Trace($"Pruning witness from blocks with number {toPrune}"); - - for (int i = 0; i < level.BlockInfos.Length; i++) - { - var blockInfo = level.BlockInfos[i]; - if (blockInfo.BlockHash is not null) - { - _witnessRepository.Delete(blockInfo.BlockHash); - } - } - } - } - } - } -} diff --git a/src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs b/src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs index 5ceaacf2495..4fc9e1077e8 100644 --- a/src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs +++ b/src/Nethermind/Nethermind.Blockchain/ReadOnlyBlockTree.cs @@ -106,13 +106,11 @@ public void UpdateHeadBlock(Hash256 blockHash) public IOwnedReadOnlyList FindHeaders(Hash256 hash, int numberOfBlocks, int skip, bool reverse) => _wrapped.FindHeaders(hash, numberOfBlocks, skip, reverse); - public BlockHeader FindLowestCommonAncestor(BlockHeader firstDescendant, BlockHeader secondDescendant, long maxSearchDepth) => _wrapped.FindLowestCommonAncestor(firstDescendant, secondDescendant, maxSearchDepth); - public Block FindBlock(long blockNumber, BlockTreeLookupOptions options) => _wrapped.FindBlock(blockNumber, options); public void DeleteInvalidBlock(Block invalidBlock) => throw new InvalidOperationException($"{nameof(ReadOnlyBlockTree)} does not expect {nameof(DeleteInvalidBlock)} calls"); - public bool IsMainChain(Hash256 blockHash) => _wrapped.IsMainChain(blockHash); + public bool IsMainChain(Hash256 blockHash, bool throwOnMissingHash = true) => _wrapped.IsMainChain(blockHash, throwOnMissingHash); public BlockHeader FindBestSuggestedHeader() => _wrapped.FindBestSuggestedHeader(); diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs index 2d2ef1695c4..7a3ce9593f4 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs @@ -256,7 +256,7 @@ public bool TryGetReceiptsIterator(long blockNumber, Hash256 blockHash, out Rece }; } - IReceiptRefDecoder refDecoder = ReceiptArrayStorageDecoder.GetRefDecoder(receiptsData); + IReceiptRefDecoder refDecoder = _storageDecoder.GetRefDecoder(receiptsData); iterator = result ? new ReceiptsIterator(receiptsData, _blocksDb, recoveryContextFactory, refDecoder) : new ReceiptsIterator(); return result; diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRootCalculator.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRootCalculator.cs index a70f397f827..5bd7d32bb59 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRootCalculator.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRootCalculator.cs @@ -13,6 +13,8 @@ public class ReceiptsRootCalculator : IReceiptsRootCalculator { public static readonly ReceiptsRootCalculator Instance = new(); + private static readonly IRlpStreamDecoder _decoder = Rlp.GetStreamDecoder(RlpDecoderKey.Trie); + public Hash256 GetReceiptsRoot(TxReceipt[] receipts, IReceiptSpec spec, Hash256? suggestedRoot) { Hash256 SkipStateAndStatusReceiptsRoot() @@ -20,7 +22,7 @@ Hash256 SkipStateAndStatusReceiptsRoot() receipts.SetSkipStateAndStatusInRlp(true); try { - return ReceiptTrie.CalculateRoot(spec, receipts, ReceiptMessageDecoder.Instance); + return ReceiptTrie.CalculateRoot(spec, receipts, _decoder); } finally { @@ -28,7 +30,7 @@ Hash256 SkipStateAndStatusReceiptsRoot() } } - Hash256 receiptsRoot = ReceiptTrie.CalculateRoot(spec, receipts, ReceiptMessageDecoder.Instance); + Hash256 receiptsRoot = ReceiptTrie.CalculateRoot(spec, receipts, _decoder); if (!spec.ValidateReceipts && receiptsRoot != suggestedRoot) { var skipStateAndStatusReceiptsRoot = SkipStateAndStatusReceiptsRoot(); diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs index a647aa0988c..551a9856a33 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncConfig.cs @@ -93,9 +93,6 @@ public interface ISyncConfig : IConfig [ConfigItem(DisabledForCli = true, HiddenFromDocs = true, DefaultValue = "1")] public long AncientReceiptsBarrierCalc => Math.Max(1, Math.Min(PivotNumberParsed, Math.Max(AncientBodiesBarrier, AncientReceiptsBarrier))); - [ConfigItem(Description = "Whether to enable the Witness protocol.", DefaultValue = "false")] - public bool WitnessProtocolEnabled { get; set; } - [ConfigItem(Description = "Whether to use the Snap sync mode.", DefaultValue = "false")] public bool SnapSync { get; set; } diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncPeer.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncPeer.cs index a51275ff58c..82e52beb0ef 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncPeer.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/ISyncPeer.cs @@ -13,11 +13,6 @@ namespace Nethermind.Blockchain.Synchronization { - public interface IWitnessPeer - { - Task GetBlockWitnessHashes(Hash256 blockHash, CancellationToken token); - } - public interface ISyncPeer : ITxPoolPeer, IPeerWithSatelliteProtocol { Node Node { get; } diff --git a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs index c896c206c75..d2645336586 100644 --- a/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs +++ b/src/Nethermind/Nethermind.Blockchain/Synchronization/SyncConfig.cs @@ -48,7 +48,6 @@ public string? PivotHash set => _pivotHash = value; } public int MaxAttemptsToUpdatePivot { get; set; } = int.MaxValue; - public bool WitnessProtocolEnabled { get; set; } = false; public bool SnapSync { get; set; } = false; public int SnapSyncAccountRangePartitionCount { get; set; } = 8; public bool FixReceipts { get; set; } = false; diff --git a/src/Nethermind/Nethermind.Blockchain/Utils/LastNStateRootTracker.cs b/src/Nethermind/Nethermind.Blockchain/Utils/LastNStateRootTracker.cs index 9a15de970ce..b9ca08c0792 100644 --- a/src/Nethermind/Nethermind.Blockchain/Utils/LastNStateRootTracker.cs +++ b/src/Nethermind/Nethermind.Blockchain/Utils/LastNStateRootTracker.cs @@ -29,7 +29,7 @@ public LastNStateRootTracker(IBlockTree blockTree, int lastN) _lastN = lastN; _blockTree.BlockAddedToMain += BlockTreeOnNewHeadBlock; - if (_blockTree.Head != null) ResetAvailableStateRoots(_blockTree.Head.Header, true); + if (_blockTree.Head is not null) ResetAvailableStateRoots(_blockTree.Head.Header, true); } private void BlockTreeOnNewHeadBlock(object? sender, BlockEventArgs e) diff --git a/src/Nethermind/Nethermind.Cli.Test/ProofCliModuleTests.cs b/src/Nethermind/Nethermind.Cli.Test/ProofCliModuleTests.cs index dac449f4a53..3eac4d9207c 100644 --- a/src/Nethermind/Nethermind.Cli.Test/ProofCliModuleTests.cs +++ b/src/Nethermind/Nethermind.Cli.Test/ProofCliModuleTests.cs @@ -7,6 +7,7 @@ using Nethermind.Cli.Modules; using Nethermind.Core.Crypto; using Nethermind.Core.Test.Builders; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc; using Nethermind.JsonRpc.Client; using Nethermind.JsonRpc.Data; diff --git a/src/Nethermind/Nethermind.Cli/Modules/EthCliModule.cs b/src/Nethermind/Nethermind.Cli/Modules/EthCliModule.cs index adba54d9821..961ce37bfe9 100644 --- a/src/Nethermind/Nethermind.Cli/Modules/EthCliModule.cs +++ b/src/Nethermind/Nethermind.Cli/Modules/EthCliModule.cs @@ -7,6 +7,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; +using Nethermind.Facade.Eth; using Nethermind.Int256; using Nethermind.JsonRpc.Data; @@ -49,6 +50,10 @@ public JsValue GetProof(string address, string[] storageKeys, string? blockParam return NodeManager.Post("eth_call", tx, blockParameter ?? "latest").Result; } + [CliFunction("eth", "simulateV1")] + public JsValue SimulateV1(ulong version, object[] blockCalls, string? blockParameter = null, bool traceTransfers = true) => + NodeManager.PostJint("eth_simulateV1", 1, blockCalls, blockParameter ?? "latest", traceTransfers).Result; + [CliFunction("eth", "getBlockByHash")] public JsValue GetBlockByHash(string hash, bool returnFullTransactionObjects) { diff --git a/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs b/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs index d30115175e0..a21d6bf5c45 100644 --- a/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs +++ b/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using FluentAssertions; using Nethermind.Blockchain; using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Receipts; @@ -55,7 +56,7 @@ private class On private readonly Dictionary _snapshotManager = new(); private readonly Dictionary _blockTrees = new(); private readonly Dictionary _blockEvents = new(); - private readonly Dictionary _producers = new(); + private readonly Dictionary _producers = new(); private readonly Dictionary _pools = new(); private On() @@ -127,8 +128,10 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f _genesis.Header.Hash = _genesis.Header.CalculateHash(); _genesis3Validators.Header.Hash = _genesis3Validators.Header.CalculateHash(); + CodeInfoRepository codeInfoRepository = new(); TransactionProcessor transactionProcessor = new(goerliSpecProvider, stateProvider, - new VirtualMachine(blockhashProvider, specProvider, nodeLogManager), + new VirtualMachine(blockhashProvider, specProvider, codeInfoRepository, nodeLogManager), + codeInfoRepository, nodeLogManager); BlockProcessor blockProcessor = new( goerliSpecProvider, @@ -137,7 +140,6 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f new BlockProcessor.BlockValidationTransactionsExecutor(transactionProcessor, stateProvider), stateProvider, NullReceiptStorage.Instance, - NullWitnessCollector.Instance, new BlockhashStore(blockTree, goerliSpecProvider, stateProvider), transactionProcessor, nodeLogManager); @@ -148,8 +150,8 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f IReadOnlyTrieStore minerTrieStore = trieStore.AsReadOnly(); WorldState minerStateProvider = new(minerTrieStore, codeDb, nodeLogManager); - VirtualMachine minerVirtualMachine = new(blockhashProvider, specProvider, nodeLogManager); - TransactionProcessor minerTransactionProcessor = new(goerliSpecProvider, minerStateProvider, minerVirtualMachine, nodeLogManager); + VirtualMachine minerVirtualMachine = new(blockhashProvider, specProvider, codeInfoRepository, nodeLogManager); + TransactionProcessor minerTransactionProcessor = new(goerliSpecProvider, minerStateProvider, minerVirtualMachine, codeInfoRepository, nodeLogManager); BlockProcessor minerBlockProcessor = new( goerliSpecProvider, @@ -158,7 +160,6 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f new BlockProcessor.BlockProductionTransactionsExecutor(minerTransactionProcessor, minerStateProvider, goerliSpecProvider, _logManager), minerStateProvider, NullReceiptStorage.Instance, - NullWitnessCollector.Instance, new BlockhashStore(blockTree, goerliSpecProvider, minerStateProvider), minerTransactionProcessor, nodeLogManager); @@ -179,7 +180,6 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f txPoolTxSource, minerProcessor, minerStateProvider, - blockTree, _timestamper, new CryptoRandom(), snapshotManager, @@ -188,11 +188,21 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f MainnetSpecProvider.Instance, _cliqueConfig, nodeLogManager); - blockProducer.Start(); - ProducedBlockSuggester suggester = new ProducedBlockSuggester(blockTree, blockProducer); + CliqueBlockProducerRunner producerRunner = new CliqueBlockProducerRunner( + blockTree, + _timestamper, + new CryptoRandom(), + snapshotManager, + blockProducer, + _cliqueConfig, + LimboLogs.Instance); + + producerRunner.Start(); + + ProducedBlockSuggester suggester = new ProducedBlockSuggester(blockTree, producerRunner); - _producers.Add(privateKey, blockProducer); + _producers.Add(privateKey, producerRunner); return this; } @@ -253,7 +263,7 @@ public On UncastVote(PrivateKey nodeId, Address address) public On IsProducingBlocks(PrivateKey nodeId, bool expected, ulong? maxInterval) { if (_logger.IsInfo) _logger.Info($"IsProducingBlocks"); - Assert.That(((IBlockProducer)_producers[nodeId]).IsProducingBlocks(maxInterval), Is.EqualTo(expected)); + Assert.That(((IBlockProducerRunner)_producers[nodeId]).IsProducingBlocks(maxInterval), Is.EqualTo(expected)); return this; } diff --git a/src/Nethermind/Nethermind.Clique.Test/CliqueRpcModuleTests.cs b/src/Nethermind/Nethermind.Clique.Test/CliqueRpcModuleTests.cs index b1f68c90b37..cf8cc178f51 100644 --- a/src/Nethermind/Nethermind.Clique.Test/CliqueRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.Clique.Test/CliqueRpcModuleTests.cs @@ -36,7 +36,6 @@ public void Sets_clique_block_producer_properly() Substitute.For(), Substitute.For(), Substitute.For(), - blockTree, Substitute.For(), Substitute.For(), Substitute.For(), @@ -48,7 +47,16 @@ public void Sets_clique_block_producer_properly() SnapshotManager snapshotManager = new(CliqueConfig.Default, new MemDb(), Substitute.For(), NullEthereumEcdsa.Instance, LimboLogs.Instance); - CliqueRpcModule bridge = new(producer, snapshotManager, blockTree); + CliqueBlockProducerRunner producerRunner = new CliqueBlockProducerRunner( + blockTree, + Substitute.For(), + Substitute.For(), + snapshotManager, + producer, + cliqueConfig, + LimboLogs.Instance); + + CliqueRpcModule bridge = new(producerRunner, snapshotManager, blockTree); Assert.DoesNotThrow(() => bridge.CastVote(TestItem.AddressB, true)); Assert.DoesNotThrow(() => bridge.UncastVote(TestItem.AddressB)); Assert.DoesNotThrow(() => bridge.CastVote(TestItem.AddressB, false)); @@ -63,7 +71,7 @@ public void Can_ask_for_block_signer() BlockHeader header = Build.A.BlockHeader.TestObject; blockFinder.FindHeader(Arg.Any()).Returns(header); snapshotManager.GetBlockSealer(header).Returns(TestItem.AddressA); - CliqueRpcModule rpcModule = new(Substitute.For(), snapshotManager, blockFinder); + CliqueRpcModule rpcModule = new(Substitute.For(), snapshotManager, blockFinder); rpcModule.clique_getBlockSigner(Keccak.Zero).Result.ResultType.Should().Be(ResultType.Success); rpcModule.clique_getBlockSigner(Keccak.Zero).Data.Should().Be(TestItem.AddressA); } @@ -74,7 +82,7 @@ public void Can_ask_for_block_signer_when_block_is_unknown() ISnapshotManager snapshotManager = Substitute.For(); IBlockFinder blockFinder = Substitute.For(); blockFinder.FindHeader(Arg.Any()).Returns((BlockHeader)null); - CliqueRpcModule rpcModule = new(Substitute.For(), snapshotManager, blockFinder); + CliqueRpcModule rpcModule = new(Substitute.For(), snapshotManager, blockFinder); rpcModule.clique_getBlockSigner(Keccak.Zero).Result.ResultType.Should().Be(ResultType.Failure); } @@ -83,7 +91,7 @@ public void Can_ask_for_block_signer_when_hash_is_null() { ISnapshotManager snapshotManager = Substitute.For(); IBlockFinder blockFinder = Substitute.For(); - CliqueRpcModule rpcModule = new(Substitute.For(), snapshotManager, blockFinder); + CliqueRpcModule rpcModule = new(Substitute.For(), snapshotManager, blockFinder); rpcModule.clique_getBlockSigner(null).Result.ResultType.Should().Be(ResultType.Failure); } } diff --git a/src/Nethermind/Nethermind.Clique.Test/CliqueTests.cs b/src/Nethermind/Nethermind.Clique.Test/CliqueTests.cs index 56bc5325969..21c436e18ca 100644 --- a/src/Nethermind/Nethermind.Clique.Test/CliqueTests.cs +++ b/src/Nethermind/Nethermind.Clique.Test/CliqueTests.cs @@ -13,7 +13,9 @@ using Nethermind.Int256; using Nethermind.Logging; using Nethermind.Serialization.Rlp; +using NSubstitute; using NUnit.Framework; +using System.Threading.Tasks; using BlockTree = Nethermind.Blockchain.BlockTree; namespace Nethermind.Clique.Test @@ -91,6 +93,37 @@ public void Test_no_signer_data_at_epoch_fails(string blockRlp) Assert.True(validSeal); } + [TestCase(Block4Rlp)] + public async Task SealBlock_SignerCanSignHeader_FullHeaderIsUsedToSign(string blockRlp) + { + IHeaderSigner signer = Substitute.For(); + signer.CanSignHeader.Returns(true); + signer.CanSign.Returns(true); + signer.Address.Returns(new Address("0x7ffc57839b00206d1ad20c69a1981b489f772031")); + signer.Sign(Arg.Any()).Returns(new Signature(new byte[65])); + CliqueSealer sut = new CliqueSealer(signer, new CliqueConfig(), _snapshotManager, LimboLogs.Instance); + Block block = Rlp.Decode(new Rlp(Bytes.FromHexString(blockRlp))); + + await sut.SealBlock(block, System.Threading.CancellationToken.None); + + signer.Received().Sign(Arg.Any()); + } + + [TestCase(Block4Rlp)] + public async Task SealBlock_SignerCannotSignHeader_HashIsUsedToSign(string blockRlp) + { + ISigner signer = Substitute.For(); + signer.CanSign.Returns(true); + signer.Address.Returns(new Address("0x7ffc57839b00206d1ad20c69a1981b489f772031")); + signer.Sign(Arg.Any()).Returns(new Signature(new byte[65])); + CliqueSealer sut = new CliqueSealer(signer, new CliqueConfig(), _snapshotManager, LimboLogs.Instance); + Block block = Rlp.Decode(new Rlp(Bytes.FromHexString(blockRlp))); + + await sut.SealBlock(block, System.Threading.CancellationToken.None); + + signer.Received().Sign(Arg.Any()); + } + public static Block GetGenesis() { Hash256 parentHash = Keccak.Zero; diff --git a/src/Nethermind/Nethermind.Config/BlocksConfig.cs b/src/Nethermind/Nethermind.Config/BlocksConfig.cs index 4a6334f40b4..9a12edfb5d4 100644 --- a/src/Nethermind/Nethermind.Config/BlocksConfig.cs +++ b/src/Nethermind/Nethermind.Config/BlocksConfig.cs @@ -24,6 +24,7 @@ public class BlocksConfig : IBlocksConfig public ulong SecondsPerSlot { get; set; } = 12; + public bool PreWarmStateOnBlockProcessing { get; set; } = true; public string ExtraData { diff --git a/src/Nethermind/Nethermind.Config/IBlocksConfig.cs b/src/Nethermind/Nethermind.Config/IBlocksConfig.cs index a78820f5f56..8f232cf2b02 100644 --- a/src/Nethermind/Nethermind.Config/IBlocksConfig.cs +++ b/src/Nethermind/Nethermind.Config/IBlocksConfig.cs @@ -34,5 +34,8 @@ public interface IBlocksConfig : IConfig [ConfigItem(Description = "The block time slot, in seconds.", DefaultValue = "12")] ulong SecondsPerSlot { get; set; } + [ConfigItem(Description = "Try to pre-warm the state when processing blocks. Can lead to 2x speedup in main loop block processing.", DefaultValue = "True")] + bool PreWarmStateOnBlockProcessing { get; set; } + byte[] GetExtraDataBytes(); } diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProcessor.cs b/src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProcessor.cs index a926ebdb7c7..6d259b36940 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProcessor.cs @@ -57,7 +57,6 @@ public AuRaBlockProcessor( blockTransactionsExecutor, stateProvider, receiptStorage, - NullWitnessCollector.Instance, new BlockhashStore(blockTree, specProvider, stateProvider), transactionProcessor, logManager, diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProducer.cs b/src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProducer.cs index 3db92361804..b74ae8cc935 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProducer.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/AuRaBlockProducer.cs @@ -27,7 +27,6 @@ public class AuRaBlockProducer : BlockProducerBase public AuRaBlockProducer(ITxSource txSource, IBlockchainProcessor processor, - IBlockProductionTrigger blockProductionTrigger, IWorldState stateProvider, ISealer sealer, IBlockTree blockTree, @@ -44,7 +43,6 @@ public AuRaBlockProducer(ITxSource txSource, processor, sealer, blockTree, - blockProductionTrigger, stateProvider, gasLimitCalculator, timestamper, diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/AuRaPlugin.cs b/src/Nethermind/Nethermind.Consensus.AuRa/AuRaPlugin.cs index 558433e9e78..f3c766da5f0 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/AuRaPlugin.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/AuRaPlugin.cs @@ -31,6 +31,8 @@ public class AuRaPlugin : IConsensusPlugin, ISynchronizationPlugin, IInitializat public string SealEngineType => Core.SealEngineType.AuRa; + private StartBlockProducerAuRa? _blockProducerStarter; + public ValueTask DisposeAsync() { @@ -40,16 +42,10 @@ public ValueTask DisposeAsync() public Task Init(INethermindApi nethermindApi) { _nethermindApi = nethermindApi as AuRaNethermindApi; - return Task.CompletedTask; - } - - public Task InitNetworkProtocol() - { - return Task.CompletedTask; - } - - public Task InitRpcModules() - { + if (_nethermindApi is not null) + { + _blockProducerStarter = new(_nethermindApi); + } return Task.CompletedTask; } @@ -63,19 +59,23 @@ public Task InitSynchronization() return Task.CompletedTask; } - public Task InitBlockProducer(IBlockProductionTrigger? blockProductionTrigger = null, ITxSource? additionalTxSource = null) + public IBlockProducer InitBlockProducer(ITxSource? additionalTxSource = null) { if (_nethermindApi is not null) { - StartBlockProducerAuRa blockProducerStarter = new(_nethermindApi); - DefaultBlockProductionTrigger ??= blockProducerStarter.CreateTrigger(); - return blockProducerStarter.BuildProducer(blockProductionTrigger ?? DefaultBlockProductionTrigger, additionalTxSource); + return _blockProducerStarter!.BuildProducer(additionalTxSource); } - return Task.FromResult(null); + return null; } - public IBlockProductionTrigger? DefaultBlockProductionTrigger { get; private set; } + public IBlockProducerRunner CreateBlockProducerRunner() + { + return new StandardBlockProducerRunner( + _blockProducerStarter.CreateTrigger(), + _nethermindApi.BlockTree, + _nethermindApi.BlockProducer!); + } public INethermindApi CreateApi(IConfigProvider configProvider, IJsonSerializer jsonSerializer, ILogManager logManager, ChainSpec chainSpec) => new AuRaNethermindApi(configProvider, jsonSerializer, logManager, chainSpec); diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContract.cs b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContract.cs index 8efb2163ab2..28f57bded5d 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContract.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/Contracts/TransactionPermissionContract.cs @@ -109,14 +109,14 @@ public PermissionConstantContract(Contract contract, IReadOnlyTxProcessorSource { } - protected override object[] CallRaw(CallInfo callInfo, IReadOnlyTransactionProcessor readOnlyTransactionProcessor) + protected override object[] CallRaw(CallInfo callInfo, IReadOnlyTxProcessingScope scope) { if (callInfo is PermissionCallInfo transactionPermissionCallInfo) { - transactionPermissionCallInfo.ToIsContract = readOnlyTransactionProcessor.IsContractDeployed(transactionPermissionCallInfo.To); + transactionPermissionCallInfo.ToIsContract = scope.WorldState.IsContract(transactionPermissionCallInfo.To); } - return base.CallRaw(callInfo, readOnlyTransactionProcessor); + return base.CallRaw(callInfo, scope); } public class PermissionCallInfo : CallInfo diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/AuRaNethermindApi.cs b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/AuRaNethermindApi.cs index 12151507209..c3822f5edd0 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/AuRaNethermindApi.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/AuRaNethermindApi.cs @@ -56,7 +56,7 @@ public TxPriorityContract.LocalDataSource? TxPriorityContractLocalDataSource { get { - if (_txPriorityContractLocalDataSource != null) return _txPriorityContractLocalDataSource; + if (_txPriorityContractLocalDataSource is not null) return _txPriorityContractLocalDataSource; IAuraConfig config = this.Config(); string? auraConfigTxPriorityConfigFilePath = config.TxPriorityConfigFilePath; diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/StartBlockProducerAuRa.cs b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/StartBlockProducerAuRa.cs index a6a8d0ff24f..4abf23d082e 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/StartBlockProducerAuRa.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/StartBlockProducerAuRa.cs @@ -20,6 +20,7 @@ using Nethermind.Consensus.Producers; using Nethermind.Consensus.Transactions; using Nethermind.Core; +using Nethermind.Core.Crypto; using Nethermind.Crypto; using Nethermind.Evm.TransactionProcessing; using Nethermind.Init.Steps; @@ -72,7 +73,7 @@ public IBlockProductionTrigger CreateTrigger() return onlyWhenNotProcessing; } - public Task BuildProducer(IBlockProductionTrigger blockProductionTrigger, ITxSource? additionalTxSource = null) + public IBlockProducer BuildProducer(ITxSource? additionalTxSource = null) { if (_api.EngineSigner is null) throw new StepDependencyException(nameof(_api.EngineSigner)); if (_api.ChainSpec is null) throw new StepDependencyException(nameof(_api.ChainSpec)); @@ -87,7 +88,6 @@ public Task BuildProducer(IBlockProductionTrigger blockProductio IBlockProducer blockProducer = new AuRaBlockProducer( producerEnv.TxSource, producerEnv.ChainProcessor, - blockProductionTrigger, producerEnv.ReadOnlyStateProvider, _api.Sealer, _api.BlockTree, @@ -100,10 +100,10 @@ public Task BuildProducer(IBlockProductionTrigger blockProductio _api.LogManager, _api.ConfigProvider.GetConfig()); - return Task.FromResult(blockProducer); + return blockProducer; } - private BlockProcessor CreateBlockProcessor(ReadOnlyTxProcessingEnv changeableTxProcessingEnv) + private BlockProcessor CreateBlockProcessor(IReadOnlyTxProcessingScope changeableTxProcessingEnv) { if (_api.RewardCalculatorSource is null) throw new StepDependencyException(nameof(_api.RewardCalculatorSource)); if (_api.ValidatorStore is null) throw new StepDependencyException(nameof(_api.ValidatorStore)); @@ -120,9 +120,9 @@ private BlockProcessor CreateBlockProcessor(ReadOnlyTxProcessingEnv changeableTx new LocalTxFilter(_api.EngineSigner)); _validator = new AuRaValidatorFactory(_api.AbiEncoder, - changeableTxProcessingEnv.StateProvider, + changeableTxProcessingEnv.WorldState, changeableTxProcessingEnv.TransactionProcessor, - changeableTxProcessingEnv.BlockTree, + _api.BlockTree, _api.CreateReadOnlyTransactionProcessorSource(), _api.ReceiptStorage, _api.ValidatorStore, @@ -152,10 +152,10 @@ private BlockProcessor CreateBlockProcessor(ReadOnlyTxProcessingEnv changeableTx _api.BlockValidator, _api.RewardCalculatorSource.Get(changeableTxProcessingEnv.TransactionProcessor), _api.BlockProducerEnvFactory.TransactionsExecutorFactory.Create(changeableTxProcessingEnv), - changeableTxProcessingEnv.StateProvider, + changeableTxProcessingEnv.WorldState, _api.ReceiptStorage, _api.LogManager, - changeableTxProcessingEnv.BlockTree, + _api.BlockTree, NullWithdrawalProcessor.Instance, _api.TransactionProcessor, _validator, @@ -164,7 +164,7 @@ private BlockProcessor CreateBlockProcessor(ReadOnlyTxProcessingEnv changeableTx contractRewriter); } - internal TxPoolTxSource CreateTxPoolTxSource(ReadOnlyTxProcessingEnv processingEnv) + internal TxPoolTxSource CreateTxPoolTxSource() { _txPriorityContract = TxAuRaFilterBuilders.CreateTxPrioritySources(_api); _localDataSource = _api.TxPriorityContractLocalDataSource; @@ -205,7 +205,7 @@ internal TxPoolTxSource CreateTxPoolTxSource(ReadOnlyTxProcessingEnv processingE return new TxPriorityTxSource( _api.TxPool, - processingEnv.StateReader, + _api.StateReader, _api.LogManager, txFilterPipeline, whitelistContractDataStore, @@ -227,27 +227,28 @@ BlockProducerEnv Create() ReadOnlyBlockTree readOnlyBlockTree = _api.BlockTree.AsReadOnly(); ReadOnlyTxProcessingEnv txProcessingEnv = _api.CreateReadOnlyTransactionProcessorSource(); - BlockProcessor blockProcessor = CreateBlockProcessor(txProcessingEnv); + IReadOnlyTxProcessingScope scope = txProcessingEnv.Build(Keccak.EmptyTreeHash); + BlockProcessor blockProcessor = CreateBlockProcessor(scope); IBlockchainProcessor blockchainProcessor = new BlockchainProcessor( readOnlyBlockTree, blockProcessor, _api.BlockPreprocessor, - txProcessingEnv.StateReader, + _api.StateReader, _api.LogManager, BlockchainProcessor.Options.NoReceipts); OneTimeChainProcessor chainProcessor = new( - txProcessingEnv.StateProvider, + scope.WorldState, blockchainProcessor); return new BlockProducerEnv() { BlockTree = readOnlyBlockTree, ChainProcessor = chainProcessor, - ReadOnlyStateProvider = txProcessingEnv.StateProvider, - TxSource = CreateTxSourceForProducer(txProcessingEnv, additionalTxSource), + ReadOnlyStateProvider = scope.WorldState, + TxSource = CreateTxSourceForProducer(additionalTxSource), ReadOnlyTxProcessingEnv = _api.CreateReadOnlyTransactionProcessorSource(), }; } @@ -255,8 +256,8 @@ BlockProducerEnv Create() return _blockProducerContext ??= Create(); } - private ITxSource CreateStandardTxSourceForProducer(ReadOnlyTxProcessingEnv processingEnv) => - CreateTxPoolTxSource(processingEnv); + private ITxSource CreateStandardTxSourceForProducer() => + CreateTxPoolTxSource(); private TxPoolTxSource CreateStandardTxPoolTxSource() { @@ -270,7 +271,7 @@ private TxPoolTxSource CreateStandardTxPoolTxSource() private ITxFilter CreateAuraTxFilterForProducer() => TxAuRaFilterBuilders.CreateAuRaTxFilterForProducer(_api, _minGasPricesContractDataStore); - private ITxSource CreateTxSourceForProducer(ReadOnlyTxProcessingEnv processingEnv, ITxSource? additionalTxSource) + private ITxSource CreateTxSourceForProducer(ITxSource? additionalTxSource) { bool CheckAddPosdaoTransactions(IList list, long auRaPosdaoTransition) { @@ -322,7 +323,7 @@ IList GetRandomContracts( if (_api.BlockTree is null) throw new StepDependencyException(nameof(_api.BlockTree)); if (_api.EngineSigner is null) throw new StepDependencyException(nameof(_api.EngineSigner)); - IList txSources = new List { CreateStandardTxSourceForProducer(processingEnv) }; + IList txSources = new List { CreateStandardTxSourceForProducer() }; bool needSigner = false; if (additionalTxSource is not null) @@ -337,7 +338,7 @@ IList GetRandomContracts( if (needSigner) { TxSealer transactionSealer = new TxSealer(_api.EngineSigner, _api.Timestamper); - txSource = new GeneratedTxSource(txSource, transactionSealer, processingEnv.StateReader, _api.LogManager); + txSource = new GeneratedTxSource(txSource, transactionSealer, _api.StateReader, _api.LogManager); } ITxFilter? txPermissionFilter = TxAuRaFilterBuilders.CreateTxPermissionFilter(_api); diff --git a/src/Nethermind/Nethermind.Consensus.Clique/CliqueBlockProducer.cs b/src/Nethermind/Nethermind.Consensus.Clique/CliqueBlockProducer.cs index 53d798834a8..8d9fb4007a1 100644 --- a/src/Nethermind/Nethermind.Consensus.Clique/CliqueBlockProducer.cs +++ b/src/Nethermind/Nethermind.Consensus.Clique/CliqueBlockProducer.cs @@ -27,54 +27,38 @@ namespace Nethermind.Consensus.Clique; -public class CliqueBlockProducer : ICliqueBlockProducer, IDisposable +public class CliqueBlockProducerRunner : ICliqueBlockProducerRunner, IDisposable { private readonly IBlockTree _blockTree; - private readonly IWorldState _stateProvider; private readonly ITimestamper _timestamper; private readonly ILogger _logger; private readonly ICryptoRandom _cryptoRandom; private readonly WiggleRandomizer _wiggle; - private readonly ITxSource _txSource; - private readonly IBlockchainProcessor _processor; - private readonly ISealer _sealer; - private readonly IGasLimitCalculator _gasLimitCalculator; - private readonly ISpecProvider _specProvider; private readonly ISnapshotManager _snapshotManager; private readonly ICliqueConfig _config; - private readonly ConcurrentDictionary _proposals = new(); - private readonly CancellationTokenSource _cancellationTokenSource = new(); private readonly System.Timers.Timer _timer = new(); private DateTime _lastProducedBlock; - public CliqueBlockProducer( - ITxSource txSource, - IBlockchainProcessor blockchainProcessor, - IWorldState stateProvider, + private CliqueBlockProducer _blockProducer; + + public CliqueBlockProducerRunner( IBlockTree blockTree, ITimestamper timestamper, ICryptoRandom cryptoRandom, ISnapshotManager snapshotManager, - ISealer cliqueSealer, - IGasLimitCalculator gasLimitCalculator, - ISpecProvider? specProvider, + CliqueBlockProducer blockProducer, ICliqueConfig config, ILogManager logManager) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _txSource = txSource ?? throw new ArgumentNullException(nameof(txSource)); - _processor = blockchainProcessor ?? throw new ArgumentNullException(nameof(blockchainProcessor)); _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); - _stateProvider = stateProvider ?? throw new ArgumentNullException(nameof(stateProvider)); _timestamper = timestamper ?? throw new ArgumentNullException(nameof(timestamper)); _cryptoRandom = cryptoRandom ?? throw new ArgumentNullException(nameof(cryptoRandom)); - _sealer = cliqueSealer ?? throw new ArgumentNullException(nameof(cliqueSealer)); - _gasLimitCalculator = gasLimitCalculator ?? throw new ArgumentNullException(nameof(gasLimitCalculator)); - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); _snapshotManager = snapshotManager ?? throw new ArgumentNullException(nameof(snapshotManager)); + _blockProducer = blockProducer; _config = config ?? throw new ArgumentNullException(nameof(config)); _wiggle = new WiggleRandomizer(_cryptoRandom, _snapshotManager); @@ -91,7 +75,7 @@ public CliqueBlockProducer( public void CastVote(Address signer, bool vote) { - bool success = _proposals.TryAdd(signer, vote); + bool success = _blockProducer.Proposals.TryAdd(signer, vote); if (!success) { throw new InvalidOperationException($"A vote for {signer} has already been cast."); @@ -102,7 +86,7 @@ public void CastVote(Address signer, bool vote) public void UncastVote(Address signer) { - bool success = _proposals.TryRemove(signer, out _); + bool success = _blockProducer.Proposals.TryRemove(signer, out _); if (!success) { throw new InvalidOperationException("Cannot uncast vote"); @@ -189,11 +173,10 @@ private void TimerOnElapsed(object sender, ElapsedEventArgs e) private Task? _producerTask; - public Task Start() + public void Start() { _blockTree.NewHeadBlock += BlockTreeOnNewHeadBlock; _producerTask = RunConsumeSignal(); - return Task.CompletedTask; } private Task RunConsumeSignal() @@ -204,7 +187,7 @@ private Task RunConsumeSignal() { try { - ConsumeSignal(); + ConsumeSignal().Wait(); if (_logger.IsDebug) _logger.Debug("Clique block producer complete."); } catch (OperationCanceledException) @@ -236,11 +219,12 @@ private void BlockTreeOnNewHeadBlock(object? sender, BlockEventArgs e) _signalsQueue.Add(e.Block); } - private void ConsumeSignal() + private async Task ConsumeSignal() { _lastProducedBlock = DateTime.UtcNow; foreach (Block signal in _signalsQueue.GetConsumingEnumerable(_cancellationTokenSource.Token)) { + // TODO: Maybe use IBlockProducer specific to clique? Block parentBlock = signal; while (_signalsQueue.TryTake(out Block? nextSignal)) { @@ -252,59 +236,12 @@ private void ConsumeSignal() try { - Block? block = PrepareBlock(parentBlock); - if (block is null) + Block? block = await _blockProducer.BuildBlock(parentBlock?.Header, token: CancellationToken.None); + if (block is not null) { - if (_logger.IsTrace) _logger.Trace("Skipping block production or block production failed"); - Metrics.FailedBlockSeals++; - continue; + _scheduledBlock = block; + _lastProducedBlock = DateTime.UtcNow; } - - if (_logger.IsInfo) _logger.Info($"Processing prepared block {block.Number}"); - Block? processedBlock = _processor.Process( - block, - ProcessingOptions.ProducingBlock, - NullBlockTracer.Instance); - if (processedBlock is null) - { - if (_logger.IsInfo) _logger.Info($"Prepared block has lost the race"); - Metrics.FailedBlockSeals++; - continue; - } - - if (_logger.IsDebug) _logger.Debug($"Sealing prepared block {processedBlock.Number}"); - - _sealer.SealBlock(processedBlock, _cancellationTokenSource.Token).ContinueWith(t => - { - if (t.IsCompletedSuccessfully) - { - if (t.Result is not null) - { - if (_logger.IsInfo) - _logger.Info($"Sealed block {t.Result.ToString(Block.Format.HashNumberDiffAndTx)}"); - _scheduledBlock = t.Result; - _lastProducedBlock = DateTime.UtcNow; - Metrics.BlocksSealed++; - } - else - { - if (_logger.IsInfo) - _logger.Info( - $"Failed to seal block {processedBlock.ToString(Block.Format.HashNumberDiffAndTx)} (null seal)"); - Metrics.FailedBlockSeals++; - } - } - else if (t.IsFaulted) - { - if (_logger.IsError) _logger.Error("Mining failed", t.Exception); - Metrics.FailedBlockSeals++; - } - else if (t.IsCanceled) - { - if (_logger.IsInfo) _logger.Info($"Sealing block {processedBlock.Number} cancelled"); - Metrics.FailedBlockSeals++; - } - }, _cancellationTokenSource.Token); } catch (Exception e) { @@ -324,7 +261,7 @@ public async Task StopAsync() await (_producerTask ?? Task.CompletedTask); } - bool IBlockProducer.IsProducingBlocks(ulong? maxProducingInterval) + bool IBlockProducerRunner.IsProducingBlocks(ulong? maxProducingInterval) { if (_producerTask is null || _producerTask.IsCompleted) return false; @@ -336,11 +273,122 @@ bool IBlockProducer.IsProducingBlocks(ulong? maxProducingInterval) public event EventHandler? BlockProduced; + + public void Dispose() + { + _cancellationTokenSource?.Dispose(); + _timer?.Dispose(); + } +} + +public class CliqueBlockProducer : IBlockProducer +{ + private readonly IWorldState _stateProvider; + private readonly ITxSource _txSource; + private readonly IBlockchainProcessor _processor; + private readonly ISealer _sealer; + private readonly IGasLimitCalculator _gasLimitCalculator; + private readonly ISpecProvider _specProvider; + private readonly ISnapshotManager _snapshotManager; + private readonly ILogger _logger; + private readonly ITimestamper _timestamper; + private readonly ICryptoRandom _cryptoRandom; + private readonly ICliqueConfig _config; + private readonly ConcurrentDictionary _proposals = new(); + + public CliqueBlockProducer( + ITxSource txSource, + IBlockchainProcessor blockchainProcessor, + IWorldState stateProvider, + ITimestamper timestamper, + ICryptoRandom cryptoRandom, + ISnapshotManager snapshotManager, + ISealer cliqueSealer, + IGasLimitCalculator gasLimitCalculator, + ISpecProvider? specProvider, + ICliqueConfig config, + ILogManager logManager + ) + { + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _txSource = txSource ?? throw new ArgumentNullException(nameof(txSource)); + _processor = blockchainProcessor ?? throw new ArgumentNullException(nameof(blockchainProcessor)); + _stateProvider = stateProvider ?? throw new ArgumentNullException(nameof(stateProvider)); + _timestamper = timestamper ?? throw new ArgumentNullException(nameof(timestamper)); + _cryptoRandom = cryptoRandom ?? throw new ArgumentNullException(nameof(cryptoRandom)); + _sealer = cliqueSealer ?? throw new ArgumentNullException(nameof(cliqueSealer)); + _gasLimitCalculator = gasLimitCalculator ?? throw new ArgumentNullException(nameof(gasLimitCalculator)); + _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); + _snapshotManager = snapshotManager ?? throw new ArgumentNullException(nameof(snapshotManager)); + _config = config ?? throw new ArgumentNullException(nameof(config)); + _logger = logManager.GetClassLogger(); + } + + public ConcurrentDictionary Proposals => _proposals; + + public async Task BuildBlock(BlockHeader? parentHeader, IBlockTracer? blockTracer = null, + PayloadAttributes? payloadAttributes = null, CancellationToken? token = null) + { + token ??= default; + Block? block = PrepareBlock(parentHeader); + if (block is null) + { + if (_logger.IsTrace) _logger.Trace("Skipping block production or block production failed"); + Metrics.FailedBlockSeals++; + return null; + } + + if (_logger.IsInfo) _logger.Info($"Processing prepared block {block.Number}"); + Block? processedBlock = _processor.Process( + block, + ProcessingOptions.ProducingBlock, + NullBlockTracer.Instance); + if (processedBlock is null) + { + if (_logger.IsInfo) _logger.Info($"Prepared block has lost the race"); + Metrics.FailedBlockSeals++; + return null; + } + + if (_logger.IsDebug) _logger.Debug($"Sealing prepared block {processedBlock.Number}"); + + try + { + Block? sealedBlock = await _sealer.SealBlock(processedBlock, token.Value); + if (sealedBlock is not null) + { + if (_logger.IsInfo) + _logger.Info($"Sealed block {sealedBlock.ToString(Block.Format.HashNumberDiffAndTx)}"); + Metrics.BlocksSealed++; + } + else + { + if (_logger.IsInfo) + _logger.Info( + $"Failed to seal block {processedBlock.ToString(Block.Format.HashNumberDiffAndTx)} (null seal)"); + Metrics.FailedBlockSeals++; + } + + return sealedBlock; + } + catch (OperationCanceledException) + { + if (_logger.IsInfo) _logger.Info($"Sealing block {processedBlock.Number} cancelled"); + Metrics.FailedBlockSeals++; + } + catch (Exception e) + { + if (_logger.IsError) _logger.Error("Mining failed", e); + Metrics.FailedBlockSeals++; + } + + return null; + } + private Hash256? _recentNotAllowedParent; - private Block? PrepareBlock(Block parentBlock) + private Block? PrepareBlock(BlockHeader parentHeader) { - BlockHeader parentHeader = parentBlock.Header; if (parentHeader.Hash is null) { if (_logger.IsError) _logger.Error( @@ -348,20 +396,20 @@ bool IBlockProducer.IsProducingBlocks(ulong? maxProducingInterval) return null; } - if (_recentNotAllowedParent == parentBlock.Hash) + if (_recentNotAllowedParent == parentHeader.Hash) { return null; } if (!_sealer.CanSeal(parentHeader.Number + 1, parentHeader.Hash)) { - if (_logger.IsTrace) _logger.Trace($"Not allowed to sign block ({parentBlock.Number + 1})"); + if (_logger.IsTrace) _logger.Trace($"Not allowed to sign block ({parentHeader.Number + 1})"); _recentNotAllowedParent = parentHeader.Hash; return null; } if (_logger.IsInfo) - _logger.Info($"Preparing new block on top of {parentBlock.ToString(Block.Format.Short)}"); + _logger.Info($"Preparing new block on top of {parentHeader}"); ulong timestamp = _timestamper.UnixTime.Seconds; @@ -370,9 +418,9 @@ bool IBlockProducer.IsProducingBlocks(ulong? maxProducingInterval) Keccak.OfAnEmptySequenceRlp, Address.Zero, 1, - parentBlock.Number + 1, - _gasLimitCalculator.GetGasLimit(parentBlock.Header), - timestamp > parentBlock.Timestamp ? timestamp : parentBlock.Timestamp + 1, + parentHeader.Number + 1, + _gasLimitCalculator.GetGasLimit(parentHeader), + timestamp > parentHeader.Timestamp ? timestamp : parentHeader.Timestamp + 1, Array.Empty()); // If the block isn't a checkpoint, cast a random vote (good enough for now) @@ -404,16 +452,16 @@ bool IBlockProducer.IsProducingBlocks(ulong? maxProducingInterval) } // Ensure the timestamp has the correct delay - header.Timestamp = Math.Max(parentBlock.Timestamp + _config.BlockPeriod, _timestamper.UnixTime.Seconds); + header.Timestamp = Math.Max(parentHeader.Timestamp + _config.BlockPeriod, _timestamper.UnixTime.Seconds); var spec = _specProvider.GetSpec(header); header.BaseFeePerGas = BaseFeeCalculator.Calculate(parentHeader, spec); // Set the correct difficulty header.Difficulty = CalculateDifficulty(snapshot, _sealer.Address); - header.TotalDifficulty = parentBlock.TotalDifficulty + header.Difficulty; + header.TotalDifficulty = parentHeader.TotalDifficulty + header.Difficulty; if (_logger.IsDebug) - _logger.Debug($"Setting total difficulty to {parentBlock.TotalDifficulty} + {header.Difficulty}."); + _logger.Debug($"Setting total difficulty to {parentHeader.TotalDifficulty} + {header.Difficulty}."); // Set extra data int mainBytesLength = Clique.ExtraVanityLength + Clique.ExtraSealLength; @@ -441,14 +489,14 @@ bool IBlockProducer.IsProducingBlocks(ulong? maxProducingInterval) _stateProvider.StateRoot = parentHeader.StateRoot!; - IEnumerable selectedTxs = _txSource.GetTransactions(parentBlock.Header, header.GasLimit); + IEnumerable selectedTxs = _txSource.GetTransactions(parentHeader, header.GasLimit); Block block = new BlockToProduce( header, selectedTxs, Array.Empty(), spec.WithdrawalsEnabled ? Enumerable.Empty() : null, spec.ConsensusRequestsEnabled ? Enumerable.Empty() : null - ); + ); header.TxRoot = TxTrie.CalculateRoot(block.Transactions); block.Header.Author = _sealer.Address; return block; @@ -465,10 +513,4 @@ private UInt256 CalculateDifficulty(Snapshot snapshot, Address signer) if (_logger.IsInfo) _logger.Info("Producing out of turn block"); return Clique.DifficultyNoTurn; } - - public void Dispose() - { - _cancellationTokenSource?.Dispose(); - _timer?.Dispose(); - } } diff --git a/src/Nethermind/Nethermind.Consensus.Clique/CliquePlugin.cs b/src/Nethermind/Nethermind.Consensus.Clique/CliquePlugin.cs index d5bc352e850..6ff7e436189 100644 --- a/src/Nethermind/Nethermind.Consensus.Clique/CliquePlugin.cs +++ b/src/Nethermind/Nethermind.Consensus.Clique/CliquePlugin.cs @@ -16,6 +16,9 @@ using Nethermind.Consensus.Transactions; using Nethermind.Consensus.Withdrawals; using Nethermind.Core.Attributes; +using Nethermind.Core.Crypto; +using Nethermind.Db; +using Nethermind.Evm.TransactionProcessing; using Nethermind.JsonRpc.Modules; using Nethermind.State; @@ -67,11 +70,11 @@ public Task Init(INethermindApi nethermindApi) return Task.CompletedTask; } - public Task InitBlockProducer(IBlockProductionTrigger? blockProductionTrigger = null, ITxSource? additionalTxSource = null) + public IBlockProducer InitBlockProducer(ITxSource? additionalTxSource = null) { if (_nethermindApi!.SealEngineType != Nethermind.Core.SealEngineType.Clique) { - return Task.FromResult((IBlockProducer)null); + return null; } (IApiWithBlockchain getFromApi, IApiWithBlockchain setInApi) = _nethermindApi!.ForProducer; @@ -99,18 +102,19 @@ public Task InitBlockProducer(IBlockProductionTrigger? blockProd getFromApi.SpecProvider, getFromApi.LogManager); + IReadOnlyTxProcessingScope scope = producerEnv.Build(Keccak.EmptyTreeHash); + BlockProcessor producerProcessor = new( getFromApi!.SpecProvider, getFromApi!.BlockValidator, NoBlockRewards.Instance, - getFromApi.BlockProducerEnvFactory.TransactionsExecutorFactory.Create(producerEnv), - producerEnv.StateProvider, + getFromApi.BlockProducerEnvFactory.TransactionsExecutorFactory.Create(scope), + scope.WorldState, NullReceiptStorage.Instance, - NullWitnessCollector.Instance, - new BlockhashStore(getFromApi.BlockTree, getFromApi.SpecProvider, producerEnv.StateProvider), + new BlockhashStore(getFromApi.BlockTree, getFromApi.SpecProvider, scope.WorldState), getFromApi.TransactionProcessor, getFromApi.LogManager, - new BlockProductionWithdrawalProcessor(new WithdrawalProcessor(producerEnv.StateProvider, getFromApi.LogManager))); + new BlockProductionWithdrawalProcessor(new WithdrawalProcessor(scope.WorldState, getFromApi.LogManager))); IBlockchainProcessor producerChainProcessor = new BlockchainProcessor( readOnlyBlockTree, @@ -121,7 +125,7 @@ public Task InitBlockProducer(IBlockProductionTrigger? blockProd BlockchainProcessor.Options.NoReceipts); OneTimeChainProcessor chainProcessor = new( - producerEnv.StateProvider, + scope.WorldState, producerChainProcessor); ITxFilterPipeline txFilterPipeline = @@ -142,8 +146,7 @@ public Task InitBlockProducer(IBlockProductionTrigger? blockProd CliqueBlockProducer blockProducer = new( additionalTxSource.Then(txPoolTxSource), chainProcessor, - producerEnv.StateProvider, - getFromApi.BlockTree!, + scope.WorldState, getFromApi.Timestamper, getFromApi.CryptoRandom, _snapshotManager!, @@ -153,14 +156,21 @@ public Task InitBlockProducer(IBlockProductionTrigger? blockProd _cliqueConfig!, getFromApi.LogManager); - getFromApi.DisposeStack.Push(blockProducer); - - return Task.FromResult((IBlockProducer)blockProducer); + return blockProducer; } - public Task InitNetworkProtocol() + public IBlockProducerRunner CreateBlockProducerRunner() { - return Task.CompletedTask; + _blockProducerRunner = new CliqueBlockProducerRunner( + _nethermindApi.BlockTree, + _nethermindApi.Timestamper, + _nethermindApi.CryptoRandom, + _snapshotManager, + (CliqueBlockProducer)_nethermindApi.BlockProducer!, + _cliqueConfig, + _nethermindApi.LogManager); + _nethermindApi.DisposeStack.Push(_blockProducerRunner); + return _blockProducerRunner; } public Task InitRpcModules() @@ -172,7 +182,7 @@ public Task InitRpcModules() (IApiWithNetwork getFromApi, _) = _nethermindApi!.ForRpc; CliqueRpcModule cliqueRpcModule = new( - getFromApi!.BlockProducer as ICliqueBlockProducer, + _blockProducerRunner, _snapshotManager!, getFromApi.BlockTree!); @@ -184,9 +194,6 @@ public Task InitRpcModules() public string SealEngineType => Nethermind.Core.SealEngineType.Clique; - [Todo("Redo clique producer to support triggers and MEV")] - public IBlockProductionTrigger DefaultBlockProductionTrigger => _nethermindApi!.ManualBlockProductionTrigger; - public ValueTask DisposeAsync() { return ValueTask.CompletedTask; } private INethermindApi? _nethermindApi; @@ -196,5 +203,6 @@ public Task InitRpcModules() private ICliqueConfig? _cliqueConfig; private IBlocksConfig? _blocksConfig; + private CliqueBlockProducerRunner _blockProducerRunner = null!; } } diff --git a/src/Nethermind/Nethermind.Consensus.Clique/CliqueRpcModule.cs b/src/Nethermind/Nethermind.Consensus.Clique/CliqueRpcModule.cs index 82581295402..07142a25b7a 100644 --- a/src/Nethermind/Nethermind.Consensus.Clique/CliqueRpcModule.cs +++ b/src/Nethermind/Nethermind.Consensus.Clique/CliqueRpcModule.cs @@ -15,11 +15,11 @@ public class CliqueRpcModule : ICliqueRpcModule { private const string CannotVoteOnNonValidatorMessage = "Not a signer node - cannot vote"; - private readonly ICliqueBlockProducer? _cliqueBlockProducer; + private readonly ICliqueBlockProducerRunner? _cliqueBlockProducer; private readonly ISnapshotManager _snapshotManager; private readonly IBlockFinder _blockTree; - public CliqueRpcModule(ICliqueBlockProducer? cliqueBlockProducer, ISnapshotManager snapshotManager, IBlockFinder blockTree) + public CliqueRpcModule(ICliqueBlockProducerRunner? cliqueBlockProducer, ISnapshotManager snapshotManager, IBlockFinder blockTree) { _cliqueBlockProducer = cliqueBlockProducer; _snapshotManager = snapshotManager ?? throw new ArgumentNullException(nameof(snapshotManager)); diff --git a/src/Nethermind/Nethermind.Consensus.Clique/CliqueSealer.cs b/src/Nethermind/Nethermind.Consensus.Clique/CliqueSealer.cs index 584a1194956..2e47118d42e 100644 --- a/src/Nethermind/Nethermind.Consensus.Clique/CliqueSealer.cs +++ b/src/Nethermind/Nethermind.Consensus.Clique/CliqueSealer.cs @@ -7,8 +7,11 @@ using System.Threading.Tasks; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Crypto; +using Nethermind.JsonRpc; using Nethermind.Logging; +using Nethermind.Serialization.Rlp; [assembly: InternalsVisibleTo("Nethermind.Clique.Test")] @@ -27,7 +30,6 @@ public CliqueSealer(ISigner signer, ICliqueConfig config, ISnapshotManager snaps _snapshotManager = snapshotManager ?? throw new ArgumentNullException(nameof(snapshotManager)); _config = config ?? throw new ArgumentNullException(nameof(config)); _signer = signer ?? throw new ArgumentNullException(nameof(signer)); - if (config.Epoch == 0) config.Epoch = Clique.DefaultEpochLength; } @@ -64,7 +66,19 @@ public CliqueSealer(ISigner signer, ICliqueConfig config, ISnapshotManager snaps // Sign all the things! Hash256 headerHash = SnapshotManager.CalculateCliqueHeaderHash(header); - Signature signature = _signer.Sign(headerHash); + Signature signature; + if (_signer is IHeaderSigner headerSigner) + { + BlockHeader clone = header.Clone(); + clone.ExtraData = SnapshotManager.SliceExtraSealFromExtraData(clone.ExtraData); + clone.Hash = headerHash; + signature = headerSigner.Sign(clone); + } + else + { + signature = _signer.Sign(headerHash); + } + // Copy signature bytes (R and S) byte[] signatureBytes = signature.Bytes; Array.Copy(signatureBytes, 0, header.ExtraData, header.ExtraData.Length - Clique.ExtraSealLength, signatureBytes.Length); diff --git a/src/Nethermind/Nethermind.Consensus.Clique/ICliqueBlockProducer.cs b/src/Nethermind/Nethermind.Consensus.Clique/ICliqueBlockProducer.cs index e5e6b742c41..e2fbc43492e 100644 --- a/src/Nethermind/Nethermind.Consensus.Clique/ICliqueBlockProducer.cs +++ b/src/Nethermind/Nethermind.Consensus.Clique/ICliqueBlockProducer.cs @@ -6,7 +6,7 @@ namespace Nethermind.Consensus.Clique { - public interface ICliqueBlockProducer : IBlockProducer + public interface ICliqueBlockProducerRunner : IBlockProducerRunner { void CastVote(Address signer, bool vote); void UncastVote(Address signer); diff --git a/src/Nethermind/Nethermind.Consensus.Clique/SnapshotManager.cs b/src/Nethermind/Nethermind.Consensus.Clique/SnapshotManager.cs index 99e68471b15..29b8e19a91f 100644 --- a/src/Nethermind/Nethermind.Consensus.Clique/SnapshotManager.cs +++ b/src/Nethermind/Nethermind.Consensus.Clique/SnapshotManager.cs @@ -75,16 +75,21 @@ private int CalculateSignersCount(BlockHeader blockHeader) public static Hash256 CalculateCliqueHeaderHash(BlockHeader blockHeader) { - int extraSeal = 65; - int shortExtraLength = blockHeader.ExtraData.Length - extraSeal; byte[] fullExtraData = blockHeader.ExtraData; - byte[] shortExtraData = blockHeader.ExtraData.Slice(0, shortExtraLength); + byte[] shortExtraData = SliceExtraSealFromExtraData(blockHeader.ExtraData); blockHeader.ExtraData = shortExtraData; Hash256 sigHash = blockHeader.CalculateHash(); blockHeader.ExtraData = fullExtraData; return sigHash; } + public static byte[] SliceExtraSealFromExtraData(byte[] extraData) + { + if (extraData.Length < Clique.ExtraSealLength) + new ArgumentException($"Cannot be less than extra seal length ({Clique.ExtraSealLength}).", nameof(extraData)); + return extraData.Slice(0, extraData.Length - Clique.ExtraSealLength); + } + private readonly object _snapshotCreationLock = new(); public ulong GetLastSignersCount() => _lastSignersCount; diff --git a/src/Nethermind/Nethermind.Consensus.Ethash/EthashPlugin.cs b/src/Nethermind/Nethermind.Consensus.Ethash/EthashPlugin.cs index 89207009a90..dae39105746 100644 --- a/src/Nethermind/Nethermind.Consensus.Ethash/EthashPlugin.cs +++ b/src/Nethermind/Nethermind.Consensus.Ethash/EthashPlugin.cs @@ -46,23 +46,19 @@ public Task Init(INethermindApi nethermindApi) return Task.CompletedTask; } - public Task InitBlockProducer(IBlockProductionTrigger? blockProductionTrigger = null, ITxSource? additionalTxSource = null) + public IBlockProducer InitBlockProducer(ITxSource? additionalTxSource = null) { - return Task.FromResult((IBlockProducer)null); + return null; } - public Task InitNetworkProtocol() - { - return Task.CompletedTask; - } + public string SealEngineType => Core.SealEngineType.Ethash; - public Task InitRpcModules() + public IBlockProducerRunner CreateBlockProducerRunner() { - return Task.CompletedTask; + return new StandardBlockProducerRunner( + _nethermindApi.ManualBlockProductionTrigger, + _nethermindApi.BlockTree, + _nethermindApi.BlockProducer!); } - - public string SealEngineType => Nethermind.Core.SealEngineType.Ethash; - - public IBlockProductionTrigger DefaultBlockProductionTrigger => _nethermindApi.ManualBlockProductionTrigger; } } diff --git a/src/Nethermind/Nethermind.Consensus.Ethash/MinedBlockProducer.cs b/src/Nethermind/Nethermind.Consensus.Ethash/MinedBlockProducer.cs index 80b111c77ad..2aaa8e1a62e 100644 --- a/src/Nethermind/Nethermind.Consensus.Ethash/MinedBlockProducer.cs +++ b/src/Nethermind/Nethermind.Consensus.Ethash/MinedBlockProducer.cs @@ -19,7 +19,6 @@ public MinedBlockProducer(ITxSource txSource, IBlockchainProcessor processor, ISealer sealer, IBlockTree blockTree, - IBlockProductionTrigger blockProductionTrigger, IWorldState stateProvider, IGasLimitCalculator gasLimitCalculator, ITimestamper timestamper, @@ -31,7 +30,6 @@ public MinedBlockProducer(ITxSource txSource, processor, sealer, blockTree, - blockProductionTrigger, stateProvider, gasLimitCalculator, timestamper, diff --git a/src/Nethermind/Nethermind.Consensus.Ethash/NethDevPlugin.cs b/src/Nethermind/Nethermind.Consensus.Ethash/NethDevPlugin.cs index 931fc547172..e1ddd19a420 100644 --- a/src/Nethermind/Nethermind.Consensus.Ethash/NethDevPlugin.cs +++ b/src/Nethermind/Nethermind.Consensus.Ethash/NethDevPlugin.cs @@ -13,7 +13,9 @@ using Nethermind.Consensus.Producers; using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Transactions; +using Nethermind.Core.Crypto; using Nethermind.Db; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; using Nethermind.State; @@ -37,11 +39,11 @@ public Task Init(INethermindApi nethermindApi) return Task.CompletedTask; } - public Task InitBlockProducer(IBlockProductionTrigger? blockProductionTrigger = null, ITxSource? additionalTxSource = null) + public IBlockProducer InitBlockProducer(ITxSource? additionalTxSource = null) { if (_nethermindApi!.SealEngineType != Nethermind.Core.SealEngineType.NethDev) { - return Task.FromResult((IBlockProducer)null); + return null; } var (getFromApi, _) = _nethermindApi!.ForProducer; @@ -70,15 +72,16 @@ public Task InitBlockProducer(IBlockProductionTrigger? blockProd getFromApi.SpecProvider, getFromApi.LogManager); + IReadOnlyTxProcessingScope scope = producerEnv.Build(Keccak.EmptyTreeHash); + BlockProcessor producerProcessor = new( getFromApi!.SpecProvider, getFromApi!.BlockValidator, NoBlockRewards.Instance, - new BlockProcessor.BlockProductionTransactionsExecutor(producerEnv, getFromApi!.SpecProvider, getFromApi.LogManager), - producerEnv.StateProvider, + new BlockProcessor.BlockProductionTransactionsExecutor(scope, getFromApi!.SpecProvider, getFromApi.LogManager), + scope.WorldState, NullReceiptStorage.Instance, - NullWitnessCollector.Instance, - new BlockhashStore(getFromApi.BlockTree, getFromApi.SpecProvider, producerEnv.StateProvider), + new BlockhashStore(getFromApi.BlockTree, getFromApi.SpecProvider, scope.WorldState), getFromApi.TransactionProcessor, getFromApi.LogManager); @@ -90,35 +93,29 @@ public Task InitBlockProducer(IBlockProductionTrigger? blockProd getFromApi.LogManager, BlockchainProcessor.Options.NoReceipts); - DefaultBlockProductionTrigger = new BuildBlocksRegularly(TimeSpan.FromMilliseconds(200)) - .IfPoolIsNotEmpty(getFromApi.TxPool) - .Or(getFromApi.ManualBlockProductionTrigger); - IBlockProducer blockProducer = new DevBlockProducer( additionalTxSource.Then(txPoolTxSource).ServeTxsOneByOne(), producerChainProcessor, - producerEnv.StateProvider, + scope.WorldState, getFromApi.BlockTree, - blockProductionTrigger ?? DefaultBlockProductionTrigger, getFromApi.Timestamper, getFromApi.SpecProvider, getFromApi.Config(), getFromApi.LogManager); - return Task.FromResult(blockProducer); + return blockProducer; } public string SealEngineType => Nethermind.Core.SealEngineType.NethDev; - public IBlockProductionTrigger DefaultBlockProductionTrigger { get; private set; } - - public Task InitNetworkProtocol() - { - return Task.CompletedTask; - } - - public Task InitRpcModules() + public IBlockProducerRunner CreateBlockProducerRunner() { - return Task.CompletedTask; + IBlockProductionTrigger trigger = new BuildBlocksRegularly(TimeSpan.FromMilliseconds(200)) + .IfPoolIsNotEmpty(_nethermindApi.TxPool) + .Or(_nethermindApi.ManualBlockProductionTrigger); + return new StandardBlockProducerRunner( + trigger, + _nethermindApi.BlockTree, + _nethermindApi.BlockProducer!); } } } diff --git a/src/Nethermind/Nethermind.Consensus.Test/ReadOnlyTxProcessingScopeTests.cs b/src/Nethermind/Nethermind.Consensus.Test/ReadOnlyTxProcessingScopeTests.cs new file mode 100644 index 00000000000..399523f0e56 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus.Test/ReadOnlyTxProcessingScopeTests.cs @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Consensus.Processing; +using Nethermind.Core.Test.Builders; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.State; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.Consensus.Test; + +public class ReadOnlyTxProcessingScopeTests +{ + [Test] + public void Test_WhenDispose_ThenStateRootWillRevert() + { + ReadOnlyTxProcessingScope env = new ReadOnlyTxProcessingScope( + Substitute.For(), + Substitute.For(), + TestItem.KeccakB + ); + + env.Dispose(); + + env.WorldState.Received().StateRoot = TestItem.KeccakB; + } +} diff --git a/src/Nethermind/Nethermind.Consensus/IBlockProducer.cs b/src/Nethermind/Nethermind.Consensus/IBlockProducer.cs index 2ad56e537c8..ff868005e66 100644 --- a/src/Nethermind/Nethermind.Consensus/IBlockProducer.cs +++ b/src/Nethermind/Nethermind.Consensus/IBlockProducer.cs @@ -1,16 +1,19 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; +using System.Threading; using System.Threading.Tasks; +using Nethermind.Consensus.Producers; using Nethermind.Core; +using Nethermind.Evm.Tracing; namespace Nethermind.Consensus; public interface IBlockProducer { - Task Start(); - Task StopAsync(); - bool IsProducingBlocks(ulong? maxProducingInterval); - event EventHandler BlockProduced; + Task BuildBlock( + BlockHeader? parentHeader = null, + IBlockTracer? blockTracer = null, + PayloadAttributes? payloadAttributes = null, + CancellationToken? cancellationToken = null); } diff --git a/src/Nethermind/Nethermind.Consensus/IBlockProducerFactory.cs b/src/Nethermind/Nethermind.Consensus/IBlockProducerFactory.cs index 9897ba185d7..46735b2501a 100644 --- a/src/Nethermind/Nethermind.Consensus/IBlockProducerFactory.cs +++ b/src/Nethermind/Nethermind.Consensus/IBlockProducerFactory.cs @@ -12,12 +12,9 @@ public interface IBlockProducerFactory /// /// Creates a block producer. /// - /// Optional parameter. If present this should be the only block production trigger for created block producer. If absent should be used. /// Optional parameter. If present this transaction source should be used before any other transaction sources, except consensus ones. Plugin still should use their own transaction sources. /// /// Can be called many times, with different parameters, each time should create a new instance. Example usage in MEV plugin. /// - Task InitBlockProducer( - IBlockProductionTrigger blockProductionTrigger, - ITxSource? additionalTxSource = null); + IBlockProducer InitBlockProducer(ITxSource? additionalTxSource = null); } diff --git a/src/Nethermind/Nethermind.Consensus/IBlockProducerRunner.cs b/src/Nethermind/Nethermind.Consensus/IBlockProducerRunner.cs new file mode 100644 index 00000000000..e2a1992bb63 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/IBlockProducerRunner.cs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading.Tasks; +using Nethermind.Core; + +namespace Nethermind.Consensus; + +public interface IBlockProducerRunner +{ + void Start(); + Task StopAsync(); + bool IsProducingBlocks(ulong? maxProducingInterval); + event EventHandler BlockProduced; +} diff --git a/src/Nethermind/Nethermind.Consensus/IHeaderSigner.cs b/src/Nethermind/Nethermind.Consensus/IHeaderSigner.cs new file mode 100644 index 00000000000..a51b1ca800a --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/IHeaderSigner.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Crypto; + +namespace Nethermind.Consensus; +public interface IHeaderSigner : ISigner +{ + bool CanSignHeader { get; } + Signature Sign(BlockHeader header); +} diff --git a/src/Nethermind/Nethermind.Consensus/IMiningConfig.cs b/src/Nethermind/Nethermind.Consensus/IMiningConfig.cs index 9881cf09b8f..4562ab03327 100644 --- a/src/Nethermind/Nethermind.Consensus/IMiningConfig.cs +++ b/src/Nethermind/Nethermind.Consensus/IMiningConfig.cs @@ -52,4 +52,9 @@ public interface IMiningConfig : IConfig [ConfigItem(HiddenFromDocs = true, DisabledForCli = true, DefaultValue = "null")] IBlocksConfig? BlocksConfig { get; } + [ConfigItem( + Description = "Url for an external signer like clef: https://github.com/ethereum/go-ethereum/blob/master/cmd/clef/tutorial.md", + HiddenFromDocs = false, + DefaultValue = "null")] + string Signer { get; set; } } diff --git a/src/Nethermind/Nethermind.Consensus/ISigner.cs b/src/Nethermind/Nethermind.Consensus/ISigner.cs index 1174e68a1d0..ec34eafa187 100644 --- a/src/Nethermind/Nethermind.Consensus/ISigner.cs +++ b/src/Nethermind/Nethermind.Consensus/ISigner.cs @@ -5,6 +5,7 @@ using Nethermind.Core.Crypto; using Nethermind.Crypto; using Nethermind.TxPool; +using System; namespace Nethermind.Consensus { diff --git a/src/Nethermind/Nethermind.Consensus/MiningConfig.cs b/src/Nethermind/Nethermind.Consensus/MiningConfig.cs index b3abd0b054e..dbf2398286a 100644 --- a/src/Nethermind/Nethermind.Consensus/MiningConfig.cs +++ b/src/Nethermind/Nethermind.Consensus/MiningConfig.cs @@ -70,4 +70,6 @@ public IBlocksConfig? BlocksConfig return _blocksConfig; } } + + public string Signer { get; set; } } diff --git a/src/Nethermind/Nethermind.Consensus/NullSigner.cs b/src/Nethermind/Nethermind.Consensus/NullSigner.cs index a6028d49dec..f4976cac687 100644 --- a/src/Nethermind/Nethermind.Consensus/NullSigner.cs +++ b/src/Nethermind/Nethermind.Consensus/NullSigner.cs @@ -22,8 +22,12 @@ public class NullSigner : ISigner, ISignerStore public PrivateKey? Key { get; } = null; + public bool CanSignHeader => false; + public void SetSigner(PrivateKey key) { } public void SetSigner(ProtectedPrivateKey key) { } + + public Signature Sign(BlockHeader header) { return new(new byte[65]); } } } diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs new file mode 100644 index 00000000000..93f3e7a556c --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockCachePreWarmer.cs @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.ObjectPool; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Core.Threading; +using Nethermind.Core.Cpu; +using Nethermind.Evm; +using Nethermind.Evm.Tracing; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Logging; +using Nethermind.State; + +namespace Nethermind.Consensus.Processing; + +public class BlockCachePreWarmer(ReadOnlyTxProcessingEnvFactory envFactory, ISpecProvider specProvider, ILogManager logManager, IWorldState? targetWorldState = null) : IBlockCachePreWarmer +{ + private readonly ObjectPool _envPool = new DefaultObjectPool(new ReadOnlyTxProcessingEnvPooledObjectPolicy(envFactory), Environment.ProcessorCount); + private readonly ObjectPool _systemTransactionPool = new DefaultObjectPool(new DefaultPooledObjectPolicy(), Environment.ProcessorCount); + private readonly ILogger _logger = logManager.GetClassLogger(); + + public Task PreWarmCaches(Block suggestedBlock, Hash256? parentStateRoot, CancellationToken cancellationToken = default) + { + if (targetWorldState is not null) + { + if (targetWorldState.ClearCache()) + { + if (_logger.IsWarn) _logger.Warn("Cashes are not empty. Clearing them."); + } + + if (!IsGenesisBlock(parentStateRoot) && Environment.ProcessorCount > 2 && !cancellationToken.IsCancellationRequested) + { + // Do not pass cancellation token to the task, we don't want exceptions to be thrown in main processing thread + return Task.Run(() => PreWarmCachesParallel(suggestedBlock, parentStateRoot, cancellationToken)); + } + } + + return Task.CompletedTask; + } + + // Parent state root is null for genesis block + private bool IsGenesisBlock(Hash256? parentStateRoot) => parentStateRoot is null; + + public void ClearCaches() => targetWorldState?.ClearCache(); + + private void PreWarmCachesParallel(Block suggestedBlock, Hash256 parentStateRoot, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) return; + + try + { + ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = Math.Max(1, RuntimeInformation.PhysicalCoreCount - 2), CancellationToken = cancellationToken }; + IReleaseSpec spec = specProvider.GetSpec(suggestedBlock.Header); + + WarmupTransactions(parallelOptions, spec, suggestedBlock, parentStateRoot); + WarmupWithdrawals(parallelOptions, spec, suggestedBlock, parentStateRoot); + + if (_logger.IsDebug) _logger.Debug($"Finished pre-warming caches for block {suggestedBlock.Number}."); + } + catch (OperationCanceledException) + { + if (_logger.IsDebug) _logger.Debug($"Pre-warming caches cancelled for block {suggestedBlock.Number}."); + } + + void WarmupWithdrawals(ParallelOptions parallelOptions, IReleaseSpec spec, Block block, Hash256 stateRoot) + { + if (parallelOptions.CancellationToken.IsCancellationRequested) return; + if (spec.WithdrawalsEnabled && block.Withdrawals is not null) + { + Parallel.For(0, block.Withdrawals.Length, parallelOptions, + i => + { + IReadOnlyTxProcessorSource env = _envPool.Get(); + using IReadOnlyTxProcessingScope scope = env.Build(stateRoot); + try + { + scope.WorldState.WarmUp(block.Withdrawals[i].Address); + } + catch (Exception ex) + { + if (_logger.IsDebug) _logger.Error($"Error pre-warming withdrawal {i}", ex); + } + finally + { + _envPool.Return(env); + } + }); + } + } + + void WarmupTransactions(ParallelOptions parallelOptions, IReleaseSpec spec, Block block, Hash256 stateRoot) + { + if (parallelOptions.CancellationToken.IsCancellationRequested) return; + Parallel.For(0, block.Transactions.Length, parallelOptions, i => + { + // If the transaction has already been processed or being processed, exit early + if (block.TransactionProcessed >= i) return; + + using ThreadExtensions.Disposable handle = Thread.CurrentThread.BoostPriority(); + Transaction tx = block.Transactions[i]; + IReadOnlyTxProcessorSource env = _envPool.Get(); + SystemTransaction systemTransaction = _systemTransactionPool.Get(); + try + { + tx.CopyTo(systemTransaction); + using IReadOnlyTxProcessingScope scope = env.Build(stateRoot); + if (spec.UseTxAccessLists) + { + scope.WorldState.WarmUp(tx.AccessList); // eip-2930 + } + TransactionResult result = scope.TransactionProcessor.Trace(systemTransaction, new BlockExecutionContext(block.Header.Clone()), NullTxTracer.Instance); + if (_logger.IsTrace) _logger.Trace($"Finished pre-warming cache for tx {tx.Hash} with {result}"); + } + catch (Exception ex) + { + if (_logger.IsDebug) _logger.Error($"Error pre-warming cache {tx.Hash}", ex); + } + finally + { + _systemTransactionPool.Return(systemTransaction); + _envPool.Return(env); + } + }); + } + } + + private class ReadOnlyTxProcessingEnvPooledObjectPolicy(ReadOnlyTxProcessingEnvFactory envFactory) : IPooledObjectPolicy + { + public IReadOnlyTxProcessorSource Create() => envFactory.Create(); + public bool Return(IReadOnlyTxProcessorSource obj) => true; + } +} diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionPicker.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionPicker.cs index 10aa4a3a9e2..e5c1c8d1543 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionPicker.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionPicker.cs @@ -16,10 +16,12 @@ public partial class BlockProcessor public class BlockProductionTransactionPicker : IBlockProductionTransactionPicker { protected readonly ISpecProvider _specProvider; + private readonly bool _ignoreEip3607; - public BlockProductionTransactionPicker(ISpecProvider specProvider) + public BlockProductionTransactionPicker(ISpecProvider specProvider, bool ignoreEip3607 = false) { _specProvider = specProvider; + _ignoreEip3607 = ignoreEip3607; } public event EventHandler? AddingTransaction; @@ -63,7 +65,7 @@ public virtual AddingTxEventArgs CanAddTransaction(Block block, Transaction curr return args.Set(TxAction.Skip, $"EIP-3860 - transaction size over max init code size"); } - if (stateProvider.IsInvalidContractSender(spec, currentTx.SenderAddress)) + if (!_ignoreEip3607 && stateProvider.IsInvalidContractSender(spec, currentTx.SenderAddress)) { return args.Set(TxAction.Skip, $"Sender is contract"); } diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionsExecutor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionsExecutor.cs index 161dd665971..bd8740c7c2f 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionsExecutor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockProductionTransactionsExecutor.cs @@ -26,12 +26,12 @@ public class BlockProductionTransactionsExecutor : IBlockProductionTransactionsE private readonly ILogger _logger; public BlockProductionTransactionsExecutor( - ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv, + IReadOnlyTxProcessingScope readOnlyTxProcessingEnv, ISpecProvider specProvider, ILogManager logManager) : this( readOnlyTxProcessingEnv.TransactionProcessor, - readOnlyTxProcessingEnv.StateProvider, + readOnlyTxProcessingEnv.WorldState, specProvider, logManager) { diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs index e2e7eb42592..d487cda83fa 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs @@ -42,13 +42,14 @@ public TxReceipt[] ProcessTransactions(Block block, ProcessingOptions processing BlockExecutionContext blkCtx = new(block.Header); for (int i = 0; i < block.Transactions.Length; i++) { + block.TransactionProcessed = i; Transaction currentTx = block.Transactions[i]; ProcessTransaction(in blkCtx, currentTx, i, receiptsTracer, processingOptions); } return receiptsTracer.TxReceipts.ToArray(); } - private void ProcessTransaction(in BlockExecutionContext blkCtx, Transaction currentTx, int index, BlockReceiptsTracer receiptsTracer, ProcessingOptions processingOptions) + protected virtual void ProcessTransaction(in BlockExecutionContext blkCtx, Transaction currentTx, int index, BlockReceiptsTracer receiptsTracer, ProcessingOptions processingOptions) { TransactionResult result = _transactionProcessor.ProcessTransaction(in blkCtx, currentTx, receiptsTracer, processingOptions, _stateProvider); if (!result) ThrowInvalidBlockException(result, blkCtx.Header, currentTx, index); diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs index 1c7d85000ee..7614f95188b 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs @@ -2,10 +2,12 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Numerics; using System.Runtime.InteropServices; using System.Threading; +using System.Threading.Tasks; using Nethermind.Blockchain; using Nethermind.Blockchain.BeaconBlockRoot; using Nethermind.Blockchain.Blocks; @@ -37,7 +39,6 @@ public partial class BlockProcessor : IBlockProcessor protected readonly IWorldState _stateProvider; private readonly IReceiptStorage _receiptStorage; private readonly IReceiptsRootCalculator _receiptsRootCalculator; - private readonly IWitnessCollector _witnessCollector; private readonly IWithdrawalProcessor _withdrawalProcessor; private readonly IBeaconBlockRootHandler _beaconBlockRootHandler; private readonly IBlockValidator _blockValidator; @@ -46,6 +47,7 @@ public partial class BlockProcessor : IBlockProcessor private readonly IConsensusRequestsProcessor _consensusRequestsProcessor; private readonly IBlockhashStore _blockhashStore; + private readonly IBlockCachePreWarmer? _preWarmer; private const int MaxUncommittedBlocks = 64; /// @@ -61,13 +63,13 @@ public BlockProcessor( IBlockProcessor.IBlockTransactionsExecutor? blockTransactionsExecutor, IWorldState? stateProvider, IReceiptStorage? receiptStorage, - IWitnessCollector? witnessCollector, IBlockhashStore? blockHashStore, ITransactionProcessor transactionProcessor, ILogManager? logManager, IWithdrawalProcessor? withdrawalProcessor = null, IBeaconBlockRootHandler? beaconBlockRootHandler = null, IReceiptsRootCalculator? receiptsRootCalculator = null, + IBlockCachePreWarmer? preWarmer = null, IConsensusRequestsProcessor? consensusRequestsProcessor = null) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); @@ -75,7 +77,6 @@ public BlockProcessor( _blockValidator = blockValidator ?? throw new ArgumentNullException(nameof(blockValidator)); _stateProvider = stateProvider ?? throw new ArgumentNullException(nameof(stateProvider)); _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage)); - _witnessCollector = witnessCollector ?? throw new ArgumentNullException(nameof(witnessCollector)); _withdrawalProcessor = withdrawalProcessor ?? new WithdrawalProcessor(stateProvider, logManager); _rewardCalculator = rewardCalculator ?? throw new ArgumentNullException(nameof(rewardCalculator)); _blockTransactionsExecutor = blockTransactionsExecutor ?? throw new ArgumentNullException(nameof(blockTransactionsExecutor)); @@ -84,10 +85,11 @@ public BlockProcessor( _consensusRequestsProcessor = consensusRequestsProcessor ?? new ConsensusRequestsProcessor(transactionProcessor); _blockhashStore = blockHashStore ?? throw new ArgumentNullException(nameof(blockHashStore)); + _preWarmer = preWarmer; ReceiptsTracer = new BlockReceiptsTracer(); } - public event EventHandler BlockProcessed; + public event EventHandler? BlockProcessed; public event EventHandler TransactionProcessed { @@ -108,30 +110,34 @@ public Block[] Process(Hash256 newBranchStateRoot, List suggestedBlocks, the previous head state.*/ Hash256 previousBranchStateRoot = CreateCheckpoint(); InitBranch(newBranchStateRoot); + Hash256 preBlockStateRoot = newBranchStateRoot; bool notReadOnly = !options.ContainsFlag(ProcessingOptions.ReadOnlyChain); int blocksCount = suggestedBlocks.Count; Block[] processedBlocks = new Block[blocksCount]; - using IDisposable tracker = _witnessCollector.TrackOnThisThread(); try { for (int i = 0; i < blocksCount; i++) { + Block suggestedBlock = suggestedBlocks[i]; if (blocksCount > 64 && i % 8 == 0) { - if (_logger.IsInfo) _logger.Info($"Processing part of a long blocks branch {i}/{blocksCount}. Block: {suggestedBlocks[i]}"); + if (_logger.IsInfo) _logger.Info($"Processing part of a long blocks branch {i}/{blocksCount}. Block: {suggestedBlock}"); } - _witnessCollector.Reset(); - (Block processedBlock, TxReceipt[] receipts) = ProcessOne(suggestedBlocks[i], options, blockTracer); + using CancellationTokenSource cancellationTokenSource = new(); + Task? preWarmTask = suggestedBlock.Transactions.Length < 3 ? + null : + _preWarmer?.PreWarmCaches(suggestedBlock, preBlockStateRoot!, cancellationTokenSource.Token); + (Block processedBlock, TxReceipt[] receipts) = ProcessOne(suggestedBlock, options, blockTracer); + cancellationTokenSource.Cancel(); + preWarmTask?.GetAwaiter().GetResult(); processedBlocks[i] = processedBlock; // be cautious here as AuRa depends on processing - PreCommitBlock(newBranchStateRoot, suggestedBlocks[i].Number); - + PreCommitBlock(newBranchStateRoot, suggestedBlock.Number); if (notReadOnly) { - _witnessCollector.Persist(processedBlock.Hash!); BlockProcessed?.Invoke(this, new BlockProcessedEventArgs(processedBlock, receipts)); } @@ -144,9 +150,12 @@ the previous head state.*/ { if (_logger.IsInfo) _logger.Info($"Commit part of a long blocks branch {i}/{blocksCount}"); previousBranchStateRoot = CreateCheckpoint(); - Hash256? newStateRoot = suggestedBlocks[i].StateRoot; + Hash256? newStateRoot = suggestedBlock.StateRoot; InitBranch(newStateRoot, false); } + + preBlockStateRoot = processedBlock.StateRoot; + _stateProvider.Reset(resizeCollections: true); } if (options.ContainsFlag(ProcessingOptions.DoNotUpdateHead)) @@ -160,6 +169,7 @@ the previous head state.*/ { _logger.Trace($"Encountered exception {ex} while processing blocks."); RestoreBranch(previousBranchStateRoot); + _preWarmer?.ClearCaches(); throw; } } @@ -232,6 +242,9 @@ private void ValidateProcessedBlock(Block suggestedBlock, ProcessingOptions opti if (_logger.IsWarn) _logger.Warn($"Suggested block TD: {suggestedBlock.TotalDifficulty}, Suggested block IsPostMerge {suggestedBlock.IsPostMerge}, Block TD: {block.TotalDifficulty}, Block IsPostMerge {block.IsPostMerge}"); throw new InvalidBlockException(suggestedBlock, error); } + + // Block is valid, copy the account changes as we use the suggested block not the processed one + suggestedBlock.AccountChanges = block.AccountChanges; } private bool ShouldComputeStateRoot(BlockHeader header) => @@ -251,7 +264,7 @@ protected virtual TxReceipt[] ProcessBlock( _beaconBlockRootHandler.ExecuteSystemCall(block, spec); _blockhashStore.ApplyHistoryBlockHashes(block.Header); - _stateProvider.Commit(spec); + _stateProvider.Commit(spec, commitStorageRoots: false); TxReceipt[] receipts = _blockTransactionsExecutor.ProcessTransactions(block, options, ReceiptsTracer, spec); @@ -267,7 +280,13 @@ protected virtual TxReceipt[] ProcessBlock( ReceiptsTracer.EndBlockTrace(); - _stateProvider.Commit(spec); + _stateProvider.Commit(spec, commitStorageRoots: true); + + if (BlockchainProcessor.IsMainProcessingThread) + { + // Get the accounts that have been changed + block.AccountChanges = _stateProvider.GetAccountChanges(); + } if (ShouldComputeStateRoot(block.Header)) { diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs index 97249d25a39..c3cfb0b7234 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs @@ -430,7 +430,9 @@ private void FireProcessingQueueEmpty() if (!readonlyChain) { - _stats.UpdateStats(lastProcessed, _blockTree, _recoveryQueue.Count, _blockQueue.Count, _stopwatch.ElapsedMicroseconds()); + Metrics.RecoveryQueueSize = _recoveryQueue.Count; + Metrics.ProcessingQueueSize = _blockQueue.Count; + _stats.UpdateStats(lastProcessed, _blockTree, _stopwatch.ElapsedMicroseconds()); } return lastProcessed; diff --git a/src/Nethermind/Nethermind.Consensus/Processing/IBlockCachePreWarmer.cs b/src/Nethermind/Nethermind.Consensus/Processing/IBlockCachePreWarmer.cs new file mode 100644 index 00000000000..609e802d44c --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Processing/IBlockCachePreWarmer.cs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Core; +using Nethermind.Core.Crypto; + +namespace Nethermind.Consensus.Processing; + +public interface IBlockCachePreWarmer +{ + Task PreWarmCaches(Block suggestedBlock, Hash256 parentStateRoot, CancellationToken cancellationToken = default); + void ClearCaches(); +} diff --git a/src/Nethermind/Nethermind.Consensus/Processing/IBlockProcessor.cs b/src/Nethermind/Nethermind.Consensus/Processing/IBlockProcessor.cs index ceb8da16d7c..88aba737926 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/IBlockProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/IBlockProcessor.cs @@ -18,9 +18,7 @@ public interface IBlockProcessor /// Initial state for the processed branch. /// List of blocks to be processed. /// Options to use for processor and transaction processor. - /// - /// Block tracer to use. By default either or - /// + /// Block tracer to use. By default either or /// List of processed blocks. Block[] Process( Hash256 newBranchStateRoot, diff --git a/src/Nethermind/Nethermind.Consensus/Processing/ProcessingStats.cs b/src/Nethermind/Nethermind.Consensus/Processing/ProcessingStats.cs index 7ef3faaecb1..d44884f538b 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/ProcessingStats.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/ProcessingStats.cs @@ -2,8 +2,8 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Generic; using System.Diagnostics; +using System.Threading; using Nethermind.Blockchain; using Nethermind.Core; using Nethermind.Core.Extensions; @@ -13,7 +13,7 @@ namespace Nethermind.Consensus.Processing { //TODO Consult on disabling of such metrics from configuration - internal class ProcessingStats + internal class ProcessingStats : IThreadPoolWorkItem { private readonly ILogger _logger; private readonly Stopwatch _processingStopwatch = new(); @@ -26,19 +26,25 @@ internal class ProcessingStats private long _lastTotalEmptyCalls; private long _lastTotalSLoad; private long _lastTotalSStore; - private long _lastStateDbReads; - private long _lastStateDbWrites; - private long _lastGen0; - private long _lastGen1; - private long _lastGen2; - private long _lastTreeNodeRlp; - private long _lastEvmExceptions; private long _lastSelfDestructs; - private long _maxMemory; private long _totalBlocks; - private long _processingMicroseconds; + private long _chunkProcessingMicroseconds; private long _lastTotalCreates; private long _lastReportMs; + private long _lastContractsAnalysed; + private long _lastCachedContractsUsed; + private long _blockProcessingMicroseconds; + private long _runningMicroseconds; + private long _runMicroseconds; + private long _reportMs; + private Block? _lastBlock; + private long _sloadOpcodeProcessing; + private long _sstoreOpcodeProcessing; + private long _callsProcessing; + private long _emptyCallsProcessing; + private long _codeDbCacheProcessing; + private long _contractAnalysedProcessing; + private long _createsProcessing; public ProcessingStats(ILogger logger) { @@ -51,19 +57,8 @@ public ProcessingStats(ILogger logger) #endif } - public void UpdateStats(Block? block, IBlockTree blockTreeCtx, int recoveryQueueSize, int blockQueueSize, long blockProcessingTimeInMicros) + public void UpdateStats(Block? block, IBlockTree blockTreeCtx, long blockProcessingTimeInMicros) { - const string resetColor = "\u001b[37m"; - const string whiteText = "\u001b[97m"; - const string yellowText = "\u001b[93m"; - const string orangeText = "\u001b[38;5;208m"; - const string redText = "\u001b[38;5;196m"; - const string greenText = "\u001b[92m"; - const string darkGreenText = "\u001b[32m"; - const string darkCyanText = "\u001b[36m"; - const string blueText = "\u001b[94m"; - const string darkGreyText = resetColor; // "\u001b[90m"; - if (block is null) { return; @@ -76,7 +71,7 @@ public void UpdateStats(Block? block, IBlockTree blockTreeCtx, int recoveryQueue _lastBlockNumber = block.Number; } - _processingMicroseconds += blockProcessingTimeInMicros; + _chunkProcessingMicroseconds += blockProcessingTimeInMicros; Metrics.Mgas += block.GasUsed / 1_000_000.0; Metrics.Transactions += block.Transactions.Length; @@ -85,161 +80,192 @@ public void UpdateStats(Block? block, IBlockTree blockTreeCtx, int recoveryQueue Metrics.LastDifficulty = block.Difficulty; Metrics.GasUsed = block.GasUsed; Metrics.GasLimit = block.GasLimit; - Metrics.RecoveryQueueSize = recoveryQueueSize; - Metrics.ProcessingQueueSize = blockQueueSize; Metrics.BlockchainHeight = block.Header.Number; Metrics.BestKnownBlockNumber = blockTreeCtx.BestKnownNumber; - long processingMicroseconds = _processingStopwatch.ElapsedMicroseconds(); - long runningMicroseconds = _runStopwatch.ElapsedMicroseconds(); - long runMicroseconds = (runningMicroseconds - _lastElapsedRunningMicroseconds); + _blockProcessingMicroseconds = _processingStopwatch.ElapsedMicroseconds(); + _runningMicroseconds = _runStopwatch.ElapsedMicroseconds(); + _runMicroseconds = (_runningMicroseconds - _lastElapsedRunningMicroseconds); - long reportMs = Environment.TickCount64; + long reportMs = _reportMs = Environment.TickCount64; if (reportMs - _lastReportMs > 1000) { - long currentStateDbReads = Db.Metrics.DbReads.GetValueOrDefault("StateDb"); - long currentStateDbWrites = Db.Metrics.DbWrites.GetValueOrDefault("StateDb"); - long currentTreeNodeRlp = Trie.Metrics.TreeNodeRlpEncodings + Trie.Metrics.TreeNodeRlpDecodings; - long evmExceptions = Evm.Metrics.EvmExceptions; - long currentSelfDestructs = Evm.Metrics.SelfDestructs; + _lastReportMs = _reportMs; + _lastBlock = block; + _sloadOpcodeProcessing = Evm.Metrics.ThreadLocalSLoadOpcode; + _sstoreOpcodeProcessing = Evm.Metrics.ThreadLocalSStoreOpcode; + _callsProcessing = Evm.Metrics.ThreadLocalCalls; + _emptyCallsProcessing = Evm.Metrics.ThreadLocalEmptyCalls; + _codeDbCacheProcessing = Db.Metrics.ThreadLocalCodeDbCache; + _contractAnalysedProcessing = Evm.Metrics.ThreadLocalContractsAnalysed; + _createsProcessing = Evm.Metrics.ThreadLocalCreates; + GenerateReport(); + } + } - long chunkBlocks = Metrics.Blocks - _lastBlockNumber; - _totalBlocks += chunkBlocks; + private void GenerateReport() => ThreadPool.UnsafeQueueUserWorkItem(this, preferLocal: false); - if (_logger.IsInfo) - { - double chunkMicroseconds = _processingMicroseconds; - double totalMicroseconds = processingMicroseconds; - long chunkTx = Metrics.Transactions - _lastTotalTx; - long chunkCalls = Evm.Metrics.Calls - _lastTotalCalls; - long chunkEmptyCalls = Evm.Metrics.EmptyCalls - _lastTotalEmptyCalls; - long chunkCreates = Evm.Metrics.Creates - _lastTotalCreates; - long chunkSload = Evm.Metrics.SloadOpcode - _lastTotalSLoad; - long chunkSstore = Evm.Metrics.SstoreOpcode - _lastTotalSStore; - double chunkMGas = Metrics.Mgas - _lastTotalMGas; - double mgasPerSecond = chunkMicroseconds == 0 ? -1 : chunkMGas / chunkMicroseconds * 1_000_000.0; - double totalMgasPerSecond = totalMicroseconds == 0 ? -1 : Metrics.Mgas / totalMicroseconds * 1_000_000.0; - double totalTxPerSecond = totalMicroseconds == 0 ? -1 : Metrics.Transactions / totalMicroseconds * 1_000_000.0; - double totalBlocksPerSecond = totalMicroseconds == 0 ? -1 : _totalBlocks / totalMicroseconds * 1_000_000.0; - double txps = chunkMicroseconds == 0 ? -1 : chunkTx / chunkMicroseconds * 1_000_000.0; - double bps = chunkMicroseconds == 0 ? -1 : chunkBlocks / chunkMicroseconds * 1_000_000.0; - double chunkMs = (chunkMicroseconds == 0 ? -1 : chunkMicroseconds / 1000.0); - double runMs = (runMicroseconds == 0 ? -1 : runMicroseconds / 1000.0); - string blockGas = Evm.Metrics.BlockMinGasPrice != float.MaxValue ? $"⛽ Gas gwei: {Evm.Metrics.BlockMinGasPrice:N2} .. {whiteText}{Math.Max(Evm.Metrics.BlockMinGasPrice, Evm.Metrics.BlockEstMedianGasPrice):N2}{resetColor} ({Evm.Metrics.BlockAveGasPrice:N2}) .. {Evm.Metrics.BlockMaxGasPrice:N2}" : ""; - string mgasColor = whiteText; + void IThreadPoolWorkItem.Execute() + { + const string resetColor = "\u001b[37m"; + const string whiteText = "\u001b[97m"; + const string yellowText = "\u001b[93m"; + const string orangeText = "\u001b[38;5;208m"; + const string redText = "\u001b[38;5;196m"; + const string greenText = "\u001b[92m"; + const string darkGreenText = "\u001b[32m"; + const string darkCyanText = "\u001b[36m"; + const string blueText = "\u001b[94m"; + const string darkGreyText = resetColor; // "\u001b[90m"; + + long currentSelfDestructs = Evm.Metrics.SelfDestructs; + + long chunkBlocks = Metrics.Blocks - _lastBlockNumber; + _totalBlocks += chunkBlocks; + + double chunkMicroseconds = _chunkProcessingMicroseconds; + double chunkMGas = Metrics.Mgas - _lastTotalMGas; + double mgasPerSecond; + if (chunkMicroseconds == 0) + { + mgasPerSecond = -1; + } + else + { + mgasPerSecond = chunkMGas / chunkMicroseconds * 1_000_000.0; + if (chunkMGas != 0) + { Metrics.MgasPerSec = mgasPerSecond; + } + } - if (chunkBlocks > 1) - { - _logger.Info($"Processed {block.Number - chunkBlocks + 1,9}...{block.Number,9} | {chunkMs,9:N2} ms | slot {runMs,7:N0} ms |{blockGas}"); - } - else - { - mgasColor = (chunkMGas / (block.GasLimit / 16_000_000.0)) switch - { - // At 30M gas limit the values are in comments - > 15 => redText, // 28.125 MGas - > 14 => orangeText, // 26.25 MGas - > 13 => yellowText, // 24.375 MGas - > 10 => darkGreenText, // 18.75 MGas - > 7 => greenText, // 13.125 MGas - > 6 => darkGreenText, // 11.25 MGas - > 5 => whiteText, // 9.375 MGas - > 4 => resetColor, // 7.5 MGas - > 3 => darkCyanText, // 5.625 MGas - _ => blueText - }; - var chunkColor = chunkMs switch - { - < 200 => greenText, - < 300 => darkGreenText, - < 500 => whiteText, - < 1000 => yellowText, - < 2000 => orangeText, - _ => redText - }; - _logger.Info($"Processed {block.Number,9} | {chunkColor}{chunkMs,9:N2}{resetColor} ms | slot {runMs,7:N0} ms |{blockGas}"); - } - - string mgasPerSecondColor = (mgasPerSecond / (block.GasLimit / 1_000_000.0)) switch + Block? block = Interlocked.Exchange(ref _lastBlock, null); + if (block is not null && _logger.IsInfo) + { + double totalMicroseconds = _blockProcessingMicroseconds; + long chunkTx = Metrics.Transactions - _lastTotalTx; + long chunkCalls = _callsProcessing - _lastTotalCalls; + long chunkEmptyCalls = _emptyCallsProcessing - _lastTotalEmptyCalls; + long chunkCreates = _createsProcessing - _lastTotalCreates; + long chunkSload = _sloadOpcodeProcessing - _lastTotalSLoad; + long chunkSstore = _sstoreOpcodeProcessing - _lastTotalSStore; + long contractsAnalysed = _contractAnalysedProcessing - _lastContractsAnalysed; + long cachedContractsUsed = _codeDbCacheProcessing - _lastCachedContractsUsed; + double totalMgasPerSecond = totalMicroseconds == 0 ? -1 : Metrics.Mgas / totalMicroseconds * 1_000_000.0; + double totalTxPerSecond = totalMicroseconds == 0 ? -1 : Metrics.Transactions / totalMicroseconds * 1_000_000.0; + double totalBlocksPerSecond = totalMicroseconds == 0 ? -1 : _totalBlocks / totalMicroseconds * 1_000_000.0; + double txps = chunkMicroseconds == 0 ? -1 : chunkTx / chunkMicroseconds * 1_000_000.0; + double bps = chunkMicroseconds == 0 ? -1 : chunkBlocks / chunkMicroseconds * 1_000_000.0; + double chunkMs = (chunkMicroseconds == 0 ? -1 : chunkMicroseconds / 1000.0); + double runMs = (_runMicroseconds == 0 ? -1 : _runMicroseconds / 1000.0); + string blockGas = Evm.Metrics.BlockMinGasPrice != float.MaxValue ? $"⛽ Gas gwei: {Evm.Metrics.BlockMinGasPrice:N2} .. {whiteText}{Math.Max(Evm.Metrics.BlockMinGasPrice, Evm.Metrics.BlockEstMedianGasPrice):N2}{resetColor} ({Evm.Metrics.BlockAveGasPrice:N2}) .. {Evm.Metrics.BlockMaxGasPrice:N2}" : ""; + string mgasColor = whiteText; + + if (chunkBlocks > 1) + { + _logger.Info($"Processed {block.Number - chunkBlocks + 1,10}...{block.Number,9} | {chunkMs,9:N2} ms | slot {runMs,7:N0} ms |{blockGas}"); + } + else + { + mgasColor = (chunkMGas / (block.GasLimit / 16_000_000.0)) switch { // At 30M gas limit the values are in comments - > 3 => greenText, // 90 MGas/s - > 2.5f => darkGreenText, // 75 MGas/s - > 2 => whiteText, // 60 MGas/s - > 1.5f => resetColor, // 45 MGas/s - > 1 => yellowText, // 30 MGas/s - > 0.5f => orangeText, // 15 MGas/s - _ => redText - }; - var sstoreColor = chunkBlocks > 1 ? "" : chunkSstore switch - { - > 3500 => redText, - > 2500 => orangeText, - > 2000 => yellowText, - > 1500 => whiteText, - > 900 when chunkCalls > 900 => whiteText, - _ => "" - }; - var callsColor = chunkBlocks > 1 ? "" : chunkCalls switch - { - > 3500 => redText, - > 2500 => orangeText, - > 2000 => yellowText, - > 1500 => whiteText, - > 900 when chunkSstore > 900 => whiteText, - _ => "" + > 15 => redText, // 28.125 MGas + > 14 => orangeText, // 26.25 MGas + > 13 => yellowText, // 24.375 MGas + > 10 => darkGreenText, // 18.75 MGas + > 7 => greenText, // 13.125 MGas + > 6 => darkGreenText, // 11.25 MGas + > 5 => whiteText, // 9.375 MGas + > 4 => resetColor, // 7.5 MGas + > 3 => darkCyanText, // 5.625 MGas + _ => blueText }; - var createsColor = chunkBlocks > 1 ? "" : chunkCreates switch + var chunkColor = chunkMs switch { - > 300 => redText, - > 200 => orangeText, - > 150 => yellowText, - > 75 => whiteText, - _ => "" + < 200 => greenText, + < 300 => darkGreenText, + < 500 => whiteText, + < 1000 => yellowText, + < 2000 => orangeText, + _ => redText }; - _logger.Info($"- Block{(chunkBlocks > 1 ? $"s {chunkBlocks,-9:N0}" : " ")}{(chunkBlocks == 1 ? mgasColor : "")} {chunkMGas,7:F2}{resetColor} MGas | {chunkTx,6:N0} txs | calls {callsColor}{chunkCalls,6:N0}{resetColor} {darkGreyText}({chunkEmptyCalls,3:N0}){resetColor} | sload {chunkSload,7:N0} | sstore {sstoreColor}{chunkSstore,6:N0}{resetColor} | create {createsColor}{chunkCreates,3:N0}{resetColor}{(currentSelfDestructs - _lastSelfDestructs > 0 ? $"{darkGreyText}({-(currentSelfDestructs - _lastSelfDestructs),3:N0}){resetColor}" : "")}"); - _logger.Info($"- Block throughput {mgasPerSecondColor}{mgasPerSecond,7:F2}{resetColor} MGas/s | {txps,9:F2} t/s | {bps,7:F2} Blk/s | recv {recoveryQueueSize,7:N0} | proc {blockQueueSize,6:N0}"); - // Only output the total throughput in debug mode - if (_logger.IsDebug) - { - _logger.Debug($"- Total throughput {totalMgasPerSecond,7:F2} MGas/s | {totalTxPerSecond,9:F2} t/s | {totalBlocksPerSecond,7:F2} Blk/s |⛽ Gas gwei: {Evm.Metrics.MinGasPrice:N2} .. {Math.Max(Evm.Metrics.MinGasPrice, Evm.Metrics.EstMedianGasPrice):N2} ({Evm.Metrics.AveGasPrice:N2}) .. {Evm.Metrics.MaxGasPrice:N2}"); - } + _logger.Info($"Processed {block.Number,10} | {chunkColor}{chunkMs,9:N2}{resetColor} ms | slot {runMs,7:N0} ms |{blockGas}"); + } - if (_logger.IsTrace) - { - long currentGen0 = GC.CollectionCount(0); - long currentGen1 = GC.CollectionCount(1); - long currentGen2 = GC.CollectionCount(2); - long currentMemory = GC.GetTotalMemory(false); - _maxMemory = Math.Max(_maxMemory, currentMemory); - _logger.Trace($"Gen0 {currentGen0 - _lastGen0,6}, Gen1 {currentGen1 - _lastGen1,6}, Gen2 {currentGen2 - _lastGen2,6}, maxmem {_maxMemory / 1000000,5}, mem {currentMemory / 1000000,5}, reads {currentStateDbReads - _lastStateDbReads,9}, writes {currentStateDbWrites - _lastStateDbWrites,9}, rlp {currentTreeNodeRlp - _lastTreeNodeRlp,9}, exceptions {evmExceptions - _lastEvmExceptions}, selfdstrcs {currentSelfDestructs - _lastSelfDestructs}"); - _lastGen0 = currentGen0; - _lastGen1 = currentGen1; - _lastGen2 = currentGen2; - } + string mgasPerSecondColor = (mgasPerSecond / (block.GasLimit / 1_000_000.0)) switch + { + // At 30M gas limit the values are in comments + > 3 => greenText, // 90 MGas/s + > 2.5f => darkGreenText, // 75 MGas/s + > 2 => whiteText, // 60 MGas/s + > 1.5f => resetColor, // 45 MGas/s + > 1 => yellowText, // 30 MGas/s + > 0.5f => orangeText, // 15 MGas/s + _ => redText + }; + var sstoreColor = chunkBlocks > 1 ? "" : chunkSstore switch + { + > 3500 => redText, + > 2500 => orangeText, + > 2000 => yellowText, + > 1500 => whiteText, + > 900 when chunkCalls > 900 => whiteText, + _ => "" + }; + var callsColor = chunkBlocks > 1 ? "" : chunkCalls switch + { + > 3500 => redText, + > 2500 => orangeText, + > 2000 => yellowText, + > 1500 => whiteText, + > 900 when chunkSstore > 900 => whiteText, + _ => "" + }; + var createsColor = chunkBlocks > 1 ? "" : chunkCreates switch + { + > 300 => redText, + > 200 => orangeText, + > 150 => yellowText, + > 75 => whiteText, + _ => "" + }; + var recoveryQueue = Metrics.RecoveryQueueSize; + var processingQueue = Metrics.ProcessingQueueSize; + + _logger.Info($"- Block{(chunkBlocks > 1 ? $"s {chunkBlocks,-9:N0}" : " ")}{(chunkBlocks == 1 ? mgasColor : "")} {chunkMGas,9:F2}{resetColor} MGas | {chunkTx,6:N0} txs | calls {callsColor}{chunkCalls,6:N0}{resetColor} {darkGreyText}({chunkEmptyCalls,3:N0}){resetColor} | sload {chunkSload,7:N0} | sstore {sstoreColor}{chunkSstore,6:N0}{resetColor} | create {createsColor}{chunkCreates,3:N0}{resetColor}{(currentSelfDestructs - _lastSelfDestructs > 0 ? $"{darkGreyText}({-(currentSelfDestructs - _lastSelfDestructs),3:N0}){resetColor}" : "")}"); + if (recoveryQueue > 0 || processingQueue > 0) + { + _logger.Info($"- Block throughput {mgasPerSecondColor}{mgasPerSecond,9:F2}{resetColor} MGas/s{(mgasPerSecond > 1000 ? "🔥" : " ")}| {txps,9:F2} t/s | {bps,7:F2} Blk/s | recover {recoveryQueue,5:N0} | process {processingQueue,5:N0}"); + } + else + { + _logger.Info($"- Block throughput {mgasPerSecondColor}{mgasPerSecond,9:F2}{resetColor} MGas/s{(mgasPerSecond > 1000 ? "🔥" : " ")}| {txps,9:F2} t/s | {bps,7:F2} Blk/s | exec code {resetColor} from cache {cachedContractsUsed,7:N0} |{resetColor} new {contractsAnalysed,6:N0}"); } - _lastReportMs = reportMs; - _lastBlockNumber = Metrics.Blocks; - _lastTotalMGas = Metrics.Mgas; - _lastElapsedRunningMicroseconds = runningMicroseconds; - _lastTotalTx = Metrics.Transactions; - _lastTotalCalls = Evm.Metrics.Calls; - _lastTotalEmptyCalls = Evm.Metrics.EmptyCalls; - _lastTotalCreates = Evm.Metrics.Creates; - _lastTotalSLoad = Evm.Metrics.SloadOpcode; - _lastTotalSStore = Evm.Metrics.SstoreOpcode; - _lastStateDbReads = currentStateDbReads; - _lastStateDbWrites = currentStateDbWrites; - _lastTreeNodeRlp = currentTreeNodeRlp; - _lastEvmExceptions = evmExceptions; - _lastSelfDestructs = currentSelfDestructs; - _processingMicroseconds = 0; + // Only output the total throughput in debug mode + if (_logger.IsDebug) + { + _logger.Debug($"- Total throughput {totalMgasPerSecond,9:F2} MGas/s | {totalTxPerSecond,9:F2} t/s | {totalBlocksPerSecond,7:F2} Blk/s |⛽ Gas gwei: {Evm.Metrics.MinGasPrice:N2} .. {Math.Max(Evm.Metrics.MinGasPrice, Evm.Metrics.EstMedianGasPrice):N2} ({Evm.Metrics.AveGasPrice:N2}) .. {Evm.Metrics.MaxGasPrice:N2}"); + } } + + _lastCachedContractsUsed = _codeDbCacheProcessing; + _lastContractsAnalysed = _contractAnalysedProcessing; + _lastBlockNumber = Metrics.Blocks; + _lastTotalMGas = Metrics.Mgas; + _lastElapsedRunningMicroseconds = _runningMicroseconds; + _lastTotalTx = Metrics.Transactions; + _lastTotalCalls = _callsProcessing; + _lastTotalEmptyCalls = _emptyCallsProcessing; + _lastTotalCreates = _createsProcessing; + _lastTotalSLoad = _sloadOpcodeProcessing; + _lastTotalSStore = _sstoreOpcodeProcessing; + _lastSelfDestructs = currentSelfDestructs; + _chunkProcessingMicroseconds = 0; } public void Start() diff --git a/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyChainProcessingEnv.cs b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyChainProcessingEnv.cs index 483a410e3ac..9d09f3a888e 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyChainProcessingEnv.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyChainProcessingEnv.cs @@ -2,12 +2,13 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using Nethermind.Blockchain; using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Receipts; using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Validators; using Nethermind.Core.Specs; -using Nethermind.Db; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; using Nethermind.State; @@ -18,44 +19,57 @@ namespace Nethermind.Consensus.Processing /// public class ReadOnlyChainProcessingEnv : IDisposable { - private readonly ReadOnlyTxProcessingEnv _txEnv; - private readonly BlockchainProcessor _blockProcessingQueue; public IBlockProcessor BlockProcessor { get; } public IBlockchainProcessor ChainProcessor { get; } public IBlockProcessingQueue BlockProcessingQueue { get; } - public IWorldState StateProvider => _txEnv.StateProvider; public ReadOnlyChainProcessingEnv( - ReadOnlyTxProcessingEnv txEnv, + IReadOnlyTxProcessingScope scope, IBlockValidator blockValidator, IBlockPreprocessorStep recoveryStep, IRewardCalculator rewardCalculator, IReceiptStorage receiptStorage, ISpecProvider specProvider, + IBlockTree blockTree, + IStateReader stateReader, ILogManager logManager, IBlockProcessor.IBlockTransactionsExecutor? blockTransactionsExecutor = null) { - _txEnv = txEnv; - IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor = - blockTransactionsExecutor ?? new BlockProcessor.BlockValidationTransactionsExecutor(_txEnv.TransactionProcessor, StateProvider); + blockTransactionsExecutor ?? new BlockProcessor.BlockValidationTransactionsExecutor(scope.TransactionProcessor, scope.WorldState); + + BlockProcessor = CreateBlockProcessor(scope, blockTree, blockValidator, rewardCalculator, receiptStorage, specProvider, logManager, transactionsExecutor); + + _blockProcessingQueue = new BlockchainProcessor(blockTree, BlockProcessor, recoveryStep, stateReader, logManager, BlockchainProcessor.Options.NoReceipts); + BlockProcessingQueue = _blockProcessingQueue; + ChainProcessor = new OneTimeChainProcessor(scope.WorldState, _blockProcessingQueue); + _blockProcessingQueue = new BlockchainProcessor(blockTree, BlockProcessor, recoveryStep, stateReader, logManager, BlockchainProcessor.Options.NoReceipts); + BlockProcessingQueue = _blockProcessingQueue; + ChainProcessor = new OneTimeChainProcessor(scope.WorldState, _blockProcessingQueue); + } - BlockProcessor = new BlockProcessor( + protected virtual IBlockProcessor CreateBlockProcessor( + IReadOnlyTxProcessingScope scope, + IBlockTree blockTree, + IBlockValidator blockValidator, + IRewardCalculator rewardCalculator, + IReceiptStorage receiptStorage, + ISpecProvider specProvider, + ILogManager logManager, + IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor + ) + { + return new BlockProcessor( specProvider, blockValidator, rewardCalculator, transactionsExecutor, - StateProvider, + scope.WorldState, receiptStorage, - NullWitnessCollector.Instance, - new BlockhashStore(txEnv.BlockTree, specProvider, StateProvider), - _txEnv.TransactionProcessor, + new BlockhashStore(blockTree, specProvider, scope.WorldState), + scope.TransactionProcessor, logManager); - - _blockProcessingQueue = new BlockchainProcessor(_txEnv.BlockTree, BlockProcessor, recoveryStep, _txEnv.StateReader, logManager, BlockchainProcessor.Options.NoReceipts); - BlockProcessingQueue = _blockProcessingQueue; - ChainProcessor = new OneTimeChainProcessor(txEnv.StateProvider, _blockProcessingQueue); } public void Dispose() diff --git a/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnv.cs b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnv.cs index 47671734515..dc13eca75dc 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnv.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnv.cs @@ -14,43 +14,58 @@ // ReSharper disable UnusedAutoPropertyAccessor.Global namespace Nethermind.Consensus.Processing { - public class ReadOnlyTxProcessingEnv : IReadOnlyTxProcessorSource + public class ReadOnlyTxProcessingEnv : ReadOnlyTxProcessingEnvBase, IReadOnlyTxProcessorSource { - public IStateReader StateReader { get; } - public IWorldState StateProvider { get; } - public ITransactionProcessor TransactionProcessor { get; set; } - public IBlockTree BlockTree { get; } - public IBlockhashProvider BlockhashProvider { get; } + protected readonly ILogManager _logManager; + + protected ITransactionProcessor? _transactionProcessor; + protected ITransactionProcessor TransactionProcessor + { + get + { + return _transactionProcessor ??= CreateTransactionProcessor(); + } + } + public IVirtualMachine Machine { get; } + public ICodeInfoRepository CodeInfoRepository { get; } public ReadOnlyTxProcessingEnv( IWorldStateManager worldStateManager, - IBlockTree? blockTree, + IBlockTree blockTree, ISpecProvider? specProvider, - ILogManager? logManager) - : this(worldStateManager, blockTree?.AsReadOnly(), specProvider, logManager) + ILogManager? logManager, + IWorldState? worldStateToWarmUp = null) + : this(worldStateManager, blockTree.AsReadOnly(), specProvider, logManager, worldStateToWarmUp) { } public ReadOnlyTxProcessingEnv( IWorldStateManager worldStateManager, - IReadOnlyBlockTree? readOnlyBlockTree, + IReadOnlyBlockTree readOnlyBlockTree, ISpecProvider? specProvider, - ILogManager? logManager) + ILogManager? logManager, + IWorldState? worldStateToWarmUp = null + ) : base(worldStateManager, readOnlyBlockTree, specProvider, logManager, worldStateToWarmUp) { - ArgumentNullException.ThrowIfNull(specProvider); - ArgumentNullException.ThrowIfNull(worldStateManager); - - StateReader = worldStateManager.GlobalStateReader; - StateProvider = worldStateManager.CreateResettableWorldState(); - + CodeInfoRepository = new CodeInfoRepository(); + Machine = new VirtualMachine(BlockhashProvider, specProvider, CodeInfoRepository, logManager); BlockTree = readOnlyBlockTree ?? throw new ArgumentNullException(nameof(readOnlyBlockTree)); BlockhashProvider = new BlockhashProvider(BlockTree, specProvider, StateProvider, logManager); - Machine = new VirtualMachine(BlockhashProvider, specProvider, logManager); - TransactionProcessor = new TransactionProcessor(specProvider, StateProvider, Machine, logManager); + _logManager = logManager; } - public IReadOnlyTransactionProcessor Build(Hash256 stateRoot) => new ReadOnlyTransactionProcessor(TransactionProcessor, StateProvider, stateRoot); + protected virtual TransactionProcessor CreateTransactionProcessor() + { + return new TransactionProcessor(SpecProvider, StateProvider, Machine, CodeInfoRepository, _logManager); + } + + public IReadOnlyTxProcessingScope Build(Hash256 stateRoot) + { + Hash256 originalStateRoot = StateProvider.StateRoot; + StateProvider.StateRoot = stateRoot; + return new ReadOnlyTxProcessingScope(TransactionProcessor, StateProvider, originalStateRoot); + } } } diff --git a/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnvBase.cs b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnvBase.cs new file mode 100644 index 00000000000..058aca9f9c2 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnvBase.cs @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Blockchain; +using Nethermind.Core.Specs; +using Nethermind.Evm; +using Nethermind.Logging; +using Nethermind.State; + +namespace Nethermind.Consensus.Processing; + +public class ReadOnlyTxProcessingEnvBase +{ + public IStateReader StateReader { get; protected set; } + protected IWorldState StateProvider { get; set; } + public IBlockTree BlockTree { get; protected set; } + public IBlockhashProvider BlockhashProvider { get; protected set; } + + public ISpecProvider SpecProvider { get; } + public ILogManager? LogManager { get; set; } + + protected ReadOnlyTxProcessingEnvBase( + IWorldStateManager worldStateManager, + IBlockTree readOnlyBlockTree, + ISpecProvider? specProvider, + ILogManager? logManager, + IWorldState? worldStateToWarmUp = null + ) + { + ArgumentNullException.ThrowIfNull(specProvider); + ArgumentNullException.ThrowIfNull(worldStateManager); + SpecProvider = specProvider; + StateReader = worldStateManager.GlobalStateReader; + StateProvider = worldStateManager.CreateResettableWorldState(worldStateToWarmUp); + BlockTree = readOnlyBlockTree ?? throw new ArgumentNullException(nameof(readOnlyBlockTree)); + BlockhashProvider = new BlockhashProvider(BlockTree, specProvider, StateProvider, logManager); + LogManager = logManager; + } +} diff --git a/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnvFactory.cs b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnvFactory.cs index 1408e3c547e..93c214be0cd 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnvFactory.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingEnvFactory.cs @@ -3,40 +3,27 @@ using Nethermind.Blockchain; using Nethermind.Core.Specs; -using Nethermind.Db; using Nethermind.Logging; using Nethermind.State; -using Nethermind.Trie.Pruning; namespace Nethermind.Consensus.Processing; -public class ReadOnlyTxProcessingEnvFactory +public class ReadOnlyTxProcessingEnvFactory( + IWorldStateManager worldStateManager, + IReadOnlyBlockTree readOnlyBlockTree, + ISpecProvider? specProvider, + ILogManager? logManager, + IWorldState? worldStateToWarmUp = null) { - private readonly IWorldStateManager _worldStateManager; - private readonly IReadOnlyBlockTree? _readOnlyBlockTree; - private readonly ISpecProvider? _specProvider; - private readonly ILogManager? _logManager; - - public ReadOnlyTxProcessingEnvFactory( - IWorldStateManager worldStateManager, - IBlockTree? blockTree, - ISpecProvider? specProvider, - ILogManager? logManager) - : this(worldStateManager, blockTree?.AsReadOnly(), specProvider, logManager) - { - } - public ReadOnlyTxProcessingEnvFactory( IWorldStateManager worldStateManager, - IReadOnlyBlockTree? readOnlyBlockTree, + IBlockTree blockTree, ISpecProvider? specProvider, - ILogManager? logManager) + ILogManager? logManager, + IWorldState? worldStateToWarmUp = null) + : this(worldStateManager, blockTree.AsReadOnly(), specProvider, logManager, worldStateToWarmUp) { - _worldStateManager = worldStateManager; - _readOnlyBlockTree = readOnlyBlockTree; - _specProvider = specProvider; - _logManager = logManager; } - public ReadOnlyTxProcessingEnv Create() => new(_worldStateManager, _readOnlyBlockTree, _specProvider, _logManager); + public ReadOnlyTxProcessingEnv Create() => new(worldStateManager, readOnlyBlockTree, specProvider, logManager, worldStateToWarmUp); } diff --git a/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingScope.cs b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingScope.cs new file mode 100644 index 00000000000..bf14441b48b --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Processing/ReadOnlyTxProcessingScope.cs @@ -0,0 +1,24 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.State; + +namespace Nethermind.Consensus.Processing; + +public class ReadOnlyTxProcessingScope( + ITransactionProcessor transactionProcessor, + IWorldState worldState, + Hash256 originalStateRoot +) : IReadOnlyTxProcessingScope +{ + public void Dispose() + { + worldState.StateRoot = originalStateRoot; + worldState.Reset(); + } + + public ITransactionProcessor TransactionProcessor => transactionProcessor; + public IWorldState WorldState => worldState; +} diff --git a/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerBase.cs b/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerBase.cs index 2aa2abc4e57..94ae0ee38f0 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerBase.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerBase.cs @@ -16,14 +16,12 @@ using Nethermind.Int256; using Nethermind.Logging; using Nethermind.State; -using Nethermind.Trie; using Metrics = Nethermind.Blockchain.Metrics; namespace Nethermind.Consensus.Producers { /// /// I think this class can be significantly simplified if we split the block production into a pipeline: - /// * signal block needed /// * prepare block frame /// * select transactions /// * seal @@ -36,7 +34,6 @@ public abstract class BlockProducerBase : IBlockProducer private IBlockchainProcessor Processor { get; } protected IBlockTree BlockTree { get; } private ITimestamper Timestamper { get; } - public event EventHandler? BlockProduced; private ISealer Sealer { get; } private IWorldState StateProvider { get; } @@ -44,13 +41,8 @@ public abstract class BlockProducerBase : IBlockProducer private readonly IDifficultyCalculator _difficultyCalculator; protected readonly ISpecProvider _specProvider; private readonly ITxSource _txSource; - private readonly IBlockProductionTrigger _trigger; - private bool _isRunning; - protected readonly SemaphoreSlim _producingBlockLock = new(1); - private CancellationTokenSource? _producerCancellationToken; - - private DateTime _lastProducedBlockDateTime; protected const int BlockProductionTimeout = 2000; + protected readonly SemaphoreSlim _producingBlockLock = new(1); protected ILogger Logger { get; } protected readonly IBlocksConfig _blocksConfig; @@ -59,7 +51,6 @@ protected BlockProducerBase( IBlockchainProcessor? processor, ISealer? sealer, IBlockTree? blockTree, - IBlockProductionTrigger? trigger, IWorldState? stateProvider, IGasLimitCalculator? gasLimitCalculator, ITimestamper? timestamper, @@ -76,59 +67,21 @@ protected BlockProducerBase( _gasLimitCalculator = gasLimitCalculator ?? throw new ArgumentNullException(nameof(gasLimitCalculator)); Timestamper = timestamper ?? throw new ArgumentNullException(nameof(timestamper)); _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); - _trigger = trigger ?? throw new ArgumentNullException(nameof(trigger)); _difficultyCalculator = difficultyCalculator ?? throw new ArgumentNullException(nameof(difficultyCalculator)); Logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _blocksConfig = blocksConfig ?? throw new ArgumentNullException(nameof(blocksConfig)); } - private void OnTriggerBlockProduction(object? sender, BlockProductionEventArgs e) - { - BlockHeader? parent = BlockTree.GetProducedBlockParent(e.ParentHeader); - e.BlockProductionTask = TryProduceAndAnnounceNewBlock(e.CancellationToken, parent, e.BlockTracer, e.PayloadAttributes); - } - - public virtual Task Start() + public async Task BuildBlock(BlockHeader? parentHeader = null, IBlockTracer? blockTracer = null, + PayloadAttributes? payloadAttributes = null, CancellationToken? token = null) { - _producerCancellationToken = new CancellationTokenSource(); - _isRunning = true; - _trigger.TriggerBlockProduction += OnTriggerBlockProduction; - _lastProducedBlockDateTime = DateTime.UtcNow; - return Task.CompletedTask; - } - - public virtual Task StopAsync() - { - _producerCancellationToken?.Cancel(); - _isRunning = false; - _trigger.TriggerBlockProduction -= OnTriggerBlockProduction; - _producerCancellationToken?.Dispose(); - return Task.CompletedTask; - } - - protected virtual bool IsRunning() => _isRunning; - - public bool IsProducingBlocks(ulong? maxProducingInterval) - { - if (Logger.IsTrace) Logger.Trace($"Checking IsProducingBlocks: maxProducingInterval {maxProducingInterval}, _lastProducedBlock {_lastProducedBlockDateTime}, IsRunning() {IsRunning()}"); - return IsRunning() && (maxProducingInterval is null || _lastProducedBlockDateTime.AddSeconds(maxProducingInterval.Value) > DateTime.UtcNow); - } - - private async Task TryProduceAndAnnounceNewBlock(CancellationToken token, BlockHeader? parentHeader, IBlockTracer? blockTracer = null, PayloadAttributes? payloadAttributes = null) - { - using CancellationTokenSource tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, _producerCancellationToken!.Token); - token = tokenSource.Token; - + token ??= default; Block? block = null; - if (await _producingBlockLock.WaitAsync(BlockProductionTimeout, token)) + if (await _producingBlockLock.WaitAsync(BlockProductionTimeout, token.Value)) { try { - block = await TryProduceNewBlock(token, parentHeader, blockTracer, payloadAttributes); - if (block is not null) - { - BlockProduced?.Invoke(this, new BlockEventArgs(block)); - } + block = await TryProduceNewBlock(token.Value, parentHeader, blockTracer, payloadAttributes); } catch (Exception e) when (!(e is TaskCanceledException)) { @@ -196,7 +149,6 @@ public bool IsProducingBlocks(ulong? maxProducingInterval) if (Logger.IsInfo) Logger.Info($"Produced block {t.Result.ToString(Block.Format.HashNumberDiffAndTx)}"); Metrics.BlocksSealed++; - _lastProducedBlockDateTime = DateTime.UtcNow; return t.Result; } else diff --git a/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerEnvFactory.cs b/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerEnvFactory.cs index 9c6270a5e19..3cc5cfde05a 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerEnvFactory.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerEnvFactory.cs @@ -12,7 +12,9 @@ using Nethermind.Consensus.Transactions; using Nethermind.Consensus.Validators; using Nethermind.Consensus.Withdrawals; +using Nethermind.Core.Crypto; using Nethermind.Core.Specs; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; using Nethermind.State; using Nethermind.TxPool; @@ -73,8 +75,11 @@ public virtual BlockProducerEnv Create(ITxSource? additionalTxSource = null) ReadOnlyTxProcessingEnv txProcessingEnv = CreateReadonlyTxProcessingEnv(_worldStateManager, readOnlyBlockTree); + IReadOnlyTxProcessingScope scope = txProcessingEnv.Build(Keccak.EmptyTreeHash); + BlockProcessor blockProcessor = - CreateBlockProcessor(txProcessingEnv, + CreateBlockProcessor( + scope, _specProvider, _blockValidator, _rewardCalculatorSource, @@ -92,14 +97,14 @@ public virtual BlockProducerEnv Create(ITxSource? additionalTxSource = null) BlockchainProcessor.Options.NoReceipts); OneTimeChainProcessor chainProcessor = new( - txProcessingEnv.StateProvider, + scope.WorldState, blockchainProcessor); return new BlockProducerEnv { BlockTree = readOnlyBlockTree, ChainProcessor = chainProcessor, - ReadOnlyStateProvider = txProcessingEnv.StateProvider, + ReadOnlyStateProvider = scope.WorldState, TxSource = CreateTxSourceForProducer(additionalTxSource, txProcessingEnv, _txPool, _blocksConfig, _transactionComparerProvider, _logManager), ReadOnlyTxProcessingEnv = txProcessingEnv }; @@ -134,7 +139,8 @@ protected virtual TxPoolTxSource CreateTxPoolTxSource( protected virtual ITxFilterPipeline CreateTxSourceFilter(IBlocksConfig blocksConfig) => TxFilterPipelineBuilder.CreateStandardFilteringPipeline(_logManager, _specProvider, blocksConfig); - protected virtual BlockProcessor CreateBlockProcessor(ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv, + protected virtual BlockProcessor CreateBlockProcessor( + IReadOnlyTxProcessingScope readOnlyTxProcessingEnv, ISpecProvider specProvider, IBlockValidator blockValidator, IRewardCalculatorSource rewardCalculatorSource, @@ -144,14 +150,13 @@ protected virtual BlockProcessor CreateBlockProcessor(ReadOnlyTxProcessingEnv re blockValidator, rewardCalculatorSource.Get(readOnlyTxProcessingEnv.TransactionProcessor), TransactionsExecutorFactory.Create(readOnlyTxProcessingEnv), - readOnlyTxProcessingEnv.StateProvider, + readOnlyTxProcessingEnv.WorldState, receiptStorage, - NullWitnessCollector.Instance, - new BlockhashStore(_blockTree, _specProvider, readOnlyTxProcessingEnv.StateProvider), + new BlockhashStore(_blockTree, _specProvider, readOnlyTxProcessingEnv.WorldState), readOnlyTxProcessingEnv.TransactionProcessor, logManager, - new BlockProductionWithdrawalProcessor(new WithdrawalProcessor(readOnlyTxProcessingEnv.StateProvider, logManager)), - consensusRequestsProcessor: _consensusRequestsProcessor); - + new BlockProductionWithdrawalProcessor(new WithdrawalProcessor(readOnlyTxProcessingEnv.WorldState, logManager)), + consensusRequestsProcessor: _consensusRequestsProcessor + ); } } diff --git a/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerTransactionsExecutorFactory.cs b/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerTransactionsExecutorFactory.cs index 3f509d1eb45..5f00e0e76a0 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerTransactionsExecutorFactory.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/BlockProducerTransactionsExecutorFactory.cs @@ -3,6 +3,7 @@ using Nethermind.Consensus.Processing; using Nethermind.Core.Specs; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; namespace Nethermind.Consensus.Producers @@ -19,7 +20,7 @@ public BlockProducerTransactionsExecutorFactory(ISpecProvider specProvider, ILog _logManager = logManager; } - public IBlockProcessor.IBlockTransactionsExecutor Create(ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv) => + public IBlockProcessor.IBlockTransactionsExecutor Create(IReadOnlyTxProcessingScope readOnlyTxProcessingEnv) => new BlockProcessor.BlockProductionTransactionsExecutor(readOnlyTxProcessingEnv, _specProvider, _logManager); } } diff --git a/src/Nethermind/Nethermind.Consensus/Producers/DevBlockProducer.cs b/src/Nethermind/Nethermind.Consensus/Producers/DevBlockProducer.cs index 162a9e2dfcb..d164fda7537 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/DevBlockProducer.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/DevBlockProducer.cs @@ -23,7 +23,6 @@ public DevBlockProducer( IBlockchainProcessor? processor, IWorldState? stateProvider, IBlockTree? blockTree, - IBlockProductionTrigger? trigger, ITimestamper? timestamper, ISpecProvider? specProvider, IBlocksConfig? blockConfig, @@ -33,7 +32,6 @@ public DevBlockProducer( processor, new NethDevSealEngine(), blockTree, - trigger, stateProvider, new FollowOtherMiners(specProvider!), timestamper, diff --git a/src/Nethermind/Nethermind.Consensus/Producers/IBlockProducerInfo.cs b/src/Nethermind/Nethermind.Consensus/Producers/IBlockProducerInfo.cs index 53c35e52ab8..1df7ce39e70 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/IBlockProducerInfo.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/IBlockProducerInfo.cs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; +using Nethermind.Core; using Nethermind.Evm.Tracing; namespace Nethermind.Consensus.Producers @@ -8,7 +10,7 @@ namespace Nethermind.Consensus.Producers public interface IBlockProducerInfo { IBlockProducer BlockProducer { get; } - IManualBlockProductionTrigger BlockProductionTrigger { get; } + IBlockProductionCondition Condition { get; } IBlockTracer BlockTracer => NullBlockTracer.Instance; } } diff --git a/src/Nethermind/Nethermind.Consensus/Producers/IBlockProductionCondition.cs b/src/Nethermind/Nethermind.Consensus/Producers/IBlockProductionCondition.cs new file mode 100644 index 00000000000..2c0feb2755d --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Producers/IBlockProductionCondition.cs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; + +namespace Nethermind.Consensus.Producers; + +public interface IBlockProductionCondition +{ + bool CanProduce(BlockHeader parentHeader); +} + +public class AlwaysOkBlockProductionCondition : IBlockProductionCondition +{ + public static IBlockProductionCondition Instance = new AlwaysOkBlockProductionCondition(); + + public bool CanProduce(BlockHeader parentHeader) + { + return true; + } +} diff --git a/src/Nethermind/Nethermind.Consensus/Producers/IBlockTransactionsExecutorFactory.cs b/src/Nethermind/Nethermind.Consensus/Producers/IBlockTransactionsExecutorFactory.cs index 9ffd775764e..d3164780c86 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/IBlockTransactionsExecutorFactory.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/IBlockTransactionsExecutorFactory.cs @@ -2,11 +2,12 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Consensus.Processing; +using Nethermind.Evm.TransactionProcessing; namespace Nethermind.Consensus.Producers { public interface IBlockTransactionsExecutorFactory { - IBlockProcessor.IBlockTransactionsExecutor Create(ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv); + IBlockProcessor.IBlockTransactionsExecutor Create(IReadOnlyTxProcessingScope readOnlyTxProcessingEnv); } } diff --git a/src/Nethermind/Nethermind.Consensus/Producers/MultipleBlockProducer.cs b/src/Nethermind/Nethermind.Consensus/Producers/MultipleBlockProducer.cs index 116e55bb3e7..57836b2e474 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/MultipleBlockProducer.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/MultipleBlockProducer.cs @@ -7,84 +7,41 @@ using System.Threading; using System.Threading.Tasks; using Nethermind.Core; +using Nethermind.Evm.Tracing; using Nethermind.Logging; namespace Nethermind.Consensus.Producers { public abstract class MultipleBlockProducer : IBlockProducer where T : IBlockProducerInfo { - private readonly IBlockProductionTrigger _blockProductionTrigger; private readonly IBestBlockPicker _bestBlockPicker; private readonly T[] _blockProducers; private readonly ILogger _logger; protected MultipleBlockProducer( - IBlockProductionTrigger blockProductionTrigger, IBestBlockPicker bestBlockPicker, ILogManager logManager, params T[] blockProducers) { if (blockProducers.Length == 0) throw new ArgumentException("Collection cannot be empty.", nameof(blockProducers)); - _blockProductionTrigger = blockProductionTrigger; _bestBlockPicker = bestBlockPicker; _blockProducers = blockProducers; _logger = logManager.GetClassLogger(); } - public Task Start() - { - for (int index = 0; index < _blockProducers.Length; index++) - { - IBlockProducer blockProducer = _blockProducers[index].BlockProducer; - blockProducer.Start(); - } - - _blockProductionTrigger.TriggerBlockProduction += OnBlockProduction; - return Task.CompletedTask; - } - - public Task StopAsync() - { - _blockProductionTrigger.TriggerBlockProduction -= OnBlockProduction; - - IList stopTasks = new List(); - for (int index = 0; index < _blockProducers.Length; index++) - { - IBlockProducer blockProducer = _blockProducers[index].BlockProducer; - stopTasks.Add(blockProducer.StopAsync()); - } - - return Task.WhenAll(stopTasks); - } - - public bool IsProducingBlocks(ulong? maxProducingInterval) - { - for (int index = 0; index < _blockProducers.Length; index++) - { - IBlockProducer blockProducer = _blockProducers[index].BlockProducer; - if (blockProducer.IsProducingBlocks(maxProducingInterval)) - { - return true; - } - } - - return false; - } - - public event EventHandler? BlockProduced; - - private void OnBlockProduction(object? sender, BlockProductionEventArgs e) - { - e.BlockProductionTask = TryProduceBlock(e.ParentHeader, e.CancellationToken); - } - - private async Task TryProduceBlock(BlockHeader? parentHeader, CancellationToken cancellationToken = default) + public async Task BuildBlock(BlockHeader? parentHeader, IBlockTracer? blockTracer = null, + PayloadAttributes? payloadAttributes = null, CancellationToken? token = null) { Task[] produceTasks = new Task[_blockProducers.Length]; for (int i = 0; i < _blockProducers.Length; i++) { T blockProducerInfo = _blockProducers[i]; - produceTasks[i] = blockProducerInfo.BlockProductionTrigger.BuildBlock(parentHeader, cancellationToken, blockProducerInfo.BlockTracer); + if (!blockProducerInfo.Condition.CanProduce(parentHeader)) + { + produceTasks[i] = Task.FromResult(null); + continue; + } + produceTasks[i] = blockProducerInfo.BlockProducer.BuildBlock(parentHeader, blockProducerInfo.BlockTracer, cancellationToken: token); } IEnumerable<(Block? Block, T BlockProducer)> blocksWithProducers; @@ -109,8 +66,6 @@ private void OnBlockProduction(object? sender, BlockProductionEventArgs e) { if (_logger.IsInfo) _logger.Info($"Picked block {bestBlock} to be included to the chain."); } - - BlockProduced?.Invoke(this, new BlockEventArgs(bestBlock)); } return bestBlock; diff --git a/src/Nethermind/Nethermind.Consensus/Producers/ProducedBlockSuggester.cs b/src/Nethermind/Nethermind.Consensus/Producers/ProducedBlockSuggester.cs index 94c1abbaecc..a061e7f6e8c 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/ProducedBlockSuggester.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/ProducedBlockSuggester.cs @@ -10,13 +10,13 @@ namespace Nethermind.Consensus.Producers public class ProducedBlockSuggester : IDisposable { private readonly IBlockTree _blockTree; - private readonly IBlockProducer _blockProducer; + private readonly IBlockProducerRunner _blockProducerRunner; - public ProducedBlockSuggester(IBlockTree blockTree, IBlockProducer blockProducer) + public ProducedBlockSuggester(IBlockTree blockTree, IBlockProducerRunner blockProducer) { _blockTree = blockTree; - _blockProducer = blockProducer; - _blockProducer.BlockProduced += OnBlockProduced; + _blockProducerRunner = blockProducer; + _blockProducerRunner.BlockProduced += OnBlockProduced; } private void OnBlockProduced(object? sender, BlockEventArgs e) @@ -28,6 +28,6 @@ private void OnBlockProduced(object? sender, BlockEventArgs e) } } - public void Dispose() => _blockProducer.BlockProduced -= OnBlockProduced; + public void Dispose() => _blockProducerRunner.BlockProduced -= OnBlockProduced; } } diff --git a/src/Nethermind/Nethermind.Consensus/Signer.cs b/src/Nethermind/Nethermind.Consensus/Signer.cs index 3a48950a129..0a22c939499 100644 --- a/src/Nethermind/Nethermind.Consensus/Signer.cs +++ b/src/Nethermind/Nethermind.Consensus/Signer.cs @@ -21,6 +21,8 @@ public class Signer : ISigner, ISignerStore public bool CanSign => _key is not null; + public bool CanSignHeader => false; + public Signer(ulong chainId, PrivateKey? key, ILogManager logManager) { _chainId = chainId; @@ -42,6 +44,11 @@ public Signature Sign(Hash256 message) return new Signature(rs, v); } + public Signature Sign(BlockHeader header) + { + return Sign(header.Hash); + } + public ValueTask Sign(Transaction tx) { Hash256 hash = Keccak.Compute(Rlp.Encode(tx, true, true, _chainId).Bytes); diff --git a/src/Nethermind/Nethermind.Consensus/StandardBlockProducerRunner.cs b/src/Nethermind/Nethermind.Consensus/StandardBlockProducerRunner.cs new file mode 100644 index 00000000000..25dc0cb8a03 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/StandardBlockProducerRunner.cs @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Blockchain; +using Nethermind.Consensus.Producers; +using Nethermind.Core; +using Nethermind.Evm.Tracing; +using Nethermind.Logging; + +namespace Nethermind.Consensus; + +public class StandardBlockProducerRunner(IBlockProductionTrigger trigger, IBlockTree blockTree, IBlockProducer blockProducer) : IBlockProducerRunner +{ + private bool _isRunning; + private CancellationTokenSource? _producerCancellationToken; + private DateTime _lastProducedBlockDateTime; + public ILogger Logger { get; } + + private void OnTriggerBlockProduction(object? sender, BlockProductionEventArgs e) + { + BlockHeader? parent = blockTree.GetProducedBlockParent(e.ParentHeader); + e.BlockProductionTask = TryProduceAndAnnounceNewBlock(e.CancellationToken, parent, e.BlockTracer, e.PayloadAttributes); + } + + private async Task TryProduceAndAnnounceNewBlock(CancellationToken token, BlockHeader? parentHeader, IBlockTracer? blockTracer = null, PayloadAttributes? payloadAttributes = null) + { + using CancellationTokenSource tokenSource = CancellationTokenSource.CreateLinkedTokenSource(token, _producerCancellationToken!.Token); + token = tokenSource.Token; + + Block? block = null; + try + { + block = await blockProducer.BuildBlock(parentHeader, blockTracer, payloadAttributes, token); + if (block is not null) + { + _lastProducedBlockDateTime = DateTime.UtcNow; + BlockProduced?.Invoke(this, new BlockEventArgs(block)); + } + } + catch (Exception e) when (!(e is TaskCanceledException)) + { + if (Logger.IsError) Logger.Error("Failed to produce block", e); + Metrics.FailedBlockSeals++; + throw; + } + + return block; + } + + public virtual void Start() + { + _producerCancellationToken = new CancellationTokenSource(); + _isRunning = true; + trigger.TriggerBlockProduction += OnTriggerBlockProduction; + _lastProducedBlockDateTime = DateTime.UtcNow; + } + + public virtual Task StopAsync() + { + _producerCancellationToken?.Cancel(); + _isRunning = false; + trigger.TriggerBlockProduction -= OnTriggerBlockProduction; + _producerCancellationToken?.Dispose(); + return Task.CompletedTask; + } + + protected virtual bool IsRunning() => _isRunning; + + public bool IsProducingBlocks(ulong? maxProducingInterval) + { + if (Logger.IsTrace) Logger.Trace($"Checking IsProducingBlocks: maxProducingInterval {maxProducingInterval}, _lastProducedBlock {_lastProducedBlockDateTime}, IsRunning() {IsRunning()}"); + return IsRunning() && (maxProducingInterval is null || _lastProducedBlockDateTime.AddSeconds(maxProducingInterval.Value) > DateTime.UtcNow); + } + + public event EventHandler? BlockProduced; +} diff --git a/src/Nethermind/Nethermind.Consensus/Validators/SimulateBlockValidatorProxy.cs b/src/Nethermind/Nethermind.Consensus/Validators/SimulateBlockValidatorProxy.cs new file mode 100644 index 00000000000..83381ed2271 --- /dev/null +++ b/src/Nethermind/Nethermind.Consensus/Validators/SimulateBlockValidatorProxy.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; + +namespace Nethermind.Consensus.Validators; + +public class SimulateBlockValidatorProxy(IBlockValidator baseBlockValidator) : IBlockValidator +{ + public bool ValidateWithdrawals(Block block, out string? error) => + baseBlockValidator.ValidateWithdrawals(block, out error); + + public bool ValidateOrphanedBlock(Block block, out string? error) => + baseBlockValidator.ValidateOrphanedBlock(block, out error); + + public bool ValidateSuggestedBlock(Block block, out string? error, bool validateHashes = true) => + baseBlockValidator.ValidateSuggestedBlock(block, out error, validateHashes); + + public bool ValidateProcessedBlock(Block processedBlock, TxReceipt[] receipts, Block suggestedBlock, out string? error) + { + error = ""; + return true; + } + + public bool Validate(BlockHeader header, BlockHeader? parent, bool isUncle, out string? error) => + baseBlockValidator.Validate(header, parent, isUncle, out error); + + public bool Validate(BlockHeader header, bool isUncle, out string? error) => + baseBlockValidator.Validate(header, isUncle, out error); +} diff --git a/src/Nethermind/Nethermind.Consensus/Withdrawals/WithdrawalProcessor.cs b/src/Nethermind/Nethermind.Consensus/Withdrawals/WithdrawalProcessor.cs index c9fe1bddde3..e315c5a5a8e 100644 --- a/src/Nethermind/Nethermind.Consensus/Withdrawals/WithdrawalProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus/Withdrawals/WithdrawalProcessor.cs @@ -31,8 +31,10 @@ public void ProcessWithdrawals(Block block, IReleaseSpec spec) if (block.Withdrawals is not null) { - foreach (var withdrawal in block.Withdrawals) + Withdrawal[] blockWithdrawals = block.Withdrawals; + for (int i = 0; i < blockWithdrawals.Length; i++) { + Withdrawal withdrawal = blockWithdrawals[i]; if (_logger.IsTrace) _logger.Trace($" {withdrawal.AmountInGwei} GWei to account {withdrawal.Address}"); // Consensus clients are using Gwei for withdrawals amount. We need to convert it to Wei before applying state changes https://github.com/ethereum/execution-apis/pull/354 diff --git a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockProducer.cs b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockProducer.cs index bb4676e4d03..7aa64141a0d 100644 --- a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockProducer.cs +++ b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockProducer.cs @@ -24,7 +24,6 @@ public TestBlockProducer( IWorldState stateProvider, ISealer sealer, IBlockTree blockTree, - IBlockProductionTrigger blockProductionTrigger, ITimestamper timestamper, ISpecProvider specProvider, ILogManager logManager, @@ -34,7 +33,6 @@ public TestBlockProducer( processor, sealer, blockTree, - blockProductionTrigger, stateProvider, new FollowOtherMiners(specProvider), timestamper, diff --git a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs index 715cca275de..8e0d79aff7e 100644 --- a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs +++ b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs @@ -9,7 +9,10 @@ using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.BeaconBlockRoot; using Nethermind.Blockchain.Find; +using Nethermind.Blockchain.FullPruning; +using Nethermind.Blockchain.Headers; using Nethermind.Blockchain.Receipts; +using Nethermind.Blockchain.Synchronization; using Nethermind.Config; using Nethermind.Consensus; using Nethermind.Consensus.Comparers; @@ -76,6 +79,7 @@ public IBlockFinder BlockFinder public IDb StateDb => DbProvider.StateDb; public TrieStore TrieStore { get; set; } = null!; public IBlockProducer BlockProducer { get; private set; } = null!; + public IBlockProducerRunner BlockProducerRunner { get; protected set; } = null!; public IDbProvider DbProvider { get; set; } = null!; public ISpecProvider SpecProvider { get; set; } = null!; @@ -117,6 +121,7 @@ protected TestBlockchain() public ProducedBlockSuggester Suggester { get; protected set; } = null!; public IConsensusRequestsProcessor? ConsensusRequestsProcessor { get; protected set; } = null!; + public ChainLevelInfoRepository ChainLevelInfoRepository { get; protected set; } = null!; public static TransactionBuilder BuildSimpleTransaction => Builders.Build.A.Transaction.SignedAndResolved(TestItem.PrivateKeyA).To(AccountB); @@ -128,7 +133,7 @@ protected virtual async Task Build(ISpecProvider? specProvider = EthereumEcdsa = new EthereumEcdsa(SpecProvider.ChainId, LogManager); DbProvider = await CreateDbProvider(); TrieStore = new TrieStore(StateDb, LogManager); - State = new WorldState(TrieStore, DbProvider.CodeDb, LogManager); + State = new WorldState(TrieStore, DbProvider.CodeDb, LogManager, new PreBlockCaches()); // Eip4788 precompile state account if (specProvider?.GenesisSpec?.IsBeaconBlockRootAvailable ?? false) @@ -161,10 +166,17 @@ protected virtual async Task Build(ISpecProvider? specProvider = WorldStateManager = new WorldStateManager(State, TrieStore, DbProvider, LimboLogs.Instance); StateReader = new StateReader(ReadOnlyTrieStore, CodeDb, LogManager); - BlockTree = Builders.Build.A.BlockTree() - .WithSpecProvider(SpecProvider) - .WithoutSettingHead - .TestObject; + ChainLevelInfoRepository = new ChainLevelInfoRepository(this.DbProvider.BlockInfosDb); + BlockTree = new BlockTree(new BlockStore(DbProvider.BlocksDb), + new HeaderStore(DbProvider.HeadersDb, DbProvider.BlockNumbersDb), + DbProvider.BlockInfosDb, + DbProvider.MetadataDb, + new BlockStore(new TestMemDb(), 100), + ChainLevelInfoRepository, + SpecProvider, + NullBloomStorage.Instance, + new SyncConfig(), + LimboLogs.Instance); ReadOnlyState = new ChainHeadReadOnlyStateProvider(BlockTree, StateReader); TransactionComparerProvider = new TransactionComparerProvider(SpecProvider, BlockTree); @@ -176,10 +188,11 @@ protected virtual async Task Build(ISpecProvider? specProvider = NonceManager = new NonceManager(chainHeadInfoProvider.AccountStateProvider); _trieStoreWatcher = new TrieStoreBoundaryWatcher(WorldStateManager, BlockTree, LogManager); - + CodeInfoRepository codeInfoRepository = new(); ReceiptStorage = new InMemoryReceiptStorage(blockTree: BlockTree); - VirtualMachine virtualMachine = new(new BlockhashProvider(BlockTree, SpecProvider, State, LogManager), SpecProvider, LogManager); - TxProcessor = new TransactionProcessor(SpecProvider, State, virtualMachine, LogManager); + VirtualMachine virtualMachine = new(new BlockhashProvider(BlockTree, SpecProvider, State, LogManager), SpecProvider, codeInfoRepository, LogManager); + TxProcessor = new TransactionProcessor(SpecProvider, State, virtualMachine, codeInfoRepository, LogManager); + BlockPreprocessorStep = new RecoverSignatures(EthereumEcdsa, TxPool, SpecProvider, LogManager); HeaderValidator = new HeaderValidator(BlockTree, Always.Valid, SpecProvider, LogManager); @@ -211,13 +224,14 @@ protected virtual async Task Build(ISpecProvider? specProvider = TxPoolTxSource txPoolTxSource = CreateTxPoolTxSource(); ITransactionComparerProvider transactionComparerProvider = new TransactionComparerProvider(SpecProvider, BlockFinder); BlockProducer = CreateTestBlockProducer(txPoolTxSource, sealer, transactionComparerProvider); - await BlockProducer.Start(); - Suggester = new ProducedBlockSuggester(BlockTree, BlockProducer); + BlockProducerRunner ??= CreateBlockProducerRunner(); + BlockProducerRunner.Start(); + Suggester = new ProducedBlockSuggester(BlockTree, BlockProducerRunner); _resetEvent = new SemaphoreSlim(0); _suggestedBlockResetEvent = new ManualResetEvent(true); BlockTree.BlockAddedToMain += BlockAddedToMain; - BlockProducer.BlockProduced += (s, e) => + BlockProducerRunner.BlockProduced += (s, e) => { _suggestedBlockResetEvent.Set(); }; @@ -287,13 +301,17 @@ protected virtual IBlockProducer CreateTestBlockProducer(TxPoolTxSource txPoolTx env.ReadOnlyStateProvider, sealer, BlockTree, - BlockProductionTrigger, Timestamper, SpecProvider, LogManager, blocksConfig); } + protected virtual IBlockProducerRunner CreateBlockProducerRunner() + { + return new StandardBlockProducerRunner(BlockProductionTrigger, BlockTree, BlockProducer); + } + public virtual ILogManager LogManager { get; set; } = LimboLogs.Instance; protected virtual TxPool.TxPool CreateTxPool() => @@ -369,7 +387,6 @@ protected virtual IBlockProcessor CreateBlockProcessor() => new BlockProcessor.BlockValidationTransactionsExecutor(TxProcessor, State), State, ReceiptStorage, - NullWitnessCollector.Instance, new BlockhashStore(BlockTree, SpecProvider, State), TxProcessor, LogManager, @@ -429,7 +446,7 @@ public void AddTransactions(params Transaction[] txs) public virtual void Dispose() { - BlockProducer?.StopAsync(); + BlockProducerRunner?.StopAsync(); if (DbProvider is not null) { CodeDb?.Dispose(); diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/BlockBuilder.cs b/src/Nethermind/Nethermind.Core.Test/Builders/BlockBuilder.cs index c8596e8021e..a43039ef8af 100644 --- a/src/Nethermind/Nethermind.Core.Test/Builders/BlockBuilder.cs +++ b/src/Nethermind/Nethermind.Core.Test/Builders/BlockBuilder.cs @@ -97,7 +97,7 @@ public BlockBuilder WithTransactions(int txCount, ISpecProvider specProvider) } BlockBuilder result = WithTransactions(txs); - Hash256 receiptHash = ReceiptTrie.CalculateRoot(specProvider.GetSpec(TestObjectInternal.Header), receipts, ReceiptMessageDecoder.Instance); + Hash256 receiptHash = ReceiptTrie.CalculateRoot(specProvider.GetSpec(TestObjectInternal.Header), receipts, Rlp.GetStreamDecoder()!); TestObjectInternal.Header.ReceiptsRoot = receiptHash; return result; } diff --git a/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeBuilder.cs b/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeBuilder.cs index 39cbca5aa2b..1f509b8e621 100644 --- a/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeBuilder.cs +++ b/src/Nethermind/Nethermind.Core.Test/Builders/BlockTreeBuilder.cs @@ -299,8 +299,7 @@ private Block CreateBlock(int splitVariant, int splitFrom, int blockIndex, Block currentBlock.Header.TxRoot = TxTrie.CalculateRoot(currentBlock.Transactions); TxReceipt[] txReceipts = receipts.ToArray(); currentBlock.Header.ReceiptsRoot = - ReceiptTrie.CalculateRoot(_specProvider.GetSpec(currentBlock.Header), txReceipts, - ReceiptMessageDecoder.Instance); + ReceiptTrie.CalculateRoot(_specProvider.GetSpec(currentBlock.Header), txReceipts, Rlp.GetStreamDecoder()!); currentBlock.Header.Hash = currentBlock.CalculateHash(); foreach (TxReceipt txReceipt in txReceipts) { diff --git a/src/Nethermind/Nethermind.Core.Test/Caching/LruCacheLowObjectTests.cs b/src/Nethermind/Nethermind.Core.Test/Caching/LruCacheLowObjectTests.cs new file mode 100644 index 00000000000..de3864b84f8 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/Caching/LruCacheLowObjectTests.cs @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Caching; +using Nethermind.Core.Test.Builders; +using Nethermind.Int256; +using NUnit.Framework; + +using Cache = Nethermind.Core.Caching.LruCacheLowObject; + +namespace Nethermind.Core.Test.Caching +{ + public class LruCacheLowObjectTests + { + private static Cache Create() + { + return new Cache(Capacity, "test")!; + } + + private const int Capacity = 16; + + private readonly Account[] _accounts = new Account[Capacity * 2]; + private readonly Address[] _addresses = new Address[Capacity * 2]; + + [SetUp] + public void Setup() + { + for (int i = 0; i < Capacity * 2; i++) + { + _accounts[i] = Build.An.Account.WithBalance((UInt256)i).TestObject; + _addresses[i] = Build.An.Address.FromNumber(i).TestObject; + } + } + + [Test] + public void At_capacity() + { + Cache cache = Create(); + for (int i = 0; i < Capacity; i++) + { + cache.Set(_addresses[i], _accounts[i]).Should().BeTrue(); + } + + Account? account = cache.Get(_addresses[Capacity - 1]); + Assert.That(account, Is.EqualTo(_accounts[Capacity - 1])); + } + + [Test] + public void Can_reset() + { + Cache cache = Create(); + cache.Set(_addresses[0], _accounts[0]).Should().BeTrue(); + cache.Set(_addresses[0], _accounts[1]).Should().BeFalse(); + cache.Get(_addresses[0]).Should().Be(_accounts[1]); + } + + [Test] + public void Can_ask_before_first_set() + { + Cache cache = Create(); + cache.Get(_addresses[0]).Should().BeNull(); + } + + [Test] + public void Can_clear() + { + Cache cache = Create(); + cache.Set(_addresses[0], _accounts[0]).Should().BeTrue(); + cache.Clear(); + cache.Get(_addresses[0]).Should().BeNull(); + cache.Set(_addresses[0], _accounts[1]).Should().BeTrue(); + cache.Get(_addresses[0]).Should().Be(_accounts[1]); + } + + [Test] + public void Beyond_capacity() + { + Cache cache = Create(); + for (int i = 0; i < Capacity * 2; i++) + { + cache.Set(_addresses[i], _accounts[i]).Should().BeTrue(); + } + + Account? account = cache.Get(_addresses[Capacity]); + account.Should().Be(_accounts[Capacity]); + } + + [Test] + public void Beyond_capacity_lru() + { + Cache cache = Create(); + for (int i = 0; i < Capacity * 2; i++) + { + for (int ii = 0; ii < Capacity / 2; ii++) + { + cache.Set(_addresses[i], _accounts[i]); + } + cache.Set(_addresses[i], _accounts[i]); + } + } + + [Test] + public void Can_set_and_then_set_null() + { + Cache cache = Create(); + cache.Set(_addresses[0], _accounts[0]).Should().BeTrue(); + cache.Set(_addresses[0], _accounts[0]).Should().BeFalse(); + cache.Set(_addresses[0], null!).Should().BeTrue(); + cache.Get(_addresses[0]).Should().Be(null); + } + + [Test] + public void Can_delete() + { + Cache cache = Create(); + cache.Set(_addresses[0], _accounts[0]); + cache.Delete(_addresses[0]).Should().BeTrue(); + cache.Get(_addresses[0]).Should().Be(null); + cache.Delete(_addresses[0]).Should().BeFalse(); + } + + [Test] + public void Clear_should_free_all_capacity() + { + Cache cache = Create(); + for (int i = 0; i < Capacity; i++) + { + cache.Set(_addresses[i], _accounts[i]).Should().BeTrue(); + } + + cache.Clear(); + + static int MapForRefill(int index) => (index + 1) % Capacity; + + // fill again + for (int i = 0; i < Capacity; i++) + { + cache.Set(_addresses[i], _accounts[MapForRefill(i)]).Should().BeTrue(); + } + + // validate + for (int i = 0; i < Capacity; i++) + { + cache.Get(_addresses[i]).Should().Be(_accounts[MapForRefill(i)]); + } + } + + [Test] + public void Delete_keeps_internal_structure() + { + int maxCapacity = 32; + int itemsToKeep = 10; + int iterations = 40; + + LruCacheLowObject cache = new(maxCapacity, "test"); + + for (int i = 0; i < iterations; i++) + { + cache.Set(i, i).Should().BeTrue(); + var remove = i - itemsToKeep; + if (remove >= 0) + cache.Delete(remove).Should().BeTrue(); + } + + int count = 0; + + for (int i = 0; i < iterations; i++) + { + if (cache.TryGet(i, out int val)) + { + count++; + val.Should().Be(i); + } + } + + count.Should().Be(itemsToKeep); + } + } +} diff --git a/src/Nethermind/Nethermind.Core.Test/Caching/LruCacheTests.cs b/src/Nethermind/Nethermind.Core.Test/Caching/LruCacheTests.cs index 35399b9e56d..95e37aa7c91 100755 --- a/src/Nethermind/Nethermind.Core.Test/Caching/LruCacheTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/Caching/LruCacheTests.cs @@ -39,7 +39,7 @@ public void At_capacity() ICache cache = Create(); for (int i = 0; i < Capacity; i++) { - cache.Set(_addresses[i], _accounts[i]); + cache.Set(_addresses[i], _accounts[i]).Should().BeTrue(); } Account? account = cache.Get(_addresses[Capacity - 1]); @@ -50,8 +50,8 @@ public void At_capacity() public void Can_reset() { ICache cache = Create(); - cache.Set(_addresses[0], _accounts[0]); - cache.Set(_addresses[0], _accounts[1]); + cache.Set(_addresses[0], _accounts[0]).Should().BeTrue(); + cache.Set(_addresses[0], _accounts[1]).Should().BeFalse(); cache.Get(_addresses[0]).Should().Be(_accounts[1]); } @@ -66,21 +66,35 @@ public void Can_ask_before_first_set() public void Can_clear() { ICache cache = Create(); - cache.Set(_addresses[0], _accounts[0]); + cache.Set(_addresses[0], _accounts[0]).Should().BeTrue(); cache.Clear(); cache.Get(_addresses[0]).Should().BeNull(); - cache.Set(_addresses[0], _accounts[1]); + cache.Set(_addresses[0], _accounts[1]).Should().BeTrue(); cache.Get(_addresses[0]).Should().Be(_accounts[1]); } [Test] - public void Beyond_capacity() + public void Beyond_capacity_lru() { ICache cache = Create(); for (int i = 0; i < Capacity * 2; i++) { + for (int ii = 0; ii < Capacity / 2; ii++) + { + cache.Set(_addresses[i], _accounts[i]); + } cache.Set(_addresses[i], _accounts[i]); } + } + + [Test] + public void Beyond_capacity() + { + ICache cache = Create(); + for (int i = 0; i < Capacity * 2; i++) + { + cache.Set(_addresses[i], _accounts[i]).Should().BeTrue(); + } Account? account = cache.Get(_addresses[Capacity]); account.Should().Be(_accounts[Capacity]); diff --git a/src/Nethermind/Nethermind.Core.Test/Caching/LruKeyCacheLowObjectTests.cs b/src/Nethermind/Nethermind.Core.Test/Caching/LruKeyCacheLowObjectTests.cs new file mode 100644 index 00000000000..cb8b07dc324 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/Caching/LruKeyCacheLowObjectTests.cs @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Core.Caching; +using Nethermind.Core.Test.Builders; +using Nethermind.Int256; +using NUnit.Framework; + +namespace Nethermind.Core.Test.Caching +{ + [TestFixture] + public class LruKeyCacheLowObjectTests + { + private const int Capacity = 16; + + private readonly Account[] _accounts = new Account[Capacity * 2]; + private readonly Address[] _addresses = new Address[Capacity * 2]; + + [SetUp] + public void Setup() + { + for (int i = 0; i < Capacity * 2; i++) + { + _accounts[i] = Build.An.Account.WithBalance((UInt256)i).TestObject; + _addresses[i] = Build.An.Address.FromNumber(i).TestObject; + } + } + + [Test] + public void At_capacity() + { + LruKeyCacheLowObject cache = new(Capacity, "test"); + for (int i = 0; i < Capacity; i++) + { + cache.Set(_addresses[i]); + } + + cache.Get(_addresses[Capacity - 1]).Should().BeTrue(); + } + + [Test] + public void Can_reset() + { + LruKeyCacheLowObject cache = new(Capacity, "test"); + cache.Set(_addresses[0]).Should().BeTrue(); + cache.Set(_addresses[0]).Should().BeFalse(); + cache.Get(_addresses[0]).Should().BeTrue(); + } + + [Test] + public void Can_ask_before_first_set() + { + LruKeyCacheLowObject cache = new(Capacity, "test"); + cache.Get(_addresses[0]).Should().BeFalse(); + } + + [Test] + public void Can_clear() + { + LruKeyCacheLowObject cache = new(Capacity, "test"); + cache.Set(_addresses[0]).Should().BeTrue(); + cache.Clear(); + cache.Get(_addresses[0]).Should().BeFalse(); + cache.Set(_addresses[0]).Should().BeTrue(); + cache.Get(_addresses[0]).Should().BeTrue(); + } + + [Test] + public void Beyond_capacity() + { + LruKeyCacheLowObject cache = new(Capacity, "test"); + for (int i = 0; i < Capacity * 2; i++) + { + cache.Set(_addresses[i]); + } + + cache.Get(_addresses[Capacity]).Should().BeTrue(); + } + + [Test] + public void Beyond_capacity_lru() + { + LruKeyCacheLowObject cache = new(Capacity, "test"); + for (int i = 0; i < Capacity * 2; i++) + { + for (int ii = 0; ii < Capacity / 2; ii++) + { + cache.Set(_addresses[i]); + } + cache.Set(_addresses[i]); + } + } + + [Test] + public void Can_delete() + { + LruKeyCacheLowObject cache = new(Capacity, "test"); + cache.Set(_addresses[0]); + cache.Delete(_addresses[0]); + cache.Get(_addresses[0]).Should().BeFalse(); + } + } +} diff --git a/src/Nethermind/Nethermind.Core.Test/Caching/LruKeyCacheNonConcurrentTests.cs b/src/Nethermind/Nethermind.Core.Test/Caching/LruKeyCacheNonConcurrentTests.cs new file mode 100644 index 00000000000..614ba3cfa4e --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/Caching/LruKeyCacheNonConcurrentTests.cs @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Core.Caching; +using Nethermind.Core.Test.Builders; +using Nethermind.Int256; +using NUnit.Framework; + +namespace Nethermind.Core.Test.Caching +{ + [TestFixture] + public class LruKeyCacheNonConcurrentTests + { + private const int Capacity = 16; + + private readonly Account[] _accounts = new Account[Capacity * 2]; + private readonly Address[] _addresses = new Address[Capacity * 2]; + + [SetUp] + public void Setup() + { + for (int i = 0; i < Capacity * 2; i++) + { + _accounts[i] = Build.An.Account.WithBalance((UInt256)i).TestObject; + _addresses[i] = Build.An.Address.FromNumber(i).TestObject; + } + } + + [Test] + public void At_capacity() + { + LruKeyCacheNonConcurrent
cache = new(Capacity, "test"); + for (int i = 0; i < Capacity; i++) + { + cache.Set(_addresses[i]).Should().BeTrue(); + } + + cache.Get(_addresses[Capacity - 1]).Should().BeTrue(); + } + + [Test] + public void Can_reset() + { + LruKeyCacheNonConcurrent
cache = new(Capacity, "test"); + cache.Set(_addresses[0]).Should().BeTrue(); + cache.Set(_addresses[0]).Should().BeFalse(); + cache.Get(_addresses[0]).Should().BeTrue(); + } + + [Test] + public void Can_ask_before_first_set() + { + LruKeyCacheNonConcurrent
cache = new(Capacity, "test"); + cache.Get(_addresses[0]).Should().BeFalse(); + } + + [Test] + public void Can_clear() + { + LruKeyCacheNonConcurrent
cache = new(Capacity, "test"); + cache.Set(_addresses[0]).Should().BeTrue(); + cache.Clear(); + cache.Get(_addresses[0]).Should().BeFalse(); + cache.Set(_addresses[0]).Should().BeTrue(); + cache.Get(_addresses[0]).Should().BeTrue(); + } + + [Test] + public void Beyond_capacity() + { + LruKeyCacheNonConcurrent
cache = new(Capacity, "test"); + for (int i = 0; i < Capacity * 2; i++) + { + cache.Set(_addresses[i]); + } + + cache.Get(_addresses[Capacity]).Should().BeTrue(); + } + + [Test] + public void Beyond_capacity_lru() + { + LruKeyCacheNonConcurrent cache = new(Capacity, "test"); + for (int i = 0; i < Capacity * 2; i++) + { + for (int ii = 0; ii < Capacity / 2; ii++) + { + cache.Set(_addresses[i]); + } + cache.Set(_addresses[i]); + } + } + + [Test] + public void Can_delete() + { + LruKeyCacheNonConcurrent
cache = new(Capacity, "test"); + cache.Set(_addresses[0]).Should().BeTrue(); + cache.Delete(_addresses[0]); + cache.Get(_addresses[0]).Should().BeFalse(); + } + } +} diff --git a/src/Nethermind/Nethermind.Core.Test/Caching/LruKeyCacheTests.cs b/src/Nethermind/Nethermind.Core.Test/Caching/LruKeyCacheTests.cs index 05185bfa433..5cf04410450 100644 --- a/src/Nethermind/Nethermind.Core.Test/Caching/LruKeyCacheTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/Caching/LruKeyCacheTests.cs @@ -33,7 +33,7 @@ public void At_capacity() LruKeyCache
cache = new(Capacity, "test"); for (int i = 0; i < Capacity; i++) { - cache.Set(_addresses[i]); + cache.Set(_addresses[i]).Should().BeTrue(); } cache.Get(_addresses[Capacity - 1]).Should().BeTrue(); @@ -72,17 +72,31 @@ public void Beyond_capacity() LruKeyCache
cache = new(Capacity, "test"); for (int i = 0; i < Capacity * 2; i++) { - cache.Set(_addresses[i]); + cache.Set(_addresses[i]).Should().BeTrue(); } cache.Get(_addresses[Capacity]).Should().BeTrue(); } + [Test] + public void Beyond_capacity_lru() + { + LruKeyCache cache = new(Capacity, "test"); + for (int i = 0; i < Capacity * 2; i++) + { + for (int ii = 0; ii < Capacity / 2; ii++) + { + cache.Set(_addresses[i]); + } + cache.Set(_addresses[i]); + } + } + [Test] public void Can_delete() { LruKeyCache
cache = new(Capacity, "test"); - cache.Set(_addresses[0]); + cache.Set(_addresses[0]).Should().BeTrue(); cache.Delete(_addresses[0]); cache.Get(_addresses[0]).Should().BeFalse(); } diff --git a/src/Nethermind/Nethermind.Core.Test/Caching/SpanLruCacheTests.cs b/src/Nethermind/Nethermind.Core.Test/Caching/SpanLruCacheTests.cs index f30d930bf53..5cf255515f4 100755 --- a/src/Nethermind/Nethermind.Core.Test/Caching/SpanLruCacheTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/Caching/SpanLruCacheTests.cs @@ -40,7 +40,7 @@ public void At_capacity() ISpanCache cache = Create(); for (int i = 0; i < Capacity; i++) { - cache.Set(_addresses[i].Bytes, _accounts[i]); + cache.Set(_addresses[i].Bytes, _accounts[i]).Should().BeTrue(); } Account? account = cache.Get(_addresses[Capacity - 1].Bytes); @@ -51,8 +51,8 @@ public void At_capacity() public void Can_reset() { ISpanCache cache = Create(); - cache.Set(_addresses[0].Bytes, _accounts[0]); - cache.Set(_addresses[0].Bytes, _accounts[1]); + cache.Set(_addresses[0].Bytes, _accounts[0]).Should().BeTrue(); + cache.Set(_addresses[0].Bytes, _accounts[1]).Should().BeFalse(); cache.Get(_addresses[0].Bytes).Should().Be(_accounts[1]); } @@ -67,10 +67,10 @@ public void Can_ask_before_first_set() public void Can_clear() { ISpanCache cache = Create(); - cache.Set(_addresses[0].Bytes, _accounts[0]); + cache.Set(_addresses[0].Bytes, _accounts[0]).Should().BeTrue(); cache.Clear(); cache.Get(_addresses[0].Bytes).Should().BeNull(); - cache.Set(_addresses[0].Bytes, _accounts[1]); + cache.Set(_addresses[0].Bytes, _accounts[1]).Should().BeTrue(); cache.Get(_addresses[0].Bytes).Should().Be(_accounts[1]); } @@ -80,13 +80,27 @@ public void Beyond_capacity() ISpanCache cache = Create(); for (int i = 0; i < Capacity * 2; i++) { - cache.Set(_addresses[i].Bytes, _accounts[i]); + cache.Set(_addresses[i].Bytes, _accounts[i]).Should().BeTrue(); } Account? account = cache.Get(_addresses[Capacity].Bytes); account.Should().Be(_accounts[Capacity]); } + [Test] + public void Beyond_capacity_lru() + { + ISpanCache cache = Create(); + for (int i = 0; i < Capacity * 2; i++) + { + for (int ii = 0; ii < Capacity / 2; ii++) + { + cache.Set(_addresses[i].Bytes, _accounts[i]); + } + cache.Set(_addresses[i].Bytes, _accounts[i]); + } + } + [Test] public void Can_set_and_then_set_null() { @@ -101,7 +115,7 @@ public void Can_set_and_then_set_null() public void Can_delete() { ISpanCache cache = Create(); - cache.Set(_addresses[0].Bytes, _accounts[0]); + cache.Set(_addresses[0].Bytes, _accounts[0]).Should().BeTrue(); cache.Delete(_addresses[0].Bytes).Should().BeTrue(); cache.Get(_addresses[0].Bytes).Should().Be(null); cache.Delete(_addresses[0].Bytes).Should().BeFalse(); diff --git a/src/Nethermind/Nethermind.Core.Test/Json/UInt256DictionaryKeyConverterTests.cs b/src/Nethermind/Nethermind.Core.Test/Json/UInt256DictionaryKeyConverterTests.cs index 533e777f8af..2f37bd6b572 100644 --- a/src/Nethermind/Nethermind.Core.Test/Json/UInt256DictionaryKeyConverterTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/Json/UInt256DictionaryKeyConverterTests.cs @@ -115,7 +115,7 @@ public void WriteJson_Dictionary_SerializedCorrectly() { new UInt256(1), new UInt256(12345) } }; string serialised = JsonSerializer.Serialize(dictionary, Options); - + serialised = serialised.Replace("\r\n", "\n"); string expected = """ { "0x1": "0x3039" diff --git a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs index a02f5663e31..d0bfa06581a 100644 --- a/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/KeccakTests.cs @@ -110,5 +110,13 @@ public void Span() Assert.That(Keccak.Compute(byteArray.AsSpan()), Is.EqualTo(Keccak.Compute(byteArray))); } + + [TestCase("0xAAAAAAAAAAAA", "f8e06bc47a06f221f1523d3b646226e6cdb322619be1da7bafd113e459bf4140")] + public void Sanity_check(string hexString, string expected) + { + byte[] bytes = Bytes.FromHexString(hexString); + ValueHash256 h = ValueKeccak.Compute(bytes); + h.Bytes.ToHexString().Should().Be(expected); + } } } diff --git a/src/Nethermind/Nethermind.Core.Test/SequenceTests.cs b/src/Nethermind/Nethermind.Core.Test/SequenceTests.cs new file mode 100644 index 00000000000..d44ba71dda4 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/SequenceTests.cs @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Buffers; +using FluentAssertions; +using Nethermind.Core.Extensions; +using NUnit.Framework; + +namespace Nethermind.Core.Test; + +public class SequenceTests +{ + [Test] + public void Sequence_whitespace_slice() + { + var end = new TestReadOnlySequenceSegment("\r\nabc"u8.ToArray(), 2); + var start = new TestReadOnlySequenceSegment(" "u8.ToArray(), 0, end); + ReadOnlySequence sequence = new ReadOnlySequence(start, 0, end, 5); + + sequence.TrimStart().ToArray().Should().Equal("abc"u8.ToArray()); + } + + private class TestReadOnlySequenceSegment : ReadOnlySequenceSegment + { + public TestReadOnlySequenceSegment(Memory memory, long runningIndex = 0, ReadOnlySequenceSegment? next = null) + { + Memory = memory; + Next = next; + RunningIndex = runningIndex; + } + } +} diff --git a/src/Nethermind/Nethermind.Core/Account.cs b/src/Nethermind/Nethermind.Core/Account.cs index 41279ec2b7b..dbe8d959740 100644 --- a/src/Nethermind/Nethermind.Core/Account.cs +++ b/src/Nethermind/Nethermind.Core/Account.cs @@ -154,8 +154,8 @@ public bool IsNull // // Method Nethermind.Core.AccountStruct:get_IsNull():bool:this (FullOpts) // G_M000_IG01: - // vzeroupper - // + // vzeroupper + // // G_M000_IG02: // vmovups ymm0, ymmword ptr [rcx] // vpor ymm0, ymm0, ymmword ptr [rcx+0x20] @@ -164,10 +164,10 @@ public bool IsNull // vptest ymm0, ymm0 // sete al // movzx rax, al - // + // // G_M000_IG03: ;; offset=0x0021 - // vzeroupper - // ret + // vzeroupper + // ret // ; Total bytes of code: 37 return (Unsafe.As>(ref Unsafe.AsRef(in _balance)) | diff --git a/src/Nethermind/Nethermind.Core/Address.cs b/src/Nethermind/Nethermind.Core/Address.cs index d56cda23269..79acbe7d5ff 100644 --- a/src/Nethermind/Nethermind.Core/Address.cs +++ b/src/Nethermind/Nethermind.Core/Address.cs @@ -35,9 +35,9 @@ public class Address : IEquatable
, IComparable
public byte[] Bytes { get; } - public Address(Hash256 keccak) : this(keccak.Bytes.Slice(12, Size).ToArray()) { } + public Address(Hash256 hash) : this(hash.Bytes.Slice(12, Size).ToArray()) { } - public Address(in ValueHash256 keccak) : this(keccak.BytesAsSpan.Slice(12, Size).ToArray()) { } + public Address(in ValueHash256 hash) : this(hash.BytesAsSpan.Slice(12, Size).ToArray()) { } public byte this[int index] => Bytes[index]; @@ -235,6 +235,14 @@ public override bool CanConvertTo(ITypeDescriptorContext? context, Type? destina } public Hash256 ToAccountPath => Keccak.Compute(Bytes); + + [SkipLocalsInit] + public ValueHash256 ToHash() + { + Span addressBytes = stackalloc byte[Hash256.Size]; + Bytes.CopyTo(addressBytes.Slice(Hash256.Size - Address.Size)); + return new ValueHash256(addressBytes); + } } public readonly struct AddressAsKey(Address key) : IEquatable diff --git a/src/Nethermind/Nethermind.Core/Block.cs b/src/Nethermind/Nethermind.Core/Block.cs index c77e9c27c5c..609455b96d3 100644 --- a/src/Nethermind/Nethermind.Core/Block.cs +++ b/src/Nethermind/Nethermind.Core/Block.cs @@ -7,6 +7,8 @@ using System.Linq; using System.Text; using Nethermind.Core.ConsensusRequests; +using System.Text.Json.Serialization; +using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Int256; @@ -117,6 +119,10 @@ public Transaction[] Transactions public Hash256? RequestsRoot => Header.RequestsRoot; // do not add setter here + [JsonIgnore] + public ArrayPoolList? AccountChanges { get; set; } + [JsonIgnore] + internal volatile int TransactionProcessed; public override string ToString() => ToString(Format.Short); diff --git a/src/Nethermind/Nethermind.Core/Bloom.cs b/src/Nethermind/Nethermind.Core/Bloom.cs index 7437a25286c..f980e1b4a0e 100644 --- a/src/Nethermind/Nethermind.Core/Bloom.cs +++ b/src/Nethermind/Nethermind.Core/Bloom.cs @@ -126,9 +126,10 @@ public void Add(LogEntry[] logEntries, Bloom? blockBloom = null) LogEntry logEntry = logEntries[entryIndex]; byte[] addressBytes = logEntry.LoggersAddress.Bytes; Set(addressBytes, blockBloom); - for (int topicIndex = 0; topicIndex < logEntry.Topics.Length; topicIndex++) + Hash256[] topics = logEntry.Topics; + for (int topicIndex = 0; topicIndex < topics.Length; topicIndex++) { - Hash256 topic = logEntry.Topics[topicIndex]; + Hash256 topic = topics[topicIndex]; Set(topic.Bytes, blockBloom); } } @@ -148,9 +149,10 @@ public bool Matches(LogEntry logEntry) { if (Matches(logEntry.LoggersAddress)) { - for (int topicIndex = 0; topicIndex < logEntry.Topics.Length; topicIndex++) + Hash256[] topics = logEntry.Topics; + for (int topicIndex = 0; topicIndex < topics.Length; topicIndex++) { - if (!Matches(logEntry.Topics[topicIndex])) + if (!Matches(topics[topicIndex])) { return false; } @@ -285,9 +287,10 @@ public readonly bool Matches(LogEntry logEntry) { if (Matches(logEntry.LoggersAddress)) { - for (int topicIndex = 0; topicIndex < logEntry.Topics.Length; topicIndex++) + Hash256[] topics = logEntry.Topics; + for (int topicIndex = 0; topicIndex < topics.Length; topicIndex++) { - if (!Matches(logEntry.Topics[topicIndex])) + if (!Matches(topics[topicIndex])) { return false; } diff --git a/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs b/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs index 0d909d05077..9dfa54cf9a0 100644 --- a/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs +++ b/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs @@ -110,4 +110,14 @@ public readonly Span AsSpan(int start, int length) if (_length == array.Length) return array; return AsSpan().ToArray(); } + + public readonly ArraySegment AsArraySegment() + { + return AsArraySegment(0, _length); + } + + public readonly ArraySegment AsArraySegment(int start, int length) + { + return new ArraySegment(_array!, start, length); + } } diff --git a/src/Nethermind/Nethermind.Core/Caching/LinkedListNode.cs b/src/Nethermind/Nethermind.Core/Caching/LinkedListNode.cs index 93011a77bb6..7a882123e39 100644 --- a/src/Nethermind/Nethermind.Core/Caching/LinkedListNode.cs +++ b/src/Nethermind/Nethermind.Core/Caching/LinkedListNode.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; @@ -21,7 +22,10 @@ public static void MoveToMostRecent([NotNull] ref LinkedListNode? leastRecent { if (node.Next == node) { - Debug.Assert(leastRecentlyUsed == node, "this should only be true for a list with only one node"); + if (leastRecentlyUsed != node) + { + InvalidNotSingleNodeList(); + } // Do nothing only one node } else @@ -33,10 +37,16 @@ public static void MoveToMostRecent([NotNull] ref LinkedListNode? leastRecent public static void Remove(ref LinkedListNode? leastRecentlyUsed, LinkedListNode node) { - Debug.Assert(leastRecentlyUsed is not null, "This method shouldn't be called on empty list!"); + if (leastRecentlyUsed is null) + { + InvalidRemoveFromEmptyList(); + } if (node.Next == node) { - Debug.Assert(leastRecentlyUsed == node, "this should only be true for a list with only one node"); + if (leastRecentlyUsed != node) + { + InvalidNotSingleNodeList(); + } leastRecentlyUsed = null; } else @@ -48,6 +58,20 @@ public static void Remove(ref LinkedListNode? leastRecentlyUsed, LinkedListNo leastRecentlyUsed = node.Next; } } + + [DoesNotReturn] + [StackTraceHidden] + static void InvalidRemoveFromEmptyList() + { + throw new InvalidOperationException("This method shouldn't be called on empty list"); + } + } + + [DoesNotReturn] + [StackTraceHidden] + static void InvalidNotSingleNodeList() + { + throw new InvalidOperationException("This should only be true for a list with only one node"); } public static void AddMostRecent([NotNull] ref LinkedListNode? leastRecentlyUsed, LinkedListNode node) diff --git a/src/Nethermind/Nethermind.Core/Caching/LruCacheLowObject.cs b/src/Nethermind/Nethermind.Core/Caching/LruCacheLowObject.cs new file mode 100644 index 00000000000..bbdceab425f --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Caching/LruCacheLowObject.cs @@ -0,0 +1,274 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Nethermind.Core.Threading; + +namespace Nethermind.Core.Caching +{ + public sealed class LruCacheLowObject + where TKey : struct, IEquatable + { + private readonly int _maxCapacity; + private readonly Dictionary _cacheMap; + private readonly McsLock _lock = new(); + private readonly string _name; + private int _leastRecentlyUsed = -1; + private Stack _freeOffsets = new(); + private readonly LruCacheItem[] _items; + + public LruCacheLowObject(int maxCapacity, string name) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxCapacity, 1); + + _name = name; + _maxCapacity = maxCapacity; + _cacheMap = new Dictionary(maxCapacity / 2); + _items = new LruCacheItem[maxCapacity]; + } + + public void Clear() + { + using var lockRelease = _lock.Acquire(); + + _leastRecentlyUsed = -1; + _cacheMap.Clear(); + _freeOffsets.Clear(); + if (RuntimeHelpers.IsReferenceOrContainsReferences()) + { + _items.AsSpan().Clear(); + } + } + + public TValue Get(TKey key) + { + using var lockRelease = _lock.Acquire(); + + if (_cacheMap.TryGetValue(key, out int offset)) + { + ref var node = ref _items[offset]; + TValue value = node.Value; + MoveToMostRecent(ref node, offset); + return value; + } + + return default!; + } + + public bool TryGet(TKey key, out TValue value) + { + using var lockRelease = _lock.Acquire(); + + if (_cacheMap.TryGetValue(key, out int offset)) + { + ref var node = ref _items[offset]; + value = node.Value; + MoveToMostRecent(ref node, offset); + return true; + } + + value = default!; + return false; + } + + public bool Set(TKey key, TValue val) + { + using var lockRelease = _lock.Acquire(); + + if (val is null) + { + return DeleteNoLock(key); + } + + if (_cacheMap.TryGetValue(key, out int offset)) + { + ref var node = ref _items[offset]; + node.Value = val; + MoveToMostRecent(ref node, offset); + return false; + } + + if (_cacheMap.Count >= _maxCapacity) + { + Replace(key, val); + } + else + { + if (_freeOffsets.Count > 0) + { + offset = _freeOffsets.Pop(); + } + else + { + offset = _cacheMap.Count; + } + ref LruCacheItem newNode = ref _items[offset]; + newNode = new(key, val); + AddMostRecent(ref newNode, offset); + _cacheMap.Add(key, offset); + } + + return true; + } + + public bool Delete(TKey key) + { + using var lockRelease = _lock.Acquire(); + + return DeleteNoLock(key); + } + + private bool DeleteNoLock(TKey key) + { + if (_cacheMap.TryGetValue(key, out int offset)) + { + ref var node = ref _items[offset]; + Remove(ref node, offset); + _cacheMap.Remove(key); + node = default; + _freeOffsets.Push(offset); + return true; + } + + return false; + } + + public bool Contains(TKey key) + { + using var lockRelease = _lock.Acquire(); + + return _cacheMap.ContainsKey(key); + } + + public int Size + { + get + { + return _cacheMap.Count; + } + } + + private void Replace(TKey key, TValue value) + { + if (_leastRecentlyUsed < 0) + { + ThrowInvalidOperationException(); + } + + int offset = _leastRecentlyUsed; + ref var node = ref _items[offset]; + + _cacheMap.Remove(node.Key); + node = new(key, value) + { + Next = node.Next, + Prev = node.Prev + }; + + MoveToMostRecent(ref node, offset); + _cacheMap.Add(key, offset); + + [DoesNotReturn] + static void ThrowInvalidOperationException() + { + throw new InvalidOperationException( + $"{nameof(LruCache)} called {nameof(Replace)} when empty."); + } + } + + + private void MoveToMostRecent(ref LruCacheItem node, int offset) + { + if (node.Next == offset) + { + if (_leastRecentlyUsed != offset) + { + InvalidNotSingleNodeList(); + } + // Do nothing only one node + } + else + { + Remove(ref node, offset); + AddMostRecent(ref node, offset); + } + } + + private void Remove(ref LruCacheItem node, int offset) + { + if (_leastRecentlyUsed < 0) + { + InvalidRemoveFromEmptyList(); + } + if (node.Next == offset) + { + if (_leastRecentlyUsed != offset) + { + InvalidNotSingleNodeList(); + } + _leastRecentlyUsed = -1; + } + else + { + _items[node.Next].Prev = node.Prev; + _items[node.Prev].Next = node.Next; + if (_leastRecentlyUsed == offset) + { + _leastRecentlyUsed = node.Next; + } + } + + static void InvalidRemoveFromEmptyList() + { + throw new InvalidOperationException("This method shouldn't be called on empty list"); + } + } + + [DoesNotReturn] + [StackTraceHidden] + static void InvalidNotSingleNodeList() + { + throw new InvalidOperationException("This should only be true for a list with only one node"); + } + + private void AddMostRecent(ref LruCacheItem node, int offset) + { + if (_leastRecentlyUsed < 0) + { + SetFirst(ref node, offset); + } + else + { + InsertMostRecent(ref node, offset); + } + } + + private void InsertMostRecent(ref LruCacheItem newNode, int offset) + { + newNode.Next = _leastRecentlyUsed; + ref var node = ref _items[_leastRecentlyUsed]; + newNode.Prev = node.Prev; + _items[node.Prev].Next = offset; + node.Prev = offset; + } + + private void SetFirst(ref LruCacheItem newNode, int offset) + { + newNode.Next = offset; + newNode.Prev = offset; + _leastRecentlyUsed = offset; + } + + private struct LruCacheItem(TKey k, TValue v) + { + public readonly TKey Key = k; + public TValue Value = v; + public int Next; + public int Prev; + } + } +} diff --git a/src/Nethermind/Nethermind.Core/Caching/LruKeyCache.cs b/src/Nethermind/Nethermind.Core/Caching/LruKeyCache.cs index cc58f807a16..d5f25705b0b 100644 --- a/src/Nethermind/Nethermind.Core/Caching/LruKeyCache.cs +++ b/src/Nethermind/Nethermind.Core/Caching/LruKeyCache.cs @@ -109,17 +109,5 @@ static void ThrowInvalidOperation() $"{nameof(LruKeyCache)} called {nameof(Replace)} when empty."); } } - - public long MemorySize => CalculateMemorySize(0, _cacheMap.Count); - - // TODO: memory size on the KeyCache will be smaller because we do not create LruCacheItems - public static long CalculateMemorySize(int keyPlusValueSize, int currentItemsCount) - { - // it may actually be different if the initial capacity not equal to max (depending on the dictionary growth path) - - const int preInit = 48 /* LinkedList */ + 80 /* Dictionary */ + 24; - int postInit = 52 /* lazy init of two internal dictionary arrays + dictionary size times (entry size + int) */ + MemorySizes.FindNextPrime(currentItemsCount) * 28 + currentItemsCount * 80 /* LinkedListNode and CacheItem times items count */; - return MemorySizes.Align(preInit + postInit + keyPlusValueSize * currentItemsCount); - } } } diff --git a/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheLowObject.cs b/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheLowObject.cs new file mode 100644 index 00000000000..dd8e3af89eb --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheLowObject.cs @@ -0,0 +1,248 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Nethermind.Core.Threading; + +namespace Nethermind.Core.Caching +{ + public sealed class LruKeyCacheLowObject + where TKey : struct, IEquatable + { + private readonly int _maxCapacity; + private readonly Dictionary _cacheMap; + private readonly McsLock _lock = new(); + private readonly string _name; + private int _leastRecentlyUsed = -1; + private Stack _freeOffsets = new(); + private readonly LruCacheItem[] _items; + + public LruKeyCacheLowObject(int maxCapacity, string name) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maxCapacity, 1); + + _name = name; + _maxCapacity = maxCapacity; + _cacheMap = new Dictionary(maxCapacity / 2); // do not initialize it at the full capacity + _items = new LruCacheItem[maxCapacity]; + } + + public void Clear() + { + using var lockRelease = _lock.Acquire(); + + _leastRecentlyUsed = -1; + _cacheMap.Clear(); + _freeOffsets.Clear(); + if (RuntimeHelpers.IsReferenceOrContainsReferences()) + { + _items.AsSpan().Clear(); + } + } + + public bool Get(TKey key) + { + using var lockRelease = _lock.Acquire(); + + if (_cacheMap.TryGetValue(key, out int offset)) + { + ref var node = ref _items[offset]; + MoveToMostRecent(ref node, offset); + return true; + } + + return false!; + } + + public bool Set(TKey key) + { + using var lockRelease = _lock.Acquire(); + + if (_cacheMap.TryGetValue(key, out int offset)) + { + ref var node = ref _items[offset]; + MoveToMostRecent(ref node, offset); + return false; + } + + if (_cacheMap.Count >= _maxCapacity) + { + Replace(key); + } + else + { + if (_freeOffsets.Count > 0) + { + offset = _freeOffsets.Pop(); + } + else + { + offset = _cacheMap.Count; + } + ref LruCacheItem newNode = ref _items[offset]; + newNode = new(key); + AddMostRecent(ref newNode, offset); + _cacheMap.Add(key, offset); + } + + return true; + } + + public void Delete(TKey key) + { + using var lockRelease = _lock.Acquire(); + + DeleteNoLock(key); + } + + private bool DeleteNoLock(TKey key) + { + if (_cacheMap.TryGetValue(key, out int offset)) + { + ref var node = ref _items[offset]; + Remove(ref node, offset); + _cacheMap.Remove(key); + node = default; + _freeOffsets.Push(offset); + return true; + } + + return false; + } + + public bool Contains(TKey key) + { + using var lockRelease = _lock.Acquire(); + + return _cacheMap.ContainsKey(key); + } + + public int Size + { + get + { + return _cacheMap.Count; + } + } + + private void Replace(TKey key) + { + if (_leastRecentlyUsed < 0) + { + ThrowInvalidOperationException(); + } + + int offset = _leastRecentlyUsed; + ref var node = ref _items[offset]; + + _cacheMap.Remove(node.Key); + node = new(key) + { + Next = node.Next, + Prev = node.Prev + }; + + MoveToMostRecent(ref node, offset); + _cacheMap.Add(key, offset); + + [DoesNotReturn] + static void ThrowInvalidOperationException() + { + throw new InvalidOperationException($"Called {nameof(Replace)} when empty."); + } + } + + private void MoveToMostRecent(ref LruCacheItem node, int offset) + { + if (node.Next == offset) + { + if (_leastRecentlyUsed != offset) + { + InvalidNotSingleNodeList(); + } + // Do nothing only one node + } + else + { + Remove(ref node, offset); + AddMostRecent(ref node, offset); + } + } + + private void Remove(ref LruCacheItem node, int offset) + { + if (_leastRecentlyUsed < 0) + { + InvalidRemoveFromEmptyList(); + } + if (node.Next == offset) + { + if (_leastRecentlyUsed != offset) + { + InvalidNotSingleNodeList(); + } + _leastRecentlyUsed = -1; + } + else + { + _items[node.Next].Prev = node.Prev; + _items[node.Prev].Next = node.Next; + if (_leastRecentlyUsed == offset) + { + _leastRecentlyUsed = node.Next; + } + } + + static void InvalidRemoveFromEmptyList() + { + throw new InvalidOperationException("This method shouldn't be called on empty list"); + } + } + + [DoesNotReturn] + [StackTraceHidden] + static void InvalidNotSingleNodeList() + { + throw new InvalidOperationException("This should only be true for a list with only one node"); + } + + private void AddMostRecent(ref LruCacheItem node, int offset) + { + if (_leastRecentlyUsed < 0) + { + SetFirst(ref node, offset); + } + else + { + InsertMostRecent(ref node, offset); + } + } + + private void InsertMostRecent(ref LruCacheItem newNode, int offset) + { + newNode.Next = _leastRecentlyUsed; + ref var node = ref _items[_leastRecentlyUsed]; + newNode.Prev = node.Prev; + _items[node.Prev].Next = offset; + node.Prev = offset; + } + + private void SetFirst(ref LruCacheItem newNode, int offset) + { + newNode.Next = offset; + newNode.Prev = offset; + _leastRecentlyUsed = offset; + } + + private struct LruCacheItem(TKey k) + { + public readonly TKey Key = k; + public int Next; + public int Prev; + } + } +} diff --git a/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheNonConcurrent.cs b/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheNonConcurrent.cs new file mode 100644 index 00000000000..ef5a2e49d2b --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Caching/LruKeyCacheNonConcurrent.cs @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Nethermind.Core.Extensions; + +namespace Nethermind.Core.Caching +{ + public sealed class LruKeyCacheNonConcurrent where TKey : notnull + { + private readonly int _maxCapacity; + private readonly string _name; + private readonly Dictionary> _cacheMap; + private LinkedListNode? _leastRecentlyUsed; + + public LruKeyCacheNonConcurrent(int maxCapacity, int startCapacity, string name) + { + _maxCapacity = maxCapacity; + _name = name ?? throw new ArgumentNullException(nameof(name)); + _cacheMap = typeof(TKey) == typeof(byte[]) + ? new Dictionary>((IEqualityComparer)Bytes.EqualityComparer) + : new Dictionary>(startCapacity); // do not initialize it at the full capacity + } + + public LruKeyCacheNonConcurrent(int maxCapacity, string name) + : this(maxCapacity, 0, name) + { + } + + public void Clear() + { + _leastRecentlyUsed = null; + _cacheMap.Clear(); + } + + public bool Get(TKey key) + { + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) + { + LinkedListNode.MoveToMostRecent(ref _leastRecentlyUsed, node); + return true; + } + + return false; + } + + public bool Set(TKey key) + { + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) + { + LinkedListNode.MoveToMostRecent(ref _leastRecentlyUsed, node); + return false; + } + else + { + if (_cacheMap.Count >= _maxCapacity) + { + Replace(key); + } + else + { + LinkedListNode newNode = new(key); + LinkedListNode.AddMostRecent(ref _leastRecentlyUsed, newNode); + _cacheMap.Add(key, newNode); + } + + return true; + } + } + + public void Delete(TKey key) + { + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) + { + LinkedListNode.Remove(ref _leastRecentlyUsed, node); + _cacheMap.Remove(key); + } + } + + private void Replace(TKey key) + { + LinkedListNode? node = _leastRecentlyUsed; + if (node is null) + { + ThrowInvalidOperation(); + } + + _cacheMap.Remove(node.Value); + node.Value = key; + LinkedListNode.MoveToMostRecent(ref _leastRecentlyUsed, node); + _cacheMap.Add(key, node); + + [DoesNotReturn] + static void ThrowInvalidOperation() + { + throw new InvalidOperationException( + $"{nameof(LruKeyCache)} called {nameof(Replace)} when empty."); + } + } + + public long MemorySize => CalculateMemorySize(0, _cacheMap.Count); + + // TODO: memory size on the KeyCache will be smaller because we do not create LruCacheItems + public static long CalculateMemorySize(int keyPlusValueSize, int currentItemsCount) + { + // it may actually be different if the initial capacity not equal to max (depending on the dictionary growth path) + + const int preInit = 48 /* LinkedList */ + 80 /* Dictionary */ + 24; + int postInit = 52 /* lazy init of two internal dictionary arrays + dictionary size times (entry size + int) */ + MemorySizes.FindNextPrime(currentItemsCount) * 28 + currentItemsCount * 80 /* LinkedListNode and CacheItem times items count */; + return MemorySizes.Align(preInit + postInit + keyPlusValueSize * currentItemsCount); + } + } +} diff --git a/src/Nethermind/Nethermind.Core/CappedArrayMemoryManager.cs b/src/Nethermind/Nethermind.Core/CappedArrayMemoryManager.cs new file mode 100644 index 00000000000..65d52241f0b --- /dev/null +++ b/src/Nethermind/Nethermind.Core/CappedArrayMemoryManager.cs @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Buffers; + +namespace Nethermind.Core.Buffers; + +public class CappedArrayMemoryManager(CappedArray? data) : MemoryManager +{ + private readonly CappedArray _data = data ?? throw new ArgumentNullException(nameof(data)); + private bool _isDisposed; + + public override Span GetSpan() + { + if (_isDisposed) + { + throw new ObjectDisposedException(nameof(CappedArrayMemoryManager)); + } + + return _data.AsSpan(); + } + + public override MemoryHandle Pin(int elementIndex = 0) + { + if (_isDisposed) + { + throw new ObjectDisposedException(nameof(CappedArrayMemoryManager)); + } + + if (elementIndex < 0 || elementIndex >= _data.Length) + { + throw new ArgumentOutOfRangeException(nameof(elementIndex)); + } + // Pinning is a no-op in this managed implementation + return new MemoryHandle(); + } + + public override void Unpin() + { + if (_isDisposed) + { + throw new ObjectDisposedException(nameof(CappedArrayMemoryManager)); + } + // Unpinning is a no-op in this managed implementation + } + + protected override void Dispose(bool disposing) + { + _isDisposed = true; + } + + protected override bool TryGetArray(out ArraySegment segment) + { + if (_isDisposed) + { + throw new ObjectDisposedException(nameof(CappedArrayMemoryManager)); + } + + segment = _data.AsArraySegment(); + return true; + } +} diff --git a/src/Nethermind/Nethermind.Core/Collections/LinkedHashSet.cs b/src/Nethermind/Nethermind.Core/Collections/LinkedHashSet.cs index 7ebcb0ad518..553448f9c60 100644 --- a/src/Nethermind/Nethermind.Core/Collections/LinkedHashSet.cs +++ b/src/Nethermind/Nethermind.Core/Collections/LinkedHashSet.cs @@ -196,7 +196,6 @@ public void UnionWith(IEnumerable? other) #endregion - #region ICollection public int Count => _dict.Count; diff --git a/src/Nethermind/Nethermind.Core/Collections/ThrowHelper.cs b/src/Nethermind/Nethermind.Core/Collections/ThrowHelper.cs index 81a93cd1935..b0d136b07f7 100644 --- a/src/Nethermind/Nethermind.Core/Collections/ThrowHelper.cs +++ b/src/Nethermind/Nethermind.Core/Collections/ThrowHelper.cs @@ -51,13 +51,6 @@ public static void ThrowArgumentOutOfRangeException_SizeHint() throw new ArgumentOutOfRangeException("sizeHint"); } - [DoesNotReturn] - [StackTraceHidden] - public static void ThrowArgumentNullException_Options() - { - throw new ArgumentNullException("options"); - } - [DoesNotReturn] [StackTraceHidden] public static void ThrowArgumentNullException_WritingStream() diff --git a/src/Nethermind/Nethermind.Init/Cpu/ConsoleExitHandler.cs b/src/Nethermind/Nethermind.Core/Cpu/ConsoleExitHandler.cs similarity index 98% rename from src/Nethermind/Nethermind.Init/Cpu/ConsoleExitHandler.cs rename to src/Nethermind/Nethermind.Core/Cpu/ConsoleExitHandler.cs index 53f6c8d2890..5b5dabc4d07 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/ConsoleExitHandler.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/ConsoleExitHandler.cs @@ -7,7 +7,7 @@ using System; using System.Diagnostics; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; internal class ConsoleExitHandler : IDisposable { diff --git a/src/Nethermind/Nethermind.Init/Cpu/CpuInfo.cs b/src/Nethermind/Nethermind.Core/Cpu/CpuInfo.cs similarity index 95% rename from src/Nethermind/Nethermind.Init/Cpu/CpuInfo.cs rename to src/Nethermind/Nethermind.Core/Cpu/CpuInfo.cs index 8e374810f34..a0c3eaadb48 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/CpuInfo.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/CpuInfo.cs @@ -4,9 +4,9 @@ // Derived from https://github.com/dotnet/BenchmarkDotNet // Licensed under the MIT License -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; -internal class CpuInfo +public class CpuInfo { public string ProcessorName { get; } public int? PhysicalProcessorCount { get; } diff --git a/src/Nethermind/Nethermind.Init/Cpu/DefaultCultureInfo.cs b/src/Nethermind/Nethermind.Core/Cpu/DefaultCultureInfo.cs similarity index 94% rename from src/Nethermind/Nethermind.Init/Cpu/DefaultCultureInfo.cs rename to src/Nethermind/Nethermind.Core/Cpu/DefaultCultureInfo.cs index 060243b5849..23465e1ceac 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/DefaultCultureInfo.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/DefaultCultureInfo.cs @@ -6,7 +6,7 @@ using System.Globalization; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; internal static class DefaultCultureInfo { diff --git a/src/Nethermind/Nethermind.Init/Cpu/Frequency.cs b/src/Nethermind/Nethermind.Core/Cpu/Frequency.cs similarity index 98% rename from src/Nethermind/Nethermind.Init/Cpu/Frequency.cs rename to src/Nethermind/Nethermind.Core/Cpu/Frequency.cs index 32e2a60a958..e2c8328b83f 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/Frequency.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/Frequency.cs @@ -6,9 +6,9 @@ using System.Globalization; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; -internal readonly struct Frequency +public readonly struct Frequency { public static readonly Frequency Zero = new Frequency(0.0); diff --git a/src/Nethermind/Nethermind.Init/Cpu/FrequencyUnit.cs b/src/Nethermind/Nethermind.Core/Cpu/FrequencyUnit.cs similarity index 94% rename from src/Nethermind/Nethermind.Init/Cpu/FrequencyUnit.cs rename to src/Nethermind/Nethermind.Core/Cpu/FrequencyUnit.cs index 9239c1e55cb..2b0de6bce90 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/FrequencyUnit.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/FrequencyUnit.cs @@ -4,9 +4,9 @@ // Derived from https://github.com/AndreyAkinshin/perfolizer // Licensed under the MIT License -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; -internal class FrequencyUnit +public class FrequencyUnit { public static readonly FrequencyUnit Hz = new FrequencyUnit("Hz", "Hertz", 1L); diff --git a/src/Nethermind/Nethermind.Init/Cpu/ProcCpuInfoKeyNames.cs b/src/Nethermind/Nethermind.Core/Cpu/ProcCpuInfoKeyNames.cs similarity index 94% rename from src/Nethermind/Nethermind.Init/Cpu/ProcCpuInfoKeyNames.cs rename to src/Nethermind/Nethermind.Core/Cpu/ProcCpuInfoKeyNames.cs index 8f2555a629b..49960868ff5 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/ProcCpuInfoKeyNames.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/ProcCpuInfoKeyNames.cs @@ -4,7 +4,7 @@ // Derived from https://github.com/dotnet/BenchmarkDotNet // Licensed under the MIT License -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; internal static class ProcCpuInfoKeyNames { diff --git a/src/Nethermind/Nethermind.Init/Cpu/ProcCpuInfoParser.cs b/src/Nethermind/Nethermind.Core/Cpu/ProcCpuInfoParser.cs similarity index 99% rename from src/Nethermind/Nethermind.Init/Cpu/ProcCpuInfoParser.cs rename to src/Nethermind/Nethermind.Core/Cpu/ProcCpuInfoParser.cs index 8db1a4c52f4..f27aa681861 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/ProcCpuInfoParser.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/ProcCpuInfoParser.cs @@ -8,7 +8,7 @@ using System.Linq; using System.Text.RegularExpressions; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; internal static partial class ProcCpuInfoParser { diff --git a/src/Nethermind/Nethermind.Init/Cpu/ProcCpuInfoProvider.cs b/src/Nethermind/Nethermind.Core/Cpu/ProcCpuInfoProvider.cs similarity index 98% rename from src/Nethermind/Nethermind.Init/Cpu/ProcCpuInfoProvider.cs rename to src/Nethermind/Nethermind.Core/Cpu/ProcCpuInfoProvider.cs index 5fc0d320b4e..c2d6fd770a7 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/ProcCpuInfoProvider.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/ProcCpuInfoProvider.cs @@ -7,7 +7,7 @@ using System; using System.Linq; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; /// /// CPU information from output of the `cat /proc/info` command. diff --git a/src/Nethermind/Nethermind.Init/Cpu/ProcessExtensions.cs b/src/Nethermind/Nethermind.Core/Cpu/ProcessExtensions.cs similarity index 99% rename from src/Nethermind/Nethermind.Init/Cpu/ProcessExtensions.cs rename to src/Nethermind/Nethermind.Core/Cpu/ProcessExtensions.cs index de38f80de2c..d725868961b 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/ProcessExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/ProcessExtensions.cs @@ -10,7 +10,7 @@ using System.Diagnostics; using System.IO; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; // we need it public to reuse it in the auto-generated dll // but we hide it from intellisense with following attribute diff --git a/src/Nethermind/Nethermind.Init/Cpu/ProcessHelper.cs b/src/Nethermind/Nethermind.Core/Cpu/ProcessHelper.cs similarity index 97% rename from src/Nethermind/Nethermind.Init/Cpu/ProcessHelper.cs rename to src/Nethermind/Nethermind.Core/Cpu/ProcessHelper.cs index 1c3bf460459..a4a3291e3f9 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/ProcessHelper.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/ProcessHelper.cs @@ -7,7 +7,7 @@ using System; using System.Diagnostics; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; internal static class ProcessHelper { diff --git a/src/Nethermind/Nethermind.Init/Cpu/RuntimeInformation.cs b/src/Nethermind/Nethermind.Core/Cpu/RuntimeInformation.cs similarity index 66% rename from src/Nethermind/Nethermind.Init/Cpu/RuntimeInformation.cs rename to src/Nethermind/Nethermind.Core/Cpu/RuntimeInformation.cs index 4dbbacf6869..f1ed2b7c93d 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/RuntimeInformation.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/RuntimeInformation.cs @@ -5,10 +5,11 @@ // Licensed under the MIT License using System; +using System.Threading.Tasks; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; -internal static class RuntimeInformation +public static class RuntimeInformation { [System.Runtime.Versioning.SupportedOSPlatformGuard("windows")] internal static bool IsWindows() => OperatingSystem.IsWindows(); // prefer linker-friendly OperatingSystem APIs @@ -19,7 +20,7 @@ internal static class RuntimeInformation [System.Runtime.Versioning.SupportedOSPlatformGuard("macos")] internal static bool IsMacOS() => OperatingSystem.IsMacOS(); - internal static CpuInfo? GetCpuInfo() + public static CpuInfo? GetCpuInfo() { if (IsWindows()) return WmicCpuInfoProvider.WmicCpuInfo.Value; @@ -31,5 +32,9 @@ internal static class RuntimeInformation return null; } + public static int PhysicalCoreCount { get; } = GetCpuInfo()?.PhysicalCoreCount ?? Environment.ProcessorCount; + public static ParallelOptions ParallelOptionsPhysicalCores { get; } = new() { MaxDegreeOfParallelism = PhysicalCoreCount }; + public static ParallelOptions ParallelOptionsLogicalCores { get; } = new() { MaxDegreeOfParallelism = Environment.ProcessorCount }; + public static bool Is64BitPlatform() => IntPtr.Size == 8; } diff --git a/src/Nethermind/Nethermind.Init/Cpu/SectionsHelper.cs b/src/Nethermind/Nethermind.Core/Cpu/SectionsHelper.cs similarity index 98% rename from src/Nethermind/Nethermind.Init/Cpu/SectionsHelper.cs rename to src/Nethermind/Nethermind.Core/Cpu/SectionsHelper.cs index f65f1a432c2..34e3cec0847 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/SectionsHelper.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/SectionsHelper.cs @@ -9,7 +9,7 @@ using System.Linq; using System.Text.RegularExpressions; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; internal static partial class SectionsHelper { diff --git a/src/Nethermind/Nethermind.Init/Cpu/SysctlCpuInfoParser.cs b/src/Nethermind/Nethermind.Core/Cpu/SysctlCpuInfoParser.cs similarity index 98% rename from src/Nethermind/Nethermind.Init/Cpu/SysctlCpuInfoParser.cs rename to src/Nethermind/Nethermind.Core/Cpu/SysctlCpuInfoParser.cs index af67becb1fb..08071d70553 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/SysctlCpuInfoParser.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/SysctlCpuInfoParser.cs @@ -6,7 +6,7 @@ using System.Collections.Generic; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; internal static class SysctlCpuInfoParser { diff --git a/src/Nethermind/Nethermind.Init/Cpu/SysctlCpuInfoProvider.cs b/src/Nethermind/Nethermind.Core/Cpu/SysctlCpuInfoProvider.cs similarity index 95% rename from src/Nethermind/Nethermind.Init/Cpu/SysctlCpuInfoProvider.cs rename to src/Nethermind/Nethermind.Core/Cpu/SysctlCpuInfoProvider.cs index 2bdd6909b5c..00b1c8b1dfb 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/SysctlCpuInfoProvider.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/SysctlCpuInfoProvider.cs @@ -6,7 +6,7 @@ using System; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; /// /// CPU information from output of the `sysctl -a` command. diff --git a/src/Nethermind/Nethermind.Init/Cpu/TimeInterval.cs b/src/Nethermind/Nethermind.Core/Cpu/TimeInterval.cs similarity index 98% rename from src/Nethermind/Nethermind.Init/Cpu/TimeInterval.cs rename to src/Nethermind/Nethermind.Core/Cpu/TimeInterval.cs index da44e32dc7c..690508a072f 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/TimeInterval.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/TimeInterval.cs @@ -6,9 +6,9 @@ using System.Globalization; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; -internal readonly struct TimeInterval +public readonly struct TimeInterval { public static readonly TimeInterval Nanosecond = TimeUnit.Nanosecond.ToInterval(1L); diff --git a/src/Nethermind/Nethermind.Init/Cpu/TimeUnit.cs b/src/Nethermind/Nethermind.Core/Cpu/TimeUnit.cs similarity index 97% rename from src/Nethermind/Nethermind.Init/Cpu/TimeUnit.cs rename to src/Nethermind/Nethermind.Core/Cpu/TimeUnit.cs index dfaea152de0..821b8a90514 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/TimeUnit.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/TimeUnit.cs @@ -7,9 +7,9 @@ using System; using System.Linq; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; -internal class TimeUnit : IEquatable +public class TimeUnit : IEquatable { public static readonly TimeUnit Nanosecond = new TimeUnit("ns", "Nanosecond", 1L); diff --git a/src/Nethermind/Nethermind.Init/Cpu/UnitPresentation.cs b/src/Nethermind/Nethermind.Core/Cpu/UnitPresentation.cs similarity index 93% rename from src/Nethermind/Nethermind.Init/Cpu/UnitPresentation.cs rename to src/Nethermind/Nethermind.Core/Cpu/UnitPresentation.cs index 900a07f78a1..bc67eaf180d 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/UnitPresentation.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/UnitPresentation.cs @@ -4,9 +4,9 @@ // Derived from https://github.com/AndreyAkinshin/perfolizer // Licensed under the MIT License -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; -internal class UnitPresentation +public class UnitPresentation { public static readonly UnitPresentation Default = new UnitPresentation(isVisible: true, 0); diff --git a/src/Nethermind/Nethermind.Init/Cpu/WmicCpuInfoKeyNames.cs b/src/Nethermind/Nethermind.Core/Cpu/WmicCpuInfoKeyNames.cs similarity index 93% rename from src/Nethermind/Nethermind.Init/Cpu/WmicCpuInfoKeyNames.cs rename to src/Nethermind/Nethermind.Core/Cpu/WmicCpuInfoKeyNames.cs index 6f690f673f9..38069667866 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/WmicCpuInfoKeyNames.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/WmicCpuInfoKeyNames.cs @@ -4,7 +4,7 @@ // Derived from https://github.com/dotnet/BenchmarkDotNet // Licensed under the MIT License -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; internal static class WmicCpuInfoKeyNames { diff --git a/src/Nethermind/Nethermind.Init/Cpu/WmicCpuInfoParser.cs b/src/Nethermind/Nethermind.Core/Cpu/WmicCpuInfoParser.cs similarity index 98% rename from src/Nethermind/Nethermind.Init/Cpu/WmicCpuInfoParser.cs rename to src/Nethermind/Nethermind.Core/Cpu/WmicCpuInfoParser.cs index f13d3819713..51d9b77bc79 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/WmicCpuInfoParser.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/WmicCpuInfoParser.cs @@ -6,7 +6,7 @@ using System.Collections.Generic; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; internal static class WmicCpuInfoParser { diff --git a/src/Nethermind/Nethermind.Init/Cpu/WmicCpuInfoProvider.cs b/src/Nethermind/Nethermind.Core/Cpu/WmicCpuInfoProvider.cs similarity index 96% rename from src/Nethermind/Nethermind.Init/Cpu/WmicCpuInfoProvider.cs rename to src/Nethermind/Nethermind.Core/Cpu/WmicCpuInfoProvider.cs index 882fc2cd970..e5e4d35a27a 100644 --- a/src/Nethermind/Nethermind.Init/Cpu/WmicCpuInfoProvider.cs +++ b/src/Nethermind/Nethermind.Core/Cpu/WmicCpuInfoProvider.cs @@ -6,7 +6,7 @@ using System; -namespace Nethermind.Init.Cpu; +namespace Nethermind.Core.Cpu; /// /// CPU information from output of the `wmic cpu get Name, NumberOfCores, NumberOfLogicalProcessors /Format:List` command. diff --git a/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs b/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs index b2002f8936f..6b123e45158 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/Hash256.cs @@ -117,12 +117,12 @@ public Hash256 ToCommitment() } public static bool operator ==(in ValueHash256 left, in ValueHash256 right) => left.Equals(in right); - public static bool operator !=(in ValueHash256 left, in ValueHash256 right) => !(left == right); public static bool operator >(in ValueHash256 left, in ValueHash256 right) => left.CompareTo(in right) > 0; public static bool operator <(in ValueHash256 left, in ValueHash256 right) => left.CompareTo(in right) < 0; public static bool operator >=(in ValueHash256 left, in ValueHash256 right) => left.CompareTo(in right) >= 0; public static bool operator <=(in ValueHash256 left, in ValueHash256 right) => left.CompareTo(in right) <= 0; + public static implicit operator Hash256(in ValueHash256 keccak) => new(keccak); } public readonly struct Hash256AsKey(Hash256 key) : IEquatable diff --git a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs index 4d8f15203f6..f3cca085977 100644 --- a/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs +++ b/src/Nethermind/Nethermind.Core/Crypto/KeccakHash.cs @@ -104,36 +104,31 @@ private static void KeccakF(Span st) ulong ema, eme, emi, emo, emu; ulong esa, ese, esi, eso, esu; - { - // Access last element to perform range check once - // and not for every ascending access - _ = st[24]; - } - aba = st[0]; - abe = st[1]; - abi = st[2]; - abo = st[3]; - abu = st[4]; - aga = st[5]; - age = st[6]; - agi = st[7]; - ago = st[8]; - agu = st[9]; - aka = st[10]; - ake = st[11]; - aki = st[12]; - ako = st[13]; - aku = st[14]; - ama = st[15]; - ame = st[16]; - ami = st[17]; - amo = st[18]; - amu = st[19]; - asa = st[20]; - ase = st[21]; - asi = st[22]; - aso = st[23]; asu = st[24]; + aso = st[23]; + asi = st[22]; + ase = st[21]; + asa = st[20]; + amu = st[19]; + amo = st[18]; + ami = st[17]; + ame = st[16]; + ama = st[15]; + aku = st[14]; + ako = st[13]; + aki = st[12]; + ake = st[11]; + aka = st[10]; + agu = st[9]; + ago = st[8]; + agi = st[7]; + age = st[6]; + aga = st[5]; + abu = st[4]; + abo = st[3]; + abi = st[2]; + abe = st[1]; + aba = st[0]; for (int round = 0; round < ROUNDS; round += 2) { @@ -151,209 +146,158 @@ private static void KeccakF(Span st) @do = bCi ^ RotateLeft(bCu, 1); du = bCo ^ RotateLeft(bCa, 1); - aba ^= da; - bCa = aba; - age ^= de; - bCe = RotateLeft(age, 44); - aki ^= di; - bCi = RotateLeft(aki, 43); - amo ^= @do; - bCo = RotateLeft(amo, 21); - asu ^= du; - bCu = RotateLeft(asu, 14); - eba = bCa ^ ((~bCe) & bCi); - eba ^= RoundConstants[round]; + bCa = aba ^ da; + bCe = RotateLeft(age ^ de, 44); + bCi = RotateLeft(aki ^ di, 43); + eba = bCa ^ ((~bCe) & bCi) ^ RoundConstants[round]; + bCo = RotateLeft(amo ^ @do, 21); ebe = bCe ^ ((~bCi) & bCo); + bCu = RotateLeft(asu ^ du, 14); ebi = bCi ^ ((~bCo) & bCu); ebo = bCo ^ ((~bCu) & bCa); ebu = bCu ^ ((~bCa) & bCe); - abo ^= @do; - bCa = RotateLeft(abo, 28); - agu ^= du; - bCe = RotateLeft(agu, 20); - aka ^= da; - bCi = RotateLeft(aka, 3); - ame ^= de; - bCo = RotateLeft(ame, 45); - asi ^= di; - bCu = RotateLeft(asi, 61); + bCa = RotateLeft(abo ^ @do, 28); + bCe = RotateLeft(agu ^ du, 20); + bCi = RotateLeft(aka ^ da, 3); ega = bCa ^ ((~bCe) & bCi); + bCo = RotateLeft(ame ^ de, 45); ege = bCe ^ ((~bCi) & bCo); + bCu = RotateLeft(asi ^ di, 61); egi = bCi ^ ((~bCo) & bCu); ego = bCo ^ ((~bCu) & bCa); egu = bCu ^ ((~bCa) & bCe); - abe ^= de; - bCa = RotateLeft(abe, 1); - agi ^= di; - bCe = RotateLeft(agi, 6); - ako ^= @do; - bCi = RotateLeft(ako, 25); - amu ^= du; - bCo = RotateLeft(amu, 8); - asa ^= da; - bCu = RotateLeft(asa, 18); + bCa = RotateLeft(abe ^ de, 1); + bCe = RotateLeft(agi ^ di, 6); + bCi = RotateLeft(ako ^ @do, 25); eka = bCa ^ ((~bCe) & bCi); + bCo = RotateLeft(amu ^ du, 8); eke = bCe ^ ((~bCi) & bCo); + bCu = RotateLeft(asa ^ da, 18); eki = bCi ^ ((~bCo) & bCu); eko = bCo ^ ((~bCu) & bCa); eku = bCu ^ ((~bCa) & bCe); - abu ^= du; - bCa = RotateLeft(abu, 27); - aga ^= da; - bCe = RotateLeft(aga, 36); - ake ^= de; - bCi = RotateLeft(ake, 10); - ami ^= di; - bCo = RotateLeft(ami, 15); - aso ^= @do; - bCu = RotateLeft(aso, 56); + bCa = RotateLeft(abu ^ du, 27); + bCe = RotateLeft(aga ^ da, 36); + bCi = RotateLeft(ake ^ de, 10); ema = bCa ^ ((~bCe) & bCi); + bCo = RotateLeft(ami ^ di, 15); eme = bCe ^ ((~bCi) & bCo); + bCu = RotateLeft(aso ^ @do, 56); emi = bCi ^ ((~bCo) & bCu); emo = bCo ^ ((~bCu) & bCa); emu = bCu ^ ((~bCa) & bCe); - abi ^= di; - bCa = RotateLeft(abi, 62); - ago ^= @do; - bCe = RotateLeft(ago, 55); - aku ^= du; - bCi = RotateLeft(aku, 39); - ama ^= da; - bCo = RotateLeft(ama, 41); - ase ^= de; - bCu = RotateLeft(ase, 2); + bCa = RotateLeft(abi ^ di, 62); + bCe = RotateLeft(ago ^ @do, 55); + bCi = RotateLeft(aku ^ du, 39); esa = bCa ^ ((~bCe) & bCi); + bCo = RotateLeft(ama ^ da, 41); ese = bCe ^ ((~bCi) & bCo); + bCu = RotateLeft(ase ^ de, 2); esi = bCi ^ ((~bCo) & bCu); eso = bCo ^ ((~bCu) & bCa); esu = bCu ^ ((~bCa) & bCe); // prepareTheta - bCa = eba ^ ega ^ eka ^ ema ^ esa; + bCe = ebe ^ ege ^ eke ^ eme ^ ese; - bCi = ebi ^ egi ^ eki ^ emi ^ esi; - bCo = ebo ^ ego ^ eko ^ emo ^ eso; bCu = ebu ^ egu ^ eku ^ emu ^ esu; - //thetaRhoPiChiIotaPrepareTheta(round+1, E, A) da = bCu ^ RotateLeft(bCe, 1); + bCa = eba ^ ega ^ eka ^ ema ^ esa; + bCi = ebi ^ egi ^ eki ^ emi ^ esi; de = bCa ^ RotateLeft(bCi, 1); + bCo = ebo ^ ego ^ eko ^ emo ^ eso; di = bCe ^ RotateLeft(bCo, 1); @do = bCi ^ RotateLeft(bCu, 1); du = bCo ^ RotateLeft(bCa, 1); - eba ^= da; - bCa = eba; - ege ^= de; - bCe = RotateLeft(ege, 44); - eki ^= di; - bCi = RotateLeft(eki, 43); - emo ^= @do; - bCo = RotateLeft(emo, 21); - esu ^= du; - bCu = RotateLeft(esu, 14); - aba = bCa ^ ((~bCe) & bCi); - aba ^= RoundConstants[round + 1]; + + bCi = RotateLeft(eki ^ di, 43); + bCe = RotateLeft(ege ^ de, 44); + bCa = eba ^ da; + aba = bCa ^ ((~bCe) & bCi) ^ RoundConstants[round + 1]; + bCo = RotateLeft(emo ^ @do, 21); abe = bCe ^ ((~bCi) & bCo); + bCu = RotateLeft(esu ^ du, 14); abi = bCi ^ ((~bCo) & bCu); abo = bCo ^ ((~bCu) & bCa); abu = bCu ^ ((~bCa) & bCe); - ebo ^= @do; - bCa = RotateLeft(ebo, 28); - egu ^= du; - bCe = RotateLeft(egu, 20); - eka ^= da; - bCi = RotateLeft(eka, 3); - eme ^= de; - bCo = RotateLeft(eme, 45); - esi ^= di; - bCu = RotateLeft(esi, 61); + bCa = RotateLeft(ebo ^ @do, 28); + bCe = RotateLeft(egu ^ du, 20); + bCi = RotateLeft(eka ^ da, 3); aga = bCa ^ ((~bCe) & bCi); + bCo = RotateLeft(eme ^ de, 45); age = bCe ^ ((~bCi) & bCo); + bCu = RotateLeft(esi ^ di, 61); agi = bCi ^ ((~bCo) & bCu); ago = bCo ^ ((~bCu) & bCa); agu = bCu ^ ((~bCa) & bCe); - ebe ^= de; - bCa = RotateLeft(ebe, 1); - egi ^= di; - bCe = RotateLeft(egi, 6); - eko ^= @do; - bCi = RotateLeft(eko, 25); - emu ^= du; - bCo = RotateLeft(emu, 8); - esa ^= da; - bCu = RotateLeft(esa, 18); + bCa = RotateLeft(ebe ^ de, 1); + bCe = RotateLeft(egi ^ di, 6); + bCi = RotateLeft(eko ^ @do, 25); aka = bCa ^ ((~bCe) & bCi); + bCo = RotateLeft(emu ^ du, 8); ake = bCe ^ ((~bCi) & bCo); + bCu = RotateLeft(esa ^ da, 18); aki = bCi ^ ((~bCo) & bCu); ako = bCo ^ ((~bCu) & bCa); aku = bCu ^ ((~bCa) & bCe); - ebu ^= du; - bCa = RotateLeft(ebu, 27); - ega ^= da; - bCe = RotateLeft(ega, 36); - eke ^= de; - bCi = RotateLeft(eke, 10); - emi ^= di; - bCo = RotateLeft(emi, 15); - eso ^= @do; - bCu = RotateLeft(eso, 56); + bCa = RotateLeft(ebu ^ du, 27); + bCe = RotateLeft(ega ^ da, 36); + bCi = RotateLeft(eke ^ de, 10); ama = bCa ^ ((~bCe) & bCi); + bCo = RotateLeft(emi ^ di, 15); ame = bCe ^ ((~bCi) & bCo); + bCu = RotateLeft(eso ^ @do, 56); ami = bCi ^ ((~bCo) & bCu); amo = bCo ^ ((~bCu) & bCa); amu = bCu ^ ((~bCa) & bCe); - ebi ^= di; - bCa = RotateLeft(ebi, 62); - ego ^= @do; - bCe = RotateLeft(ego, 55); - eku ^= du; - bCi = RotateLeft(eku, 39); - ema ^= da; - bCo = RotateLeft(ema, 41); - ese ^= de; - bCu = RotateLeft(ese, 2); + bCa = RotateLeft(ebi ^ di, 62); + bCe = RotateLeft(ego ^ @do, 55); + bCi = RotateLeft(eku ^ du, 39); asa = bCa ^ ((~bCe) & bCi); + bCo = RotateLeft(ema ^ da, 41); ase = bCe ^ ((~bCi) & bCo); + bCu = RotateLeft(ese ^ de, 2); asi = bCi ^ ((~bCo) & bCu); aso = bCo ^ ((~bCu) & bCa); asu = bCu ^ ((~bCa) & bCe); } //copyToState(state, A) - st[0] = aba; - st[1] = abe; - st[2] = abi; - st[3] = abo; - st[4] = abu; - st[5] = aga; - st[6] = age; - st[7] = agi; - st[8] = ago; - st[9] = agu; - st[10] = aka; - st[11] = ake; - st[12] = aki; - st[13] = ako; - st[14] = aku; - st[15] = ama; - st[16] = ame; - st[17] = ami; - st[18] = amo; - st[19] = amu; - st[20] = asa; - st[21] = ase; - st[22] = asi; - st[23] = aso; st[24] = asu; + st[23] = aso; + st[22] = asi; + st[21] = ase; + st[20] = asa; + st[19] = amu; + st[18] = amo; + st[17] = ami; + st[16] = ame; + st[15] = ama; + st[14] = aku; + st[13] = ako; + st[12] = aki; + st[11] = ake; + st[10] = aka; + st[9] = agu; + st[8] = ago; + st[7] = agi; + st[6] = age; + st[5] = aga; + st[4] = abu; + st[3] = abo; + st[2] = abi; + st[1] = abe; + st[0] = aba; } public static Span ComputeHash(ReadOnlySpan input, int size = HASH_SIZE) diff --git a/src/Nethermind/Nethermind.Core/Extensions/ReadOnlySequenceExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/ReadOnlySequenceExtensions.cs new file mode 100644 index 00000000000..f53858d950b --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Extensions/ReadOnlySequenceExtensions.cs @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Buffers; + +namespace Nethermind.Core.Extensions; + +public static class ReadOnlySequenceExtensions +{ + private static readonly byte[] WhitespaceChars = " \t\r\n"u8.ToArray(); + + public static ReadOnlySequence TrimStart(this ReadOnlySequence sequence, byte[]? chars = null) + { + SequencePosition position = sequence.Start; + ReadOnlySpan charsSpan = chars ?? WhitespaceChars; + while (sequence.TryGet(ref position, out ReadOnlyMemory memory)) + { + ReadOnlySpan span = memory.Span; + int index = span.IndexOfAnyExcept(charsSpan); + + if (index != 0) + { + // if index == -1, then the whole span is whitespace + sequence = sequence.Slice(index != -1 ? index : span.Length); + } + else + { + return sequence; + } + } + + return sequence; + } +} diff --git a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs index 6d0a81ce13b..717ae542c71 100644 --- a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs +++ b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs @@ -33,7 +33,7 @@ public interface IReadOnlyKeyValueStore bool KeyExists(ReadOnlySpan key) { Span span = GetSpan(key); - bool result = span.IsNull(); + bool result = !span.IsNull(); DangerousReleaseMemory(span); return result; } @@ -55,7 +55,7 @@ public interface IWriteOnlyKeyValueStore /// is preferable. Unless you plan to reuse the array somehow (pool), then you'd just use span. /// public bool PreferWriteByArray => false; - void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) => Set(key, value.ToArray(), flags); + void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags flags = WriteFlags.None) => Set(key, value.IsNull() ? null : value.ToArray(), flags); void Remove(ReadOnlySpan key) => Set(key, null); } diff --git a/src/Nethermind/Nethermind.Core/ProductInfo.cs b/src/Nethermind/Nethermind.Core/ProductInfo.cs index 70a72ab5b2a..b23e63c3006 100644 --- a/src/Nethermind/Nethermind.Core/ProductInfo.cs +++ b/src/Nethermind/Nethermind.Core/ProductInfo.cs @@ -12,24 +12,29 @@ public static class ProductInfo { static ProductInfo() { - var assembly = Assembly.GetEntryAssembly(); - var infoAttr = assembly?.GetCustomAttribute(); - var metadataAttrs = assembly?.GetCustomAttributes(); - var productAttr = assembly?.GetCustomAttribute(); - var commit = metadataAttrs?.FirstOrDefault(a => a.Key.Equals("Commit", StringComparison.Ordinal))?.Value; + var assembly = Assembly.GetEntryAssembly()!; + var metadataAttrs = assembly.GetCustomAttributes()!; + var productAttr = assembly.GetCustomAttribute()!; + var versionAttr = assembly.GetCustomAttribute()!; var timestamp = metadataAttrs?.FirstOrDefault(a => a.Key.Equals("BuildTimestamp", StringComparison.Ordinal))?.Value; BuildTimestamp = long.TryParse(timestamp, out var t) ? DateTimeOffset.FromUnixTimeSeconds(t) : DateTimeOffset.MinValue; - Commit = commit ?? string.Empty; Name = productAttr?.Product ?? "Nethermind"; OS = Platform.GetPlatformName(); OSArchitecture = RuntimeInformation.OSArchitecture.ToString().ToLowerInvariant(); Runtime = RuntimeInformation.FrameworkDescription; - Version = infoAttr?.InformationalVersion ?? string.Empty; + Version = versionAttr.InformationalVersion; + + var index = Version.IndexOf('+', StringComparison.Ordinal); + + if (index != -1) + { + Commit = Version[(index + 1)..]; + Version = Version[..Math.Min(index + 9, Version.Length - 1)]; + } - ClientCode = "NM"; ClientId = $"{Name}/v{Version}/{OS.ToLowerInvariant()}-{OSArchitecture}/dotnet{Runtime[5..]}"; } @@ -37,9 +42,9 @@ static ProductInfo() public static string ClientId { get; } - public static string ClientCode { get; } + public static string ClientCode { get; } = "NM"; - public static string Commit { get; } + public static string Commit { get; set; } = string.Empty; public static string Name { get; } @@ -51,11 +56,11 @@ static ProductInfo() public static string Version { get; } - public static string Network { get; set; } = ""; + public static string Network { get; set; } = string.Empty; - public static string Instance { get; set; } = ""; + public static string Instance { get; set; } = string.Empty; - public static string SyncType { get; set; } = ""; + public static string SyncType { get; set; } = string.Empty; - public static string PruningMode { get; set; } = ""; + public static string PruningMode { get; set; } = string.Empty; } diff --git a/src/Nethermind/Nethermind.Core/Resettables/ResettableDictionary.cs b/src/Nethermind/Nethermind.Core/Resettables/ResettableDictionary.cs index 7dd63feef16..04607484640 100644 --- a/src/Nethermind/Nethermind.Core/Resettables/ResettableDictionary.cs +++ b/src/Nethermind/Nethermind.Core/Resettables/ResettableDictionary.cs @@ -111,13 +111,19 @@ public TValue this[TKey key] public ICollection Keys => _wrapped.Keys; public ICollection Values => _wrapped.Values; - public void Reset() + public void Reset(bool resizeCollections = true) { if (_wrapped.Count == 0) { return; } + if (!resizeCollections) + { + _wrapped.Clear(); + return; + } + if (_wrapped.Count < _currentCapacity / _resetRatio && _currentCapacity != _startCapacity) { _currentCapacity = Math.Max(_startCapacity, _currentCapacity / _resetRatio); diff --git a/src/Nethermind/Nethermind.Core/Resettables/ResettableHashSet.cs b/src/Nethermind/Nethermind.Core/Resettables/ResettableHashSet.cs index c3f81bd7f57..4db6c75bd79 100644 --- a/src/Nethermind/Nethermind.Core/Resettables/ResettableHashSet.cs +++ b/src/Nethermind/Nethermind.Core/Resettables/ResettableHashSet.cs @@ -61,12 +61,17 @@ public bool Remove(T item) public int Count => _wrapped.Count; public bool IsReadOnly => false; - public void Reset() + public void Reset(bool resizeCollections = true) { if (_wrapped.Count == 0) { return; } + if (!resizeCollections) + { + _wrapped.Clear(); + return; + } if (_wrapped.Count < _currentCapacity / _resetRatio && _currentCapacity != _startCapacity) { diff --git a/src/Nethermind/Nethermind.Core/Specs/IReleaseSpec.cs b/src/Nethermind/Nethermind.Core/Specs/IReleaseSpec.cs index f5f20d2eb93..b3b5d0ee5ac 100644 --- a/src/Nethermind/Nethermind.Core/Specs/IReleaseSpec.cs +++ b/src/Nethermind/Nethermind.Core/Specs/IReleaseSpec.cs @@ -176,11 +176,6 @@ public interface IReleaseSpec : IEip1559Spec, IReceiptSpec /// bool IsEip2200Enabled { get; } - /// - /// Berlin subroutines -> https://github.com/ethereum/EIPs/issues/2315 - /// - bool IsEip2315Enabled { get; } - /// /// Berlin BLS crypto precompiles /// @@ -356,8 +351,6 @@ public interface IReleaseSpec : IEip1559Spec, IReceiptSpec public bool ShiftOpcodesEnabled => IsEip145Enabled; - public bool SubroutinesEnabled => IsEip2315Enabled; - public bool RevertOpcodeEnabled => IsEip140Enabled; public bool ExtCodeHashOpcodeEnabled => IsEip1052Enabled; diff --git a/src/Nethermind/Nethermind.Core/Threading/ThreadExtensions.cs b/src/Nethermind/Nethermind.Core/Threading/ThreadExtensions.cs index 31d892ba8b6..002c6dadbe5 100644 --- a/src/Nethermind/Nethermind.Core/Threading/ThreadExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Threading/ThreadExtensions.cs @@ -10,7 +10,7 @@ public static class ThreadExtensions { public readonly struct Disposable : IDisposable { - private readonly Thread _thread; + private readonly Thread? _thread; private readonly ThreadPriority _previousPriority; internal Disposable(Thread thread) @@ -22,7 +22,10 @@ internal Disposable(Thread thread) public void Dispose() { - _thread.Priority = _previousPriority; + if (_thread is not null && Thread.CurrentThread == _thread) + { + _thread.Priority = _previousPriority; + } } } diff --git a/src/Nethermind/Nethermind.Core/Transaction.cs b/src/Nethermind/Nethermind.Core/Transaction.cs index f83f0ea5d75..1371eba41de 100644 --- a/src/Nethermind/Nethermind.Core/Transaction.cs +++ b/src/Nethermind/Nethermind.Core/Transaction.cs @@ -266,6 +266,33 @@ public bool Return(Transaction obj) return true; } } + + public void CopyTo(Transaction tx) + { + tx.ChainId = ChainId; + tx.Type = Type; + tx.SourceHash = SourceHash; + tx.Mint = Mint; + tx.IsOPSystemTransaction = IsOPSystemTransaction; + tx.Nonce = Nonce; + tx.GasPrice = GasPrice; + tx.GasBottleneck = GasBottleneck; + tx.DecodedMaxFeePerGas = DecodedMaxFeePerGas; + tx.GasLimit = GasLimit; + tx.To = To; + tx.Value = Value; + tx.Data = Data; + tx.SenderAddress = SenderAddress; + tx.Signature = Signature; + tx.Timestamp = Timestamp; + tx.AccessList = AccessList; + tx.MaxFeePerBlobGas = MaxFeePerBlobGas; + tx.BlobVersionedHashes = BlobVersionedHashes; + tx.NetworkWrapper = NetworkWrapper; + tx.IsServiceTransaction = IsServiceTransaction; + tx.PoolIndex = PoolIndex; + tx._size = _size; + } } /// diff --git a/src/Nethermind/Nethermind.Crypto/AesEngineX86Intrinsic.cs b/src/Nethermind/Nethermind.Crypto/AesEngineX86Intrinsic.cs deleted file mode 100644 index 1f5751eaf3e..00000000000 --- a/src/Nethermind/Nethermind.Crypto/AesEngineX86Intrinsic.cs +++ /dev/null @@ -1,425 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only -// Modified from BouncyCastle MIT - -#pragma warning disable CA1857 // A constant is expected for the parameter - -using System; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -using Org.BouncyCastle.Crypto; -using Org.BouncyCastle.Crypto.Parameters; - -using Aes = System.Runtime.Intrinsics.X86.Aes; -using Sse2 = System.Runtime.Intrinsics.X86.Sse2; - -namespace Nethermind.Crypto; - -public sealed class AesEngineX86Intrinsic : IBlockCipher -{ - public static bool IsSupported => Aes.IsSupported; - public bool IsPartialBlockOkay => false; - public void Reset() { } - - public AesEngineX86Intrinsic() - { - if (!IsSupported) - throw new PlatformNotSupportedException(nameof(AesEngineX86Intrinsic)); - } - - public string AlgorithmName => "AES"; - - public int GetBlockSize() => 16; - - private AesEncoderDecoder _implementation; - - public void Init(bool forEncryption, ICipherParameters parameters) - { - if (parameters is not KeyParameter keyParameter) - { - ArgumentNullException.ThrowIfNull(parameters, nameof(parameters)); - throw new ArgumentException("invalid type: " + parameters.GetType(), nameof(parameters)); - } - - Vector128[] roundKeys = CreateRoundKeys(keyParameter.GetKey(), forEncryption); - _implementation = AesEncoderDecoder.Init(forEncryption, roundKeys); - } - - public int ProcessBlock(byte[] inBuf, int inOff, byte[] outBuf, int outOff) - { - Check.DataLength(inBuf, inOff, 16); - Check.OutputLength(outBuf, outOff, 16); - - Vector128 state = Unsafe.As>(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(inBuf), inOff)); - - _implementation.ProcessRounds(ref state); - - Unsafe.As>(ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(outBuf), outOff)) = state; - - return 16; - } - - private static Vector128[] CreateRoundKeys(byte[] key, bool forEncryption) - { - Vector128[] K = key.Length switch - { - 16 => KeyLength16(key), - 24 => KeyLength24(key), - 32 => KeyLength32(key), - _ => throw new ArgumentException("Key length not 128/192/256 bits.") - }; - - if (!forEncryption) - { - for (int i = 1, last = K.Length - 1; i < last; ++i) - { - K[i] = Aes.InverseMixColumns(K[i]); - } - - Array.Reverse(K); - } - - return K; - - [SkipLocalsInit] - static Vector128[] KeyLength16(byte[] key) - { - ReadOnlySpan rcon = stackalloc byte[] { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36 }; - - Vector128 s = MemoryMarshal.Read>(key.AsSpan(0, 16)); - Vector128[] K = new Vector128[11]; - K[0] = s; - - for (int round = 0; round < 10;) - { - Vector128 t = Aes.KeygenAssist(s, rcon[round++]); - t = Sse2.Shuffle(t.AsInt32(), 0xFF).AsByte(); - s = Sse2.Xor(s, Sse2.ShiftLeftLogical128BitLane(s, 8)); - s = Sse2.Xor(s, Sse2.ShiftLeftLogical128BitLane(s, 4)); - s = Sse2.Xor(s, t); - K[round] = s; - } - - return K; - } - - static Vector128[] KeyLength24(byte[] key) - { - Vector128 s1 = MemoryMarshal.Read>(key.AsSpan(0, 16)); - Vector128 s2 = MemoryMarshal.Read>(key.AsSpan(16, 8)).ToVector128(); - Vector128[] K = new Vector128[13]; - K[0] = s1; - - byte rcon = 0x01; - for (int round = 0; ;) - { - Vector128 t1 = Aes.KeygenAssist(s2, rcon); rcon <<= 1; - t1 = Sse2.Shuffle(t1.AsInt32(), 0x55).AsByte(); - - s1 = Sse2.Xor(s1, Sse2.ShiftLeftLogical128BitLane(s1, 8)); - s1 = Sse2.Xor(s1, Sse2.ShiftLeftLogical128BitLane(s1, 4)); - s1 = Sse2.Xor(s1, t1); - - K[++round] = Sse2.Xor(s2, Sse2.ShiftLeftLogical128BitLane(s1, 8)); - - Vector128 s3 = Sse2.Xor(s2, Sse2.ShiftRightLogical128BitLane(s1, 12)); - s3 = Sse2.Xor(s3, Sse2.ShiftLeftLogical128BitLane(s3, 4)); - - K[++round] = Sse2.Xor( - Sse2.ShiftRightLogical128BitLane(s1, 8), - Sse2.ShiftLeftLogical128BitLane(s3, 8)); - - Vector128 t2 = Aes.KeygenAssist(s3, rcon); rcon <<= 1; - t2 = Sse2.Shuffle(t2.AsInt32(), 0x55).AsByte(); - - s1 = Sse2.Xor(s1, Sse2.ShiftLeftLogical128BitLane(s1, 8)); - s1 = Sse2.Xor(s1, Sse2.ShiftLeftLogical128BitLane(s1, 4)); - s1 = Sse2.Xor(s1, t2); - - K[++round] = s1; - - if (round == 12) - break; - - s2 = Sse2.Xor(s3, Sse2.ShiftRightLogical128BitLane(s1, 12)); - s2 = Sse2.Xor(s2, Sse2.ShiftLeftLogical128BitLane(s2, 4)); - s2 = s2.WithUpper(Vector64.Zero); - } - - return K; - } - - static Vector128[] KeyLength32(byte[] key) - { - Vector128 s1 = MemoryMarshal.Read>(key.AsSpan(0, 16)); - Vector128 s2 = MemoryMarshal.Read>(key.AsSpan(16, 16)); - Vector128[] K = new Vector128[15]; - K[0] = s1; - K[1] = s2; - - byte rcon = 0x01; - for (int round = 1; ;) - { - Vector128 t1 = Aes.KeygenAssist(s2, rcon); rcon <<= 1; - t1 = Sse2.Shuffle(t1.AsInt32(), 0xFF).AsByte(); - s1 = Sse2.Xor(s1, Sse2.ShiftLeftLogical128BitLane(s1, 8)); - s1 = Sse2.Xor(s1, Sse2.ShiftLeftLogical128BitLane(s1, 4)); - s1 = Sse2.Xor(s1, t1); - K[++round] = s1; - - if (round == 14) - break; - - Vector128 t2 = Aes.KeygenAssist(s1, 0x00); - t2 = Sse2.Shuffle(t2.AsInt32(), 0xAA).AsByte(); - s2 = Sse2.Xor(s2, Sse2.ShiftLeftLogical128BitLane(s2, 8)); - s2 = Sse2.Xor(s2, Sse2.ShiftLeftLogical128BitLane(s2, 4)); - s2 = Sse2.Xor(s2, t2); - K[++round] = s2; - } - - return K; - } - } - - private abstract class AesEncoderDecoder - { - protected readonly Vector128[] _roundKeys; - - public AesEncoderDecoder(Vector128[] roundKeys) - { - _roundKeys = roundKeys; - } - - public static AesEncoderDecoder Init(bool forEncryption, Vector128[] roundKeys) - { - if (roundKeys.Length == 11) - { - return forEncryption ? new Encode128(roundKeys) : new Decode128(roundKeys); - } - else if (roundKeys.Length == 13) - { - return forEncryption ? new Encode192(roundKeys) : new Decode192(roundKeys); - } - else - { - return forEncryption ? new Encode256(roundKeys) : new Decode256(roundKeys); - } - } - - public abstract void ProcessRounds(ref Vector128 state); - - private sealed class Encode128 : AesEncoderDecoder - { - public Encode128(Vector128[] roundKeys) : base(roundKeys) { } - - public override void ProcessRounds(ref Vector128 state) - { - // Take local reference to array so Jit can reason length doesn't change in method - Vector128[] roundKeys = _roundKeys; - { - // Get the Jit to bounds check once rather than each increasing array access - Vector128 temp = roundKeys[10]; - } - - // Operate on non-ref local so it remains in register rather than operating on memory - Vector128 state2 = Sse2.Xor(state, roundKeys[0]); - state2 = Aes.Encrypt(state2, roundKeys[1]); - state2 = Aes.Encrypt(state2, roundKeys[2]); - state2 = Aes.Encrypt(state2, roundKeys[3]); - state2 = Aes.Encrypt(state2, roundKeys[4]); - state2 = Aes.Encrypt(state2, roundKeys[5]); - state2 = Aes.Encrypt(state2, roundKeys[6]); - state2 = Aes.Encrypt(state2, roundKeys[7]); - state2 = Aes.Encrypt(state2, roundKeys[8]); - state2 = Aes.Encrypt(state2, roundKeys[9]); - state2 = Aes.EncryptLast(state2, roundKeys[10]); - // Copy back to ref - state = state2; - } - } - - private sealed class Decode128 : AesEncoderDecoder - { - public Decode128(Vector128[] roundKeys) : base(roundKeys) { } - - public override void ProcessRounds(ref Vector128 state) - { - // Take local reference to array so Jit can reason length doesn't change in method - Vector128[] roundKeys = _roundKeys; - { - // Get the Jit to bounds check once rather than each increasing array access - Vector128 temp = roundKeys[10]; - } - - // Operate on non-ref local so it remains in register rather than operating on memory - Vector128 state2 = Sse2.Xor(state, roundKeys[0]); - state2 = Aes.Decrypt(state2, roundKeys[1]); - state2 = Aes.Decrypt(state2, roundKeys[2]); - state2 = Aes.Decrypt(state2, roundKeys[3]); - state2 = Aes.Decrypt(state2, roundKeys[4]); - state2 = Aes.Decrypt(state2, roundKeys[5]); - state2 = Aes.Decrypt(state2, roundKeys[6]); - state2 = Aes.Decrypt(state2, roundKeys[7]); - state2 = Aes.Decrypt(state2, roundKeys[8]); - state2 = Aes.Decrypt(state2, roundKeys[9]); - state2 = Aes.DecryptLast(state2, roundKeys[10]); - // Copy back to ref - state = state2; - } - } - - private sealed class Encode192 : AesEncoderDecoder - { - public Encode192(Vector128[] roundKeys) : base(roundKeys) { } - - public override void ProcessRounds(ref Vector128 state) - { - // Take local reference to array so Jit can reason length doesn't change in method - Vector128[] roundKeys = _roundKeys; - { - // Get the Jit to bounds check once rather than each increasing array access - Vector128 temp = roundKeys[12]; - } - - // Operate on non-ref local so it remains in register rather than operating on memory - Vector128 state2 = Sse2.Xor(state, roundKeys[0]); - state2 = Aes.Encrypt(state2, roundKeys[1]); - state2 = Aes.Encrypt(state2, roundKeys[2]); - state2 = Aes.Encrypt(state2, roundKeys[3]); - state2 = Aes.Encrypt(state2, roundKeys[4]); - state2 = Aes.Encrypt(state2, roundKeys[5]); - state2 = Aes.Encrypt(state2, roundKeys[6]); - state2 = Aes.Encrypt(state2, roundKeys[7]); - state2 = Aes.Encrypt(state2, roundKeys[8]); - state2 = Aes.Encrypt(state2, roundKeys[9]); - state2 = Aes.Encrypt(state2, roundKeys[10]); - state2 = Aes.Encrypt(state2, roundKeys[11]); - state2 = Aes.EncryptLast(state2, roundKeys[12]); - // Copy back to ref - state = state2; - } - } - - private sealed class Decode192 : AesEncoderDecoder - { - public Decode192(Vector128[] roundKeys) : base(roundKeys) { } - - public override void ProcessRounds(ref Vector128 state) - { - // Take local reference to array so Jit can reason length doesn't change in method - Vector128[] roundKeys = _roundKeys; - { - // Get the Jit to bounds check once rather than each increasing array access - Vector128 temp = roundKeys[12]; - } - - // Operate on non-ref local so it remains in register rather than operating on memory - Vector128 state2 = Sse2.Xor(state, roundKeys[0]); - state2 = Aes.Decrypt(state2, roundKeys[1]); - state2 = Aes.Decrypt(state2, roundKeys[2]); - state2 = Aes.Decrypt(state2, roundKeys[3]); - state2 = Aes.Decrypt(state2, roundKeys[4]); - state2 = Aes.Decrypt(state2, roundKeys[5]); - state2 = Aes.Decrypt(state2, roundKeys[6]); - state2 = Aes.Decrypt(state2, roundKeys[7]); - state2 = Aes.Decrypt(state2, roundKeys[8]); - state2 = Aes.Decrypt(state2, roundKeys[9]); - state2 = Aes.Decrypt(state2, roundKeys[10]); - state2 = Aes.Decrypt(state2, roundKeys[11]); - state2 = Aes.DecryptLast(state2, roundKeys[12]); - // Copy back to ref - state = state2; - } - } - - private sealed class Encode256 : AesEncoderDecoder - { - public Encode256(Vector128[] roundKeys) : base(roundKeys) { } - - public override void ProcessRounds(ref Vector128 state) - { - // Take local reference to array so Jit can reason length doesn't change in method - Vector128[] roundKeys = _roundKeys; - { - // Get the Jit to bounds check once rather than each increasing array access - Vector128 temp = roundKeys[14]; - } - - // Operate on non-ref local so it remains in register rather than operating on memory - Vector128 state2 = Sse2.Xor(state, roundKeys[0]); - state2 = Aes.Encrypt(state2, roundKeys[1]); - state2 = Aes.Encrypt(state2, roundKeys[2]); - state2 = Aes.Encrypt(state2, roundKeys[3]); - state2 = Aes.Encrypt(state2, roundKeys[4]); - state2 = Aes.Encrypt(state2, roundKeys[5]); - state2 = Aes.Encrypt(state2, roundKeys[6]); - state2 = Aes.Encrypt(state2, roundKeys[7]); - state2 = Aes.Encrypt(state2, roundKeys[8]); - state2 = Aes.Encrypt(state2, roundKeys[9]); - state2 = Aes.Encrypt(state2, roundKeys[10]); - state2 = Aes.Encrypt(state2, roundKeys[11]); - state2 = Aes.Encrypt(state2, roundKeys[12]); - state2 = Aes.Encrypt(state2, roundKeys[13]); - state2 = Aes.EncryptLast(state2, roundKeys[14]); - // Copy back to ref - state = state2; - } - } - - private sealed class Decode256 : AesEncoderDecoder - { - public Decode256(Vector128[] roundKeys) : base(roundKeys) { } - - public override void ProcessRounds(ref Vector128 state) - { - // Take local reference to array so Jit can reason length doesn't change in method - Vector128[] roundKeys = _roundKeys; - { - // Get the Jit to bounds check once rather than each increasing array access - Vector128 temp = roundKeys[14]; - } - - // Operate on non-ref local so it remains in register rather than operating on memory - Vector128 state2 = Sse2.Xor(state, roundKeys[0]); - state2 = Aes.Decrypt(state2, roundKeys[1]); - state2 = Aes.Decrypt(state2, roundKeys[2]); - state2 = Aes.Decrypt(state2, roundKeys[3]); - state2 = Aes.Decrypt(state2, roundKeys[4]); - state2 = Aes.Decrypt(state2, roundKeys[5]); - state2 = Aes.Decrypt(state2, roundKeys[6]); - state2 = Aes.Decrypt(state2, roundKeys[7]); - state2 = Aes.Decrypt(state2, roundKeys[8]); - state2 = Aes.Decrypt(state2, roundKeys[9]); - state2 = Aes.Decrypt(state2, roundKeys[10]); - state2 = Aes.Decrypt(state2, roundKeys[11]); - state2 = Aes.Decrypt(state2, roundKeys[12]); - state2 = Aes.Decrypt(state2, roundKeys[13]); - state2 = Aes.DecryptLast(state2, roundKeys[14]); - // Copy back to ref - state = state2; - } - } - } - - private static class Check - { - public static void DataLength(byte[] buf, int off, int len) - { - if (off > (buf.Length - len)) ThrowDataLengthException(); - - static void ThrowDataLengthException() => throw new DataLengthException("input buffer too short"); - } - - public static void OutputLength(byte[] buf, int off, int len) - { - if (off > (buf.Length - len)) ThrowOutputLengthException(); - - static void ThrowOutputLengthException() => throw new OutputLengthException("output buffer too short"); - } - } -} - -#pragma warning restore CA1857 // A constant is expected for the parameter diff --git a/src/Nethermind/Nethermind.Crypto/EciesCipher.cs b/src/Nethermind/Nethermind.Crypto/EciesCipher.cs index 078ccd9666b..7e9f1d89409 100644 --- a/src/Nethermind/Nethermind.Crypto/EciesCipher.cs +++ b/src/Nethermind/Nethermind.Crypto/EciesCipher.cs @@ -6,91 +6,89 @@ using Nethermind.Core.Extensions; using Org.BouncyCastle.Crypto; using Org.BouncyCastle.Crypto.Digests; -using Org.BouncyCastle.Crypto.Engines; using Org.BouncyCastle.Crypto.Macs; using Org.BouncyCastle.Crypto.Modes; using Org.BouncyCastle.Crypto.Parameters; -namespace Nethermind.Crypto +namespace Nethermind.Crypto; + +/// +/// Code adapted from ethereumJ (https://github.com/ethereum/ethereumj) +/// +public class EciesCipher : IEciesCipher { - /// - /// Code adapted from ethereumJ (https://github.com/ethereum/ethereumj) - /// - public class EciesCipher : IEciesCipher - { - private const int KeySize = 128; - private readonly ICryptoRandom _cryptoRandom; - private readonly PrivateKeyGenerator _keyGenerator; + private const int KeySize = 128; + private readonly ICryptoRandom _cryptoRandom; + private readonly PrivateKeyGenerator _keyGenerator; - public EciesCipher(ICryptoRandom cryptoRandom) - { - _cryptoRandom = cryptoRandom; - _keyGenerator = new PrivateKeyGenerator(cryptoRandom); - } + public EciesCipher(ICryptoRandom cryptoRandom) + { + _cryptoRandom = cryptoRandom; + _keyGenerator = new PrivateKeyGenerator(cryptoRandom); + } - private static readonly int ephemBytesLength = 2 * ((BouncyCrypto.DomainParameters.Curve.FieldSize + 7) / 8) + 1; + private static readonly int ephemBytesLength = 2 * ((BouncyCrypto.DomainParameters.Curve.FieldSize + 7) / 8) + 1; - public (bool, byte[]) Decrypt(PrivateKey privateKey, byte[] cipherText, byte[]? macData = null) + public (bool, byte[]) Decrypt(PrivateKey privateKey, byte[] cipherText, byte[]? macData = null) + { + if (cipherText[0] != 4) // if not a compressed public key then probably we need to use EIP8 { - if (cipherText[0] != 4) // if not a compressed public key then probably we need to use EIP8 - { - return (false, null); - } + return (false, null); + } - Span ephemBytes = cipherText.AsSpan(0, ephemBytesLength); - byte[] iv = cipherText.Slice(ephemBytesLength, KeySize / 8); - byte[] cipherBody = cipherText.Slice(ephemBytesLength + KeySize / 8); + Span ephemBytes = cipherText.AsSpan(0, ephemBytesLength); + byte[] iv = cipherText.Slice(ephemBytesLength, KeySize / 8); + byte[] cipherBody = cipherText.Slice(ephemBytesLength + KeySize / 8); - byte[] plaintext = Decrypt(new PublicKey(ephemBytes), privateKey, iv, cipherBody, macData); - return (true, plaintext); - } + byte[] plaintext = Decrypt(new PublicKey(ephemBytes), privateKey, iv, cipherBody, macData); + return (true, plaintext); + } - public byte[] Encrypt(PublicKey recipientPublicKey, byte[] plainText, byte[] macData) - { - byte[] iv = _cryptoRandom.GenerateRandomBytes(KeySize / 8); - PrivateKey ephemeralPrivateKey = _keyGenerator.Generate(); - IIesEngine iesEngine = MakeIesEngine(true, recipientPublicKey, ephemeralPrivateKey, iv); - byte[] cipher = iesEngine.ProcessBlock(plainText, 0, plainText.Length, macData); + public byte[] Encrypt(PublicKey recipientPublicKey, byte[] plainText, byte[] macData) + { + byte[] iv = _cryptoRandom.GenerateRandomBytes(KeySize / 8); + PrivateKey ephemeralPrivateKey = _keyGenerator.Generate(); + IIesEngine iesEngine = MakeIesEngine(true, recipientPublicKey, ephemeralPrivateKey, iv); + byte[] cipher = iesEngine.ProcessBlock(plainText, 0, plainText.Length, macData); - byte[] prefixedBytes = ephemeralPrivateKey.PublicKey.PrefixedBytes; + byte[] prefixedBytes = ephemeralPrivateKey.PublicKey.PrefixedBytes; - byte[] outputArray = new byte[prefixedBytes.Length + iv.Length + cipher.Length]; - Span outputSpan = outputArray; + byte[] outputArray = new byte[prefixedBytes.Length + iv.Length + cipher.Length]; + Span outputSpan = outputArray; - prefixedBytes.AsSpan().CopyTo(outputSpan); - outputSpan = outputSpan[prefixedBytes.Length..]; + prefixedBytes.AsSpan().CopyTo(outputSpan); + outputSpan = outputSpan[prefixedBytes.Length..]; - iv.AsSpan().CopyTo(outputSpan); - outputSpan = outputSpan[iv.Length..]; + iv.AsSpan().CopyTo(outputSpan); + outputSpan = outputSpan[iv.Length..]; - cipher.AsSpan().CopyTo(outputSpan); + cipher.AsSpan().CopyTo(outputSpan); - return outputArray; - } + return outputArray; + } - private readonly OptimizedKdf _optimizedKdf = new(); + private readonly OptimizedKdf _optimizedKdf = new(); - private static byte[] Decrypt(PublicKey ephemeralPublicKey, PrivateKey privateKey, byte[] iv, byte[] ciphertextBody, byte[] macData) - { - IIesEngine iesEngine = MakeIesEngine(false, ephemeralPublicKey, privateKey, iv); - return iesEngine.ProcessBlock(ciphertextBody, 0, ciphertextBody.Length, macData); - } + private static byte[] Decrypt(PublicKey ephemeralPublicKey, PrivateKey privateKey, byte[] iv, byte[] ciphertextBody, byte[] macData) + { + IIesEngine iesEngine = MakeIesEngine(false, ephemeralPublicKey, privateKey, iv); + return iesEngine.ProcessBlock(ciphertextBody, 0, ciphertextBody.Length, macData); + } - private static readonly IesParameters _iesParameters = new IesWithCipherParameters(Array.Empty(), Array.Empty(), KeySize, KeySize); + private static readonly IesParameters _iesParameters = new IesWithCipherParameters(Array.Empty(), Array.Empty(), KeySize, KeySize); - private static IIesEngine MakeIesEngine(bool isEncrypt, PublicKey publicKey, PrivateKey privateKey, byte[] iv) - { - IBlockCipher aesFastEngine = AesEngineX86Intrinsic.IsSupported ? new AesEngineX86Intrinsic() : new AesEngine(); + private static IIesEngine MakeIesEngine(bool isEncrypt, PublicKey publicKey, PrivateKey privateKey, byte[] iv) + { + IBlockCipher aesFastEngine = AesUtilities.CreateEngine(); - EthereumIesEngine iesEngine = new( - new HMac(new Sha256Digest()), - new Sha256Digest(), - new BufferedBlockCipher(new SicBlockCipher(aesFastEngine))); + EthereumIesEngine iesEngine = new( + new HMac(new Sha256Digest()), + new Sha256Digest(), + new BufferedBlockCipher(new SicBlockCipher(aesFastEngine))); - ParametersWithIV parametersWithIV = new(_iesParameters, iv); - byte[] secret = SecP256k1.EcdhSerialized(publicKey.Bytes, privateKey.KeyBytes); - iesEngine.Init(isEncrypt, OptimizedKdf.Derive(secret), parametersWithIV); - return iesEngine; - } + ParametersWithIV parametersWithIV = new(_iesParameters, iv); + byte[] secret = SecP256k1.EcdhSerialized(publicKey.Bytes, privateKey.KeyBytes); + iesEngine.Init(isEncrypt, OptimizedKdf.Derive(secret), parametersWithIV); + return iesEngine; } } diff --git a/src/Nethermind/Nethermind.Crypto/EthereumIesEngine.cs b/src/Nethermind/Nethermind.Crypto/EthereumIesEngine.cs index 5520deab0fc..d50a2631c97 100644 --- a/src/Nethermind/Nethermind.Crypto/EthereumIesEngine.cs +++ b/src/Nethermind/Nethermind.Crypto/EthereumIesEngine.cs @@ -158,7 +158,7 @@ private byte[] DecryptBlock(byte[] inEnc, int inOff, int inLen, byte[]? macData) _mac.DoFinal(t2, 0); - if (!Arrays.ConstantTimeAreEqual(t1, t2)) + if (!Arrays.FixedTimeEquals(t1, t2)) { throw new InvalidCipherTextException("Invalid MAC."); } diff --git a/src/Nethermind/Nethermind.Crypto/KzgPolynomialCommitments.cs b/src/Nethermind/Nethermind.Crypto/KzgPolynomialCommitments.cs index d6abf80311d..758a5fe86e2 100644 --- a/src/Nethermind/Nethermind.Crypto/KzgPolynomialCommitments.cs +++ b/src/Nethermind/Nethermind.Crypto/KzgPolynomialCommitments.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers; using System.IO; using System.Security.Cryptography; using System.Threading; @@ -90,15 +91,35 @@ public static bool VerifyProof(ReadOnlySpan commitment, ReadOnlySpan public static bool AreProofsValid(byte[][] blobs, byte[][] commitments, byte[][] proofs) { - byte[] flatBlobs = new byte[blobs.Length * Ckzg.Ckzg.BytesPerBlob]; - byte[] flatCommitments = new byte[blobs.Length * Ckzg.Ckzg.BytesPerCommitment]; - byte[] flatProofs = new byte[blobs.Length * Ckzg.Ckzg.BytesPerProof]; + if (blobs.Length is 1 && commitments.Length is 1 && proofs.Length is 1) + { + try + { + return Ckzg.Ckzg.VerifyBlobKzgProof(blobs[0], commitments[0], proofs[0], _ckzgSetup); + } + catch (Exception e) when (e is ArgumentException or ApplicationException or InsufficientMemoryException) + { + return false; + } + } + + int length = blobs.Length * Ckzg.Ckzg.BytesPerBlob; + byte[] flatBlobsArray = ArrayPool.Shared.Rent(length); + Span flatBlobs = new(flatBlobsArray, 0, length); + + length = blobs.Length * Ckzg.Ckzg.BytesPerCommitment; + byte[] flatCommitmentsArray = ArrayPool.Shared.Rent(length); + Span flatCommitments = new(flatCommitmentsArray, 0, length); + + length = blobs.Length * Ckzg.Ckzg.BytesPerProof; + byte[] flatProofsArray = ArrayPool.Shared.Rent(length); + Span flatProofs = new(flatProofsArray, 0, length); for (int i = 0; i < blobs.Length; i++) { - Array.Copy(blobs[i], 0, flatBlobs, i * Ckzg.Ckzg.BytesPerBlob, Ckzg.Ckzg.BytesPerBlob); - Array.Copy(commitments[i], 0, flatCommitments, i * Ckzg.Ckzg.BytesPerCommitment, Ckzg.Ckzg.BytesPerCommitment); - Array.Copy(proofs[i], 0, flatProofs, i * Ckzg.Ckzg.BytesPerProof, Ckzg.Ckzg.BytesPerProof); + blobs[i].CopyTo(flatBlobs.Slice(i * Ckzg.Ckzg.BytesPerBlob, Ckzg.Ckzg.BytesPerBlob)); + commitments[i].CopyTo(flatCommitments.Slice(i * Ckzg.Ckzg.BytesPerCommitment, Ckzg.Ckzg.BytesPerCommitment)); + proofs[i].CopyTo(flatProofs.Slice(i * Ckzg.Ckzg.BytesPerProof, Ckzg.Ckzg.BytesPerProof)); } try @@ -110,10 +131,16 @@ public static bool AreProofsValid(byte[][] blobs, byte[][] commitments, byte[][] { return false; } + finally + { + ArrayPool.Shared.Return(flatBlobsArray); + ArrayPool.Shared.Return(flatCommitmentsArray); + ArrayPool.Shared.Return(flatProofsArray); + } } /// - /// Method to genereate correct data for tests only, not safe + /// Method to generate correct data for tests only, not safe /// public static void KzgifyBlob(ReadOnlySpan blob, Span commitment, Span proof, Span hashV1) { diff --git a/src/Nethermind/Nethermind.Crypto/Nethermind.Crypto.csproj b/src/Nethermind/Nethermind.Crypto/Nethermind.Crypto.csproj index 0c5702026d5..cbd9d76af1a 100644 --- a/src/Nethermind/Nethermind.Crypto/Nethermind.Crypto.csproj +++ b/src/Nethermind/Nethermind.Crypto/Nethermind.Crypto.csproj @@ -11,10 +11,10 @@ + - diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs index c4d54c4ece6..b18a43b70b5 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs @@ -26,7 +26,7 @@ public class DbConfig : IDbConfig public bool? DisableCompression { get; set; } = false; public bool? UseLz4 { get; set; } = false; public ulong? CompactionReadAhead { get; set; } = (ulong)256.KiB(); - public IDictionary? AdditionalRocksDbOptions { get; set; } + public string? AdditionalRocksDbOptions { get; set; } public ulong? MaxBytesForLevelBase { get; set; } = (ulong)256.MiB(); public ulong TargetFileSizeBase { get; set; } = (ulong)64.MiB(); public int TargetFileSizeMultiplier { get; set; } = 1; @@ -36,7 +36,7 @@ public class DbConfig : IDbConfig public bool AllowMmapReads { get; set; } = false; public bool? VerifyChecksum { get; set; } = true; public double MaxBytesForLevelMultiplier { get; set; } = 10; - public ulong? MaxCompactionBytes { get; set; } = null; + public ulong? MaxCompactionBytes { get; set; } = (ulong)4.GiB(); public int MinWriteBufferNumberToMerge { get; set; } = 1; public ulong? RowCacheSize { get; set; } = null; public bool OptimizeFiltersForHits { get; set; } = true; @@ -51,6 +51,10 @@ public class DbConfig : IDbConfig public int? UseRibbonFilterStartingFromLevel { get; set; } public ulong BytesPerSync { get; set; } = 0; public double? DataBlockIndexUtilRatio { get; set; } + public bool EnableFileWarmer { get; set; } = false; + public double CompressibilityHint { get; set; } = 1.0; + + public ulong BlobTransactionsDbBlockCacheSize { get; set; } = (ulong)32.MiB(); public ulong ReceiptsDbWriteBufferSize { get; set; } = (ulong)2.MiB(); public uint ReceiptsDbWriteBufferNumber { get; set; } = 2; @@ -62,8 +66,9 @@ public class DbConfig : IDbConfig public bool? ReceiptsDbUseDirectReads { get; set; } public bool? ReceiptsDbUseDirectIoForFlushAndCompactions { get; set; } public ulong? ReceiptsDbCompactionReadAhead { get; set; } - public ulong ReceiptsDbTargetFileSizeBase { get; set; } = (ulong)256.MiB(); - public IDictionary? ReceiptsDbAdditionalRocksDbOptions { get; set; } + public ulong ReceiptsDbTargetFileSizeBase { get; set; } = (ulong)64.MiB(); + public double ReceiptsDbCompressibilityHint { get; set; } = 0.35; + public string? ReceiptsDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst"; public ulong BlocksDbWriteBufferSize { get; set; } = (ulong)64.MiB(); public uint BlocksDbWriteBufferNumber { get; set; } = 2; @@ -75,7 +80,7 @@ public class DbConfig : IDbConfig public bool? BlocksDbUseDirectReads { get; set; } public bool? BlocksDbUseDirectIoForFlushAndCompactions { get; set; } public ulong? BlocksDbCompactionReadAhead { get; set; } - public IDictionary? BlocksDbAdditionalRocksDbOptions { get; set; } + public string? BlocksDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst"; public ulong HeadersDbWriteBufferSize { get; set; } = (ulong)8.MiB(); public uint HeadersDbWriteBufferNumber { get; set; } = 2; @@ -87,7 +92,7 @@ public class DbConfig : IDbConfig public bool? HeadersDbUseDirectReads { get; set; } public bool? HeadersDbUseDirectIoForFlushAndCompactions { get; set; } public ulong? HeadersDbCompactionReadAhead { get; set; } - public IDictionary? HeadersDbAdditionalRocksDbOptions { get; set; } + public string? HeadersDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst"; public ulong? HeadersDbMaxBytesForLevelBase { get; set; } = (ulong)128.MiB(); public ulong BlockNumbersDbWriteBufferSize { get; set; } = (ulong)8.MiB(); @@ -103,7 +108,7 @@ public class DbConfig : IDbConfig public bool? BlockNumbersDbUseDirectReads { get; set; } public bool? BlockNumbersDbUseDirectIoForFlushAndCompactions { get; set; } public ulong? BlockNumbersDbCompactionReadAhead { get; set; } - public IDictionary? BlockNumbersDbAdditionalRocksDbOptions { get; set; } + public string? BlockNumbersDbAdditionalRocksDbOptions { get; set; } public ulong? BlockNumbersDbMaxBytesForLevelBase { get; set; } = (ulong)16.MiB(); public ulong BlockInfosDbWriteBufferSize { get; set; } = (ulong)4.MiB(); @@ -116,7 +121,7 @@ public class DbConfig : IDbConfig public bool? BlockInfosDbUseDirectReads { get; set; } public bool? BlockInfosDbUseDirectIoForFlushAndCompactions { get; set; } public ulong? BlockInfosDbCompactionReadAhead { get; set; } - public IDictionary? BlockInfosDbAdditionalRocksDbOptions { get; set; } + public string? BlockInfosDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst"; public ulong PendingTxsDbWriteBufferSize { get; set; } = (ulong)4.MiB(); public uint PendingTxsDbWriteBufferNumber { get; set; } = 4; @@ -128,7 +133,7 @@ public class DbConfig : IDbConfig public bool? PendingTxsDbUseDirectReads { get; set; } public bool? PendingTxsDbUseDirectIoForFlushAndCompactions { get; set; } public ulong? PendingTxsDbCompactionReadAhead { get; set; } - public IDictionary? PendingTxsDbAdditionalRocksDbOptions { get; set; } + public string? PendingTxsDbAdditionalRocksDbOptions { get; set; } public ulong CodeDbWriteBufferSize { get; set; } = (ulong)1.MiB(); public uint CodeDbWriteBufferNumber { get; set; } = 2; @@ -143,7 +148,7 @@ public class DbConfig : IDbConfig public bool? CodeUseDirectReads { get; set; } public bool? CodeUseDirectIoForFlushAndCompactions { get; set; } public ulong? CodeCompactionReadAhead { get; set; } - public IDictionary? CodeDbAdditionalRocksDbOptions { get; set; } + public string? CodeDbAdditionalRocksDbOptions { get; set; } public ulong BloomDbWriteBufferSize { get; set; } = (ulong)1.KiB(); public uint BloomDbWriteBufferNumber { get; set; } = 4; @@ -151,32 +156,7 @@ public class DbConfig : IDbConfig public bool BloomDbCacheIndexAndFilterBlocks { get; set; } = false; public int? BloomDbMaxOpenFiles { get; set; } public long? BloomDbMaxBytesPerSec { get; set; } - public IDictionary? BloomDbAdditionalRocksDbOptions { get; set; } - - public ulong WitnessDbWriteBufferSize { get; set; } = (ulong)1.KiB(); - public uint WitnessDbWriteBufferNumber { get; set; } = 4; - public ulong WitnessDbBlockCacheSize { get; set; } = 0; - public bool WitnessDbCacheIndexAndFilterBlocks { get; set; } = false; - public int? WitnessDbMaxOpenFiles { get; set; } - public long? WitnessDbMaxBytesPerSec { get; set; } - public int? WitnessDbBlockSize { get; set; } - public bool? WitnessUseDirectReads { get; set; } - public bool? WitnessUseDirectIoForFlushAndCompactions { get; set; } - public ulong? WitnessCompactionReadAhead { get; set; } - public IDictionary? WitnessDbAdditionalRocksDbOptions { get; set; } - - // TODO - profile and customize - public ulong CanonicalHashTrieDbWriteBufferSize { get; set; } = (ulong)2.MB(); - public uint CanonicalHashTrieDbWriteBufferNumber { get; set; } = 4; - public ulong CanonicalHashTrieDbBlockCacheSize { get; set; } = 0; - public bool CanonicalHashTrieDbCacheIndexAndFilterBlocks { get; set; } = false; - public int? CanonicalHashTrieDbMaxOpenFiles { get; set; } - public long? CanonicalHashTrieDbMaxBytesPerSec { get; set; } - public int? CanonicalHashTrieDbBlockSize { get; set; } - public bool? CanonicalHashTrieUseDirectReads { get; set; } - public bool? CanonicalHashTrieUseDirectIoForFlushAndCompactions { get; set; } - public ulong? CanonicalHashTrieCompactionReadAhead { get; set; } - public IDictionary? CanonicalHashTrieDbAdditionalRocksDbOptions { get; set; } + public string? BloomDbAdditionalRocksDbOptions { get; set; } public ulong MetadataDbWriteBufferSize { get; set; } = (ulong)1.KiB(); public uint MetadataDbWriteBufferNumber { get; set; } = 4; @@ -188,7 +168,7 @@ public class DbConfig : IDbConfig public bool? MetadataUseDirectReads { get; set; } public bool? MetadataUseDirectIoForFlushAndCompactions { get; set; } public ulong? MetadataCompactionReadAhead { get; set; } - public IDictionary? MetadataDbAdditionalRocksDbOptions { get; set; } + public string? MetadataDbAdditionalRocksDbOptions { get; set; } public ulong StateDbWriteBufferSize { get; set; } = (ulong)64.MB(); public uint StateDbWriteBufferNumber { get; set; } = 4; @@ -223,7 +203,9 @@ public class DbConfig : IDbConfig public int? StateDbBloomFilterBitsPerKey { get; set; } = 15; public int? StateDbUseRibbonFilterStartingFromLevel { get; set; } = 2; public double? StateDbDataBlockIndexUtilRatio { get; set; } = 0.5; - public IDictionary? StateDbAdditionalRocksDbOptions { get; set; } + public bool StateDbEnableFileWarmer { get; set; } = false; + public double StateDbCompressibilityHint { get; set; } = 0.45; + public string? StateDbAdditionalRocksDbOptions { get; set; } public uint RecycleLogFileNum { get; set; } = 0; public bool WriteAheadLogSync { get; set; } = false; diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs index 8cb1fa84edb..b19730fdf50 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs @@ -27,7 +27,7 @@ public interface IDbConfig : IConfig bool? DisableCompression { get; set; } bool? UseLz4 { get; set; } ulong? CompactionReadAhead { get; set; } - IDictionary? AdditionalRocksDbOptions { get; set; } + string? AdditionalRocksDbOptions { get; set; } ulong? MaxBytesForLevelBase { get; set; } ulong TargetFileSizeBase { get; set; } int TargetFileSizeMultiplier { get; set; } @@ -52,6 +52,10 @@ public interface IDbConfig : IConfig int? UseRibbonFilterStartingFromLevel { get; set; } ulong BytesPerSync { get; set; } double? DataBlockIndexUtilRatio { get; set; } + bool EnableFileWarmer { get; set; } + double CompressibilityHint { get; set; } + + ulong BlobTransactionsDbBlockCacheSize { get; set; } ulong ReceiptsDbWriteBufferSize { get; set; } uint ReceiptsDbWriteBufferNumber { get; set; } @@ -64,7 +68,8 @@ public interface IDbConfig : IConfig bool? ReceiptsDbUseDirectIoForFlushAndCompactions { get; set; } ulong? ReceiptsDbCompactionReadAhead { get; set; } ulong ReceiptsDbTargetFileSizeBase { get; set; } - IDictionary? ReceiptsDbAdditionalRocksDbOptions { get; set; } + double ReceiptsDbCompressibilityHint { get; set; } + string? ReceiptsDbAdditionalRocksDbOptions { get; set; } ulong BlocksDbWriteBufferSize { get; set; } uint BlocksDbWriteBufferNumber { get; set; } @@ -76,7 +81,7 @@ public interface IDbConfig : IConfig bool? BlocksDbUseDirectReads { get; set; } bool? BlocksDbUseDirectIoForFlushAndCompactions { get; set; } ulong? BlocksDbCompactionReadAhead { get; set; } - IDictionary? BlocksDbAdditionalRocksDbOptions { get; set; } + string? BlocksDbAdditionalRocksDbOptions { get; set; } ulong HeadersDbWriteBufferSize { get; set; } uint HeadersDbWriteBufferNumber { get; set; } @@ -88,7 +93,7 @@ public interface IDbConfig : IConfig bool? HeadersDbUseDirectReads { get; set; } bool? HeadersDbUseDirectIoForFlushAndCompactions { get; set; } ulong? HeadersDbCompactionReadAhead { get; set; } - IDictionary? HeadersDbAdditionalRocksDbOptions { get; set; } + string? HeadersDbAdditionalRocksDbOptions { get; set; } ulong? HeadersDbMaxBytesForLevelBase { get; set; } ulong BlockNumbersDbWriteBufferSize { get; set; } @@ -104,7 +109,7 @@ public interface IDbConfig : IConfig bool? BlockNumbersDbUseDirectReads { get; set; } bool? BlockNumbersDbUseDirectIoForFlushAndCompactions { get; set; } ulong? BlockNumbersDbCompactionReadAhead { get; set; } - IDictionary? BlockNumbersDbAdditionalRocksDbOptions { get; set; } + string? BlockNumbersDbAdditionalRocksDbOptions { get; set; } ulong? BlockNumbersDbMaxBytesForLevelBase { get; set; } ulong BlockInfosDbWriteBufferSize { get; set; } @@ -117,7 +122,7 @@ public interface IDbConfig : IConfig bool? BlockInfosDbUseDirectReads { get; set; } bool? BlockInfosDbUseDirectIoForFlushAndCompactions { get; set; } ulong? BlockInfosDbCompactionReadAhead { get; set; } - IDictionary? BlockInfosDbAdditionalRocksDbOptions { get; set; } + string? BlockInfosDbAdditionalRocksDbOptions { get; set; } ulong PendingTxsDbWriteBufferSize { get; set; } uint PendingTxsDbWriteBufferNumber { get; set; } @@ -129,7 +134,7 @@ public interface IDbConfig : IConfig bool? PendingTxsDbUseDirectReads { get; set; } bool? PendingTxsDbUseDirectIoForFlushAndCompactions { get; set; } ulong? PendingTxsDbCompactionReadAhead { get; set; } - IDictionary? PendingTxsDbAdditionalRocksDbOptions { get; set; } + string? PendingTxsDbAdditionalRocksDbOptions { get; set; } ulong CodeDbWriteBufferSize { get; set; } uint CodeDbWriteBufferNumber { get; set; } @@ -144,7 +149,7 @@ public interface IDbConfig : IConfig bool? CodeUseDirectReads { get; set; } bool? CodeUseDirectIoForFlushAndCompactions { get; set; } ulong? CodeCompactionReadAhead { get; set; } - IDictionary? CodeDbAdditionalRocksDbOptions { get; set; } + string? CodeDbAdditionalRocksDbOptions { get; set; } ulong BloomDbWriteBufferSize { get; set; } uint BloomDbWriteBufferNumber { get; set; } @@ -152,31 +157,7 @@ public interface IDbConfig : IConfig bool BloomDbCacheIndexAndFilterBlocks { get; set; } int? BloomDbMaxOpenFiles { get; set; } long? BloomDbMaxBytesPerSec { get; set; } - IDictionary? BloomDbAdditionalRocksDbOptions { get; set; } - - ulong WitnessDbWriteBufferSize { get; set; } - uint WitnessDbWriteBufferNumber { get; set; } - ulong WitnessDbBlockCacheSize { get; set; } - bool WitnessDbCacheIndexAndFilterBlocks { get; set; } - int? WitnessDbMaxOpenFiles { get; set; } - long? WitnessDbMaxBytesPerSec { get; set; } - int? WitnessDbBlockSize { get; set; } - bool? WitnessUseDirectReads { get; set; } - bool? WitnessUseDirectIoForFlushAndCompactions { get; set; } - ulong? WitnessCompactionReadAhead { get; set; } - IDictionary? WitnessDbAdditionalRocksDbOptions { get; set; } - - ulong CanonicalHashTrieDbWriteBufferSize { get; set; } - uint CanonicalHashTrieDbWriteBufferNumber { get; set; } - ulong CanonicalHashTrieDbBlockCacheSize { get; set; } - bool CanonicalHashTrieDbCacheIndexAndFilterBlocks { get; set; } - int? CanonicalHashTrieDbMaxOpenFiles { get; set; } - long? CanonicalHashTrieDbMaxBytesPerSec { get; set; } - int? CanonicalHashTrieDbBlockSize { get; set; } - bool? CanonicalHashTrieUseDirectReads { get; set; } - bool? CanonicalHashTrieUseDirectIoForFlushAndCompactions { get; set; } - ulong? CanonicalHashTrieCompactionReadAhead { get; set; } - IDictionary? CanonicalHashTrieDbAdditionalRocksDbOptions { get; set; } + string? BloomDbAdditionalRocksDbOptions { get; set; } ulong MetadataDbWriteBufferSize { get; set; } uint MetadataDbWriteBufferNumber { get; set; } @@ -188,7 +169,7 @@ public interface IDbConfig : IConfig bool? MetadataUseDirectReads { get; set; } bool? MetadataUseDirectIoForFlushAndCompactions { get; set; } ulong? MetadataCompactionReadAhead { get; set; } - IDictionary? MetadataDbAdditionalRocksDbOptions { get; set; } + string? MetadataDbAdditionalRocksDbOptions { get; set; } ulong StateDbWriteBufferSize { get; set; } uint StateDbWriteBufferNumber { get; set; } @@ -223,7 +204,9 @@ public interface IDbConfig : IConfig int? StateDbBloomFilterBitsPerKey { get; set; } int? StateDbUseRibbonFilterStartingFromLevel { get; set; } double? StateDbDataBlockIndexUtilRatio { get; set; } - IDictionary? StateDbAdditionalRocksDbOptions { get; set; } + bool StateDbEnableFileWarmer { get; set; } + double StateDbCompressibilityHint { get; set; } + string? StateDbAdditionalRocksDbOptions { get; set; } /// /// Enables DB Statistics - https://github.com/facebook/rocksdb/wiki/Statistics diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs index 9628a384789..5ca7bb5a6de 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Generic; using System.IO; using System.Reflection; using Nethermind.Core.Extensions; @@ -12,6 +11,7 @@ namespace Nethermind.Db.Rocks.Config; public class PerTableDbConfig { private readonly string _tableName; + private readonly string? _columnName; private readonly IDbConfig _dbConfig; private readonly DbSettings _settings; @@ -20,10 +20,7 @@ public PerTableDbConfig(IDbConfig dbConfig, DbSettings dbSettings, string? colum _dbConfig = dbConfig; _settings = dbSettings; _tableName = _settings.DbName; - if (columnName is not null) - { - _tableName += columnName; - } + _columnName = columnName; } public bool CacheIndexAndFilterBlocks => _settings.CacheIndexAndFilterBlocks ?? ReadConfig(nameof(CacheIndexAndFilterBlocks)); @@ -34,7 +31,7 @@ public PerTableDbConfig(IDbConfig dbConfig, DbSettings dbSettings, string? colum public ulong WriteBufferNumber => _settings.WriteBufferNumber ?? ReadConfig(nameof(WriteBufferNumber)); - public IDictionary? AdditionalRocksDbOptions => ReadConfig?>(nameof(AdditionalRocksDbOptions)); + public string? AdditionalRocksDbOptions => ReadConfig(nameof(AdditionalRocksDbOptions)); public int? MaxOpenFiles => ReadConfig(nameof(MaxOpenFiles)); public long? MaxBytesPerSec => ReadConfig(nameof(MaxBytesPerSec)); @@ -73,46 +70,74 @@ public PerTableDbConfig(IDbConfig dbConfig, DbSettings dbSettings, string? colum public int? UseRibbonFilterStartingFromLevel => ReadConfig(nameof(UseRibbonFilterStartingFromLevel)); public ulong BytesPerSync => ReadConfig(nameof(BytesPerSync)); public double? DataBlockIndexUtilRatio => ReadConfig(nameof(DataBlockIndexUtilRatio)); + public bool EnableFileWarmer => ReadConfig(nameof(EnableFileWarmer)); + public double CompressibilityHint => ReadConfig(nameof(CompressibilityHint)); private T? ReadConfig(string propertyName) { - return ReadConfig(_dbConfig, propertyName, GetPrefix()); + return ReadConfig(_dbConfig, propertyName, GetPrefixes()); } - private string GetPrefix() + private string[] GetPrefixes() { - return _tableName.StartsWith("State") ? "StateDb" : string.Concat(_tableName, "Db"); + if (_tableName.StartsWith("State")) + { + return ["StateDb"]; + } + + if (_columnName != null) + { + return [ + string.Concat(_tableName, _columnName, "Db"), + string.Concat(_tableName, "Db"), + ]; + } + + return [string.Concat(_tableName, "Db")]; } - private static T? ReadConfig(IDbConfig dbConfig, string propertyName, string prefix) + private static T? ReadConfig(IDbConfig dbConfig, string propertyName, string[] prefixes) { - string prefixed = string.Concat(prefix, propertyName); - try { Type type = dbConfig.GetType(); - PropertyInfo? propertyInfo = type.GetProperty(prefixed, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + PropertyInfo? propertyInfo; - if (propertyInfo is not null && propertyInfo.PropertyType.CanBeAssignedNull()) + foreach (var prefix in prefixes) { - // If its nullable check if its null first - T? val = (T?)propertyInfo?.GetValue(dbConfig); - if (val is not null) + string prefixed = string.Concat(prefix, propertyName); + + propertyInfo = type.GetProperty(prefixed, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + if (propertyInfo is not null) { - return val; + if (propertyInfo.PropertyType.CanBeAssignedNull()) + { + // If its nullable check if its null first + object? valObj = propertyInfo.GetValue(dbConfig); + if (valObj is not null) + { + T? val = (T?)valObj; + if (val is not null) + { + return val; + } + } + } + else + { + // If not nullable just use it directly + return (T?)propertyInfo.GetValue(dbConfig); + } } - - // Use generic one even if its available - propertyInfo = type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); } - // if no custom db property default to generic one - propertyInfo ??= type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + // Use generic one even if its available + propertyInfo = type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); return (T?)propertyInfo?.GetValue(dbConfig); } catch (Exception e) { - throw new InvalidDataException($"Unable to read {prefixed} property from DB config", e); + throw new InvalidDataException($"Unable to read property from DB config. Prefixes: ${prefixes}", e); } } } diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index 6be2d3c4abe..8b5ac589718 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -7,8 +7,12 @@ using System.Collections.Generic; using System.IO; using System.IO.Abstractions; +using System.Linq; using System.Reflection; +using System.Runtime.InteropServices; using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; using ConcurrentCollections; using Nethermind.Config; using Nethermind.Core; @@ -96,12 +100,6 @@ public DbOnTheRocks( _rocksDbNative = rocksDbNative ?? RocksDbSharp.Native.Instance; _perTableDbConfig = new PerTableDbConfig(dbConfig, _settings); _db = Init(basePath, dbSettings.DbPath, dbConfig, logManager, columnFamilies, dbSettings.DeleteOnStart, sharedCache); - - if (_perTableDbConfig.AdditionalRocksDbOptions is not null) - { - ApplyOptions(_perTableDbConfig.AdditionalRocksDbOptions); - } - _iteratorManager = new IteratorManager(_db, null, _readAheadReadOptions); } @@ -183,6 +181,11 @@ private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogMan } } + if (_perTableDbConfig.EnableFileWarmer) + { + WarmupFile(_fullPath, db); + } + return db; } catch (DllNotFoundException e) when (e.Message.Contains("libdl")) @@ -202,6 +205,81 @@ private RocksDb Init(string basePath, string dbPath, IDbConfig dbConfig, ILogMan } + private void WarmupFile(string basePath, RocksDb db) + { + long availableMemory = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes; + _logger.Info($"Warming up database {Name} assuming {availableMemory} bytes of available memory"); + List<(FileMetadata metadata, DateTime creationTime)> fileMetadatas = new(); + + foreach (LiveFileMetadata liveFileMetadata in db.GetLiveFilesMetadata()) + { + string fullPath = Path.Join(basePath, liveFileMetadata.FileMetadata.FileName); + try + { + DateTime creationTime = File.GetCreationTimeUtc(fullPath); + fileMetadatas.Add((liveFileMetadata.FileMetadata, creationTime)); + } + catch (IOException) + { + // Maybe the file is gone or something. We ignore it. + } + } + + fileMetadatas.Sort((item1, item2) => + { + // Sort them by level so that lower level get priority + int levelDiff = item1.metadata.FileLevel - item2.metadata.FileLevel; + if (levelDiff != 0) return levelDiff; + + // Otherwise, we pick which file is newest. + return item2.creationTime.CompareTo(item1.creationTime); + }); + + long totalSize = 0; + fileMetadatas = fileMetadatas.TakeWhile(metadata => + { + availableMemory -= (long)metadata.metadata.FileSize; + bool take = availableMemory > 0; + if (take) + { + totalSize += (long)metadata.metadata.FileSize; + } + return take; + }) + // We reverse them again so that lower level goes last so that it is the freshest. + // Not all of the available memory is actually available so we are probably over reading things. + .Reverse() + .ToList(); + + long totalRead = 0; + Parallel.ForEach(fileMetadatas, (task) => + { + string fullPath = Path.Join(basePath, task.metadata.FileName); + _logger.Info($"{(totalRead * 100 / (double)totalSize):00.00}% Warming up file {fullPath}"); + + try + { + byte[] buffer = new byte[512.KiB()]; + using FileStream stream = File.OpenRead(fullPath); + int readCount = buffer.Length; + while (readCount == buffer.Length) + { + readCount = stream.Read(buffer); + Interlocked.Add(ref totalRead, readCount); + } + } + catch (FileNotFoundException) + { + // Happens sometimes. We do nothing here. + } + catch (IOException e) + { + // Something unusual, but nothing noteworthy. + _logger.Warn($"Exception warming up {fullPath} {e}"); + } + }); + } + private void CreateMarkerIfCorrupt(RocksDbSharpException rocksDbException) { if (rocksDbException.Message.Contains("Corruption:")) @@ -625,6 +703,20 @@ protected virtual void BuildOptions(PerTableDbConfig dbConfig, Options opt options.EnableStatistics(); } options.SetStatsDumpPeriodSec(dbConfig.StatsDumpPeriodSec); + + if (dbConfig.AdditionalRocksDbOptions is not null) + { + IntPtr optsPtr = Marshal.StringToHGlobalAnsi(dbConfig.AdditionalRocksDbOptions); + try + { + _rocksDbNative.rocksdb_get_options_from_string(options.Handle, optsPtr, options.Handle); + } + finally + { + Marshal.FreeHGlobal(optsPtr); + } + } + #endregion #region read-write options @@ -1408,7 +1500,7 @@ public virtual void Tune(ITunableDb.TuneType type) case ITunableDb.TuneType.HeavyWrite: // Compaction spikes are clear at this point. Will definitely affect attestation performance. // Its unclear if it improve or slow down sync time. Seems to be the sweet spot. - ApplyOptions(GetHeavyWriteOptions((ulong)4.GiB())); + ApplyOptions(GetHeavyWriteOptions((ulong)2.GiB())); break; case ITunableDb.TuneType.AggressiveHeavyWrite: // For when, you are desperate, but don't wanna disable compaction completely, because you don't want @@ -1517,8 +1609,10 @@ private IDictionary GetHeavyWriteOptions(ulong l0SizeTarget) // but no io, only cpu. // bufferSize*maxBufferNumber = 128MB, which is the max memory used, which tend to be the case as its now // stalled by compaction instead of flush. - ulong bufferSize = (ulong)16.MiB(); - ulong l0FileSize = bufferSize * (ulong)_perTableDbConfig.MinWriteBufferNumberToMerge; + // The buffer is not compressed unlike l0File, so to account for it, its size need to be slightly larger. + ulong targetFileSize = (ulong)16.MiB(); + ulong bufferSize = (ulong)(targetFileSize / _perTableDbConfig.CompressibilityHint); + ulong l0FileSize = targetFileSize * (ulong)_perTableDbConfig.MinWriteBufferNumberToMerge; ulong maxBufferNumber = 8; // Guide recommend to have l0 and l1 to be the same size. They have to be compacted together so if l1 is larger, @@ -1680,7 +1774,7 @@ private void Return(Iterator iterator, ReadFlags flags) holder.Usage++; Iterator? oldIterator = Interlocked.Exchange(ref holder.Iterator, iterator); - if (oldIterator != null) + if (oldIterator is not null) { // Well... this is weird. I'll just dispose it. oldIterator.Dispose(); diff --git a/src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs b/src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs index 75e9b29a07e..d9d65b1d024 100644 --- a/src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs @@ -13,8 +13,8 @@ public class PerTableDbConfigTests public void CanReadAllConfigForAllTable() { DbConfig dbConfig = new DbConfig(); - string[] tables = new[] - { + string[] tables = + [ DbNames.Storage, DbNames.State, DbNames.Code, @@ -23,10 +23,8 @@ public void CanReadAllConfigForAllTable() DbNames.Receipts, DbNames.BlockInfos, DbNames.Bloom, - DbNames.Witness, - DbNames.CHT, - DbNames.Metadata, - }; + DbNames.Metadata + ]; foreach (string table in tables) { @@ -40,6 +38,17 @@ public void CanReadAllConfigForAllTable() } } + [Test] + public void When_ColumnDb_UsePerTableConfig() + { + DbConfig dbConfig = new DbConfig(); + dbConfig.MaxOpenFiles = 2; + dbConfig.ReceiptsDbMaxOpenFiles = 3; + + PerTableDbConfig config = new PerTableDbConfig(dbConfig, new DbSettings(DbNames.Receipts, ""), "Blocks"); + config.MaxOpenFiles.Should().Be(3); + } + [Test] public void When_PerTableConfigIsAvailable_UsePerTableConfig() { diff --git a/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs b/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs index 359f6a3bd48..7b7ff34f41c 100644 --- a/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/DbOnTheRocksTests.cs @@ -19,6 +19,7 @@ using Nethermind.Db.Rocks.Config; using Nethermind.Logging; using NSubstitute; +using NSubstitute.ExceptionExtensions; using NUnit.Framework; using RocksDbSharp; using IWriteBatch = Nethermind.Core.IWriteBatch; @@ -47,7 +48,7 @@ public void TearDown() public void WriteOptions_is_correct() { IDbConfig config = new DbConfig(); - DbOnTheRocks db = new(DbPath, GetRocksDbSettings(DbPath, "Blocks"), config, LimboLogs.Instance); + using DbOnTheRocks db = new(DbPath, GetRocksDbSettings(DbPath, "Blocks"), config, LimboLogs.Instance); WriteOptions? options = db.WriteFlagsToWriteOptions(WriteFlags.LowPriority); Native.Instance.rocksdb_writeoptions_get_low_pri(options.Handle).Should().BeTrue(); @@ -129,6 +130,47 @@ public void Dispose_wont_cause_ObjectDisposedException_when_batch_is_still_open( db.Dispose(); } + [Test] + public void CanOpenWithFileWarmer() + { + IDbConfig config = new DbConfig(); + config.EnableFileWarmer = true; + { + using DbOnTheRocks db = new("testFileWarmer", GetRocksDbSettings("testFileWarmer", "FileWarmerTest"), config, LimboLogs.Instance); + for (int i = 0; i < 1000; i++) + { + db[i.ToBigEndianByteArray()] = i.ToBigEndianByteArray(); + } + } + + { + using DbOnTheRocks db = new("testFileWarmer", GetRocksDbSettings("testFileWarmer", "FileWarmerTest"), config, LimboLogs.Instance); + } + } + + [TestCase("compaction_pri=kByCompensatedSize", true)] + [TestCase("compaction_pri=kByCompensatedSize;num_levels=4", true)] + [TestCase("compaction_pri=kSomethingElse", false)] + public void CanOpenWithAdditionalConfig(string opts, bool success) + { + IDbConfig config = new DbConfig(); + config.AdditionalRocksDbOptions = opts; + + Action act = () => + { + using DbOnTheRocks db = new("testFileWarmer", GetRocksDbSettings("testFileWarmer", "FileWarmerTest"), config, LimboLogs.Instance); + }; + + if (success) + { + act.Should().NotThrow(); + } + else + { + act.Should().Throw(); + } + } + [Test] public void Corrupted_exception_on_open_would_create_marker() { diff --git a/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs b/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs index 7c5dd5f1582..6b927954aae 100644 --- a/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs @@ -85,7 +85,6 @@ private void AssertStandardDbs(IDbProvider dbProvider, Type dbType, Type receipt dbProvider.BlockInfosDb.Should().BeOfType(dbType); dbProvider.BlocksDb.Should().BeOfType(dbType); dbProvider.BloomDb.Should().BeOfType(dbType); - dbProvider.ChtDb.Should().BeOfType(dbType); dbProvider.HeadersDb.Should().BeOfType(dbType); dbProvider.ReceiptsDb.Should().BeOfType(receiptsDb); dbProvider.CodeDb.Should().BeOfType(dbType); diff --git a/src/Nethermind/Nethermind.Db/DbNames.cs b/src/Nethermind/Nethermind.Db/DbNames.cs index c6004120c0b..7b8459430f9 100644 --- a/src/Nethermind/Nethermind.Db/DbNames.cs +++ b/src/Nethermind/Nethermind.Db/DbNames.cs @@ -15,8 +15,6 @@ public static class DbNames public const string BlockInfos = "blockInfos"; public const string BadBlocks = "badBlocks"; public const string Bloom = "bloom"; - public const string Witness = "witness"; - public const string CHT = "canonicalHashTrie"; public const string Metadata = "metadata"; public const string BlobTransactions = "blobTransactions"; } diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index b54121bf3a7..7786271bc74 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -20,11 +20,6 @@ public interface IDbProvider : IDisposable // BloomDB progress / config (does not contain blooms - they are kept in bloom storage) public IDb BloomDb => GetDb(DbNames.Bloom); - // LES (ignore) - public IDb ChtDb => GetDb(DbNames.CHT); - - public IDb WitnessDb => GetDb(DbNames.Witness); - public IDb MetadataDb => GetDb(DbNames.Metadata); public IColumnsDb BlobTransactionsDb => GetColumnDb(DbNames.BlobTransactions); diff --git a/src/Nethermind/Nethermind.Db/Metrics.cs b/src/Nethermind/Nethermind.Db/Metrics.cs index 2d9a7c5ad0e..7a2fdb8dc06 100644 --- a/src/Nethermind/Nethermind.Db/Metrics.cs +++ b/src/Nethermind/Nethermind.Db/Metrics.cs @@ -1,9 +1,9 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.Collections.Concurrent; -using System.Collections.Generic; using System.ComponentModel; +using System.Threading; + using Nethermind.Core.Attributes; namespace Nethermind.Db @@ -12,11 +12,76 @@ public static class Metrics { [CounterMetric] [Description("Number of Code DB cache reads.")] - public static long CodeDbCache { get; set; } + public static long CodeDbCache + { + get + { + long total = 0; + foreach (var value in _codeDbCache.Values) + { + total += value; + } + return total; + } + } + private static ThreadLocal _codeDbCache = new(trackAllValues: true); + [Description("Number of Code DB cache reads on thread.")] + public static long ThreadLocalCodeDbCache => _codeDbCache.Value; + public static void IncrementCodeDbCache() => _codeDbCache.Value++; + + [CounterMetric] + [Description("Number of State Trie cache hits.")] + public static long StateTreeCache + { + get + { + long total = 0; + foreach (var value in _stateTreeCacheHits.Values) + { + total += value; + } + return total; + } + } + + private static ThreadLocal _stateTreeCacheHits = new(trackAllValues: true); + public static void IncrementStateTreeCacheHits() => _stateTreeCacheHits.Value++; [CounterMetric] [Description("Number of State Trie reads.")] - public static long StateTreeReads { get; set; } + public static long StateTreeReads + { + get + { + long total = 0; + foreach (var value in _stateTreeReads.Values) + { + total += value; + } + return total; + } + } + private static ThreadLocal _stateTreeReads = new(trackAllValues: true); + [Description("Number of State Trie reads on thread.")] + public static long ThreadLocalStateTreeReads => _stateTreeReads.Value; + public static void IncrementStateTreeReads() => _stateTreeReads.Value++; + + [CounterMetric] + [Description("Number of State Reader reads.")] + public static long StateReaderReads + { + get + { + long total = 0; + foreach (var value in _stateReaderReads.Values) + { + total += value; + } + return total; + } + } + private static ThreadLocal _stateReaderReads = new(trackAllValues: true); + public static void IncrementStateReaderReads() => _stateReaderReads.Value++; [CounterMetric] [Description("Number of Blocks Trie writes.")] @@ -26,9 +91,45 @@ public static class Metrics [Description("Number of State DB duplicate writes during full pruning.")] public static int StateDbInPruningWrites; + [CounterMetric] + [Description("Number of storage trie cache hits.")] + public static long StorageTreeCache + { + get + { + long total = 0; + foreach (var value in _storageTreeCache.Values) + { + total += value; + } + return total; + } + } + private static ThreadLocal _storageTreeCache = new(trackAllValues: true); + public static void IncrementStorageTreeCache() => _storageTreeCache.Value++; + [CounterMetric] [Description("Number of storage trie reads.")] - public static long StorageTreeReads { get; set; } + public static long StorageTreeReads + { + get + { + long total = 0; + foreach (var value in _storageTreeReads.Values) + { + total += value; + } + return total; + } + } + private static ThreadLocal _storageTreeReads = new(trackAllValues: true); + [Description("Number of storage trie reads on thread.")] + public static long ThreadLocalStorageTreeReads => _storageTreeReads.Value; + public static void IncrementStorageTreeReads() => _storageTreeReads.Value++; + + [CounterMetric] + [Description("Number of storage reader reads.")] + public static long StorageReaderReads { get; set; } [CounterMetric] [Description("Number of storage trie writes.")] diff --git a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs index 6feee30504c..495a51fbe5e 100644 --- a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs +++ b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs @@ -3,38 +3,30 @@ using System; using System.Collections.Generic; +using System.Linq; using Nethermind.Core; namespace Nethermind.Db { - public class ReadOnlyDb : IReadOnlyDb + public class ReadOnlyDb(IDb wrappedDb, bool createInMemWriteStore) : IReadOnlyDb { private readonly MemDb _memDb = new(); - private readonly IDb _wrappedDb; - private readonly bool _createInMemWriteStore; - - public ReadOnlyDb(IDb wrappedDb, bool createInMemWriteStore) - { - _wrappedDb = wrappedDb; - _createInMemWriteStore = createInMemWriteStore; - } - public void Dispose() { _memDb.Dispose(); } - public string Name { get; } = "ReadOnlyDb"; + public string Name { get => wrappedDb.Name; } public byte[]? Get(ReadOnlySpan key, ReadFlags flags = ReadFlags.None) { - return _memDb.Get(key, flags) ?? _wrappedDb.Get(key, flags); + return _memDb.Get(key, flags) ?? wrappedDb.Get(key, flags); } public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None) { - if (!_createInMemWriteStore) + if (!createInMemWriteStore) { throw new InvalidOperationException($"This {nameof(ReadOnlyDb)} did not expect any writes."); } @@ -46,7 +38,7 @@ public KeyValuePair[] this[byte[][] keys] { get { - var result = _wrappedDb[keys]; + var result = wrappedDb[keys]; var memResult = _memDb[keys]; for (int i = 0; i < memResult.Length; i++) { @@ -61,43 +53,34 @@ public KeyValuePair[] this[byte[][] keys] } } - public IEnumerable> GetAll(bool ordered = false) => _memDb.GetAll(); + public IEnumerable> GetAll(bool ordered = false) => _memDb.GetAll().Union(wrappedDb.GetAll()); - public IEnumerable GetAllKeys(bool ordered = false) => _memDb.GetAllKeys(); + public IEnumerable GetAllKeys(bool ordered = false) => _memDb.GetAllKeys().Union(wrappedDb.GetAllKeys()); - public IEnumerable GetAllValues(bool ordered = false) => _memDb.GetAllValues(); + public IEnumerable GetAllValues(bool ordered = false) => _memDb.GetAllValues().Union(wrappedDb.GetAllValues()); - public IWriteBatch StartWriteBatch() - { - return this.LikeABatch(); - } + public IWriteBatch StartWriteBatch() => this.LikeABatch(); - public IDbMeta.DbMetric GatherMetric(bool includeSharedCache = false) => _wrappedDb.GatherMetric(includeSharedCache); + public IDbMeta.DbMetric GatherMetric(bool includeSharedCache = false) => wrappedDb.GatherMetric(includeSharedCache); public void Remove(ReadOnlySpan key) { } - public bool KeyExists(ReadOnlySpan key) - { - return _memDb.KeyExists(key) || _wrappedDb.KeyExists(key); - } + public bool KeyExists(ReadOnlySpan key) => _memDb.KeyExists(key) || wrappedDb.KeyExists(key); public void Flush() { - _wrappedDb.Flush(); + wrappedDb.Flush(); _memDb.Flush(); } - public void Clear() { throw new InvalidOperationException(); } + public void Clear() => throw new InvalidOperationException(); - public virtual void ClearTempChanges() - { - _memDb.Clear(); - } + public virtual void ClearTempChanges() => _memDb.Clear(); - public Span GetSpan(ReadOnlySpan key) => _memDb.Get(key).AsSpan(); + public Span GetSpan(ReadOnlySpan key) => Get(key).AsSpan(); public void PutSpan(ReadOnlySpan keyBytes, ReadOnlySpan value, WriteFlags writeFlags = WriteFlags.None) { - if (!_createInMemWriteStore) + if (!createInMemWriteStore) { throw new InvalidOperationException($"This {nameof(ReadOnlyDb)} did not expect any writes."); } diff --git a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs index e87d5bb4e48..c5fe7f0bf3d 100644 --- a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs +++ b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs @@ -52,8 +52,6 @@ DbFactory is not MemDbFactory RegisterDb(BuildDbSettings(DbNames.Code)); RegisterDb(BuildDbSettings(DbNames.Bloom)); - RegisterDb(BuildDbSettings(DbNames.CHT)); - RegisterDb(BuildDbSettings(DbNames.Witness)); if (useReceiptsDb) { RegisterColumnsDb(BuildDbSettings(DbNames.Receipts)); diff --git a/src/Nethermind/Nethermind.EthStats.Test/EthStatsPluginTests.cs b/src/Nethermind/Nethermind.EthStats.Test/EthStatsPluginTests.cs index 06092269a99..2deefaa5e7e 100644 --- a/src/Nethermind/Nethermind.EthStats.Test/EthStatsPluginTests.cs +++ b/src/Nethermind/Nethermind.EthStats.Test/EthStatsPluginTests.cs @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.Threading.Tasks; using Nethermind.Api; +using Nethermind.Api.Extensions; using Nethermind.EthStats.Configs; using Nethermind.Runner.Test.Ethereum; using NUnit.Framework; @@ -14,7 +14,7 @@ public class EthStatsPluginTests public IEthStatsConfig StatsConfig { get; private set; } = null!; private NethermindApi _context = null!; #pragma warning disable NUnit1032 - private EthStatsPlugin _plugin = null!; + private INethermindPlugin _plugin = null!; #pragma warning restore NUnit1032 [SetUp] @@ -32,6 +32,7 @@ public void Setup() public void Init_eth_stats_plugin_does_not_throw_exception(bool enabled) { StatsConfig = new EthStatsConfig() { Enabled = enabled }; + Assert.DoesNotThrow(() => _plugin.InitRlpDecoders(_context)); Assert.DoesNotThrowAsync(async () => await _plugin.Init(_context)); Assert.DoesNotThrowAsync(async () => await _plugin.InitNetworkProtocol()); Assert.DoesNotThrowAsync(async () => await _plugin.InitRpcModules()); diff --git a/src/Nethermind/Nethermind.EthStats/EthStatsPlugin.cs b/src/Nethermind/Nethermind.EthStats/EthStatsPlugin.cs index 0bb60072a5e..b57e083dbe6 100644 --- a/src/Nethermind/Nethermind.EthStats/EthStatsPlugin.cs +++ b/src/Nethermind/Nethermind.EthStats/EthStatsPlugin.cs @@ -118,6 +118,4 @@ public async Task InitNetworkProtocol() await _ethStatsIntegration.InitAsync(); } } - - public Task InitRpcModules() => Task.CompletedTask; } diff --git a/src/Nethermind/Nethermind.Evm.Benchmark/EvmBenchmarks.cs b/src/Nethermind/Nethermind.Evm.Benchmark/EvmBenchmarks.cs index 4d16a92e69d..4399d6eaca1 100644 --- a/src/Nethermind/Nethermind.Evm.Benchmark/EvmBenchmarks.cs +++ b/src/Nethermind/Nethermind.Evm.Benchmark/EvmBenchmarks.cs @@ -44,8 +44,8 @@ public void GlobalSetup() _stateProvider = new WorldState(trieStore, codeDb, new OneLoggerLogManager(NullLogger.Instance)); _stateProvider.CreateAccount(Address.Zero, 1000.Ether()); _stateProvider.Commit(_spec); - - _virtualMachine = new VirtualMachine(_blockhashProvider, MainnetSpecProvider.Instance, LimboLogs.Instance); + CodeInfoRepository codeInfoRepository = new(); + _virtualMachine = new VirtualMachine(_blockhashProvider, MainnetSpecProvider.Instance, codeInfoRepository, LimboLogs.Instance); _environment = new ExecutionEnvironment ( diff --git a/src/Nethermind/Nethermind.Evm.Benchmark/MultipleUnsignedOperations.cs b/src/Nethermind/Nethermind.Evm.Benchmark/MultipleUnsignedOperations.cs index 75d82db72fb..9f400db12dc 100644 --- a/src/Nethermind/Nethermind.Evm.Benchmark/MultipleUnsignedOperations.cs +++ b/src/Nethermind/Nethermind.Evm.Benchmark/MultipleUnsignedOperations.cs @@ -76,7 +76,8 @@ public void GlobalSetup() _stateProvider.Commit(_spec); Console.WriteLine(MuirGlacier.Instance); - _virtualMachine = new VirtualMachine(_blockhashProvider, MainnetSpecProvider.Instance, new OneLoggerLogManager(NullLogger.Instance)); + CodeInfoRepository codeInfoRepository = new(); + _virtualMachine = new VirtualMachine(_blockhashProvider, MainnetSpecProvider.Instance, codeInfoRepository, new OneLoggerLogManager(NullLogger.Instance)); _environment = new ExecutionEnvironment ( diff --git a/src/Nethermind/Nethermind.Evm.Benchmark/StaticCallBenchmarks.cs b/src/Nethermind/Nethermind.Evm.Benchmark/StaticCallBenchmarks.cs index c559f5d52fd..ca0bad688ec 100644 --- a/src/Nethermind/Nethermind.Evm.Benchmark/StaticCallBenchmarks.cs +++ b/src/Nethermind/Nethermind.Evm.Benchmark/StaticCallBenchmarks.cs @@ -87,7 +87,8 @@ public void GlobalSetup() _stateProvider.Commit(_spec); Console.WriteLine(MuirGlacier.Instance); - _virtualMachine = new VirtualMachine(_blockhashProvider, MainnetSpecProvider.Instance, new OneLoggerLogManager(NullLogger.Instance)); + CodeInfoRepository codeInfoRepository = new(); + _virtualMachine = new VirtualMachine(_blockhashProvider, MainnetSpecProvider.Instance, codeInfoRepository, new OneLoggerLogManager(NullLogger.Instance)); _environment = new ExecutionEnvironment ( diff --git a/src/Nethermind/Nethermind.Evm.Test/CodeAnalysis/CodeInfoTests.cs b/src/Nethermind/Nethermind.Evm.Test/CodeAnalysis/CodeInfoTests.cs index a76c67e2eb6..67490a397e1 100644 --- a/src/Nethermind/Nethermind.Evm.Test/CodeAnalysis/CodeInfoTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/CodeAnalysis/CodeInfoTests.cs @@ -28,23 +28,7 @@ public void Validates_when_only_jump_dest_present(int destination, bool isValid) CodeInfo codeInfo = new(code); - codeInfo.ValidateJump(destination, false).Should().Be(isValid); - } - - [TestCase(-1, false)] - [TestCase(0, true)] - [TestCase(1, false)] - public void Validates_when_only_begin_sub_present(int destination, bool isValid) - { - byte[] code = - { - (byte)Instruction.BEGINSUB - }; - - CodeInfo codeInfo = new(code); - - - codeInfo.ValidateJump(destination, true).Should().Be(isValid); + codeInfo.ValidateJump(destination).Should().Be(isValid); } [Test] @@ -58,23 +42,7 @@ public void Validates_when_push_with_data_like_jump_dest() CodeInfo codeInfo = new(code); - codeInfo.ValidateJump(1, true).Should().BeFalse(); - codeInfo.ValidateJump(1, false).Should().BeFalse(); - } - - [Test] - public void Validates_when_push_with_data_like_begin_sub() - { - byte[] code = - { - (byte)Instruction.PUSH1, - (byte)Instruction.BEGINSUB - }; - - CodeInfo codeInfo = new(code); - - codeInfo.ValidateJump(1, true).Should().BeFalse(); - codeInfo.ValidateJump(1, false).Should().BeFalse(); + codeInfo.ValidateJump(1).Should().BeFalse(); } [Test] @@ -89,7 +57,7 @@ public void Validate_CodeBitmap_With_Push10() CodeInfo codeInfo = new(code); - codeInfo.ValidateJump(11, false).Should().BeTrue(); + codeInfo.ValidateJump(11).Should().BeTrue(); } [Test] @@ -104,7 +72,7 @@ public void Validate_CodeBitmap_With_Push30() CodeInfo codeInfo = new(code); - codeInfo.ValidateJump(31, false).Should().BeTrue(); + codeInfo.ValidateJump(31).Should().BeTrue(); } [Test] @@ -117,7 +85,7 @@ public void Small_Jumpdest() CodeInfo codeInfo = new(code); - codeInfo.ValidateJump(10, false).Should().BeTrue(); + codeInfo.ValidateJump(10).Should().BeTrue(); } [Test] @@ -130,7 +98,7 @@ public void Small_Push1() CodeInfo codeInfo = new(code); - codeInfo.ValidateJump(10, false).Should().BeFalse(); + codeInfo.ValidateJump(10).Should().BeFalse(); } [Test] @@ -140,7 +108,7 @@ public void Jumpdest_Over10k() CodeInfo codeInfo = new(code); - codeInfo.ValidateJump(10, false).Should().BeTrue(); + codeInfo.ValidateJump(10).Should().BeTrue(); } [Test] @@ -150,7 +118,7 @@ public void Push1_Over10k() CodeInfo codeInfo = new(code); - codeInfo.ValidateJump(10, false).Should().BeFalse(); + codeInfo.ValidateJump(10).Should().BeFalse(); } [Test] @@ -164,8 +132,8 @@ public void Push1Jumpdest_Over10k() CodeInfo codeInfo = new(code); - codeInfo.ValidateJump(10, false).Should().BeFalse(); - codeInfo.ValidateJump(11, false).Should().BeFalse(); // 0x5b but not JUMPDEST but data + codeInfo.ValidateJump(10).Should().BeFalse(); + codeInfo.ValidateJump(11).Should().BeFalse(); // 0x5b but not JUMPDEST but data } [TestCase(1)] @@ -231,15 +199,15 @@ public void PushNJumpdest_Over10k(int n) for (i = 0; i < Vector256.Count * 2 + Vector128.Count; i++) { - codeInfo.ValidateJump(i, false).Should().BeTrue(); + codeInfo.ValidateJump(i).Should().BeTrue(); } for (; i < Vector256.Count * 3; i++) { - codeInfo.ValidateJump(i, false).Should().BeFalse(); + codeInfo.ValidateJump(i).Should().BeFalse(); } for (; i < code.Length; i++) { - codeInfo.ValidateJump(i, false).Should().BeFalse(); // Are 0x5b but not JUMPDEST but data + codeInfo.ValidateJump(i).Should().BeFalse(); // Are 0x5b but not JUMPDEST but data } } } diff --git a/src/Nethermind/Nethermind.Evm.Test/Eip3198BaseFeeTests.cs b/src/Nethermind/Nethermind.Evm.Test/Eip3198BaseFeeTests.cs index 6b20654a11f..75430c258c1 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Eip3198BaseFeeTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Eip3198BaseFeeTests.cs @@ -26,7 +26,7 @@ public class Eip3198BaseFeeTests : VirtualMachineTestsBase [TestCase(false, 0, false)] public void Base_fee_opcode_should_return_expected_results(bool eip3198Enabled, int baseFee, bool send1559Tx) { - _processor = new TransactionProcessor(SpecProvider, TestState, Machine, LimboLogs.Instance); + _processor = new TransactionProcessor(SpecProvider, TestState, Machine, CodeInfoRepository, LimboLogs.Instance); byte[] code = Prepare.EvmCode .Op(Instruction.BASEFEE) .PushData(0) diff --git a/src/Nethermind/Nethermind.Evm.Test/Eip3529RefundsTests.cs b/src/Nethermind/Nethermind.Evm.Test/Eip3529RefundsTests.cs index fe3e7a58e32..81c276ef05d 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Eip3529RefundsTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Eip3529RefundsTests.cs @@ -73,7 +73,7 @@ private void Test(string codeHex, long gasUsed, long refund, byte originalValue, TestState.CreateAccount(Recipient, 1.Ether()); TestState.Set(new StorageCell(Recipient, 0), new[] { originalValue }); TestState.Commit(eip3529Enabled ? London.Instance : Berlin.Instance); - _processor = new TransactionProcessor(SpecProvider, TestState, Machine, LimboLogs.Instance); + _processor = new TransactionProcessor(SpecProvider, TestState, Machine, CodeInfoRepository, LimboLogs.Instance); long blockNumber = eip3529Enabled ? MainnetSpecProvider.LondonBlockNumber : MainnetSpecProvider.LondonBlockNumber - 1; (Block block, Transaction transaction) = PrepareTx(blockNumber, 100000, Bytes.FromHexString(codeHex)); diff --git a/src/Nethermind/Nethermind.Evm.Test/Eip3541Tests.cs b/src/Nethermind/Nethermind.Evm.Test/Eip3541Tests.cs index 4154e94f095..0c5f2219dd3 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Eip3541Tests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Eip3541Tests.cs @@ -96,7 +96,7 @@ void DeployCodeAndAssertTx(string code, bool eip3541Enabled, ContractDeployment break; } - _processor = new TransactionProcessor(SpecProvider, TestState, Machine, LimboLogs.Instance); + _processor = new TransactionProcessor(SpecProvider, TestState, Machine, CodeInfoRepository, LimboLogs.Instance); long blockNumber = eip3541Enabled ? MainnetSpecProvider.LondonBlockNumber : MainnetSpecProvider.LondonBlockNumber - 1; (Block block, Transaction transaction) = PrepareTx(blockNumber, 100000, createContract); diff --git a/src/Nethermind/Nethermind.Evm.Test/Eip7516BlobBaseFeeTests.cs b/src/Nethermind/Nethermind.Evm.Test/Eip7516BlobBaseFeeTests.cs index 1a1036de336..f62d8c595f6 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Eip7516BlobBaseFeeTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Eip7516BlobBaseFeeTests.cs @@ -27,7 +27,7 @@ public class Eip7516BlobBaseFeeTests : VirtualMachineTestsBase [TestCase(false, 0ul)] public void Blob_Base_fee_opcode_should_return_expected_results(bool eip7516Enabled, ulong excessBlobGas) { - _processor = new TransactionProcessor(SpecProvider, TestState, Machine, LimboLogs.Instance); + _processor = new TransactionProcessor(SpecProvider, TestState, Machine, CodeInfoRepository, LimboLogs.Instance); byte[] code = Prepare.EvmCode .Op(Instruction.BLOBBASEFEE) .PushData(0) diff --git a/src/Nethermind/Nethermind.Evm.Test/EvmPooledMemoryTests.cs b/src/Nethermind/Nethermind.Evm.Test/EvmPooledMemoryTests.cs index 18529a85794..ab88c978cae 100644 --- a/src/Nethermind/Nethermind.Evm.Test/EvmPooledMemoryTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/EvmPooledMemoryTests.cs @@ -158,14 +158,17 @@ private static string run(byte[] input) new MemDb(), LimboLogs.Instance); ISpecProvider specProvider = new TestSpecProvider(London.Instance); + CodeInfoRepository codeInfoRepository = new(); VirtualMachine virtualMachine = new( new TestBlockhashProvider(specProvider), specProvider, + codeInfoRepository, LimboLogs.Instance); TransactionProcessor transactionProcessor = new TransactionProcessor( specProvider, stateProvider, virtualMachine, + codeInfoRepository, LimboLogs.Instance); stateProvider.CreateAccount(to, 123); diff --git a/src/Nethermind/Nethermind.Evm.Test/InstructionTests.cs b/src/Nethermind/Nethermind.Evm.Test/InstructionTests.cs index 2a9f1677613..74ea7de35c1 100644 --- a/src/Nethermind/Nethermind.Evm.Test/InstructionTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/InstructionTests.cs @@ -20,42 +20,5 @@ public void Return_prevrandao_name_for_prevrandao_opcode_for_post_merge() { Instruction.PREVRANDAO.GetName(true, Cancun.Instance).Should().Be("PREVRANDAO"); } - - [Test] - public void Return_tload_name_for_beginsub_opcode_for_eip1153() - { - Instruction.BEGINSUB.GetName(true, Cancun.Instance).Should().Be("TLOAD"); - } - - [Test] - public void Return_beginsub_name_for_beginsub_opcode_for_eip1153() - { - Instruction.BEGINSUB.GetName(true, Shanghai.Instance).Should().Be("BEGINSUB"); - } - - [Test] - public void Return_returnsub_name_for_returnsub_opcode_for_eip1153() - { - Instruction.RETURNSUB.GetName(true, Shanghai.Instance).Should().Be("RETURNSUB"); - } - - [Test] - public void Return_tstore_name_for_returnsub_opcode_for_eip1153() - { - Instruction.RETURNSUB.GetName(true, Cancun.Instance).Should().Be("TSTORE"); - } - - - [Test] - public void Return_mcopy_name_for_mcopy_opcode_post_eip_5656() - { - Instruction.MCOPY.GetName(true, Cancun.Instance).Should().Be("MCOPY"); - } - - [Test] - public void Return_jumpsub_name_for_mcopy_opcode_pre_eip_5656() - { - Instruction.MCOPY.GetName(true, Shanghai.Instance).Should().Be("JUMPSUB"); - } } } diff --git a/src/Nethermind/Nethermind.Evm.Test/Tracing/GasEstimationTests.cs b/src/Nethermind/Nethermind.Evm.Test/Tracing/GasEstimationTests.cs index 2e41c2443ed..def68b3ca1d 100644 --- a/src/Nethermind/Nethermind.Evm.Test/Tracing/GasEstimationTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/Tracing/GasEstimationTests.cs @@ -362,9 +362,9 @@ public TestEnvironment() _stateProvider.Commit(_specProvider.GenesisSpec); _stateProvider.CommitTree(0); - VirtualMachine virtualMachine = new(new TestBlockhashProvider(_specProvider), _specProvider, LimboLogs.Instance); - _transactionProcessor = new TransactionProcessor(_specProvider, _stateProvider, - virtualMachine, LimboLogs.Instance); + CodeInfoRepository codeInfoRepository = new(); + VirtualMachine virtualMachine = new(new TestBlockhashProvider(_specProvider), _specProvider, codeInfoRepository, LimboLogs.Instance); + _transactionProcessor = new TransactionProcessor(_specProvider, _stateProvider, virtualMachine, codeInfoRepository, LimboLogs.Instance); _ethereumEcdsa = new EthereumEcdsa(_specProvider.ChainId, LimboLogs.Instance); tracer = new(); diff --git a/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorEip4844Tests.cs b/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorEip4844Tests.cs index 7312cf923f7..562b27e937e 100644 --- a/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorEip4844Tests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorEip4844Tests.cs @@ -23,10 +23,10 @@ namespace Nethermind.Evm.Test; [TestFixture] internal class TransactionProcessorEip4844Tests { - private ISpecProvider _specProvider; - private IEthereumEcdsa _ethereumEcdsa; - private TransactionProcessor _transactionProcessor; - private IWorldState _stateProvider; + private ISpecProvider _specProvider = null!; + private IEthereumEcdsa _ethereumEcdsa = null!; + private TransactionProcessor _transactionProcessor = null!; + private IWorldState _stateProvider = null!; [SetUp] public void Setup() @@ -35,8 +35,9 @@ public void Setup() _specProvider = new TestSpecProvider(Cancun.Instance); TrieStore trieStore = new(stateDb, LimboLogs.Instance); _stateProvider = new WorldState(trieStore, new MemDb(), LimboLogs.Instance); - VirtualMachine virtualMachine = new(new TestBlockhashProvider(_specProvider), _specProvider, LimboLogs.Instance); - _transactionProcessor = new TransactionProcessor(_specProvider, _stateProvider, virtualMachine, LimboLogs.Instance); + CodeInfoRepository codeInfoRepository = new(); + VirtualMachine virtualMachine = new(new TestBlockhashProvider(_specProvider), _specProvider, codeInfoRepository, LimboLogs.Instance); + _transactionProcessor = new TransactionProcessor(_specProvider, _stateProvider, virtualMachine, codeInfoRepository, LimboLogs.Instance); _ethereumEcdsa = new EthereumEcdsa(_specProvider.ChainId, LimboLogs.Instance); } diff --git a/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorFeeTests.cs b/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorFeeTests.cs index 09d61e31b55..e0185255a0b 100644 --- a/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorFeeTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorFeeTests.cs @@ -42,8 +42,9 @@ public void Setup() _stateProvider.Commit(_specProvider.GenesisSpec); _stateProvider.CommitTree(0); - VirtualMachine virtualMachine = new(new TestBlockhashProvider(_specProvider), _specProvider, LimboLogs.Instance); - _transactionProcessor = new TransactionProcessor(_specProvider, _stateProvider, virtualMachine, LimboLogs.Instance); + CodeInfoRepository codeInfoRepository = new(); + VirtualMachine virtualMachine = new(new TestBlockhashProvider(_specProvider), _specProvider, codeInfoRepository, LimboLogs.Instance); + _transactionProcessor = new TransactionProcessor(_specProvider, _stateProvider, virtualMachine, codeInfoRepository, LimboLogs.Instance); _ethereumEcdsa = new EthereumEcdsa(_specProvider.ChainId, LimboLogs.Instance); } diff --git a/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorTests.cs b/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorTests.cs index 5d30cc00eee..3a7139dc4b6 100644 --- a/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorTests.cs +++ b/src/Nethermind/Nethermind.Evm.Test/TransactionProcessorTests.cs @@ -58,8 +58,9 @@ public void Setup() _stateProvider.Commit(_specProvider.GenesisSpec); _stateProvider.CommitTree(0); - VirtualMachine virtualMachine = new(new TestBlockhashProvider(_specProvider), _specProvider, LimboLogs.Instance); - _transactionProcessor = new TransactionProcessor(_specProvider, _stateProvider, virtualMachine, LimboLogs.Instance); + CodeInfoRepository codeInfoRepository = new(); + VirtualMachine virtualMachine = new(new TestBlockhashProvider(_specProvider), _specProvider, codeInfoRepository, LimboLogs.Instance); + _transactionProcessor = new TransactionProcessor(_specProvider, _stateProvider, virtualMachine, codeInfoRepository, LimboLogs.Instance); _ethereumEcdsa = new EthereumEcdsa(_specProvider.ChainId, LimboLogs.Instance); } diff --git a/src/Nethermind/Nethermind.Evm.Test/VirtualMachineTestsBase.cs b/src/Nethermind/Nethermind.Evm.Test/VirtualMachineTestsBase.cs index f914566a102..9cafd39c208 100644 --- a/src/Nethermind/Nethermind.Evm.Test/VirtualMachineTestsBase.cs +++ b/src/Nethermind/Nethermind.Evm.Test/VirtualMachineTestsBase.cs @@ -35,6 +35,7 @@ public class VirtualMachineTestsBase private IDb _stateDb; protected VirtualMachine Machine { get; private set; } + protected CodeInfoRepository CodeInfoRepository { get; private set; } protected IWorldState TestState { get; private set; } protected static Address Contract { get; } = new("0xd75a3a95360e44a3874e691fb48d77855f127069"); protected static Address Sender { get; } = TestItem.AddressA; @@ -67,8 +68,9 @@ public virtual void Setup() TestState = new WorldState(trieStore, codeDb, logManager); _ethereumEcdsa = new EthereumEcdsa(SpecProvider.ChainId, logManager); IBlockhashProvider blockhashProvider = new TestBlockhashProvider(SpecProvider); - Machine = new VirtualMachine(blockhashProvider, SpecProvider, logManager); - _processor = new TransactionProcessor(SpecProvider, TestState, Machine, logManager); + CodeInfoRepository = new CodeInfoRepository(); + Machine = new VirtualMachine(blockhashProvider, SpecProvider, CodeInfoRepository, logManager); + _processor = new TransactionProcessor(SpecProvider, TestState, Machine, CodeInfoRepository, logManager); } [TearDown] diff --git a/src/Nethermind/Nethermind.Evm/ByteCodeBuilder.cs b/src/Nethermind/Nethermind.Evm/ByteCodeBuilder.cs index 3f1407b117c..a44ab9dae90 100644 --- a/src/Nethermind/Nethermind.Evm/ByteCodeBuilder.cs +++ b/src/Nethermind/Nethermind.Evm/ByteCodeBuilder.cs @@ -274,10 +274,10 @@ public Prepare PushData(string data) return this; } - public Prepare PushData(byte[] data) + public Prepare PushData(ReadOnlyMemory data) { _byteCode.Add((byte)(Instruction.PUSH1 + (byte)data.Length - 1)); - _byteCode.AddRange(data); + _byteCode.AddRange(data.Span); return this; } diff --git a/src/Nethermind/Nethermind.Evm/ByteCodeBuilderExtensions.cs b/src/Nethermind/Nethermind.Evm/ByteCodeBuilderExtensions.cs index 8ceccf331af..b94b33475f3 100644 --- a/src/Nethermind/Nethermind.Evm/ByteCodeBuilderExtensions.cs +++ b/src/Nethermind/Nethermind.Evm/ByteCodeBuilderExtensions.cs @@ -101,10 +101,6 @@ public static Prepare SWAPx(this Prepare @this, byte i) => @this.Op(Instruction.SWAP1 + i - 1); public static Prepare DUPx(this Prepare @this, byte i) => @this.Op(Instruction.DUP1 + i - 1); - public static Prepare BEGINSUB(this Prepare @this) - => @this.Op(Instruction.BEGINSUB); - public static Prepare RETURNSUB(this Prepare @this) - => @this.Op(Instruction.RETURNSUB); public static Prepare INVALID(this Prepare @this) => @this.Op(Instruction.INVALID); #endregion @@ -116,9 +112,6 @@ public static Prepare SELFDESTRUCT(this Prepare @this, Address? address = null) public static Prepare EXTCODEHASH(this Prepare @this, Address? address = null) => @this.PushSingle(address) .Op(Instruction.EXTCODEHASH); - public static Prepare JUMPSUB(this Prepare @this, UInt256? pos = null) - => @this.PushSingle(pos) - .Op(Instruction.JUMPSUB); public static Prepare PUSHx(this Prepare @this, byte[] args) => @this.PushData(args); public static Prepare MLOAD(this Prepare @this, UInt256? pos = null) diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs index 6d167abac09..e1bb762bceb 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/CodeInfo.cs @@ -29,6 +29,7 @@ public CodeInfo(ReadOnlyMemory code) } public bool IsPrecompile => Precompile is not null; + public bool IsEmpty => ReferenceEquals(_analyzer, _emptyAnalyzer) && !IsPrecompile; public CodeInfo(IPrecompile precompile) { @@ -37,14 +38,22 @@ public CodeInfo(IPrecompile precompile) _analyzer = _emptyAnalyzer; } - public bool ValidateJump(int destination, bool isSubroutine) + public bool ValidateJump(int destination) { - return _analyzer.ValidateJump(destination, isSubroutine); + return _analyzer.ValidateJump(destination); } void IThreadPoolWorkItem.Execute() { _analyzer.Execute(); } + + public void AnalyseInBackgroundIfRequired() + { + if (!ReferenceEquals(_analyzer, _emptyAnalyzer) && _analyzer.RequiresAnalysis) + { + ThreadPool.UnsafeQueueUserWorkItem(this, preferLocal: false); + } + } } } diff --git a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs index 8e944c18101..8ed08d2a21a 100644 --- a/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs +++ b/src/Nethermind/Nethermind.Evm/CodeAnalysis/JumpDestinationAnalyzer.cs @@ -6,39 +6,76 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; +using System.Threading; namespace Nethermind.Evm.CodeAnalysis { public sealed class JumpDestinationAnalyzer(ReadOnlyMemory code) { - private const int PUSH1 = 0x60; + private const int PUSH1 = (int)Instruction.PUSH1; private const int PUSHx = PUSH1 - 1; - private const int JUMPDEST = 0x5b; - private const int BEGINSUB = 0x5c; + private const int JUMPDEST = (int)Instruction.JUMPDEST; private const int BitShiftPerInt64 = 6; private static readonly long[]? _emptyJumpDestinationBitmap = new long[1]; private long[]? _jumpDestinationBitmap = code.Length == 0 ? _emptyJumpDestinationBitmap : null; + private object? _analysisComplete; private ReadOnlyMemory MachineCode { get; } = code; - public bool ValidateJump(int destination, bool isSubroutine) + public bool ValidateJump(int destination) { - ReadOnlySpan machineCode = MachineCode.Span; - _jumpDestinationBitmap ??= CreateJumpDestinationBitmap(machineCode); + _jumpDestinationBitmap ??= CreateOrWaitForJumpDestinationBitmap(); - var result = false; // Cast to uint to change negative numbers to very int high numbers // Then do length check, this both reduces check by 1 and eliminates the bounds // check from accessing the span. - if ((uint)destination < (uint)machineCode.Length && IsJumpDestination(_jumpDestinationBitmap, destination)) + return (uint)destination < (uint)MachineCode.Length && IsJumpDestination(_jumpDestinationBitmap, destination); + } + + private long[] CreateOrWaitForJumpDestinationBitmap() + { + object? previous = Volatile.Read(ref _analysisComplete); + if (previous is null) { - // Store byte to int, as less expensive operations at word size - int codeByte = machineCode[destination]; - result = isSubroutine ? codeByte == BEGINSUB : codeByte == JUMPDEST; + ManualResetEventSlim analysisComplete = new(initialState: false); + previous = Interlocked.CompareExchange(ref _analysisComplete, analysisComplete, null); + if (previous is null) + { + // Not already in progress, so start it. + var bitmap = CreateJumpDestinationBitmap(); + _jumpDestinationBitmap = bitmap; + // Release the MRES to be GC'd + _analysisComplete = bitmap; + // Signal complete. + analysisComplete.Set(); + return bitmap; + } + } + + if (previous is ManualResetEventSlim resetEvent) + { + Thread thread = Thread.CurrentThread; + ThreadPriority priority = thread.Priority; + try + { + // We are waiting, so drop priority to normal (BlockProcessing runs at higher priority). + thread.Priority = ThreadPriority.Normal; + + // Already in progress, wait for completion. + resetEvent.Wait(); + } + finally + { + // Restore the priority of the thread. + thread.Priority = priority; + } + + return _jumpDestinationBitmap; } - return result; + // Must be the bitmap, and lost check->create benign data race + return (long[])previous; } /// @@ -64,8 +101,10 @@ private static int GetInt64ArrayLengthFromBitLength(int n) => /// Collects data locations in code. /// An unset bit means the byte is an opcode, a set bit means it's data. /// - private static long[] CreateJumpDestinationBitmap(ReadOnlySpan code) + private long[] CreateJumpDestinationBitmap() { + Metrics.IncrementContractsAnalysed(); + ReadOnlySpan code = MachineCode.Span; long[] jumpDestinationBitmap = new long[GetInt64ArrayLengthFromBitLength(code.Length)]; int programCounter = 0; // We accumulate each array segment to a register and then flush to memory when we move to next. @@ -90,12 +129,8 @@ private static long[] CreateJumpDestinationBitmap(ReadOnlySpan code) { // Check the bytes for any JUMPDESTs. Vector128 dest = Sse2.CompareEqual(data, Vector128.Create((sbyte)JUMPDEST)); - // Check the bytes for any BEGINSUBs. - Vector128 sub = Sse2.CompareEqual(data, Vector128.Create((sbyte)BEGINSUB)); - // Merge the two results. - Vector128 combined = Sse2.Or(dest, sub); // Extract the checks as a set of int flags. - int flags = Sse2.MoveMask(combined); + int flags = Sse2.MoveMask(dest); // Shift up flags by depending which side of long we are on, and merge to current set. currentFlags |= (long)flags << (programCounter & (32 + 16)); // Forward programCounter by Vector128 stride. @@ -110,7 +145,7 @@ private static long[] CreateJumpDestinationBitmap(ReadOnlySpan code) // access here. int op = Unsafe.Add(ref MemoryMarshal.GetReference(code), programCounter); - if ((uint)op - JUMPDEST <= BEGINSUB - JUMPDEST) + if (op == JUMPDEST) { // Accumulate Jump Destinations to register, shift will wrap and single bit // so can shift by the whole programCounter. @@ -159,6 +194,7 @@ private static long[] CreateJumpDestinationBitmap(ReadOnlySpan code) /// /// Checks if the position is in a code segment. /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsJumpDestination(long[] bitvec, int pos) { int vecIndex = pos >> BitShiftPerInt64; @@ -177,8 +213,34 @@ private static void MarkJumpDestinations(long[] jumpDestinationBitmap, int pos, public void Execute() { - // This is to support background thread preparation of the bitmap. - _jumpDestinationBitmap ??= CreateJumpDestinationBitmap(MachineCode.Span); + if (_jumpDestinationBitmap is null && Volatile.Read(ref _analysisComplete) is null) + { + ManualResetEventSlim analysisComplete = new(initialState: false); + if (Interlocked.CompareExchange(ref _analysisComplete, analysisComplete, null) is null) + { + Thread thread = Thread.CurrentThread; + ThreadPriority priority = thread.Priority; + try + { + // Boost the priority of the thread as block processing may be waiting on this. + thread.Priority = ThreadPriority.AboveNormal; + + _jumpDestinationBitmap ??= CreateJumpDestinationBitmap(); + + // Release the MRES to be GC'd + _analysisComplete = _jumpDestinationBitmap; + // Signal complete. + analysisComplete.Set(); + } + finally + { + // Restore the priority of the thread. + thread.Priority = priority; + } + } + } } + + public bool RequiresAnalysis => _jumpDestinationBitmap is null; } } diff --git a/src/Nethermind/Nethermind.Evm/CodeInfoRepository.cs b/src/Nethermind/Nethermind.Evm/CodeInfoRepository.cs new file mode 100644 index 00000000000..2cff92aa7ca --- /dev/null +++ b/src/Nethermind/Nethermind.Evm/CodeInfoRepository.cs @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Frozen; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics; +using System.Threading; +using Nethermind.Core; +using Nethermind.Core.Caching; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Evm.CodeAnalysis; +using Nethermind.Evm.Precompiles; +using Nethermind.Evm.Precompiles.Bls; +using Nethermind.Evm.Precompiles.Snarks; +using Nethermind.State; + +namespace Nethermind.Evm; + +public class CodeInfoRepository : ICodeInfoRepository +{ + internal sealed class CodeLruCache + { + private const int CacheCount = 16; + private const int CacheMax = CacheCount - 1; + private readonly LruCacheLowObject[] _caches; + + public CodeLruCache() + { + _caches = new LruCacheLowObject[CacheCount]; + for (int i = 0; i < _caches.Length; i++) + { + // Cache per nibble to reduce contention as TxPool is very parallel + _caches[i] = new LruCacheLowObject(MemoryAllowance.CodeCacheSize / CacheCount, $"VM bytecodes {i}"); + } + } + + public CodeInfo? Get(in ValueHash256 codeHash) + { + LruCacheLowObject cache = _caches[GetCacheIndex(codeHash)]; + return cache.Get(codeHash); + } + + public bool Set(in ValueHash256 codeHash, CodeInfo codeInfo) + { + LruCacheLowObject cache = _caches[GetCacheIndex(codeHash)]; + return cache.Set(codeHash, codeInfo); + } + + private static int GetCacheIndex(in ValueHash256 codeHash) => codeHash.Bytes[^1] & CacheMax; + + public bool TryGet(in ValueHash256 codeHash, [NotNullWhen(true)] out CodeInfo? codeInfo) + { + codeInfo = Get(codeHash); + return codeInfo is not null; + } + } + + + private static readonly FrozenDictionary _precompiles = InitializePrecompiledContracts(); + private static readonly CodeLruCache _codeCache = new(); + + private static FrozenDictionary InitializePrecompiledContracts() + { + return new Dictionary + { + [EcRecoverPrecompile.Address] = new(EcRecoverPrecompile.Instance), + [Sha256Precompile.Address] = new(Sha256Precompile.Instance), + [Ripemd160Precompile.Address] = new(Ripemd160Precompile.Instance), + [IdentityPrecompile.Address] = new(IdentityPrecompile.Instance), + + [Bn254AddPrecompile.Address] = new(Bn254AddPrecompile.Instance), + [Bn254MulPrecompile.Address] = new(Bn254MulPrecompile.Instance), + [Bn254PairingPrecompile.Address] = new(Bn254PairingPrecompile.Instance), + [ModExpPrecompile.Address] = new(ModExpPrecompile.Instance), + + [Blake2FPrecompile.Address] = new(Blake2FPrecompile.Instance), + + [G1AddPrecompile.Address] = new(G1AddPrecompile.Instance), + [G1MulPrecompile.Address] = new(G1MulPrecompile.Instance), + [G1MultiExpPrecompile.Address] = new(G1MultiExpPrecompile.Instance), + [G2AddPrecompile.Address] = new(G2AddPrecompile.Instance), + [G2MulPrecompile.Address] = new(G2MulPrecompile.Instance), + [G2MultiExpPrecompile.Address] = new(G2MultiExpPrecompile.Instance), + [PairingPrecompile.Address] = new(PairingPrecompile.Instance), + [MapToG1Precompile.Address] = new(MapToG1Precompile.Instance), + [MapToG2Precompile.Address] = new(MapToG2Precompile.Instance), + + [PointEvaluationPrecompile.Address] = new(PointEvaluationPrecompile.Instance), + }.ToFrozenDictionary(); + } + + public CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IReleaseSpec vmSpec) + { + if (codeSource.IsPrecompile(vmSpec)) + { + return _precompiles[codeSource]; + } + + CodeInfo? cachedCodeInfo = null; + ValueHash256 codeHash = worldState.GetCodeHash(codeSource); + if (codeHash == Keccak.OfAnEmptyString.ValueHash256) + { + cachedCodeInfo = CodeInfo.Empty; + } + + cachedCodeInfo ??= _codeCache.Get(codeHash); + if (cachedCodeInfo is null) + { + byte[]? code = worldState.GetCode(codeHash); + + if (code is null) + { + MissingCode(codeSource, codeHash); + } + + cachedCodeInfo = new CodeInfo(code); + cachedCodeInfo.AnalyseInBackgroundIfRequired(); + _codeCache.Set(codeHash, cachedCodeInfo); + } + else + { + Db.Metrics.IncrementCodeDbCache(); + } + + return cachedCodeInfo; + + [DoesNotReturn] + [StackTraceHidden] + static void MissingCode(Address codeSource, in ValueHash256 codeHash) + { + throw new NullReferenceException($"Code {codeHash} missing in the state for address {codeSource}"); + } + } + + public CodeInfo GetOrAdd(ValueHash256 codeHash, ReadOnlySpan initCode) + { + if (!_codeCache.TryGet(codeHash, out CodeInfo? codeInfo)) + { + codeInfo = new(initCode.ToArray()); + + // Prime the code cache as likely to be used by more txs + _codeCache.Set(codeHash, codeInfo); + } + + return codeInfo; + } + + + public void InsertCode(IWorldState state, ReadOnlyMemory code, Address codeOwner, IReleaseSpec spec, bool isSystemEnv) + { + CodeInfo codeInfo = new(code); + codeInfo.AnalyseInBackgroundIfRequired(); + + Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.Span); + state.InsertCode(codeOwner, codeHash, code, spec, isSystemEnv); + _codeCache.Set(codeHash, codeInfo); + } +} diff --git a/src/Nethermind/Nethermind.Evm/ICodeInfoRepository.cs b/src/Nethermind/Nethermind.Evm/ICodeInfoRepository.cs new file mode 100644 index 00000000000..ec8da29ca6a --- /dev/null +++ b/src/Nethermind/Nethermind.Evm/ICodeInfoRepository.cs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Evm.CodeAnalysis; +using Nethermind.State; + +namespace Nethermind.Evm; + +public interface ICodeInfoRepository +{ + CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IReleaseSpec vmSpec); + CodeInfo GetOrAdd(ValueHash256 codeHash, ReadOnlySpan initCode); + void InsertCode(IWorldState state, ReadOnlyMemory code, Address codeOwner, IReleaseSpec spec, bool isSystemEnv); +} diff --git a/src/Nethermind/Nethermind.Evm/IVirtualMachine.cs b/src/Nethermind/Nethermind.Evm/IVirtualMachine.cs index 112e31251cc..8099d6d7b6c 100644 --- a/src/Nethermind/Nethermind.Evm/IVirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/IVirtualMachine.cs @@ -17,8 +17,5 @@ public interface IVirtualMachine { TransactionSubstate Run(EvmState state, IWorldState worldState, ITxTracer txTracer) where TTracingActions : struct, IIsTracing; - - CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IReleaseSpec spec); - void InsertCode(ReadOnlyMemory code, Address codeOwner, IReleaseSpec spec, bool isSystemEnv); } } diff --git a/src/Nethermind/Nethermind.Evm/Instruction.cs b/src/Nethermind/Nethermind.Evm/Instruction.cs index 08aad72e235..61d5f1e2502 100644 --- a/src/Nethermind/Nethermind.Evm/Instruction.cs +++ b/src/Nethermind/Nethermind.Evm/Instruction.cs @@ -81,9 +81,6 @@ public enum Instruction : byte MSIZE = 0x59, GAS = 0x5a, JUMPDEST = 0x5b, - BEGINSUB = 0x5c, - RETURNSUB = 0x5d, - JUMPSUB = 0x5e, MCOPY = 0x5e, PUSH0 = 0x5f, // EIP-3855 @@ -184,9 +181,6 @@ public static class InstructionExtensions instruction switch { Instruction.PREVRANDAO when !isPostMerge => "DIFFICULTY", - Instruction.TLOAD or Instruction.BEGINSUB => spec?.TransientStorageEnabled == true ? "TLOAD" : "BEGINSUB", - Instruction.TSTORE or Instruction.RETURNSUB => spec?.TransientStorageEnabled == true ? "TSTORE" : "RETURNSUB", - Instruction.JUMPSUB or Instruction.MCOPY => spec?.IsEip5656Enabled == true ? "MCOPY" : "JUMPSUB", _ => FastEnum.IsDefined(instruction) ? FastEnum.GetName(instruction) : null }; } diff --git a/src/Nethermind/Nethermind.Evm/MemoryAllowance.cs b/src/Nethermind/Nethermind.Evm/MemoryAllowance.cs index 22c3c95140c..3cc5a7983b1 100644 --- a/src/Nethermind/Nethermind.Evm/MemoryAllowance.cs +++ b/src/Nethermind/Nethermind.Evm/MemoryAllowance.cs @@ -5,6 +5,6 @@ namespace Nethermind.Evm { public static class MemoryAllowance { - public static int CodeCacheSize { get; } = 16_384; + public static int CodeCacheSize { get; } = 4_096 + 1_024; } } diff --git a/src/Nethermind/Nethermind.Evm/Metrics.cs b/src/Nethermind/Nethermind.Evm/Metrics.cs index 06147b13d47..8c3286facc9 100644 --- a/src/Nethermind/Nethermind.Evm/Metrics.cs +++ b/src/Nethermind/Nethermind.Evm/Metrics.cs @@ -3,6 +3,7 @@ using System.ComponentModel; using System.Runtime.CompilerServices; +using System.Threading; using Nethermind.Core.Attributes; @@ -21,15 +22,60 @@ public class Metrics [CounterMetric] [Description("Number of calls to other contracts.")] - public static long Calls { get; set; } + public static long Calls + { + get + { + long total = 0; + foreach (var value in _calls.Values) + { + total += value; + } + return total; + } + } + private static ThreadLocal _calls = new(trackAllValues: true); + [Description("Number of calls to other contracts on thread.")] + public static long ThreadLocalCalls => _calls.Value; + public static void IncrementCalls() => _calls.Value++; [CounterMetric] [Description("Number of SLOAD opcodes executed.")] - public static long SloadOpcode { get; set; } + public static long SloadOpcode + { + get + { + long total = 0; + foreach (var value in _sLoadOpcode.Values) + { + total += value; + } + return total; + } + } + private static ThreadLocal _sLoadOpcode = new(trackAllValues: true); + [Description("Number of SLOAD opcodes executed on thread.")] + public static long ThreadLocalSLoadOpcode => _sLoadOpcode.Value; + public static void IncrementSLoadOpcode() => _sLoadOpcode.Value++; [CounterMetric] [Description("Number of SSTORE opcodes executed.")] - public static long SstoreOpcode { get; set; } + public static long SstoreOpcode + { + get + { + long total = 0; + foreach (var value in _sStoreOpcode.Values) + { + total += value; + } + return total; + } + } + private static ThreadLocal _sStoreOpcode = new(trackAllValues: true); + [Description("Number of SSTORE opcodes executed on thread.")] + public static long ThreadLocalSStoreOpcode => _sStoreOpcode.Value; + public static void IncrementSStoreOpcode() => _sStoreOpcode.Value++; [Description("Number of TLOAD opcodes executed.")] public static long TloadOpcode { get; set; } @@ -40,8 +86,8 @@ public class Metrics [Description("Number of MCOPY opcodes executed.")] public static long MCopyOpcode { get; set; } - [Description("Number of MODEXP precompiles executed.")] - public static long ModExpOpcode { get; set; } + [Description("Number of EXP opcodes executed.")] + public static long ExpOpcode { get; set; } [Description("Number of BLOCKHASH opcodes executed.")] public static long BlockhashOpcode { get; set; } @@ -72,11 +118,60 @@ public class Metrics [CounterMetric] [Description("Number of calls made to addresses without code.")] - public static long EmptyCalls { get; set; } + public static long EmptyCalls + { + get + { + long total = 0; + foreach (var value in _emptyCalls.Values) + { + total += value; + } + return total; + } + } + private static ThreadLocal _emptyCalls = new(trackAllValues: true); + [Description("Number of calls made to addresses without code on thread.")] + public static long ThreadLocalEmptyCalls => _emptyCalls.Value; + public static void IncrementEmptyCalls() => _emptyCalls.Value++; [CounterMetric] [Description("Number of contract create calls.")] - public static long Creates { get; set; } + public static long Creates + { + get + { + long total = 0; + foreach (var value in _creates.Values) + { + total += value; + } + return total; + } + } + private static ThreadLocal _creates = new(trackAllValues: true); + [Description("Number of contract create calls on thread.")] + public static long ThreadLocalCreates => _creates.Value; + public static void IncrementCreates() => _creates.Value++; + + [Description("Number of contracts' code analysed for jump destinations.")] + public static long ContractsAnalysed + { + get + { + long total = 0; + foreach (var value in _contractsAnalysed.Values) + { + total += value; + } + return total; + } + } + private static ThreadLocal _contractsAnalysed = new(trackAllValues: true); + [Description("Number of contracts' code analysed for jump destinations on thread.")] + public static long ThreadLocalContractsAnalysed => _contractsAnalysed.Value; + public static void IncrementContractsAnalysed() => _contractsAnalysed.Value++; + internal static long Transactions { get; set; } internal static float AveGasPrice { get; set; } internal static float MinGasPrice { get; set; } = float.MaxValue; diff --git a/src/Nethermind/Nethermind.Evm/Precompiles/IdentityPrecompile.cs b/src/Nethermind/Nethermind.Evm/Precompiles/IdentityPrecompile.cs index e26dc714acc..4b9273a7c1d 100644 --- a/src/Nethermind/Nethermind.Evm/Precompiles/IdentityPrecompile.cs +++ b/src/Nethermind/Nethermind.Evm/Precompiles/IdentityPrecompile.cs @@ -29,7 +29,7 @@ public long DataGasCost(in ReadOnlyMemory inputData, IReleaseSpec releaseS public (ReadOnlyMemory, bool) Run(in ReadOnlyMemory inputData, IReleaseSpec releaseSpec) { - return (inputData, true); + return (inputData.ToArray(), true); } } } diff --git a/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs index f319531a9f0..4a8587a7690 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs @@ -12,6 +12,7 @@ namespace Nethermind.Evm.Tracing; public class BlockReceiptsTracer : IBlockTracer, ITxTracer, IJournal, ITxTracerWrapper { + private IBlockTracer _otherTracer = NullBlockTracer.Instance; protected Block Block = null!; public bool IsTracingReceipt => true; public bool IsTracingActions => _currentTxTracer.IsTracingActions; @@ -29,8 +30,6 @@ public class BlockReceiptsTracer : IBlockTracer, ITxTracer, IJournal, ITxTr public bool IsTracingFees => _currentTxTracer.IsTracingFees; public bool IsTracingLogs => _currentTxTracer.IsTracingLogs; - private IBlockTracer _otherTracer = NullBlockTracer.Instance; - public void MarkAsSuccess(Address recipient, long gasSpent, byte[] output, LogEntry[] logs, Hash256? stateRoot = null) { _txReceipts.Add(BuildReceipt(recipient, gasSpent, StatusCode.Success, logs, stateRoot)); diff --git a/src/Nethermind/Nethermind.Evm/Tracing/CancellationBlockTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/CancellationBlockTracer.cs index 2fabb220ca2..311a2fedd26 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/CancellationBlockTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/CancellationBlockTracer.cs @@ -7,54 +7,46 @@ namespace Nethermind.Evm.Tracing { - public class CancellationBlockTracer : IBlockTracer + public class CancellationBlockTracer(IBlockTracer innerTracer, CancellationToken token = default) : IBlockTracer { - private readonly IBlockTracer _innerTracer; - private readonly CancellationToken _token; private bool _isTracingRewards; - public CancellationBlockTracer(IBlockTracer innerTracer, CancellationToken token = default) - { - _innerTracer = innerTracer; - _token = token; - } - public bool IsTracingRewards { - get => _isTracingRewards || _innerTracer.IsTracingRewards; + get => _isTracingRewards || innerTracer.IsTracingRewards; set => _isTracingRewards = value; } public void ReportReward(Address author, string rewardType, UInt256 rewardValue) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingRewards) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingRewards) { - _innerTracer.ReportReward(author, rewardType, rewardValue); + innerTracer.ReportReward(author, rewardType, rewardValue); } } public void StartNewBlockTrace(Block block) { - _token.ThrowIfCancellationRequested(); - _innerTracer.StartNewBlockTrace(block); + token.ThrowIfCancellationRequested(); + innerTracer.StartNewBlockTrace(block); } public ITxTracer StartNewTxTrace(Transaction? tx) { - _token.ThrowIfCancellationRequested(); - return _innerTracer.StartNewTxTrace(tx).WithCancellation(_token); + token.ThrowIfCancellationRequested(); + return innerTracer.StartNewTxTrace(tx).WithCancellation(token); } public void EndTxTrace() { - _token.ThrowIfCancellationRequested(); - _innerTracer.EndTxTrace(); + token.ThrowIfCancellationRequested(); + innerTracer.EndTxTrace(); } public void EndBlockTrace() { - _innerTracer.EndBlockTrace(); + innerTracer.EndBlockTrace(); } } } diff --git a/src/Nethermind/Nethermind.Evm/Tracing/CancellationTxTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/CancellationTxTracer.cs index bc6badef0df..5bfd05b9d4a 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/CancellationTxTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/CancellationTxTracer.cs @@ -4,16 +4,15 @@ using System; using System.Collections.Generic; using System.Threading; +using System.Linq; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Int256; namespace Nethermind.Evm.Tracing; -public class CancellationTxTracer : ITxTracer, ITxTracerWrapper +public class CancellationTxTracer(ITxTracer innerTracer, CancellationToken token = default) : ITxTracer, ITxTracerWrapper { - private readonly ITxTracer _innerTracer; - private readonly CancellationToken _token; private readonly bool _isTracingReceipt; private readonly bool _isTracingActions; private readonly bool _isTracingOpLevelStorage; @@ -29,435 +28,431 @@ public class CancellationTxTracer : ITxTracer, ITxTracerWrapper private readonly bool _isTracingFees; private readonly bool _isTracingOpLevelLogs; - public ITxTracer InnerTracer => _innerTracer; - - public CancellationTxTracer(ITxTracer innerTracer, CancellationToken token = default) - { - _innerTracer = innerTracer; - _token = token; - } + public ITxTracer InnerTracer => innerTracer; public bool IsCancelable => true; - public bool IsCancelled => _token.IsCancellationRequested; + public bool IsCancelled => token.IsCancellationRequested; public bool IsTracingReceipt { - get => _isTracingReceipt || _innerTracer.IsTracingReceipt; + get => _isTracingReceipt || innerTracer.IsTracingReceipt; init => _isTracingReceipt = value; } public bool IsTracingActions { - get => _isTracingActions || _innerTracer.IsTracingActions; + get => _isTracingActions || innerTracer.IsTracingActions; init => _isTracingActions = value; } public bool IsTracingOpLevelStorage { - get => _isTracingOpLevelStorage || _innerTracer.IsTracingOpLevelStorage; + get => _isTracingOpLevelStorage || innerTracer.IsTracingOpLevelStorage; init => _isTracingOpLevelStorage = value; } public bool IsTracingMemory { - get => _isTracingMemory || _innerTracer.IsTracingMemory; + get => _isTracingMemory || innerTracer.IsTracingMemory; init => _isTracingMemory = value; } public bool IsTracingInstructions { - get => _isTracingInstructions || _innerTracer.IsTracingInstructions; + get => _isTracingInstructions || innerTracer.IsTracingInstructions; init => _isTracingInstructions = value; } public bool IsTracingRefunds { - get => _isTracingRefunds || _innerTracer.IsTracingRefunds; + get => _isTracingRefunds || innerTracer.IsTracingRefunds; init => _isTracingRefunds = value; } public bool IsTracingCode { - get => _isTracingCode || _innerTracer.IsTracingCode; + get => _isTracingCode || innerTracer.IsTracingCode; init => _isTracingCode = value; } public bool IsTracingStack { - get => _isTracingStack || _innerTracer.IsTracingStack; + get => _isTracingStack || innerTracer.IsTracingStack; init => _isTracingStack = value; } public bool IsTracingState { - get => _isTracingState || _innerTracer.IsTracingState; + get => _isTracingState || innerTracer.IsTracingState; init => _isTracingState = value; } public bool IsTracingStorage { - get => _isTracingStorage || _innerTracer.IsTracingStorage; + get => _isTracingStorage || innerTracer.IsTracingStorage; init => _isTracingStorage = value; } public bool IsTracingBlockHash { - get => _isTracingBlockHash || _innerTracer.IsTracingBlockHash; + get => _isTracingBlockHash || innerTracer.IsTracingBlockHash; init => _isTracingBlockHash = value; } public bool IsTracingAccess { - get => _isTracingBlockAccess || _innerTracer.IsTracingAccess; + get => _isTracingBlockAccess || innerTracer.IsTracingAccess; init => _isTracingBlockAccess = value; } public bool IsTracingFees { - get => _isTracingFees || _innerTracer.IsTracingFees; + get => _isTracingFees || innerTracer.IsTracingFees; init => _isTracingFees = value; } + public bool IsTracingLogs { - get => _isTracingOpLevelLogs || _innerTracer.IsTracingLogs; + get => _isTracingOpLevelLogs || innerTracer.IsTracingLogs; init => _isTracingOpLevelLogs = value; } + public void ReportBalanceChange(Address address, UInt256? before, UInt256? after) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingState) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingState) { - _innerTracer.ReportBalanceChange(address, before, after); + innerTracer.ReportBalanceChange(address, before, after); } } public void ReportCodeChange(Address address, byte[] before, byte[] after) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingState) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingState) { - _innerTracer.ReportCodeChange(address, before, after); + innerTracer.ReportCodeChange(address, before, after); } } public void ReportNonceChange(Address address, UInt256? before, UInt256? after) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingState) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingState) { - _innerTracer.ReportNonceChange(address, before, after); + innerTracer.ReportNonceChange(address, before, after); } } public void ReportAccountRead(Address address) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingState) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingState) { - _innerTracer.ReportAccountRead(address); + innerTracer.ReportAccountRead(address); } } public void ReportStorageChange(in StorageCell storageCell, byte[] before, byte[] after) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingStorage) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingStorage) { - _innerTracer.ReportStorageChange(storageCell, before, after); + innerTracer.ReportStorageChange(storageCell, before, after); } } public void ReportStorageRead(in StorageCell storageCell) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingStorage) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingStorage) { - _innerTracer.ReportStorageRead(storageCell); + innerTracer.ReportStorageRead(storageCell); } } public void MarkAsSuccess(Address recipient, long gasSpent, byte[] output, LogEntry[] logs, Hash256? stateRoot = null) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingReceipt) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingReceipt) { - _innerTracer.MarkAsSuccess(recipient, gasSpent, output, logs, stateRoot); + innerTracer.MarkAsSuccess(recipient, gasSpent, output, logs, stateRoot); } } public void MarkAsFailed(Address recipient, long gasSpent, byte[] output, string error, Hash256? stateRoot = null) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingReceipt) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingReceipt) { - _innerTracer.MarkAsFailed(recipient, gasSpent, output, error, stateRoot); + innerTracer.MarkAsFailed(recipient, gasSpent, output, error, stateRoot); } } public void StartOperation(int pc, Instruction opcode, long gas, in ExecutionEnvironment env) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingInstructions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingInstructions) { - _innerTracer.StartOperation(pc, opcode, gas, env); + innerTracer.StartOperation(pc, opcode, gas, env); } } public void ReportOperationError(EvmExceptionType error) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingInstructions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingInstructions) { - _innerTracer.ReportOperationError(error); + innerTracer.ReportOperationError(error); } } public void ReportOperationRemainingGas(long gas) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingInstructions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingInstructions) { - _innerTracer.ReportOperationRemainingGas(gas); + innerTracer.ReportOperationRemainingGas(gas); } } public void ReportLog(LogEntry log) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingLogs) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingLogs) { - _innerTracer.ReportLog(log); + innerTracer.ReportLog(log); } } public void SetOperationStack(TraceStack stack) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingStack) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingStack) { - _innerTracer.SetOperationStack(stack); + innerTracer.SetOperationStack(stack); } } public void ReportStackPush(in ReadOnlySpan stackItem) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingInstructions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingInstructions) { - _innerTracer.ReportStackPush(stackItem); + innerTracer.ReportStackPush(stackItem); } } public void ReportStackPush(in ZeroPaddedSpan stackItem) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingInstructions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingInstructions) { - _innerTracer.ReportStackPush(stackItem); + innerTracer.ReportStackPush(stackItem); } } public void ReportStackPush(byte stackItem) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingInstructions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingInstructions) { - _innerTracer.ReportStackPush(stackItem); + innerTracer.ReportStackPush(stackItem); } } public void SetOperationMemory(TraceMemory memoryTrace) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingMemory) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingMemory) { - _innerTracer.SetOperationMemory(memoryTrace); + innerTracer.SetOperationMemory(memoryTrace); } } public void SetOperationMemorySize(ulong newSize) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingMemory) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingMemory) { - _innerTracer.SetOperationMemorySize(newSize); + innerTracer.SetOperationMemorySize(newSize); } } public void ReportMemoryChange(long offset, in ReadOnlySpan data) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingInstructions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingInstructions) { - _innerTracer.ReportMemoryChange(offset, data); + innerTracer.ReportMemoryChange(offset, data); } } public void ReportMemoryChange(long offset, in ZeroPaddedSpan data) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingInstructions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingInstructions) { - _innerTracer.ReportMemoryChange(offset, data); + innerTracer.ReportMemoryChange(offset, data); } } public void ReportMemoryChange(long offset, byte data) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingInstructions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingInstructions) { - _innerTracer.ReportMemoryChange(offset, data); + innerTracer.ReportMemoryChange(offset, data); } } public void ReportStorageChange(in ReadOnlySpan key, in ReadOnlySpan value) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingInstructions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingInstructions) { - _innerTracer.ReportStorageChange(key, value); + innerTracer.ReportStorageChange(key, value); } } public void SetOperationStorage(Address address, UInt256 storageIndex, ReadOnlySpan newValue, ReadOnlySpan currentValue) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingOpLevelStorage) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingOpLevelStorage) { - _innerTracer.SetOperationStorage(address, storageIndex, newValue, currentValue); + innerTracer.SetOperationStorage(address, storageIndex, newValue, currentValue); } } public void LoadOperationStorage(Address address, UInt256 storageIndex, ReadOnlySpan value) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingOpLevelStorage) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingOpLevelStorage) { - _innerTracer.LoadOperationStorage(address, storageIndex, value); + innerTracer.LoadOperationStorage(address, storageIndex, value); } } public void ReportSelfDestruct(Address address, UInt256 balance, Address refundAddress) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingActions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingActions) { - _innerTracer.ReportSelfDestruct(address, balance, refundAddress); + innerTracer.ReportSelfDestruct(address, balance, refundAddress); } } public void ReportAction(long gas, UInt256 value, Address from, Address to, ReadOnlyMemory input, ExecutionType callType, bool isPrecompileCall = false) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingActions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingActions) { - _innerTracer.ReportAction(gas, value, from, to, input, callType, isPrecompileCall); + innerTracer.ReportAction(gas, value, from, to, input, callType, isPrecompileCall); } } public void ReportActionEnd(long gas, ReadOnlyMemory output) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingActions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingActions) { - _innerTracer.ReportActionEnd(gas, output); + innerTracer.ReportActionEnd(gas, output); } } public void ReportActionError(EvmExceptionType evmExceptionType) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingActions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingActions) { - _innerTracer.ReportActionError(evmExceptionType); + innerTracer.ReportActionError(evmExceptionType); } } public void ReportActionRevert(long gasLeft, ReadOnlyMemory output) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingActions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingActions) { - _innerTracer.ReportActionRevert(gasLeft, output); + innerTracer.ReportActionRevert(gasLeft, output); } } public void ReportActionEnd(long gas, Address deploymentAddress, ReadOnlyMemory deployedCode) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingActions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingActions) { - _innerTracer.ReportActionEnd(gas, deploymentAddress, deployedCode); + innerTracer.ReportActionEnd(gas, deploymentAddress, deployedCode); } } public void ReportBlockHash(Hash256 blockHash) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingBlockHash) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingBlockHash) { - _innerTracer.ReportBlockHash(blockHash); + innerTracer.ReportBlockHash(blockHash); } } public void ReportByteCode(ReadOnlyMemory byteCode) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingCode) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingCode) { - _innerTracer.ReportByteCode(byteCode); + innerTracer.ReportByteCode(byteCode); } } public void ReportGasUpdateForVmTrace(long refund, long gasAvailable) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingInstructions) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingInstructions) { - _innerTracer.ReportGasUpdateForVmTrace(refund, gasAvailable); + innerTracer.ReportGasUpdateForVmTrace(refund, gasAvailable); } } public void ReportRefund(long refund) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingRefunds) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingRefunds) { - _innerTracer.ReportRefund(refund); + innerTracer.ReportRefund(refund); } } public void ReportExtraGasPressure(long extraGasPressure) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingRefunds) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingRefunds) { - _innerTracer.ReportExtraGasPressure(extraGasPressure); + innerTracer.ReportExtraGasPressure(extraGasPressure); } } public void ReportAccess(IReadOnlySet
accessedAddresses, IReadOnlySet accessedStorageCells) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingAccess) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingAccess) { - _innerTracer.ReportAccess(accessedAddresses, accessedStorageCells); + innerTracer.ReportAccess(accessedAddresses, accessedStorageCells); } } public void ReportFees(UInt256 fees, UInt256 burntFees) { - _token.ThrowIfCancellationRequested(); - if (_innerTracer.IsTracingFees) + token.ThrowIfCancellationRequested(); + if (innerTracer.IsTracingFees) { - _innerTracer.ReportFees(fees, burntFees); + innerTracer.ReportFees(fees, burntFees); } } public void Dispose() { - _innerTracer.Dispose(); + innerTracer.Dispose(); } } diff --git a/src/Nethermind/Nethermind.Evm/Tracing/ITxLogsMutator.cs b/src/Nethermind/Nethermind.Evm/Tracing/ITxLogsMutator.cs new file mode 100644 index 00000000000..fad85dc3b1f --- /dev/null +++ b/src/Nethermind/Nethermind.Evm/Tracing/ITxLogsMutator.cs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using Nethermind.Core; + +namespace Nethermind.Evm.Tracing; + +public interface ITxLogsMutator +{ + bool IsMutatingLogs { get; } + + void SetLogsToMutate(ICollection logsToMutate); +} diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/BuildUpTransactionProcessorAdapter.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/BuildUpTransactionProcessorAdapter.cs index d0215a41aa3..2ac0a3b8fcb 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/BuildUpTransactionProcessorAdapter.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/BuildUpTransactionProcessorAdapter.cs @@ -6,16 +6,9 @@ namespace Nethermind.Evm.TransactionProcessing { - public class BuildUpTransactionProcessorAdapter : ITransactionProcessorAdapter + public class BuildUpTransactionProcessorAdapter(ITransactionProcessor transactionProcessor) : ITransactionProcessorAdapter { - private readonly ITransactionProcessor _transactionProcessor; - - public BuildUpTransactionProcessorAdapter(ITransactionProcessor transactionProcessor) - { - _transactionProcessor = transactionProcessor; - } - public TransactionResult Execute(Transaction transaction, in BlockExecutionContext blkCtx, ITxTracer txTracer) => - _transactionProcessor.BuildUp(transaction, in blkCtx, txTracer); + transactionProcessor.BuildUp(transaction, in blkCtx, txTracer); } } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/IReadOnlyTransactionProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/IReadOnlyTransactionProcessor.cs deleted file mode 100644 index 20b46e3c280..00000000000 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/IReadOnlyTransactionProcessor.cs +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using Nethermind.Core; - -namespace Nethermind.Evm.TransactionProcessing -{ - public interface IReadOnlyTransactionProcessor : ITransactionProcessor, IDisposable - { - bool IsContractDeployed(Address address); - } -} diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/IReadOnlyTxProcessingScope.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/IReadOnlyTxProcessingScope.cs new file mode 100644 index 00000000000..ce7bb61e8c3 --- /dev/null +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/IReadOnlyTxProcessingScope.cs @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.State; + +namespace Nethermind.Evm.TransactionProcessing; + +public interface IReadOnlyTxProcessingScope : IDisposable +{ + ITransactionProcessor TransactionProcessor { get; } + IWorldState WorldState { get; } +} diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/IReadOnlyTxProcessorSource.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/IReadOnlyTxProcessorSource.cs index 65d44d66986..97f7f5f960a 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/IReadOnlyTxProcessorSource.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/IReadOnlyTxProcessorSource.cs @@ -7,6 +7,6 @@ namespace Nethermind.Evm.TransactionProcessing { public interface IReadOnlyTxProcessorSource { - IReadOnlyTransactionProcessor Build(Hash256 stateRoot); + IReadOnlyTxProcessingScope Build(Hash256 stateRoot); } } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/ReadOnlyTransactionProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/ReadOnlyTransactionProcessor.cs deleted file mode 100644 index 026427b6413..00000000000 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/ReadOnlyTransactionProcessor.cs +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using Nethermind.Core; -using Nethermind.Core.Crypto; -using Nethermind.Evm.Tracing; -using Nethermind.State; - -namespace Nethermind.Evm.TransactionProcessing -{ - public class ReadOnlyTransactionProcessor : IReadOnlyTransactionProcessor - { - private readonly ITransactionProcessor _transactionProcessor; - private readonly IWorldState _stateProvider; - private readonly Hash256 _stateBefore; - - public ReadOnlyTransactionProcessor(ITransactionProcessor transactionProcessor, IWorldState stateProvider, Hash256 startState) - { - _transactionProcessor = transactionProcessor ?? throw new ArgumentNullException(nameof(transactionProcessor)); - _stateProvider = stateProvider ?? throw new ArgumentNullException(nameof(stateProvider)); - _stateBefore = _stateProvider.StateRoot; - _stateProvider.StateRoot = startState ?? throw new ArgumentNullException(nameof(startState)); - } - - public TransactionResult Execute(Transaction transaction, in BlockExecutionContext blCtx, ITxTracer txTracer) => - _transactionProcessor.Execute(transaction, in blCtx, txTracer); - - public TransactionResult CallAndRestore(Transaction transaction, in BlockExecutionContext blCtx, ITxTracer txTracer) => - _transactionProcessor.CallAndRestore(transaction, in blCtx, txTracer); - - public TransactionResult BuildUp(Transaction transaction, in BlockExecutionContext blCtx, ITxTracer txTracer) => - _transactionProcessor.BuildUp(transaction, in blCtx, txTracer); - - public TransactionResult Trace(Transaction transaction, in BlockExecutionContext blCtx, ITxTracer txTracer) => - _transactionProcessor.Trace(transaction, in blCtx, txTracer); - - - public bool IsContractDeployed(Address address) => _stateProvider.IsContract(address); - - public void Dispose() - { - _stateProvider.StateRoot = _stateBefore; - _stateProvider.Reset(); - } - } -} diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/SystemTxProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/SystemTxProcessor.cs index f16eaa22c15..333ce0dcd88 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/SystemTxProcessor.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/SystemTxProcessor.cs @@ -32,10 +32,12 @@ public class SystemTxProcessor : ITransactionProcessor protected IWorldState WorldState { get; private init; } protected IVirtualMachine VirtualMachine { get; private init; } + private readonly ICodeInfoRepository _codeInfoRepository; public SystemTxProcessor( IReleaseSpec? spec, IWorldState? worldState, IVirtualMachine? virtualMachine, + ICodeInfoRepository? codeInfoRepository, EthereumEcdsa? ecdsa, ILogger? logger) { @@ -43,6 +45,7 @@ public SystemTxProcessor( Spec = spec ?? throw new ArgumentNullException(nameof(spec)); WorldState = worldState ?? throw new ArgumentNullException(nameof(worldState)); VirtualMachine = virtualMachine ?? throw new ArgumentNullException(nameof(virtualMachine)); + _codeInfoRepository = codeInfoRepository ?? throw new ArgumentNullException(nameof(codeInfoRepository)); Ecdsa = ecdsa ?? throw new ArgumentNullException(nameof(ecdsa)); } public TransactionResult CallAndRestore(Transaction transaction, in BlockExecutionContext blCtx, ITxTracer txTracer) => @@ -304,7 +307,7 @@ protected virtual ExecutionEnvironment BuildExecutionEnvironment( TxExecutionContext executionContext = new(in blCtx, tx.SenderAddress, effectiveGasPrice, tx.BlobVersionedHashes); CodeInfo codeInfo = tx.IsContractCreation ? new(tx.Data?.AsArray() ?? Array.Empty()) - : VirtualMachine.GetCachedCodeInfo(WorldState, recipient, spec); + : _codeInfoRepository.GetCachedCodeInfo(WorldState, recipient, spec); byte[] inputData = tx.IsMessageCall ? tx.Data.AsArray() ?? Array.Empty() : Array.Empty(); @@ -415,7 +418,7 @@ protected void ExecuteEvmCall( if (unspentGas >= codeDepositGasCost) { var code = substate.Output.ToArray(); - VirtualMachine.InsertCode(code, env.ExecutingAccount, spec, env.IsSystemEnv); + _codeInfoRepository.InsertCode(WorldState, code, env.ExecutingAccount, spec, env.IsSystemEnv); unspentGas -= codeDepositGasCost; } @@ -454,7 +457,7 @@ protected void PrepareAccountForContractDeployment(Address contractAddress, IRel { if (WorldState.AccountExists(contractAddress)) { - CodeInfo codeInfo = VirtualMachine.GetCachedCodeInfo(WorldState, contractAddress, spec); + CodeInfo codeInfo = _codeInfoRepository.GetCachedCodeInfo(WorldState, contractAddress, spec); bool codeIsNotEmpty = codeInfo.MachineCode.Length != 0; bool accountNonceIsNotZero = WorldState.GetNonce(contractAddress) != 0; diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs index fd6bb9cd4d1..b6ee586f42a 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs @@ -7,8 +7,6 @@ using System.IO; using System.Linq; using System.Runtime.CompilerServices; -using System.Threading; - using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Specs; @@ -33,22 +31,27 @@ public class TransactionProcessor : ITransactionProcessor protected ISpecProvider SpecProvider { get; private init; } protected IWorldState WorldState { get; private init; } protected IVirtualMachine VirtualMachine { get; private init; } + private readonly ICodeInfoRepository _codeInfoRepository; public TransactionProcessor( ISpecProvider? specProvider, IWorldState? worldState, IVirtualMachine? virtualMachine, + ICodeInfoRepository? codeInfoRepository, ILogManager? logManager) { ArgumentNullException.ThrowIfNull(logManager, nameof(logManager)); ArgumentNullException.ThrowIfNull(specProvider, nameof(specProvider)); ArgumentNullException.ThrowIfNull(worldState, nameof(worldState)); ArgumentNullException.ThrowIfNull(virtualMachine, nameof(virtualMachine)); + ArgumentNullException.ThrowIfNull(codeInfoRepository, nameof(codeInfoRepository)); Logger = logManager.GetClassLogger(); SpecProvider = specProvider; WorldState = worldState; VirtualMachine = virtualMachine; + _codeInfoRepository = codeInfoRepository; + Ecdsa = new EthereumEcdsa(specProvider.ChainId, logManager); } @@ -75,7 +78,7 @@ protected virtual TransactionResult Execute(Transaction tx, in BlockExecutionCon if (tx.IsSystem()) { - ITransactionProcessor systemProcessor = new SystemTxProcessor(spec, WorldState, VirtualMachine, Ecdsa, Logger); + ITransactionProcessor systemProcessor = new SystemTxProcessor(spec, WorldState, VirtualMachine, _codeInfoRepository, Ecdsa, Logger); return systemProcessor.Execute(tx, blCtx, tracer); } @@ -99,7 +102,7 @@ protected virtual TransactionResult Execute(Transaction tx, in BlockExecutionCon if (!(result = BuyGas(tx, header, spec, tracer, opts, effectiveGasPrice, out UInt256 premiumPerGas, out UInt256 senderReservedGasPayment))) return result; if (!(result = IncrementNonce(tx, header, spec, tracer, opts))) return result; - if (commit) WorldState.Commit(spec, tracer.IsTracingState ? tracer : NullTxTracer.Instance); + if (commit) WorldState.Commit(spec, tracer.IsTracingState ? tracer : NullTxTracer.Instance, commitStorageRoots: false); ExecutionEnvironment env = BuildExecutionEnvironment(tx, in blCtx, spec, effectiveGasPrice); @@ -127,7 +130,7 @@ protected virtual TransactionResult Execute(Transaction tx, in BlockExecutionCon } else if (commit) { - WorldState.Commit(spec, tracer.IsTracingState ? tracer : NullStateTracer.Instance); + WorldState.Commit(spec, tracer.IsTracingState ? tracer : NullStateTracer.Instance, commitStorageRoots: !spec.IsEip658Enabled); } if (tracer.IsTracingReceipt) @@ -389,7 +392,9 @@ protected ExecutionEnvironment BuildExecutionEnvironment( CodeInfo codeInfo = tx.IsContractCreation ? new(tx.Data ?? Memory.Empty) - : VirtualMachine.GetCachedCodeInfo(WorldState, recipient, spec); + : _codeInfoRepository.GetCachedCodeInfo(WorldState, recipient, spec); + + codeInfo.AnalyseInBackgroundIfRequired(); byte[] inputData = tx.IsMessageCall ? tx.Data.AsArray() ?? Array.Empty() : Array.Empty(); @@ -499,7 +504,8 @@ protected void ExecuteEvmCall( if (unspentGas >= codeDepositGasCost) { var code = substate.Output.ToArray(); - VirtualMachine.InsertCode(code, env.ExecutingAccount, spec, env.IsSystemEnv); + _codeInfoRepository.InsertCode(WorldState, code, env.ExecutingAccount, spec, env.IsSystemEnv); + unspentGas -= codeDepositGasCost; } } @@ -553,7 +559,7 @@ protected void PrepareAccountForContractDeployment(Address contractAddress, IRel { if (WorldState.AccountExists(contractAddress)) { - CodeInfo codeInfo = VirtualMachine.GetCachedCodeInfo(WorldState, contractAddress, spec); + CodeInfo codeInfo = _codeInfoRepository.GetCachedCodeInfo(WorldState, contractAddress, spec); bool codeIsNotEmpty = codeInfo.MachineCode.Length != 0; bool accountNonceIsNotZero = WorldState.GetNonce(contractAddress) != 0; diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index b219520dfb3..51b7b742fbf 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -6,23 +6,18 @@ using System.Collections.Generic; using System.Runtime.CompilerServices; using Nethermind.Core; -using Nethermind.Core.Caching; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Evm.CodeAnalysis; using Nethermind.Evm.Precompiles; -using Nethermind.Evm.Precompiles.Bls; -using Nethermind.Evm.Precompiles.Snarks; using Nethermind.Evm.Tracing; using Nethermind.Logging; using Nethermind.State; using System.Diagnostics.CodeAnalysis; -using System.Diagnostics; using System.Runtime.Intrinsics; using static Nethermind.Evm.VirtualMachine; using static System.Runtime.CompilerServices.Unsafe; -using ValueHash256 = Nethermind.Core.Crypto.ValueHash256; #if DEBUG using Nethermind.Evm.Tracing.Debugger; @@ -34,23 +29,23 @@ namespace Nethermind.Evm; using System.Collections.Frozen; using System.Linq; +using System.Runtime.InteropServices; using System.Threading; using Int256; using Nethermind.Crypto; +using Nethermind.Core.Collections; + public class VirtualMachine : IVirtualMachine { public const int MaxCallDepth = 1024; - internal static FrozenDictionary PrecompileCode { get; } = InitializePrecompiledContracts(); - internal static LruCache CodeCache { get; } = new(MemoryAllowance.CodeCacheSize, MemoryAllowance.CodeCacheSize, "VM bytecodes"); - - private readonly static UInt256 P255Int = (UInt256)System.Numerics.BigInteger.Pow(2, 255); + private static readonly UInt256 P255Int = (UInt256)System.Numerics.BigInteger.Pow(2, 255); internal static ref readonly UInt256 P255 => ref P255Int; internal static readonly UInt256 BigInt256 = 256; internal static readonly UInt256 BigInt32 = 32; - internal static readonly byte[] BytesZero = { 0 }; + internal static readonly byte[] BytesZero = [0]; internal static readonly byte[] BytesZero32 = { @@ -73,61 +68,19 @@ public class VirtualMachine : IVirtualMachine public VirtualMachine( IBlockhashProvider? blockhashProvider, ISpecProvider? specProvider, + ICodeInfoRepository codeInfoRepository, ILogManager? logManager) { ILogger logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - if (!logger.IsTrace) - { - _evm = new VirtualMachine(blockhashProvider, specProvider, logger); - } - else - { - _evm = new VirtualMachine(blockhashProvider, specProvider, logger); - } - } - - public CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IReleaseSpec spec) - => _evm.GetCachedCodeInfo(worldState, codeSource, spec); - - public void InsertCode(ReadOnlyMemory code, Address codeOwner, IReleaseSpec spec, bool isSystemEnv) - { - _evm.InsertCode(code, codeOwner, spec, isSystemEnv); + _evm = logger.IsTrace + ? new VirtualMachine(blockhashProvider, specProvider, codeInfoRepository, logger) + : new VirtualMachine(blockhashProvider, specProvider, codeInfoRepository, logger); } public TransactionSubstate Run(EvmState state, IWorldState worldState, ITxTracer txTracer) where TTracingActions : struct, VirtualMachine.IIsTracing => _evm.Run(state, worldState, txTracer); - private static FrozenDictionary InitializePrecompiledContracts() - { - return new Dictionary - { - [EcRecoverPrecompile.Address] = new(EcRecoverPrecompile.Instance), - [Sha256Precompile.Address] = new(Sha256Precompile.Instance), - [Ripemd160Precompile.Address] = new(Ripemd160Precompile.Instance), - [IdentityPrecompile.Address] = new(IdentityPrecompile.Instance), - - [Bn254AddPrecompile.Address] = new(Bn254AddPrecompile.Instance), - [Bn254MulPrecompile.Address] = new(Bn254MulPrecompile.Instance), - [Bn254PairingPrecompile.Address] = new(Bn254PairingPrecompile.Instance), - [ModExpPrecompile.Address] = new(ModExpPrecompile.Instance), - - [Blake2FPrecompile.Address] = new(Blake2FPrecompile.Instance), - - [G1AddPrecompile.Address] = new(G1AddPrecompile.Instance), - [G1MulPrecompile.Address] = new(G1MulPrecompile.Instance), - [G1MultiExpPrecompile.Address] = new(G1MultiExpPrecompile.Instance), - [G2AddPrecompile.Address] = new(G2AddPrecompile.Instance), - [G2MulPrecompile.Address] = new(G2MulPrecompile.Instance), - [G2MultiExpPrecompile.Address] = new(G2MultiExpPrecompile.Instance), - [PairingPrecompile.Address] = new(PairingPrecompile.Instance), - [MapToG1Precompile.Address] = new(MapToG1Precompile.Instance), - [MapToG2Precompile.Address] = new(MapToG2Precompile.Instance), - - [PointEvaluationPrecompile.Address] = new(PointEvaluationPrecompile.Instance), - }.ToFrozenDictionary(); - } - internal readonly ref struct CallResult { public static CallResult InvalidSubroutineEntry => new(EvmExceptionType.InvalidSubroutineEntry); @@ -141,8 +94,8 @@ internal readonly ref struct CallResult public static CallResult StackUnderflowException => new(EvmExceptionType.StackUnderflow); // TODO: use these to avoid CALL POP attacks public static CallResult InvalidCodeException => new(EvmExceptionType.InvalidCode); public static CallResult AuthorizedNotSet => new(EvmExceptionType.AuthorizedNotSet); - - public static CallResult Empty => new(Array.Empty(), null); + public static CallResult Empty => new(default, null); + public static object BoxedEmpty { get; } = new object(); public CallResult(EvmState stateToExecute) { @@ -162,7 +115,7 @@ private CallResult(EvmExceptionType exceptionType) ExceptionType = exceptionType; } - public CallResult(byte[] output, bool? precompileSuccess, bool shouldRevert = false, EvmExceptionType exceptionType = EvmExceptionType.None) + public CallResult(ReadOnlyMemory output, bool? precompileSuccess, bool shouldRevert = false, EvmExceptionType exceptionType = EvmExceptionType.None) { StateToExecute = null; Output = output; @@ -193,22 +146,25 @@ internal sealed class VirtualMachine : IVirtualMachine where TLogger : private readonly ISpecProvider _specProvider; private readonly ILogger _logger; private IWorldState _worldState; - private IWorldState _state; + private IWorldState _state = null!; private readonly Stack _stateStack = new(); private (Address Address, bool ShouldDelete) _parityTouchBugAccount = (Address.FromNumber(3), false); private ReadOnlyMemory _returnDataBuffer = Array.Empty(); private ITxTracer _txTracer = NullTxTracer.Instance; + private readonly ICodeInfoRepository _codeInfoRepository; private EthereumEcdsa _ecdsa; public VirtualMachine( IBlockhashProvider? blockhashProvider, ISpecProvider? specProvider, - ILogger logger) + ICodeInfoRepository codeInfoRepository, + ILogger? logger) { - _logger = logger; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); _blockhashProvider = blockhashProvider ?? throw new ArgumentNullException(nameof(blockhashProvider)); _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); + _codeInfoRepository = codeInfoRepository ?? throw new ArgumentNullException(nameof(codeInfoRepository)); _chainId = ((UInt256)specProvider.ChainId).ToBigEndian(); _ecdsa = new EthereumEcdsa(specProvider.ChainId, LimboLogs.Instance); } @@ -263,18 +219,21 @@ public TransactionSubstate Run(EvmState state, IWorldState worl { if (typeof(TTracingActions) == typeof(IsTracing) && !currentState.IsContinuation) { - _txTracer.ReportAction(currentState.GasAvailable, currentState.Env.Value, currentState.From, currentState.To, currentState.ExecutionType.IsAnyCreate() ? currentState.Env.CodeInfo.MachineCode : currentState.Env.InputData, currentState.ExecutionType); + _txTracer.ReportAction(currentState.GasAvailable, + currentState.Env.Value, + currentState.From, + currentState.To, + currentState.ExecutionType.IsAnyCreate() + ? currentState.Env.CodeInfo.MachineCode + : currentState.Env.InputData, + currentState.ExecutionType); + if (_txTracer.IsTracingCode) _txTracer.ReportByteCode(currentState.Env.CodeInfo.MachineCode); } - if (!_txTracer.IsTracingInstructions) - { - callResult = ExecuteCall(currentState, previousCallResult, previousCallOutput, previousCallOutputDestination, spec); - } - else - { - callResult = ExecuteCall(currentState, previousCallResult, previousCallOutput, previousCallOutputDestination, spec); - } + callResult = !_txTracer.IsTracingInstructions + ? ExecuteCall(currentState, previousCallResult, previousCallOutput, previousCallOutputDestination, spec) + : ExecuteCall(currentState, previousCallResult, previousCallOutput, previousCallOutputDestination, spec); if (!callResult.IsReturn) { @@ -391,7 +350,8 @@ public TransactionSubstate Run(EvmState state, IWorldState worl if (gasAvailableForCodeDeposit >= codeDepositGasCost && !invalidCode) { ReadOnlyMemory code = callResult.Output; - InsertCode(code, callCodeOwner, spec, state.Env.IsSystemEnv); + _codeInfoRepository.InsertCode(_state, code, callCodeOwner, spec, state.Env.IsSystemEnv); + currentState.GasAvailable -= codeDepositGasCost; if (typeof(TTracingActions) == typeof(IsTracing)) @@ -499,17 +459,6 @@ public TransactionSubstate Run(EvmState state, IWorldState worl } } - public void InsertCode(ReadOnlyMemory code, Address callCodeOwner, IReleaseSpec spec, bool IsSystemTxEnv) - { - var codeInfo = new CodeInfo(code); - // Start generating the JumpDestinationBitmap in background. - ThreadPool.UnsafeQueueUserWorkItem(codeInfo, preferLocal: false); - - Hash256 codeHash = code.Length == 0 ? Keccak.OfAnEmptyString : Keccak.Compute(code.Span); - _state.InsertCode(callCodeOwner, codeHash, code, spec, IsSystemTxEnv); - CodeCache.Set(codeHash, codeInfo); - } - private void RevertParityTouchBugAccount(IReleaseSpec spec, bool isSystemEnv) { if (_parityTouchBugAccount.ShouldDelete) @@ -523,50 +472,6 @@ private void RevertParityTouchBugAccount(IReleaseSpec spec, bool isSystemEnv) } } - public CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IReleaseSpec vmSpec) - { - if (codeSource.IsPrecompile(vmSpec)) - { - return PrecompileCode[codeSource]; - } - - CodeInfo cachedCodeInfo = null; - ValueHash256 codeHash = worldState.GetCodeHash(codeSource); - if (codeHash == Keccak.OfAnEmptyString.ValueHash256) - { - cachedCodeInfo = CodeInfo.Empty; - } - - cachedCodeInfo ??= CodeCache.Get(codeHash); - if (cachedCodeInfo is null) - { - byte[]? code = worldState.GetCode(codeHash); - - if (code is null) - { - MissingCode(codeSource, codeHash); - } - - cachedCodeInfo = new CodeInfo(code); - CodeCache.Set(codeHash, cachedCodeInfo); - } - else - { - Db.Metrics.CodeDbCache++; - // need to touch code so that any collectors that track database access are informed - worldState.TouchCode(codeHash); - } - - return cachedCodeInfo; - - [DoesNotReturn] - [StackTraceHidden] - static void MissingCode(Address codeSource, in ValueHash256 codeHash) - { - throw new NullReferenceException($"Code {codeHash} missing in the state for address {codeSource}"); - } - } - private static bool UpdateGas(long gasCost, ref long gasAvailable) { if (gasAvailable < gasCost) @@ -695,7 +600,7 @@ private CallResult ExecutePrecompile(EvmState state, IReleaseSpec spec) try { (ReadOnlyMemory output, bool success) = precompile.Run(callData, spec); - CallResult callResult = new(output.ToArray(), success, !success); + CallResult callResult = new(output, success, !success); return callResult; } catch (DllNotFoundException exception) @@ -706,7 +611,7 @@ private CallResult ExecutePrecompile(EvmState state, IReleaseSpec spec) catch (Exception exception) { if (_logger.IsError) _logger.Error($"Precompiled contract ({precompile.GetType()}) execution exception", exception); - CallResult callResult = new(Array.Empty(), false, true); + CallResult callResult = new(default, false, true); return callResult; } } @@ -742,7 +647,7 @@ private CallResult ExecuteCall(EvmState vmState, ReadOnlyM { if (!vmState.IsTopLevel) { - Metrics.EmptyCalls++; + Metrics.IncrementEmptyCalls(); } goto Empty; } @@ -986,7 +891,7 @@ private CallResult ExecuteCode externalCode = GetCachedCodeInfo(_worldState, address, spec).MachineCode; + ReadOnlyMemory externalCode = _codeInfoRepository.GetCachedCodeInfo(_worldState, address, spec).MachineCode; slice = externalCode.SliceWithZeroPadding(b, (int)result); vmState.Memory.Save(in a, in slice); if (typeof(TTracingInstructions) == typeof(IsTracing)) @@ -1843,7 +1748,7 @@ private CallResult ExecuteCode value = _state.GetTransientState(in storageCell); - stack.PushBytes(value); + if (!stack.PopUInt256(out result)) goto StackUnderflow; + storageCell = new(env.ExecutingAccount, result); - if (typeof(TTracingStorage) == typeof(IsTracing)) - { - if (gasAvailable < 0) goto OutOfGas; - _txTracer.LoadOperationTransientStorage(storageCell.Address, result, value); - } + ReadOnlySpan value = _state.GetTransientState(in storageCell); + stack.PushBytes(value); - break; - } - else + if (typeof(TTracingStorage) == typeof(IsTracing)) { - if (!spec.SubroutinesEnabled) goto InvalidInstruction; - - // why do we even need the cost of it? - gasAvailable -= GasCostOf.Base; - - goto InvalidSubroutineEntry; + if (gasAvailable < 0) goto OutOfGas; + _txTracer.LoadOperationTransientStorage(storageCell.Address, result, value); } + break; } - case Instruction.RETURNSUB | Instruction.TSTORE: + case Instruction.TSTORE: { - if (spec.TransientStorageEnabled) + if (!spec.TransientStorageEnabled) goto InvalidInstruction; { Metrics.TstoreOpcode++; @@ -2055,24 +1954,10 @@ private CallResult ExecuteCode(EvmState vmState, [MethodImpl(MethodImplOptions.NoInlining)] private void InstructionExtCodeSize(Address address, ref EvmStack stack, IReleaseSpec spec) where TTracingInstructions : struct, IIsTracing { - int codeLength = GetCachedCodeInfo(_worldState, address, spec).MachineCode.Span.Length; - UInt256 result = (UInt256)codeLength; + ReadOnlyMemory accountCode = _codeInfoRepository.GetCachedCodeInfo(_worldState, address, spec).MachineCode; + UInt256 result = (UInt256)accountCode.Span.Length; stack.PushUInt256(in result); } @@ -2309,7 +2169,7 @@ private EvmExceptionType InstructionCall( returnData = null; ref readonly ExecutionEnvironment env = ref vmState.Env; - Metrics.Calls++; + Metrics.IncrementCalls(); if (instruction == Instruction.DELEGATECALL && !spec.DelegateCallEnabled || instruction == Instruction.STATICCALL && !spec.StaticCallEnabled || @@ -2361,11 +2221,7 @@ private EvmExceptionType InstructionCall( if (typeof(TLogger) == typeof(IsTracing)) { - _logger.Trace($"caller {caller}"); - _logger.Trace($"code source {codeSource}"); - _logger.Trace($"target {target}"); - _logger.Trace($"value {callValue}"); - _logger.Trace($"transfer value {transferValue}"); + TraceCallDetails(codeSource, ref callValue, ref transferValue, caller, target); } long gasExtra = 0L; @@ -2389,6 +2245,9 @@ private EvmExceptionType InstructionCall( !UpdateMemoryCost(vmState, ref gasAvailable, in outputOffset, outputLength) || !UpdateGas(gasExtra, ref gasAvailable)) return EvmExceptionType.OutOfGas; + CodeInfo codeInfo = _codeInfoRepository.GetCachedCodeInfo(_worldState, codeSource, spec); + codeInfo.AnalyseInBackgroundIfRequired(); + if (spec.Use63Over64Rule) { UInt256 sixtyFourthDeducted = (UInt256)(gasAvailable - gasAvailable / 64); @@ -2436,11 +2295,19 @@ private EvmExceptionType InstructionCall( return EvmExceptionType.None; } - ReadOnlyMemory callData = vmState.Memory.Load(in dataOffset, dataLength); - Snapshot snapshot = _worldState.TakeSnapshot(); _state.SubtractFromBalance(caller, transferValue, spec, env.IsSystemEnv); + if (codeInfo.IsEmpty && typeof(TTracingInstructions) != typeof(IsTracing) && !_txTracer.IsTracingActions) + { + // Non contract call, no need to construct call frame can just credit balance and return gas + _returnDataBuffer = default; + stack.PushBytes(StatusCode.SuccessBytes.Span); + UpdateGasUp(gasLimitUl, ref gasAvailable); + return FastCall(spec, out returnData, in transferValue, target); + } + + ReadOnlyMemory callData = vmState.Memory.Load(in dataOffset, dataLength); ExecutionEnvironment callEnv = new ( txExecutionContext: in env.TxExecutionContext, @@ -2451,7 +2318,7 @@ private EvmExceptionType InstructionCall( transferValue: transferValue, value: callValue, inputData: callData, - codeInfo: GetCachedCodeInfo(_worldState, codeSource, spec), + codeInfo: codeInfo, isSystemExecutionEnv: env.IsSystemEnv ); if (typeof(TLogger) == typeof(IsTracing)) _logger.Trace($"Tx call gas {gasLimitUl}"); @@ -2477,6 +2344,32 @@ private EvmExceptionType InstructionCall( isCreateOnPreExistingAccount: false); return EvmExceptionType.None; + + EvmExceptionType FastCall(IReleaseSpec spec, out object returnData, in UInt256 transferValue, Address target) + { + if (!_state.AccountExists(target)) + { + _state.CreateAccount(target, transferValue); + } + else + { + _state.AddToBalance(target, transferValue, spec); + } + Metrics.IncrementEmptyCalls(); + + returnData = CallResult.BoxedEmpty; + return EvmExceptionType.None; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + void TraceCallDetails(Address codeSource, ref UInt256 callValue, ref UInt256 transferValue, Address caller, Address target) + { + _logger.Trace($"caller {caller}"); + _logger.Trace($"code source {codeSource}"); + _logger.Trace($"target {target}"); + _logger.Trace($"value {callValue}"); + _logger.Trace($"transfer value {transferValue}"); + } } [SkipLocalsInit] @@ -2650,7 +2543,7 @@ private EvmExceptionType InstructionSelfDestruct(EvmState vmState, ref Snapshot snapshot = _worldState.TakeSnapshot(); bool accountExists = _state.AccountExists(contractAddress); - if (accountExists && (GetCachedCodeInfo(_worldState, contractAddress, spec).MachineCode.Length != 0 || + if (accountExists && (_codeInfoRepository.GetCachedCodeInfo(_worldState, contractAddress, spec).MachineCode.Length != 0 || _state.GetNonce(contractAddress) != 0)) { /* we get the snapshot before this as there is a possibility with that we will touch an empty account and remove it even if the REVERT operation follows */ @@ -2675,6 +2568,7 @@ private EvmExceptionType InstructionSelfDestruct(EvmState vmState, ref // pointing to data in this tx and will become invalid // for another tx as returned to pool. CodeInfo codeInfo = new(initCode); + codeInfo.AnalyseInBackgroundIfRequired(); ExecutionEnvironment callEnv = new ( @@ -2743,7 +2637,7 @@ private EvmExceptionType InstructionSLoad where TTracingInstructions : struct, IIsTracing where TTracingStorage : struct, IIsTracing { - Metrics.SloadOpcode++; + Metrics.IncrementSLoadOpcode(); gasAvailable -= spec.GetSLoadCost(); if (!stack.PopUInt256(out UInt256 result)) return EvmExceptionType.StackUnderflow; @@ -2771,7 +2665,7 @@ private EvmExceptionType InstructionSStore int.MaxValue) { @@ -2962,7 +2856,7 @@ private static bool Jump(in UInt256 jumpDest, ref int programCounter, in Executi } int jumpDestInt = (int)jumpDest; - if (!env.CodeInfo.ValidateJump(jumpDestInt, isSubroutine)) + if (!env.CodeInfo.ValidateJump(jumpDestInt)) { // https://github.com/NethermindEth/nethermind/issues/140 // TODO: add a test, validating inside the condition was not covered by existing tests and fails on 61363 Ropsten diff --git a/src/Nethermind/Nethermind.ExternalSigner.Plugin/ClefSigner.cs b/src/Nethermind/Nethermind.ExternalSigner.Plugin/ClefSigner.cs new file mode 100644 index 00000000000..16b5c8c0026 --- /dev/null +++ b/src/Nethermind/Nethermind.ExternalSigner.Plugin/ClefSigner.cs @@ -0,0 +1,140 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using DotNetty.Buffers; +using Nethermind.Consensus; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Crypto; +using Nethermind.JsonRpc.Client; +using Nethermind.Serialization.Rlp; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace Nethermind.JsonRpc; +public class ClefSigner : IHeaderSigner, ISignerStore +{ + private readonly IJsonRpcClient rpcClient; + private HeaderDecoder _headerDecoder; + + private ClefSigner(IJsonRpcClient rpcClient, Address author) + { + this.rpcClient = rpcClient; + Address = author; + _headerDecoder = new HeaderDecoder(); + } + + public static async Task Create(IJsonRpcClient jsonRpcClient, Address? blockAuthorAccount = null) + { + ClefSigner signer = new(jsonRpcClient, await GetSignerAddress(jsonRpcClient, blockAuthorAccount)); + return signer; + } + + public Address Address { get; private set; } + + public bool CanSign => true; + + public bool CanSignHeader => true; + + public PrivateKey? Key => throw new InvalidOperationException("Cannot get private keys from remote signer."); + + /// + /// Clef will not sign data directly, but will parse and sign data in the format: + /// keccak256("\x19Ethereum Signed Message:\n${message length}${message}") + /// + /// Message to be signed. + /// of . + public Signature Sign(Hash256 message) + { + var signed = rpcClient.Post( + "account_signData", + "text/plain", + Address.ToString(), + message).GetAwaiter().GetResult(); + if (signed == null) + ThrowInvalidOperationSignFailed(); + var bytes = Bytes.FromHexString(signed); + return new Signature(bytes); + } + + /// + /// Used to sign a clique header. The full Rlp of the header has to be sent, + /// since clef does not sign data directly, but will parse and decide itself what to sign. + /// + /// Full Rlp of the clique header. + /// of the hash of the clique header. + public Signature Sign(BlockHeader header) + { + if (header is null) throw new ArgumentNullException(nameof(header)); + int contentLength = _headerDecoder.GetLength(header, RlpBehaviors.None); + IByteBuffer buffer = PooledByteBufferAllocator.Default.Buffer(contentLength); + try + { + RlpStream rlpStream = new NettyRlpStream(buffer); + rlpStream.Encode(header); + string? signed = rpcClient.Post( + "account_signData", + "application/x-clique-header", + Address.ToString(), + buffer.AsSpan().ToHexString(true)).GetAwaiter().GetResult(); + if (signed is null) + ThrowInvalidOperationSignFailed(); + byte[] bytes = Bytes.FromHexString(signed); + + //Clef will set recid to 0/1, without the VOffset + if (bytes.Length == 65 && (bytes[64] == 0 || bytes[64] == 1)) + return new Signature(bytes.AsSpan(0, 64), bytes[64]); + return new Signature(bytes); + } + finally + { + buffer.Release(); + } + } + + public ValueTask Sign(Transaction tx) => + throw new NotImplementedException("Remote signing of transactions is not supported."); + + private async static Task
GetSignerAddress(IJsonRpcClient rpcClient, Address? blockAuthorAccount) + { + var accounts = await rpcClient.Post("account_list"); + if (accounts is null) + throw new InvalidOperationException("Remote signer 'account_list' response is invalid."); + if (!accounts.Any()) + throw new InvalidOperationException("Remote signer has not been configured with any signers."); + if (blockAuthorAccount != null) + { + if (accounts.Any(a => new Address(a).Bytes.SequenceEqual(blockAuthorAccount.Bytes))) + return blockAuthorAccount; + else + throw new InvalidOperationException($"Remote signer cannot sign for {blockAuthorAccount}."); + } + else + { + return new Address(accounts[0]); + } + } + + public void SetSigner(PrivateKey key) + { + ThrowInvalidOperationSetSigner(); + } + + public void SetSigner(ProtectedPrivateKey key) + { + ThrowInvalidOperationSetSigner(); + } + + [DoesNotReturn] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void ThrowInvalidOperationSignFailed() => + throw new InvalidOperationException("Remote signer failed to sign the request."); + + [DoesNotReturn] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void ThrowInvalidOperationSetSigner() => + throw new InvalidOperationException("Cannot set a signer when using a remote signer."); +} + + diff --git a/src/Nethermind/Nethermind.ExternalSigner.Plugin/ClefSignerPlugin.cs b/src/Nethermind/Nethermind.ExternalSigner.Plugin/ClefSignerPlugin.cs new file mode 100644 index 00000000000..19daea61b1d --- /dev/null +++ b/src/Nethermind/Nethermind.ExternalSigner.Plugin/ClefSignerPlugin.cs @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Api; +using Nethermind.Api.Extensions; +using Nethermind.Core; +using Nethermind.JsonRpc.Client; +using Nethermind.JsonRpc; +using Nethermind.Network; +using Nethermind.Consensus; +using Nethermind.KeyStore.Config; +using System.Configuration; + +namespace Nethermind.ExternalSigner.Plugin; + +public class ClefSignerPlugin : INethermindPlugin +{ + private INethermindApi? _nethermindApi; + + public string Name => "Clef signer"; + + public string Description => "Enabled signing from a remote Clef instance over Json RPC."; + + public string Author => "Nethermind"; + + public ValueTask DisposeAsync() + { + return ValueTask.CompletedTask; + } + + public async Task Init(INethermindApi nethermindApi) + { + _nethermindApi = nethermindApi ?? throw new ArgumentNullException(nameof(nethermindApi)); + + if (_nethermindApi == null) + throw new InvalidOperationException("Init() must be called first."); + + IMiningConfig miningConfig = _nethermindApi.Config(); + if (miningConfig.Enabled) + { + if (!string.IsNullOrEmpty(miningConfig.Signer)) + { + Uri? uri; + if (!Uri.TryCreate(miningConfig.Signer, UriKind.Absolute, out uri)) + { + throw new ConfigurationErrorsException($"{miningConfig.Signer} must have be a valid uri."); + } + ClefSigner signer = + await SetupExternalSigner(uri, _nethermindApi.Config().BlockAuthorAccount); + _nethermindApi.EngineSigner = signer; + } + } + + } + + public Task InitNetworkProtocol() + { + return Task.CompletedTask; + } + + public Task InitRpcModules() + { + return Task.CompletedTask; + } + + private async Task SetupExternalSigner(Uri urlSigner, string blockAuthorAccount) + { + try + { + Address? address = string.IsNullOrEmpty(blockAuthorAccount) ? null : new Address(blockAuthorAccount); + BasicJsonRpcClient rpcClient = new(urlSigner, _nethermindApi!.EthereumJsonSerializer, _nethermindApi.LogManager, TimeSpan.FromSeconds(10)); + _nethermindApi.DisposeStack.Push(rpcClient); + return await ClefSigner.Create(rpcClient, address); + } + catch (HttpRequestException e) + { + throw new NetworkingException($"Remote signer at {urlSigner} did not respond.", NetworkExceptionType.TargetUnreachable, e); + } + } +} diff --git a/src/Nethermind/Nethermind.ExternalSigner.Plugin/Nethermind.ExternalSigner.Plugin.csproj b/src/Nethermind/Nethermind.ExternalSigner.Plugin/Nethermind.ExternalSigner.Plugin.csproj new file mode 100644 index 00000000000..31ecca4eede --- /dev/null +++ b/src/Nethermind/Nethermind.ExternalSigner.Plugin/Nethermind.ExternalSigner.Plugin.csproj @@ -0,0 +1,14 @@ + + + + net8.0 + enable + enable + + + + + + + + diff --git a/src/Nethermind/Nethermind.Facade.Test/BlockchainBridgeTests.cs b/src/Nethermind/Nethermind.Facade.Test/BlockchainBridgeTests.cs index 69d8ef8077c..77fa79f865b 100644 --- a/src/Nethermind/Nethermind.Facade.Test/BlockchainBridgeTests.cs +++ b/src/Nethermind/Nethermind.Facade.Test/BlockchainBridgeTests.cs @@ -29,6 +29,7 @@ using NUnit.Framework; using Nethermind.Config; using Nethermind.Evm; +using Nethermind.Facade.Simulate; using Nethermind.State; namespace Nethermind.Facade.Test @@ -47,6 +48,20 @@ public class BlockchainBridgeTests private ISpecProvider _specProvider; private IDbProvider _dbProvider; + private class TestReadOnlyTxProcessingEnv : ReadOnlyTxProcessingEnv + { + public TestReadOnlyTxProcessingEnv( + IWorldStateManager worldStateManager, + IBlockTree blockTree, + ISpecProvider specProvider, + ILogManager logManager, + ITransactionProcessor transactionProcessor) : + base(worldStateManager, blockTree, specProvider, logManager) + { + _transactionProcessor = transactionProcessor; + } + } + [SetUp] public async Task SetUp() { @@ -67,16 +82,24 @@ public async Task SetUp() IWorldStateManager readOnlyWorldStateManager = new ReadOnlyWorldStateManager(dbProvider, trieStore, LimboLogs.Instance); - ReadOnlyTxProcessingEnv processingEnv = new( + IReadOnlyBlockTree readOnlyBlockTree = _blockTree.AsReadOnly(); + ReadOnlyTxProcessingEnv processingEnv = new TestReadOnlyTxProcessingEnv( readOnlyWorldStateManager, - new ReadOnlyBlockTree(_blockTree), + readOnlyBlockTree, _specProvider, - LimboLogs.Instance); + LimboLogs.Instance, + _transactionProcessor); - processingEnv.TransactionProcessor = _transactionProcessor; + SimulateReadOnlyBlocksProcessingEnvFactory simulateProcessingEnvFactory = new SimulateReadOnlyBlocksProcessingEnvFactory( + readOnlyWorldStateManager, + readOnlyBlockTree, + new ReadOnlyDbProvider(_dbProvider, true), + _specProvider, + LimboLogs.Instance); _blockchainBridge = new BlockchainBridge( processingEnv, + simulateProcessingEnvFactory, _txPool, _receiptStorage, _filterStore, @@ -195,10 +218,17 @@ public void Bridge_head_is_correct(long headNumber) IWorldStateManager readOnlyWorldStateManager = new ReadOnlyWorldStateManager(dbProvider, trieStore, LimboLogs.Instance); - + IReadOnlyBlockTree roBlockTree = _blockTree.AsReadOnly(); ReadOnlyTxProcessingEnv processingEnv = new( readOnlyWorldStateManager, - new ReadOnlyBlockTree(_blockTree), + roBlockTree, + _specProvider, + LimboLogs.Instance); + + SimulateReadOnlyBlocksProcessingEnvFactory simulateProcessingEnv = new SimulateReadOnlyBlocksProcessingEnvFactory( + readOnlyWorldStateManager, + roBlockTree, + new ReadOnlyDbProvider(_dbProvider, true), _specProvider, LimboLogs.Instance); @@ -210,6 +240,7 @@ public void Bridge_head_is_correct(long headNumber) _blockchainBridge = new BlockchainBridge( processingEnv, + simulateProcessingEnv, _txPool, _receiptStorage, _filterStore, diff --git a/src/Nethermind/Nethermind.Facade/BlockchainBridge.cs b/src/Nethermind/Nethermind.Facade/BlockchainBridge.cs index 5c82dede5a9..91675b757e1 100644 --- a/src/Nethermind/Nethermind.Facade/BlockchainBridge.cs +++ b/src/Nethermind/Nethermind.Facade/BlockchainBridge.cs @@ -19,13 +19,15 @@ using Block = Nethermind.Core.Block; using System.Threading; using Nethermind.Consensus.Processing; -using Nethermind.Core.Eip2930; using Nethermind.Core.Specs; using Nethermind.Evm.TransactionProcessing; using Nethermind.Facade.Filters; using Nethermind.State; using Nethermind.Core.Extensions; using Nethermind.Config; +using Nethermind.Facade.Proxy.Models.Simulate; +using Nethermind.Facade.Simulate; +using Transaction = Nethermind.Core.Transaction; namespace Nethermind.Facade { @@ -37,7 +39,9 @@ public interface IBlockchainBridgeFactory [Todo(Improve.Refactor, "I want to remove BlockchainBridge, split it into something with logging, state and tx processing. Then we can start using independent modules.")] public class BlockchainBridge : IBlockchainBridge { - private readonly ReadOnlyTxProcessingEnv _processingEnv; + private readonly IReadOnlyTxProcessorSource _processingEnv; + private readonly IBlockTree _blockTree; + private readonly IStateReader _stateReader; private readonly ITxPool _txPool; private readonly IFilterStore _filterStore; private readonly IEthereumEcdsa _ecdsa; @@ -47,8 +51,10 @@ public class BlockchainBridge : IBlockchainBridge private readonly ILogFinder _logFinder; private readonly ISpecProvider _specProvider; private readonly IBlocksConfig _blocksConfig; + private readonly SimulateBridgeHelper _simulateBridgeHelper; public BlockchainBridge(ReadOnlyTxProcessingEnv processingEnv, + SimulateReadOnlyBlocksProcessingEnvFactory simulateProcessingEnvFactory, ITxPool? txPool, IReceiptFinder? receiptStorage, IFilterStore? filterStore, @@ -61,6 +67,8 @@ public BlockchainBridge(ReadOnlyTxProcessingEnv processingEnv, bool isMining) { _processingEnv = processingEnv ?? throw new ArgumentNullException(nameof(processingEnv)); + _blockTree = processingEnv.BlockTree; + _stateReader = processingEnv.StateReader; _txPool = txPool ?? throw new ArgumentNullException(nameof(_txPool)); _receiptFinder = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage)); _filterStore = filterStore ?? throw new ArgumentNullException(nameof(filterStore)); @@ -71,13 +79,16 @@ public BlockchainBridge(ReadOnlyTxProcessingEnv processingEnv, _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); _blocksConfig = blocksConfig; IsMining = isMining; + _simulateBridgeHelper = new SimulateBridgeHelper( + simulateProcessingEnvFactory ?? throw new ArgumentNullException(nameof(simulateProcessingEnvFactory)), + _blocksConfig); } public Block? HeadBlock { get { - return _processingEnv.BlockTree.Head; + return _blockTree.Head; } } @@ -88,7 +99,7 @@ public Block? HeadBlock Hash256 blockHash = _receiptFinder.FindBlockHash(txHash); if (blockHash is not null) { - Block? block = _processingEnv.BlockTree.FindBlock(blockHash, BlockTreeLookupOptions.RequireCanonical); + Block? block = _blockTree.FindBlock(blockHash, BlockTreeLookupOptions.RequireCanonical); if (block is not null) { TxReceipt[] txReceipts = _receiptFinder.Get(block); @@ -108,7 +119,7 @@ public Block? HeadBlock Hash256 blockHash = _receiptFinder.FindBlockHash(txHash); if (blockHash is not null) { - Block block = _processingEnv.BlockTree.FindBlock(blockHash, BlockTreeLookupOptions.TotalDifficultyNotNeeded); + Block block = _blockTree.FindBlock(blockHash, BlockTreeLookupOptions.TotalDifficultyNotNeeded); TxReceipt txReceipt = _receiptFinder.Get(block).ForTransaction(txHash); return (txReceipt, block?.Transactions[txReceipt.Index], block?.BaseFeePerGas); } @@ -127,31 +138,6 @@ public Block? HeadBlock return blockHash is not null ? _receiptFinder.Get(blockHash).ForTransaction(txHash) : null; } - public class CallOutput - { - public CallOutput() - { - } - - public CallOutput(byte[] outputData, long gasSpent, string error, bool inputError = false) - { - Error = error; - OutputData = outputData; - GasSpent = gasSpent; - InputError = inputError; - } - - public string? Error { get; set; } - - public byte[] OutputData { get; set; } - - public long GasSpent { get; set; } - - public bool InputError { get; set; } - - public AccessList? AccessList { get; set; } - } - public CallOutput Call(BlockHeader header, Transaction tx, CancellationToken cancellationToken) { CallOutputTracer callOutputTracer = new(); @@ -166,9 +152,31 @@ public CallOutput Call(BlockHeader header, Transaction tx, CancellationToken can }; } + public SimulateOutput Simulate(BlockHeader header, SimulatePayload payload, CancellationToken cancellationToken) + { + SimulateBlockTracer simulateOutputTracer = new(payload.TraceTransfers, payload.ReturnFullTransactionObjects, _specProvider); + BlockReceiptsTracer tracer = new(); + tracer.SetOtherTracer(simulateOutputTracer); + SimulateOutput result = new(); + try + { + if (!_simulateBridgeHelper.TrySimulate(header, payload, new CancellationBlockTracer(tracer, cancellationToken), out string error)) + { + result.Error = error; + } + } + catch (Exception ex) + { + result.Error = ex.ToString(); + } + + result.Items = simulateOutputTracer.Results; + return result; + } + public CallOutput EstimateGas(BlockHeader header, Transaction tx, int errorMargin, CancellationToken cancellationToken) { - using IReadOnlyTransactionProcessor? readOnlyTransactionProcessor = _processingEnv.Build(header.StateRoot!); + using IReadOnlyTxProcessingScope scope = _processingEnv.Build(header.StateRoot!); EstimateGasTracer estimateGasTracer = new(); TransactionResult tryCallResult = TryCallAndRestore( @@ -177,7 +185,7 @@ public CallOutput EstimateGas(BlockHeader header, Transaction tx, int errorMargi true, estimateGasTracer.WithCancellation(cancellationToken)); - GasEstimator gasEstimator = new(readOnlyTransactionProcessor, _processingEnv.StateProvider, + GasEstimator gasEstimator = new(scope.TransactionProcessor, scope.WorldState, _specProvider, _blocksConfig); long estimate = gasEstimator.Estimate(tx, header, estimateGasTracer, errorMargin, cancellationToken); @@ -194,7 +202,7 @@ public CallOutput CreateAccessList(BlockHeader header, Transaction tx, Cancellat CallOutputTracer callOutputTracer = new(); AccessTxTracer accessTxTracer = optimize ? new(tx.SenderAddress, - tx.GetRecipient(tx.IsContractCreation ? _processingEnv.StateReader.GetNonce(header.StateRoot, tx.SenderAddress) : 0), header.GasBeneficiary) + tx.GetRecipient(tx.IsContractCreation ? _stateReader.GetNonce(header.StateRoot, tx.SenderAddress) : 0), header.GasBeneficiary) : new(header.GasBeneficiary); TransactionResult tryCallResult = TryCallAndRestore(header, tx, false, @@ -235,7 +243,7 @@ private TransactionResult CallAndRestore( transaction.SenderAddress ??= Address.SystemUser; Hash256 stateRoot = blockHeader.StateRoot!; - using IReadOnlyTransactionProcessor transactionProcessor = _processingEnv.Build(stateRoot); + using IReadOnlyTxProcessingScope scope = _processingEnv.Build(stateRoot); if (transaction.Nonce == 0) { @@ -277,17 +285,17 @@ private TransactionResult CallAndRestore( callHeader.MixHash = blockHeader.MixHash; callHeader.IsPostMerge = blockHeader.Difficulty == 0; transaction.Hash = transaction.CalculateHash(); - return transactionProcessor.CallAndRestore(transaction, new(callHeader), tracer); + return scope.TransactionProcessor.CallAndRestore(transaction, new(callHeader), tracer); } public ulong GetChainId() { - return _processingEnv.BlockTree.ChainId; + return _blockTree.ChainId; } private UInt256 GetNonce(Hash256 stateRoot, Address address) { - return _processingEnv.StateReader.GetNonce(stateRoot, address); + return _stateReader.GetNonce(stateRoot, address); } public bool FilterExists(int filterId) => _filterStore.FilterExists(filterId); @@ -343,7 +351,7 @@ public int NewFilter(BlockParameter? fromBlock, BlockParameter? toBlock, public int NewBlockFilter() { - BlockFilter filter = _filterStore.CreateBlockFilter(_processingEnv.BlockTree.Head!.Number); + BlockFilter filter = _filterStore.CreateBlockFilter(_blockTree.Head!.Number); _filterStore.SaveFilter(filter); return filter.Id; } @@ -388,12 +396,12 @@ public Hash256[] GetPendingTransactionFilterChanges(int filterId) => public void RunTreeVisitor(ITreeVisitor treeVisitor, Hash256 stateRoot) { - _processingEnv.StateReader.RunTreeVisitor(treeVisitor, stateRoot); + _stateReader.RunTreeVisitor(treeVisitor, stateRoot); } public bool HasStateForRoot(Hash256 stateRoot) { - return _processingEnv.StateReader.HasStateForRoot(stateRoot); + return _stateReader.HasStateForRoot(stateRoot); } public IEnumerable FindLogs(LogFilter filter, BlockHeader fromBlock, BlockHeader toBlock, CancellationToken cancellationToken = default) diff --git a/src/Nethermind/Nethermind.Facade/BlockchainBridgeExtensions.cs b/src/Nethermind/Nethermind.Facade/BlockchainBridgeExtensions.cs new file mode 100644 index 00000000000..a5af9f8b625 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/BlockchainBridgeExtensions.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; + +namespace Nethermind.Facade; + +public static class BlockchainBridgeExtensions +{ + public static bool HasStateForBlock(this IBlockchainBridge blockchainBridge, BlockHeader header) => + blockchainBridge.HasStateForRoot(header.StateRoot!); +} diff --git a/src/Nethermind/Nethermind.Facade/CallOutput.cs b/src/Nethermind/Nethermind.Facade/CallOutput.cs new file mode 100644 index 00000000000..42034a699a0 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/CallOutput.cs @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core.Eip2930; + +namespace Nethermind.Facade; + +public class CallOutput +{ + public string? Error { get; set; } + + public byte[] OutputData { get; set; } = Array.Empty(); + + public long GasSpent { get; set; } + + public bool InputError { get; set; } + + public AccessList? AccessList { get; set; } +} diff --git a/src/Nethermind/Nethermind.JsonRpc/Data/AccessListItemForRpc.cs b/src/Nethermind/Nethermind.Facade/Eth/AccessListItemForRpc.cs similarity index 98% rename from src/Nethermind/Nethermind.JsonRpc/Data/AccessListItemForRpc.cs rename to src/Nethermind/Nethermind.Facade/Eth/AccessListItemForRpc.cs index 17533dd0f25..0269aeeb0e6 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Data/AccessListItemForRpc.cs +++ b/src/Nethermind/Nethermind.Facade/Eth/AccessListItemForRpc.cs @@ -12,7 +12,7 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace Nethermind.JsonRpc.Data +namespace Nethermind.Facade.Eth { public struct AccessListItemForRpc : IEquatable { diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/BlockForRpc.cs b/src/Nethermind/Nethermind.Facade/Eth/BlockForRpc.cs similarity index 99% rename from src/Nethermind/Nethermind.JsonRpc/Modules/Eth/BlockForRpc.cs rename to src/Nethermind/Nethermind.Facade/Eth/BlockForRpc.cs index 6c29b8f76cb..21ea75ed6d4 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/BlockForRpc.cs +++ b/src/Nethermind/Nethermind.Facade/Eth/BlockForRpc.cs @@ -15,7 +15,7 @@ using System.Runtime.CompilerServices; using Nethermind.Core.ConsensusRequests; -namespace Nethermind.JsonRpc.Modules.Eth; +namespace Nethermind.Facade.Eth; public class BlockForRpc { diff --git a/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpc.cs b/src/Nethermind/Nethermind.Facade/Eth/TransactionForRpc.cs similarity index 99% rename from src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpc.cs rename to src/Nethermind/Nethermind.Facade/Eth/TransactionForRpc.cs index 9806860fad7..ab7a7bbc8ba 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpc.cs +++ b/src/Nethermind/Nethermind.Facade/Eth/TransactionForRpc.cs @@ -10,8 +10,9 @@ using Nethermind.Core.Eip2930; using Nethermind.Core.Extensions; using Nethermind.Int256; +using Nethermind.JsonRpc.Data; -namespace Nethermind.JsonRpc.Data; +namespace Nethermind.Facade.Eth; public class TransactionForRpc { diff --git a/src/Nethermind/Nethermind.Facade/IBlockchainBridge.cs b/src/Nethermind/Nethermind.Facade/IBlockchainBridge.cs index f998ae99fd0..ab0ffa8d309 100644 --- a/src/Nethermind/Nethermind.Facade/IBlockchainBridge.cs +++ b/src/Nethermind/Nethermind.Facade/IBlockchainBridge.cs @@ -9,8 +9,11 @@ using Nethermind.Core.Crypto; using Nethermind.Evm; using Nethermind.Facade.Filters; +using Nethermind.Facade.Simulate; +using Nethermind.Facade.Proxy.Models.Simulate; using Nethermind.Int256; using Nethermind.Trie; +using static Nethermind.Facade.BlockchainBridge; using Block = Nethermind.Core.Block; namespace Nethermind.Facade @@ -24,9 +27,11 @@ public interface IBlockchainBridge : ILogFinder TxReceipt GetReceipt(Hash256 txHash); (TxReceipt? Receipt, TxGasInfo? GasInfo, int LogIndexStart) GetReceiptAndGasInfo(Hash256 txHash); (TxReceipt? Receipt, Transaction Transaction, UInt256? baseFee) GetTransaction(Hash256 txHash, bool checkTxnPool = true); - BlockchainBridge.CallOutput Call(BlockHeader header, Transaction tx, CancellationToken cancellationToken); - BlockchainBridge.CallOutput EstimateGas(BlockHeader header, Transaction tx, int errorMarginBasisPoints, CancellationToken cancellationToken); - BlockchainBridge.CallOutput CreateAccessList(BlockHeader header, Transaction tx, CancellationToken cancellationToken, bool optimize); + CallOutput Call(BlockHeader header, Transaction tx, CancellationToken cancellationToken); + SimulateOutput Simulate(BlockHeader header, SimulatePayload payload, CancellationToken cancellationToken); + CallOutput EstimateGas(BlockHeader header, Transaction tx, int errorMarginBasisPoints, CancellationToken cancellationToken); + + CallOutput CreateAccessList(BlockHeader header, Transaction tx, CancellationToken cancellationToken, bool optimize); ulong GetChainId(); int NewBlockFilter(); diff --git a/src/Nethermind/Nethermind.Facade/Nethermind.Facade.csproj b/src/Nethermind/Nethermind.Facade/Nethermind.Facade.csproj index 200aa01482e..7a90f4758a8 100644 --- a/src/Nethermind/Nethermind.Facade/Nethermind.Facade.csproj +++ b/src/Nethermind/Nethermind.Facade/Nethermind.Facade.csproj @@ -2,6 +2,7 @@ annotations + True diff --git a/src/Nethermind/Nethermind.Facade/OverridableCodeInfoRepository.cs b/src/Nethermind/Nethermind.Facade/OverridableCodeInfoRepository.cs new file mode 100644 index 00000000000..3df49a9e594 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/OverridableCodeInfoRepository.cs @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Evm; +using Nethermind.Evm.CodeAnalysis; +using Nethermind.State; + +namespace Nethermind.Facade; + +public class OverridableCodeInfoRepository(ICodeInfoRepository codeInfoRepository) : ICodeInfoRepository +{ + private readonly Dictionary _codeOverwrites = new(); + + public CodeInfo GetCachedCodeInfo(IWorldState worldState, Address codeSource, IReleaseSpec vmSpec) => + _codeOverwrites.TryGetValue(codeSource, out CodeInfo result) + ? result + : codeInfoRepository.GetCachedCodeInfo(worldState, codeSource, vmSpec); + + public CodeInfo GetOrAdd(ValueHash256 codeHash, ReadOnlySpan initCode) => codeInfoRepository.GetOrAdd(codeHash, initCode); + + public void InsertCode(IWorldState state, ReadOnlyMemory code, Address codeOwner, IReleaseSpec spec, bool isSystemEnv) => + codeInfoRepository.InsertCode(state, code, codeOwner, spec, isSystemEnv); + + + public void SetCodeOverwrite( + IWorldState worldState, + IReleaseSpec vmSpec, + Address key, + CodeInfo value, + Address? redirectAddress = null) + { + if (redirectAddress is not null) + { + _codeOverwrites[redirectAddress] = GetCachedCodeInfo(worldState, key, vmSpec); + } + + _codeOverwrites[key] = value; + } +} diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/AccountOverride.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/AccountOverride.cs new file mode 100644 index 00000000000..5b8a26ab431 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/AccountOverride.cs @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Int256; + +namespace Nethermind.Facade.Proxy.Models; + +public class AccountOverride +{ + public UInt256? Nonce { get; set; } + public UInt256? Balance { get; set; } + public byte[]? Code { get; set; } + public Address? MovePrecompileToAddress { get; set; } + + /// + /// Storage for AccountOverrideState + /// + public Dictionary? State { get; set; } + + /// + /// Storage difference for AccountOverrideStateDiff + /// + public Dictionary? StateDiff { get; set; } +} diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/CallTransactionModel.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/CallTransactionModel.cs deleted file mode 100644 index 697b11eb3d7..00000000000 --- a/src/Nethermind/Nethermind.Facade/Proxy/Models/CallTransactionModel.cs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core; -using Nethermind.Int256; -using static Nethermind.Core.Extensions.MemoryExtensions; - -namespace Nethermind.Facade.Proxy.Models -{ - public class CallTransactionModel - { - public Address From { get; set; } - public Address To { get; set; } - public UInt256 Gas { get; set; } - public UInt256 GasPrice { get; set; } - public UInt256 Value { get; set; } - public byte[] Data { get; set; } - - public static CallTransactionModel FromTransaction(Transaction transaction) - => new() - { - From = transaction.SenderAddress, - To = transaction.To, - Data = transaction.Data.AsArray(), - Value = transaction.Value, - Gas = (UInt256)transaction.GasLimit, - GasPrice = transaction.GasPrice - }; - } -} diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/BlockOverride.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/BlockOverride.cs new file mode 100644 index 00000000000..ad017bec29e --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/BlockOverride.cs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Config; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Int256; + +namespace Nethermind.Facade.Proxy.Models.Simulate; + +public class BlockOverride +{ + public ulong? Number { get; set; } + public Hash256 PrevRandao { get; set; } = Keccak.Zero; + public ulong? Time { get; set; } + public ulong? GasLimit { get; set; } + public Address? FeeRecipient { get; set; } + public UInt256? BaseFeePerGas { get; set; } + + public BlockHeader GetBlockHeader(BlockHeader parent, IBlocksConfig cfg, IReleaseSpec spec) + { + ulong newTime = Time ?? checked(parent.Timestamp + cfg.SecondsPerSlot); + + long newGasLimit = GasLimit switch + { + null => parent.GasLimit, + <= long.MaxValue => (long)GasLimit, + _ => throw new OverflowException($"GasLimit value is too large, max value {ulong.MaxValue}") + }; + + long newBlockNumber = Number switch + { + null => checked(parent.Number + 1), + <= long.MaxValue => (long)Number, + _ => throw new OverflowException($"Block Number value is too large, max value {ulong.MaxValue}") + }; + + Address newFeeRecipientAddress = FeeRecipient ?? parent.Beneficiary!; + UInt256 newDifficulty = parent.Difficulty == 0 ? 0 : parent.Difficulty + 1; + BlockHeader result = new( + parent.Hash!, + Keccak.OfAnEmptySequenceRlp, + newFeeRecipientAddress, + newDifficulty, + newBlockNumber, + newGasLimit, + newTime, + Array.Empty()) + { + BaseFeePerGas = BaseFeePerGas ?? BaseFeeCalculator.Calculate(parent, spec), + MixHash = PrevRandao, + IsPostMerge = parent.Difficulty == 0, + TotalDifficulty = parent.TotalDifficulty + newDifficulty, + SealEngineType = parent.SealEngineType + }; + + return result; + } +} diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/BlockStateCall.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/BlockStateCall.cs new file mode 100644 index 00000000000..db4efb549ba --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/BlockStateCall.cs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using Nethermind.Core; + +namespace Nethermind.Facade.Proxy.Models.Simulate; + +public class BlockStateCall +{ + public BlockOverride? BlockOverrides { get; set; } + public Dictionary? StateOverrides { get; set; } + public T[]? Calls { get; set; } = Array.Empty(); +} diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/Error.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/Error.cs new file mode 100644 index 00000000000..a53bdd9a9e5 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/Error.cs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Facade.Proxy.Models.Simulate; + +public class Error +{ + public int Code { get; set; } + public string Message { get; set; } + public string Data { get; set; } +} diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/Log.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/Log.cs new file mode 100644 index 00000000000..1e0f05d12c3 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/Log.cs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Crypto; + +namespace Nethermind.Facade.Proxy.Models.Simulate; + +public class Log +{ + public Address Address { get; set; } + public Hash256[] Topics { get; set; } + public byte[] Data { get; set; } + public ulong BlockNumber { get; set; } + public Hash256 TransactionHash { get; set; } + public ulong TransactionIndex { get; set; } + public Hash256 BlockHash { get; set; } + + public ulong LogIndex { get; set; } + public bool Removed { get; set; } = false; +} diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/ResultType.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/ResultType.cs new file mode 100644 index 00000000000..78e8707e914 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/ResultType.cs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Facade.Proxy.Models.Simulate; + +public enum ResultType +{ + Failure, + Success, + Invalid +} diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/SimulateBlockResult.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/SimulateBlockResult.cs new file mode 100644 index 00000000000..4b4e300fe10 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/SimulateBlockResult.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Reflection; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Facade.Eth; +using Nethermind.Int256; + +namespace Nethermind.Facade.Proxy.Models.Simulate; + +public class SimulateBlockResult(Block source, bool includeFullTransactionData, ISpecProvider specProvider) + : BlockForRpc(source, includeFullTransactionData, specProvider) +{ + public List Calls { get; set; } = new(); + public UInt256 BlobBaseFee { get; set; } +} diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/SimulateCallResult.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/SimulateCallResult.cs new file mode 100644 index 00000000000..e07ec128b4d --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/SimulateCallResult.cs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using System.Linq; + +namespace Nethermind.Facade.Proxy.Models.Simulate; + +public class SimulateCallResult +{ + public ulong Status { get; set; } + public byte[]? ReturnData { get; set; } + public ulong? GasUsed { get; set; } + public Error? Error { get; set; } + public IEnumerable Logs { get; set; } = Enumerable.Empty(); +} diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/SimulatePayload.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/SimulatePayload.cs new file mode 100644 index 00000000000..ad81f77158e --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/SimulatePayload.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; + +namespace Nethermind.Facade.Proxy.Models.Simulate; + +public class SimulatePayload +{ + /// + /// Definition of blocks that can contain calls and overrides + /// + public List>? BlockStateCalls { get; set; } + + /// + /// Should trace ETH Transfers + /// + public bool TraceTransfers { get; set; } = false; + + /// + /// When true, the simulate does all validations that a normal EVM would do, except contract sender and signature + /// checks. When false, multicall behaves like eth_call. + /// + public bool Validation { get; set; } = false; + + /// + /// When true, the simulate returns Full Tx Objects + /// + public bool ReturnFullTransactionObjects { get; set; } = false; +} diff --git a/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/TransactionWithSourceDetails.cs b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/TransactionWithSourceDetails.cs new file mode 100644 index 00000000000..b3fed70ba7c --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Proxy/Models/Simulate/TransactionWithSourceDetails.cs @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; + +namespace Nethermind.Facade.Proxy.Models.Simulate; + +public class TransactionWithSourceDetails +{ + public bool HadGasLimitInRequest; + public bool HadNonceInRequest; + public Transaction Transaction { get; set; } +} diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateBlockTracer.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateBlockTracer.cs new file mode 100644 index 00000000000..c6ab396eedc --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateBlockTracer.cs @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using System.Linq; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Evm; +using Nethermind.Evm.Tracing; +using Nethermind.Facade.Proxy.Models.Simulate; +using Nethermind.Int256; + +namespace Nethermind.Facade.Simulate; + +public class SimulateBlockTracer(bool isTracingLogs, bool includeFullTxData, ISpecProvider spec) : BlockTracer +{ + private readonly List _txTracers = new(); + + private Block _currentBlock = null!; + public List Results { get; } = new(); + + public override void StartNewBlockTrace(Block block) + { + _txTracers.Clear(); + _currentBlock = block; + } + + public override ITxTracer StartNewTxTrace(Transaction? tx) + { + + if (tx?.Hash is not null) + { + ulong txIndex = (ulong)_txTracers.Count; + SimulateTxMutatorTracer result = new(isTracingLogs, tx.Hash, (ulong)_currentBlock.Number, + _currentBlock.Hash, txIndex); + _txTracers.Add(result); + return result; + } + + return NullTxTracer.Instance; + } + + public override void EndBlockTrace() + { + SimulateBlockResult? result = new(_currentBlock, includeFullTxData, spec) + { + Calls = _txTracers.Select(t => t.TraceResult).ToList(), + }; + if (_currentBlock.Header.ExcessBlobGas is not null) + { + if (!BlobGasCalculator.TryCalculateBlobGasPricePerUnit(_currentBlock.Header.ExcessBlobGas.Value, + out UInt256 blobGasPricePerUnit)) + { + result.BlobBaseFee = blobGasPricePerUnit; + } + } + + Results.Add(result); + } +} diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateBlockhashProvider.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateBlockhashProvider.cs new file mode 100644 index 00000000000..8f054a809ca --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateBlockhashProvider.cs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Blockchain; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Evm; + +namespace Nethermind.Facade.Simulate; + +public sealed class SimulateBlockhashProvider(IBlockhashProvider blockhashProvider, IBlockTree blockTree) + : IBlockhashProvider +{ + public Hash256? GetBlockhash(BlockHeader currentBlock, in long number) + { + long bestKnown = blockTree.BestKnownNumber; + return bestKnown < number && blockTree.BestSuggestedHeader is not null + ? blockhashProvider.GetBlockhash(blockTree.BestSuggestedHeader!, in bestKnown) + : blockhashProvider.GetBlockhash(currentBlock, in number); + } +} diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateBridgeHelper.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateBridgeHelper.cs new file mode 100644 index 00000000000..4993b53ec6e --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateBridgeHelper.cs @@ -0,0 +1,285 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.InteropServices; +using Nethermind.Blockchain; +using Nethermind.Config; +using Nethermind.Consensus.Processing; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Crypto; +using Nethermind.Evm.Tracing; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Facade.Proxy.Models.Simulate; +using Nethermind.Int256; +using Nethermind.State; +using Transaction = Nethermind.Core.Transaction; + +namespace Nethermind.Facade.Simulate; + +public class SimulateBridgeHelper(SimulateReadOnlyBlocksProcessingEnvFactory simulateProcessingEnvFactory, IBlocksConfig blocksConfig) +{ + private const ProcessingOptions SimulateProcessingOptions = + ProcessingOptions.ForceProcessing + | ProcessingOptions.IgnoreParentNotOnMainChain + | ProcessingOptions.MarkAsProcessed + | ProcessingOptions.StoreReceipts; + + private void PrepareState(BlockHeader blockHeader, + BlockHeader parent, + BlockStateCall blockStateCall, + IWorldState stateProvider, + OverridableCodeInfoRepository codeInfoRepository, + IReleaseSpec releaseSpec) + { + stateProvider.StateRoot = parent.StateRoot!; + stateProvider.ApplyStateOverrides(codeInfoRepository, blockStateCall.StateOverrides, releaseSpec, blockHeader.Number); + + IEnumerable
senders = blockStateCall.Calls?.Select(details => details.Transaction.SenderAddress) ?? Enumerable.Empty(); + IEnumerable
targets = blockStateCall.Calls?.Select(details => details.Transaction.To!) ?? Enumerable.Empty(); + foreach (Address address in senders.Union(targets).Where(t => t is not null)) + { + stateProvider.CreateAccountIfNotExists(address, 0, 1); + } + + stateProvider.Commit(releaseSpec); + stateProvider.CommitTree(blockHeader.Number - 1); + stateProvider.RecalculateStateRoot(); + + blockHeader.StateRoot = stateProvider.StateRoot; + } + + public bool TrySimulate( + BlockHeader parent, + SimulatePayload payload, + IBlockTracer tracer, + [NotNullWhen(false)] out string? error) => + TrySimulate(parent, payload, tracer, simulateProcessingEnvFactory.Create(payload.Validation), out error); + + + private bool TrySimulate( + BlockHeader parent, + SimulatePayload payload, + IBlockTracer tracer, + SimulateReadOnlyBlocksProcessingEnv env, + [NotNullWhen(false)] out string? error) + { + IBlockTree blockTree = env.BlockTree; + IWorldState stateProvider = env.WorldState; + parent = GetParent(parent, payload, blockTree); + IReleaseSpec spec = env.SpecProvider.GetSpec(parent); + + if (payload.BlockStateCalls is not null) + { + Dictionary nonceCache = new(); + List suggestedBlocks = [null]; + + foreach (BlockStateCall blockCall in payload.BlockStateCalls) + { + nonceCache.Clear(); + BlockHeader callHeader = GetCallHeader(blockCall, parent, payload.Validation, spec); //currentSpec is still parent spec + spec = env.SpecProvider.GetSpec(callHeader); + PrepareState(callHeader, parent, blockCall, env.WorldState, env.CodeInfoRepository, spec); + + if (blockCall.BlockOverrides is { BaseFeePerGas: not null }) + { + callHeader.BaseFeePerGas = blockCall.BlockOverrides.BaseFeePerGas.Value; + } + else if (!payload.Validation) + { + callHeader.BaseFeePerGas = 0; + } + callHeader.Hash = callHeader.CalculateHash(); + + Transaction[] transactions = CreateTransactions(payload, blockCall, callHeader, stateProvider, nonceCache); + if (!TryGetBlock(payload, env, callHeader, transactions, out Block currentBlock, out error)) + { + return false; + } + + ProcessingOptions processingFlags = SimulateProcessingOptions; + + if (!payload.Validation) + { + processingFlags |= ProcessingOptions.NoValidation; + } + + suggestedBlocks[0] = currentBlock; + + IBlockProcessor processor = env.GetProcessor(payload.Validation); + Block processedBlock = + processor.Process(stateProvider.StateRoot, suggestedBlocks, processingFlags, tracer)[0]; + + FinalizeStateAndBlock(stateProvider, processedBlock, spec, currentBlock, blockTree); + parent = processedBlock.Header; + } + } + + error = null; + return true; + } + + private static void FinalizeStateAndBlock(IWorldState stateProvider, Block processedBlock, IReleaseSpec currentSpec, Block currentBlock, IBlockTree blockTree) + { + stateProvider.StateRoot = processedBlock.StateRoot!; + stateProvider.Commit(currentSpec); + stateProvider.CommitTree(currentBlock.Number); + blockTree.SuggestBlock(processedBlock, BlockTreeSuggestOptions.ForceSetAsMain); + blockTree.UpdateHeadBlock(processedBlock.Hash!); + } + + private static BlockHeader GetParent(BlockHeader parent, SimulatePayload payload, IBlockTree blockTree) + { + Block? latestBlock = blockTree.FindLatestBlock(); + long latestBlockNumber = latestBlock?.Number ?? 0; + + if (latestBlockNumber < parent.Number) + { + parent = latestBlock?.Header ?? blockTree.Head!.Header; + } + + BlockStateCall? firstBlock = payload.BlockStateCalls?.FirstOrDefault(); + + ulong lastKnown = (ulong)latestBlockNumber; + if (firstBlock?.BlockOverrides?.Number > 0 && firstBlock.BlockOverrides?.Number < lastKnown) + { + Block? searchResult = blockTree.FindBlock((long)firstBlock.BlockOverrides.Number); + if (searchResult is not null) + { + parent = searchResult.Header; + } + } + + return parent; + } + + private static bool TryGetBlock( + SimulatePayload payload, + SimulateReadOnlyBlocksProcessingEnv env, + BlockHeader callHeader, + Transaction[] transactions, + out Block currentBlock, + [NotNullWhen(false)] out string? error) + { + IWorldState stateProvider = env.WorldState; + Snapshot shoot = stateProvider.TakeSnapshot(); + currentBlock = new Block(callHeader); + LinkedHashSet testedTxs = new(); + for (int index = 0; index < transactions.Length; index++) + { + Transaction transaction = transactions[index]; + BlockProcessor.AddingTxEventArgs? args = env.BlockTransactionPicker.CanAddTransaction(currentBlock, transaction, testedTxs, stateProvider); + + if (args.Action is BlockProcessor.TxAction.Stop or BlockProcessor.TxAction.Skip && payload.Validation) + { + error = $"invalid transaction index: {index} at block number: {callHeader.Number}, Reason: {args.Reason}"; + return false; + } + + stateProvider.IncrementNonce(transaction.SenderAddress!); + testedTxs.Add(transaction); + } + + stateProvider.Restore(shoot); + stateProvider.RecalculateStateRoot(); + + currentBlock = currentBlock.WithReplacedBody(currentBlock.Body.WithChangedTransactions(testedTxs.ToArray())); + error = null; + return true; + } + + private Transaction[] CreateTransactions(SimulatePayload payload, + BlockStateCall callInputBlock, + BlockHeader callHeader, + IWorldState stateProvider, + Dictionary nonceCache) + { + int notSpecifiedGasTxsCount = callInputBlock.Calls?.Count(details => !details.HadGasLimitInRequest) ?? 0; + long gasSpecified = callInputBlock.Calls?.Where(details => details.HadGasLimitInRequest).Sum(details => details.Transaction.GasLimit) ?? 0; + if (notSpecifiedGasTxsCount > 0) + { + long gasPerTx = callHeader.GasLimit - gasSpecified / notSpecifiedGasTxsCount; + IEnumerable notSpecifiedGasTxs = callInputBlock.Calls?.Where(details => !details.HadGasLimitInRequest) ?? Enumerable.Empty(); + foreach (TransactionWithSourceDetails call in notSpecifiedGasTxs) + { + call.Transaction.GasLimit = gasPerTx; + } + } + + return callInputBlock.Calls?.Select(t => CreateTransaction(t, callHeader, stateProvider, nonceCache, payload.Validation)).ToArray() ?? Array.Empty(); + } + + private Transaction CreateTransaction(TransactionWithSourceDetails transactionDetails, + BlockHeader callHeader, + IWorldState stateProvider, + Dictionary nonceCache, + bool validate) + { + Transaction? transaction = transactionDetails.Transaction; + transaction.SenderAddress ??= Address.Zero; + transaction.To ??= Address.Zero; + transaction.Data ??= Memory.Empty; + + if (!transactionDetails.HadNonceInRequest) + { + ref UInt256 cachedNonce = ref CollectionsMarshal.GetValueRefOrAddDefault(nonceCache, transaction.SenderAddress, out bool exist); + if (!exist) + { + if (stateProvider.TryGetAccount(transaction.SenderAddress, out AccountStruct test)) + { + cachedNonce = test.Nonce; + } + // else // Todo think if we shall create account here + } + else + { + cachedNonce++; + } + + transaction.Nonce = cachedNonce; + } + + if (validate) + { + if (transaction.GasPrice == 0) + { + transaction.GasPrice = callHeader.BaseFeePerGas; + } + + if (transaction.Type == TxType.EIP1559 && transaction.DecodedMaxFeePerGas == 0) + { + transaction.DecodedMaxFeePerGas = transaction.GasPrice == 0 + ? callHeader.BaseFeePerGas + 1 + : transaction.GasPrice; + } + } + + transaction.Hash ??= transaction.CalculateHash(); + + return transaction; + } + + private BlockHeader GetCallHeader(BlockStateCall block, BlockHeader parent, bool payloadValidation, IReleaseSpec parentSpec) => + block.BlockOverrides is not null + ? block.BlockOverrides.GetBlockHeader(parent, blocksConfig, parentSpec) + : new BlockHeader( + parent.Hash!, + Keccak.OfAnEmptySequenceRlp, + Address.Zero, + UInt256.Zero, + parent.Number + 1, + parent.GasLimit, + parent.Timestamp + 1, + Array.Empty()) + { + BaseFeePerGas = !payloadValidation ? 0 : BaseFeeCalculator.Calculate(parent, parentSpec), + MixHash = parent.MixHash, + IsPostMerge = parent.Difficulty == 0 + }; +} diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryBlockStore.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryBlockStore.cs new file mode 100644 index 00000000000..4b76d9bcdb5 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryBlockStore.cs @@ -0,0 +1,94 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using Nethermind.Blockchain.Blocks; +using Nethermind.Core; +using Nethermind.Core.Buffers; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Serialization.Rlp; + +namespace Nethermind.Facade.Simulate; + +public class SimulateDictionaryBlockStore(IBlockStore readonlyBaseBlockStore) : IBlockStore +{ + private readonly Dictionary _blockDict = new(); + private readonly Dictionary _blockNumDict = new(); + private readonly Dictionary _metadataDict = new(Bytes.EqualityComparer); + private readonly BlockDecoder _blockDecoder = new(); + + public void Insert(Block block, WriteFlags writeFlags = WriteFlags.None) + { + _blockDict[block.Hash] = block; + _blockNumDict[block.Number] = block; + } + + public void Delete(long blockNumber, Hash256 blockHash) + { + _blockDict.Remove(blockHash); + _blockNumDict.Remove(blockNumber); + } + + public Block? Get(long blockNumber, Hash256 blockHash, RlpBehaviors rlpBehaviors = RlpBehaviors.None, bool shouldCache = true) + { + if (_blockNumDict.TryGetValue(blockNumber, out Block block)) + { + return block; + } + + block = readonlyBaseBlockStore.Get(blockNumber, blockHash, rlpBehaviors, false); + if (block is not null && shouldCache) + { + Cache(block); + } + return block; + } + + public byte[]? GetRaw(long blockNumber, Hash256 blockHash) + { + if (_blockNumDict.TryGetValue(blockNumber, out Block block)) + { + using NettyRlpStream newRlp = _blockDecoder.EncodeToNewNettyStream(block); + return newRlp.AsSpan().ToArray(); + } + return readonlyBaseBlockStore.GetRaw(blockNumber, blockHash); + } + + public IEnumerable GetAll() + { + var allBlocks = new HashSet(readonlyBaseBlockStore.GetAll()); + foreach (Block block in _blockDict.Values) + { + allBlocks.Add(block); + } + return allBlocks; + } + + public ReceiptRecoveryBlock? GetReceiptRecoveryBlock(long blockNumber, Hash256 blockHash) + { + if (_blockNumDict.TryGetValue(blockNumber, out Block block)) + { + using NettyRlpStream newRlp = _blockDecoder.EncodeToNewNettyStream(block); + using var memoryManager = new CappedArrayMemoryManager(newRlp.Data); + return BlockDecoder.DecodeToReceiptRecoveryBlock(memoryManager, memoryManager.Memory, RlpBehaviors.None); + } + return readonlyBaseBlockStore.GetReceiptRecoveryBlock(blockNumber, blockHash); + } + + public void Cache(Block block) + { + Insert(block); + } + + public void SetMetadata(byte[] key, byte[] value) + { + _metadataDict[key] = value; + readonlyBaseBlockStore.SetMetadata(key, value); + } + + public byte[]? GetMetadata(byte[] key) + { + return _metadataDict.TryGetValue(key, out var value) ? value : readonlyBaseBlockStore.GetMetadata(key); + } +} diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryHeaderStore.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryHeaderStore.cs new file mode 100644 index 00000000000..715df65eabb --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateDictionaryHeaderStore.cs @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using Nethermind.Blockchain.Headers; +using Nethermind.Core; +using Nethermind.Core.Crypto; + +namespace Nethermind.Facade.Simulate; + +/// +/// This type is needed for two things: +/// - Bypass issue of networking compatibility and RLPs not supporting BaseFeePerGas of 0 +/// - Improve performance to get faster local caching without re-encoding of data in Simulate blocks +/// +/// +public class SimulateDictionaryHeaderStore(IHeaderStore readonlyBaseHeaderStore) : IHeaderStore +{ + private readonly Dictionary _headerDict = new(); + private readonly Dictionary _blockNumberDict = new(); + + public void Insert(BlockHeader header) + { + _headerDict[header.Hash] = header; + InsertBlockNumber(header.Hash, header.Number); + } + + public BlockHeader? Get(Hash256 blockHash, bool shouldCache = false, long? blockNumber = null) + { + if (blockNumber is null) + { + blockNumber = GetBlockNumber(blockHash); + } + + if (blockNumber.HasValue && _headerDict.TryGetValue(blockHash, out BlockHeader? header)) + { + if (shouldCache) + { + Cache(header); + } + return header; + } + + header = readonlyBaseHeaderStore.Get(blockHash, false, blockNumber); + if (header is not null && shouldCache) + { + Cache(header); + } + return header; + } + + public void Cache(BlockHeader header) + { + Insert(header); + } + + public void Delete(Hash256 blockHash) + { + _headerDict.Remove(blockHash); + _blockNumberDict.Remove(blockHash); + } + + public void InsertBlockNumber(Hash256 blockHash, long blockNumber) + { + _blockNumberDict[blockHash] = blockNumber; + } + + public long? GetBlockNumber(Hash256 blockHash) + { + return _blockNumberDict.TryGetValue(blockHash, out var blockNumber) ? blockNumber : readonlyBaseHeaderStore.GetBlockNumber(blockHash); + } +} diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateOutput.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateOutput.cs new file mode 100644 index 00000000000..939de8f3021 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateOutput.cs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using Nethermind.Facade.Proxy.Models.Simulate; + +namespace Nethermind.Facade.Simulate; + +public class SimulateOutput +{ + + public string? Error { get; set; } + public int? ErrorCode { get; set; } + + public IReadOnlyList Items { get; set; } +} diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateReadOnlyBlocksProcessingEnv.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateReadOnlyBlocksProcessingEnv.cs new file mode 100644 index 00000000000..8d152eb201d --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateReadOnlyBlocksProcessingEnv.cs @@ -0,0 +1,117 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Blockchain; +using Nethermind.Blockchain.Blocks; +using Nethermind.Blockchain.Receipts; +using Nethermind.Consensus.Processing; +using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Validators; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Db; +using Nethermind.Evm; +using Nethermind.Evm.Tracing; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Logging; +using Nethermind.State; +using static Nethermind.Consensus.Processing.BlockProcessor; + +namespace Nethermind.Facade.Simulate; + +public class SimulateBlockValidationTransactionsExecutor : BlockValidationTransactionsExecutor +{ + public SimulateBlockValidationTransactionsExecutor(ITransactionProcessor transactionProcessor, IWorldState stateProvider) : base(transactionProcessor, stateProvider) + { + } + + public SimulateBlockValidationTransactionsExecutor(ITransactionProcessorAdapter transactionProcessor, IWorldState stateProvider) : base(transactionProcessor, stateProvider) + { + } + + protected override void ProcessTransaction(in BlockExecutionContext blkCtx, Transaction currentTx, int index, + BlockReceiptsTracer receiptsTracer, ProcessingOptions processingOptions) + { + processingOptions |= ProcessingOptions.ForceProcessing | ProcessingOptions.DoNotVerifyNonce | ProcessingOptions.NoValidation; + base.ProcessTransaction(in blkCtx, currentTx, index, receiptsTracer, processingOptions); + } +} + +public class SimulateReadOnlyBlocksProcessingEnv : ReadOnlyTxProcessingEnvBase, IDisposable +{ + private readonly IBlockValidator _blockValidator; + private readonly ILogManager? _logManager; + private readonly TransactionProcessor _transactionProcessor; + public IWorldState WorldState => StateProvider; + + public SimulateReadOnlyBlocksProcessingEnv( + IWorldStateManager worldStateManager, + IReadOnlyBlockTree baseBlockTree, + IReadOnlyDbProvider readOnlyDbProvider, + IBlockTree blockTree, + ISpecProvider specProvider, + ILogManager? logManager = null, + bool validate = false) + : base(worldStateManager, blockTree, specProvider, logManager) + { + ReadOnlyBlockTree = baseBlockTree; + DbProvider = readOnlyDbProvider; + WorldStateManager = worldStateManager; + _logManager = logManager; + + BlockTree = new BlockTreeOverlay(ReadOnlyBlockTree, blockTree); + BlockhashProvider = new SimulateBlockhashProvider(new BlockhashProvider(BlockTree, specProvider, StateProvider, logManager), BlockTree); + StateProvider = WorldStateManager.GlobalWorldState; + StateReader = WorldStateManager.GlobalStateReader; + CodeInfoRepository = new OverridableCodeInfoRepository(new CodeInfoRepository()); + VirtualMachine = new SimulateVirtualMachine(new VirtualMachine(BlockhashProvider, specProvider, CodeInfoRepository, logManager)); + _transactionProcessor = new SimulateTransactionProcessor(SpecProvider, StateProvider, VirtualMachine, CodeInfoRepository, _logManager, validate); + _blockValidator = CreateValidator(); + BlockTransactionPicker = new BlockProductionTransactionPicker(specProvider, true); + } + + public IWorldStateManager WorldStateManager { get; } + public IVirtualMachine VirtualMachine { get; } + public IReadOnlyDbProvider DbProvider { get; } + public IReadOnlyBlockTree ReadOnlyBlockTree { get; set; } + public OverridableCodeInfoRepository CodeInfoRepository { get; } + public BlockProductionTransactionPicker BlockTransactionPicker { get; } + + public void Dispose() + { + DbProvider.Dispose(); + } + + private SimulateBlockValidatorProxy CreateValidator() + { + HeaderValidator headerValidator = new( + BlockTree, + Always.Valid, + SpecProvider, + _logManager); + + BlockValidator blockValidator = new( + new TxValidator(SpecProvider!.ChainId), + headerValidator, + Always.Valid, + SpecProvider, + _logManager); + + return new SimulateBlockValidatorProxy(blockValidator); + } + + public IBlockProcessor GetProcessor(bool validate) => + new BlockProcessor(SpecProvider, + _blockValidator, + NoBlockRewards.Instance, + validate + ? new BlockValidationTransactionsExecutor(_transactionProcessor, StateProvider) + : new SimulateBlockValidationTransactionsExecutor(_transactionProcessor, StateProvider), + StateProvider, + NullReceiptStorage.Instance, + new BlockhashStore(BlockTree, SpecProvider, StateProvider), + _transactionProcessor, + _logManager); +} diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateReadOnlyBlocksProcessingEnvFactory.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateReadOnlyBlocksProcessingEnvFactory.cs new file mode 100644 index 00000000000..99078525673 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateReadOnlyBlocksProcessingEnvFactory.cs @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Blockchain; +using Nethermind.Blockchain.Blocks; +using Nethermind.Blockchain.Headers; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Core.Specs; +using Nethermind.Db; +using Nethermind.Db.Blooms; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.State.Repositories; +using Nethermind.Trie.Pruning; + +namespace Nethermind.Facade.Simulate; + +public class SimulateReadOnlyBlocksProcessingEnvFactory( + IWorldStateManager worldStateManager, + IReadOnlyBlockTree baseBlockTree, + IDbProvider dbProvider, + ISpecProvider specProvider, + ILogManager? logManager = null) +{ + public SimulateReadOnlyBlocksProcessingEnv Create(bool validate) + { + IReadOnlyDbProvider editableDbProvider = new ReadOnlyDbProvider(dbProvider, true); + OverlayTrieStore overlayTrieStore = new(editableDbProvider.StateDb, worldStateManager.TrieStore, logManager); + OverlayWorldStateManager overlayWorldStateManager = new(editableDbProvider, overlayTrieStore, logManager); + BlockTree tempBlockTree = CreateTempBlockTree(editableDbProvider, specProvider, logManager, editableDbProvider); + + return new SimulateReadOnlyBlocksProcessingEnv( + overlayWorldStateManager, + baseBlockTree, + editableDbProvider, + tempBlockTree, + specProvider, + logManager, + validate); + } + + private static BlockTree CreateTempBlockTree(IReadOnlyDbProvider readOnlyDbProvider, ISpecProvider? specProvider, ILogManager? logManager, IReadOnlyDbProvider editableDbProvider) + { + IBlockStore mainblockStore = new BlockStore(editableDbProvider.BlocksDb); + IHeaderStore mainHeaderStore = new HeaderStore(editableDbProvider.HeadersDb, editableDbProvider.BlockNumbersDb); + SimulateDictionaryHeaderStore tmpHeaderStore = new(mainHeaderStore); + const int badBlocksStored = 1; + + SimulateDictionaryBlockStore tmpBlockStore = new(mainblockStore); + IBlockStore badBlockStore = new BlockStore(editableDbProvider.BadBlocksDb, badBlocksStored); + + return new(tmpBlockStore, + tmpHeaderStore, + editableDbProvider.BlockInfosDb, + editableDbProvider.MetadataDb, + badBlockStore, + new ChainLevelInfoRepository(readOnlyDbProvider.BlockInfosDb), + specProvider, + NullBloomStorage.Instance, + new SyncConfig(), + logManager); + } +} diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateTransactionProcessor.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateTransactionProcessor.cs new file mode 100644 index 00000000000..79499458247 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateTransactionProcessor.cs @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Evm; +using Nethermind.Evm.Tracing; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Logging; +using Nethermind.State; + +namespace Nethermind.Facade.Simulate; + +public class SimulateTransactionProcessor( + ISpecProvider? specProvider, + IWorldState? worldState, + IVirtualMachine? virtualMachine, + ICodeInfoRepository? codeInfoRepository, + ILogManager? logManager, + bool validate) + : TransactionProcessor(specProvider, worldState, virtualMachine, codeInfoRepository, logManager), ITransactionProcessor +{ + protected override TransactionResult Execute(Transaction tx, in BlockExecutionContext blCtx, ITxTracer tracer, ExecutionOptions opts) + { + if (!validate) + { + opts |= ExecutionOptions.NoValidation; + } + + return base.Execute(tx, in blCtx, tracer, opts); + } +} diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateTxMutatorTracer.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateTxMutatorTracer.cs new file mode 100644 index 00000000000..4234b78edf9 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateTxMutatorTracer.cs @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Linq; +using Nethermind.Abi; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Evm; +using Nethermind.Evm.Tracing; +using Nethermind.Facade.Proxy.Models.Simulate; +using Nethermind.Int256; +using Log = Nethermind.Facade.Proxy.Models.Simulate.Log; + +namespace Nethermind.Facade.Simulate; + +internal sealed class SimulateTxMutatorTracer : TxTracer, ITxLogsMutator +{ + public const int ExecutionError = -32015; + + private static readonly Hash256 transferSignature = + new AbiSignature("Transfer", AbiType.Address, AbiType.Address, AbiType.UInt256).Hash; + + private static readonly Address Erc20Sender = new("0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"); + private readonly Hash256 _currentBlockHash; + private readonly ulong _currentBlockNumber; + private readonly Hash256 _txHash; + private readonly ulong _txIndex; + private ICollection? _logsToMutate; + + public SimulateTxMutatorTracer(bool isTracingTransfers, Hash256 txHash, ulong currentBlockNumber, Hash256 currentBlockHash, + ulong txIndex) + { + _txHash = txHash; + _currentBlockNumber = currentBlockNumber; + _currentBlockHash = currentBlockHash; + _txIndex = txIndex; + IsTracingReceipt = true; + IsTracingActions = IsMutatingLogs = isTracingTransfers; + } + + public SimulateCallResult? TraceResult { get; set; } + + public bool IsMutatingLogs { get; } + + public void SetLogsToMutate(ICollection logsToMutate) => _logsToMutate = logsToMutate; + + public override void ReportAction(long gas, UInt256 value, Address from, Address to, ReadOnlyMemory input, ExecutionType callType, bool isPrecompileCall = false) + { + base.ReportAction(gas, value, from, to, input, callType, isPrecompileCall); + if (value > UInt256.Zero) + { + var data = AbiEncoder.Instance.Encode(AbiEncodingStyle.Packed, new AbiSignature("", AbiType.UInt256), + value); + _logsToMutate?.Add(new LogEntry(Erc20Sender, data, [transferSignature, from.ToHash(), to.ToHash()])); + } + } + + public override void MarkAsSuccess(Address recipient, long gasSpent, byte[] output, LogEntry[] logs, + Hash256? stateRoot = null) + { + TraceResult = new SimulateCallResult + { + GasUsed = (ulong)gasSpent, + ReturnData = output, + Status = StatusCode.Success, + Logs = logs.Select((entry, i) => new Log + { + Address = entry.LoggersAddress, + Topics = entry.Topics, + Data = entry.Data, + LogIndex = (ulong)i, + TransactionHash = _txHash, + TransactionIndex = _txIndex, + BlockHash = _currentBlockHash, + BlockNumber = _currentBlockNumber + }).ToList() + }; + } + + public override void MarkAsFailed(Address recipient, long gasSpent, byte[] output, string error, + Hash256? stateRoot = null) + { + TraceResult = new SimulateCallResult + { + GasUsed = (ulong)gasSpent, + Error = new Error + { + Code = ExecutionError, // revert error code stub + Message = error + }, + ReturnData = null, + Status = StatusCode.Failure + }; + } +} diff --git a/src/Nethermind/Nethermind.Facade/Simulate/SimulateVirtualMachine.cs b/src/Nethermind/Nethermind.Facade/Simulate/SimulateVirtualMachine.cs new file mode 100644 index 00000000000..5ea81093e80 --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/Simulate/SimulateVirtualMachine.cs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Diagnostics.CodeAnalysis; +using Nethermind.Evm; +using Nethermind.Evm.Tracing; +using Nethermind.State; + +namespace Nethermind.Facade.Simulate; + +public class SimulateVirtualMachine(IVirtualMachine virtualMachine) : IVirtualMachine +{ + public TransactionSubstate Run(EvmState state, IWorldState worldState, ITxTracer txTracer) where TTracingActions : struct, VirtualMachine.IIsTracing + { + if (typeof(TTracingActions) == typeof(VirtualMachine.IsTracing) && TryGetLogsMutator(txTracer, out ITxLogsMutator logsMutator)) + { + logsMutator.SetLogsToMutate(state.Logs); + } + + return virtualMachine.Run(state, worldState, txTracer); + } + + private static bool TryGetLogsMutator(ITxTracer txTracer, [NotNullWhen(true)] out ITxLogsMutator? txLogsMutator) + { + switch (txTracer) + { + case ITxLogsMutator { IsMutatingLogs: true } logsMutator: + txLogsMutator = logsMutator; + return true; + case ITxTracerWrapper txTracerWrapper: + return TryGetLogsMutator(txTracerWrapper.InnerTracer, out txLogsMutator); + default: + txLogsMutator = null; + return false; + } + } +} diff --git a/src/Nethermind/Nethermind.Facade/StateOverridesExtensions.cs b/src/Nethermind/Nethermind.Facade/StateOverridesExtensions.cs new file mode 100644 index 00000000000..080e6c2a81b --- /dev/null +++ b/src/Nethermind/Nethermind.Facade/StateOverridesExtensions.cs @@ -0,0 +1,132 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; +using Nethermind.Evm.CodeAnalysis; +using Nethermind.Facade.Proxy.Models; +using Nethermind.Int256; +using Nethermind.State; +using Nethermind.Trie; + +namespace Nethermind.Facade; + +public static class StateOverridesExtensions +{ + public static void ApplyStateOverrides( + this IWorldState state, + OverridableCodeInfoRepository overridableCodeInfoRepository, + Dictionary? overrides, + IReleaseSpec spec, + long blockNumber) + { + if (overrides is not null) + { + foreach ((Address address, AccountOverride accountOverride) in overrides) + { + if (!state.TryGetAccount(address, out AccountStruct account)) + { + state.CreateAccount(address, accountOverride.Balance ?? UInt256.Zero, accountOverride.Nonce ?? UInt256.Zero); + } + else + { + state.UpdateBalance(spec, account, accountOverride, address); + state.UpdateNonce(account, accountOverride, address); + } + + state.UpdateCode(overridableCodeInfoRepository, spec, accountOverride, address); + state.UpdateState(accountOverride, address); + } + } + + state.Commit(spec); + state.CommitTree(blockNumber); + state.RecalculateStateRoot(); + } + + private static void UpdateState(this IWorldState stateProvider, AccountOverride accountOverride, Address address) + { + void ApplyState(Dictionary diff) + { + foreach ((UInt256 index, Hash256 value) in diff) + { + stateProvider.Set(new StorageCell(address, index), value.Bytes.WithoutLeadingZeros().ToArray()); + } + } + + if (accountOverride.State is not null) + { + stateProvider.ClearStorage(address); + ApplyState(accountOverride.State); + } + else if (accountOverride.StateDiff is not null) + { + ApplyState(accountOverride.StateDiff); + } + } + + private static void UpdateCode( + this IWorldState stateProvider, + OverridableCodeInfoRepository overridableCodeInfoRepository, + IReleaseSpec currentSpec, + AccountOverride accountOverride, + Address address) + { + if (accountOverride.Code is not null) + { + overridableCodeInfoRepository.SetCodeOverwrite( + stateProvider, + currentSpec, + address, + new CodeInfo(accountOverride.Code), + accountOverride.MovePrecompileToAddress); + } + } + + private static void UpdateNonce( + this IWorldState stateProvider, + in AccountStruct account, + AccountOverride accountOverride, + Address address) + { + if (accountOverride.Nonce is not null) + { + UInt256 nonce = account.Nonce; + UInt256 newNonce = accountOverride.Nonce.Value; + if (nonce > newNonce) + { + stateProvider.DecrementNonce(address, nonce - newNonce); + } + else if (nonce < accountOverride.Nonce) + { + stateProvider.IncrementNonce(address, newNonce - nonce); + } + } + } + + private static void UpdateBalance( + this IWorldState stateProvider, + IReleaseSpec spec, + in AccountStruct account, + AccountOverride accountOverride, + Address address) + { + if (accountOverride.Balance is not null) + { + UInt256 balance = account.Balance; + UInt256 newBalance = accountOverride.Balance.Value; + if (balance > newBalance) + { + stateProvider.SubtractFromBalance(address, balance - newBalance, spec); + } + else if (balance < newBalance) + { + stateProvider.AddToBalance(address, newBalance - balance, spec); + } + } + } +} diff --git a/src/Nethermind/Nethermind.HealthChecks.Test/NodeHealthServiceTests.cs b/src/Nethermind/Nethermind.HealthChecks.Test/NodeHealthServiceTests.cs index 93bd1a17196..c3336e8370a 100644 --- a/src/Nethermind/Nethermind.HealthChecks.Test/NodeHealthServiceTests.cs +++ b/src/Nethermind/Nethermind.HealthChecks.Test/NodeHealthServiceTests.cs @@ -38,13 +38,13 @@ public void CheckHealth_returns_expected_results([ValueSource(nameof(CheckHealth ISyncServer syncServer = Substitute.For(); IReceiptStorage receiptStorage = Substitute.For(); IBlockchainProcessor blockchainProcessor = Substitute.For(); - IBlockProducer blockProducer = Substitute.For(); + IBlockProducerRunner blockProducerRunner = Substitute.For(); ISyncConfig syncConfig = Substitute.For(); IHealthHintService healthHintService = Substitute.For(); INethermindApi api = Substitute.For(); api.SpecProvider = Substitute.For(); blockchainProcessor.IsProcessingBlocks(Arg.Any()).Returns(test.IsProcessingBlocks); - blockProducer.IsProducingBlocks(Arg.Any()).Returns(test.IsProducingBlocks); + blockProducerRunner.IsProducingBlocks(Arg.Any()).Returns(test.IsProducingBlocks); syncServer.GetPeerCount().Returns(test.PeerCount); IDriveInfo drive = Substitute.For(); @@ -65,7 +65,7 @@ public void CheckHealth_returns_expected_results([ValueSource(nameof(CheckHealth IEthSyncingInfo ethSyncingInfo = new EthSyncingInfo(blockFinder, receiptStorage, syncConfig, Substitute.For(), Substitute.For(), LimboLogs.Instance); NodeHealthService nodeHealthService = - new(syncServer, blockchainProcessor, blockProducer, new HealthChecksConfig(), + new(syncServer, blockchainProcessor, blockProducerRunner, new HealthChecksConfig(), healthHintService, ethSyncingInfo, new EngineRpcCapabilitiesProvider(api.SpecProvider), api, new[] { drive }, test.IsMining); CheckHealthResult result = nodeHealthService.CheckHealth(); Assert.That(result.Healthy, Is.EqualTo(test.ExpectedHealthy)); @@ -84,7 +84,7 @@ public void post_merge_health_checks([ValueSource(nameof(CheckHealthPostMergeTes IBlockTree blockFinder = Substitute.For(); ISyncServer syncServer = Substitute.For(); IBlockchainProcessor blockchainProcessor = Substitute.For(); - IBlockProducer blockProducer = Substitute.For(); + IBlockProducerRunner blockProducerRunner = Substitute.For(); IHealthHintService healthHintService = Substitute.For(); ISyncModeSelector syncModeSelector = new StaticSelector(test.SyncMode); INethermindApi api = Substitute.For(); @@ -133,7 +133,7 @@ public void post_merge_health_checks([ValueSource(nameof(CheckHealthPostMergeTes IEthSyncingInfo ethSyncingInfo = new EthSyncingInfo(blockFinder, new InMemoryReceiptStorage(), new SyncConfig(), syncModeSelector, Substitute.For(), new TestLogManager()); NodeHealthService nodeHealthService = - new(syncServer, blockchainProcessor, blockProducer, new HealthChecksConfig(), + new(syncServer, blockchainProcessor, blockProducerRunner, new HealthChecksConfig(), healthHintService, ethSyncingInfo, customProvider, api, new[] { drive }, false); nodeHealthService.CheckHealth(); diff --git a/src/Nethermind/Nethermind.HealthChecks/HealthChecksPlugin.cs b/src/Nethermind/Nethermind.HealthChecks/HealthChecksPlugin.cs index 3b4478eb748..e56363a30e3 100644 --- a/src/Nethermind/Nethermind.HealthChecks/HealthChecksPlugin.cs +++ b/src/Nethermind/Nethermind.HealthChecks/HealthChecksPlugin.cs @@ -119,10 +119,6 @@ public void AddServices(IServiceCollection service) .AddInMemoryStorage(); } } - public Task InitNetworkProtocol() - { - return Task.CompletedTask; - } public Task InitRpcModules() { @@ -142,7 +138,7 @@ public Task InitRpcModules() } _nodeHealthService = new NodeHealthService(_api.SyncServer, - _api.BlockchainProcessor!, _api.BlockProducer!, _healthChecksConfig, _api.HealthHintService!, + _api.BlockchainProcessor!, _api.BlockProducerRunner!, _healthChecksConfig, _api.HealthHintService!, _api.EthSyncingInfo!, _api.RpcCapabilitiesProvider, _api, drives, _initConfig.IsMining); if (_healthChecksConfig.Enabled) diff --git a/src/Nethermind/Nethermind.HealthChecks/NodeHealthService.cs b/src/Nethermind/Nethermind.HealthChecks/NodeHealthService.cs index f31ca24636f..4f11d3ea5f9 100644 --- a/src/Nethermind/Nethermind.HealthChecks/NodeHealthService.cs +++ b/src/Nethermind/Nethermind.HealthChecks/NodeHealthService.cs @@ -28,7 +28,7 @@ public class NodeHealthService : INodeHealthService { private readonly ISyncServer _syncServer; private readonly IBlockchainProcessor _blockchainProcessor; - private readonly IBlockProducer _blockProducer; + private readonly IBlockProducerRunner _blockProducerRunner; private readonly IHealthChecksConfig _healthChecksConfig; private readonly IHealthHintService _healthHintService; private readonly IEthSyncingInfo _ethSyncingInfo; @@ -39,7 +39,7 @@ public class NodeHealthService : INodeHealthService public NodeHealthService(ISyncServer syncServer, IBlockchainProcessor blockchainProcessor, - IBlockProducer blockProducer, + IBlockProducerRunner blockProducerRunner, IHealthChecksConfig healthChecksConfig, IHealthHintService healthHintService, IEthSyncingInfo ethSyncingInfo, @@ -53,7 +53,7 @@ public NodeHealthService(ISyncServer syncServer, _healthChecksConfig = healthChecksConfig; _healthHintService = healthHintService; _blockchainProcessor = blockchainProcessor; - _blockProducer = blockProducer; + _blockProducerRunner = blockProducerRunner; _ethSyncingInfo = ethSyncingInfo; _rpcCapabilitiesProvider = rpcCapabilitiesProvider; _api = api; @@ -232,7 +232,7 @@ private static bool CheckPeers(ICollection<(string Description, string LongDescr private bool IsProducingBlocks(ICollection<(string Description, string LongDescription)> messages, ICollection errors) { ulong? maxIntervalHint = GetBlockProducerIntervalHint(); - bool producingBlocks = _blockProducer.IsProducingBlocks(maxIntervalHint); + bool producingBlocks = _blockProducerRunner.IsProducingBlocks(maxIntervalHint); if (producingBlocks == false) { errors.Add(ErrorStrings.NotProducingBlocks); diff --git a/src/Nethermind/Nethermind.Init/InitializeStateDb.cs b/src/Nethermind/Nethermind.Init/InitializeStateDb.cs index f220854d2f3..96680ddbade 100644 --- a/src/Nethermind/Nethermind.Init/InitializeStateDb.cs +++ b/src/Nethermind/Nethermind.Init/InitializeStateDb.cs @@ -10,6 +10,7 @@ using Nethermind.Blockchain; using Nethermind.Blockchain.FullPruning; using Nethermind.Blockchain.Synchronization; +using Nethermind.Config; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -20,9 +21,7 @@ using Nethermind.Logging; using Nethermind.Serialization.Json; using Nethermind.State; -using Nethermind.State.Witnesses; using Nethermind.Synchronization.Trie; -using Nethermind.Synchronization.Witness; using Nethermind.Trie; using Nethermind.Trie.Pruning; @@ -54,6 +53,7 @@ public Task Execute(CancellationToken cancellationToken) ISyncConfig syncConfig = getApi.Config(); IPruningConfig pruningConfig = getApi.Config(); IInitConfig initConfig = getApi.Config(); + IBlocksConfig blockConfig = getApi.Config(); _api.NodeStorageFactory.DetectCurrentKeySchemeFrom(getApi.DbProvider.StateDb); @@ -76,7 +76,7 @@ public Task Execute(CancellationToken cancellationToken) if (pruningConfig.PruningBoundary < 64) { - if (_logger.IsWarn) _logger.Warn($"Prunig boundary must be at least 64. Setting to 64."); + if (_logger.IsWarn) _logger.Warn($"Pruning boundary must be at least 64. Setting to 64."); pruningConfig.PruningBoundary = 64; } @@ -86,23 +86,9 @@ public Task Execute(CancellationToken cancellationToken) syncConfig.DownloadBodiesInFastSync = true; } - IWitnessCollector witnessCollector; - if (syncConfig.WitnessProtocolEnabled) - { - WitnessCollector witnessCollectorImpl = new(getApi.DbProvider.WitnessDb, _api.LogManager); - witnessCollector = setApi.WitnessCollector = witnessCollectorImpl; - setApi.WitnessRepository = witnessCollectorImpl.WithPruning(getApi.BlockTree!, getApi.LogManager); - } - else - { - witnessCollector = setApi.WitnessCollector = NullWitnessCollector.Instance; - setApi.WitnessRepository = NullWitnessCollector.Instance; - } - _api.NodeStorageFactory.DetectCurrentKeySchemeFrom(getApi.DbProvider.StateDb); - IKeyValueStore codeDb = getApi.DbProvider.CodeDb - .WitnessedBy(witnessCollector); - IKeyValueStoreWithBatching stateWitnessedBy = getApi.DbProvider.StateDb.WitnessedBy(witnessCollector); + IKeyValueStore codeDb = getApi.DbProvider.CodeDb; + IKeyValueStoreWithBatching stateDb = getApi.DbProvider.StateDb; IPersistenceStrategy persistenceStrategy; IPruningStrategy pruningStrategy; if (pruningConfig.Mode.IsMemory()) @@ -115,6 +101,12 @@ public Task Execute(CancellationToken cancellationToken) persistenceStrategy = persistenceStrategy.Or(triggerPersistenceStrategy); } + if ((_api.NodeStorageFactory.CurrentKeyScheme != INodeStorage.KeyScheme.Hash || initConfig.StateDbKeyScheme == INodeStorage.KeyScheme.HalfPath) + && pruningConfig.CacheMb > 2000) + { + if (_logger.IsWarn) _logger.Warn($"Detected {pruningConfig.CacheMb}MB of pruning cache config. Pruning cache more than 2000MB is not recommended as it may cause long memory pruning time which affect attestation."); + } + pruningStrategy = Prune .WhenCacheReaches(pruningConfig.CacheMb.MB()) // Use of ratio, as the effectiveness highly correlate with the amount of keys per snapshot save which @@ -128,7 +120,7 @@ public Task Execute(CancellationToken cancellationToken) persistenceStrategy = Persist.EveryBlock; } - INodeStorage mainNodeStorage = _api.NodeStorageFactory.WrapKeyValueStore(stateWitnessedBy); + INodeStorage mainNodeStorage = _api.NodeStorageFactory.WrapKeyValueStore(stateDb); TrieStore trieStore = syncConfig.TrieHealing ? new HealingTrieStore( @@ -145,15 +137,25 @@ public Task Execute(CancellationToken cancellationToken) // TODO: Needed by node serving. Probably should use `StateReader` instead. setApi.TrieStore = trieStore; + ITrieStore mainWorldTrieStore = trieStore; + PreBlockCaches? preBlockCaches = null; + if (blockConfig.PreWarmStateOnBlockProcessing) + { + preBlockCaches = new PreBlockCaches(); + mainWorldTrieStore = new PreCachedTrieStore(trieStore, preBlockCaches.RlpCache); + } + IWorldState worldState = syncConfig.TrieHealing ? new HealingWorldState( - trieStore, + mainWorldTrieStore, codeDb, - getApi.LogManager) + getApi.LogManager, + preBlockCaches) : new WorldState( - trieStore, + mainWorldTrieStore, codeDb, - getApi.LogManager); + getApi.LogManager, + preBlockCaches); // This is probably the point where a different state implementation would switch. IWorldStateManager stateManager = setApi.WorldStateManager = new WorldStateManager( @@ -165,7 +167,7 @@ public Task Execute(CancellationToken cancellationToken) // TODO: Don't forget this TrieStoreBoundaryWatcher trieStoreBoundaryWatcher = new(stateManager, _api.BlockTree!, _api.LogManager); getApi.DisposeStack.Push(trieStoreBoundaryWatcher); - getApi.DisposeStack.Push(trieStore); + getApi.DisposeStack.Push(mainWorldTrieStore); setApi.WorldState = stateManager.GlobalWorldState; setApi.StateReader = stateManager.GlobalStateReader; diff --git a/src/Nethermind/Nethermind.Init/Steps/InitRlp.cs b/src/Nethermind/Nethermind.Init/Steps/InitRlp.cs index 81b8516e3e2..9a8d7ec4051 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitRlp.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitRlp.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using Nethermind.Api; +using Nethermind.Api.Extensions; using Nethermind.Core.Attributes; using Nethermind.Network; using Nethermind.Serialization.Rlp; @@ -33,6 +34,11 @@ public virtual Task Execute(CancellationToken _) Rlp.RegisterDecoders(assembly); } + foreach (INethermindPlugin plugin in _api.Plugins) + { + plugin.InitRlpDecoders(_api); + } + return Task.CompletedTask; } } diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockProducer.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockProducer.cs index e7e2b9225a0..380c4baa4f1 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockProducer.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockProducer.cs @@ -24,15 +24,24 @@ public InitializeBlockProducer(INethermindApi api) _api = api; } - public async Task Execute(CancellationToken _) + public Task Execute(CancellationToken _) { if (_api.BlockProductionPolicy!.ShouldStartBlockProduction()) { - _api.BlockProducer = await BuildProducer(); + _api.BlockProducer = BuildProducer(); + + _api.BlockProducerRunner = _api.GetConsensusPlugin()!.CreateBlockProducerRunner(); + + foreach (IConsensusWrapperPlugin wrapperPlugin in _api.GetConsensusWrapperPlugins().OrderBy((p) => p.Priority)) + { + _api.BlockProducerRunner = wrapperPlugin.InitBlockProducerRunner(_api.BlockProducerRunner); + } } + + return Task.CompletedTask; } - protected virtual async Task BuildProducer() + protected virtual IBlockProducer BuildProducer() { _api.BlockProducerEnvFactory = new BlockProducerEnvFactory( _api.WorldStateManager!, @@ -59,7 +68,7 @@ protected virtual async Task BuildProducer() blockProducerFactory = new ConsensusWrapperToBlockProducerFactoryAdapter(wrapperPlugin, blockProducerFactory); } - return await blockProducerFactory.InitBlockProducer(consensusPlugin.DefaultBlockProductionTrigger); + return blockProducerFactory.InitBlockProducer(); } else { @@ -71,9 +80,9 @@ private class ConsensusWrapperToBlockProducerFactoryAdapter( IConsensusWrapperPlugin consensusWrapperPlugin, IBlockProducerFactory baseBlockProducerFactory) : IBlockProducerFactory { - public Task InitBlockProducer(IBlockProductionTrigger blockProductionTrigger, ITxSource? additionalTxSource = null) + public IBlockProducer InitBlockProducer(ITxSource? additionalTxSource = null) { - return consensusWrapperPlugin.InitBlockProducer(baseBlockProducerFactory, blockProductionTrigger, additionalTxSource); + return consensusWrapperPlugin.InitBlockProducer(baseBlockProducerFactory, additionalTxSource); } } } diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs index b6c9aa3c61a..b90e73e2e63 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs @@ -37,9 +37,7 @@ using Nethermind.Logging; using Nethermind.Serialization.Json; using Nethermind.State; -using Nethermind.State.Witnesses; using Nethermind.Synchronization.Trie; -using Nethermind.Synchronization.Witness; using Nethermind.Trie; using Nethermind.Trie.Pruning; using Nethermind.TxPool; @@ -163,16 +161,20 @@ protected virtual ITransactionProcessor CreateTransactionProcessor() { if (_api.SpecProvider is null) throw new StepDependencyException(nameof(_api.SpecProvider)); - VirtualMachine virtualMachine = CreateVirtualMachine(); + CodeInfoRepository codeInfoRepository = new(); + VirtualMachine virtualMachine = CreateVirtualMachine(codeInfoRepository); - return new TransactionProcessor( + TransactionProcessor transactionProcessor = new( _api.SpecProvider, _api.WorldState, virtualMachine, + codeInfoRepository, _api.LogManager); + + return transactionProcessor; } - protected virtual VirtualMachine CreateVirtualMachine() + protected VirtualMachine CreateVirtualMachine(CodeInfoRepository codeInfoRepository) { if (_api.BlockTree is null) throw new StepDependencyException(nameof(_api.BlockTree)); if (_api.SpecProvider is null) throw new StepDependencyException(nameof(_api.SpecProvider)); @@ -182,10 +184,13 @@ protected virtual VirtualMachine CreateVirtualMachine() BlockhashProvider blockhashProvider = new( _api.BlockTree, _api.SpecProvider, _api.WorldState, _api.LogManager); - return new VirtualMachine( + VirtualMachine virtualMachine = new( blockhashProvider, _api.SpecProvider, + codeInfoRepository, _api.LogManager); + + return virtualMachine; } protected virtual IHealthHintService CreateHealthHintService() => @@ -220,9 +225,16 @@ protected virtual BlockProcessor CreateBlockProcessor() if (_api.RewardCalculatorSource is null) throw new StepDependencyException(nameof(_api.RewardCalculatorSource)); if (_api.TransactionProcessor is null) throw new StepDependencyException(nameof(_api.TransactionProcessor)); if (_api.BlockTree is null) throw new StepDependencyException(nameof(_api.BlockTree)); + if (_api.WorldStateManager is null) throw new StepDependencyException(nameof(_api.WorldStateManager)); + if (_api.SpecProvider is null) throw new StepDependencyException(nameof(_api.SpecProvider)); + IBlocksConfig blocksConfig = _api.Config(); IWorldState worldState = _api.WorldState!; + BlockCachePreWarmer? preWarmer = blocksConfig.PreWarmStateOnBlockProcessing + ? new(new(_api.WorldStateManager, _api.BlockTree, _api.SpecProvider, _api.LogManager, worldState), _api.SpecProvider, _api.LogManager, worldState) + : null; + return new BlockProcessor( _api.SpecProvider, _api.BlockValidator, @@ -230,10 +242,11 @@ protected virtual BlockProcessor CreateBlockProcessor() new BlockProcessor.BlockValidationTransactionsExecutor(_api.TransactionProcessor, worldState), worldState, _api.ReceiptStorage, - _api.WitnessCollector, new BlockhashStore(_api.BlockTree, _api.SpecProvider!, worldState), _api.TransactionProcessor, - _api.LogManager); + _api.LogManager, + preWarmer: preWarmer + ); } // TODO: remove from here - move to consensus? diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index 35bd10e7e2b..c3c79865a07 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -35,10 +35,8 @@ using Nethermind.Stats.Model; using Nethermind.Synchronization; using Nethermind.Synchronization.Blocks; -using Nethermind.Synchronization.LesSync; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; -using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SnapSync; using Nethermind.Synchronization.Trie; using Nethermind.TxPool; @@ -102,8 +100,6 @@ private async Task Initialize(CancellationToken cancellationToken) NetworkDiagTracer.Start(_api.LogManager); } - CanonicalHashTrie cht = new CanonicalHashTrie(_api.DbProvider!.ChtDb); - _api.BetterPeerStrategy = new TotalDifficultyBetterPeerStrategy(_api.LogManager); int maxPeersCount = _networkConfig.ActivePeersMaxCount; @@ -179,13 +175,10 @@ private async Task Initialize(CancellationToken cancellationToken) _api.SyncPeerPool, _api.SyncModeSelector, _api.Config(), - _api.WitnessRepository, _api.GossipPolicy, _api.SpecProvider!, - _api.LogManager, - cht); + _api.LogManager); - _ = syncServer.BuildCHT(); _api.DisposeStack.Push(syncServer); InitDiscovery(); @@ -534,11 +527,6 @@ private async Task InitPeer() _api.ProtocolsManager!.AddSupportedCapability(new Capability(Protocol.Snap, 1)); } - if (_syncConfig.WitnessProtocolEnabled) - { - _api.ProtocolsManager.AddSupportedCapability(new Capability(Protocol.Wit, 0)); - } - _api.ProtocolValidator = protocolValidator; NodesLoader nodesLoader = new(_networkConfig, _api.NodeStatsManager, peerStorage, _api.RlpxPeer, _api.LogManager); diff --git a/src/Nethermind/Nethermind.Init/Steps/LogHardwareInfo.cs b/src/Nethermind/Nethermind.Init/Steps/LogHardwareInfo.cs index 41619882327..3132f1b30ba 100644 --- a/src/Nethermind/Nethermind.Init/Steps/LogHardwareInfo.cs +++ b/src/Nethermind/Nethermind.Init/Steps/LogHardwareInfo.cs @@ -4,6 +4,7 @@ using System.Threading; using System.Threading.Tasks; using Nethermind.Api; +using Nethermind.Core.Cpu; using ILogger = Nethermind.Logging.ILogger; namespace Nethermind.Init.Steps; @@ -25,7 +26,7 @@ public Task Execute(CancellationToken cancellationToken) try { - var cpu = Cpu.RuntimeInformation.GetCpuInfo(); + var cpu = RuntimeInformation.GetCpuInfo(); if (cpu is not null) { _logger.Info($"CPU: {cpu.ProcessorName} ({cpu.PhysicalCoreCount}C{cpu.LogicalCoreCount}T)"); diff --git a/src/Nethermind/Nethermind.Init/Steps/RegisterPluginRpcModules.cs b/src/Nethermind/Nethermind.Init/Steps/RegisterPluginRpcModules.cs index 82d0110ee91..84dccd3ea34 100644 --- a/src/Nethermind/Nethermind.Init/Steps/RegisterPluginRpcModules.cs +++ b/src/Nethermind/Nethermind.Init/Steps/RegisterPluginRpcModules.cs @@ -1,33 +1,12 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; -using System.Linq; using System.Threading; using System.Threading.Tasks; using Nethermind.Api; using Nethermind.Api.Extensions; -using Nethermind.Blockchain.FullPruning; -using Nethermind.Core; -using Nethermind.Init.Steps.Migrations; -using Nethermind.JsonRpc; using Nethermind.JsonRpc.Modules; -using Nethermind.JsonRpc.Modules.Admin; -using Nethermind.JsonRpc.Modules.DebugModule; -using Nethermind.JsonRpc.Modules.Eth; using Nethermind.JsonRpc.Modules.Evm; -using Nethermind.JsonRpc.Modules.Net; -using Nethermind.JsonRpc.Modules.Parity; -using Nethermind.JsonRpc.Modules.Personal; -using Nethermind.JsonRpc.Modules.Proof; -using Nethermind.JsonRpc.Modules.Subscribe; -using Nethermind.JsonRpc.Modules.Trace; -using Nethermind.JsonRpc.Modules.TxPool; -using Nethermind.JsonRpc.Modules.Web3; -using Nethermind.JsonRpc.Modules.Witness; -using Nethermind.Logging; -using Nethermind.Network.Config; -using Nethermind.JsonRpc.Modules.Rpc; namespace Nethermind.Init.Steps; diff --git a/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs b/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs index f1756256fd1..db5341d33b4 100644 --- a/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs +++ b/src/Nethermind/Nethermind.Init/Steps/RegisterRpcModules.cs @@ -6,8 +6,8 @@ using System.Threading; using System.Threading.Tasks; using Nethermind.Api; -using Nethermind.Api.Extensions; using Nethermind.Blockchain.FullPruning; +using Nethermind.Config; using Nethermind.Core; using Nethermind.Init.Steps.Migrations; using Nethermind.JsonRpc; @@ -16,7 +16,6 @@ using Nethermind.JsonRpc.Modules.DebugModule; using Nethermind.JsonRpc.Modules.Eth; using Nethermind.JsonRpc.Modules.Eth.FeeHistory; -using Nethermind.JsonRpc.Modules.Evm; using Nethermind.JsonRpc.Modules.Net; using Nethermind.JsonRpc.Modules.Parity; using Nethermind.JsonRpc.Modules.Personal; @@ -25,11 +24,9 @@ using Nethermind.JsonRpc.Modules.Trace; using Nethermind.JsonRpc.Modules.TxPool; using Nethermind.JsonRpc.Modules.Web3; -using Nethermind.JsonRpc.Modules.Witness; using Nethermind.Logging; using Nethermind.Network.Config; using Nethermind.JsonRpc.Modules.Rpc; -using Nethermind.Serialization.Json; namespace Nethermind.Init.Steps; @@ -37,43 +34,37 @@ namespace Nethermind.Init.Steps; public class RegisterRpcModules : IStep { private readonly INethermindApi _api; + private readonly IJsonRpcConfig _jsonRpcConfig; public RegisterRpcModules(INethermindApi api) { _api = api; + _jsonRpcConfig = _api.Config(); } public virtual async Task Execute(CancellationToken cancellationToken) { - if (_api.BlockTree is null) throw new StepDependencyException(nameof(_api.BlockTree)); - if (_api.ReceiptFinder is null) throw new StepDependencyException(nameof(_api.ReceiptFinder)); - if (_api.BloomStorage is null) throw new StepDependencyException(nameof(_api.BloomStorage)); - if (_api.LogManager is null) throw new StepDependencyException(nameof(_api.LogManager)); + StepDependencyException.ThrowIfNull(_api.BlockTree); + StepDependencyException.ThrowIfNull(_api.ReceiptFinder); + StepDependencyException.ThrowIfNull(_api.BloomStorage); + StepDependencyException.ThrowIfNull(_api.LogManager); - IJsonRpcConfig jsonRpcConfig = _api.Config(); - if (!jsonRpcConfig.Enabled) + if (!_jsonRpcConfig.Enabled) { return; } - if (_api.FileSystem is null) throw new StepDependencyException(nameof(_api.FileSystem)); - if (_api.TxPool is null) throw new StepDependencyException(nameof(_api.TxPool)); - if (_api.Wallet is null) throw new StepDependencyException(nameof(_api.Wallet)); - if (_api.SpecProvider is null) throw new StepDependencyException(nameof(_api.SpecProvider)); - if (_api.SyncModeSelector is null) throw new StepDependencyException(nameof(_api.SyncModeSelector)); - if (_api.TxSender is null) throw new StepDependencyException(nameof(_api.TxSender)); - if (_api.StateReader is null) throw new StepDependencyException(nameof(_api.StateReader)); - if (_api.WorldStateManager is null) throw new StepDependencyException(nameof(_api.WorldStateManager)); - if (_api.PeerManager is null) throw new StepDependencyException(nameof(_api.PeerManager)); - - if (jsonRpcConfig.Enabled) - { - _api.RpcModuleProvider = new RpcModuleProvider(_api.FileSystem, jsonRpcConfig, _api.LogManager); - } - else - { - _api.RpcModuleProvider ??= NullModuleProvider.Instance; - } + StepDependencyException.ThrowIfNull(_api.FileSystem); + StepDependencyException.ThrowIfNull(_api.TxPool); + StepDependencyException.ThrowIfNull(_api.Wallet); + StepDependencyException.ThrowIfNull(_api.SpecProvider); + StepDependencyException.ThrowIfNull(_api.SyncModeSelector); + StepDependencyException.ThrowIfNull(_api.TxSender); + StepDependencyException.ThrowIfNull(_api.StateReader); + StepDependencyException.ThrowIfNull(_api.WorldStateManager); + StepDependencyException.ThrowIfNull(_api.PeerManager); + + _api.RpcModuleProvider = new RpcModuleProvider(_api.FileSystem, _jsonRpcConfig, _api.LogManager); IRpcModuleProvider rpcModuleProvider = _api.RpcModuleProvider; @@ -81,55 +72,37 @@ public virtual async Task Execute(CancellationToken cancellationToken) ILogger logger = _api.LogManager.GetClassLogger(); IInitConfig initConfig = _api.Config(); - IJsonRpcConfig rpcConfig = _api.Config(); INetworkConfig networkConfig = _api.Config(); + // lets add threads to support parallel eth_getLogs ThreadPool.GetMinThreads(out int workerThreads, out int completionPortThreads); ThreadPool.SetMinThreads(workerThreads + Environment.ProcessorCount, completionPortThreads + Environment.ProcessorCount); - if (_api.ReceiptStorage is null) throw new StepDependencyException(nameof(_api.ReceiptStorage)); - if (_api.GasPriceOracle is null) throw new StepDependencyException(nameof(_api.GasPriceOracle)); - if (_api.EthSyncingInfo is null) throw new StepDependencyException(nameof(_api.EthSyncingInfo)); + StepDependencyException.ThrowIfNull(_api.ReceiptStorage); + StepDependencyException.ThrowIfNull(_api.GasPriceOracle); + StepDependencyException.ThrowIfNull(_api.EthSyncingInfo); + RpcLimits.Init(_jsonRpcConfig.RequestQueueLimit); + RegisterEthRpcModule(rpcModuleProvider); - var feeHistoryOracle = new FeeHistoryOracle(_api.BlockTree, _api.ReceiptStorage, _api.SpecProvider); - _api.DisposeStack.Push(feeHistoryOracle); - EthModuleFactory ethModuleFactory = new( - _api.TxPool, - _api.TxSender, - _api.Wallet, - _api.BlockTree, - rpcConfig, - _api.LogManager, - _api.StateReader, - _api, - _api.SpecProvider, - _api.ReceiptStorage, - _api.GasPriceOracle, - _api.EthSyncingInfo, - feeHistoryOracle); - - RpcLimits.Init(rpcConfig.RequestQueueLimit); - rpcModuleProvider.RegisterBounded(ethModuleFactory, rpcConfig.EthModuleConcurrentInstances ?? Environment.ProcessorCount, rpcConfig.Timeout); - if (_api.DbProvider is null) throw new StepDependencyException(nameof(_api.DbProvider)); - if (_api.BlockPreprocessor is null) throw new StepDependencyException(nameof(_api.BlockPreprocessor)); - if (_api.BlockValidator is null) throw new StepDependencyException(nameof(_api.BlockValidator)); - if (_api.RewardCalculatorSource is null) throw new StepDependencyException(nameof(_api.RewardCalculatorSource)); - if (_api.KeyStore is null) throw new StepDependencyException(nameof(_api.KeyStore)); - if (_api.PeerPool is null) throw new StepDependencyException(nameof(_api.PeerPool)); - if (_api.WitnessRepository is null) throw new StepDependencyException(nameof(_api.WitnessRepository)); - if (_api.BadBlocksStore is null) throw new StepDependencyException(nameof(_api.BadBlocksStore)); + StepDependencyException.ThrowIfNull(_api.DbProvider); + StepDependencyException.ThrowIfNull(_api.BlockPreprocessor); + StepDependencyException.ThrowIfNull(_api.BlockValidator); + StepDependencyException.ThrowIfNull(_api.RewardCalculatorSource); + StepDependencyException.ThrowIfNull(_api.KeyStore); + StepDependencyException.ThrowIfNull(_api.PeerPool); + StepDependencyException.ThrowIfNull(_api.BadBlocksStore); ProofModuleFactory proofModuleFactory = new(_api.WorldStateManager, _api.BlockTree, _api.BlockPreprocessor, _api.ReceiptFinder, _api.SpecProvider, _api.LogManager); - rpcModuleProvider.RegisterBounded(proofModuleFactory, 2, rpcConfig.Timeout); + rpcModuleProvider.RegisterBounded(proofModuleFactory, 2, _jsonRpcConfig.Timeout); DebugModuleFactory debugModuleFactory = new( _api.WorldStateManager, _api.DbProvider, _api.BlockTree, - rpcConfig, + _jsonRpcConfig, _api.BlockValidator, _api.BlockPreprocessor, _api.RewardCalculatorSource, @@ -141,23 +114,11 @@ public virtual async Task Execute(CancellationToken cancellationToken) _api.BadBlocksStore, _api.FileSystem, _api.LogManager); - rpcModuleProvider.RegisterBoundedByCpuCount(debugModuleFactory, rpcConfig.Timeout); + rpcModuleProvider.RegisterBoundedByCpuCount(debugModuleFactory, _jsonRpcConfig.Timeout); - TraceModuleFactory traceModuleFactory = new( - _api.WorldStateManager, - _api.BlockTree, - rpcConfig, - _api.BlockPreprocessor, - _api.RewardCalculatorSource, - _api.ReceiptStorage, - _api.SpecProvider, - _api.PoSSwitcher, - _api.LogManager); - - rpcModuleProvider.RegisterBoundedByCpuCount(traceModuleFactory, rpcConfig.Timeout); + RegisterTraceRpcModule(rpcModuleProvider); - if (_api.EthereumEcdsa is null) throw new StepDependencyException(nameof(_api.EthereumEcdsa)); - if (_api.Wallet is null) throw new StepDependencyException(nameof(_api.Wallet)); + StepDependencyException.ThrowIfNull(_api.EthereumEcdsa); PersonalRpcModule personalRpcModule = new( _api.EthereumEcdsa, @@ -165,9 +126,9 @@ public virtual async Task Execute(CancellationToken cancellationToken) _api.KeyStore); rpcModuleProvider.RegisterSingle(personalRpcModule); - if (_api.PeerManager is null) throw new StepDependencyException(nameof(_api.PeerManager)); - if (_api.StaticNodesManager is null) throw new StepDependencyException(nameof(_api.StaticNodesManager)); - if (_api.Enode is null) throw new StepDependencyException(nameof(_api.Enode)); + StepDependencyException.ThrowIfNull(_api.PeerManager); + StepDependencyException.ThrowIfNull(_api.StaticNodesManager); + StepDependencyException.ThrowIfNull(_api.Enode); ManualPruningTrigger pruningTrigger = new(); _api.PruningTrigger.Add(pruningTrigger); @@ -181,13 +142,13 @@ public virtual async Task Execute(CancellationToken cancellationToken) pruningTrigger); rpcModuleProvider.RegisterSingle(adminRpcModule); - if (_api.TxPoolInfoProvider is null) throw new StepDependencyException(nameof(_api.TxPoolInfoProvider)); + StepDependencyException.ThrowIfNull(_api.TxPoolInfoProvider); TxPoolRpcModule txPoolRpcModule = new(_api.TxPoolInfoProvider, _api.LogManager); rpcModuleProvider.RegisterSingle(txPoolRpcModule); - if (_api.SyncServer is null) throw new StepDependencyException(nameof(_api.SyncServer)); - if (_api.EngineSignerStore is null) throw new StepDependencyException(nameof(_api.EngineSignerStore)); + StepDependencyException.ThrowIfNull(_api.SyncServer); + StepDependencyException.ThrowIfNull(_api.EngineSignerStore); NetRpcModule netRpcModule = new(_api.LogManager, new NetBridge(_api.Enode, _api.SyncServer)); rpcModuleProvider.RegisterSingle(netRpcModule); @@ -204,14 +165,11 @@ public virtual async Task Execute(CancellationToken cancellationToken) _api.PeerManager); rpcModuleProvider.RegisterSingle(parityRpcModule); - WitnessRpcModule witnessRpcModule = new(_api.WitnessRepository, _api.BlockTree); - rpcModuleProvider.RegisterSingle(witnessRpcModule); - - if (_api.ReceiptMonitor is null) throw new StepDependencyException(nameof(_api.ReceiptMonitor)); + StepDependencyException.ThrowIfNull(_api.ReceiptMonitor); JsonRpcLocalStats jsonRpcLocalStats = new( _api.Timestamper, - jsonRpcConfig, + _jsonRpcConfig, _api.LogManager); _api.JsonRpcLocalStats = jsonRpcLocalStats; @@ -244,4 +202,75 @@ public virtual async Task Execute(CancellationToken cancellationToken) await Task.CompletedTask; } + + protected ModuleFactoryBase CreateEthModuleFactory() + { + StepDependencyException.ThrowIfNull(_api.BlockTree); + StepDependencyException.ThrowIfNull(_api.ReceiptStorage); + StepDependencyException.ThrowIfNull(_api.SpecProvider); + StepDependencyException.ThrowIfNull(_api.TxPool); + StepDependencyException.ThrowIfNull(_api.TxSender); + StepDependencyException.ThrowIfNull(_api.Wallet); + StepDependencyException.ThrowIfNull(_api.StateReader); + StepDependencyException.ThrowIfNull(_api.GasPriceOracle); + StepDependencyException.ThrowIfNull(_api.EthSyncingInfo); + StepDependencyException.ThrowIfNull(_api.EthSyncingInfo); + + var feeHistoryOracle = new FeeHistoryOracle(_api.BlockTree, _api.ReceiptStorage, _api.SpecProvider); + _api.DisposeStack.Push(feeHistoryOracle); + + IBlocksConfig blockConfig = _api.Config(); + ulong secondsPerSlot = blockConfig.SecondsPerSlot; + + return new EthModuleFactory( + _api.TxPool, + _api.TxSender, + _api.Wallet, + _api.BlockTree, + _jsonRpcConfig, + _api.LogManager, + _api.StateReader, + _api, + _api.SpecProvider, + _api.ReceiptStorage, + _api.GasPriceOracle, + _api.EthSyncingInfo, + feeHistoryOracle, + secondsPerSlot); + } + + protected virtual void RegisterEthRpcModule(IRpcModuleProvider rpcModuleProvider) + { + ModuleFactoryBase ethModuleFactory = CreateEthModuleFactory(); + + rpcModuleProvider.RegisterBounded(ethModuleFactory, + _jsonRpcConfig.EthModuleConcurrentInstances ?? Environment.ProcessorCount, _jsonRpcConfig.Timeout); + } + + protected ModuleFactoryBase CreateTraceModuleFactory() + { + StepDependencyException.ThrowIfNull(_api.WorldStateManager); + StepDependencyException.ThrowIfNull(_api.BlockTree); + StepDependencyException.ThrowIfNull(_api.RewardCalculatorSource); + StepDependencyException.ThrowIfNull(_api.ReceiptStorage); + StepDependencyException.ThrowIfNull(_api.SpecProvider); + + return new TraceModuleFactory( + _api.WorldStateManager, + _api.BlockTree, + _jsonRpcConfig, + _api.BlockPreprocessor, + _api.RewardCalculatorSource, + _api.ReceiptStorage, + _api.SpecProvider, + _api.PoSSwitcher, + _api.LogManager); + } + + protected virtual void RegisterTraceRpcModule(IRpcModuleProvider rpcModuleProvider) + { + ModuleFactoryBase traceModuleFactory = CreateTraceModuleFactory(); + + rpcModuleProvider.RegisterBoundedByCpuCount(traceModuleFactory, _jsonRpcConfig.Timeout); + } } diff --git a/src/Nethermind/Nethermind.Init/Steps/SetupKeyStore.cs b/src/Nethermind/Nethermind.Init/Steps/SetupKeyStore.cs index 8250541cbe4..cd181ef667b 100644 --- a/src/Nethermind/Nethermind.Init/Steps/SetupKeyStore.cs +++ b/src/Nethermind/Nethermind.Init/Steps/SetupKeyStore.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Nethermind.Api; using Nethermind.Config; +using Nethermind.Consensus; using Nethermind.Crypto; using Nethermind.KeyStore; using Nethermind.KeyStore.Config; @@ -60,7 +61,11 @@ await Task.Run(() => INodeKeyManager nodeKeyManager = new NodeKeyManager(get.CryptoRandom, get.KeyStore, keyStoreConfig, get.LogManager, passwordProvider, get.FileSystem); ProtectedPrivateKey? nodeKey = set.NodeKey = nodeKeyManager.LoadNodeKey(); - set.OriginalSignerKey = nodeKeyManager.LoadSignerKey(); + IMiningConfig miningConfig = get.Config(); + //Don't load the local key if an external signer is configured + if (!miningConfig.Enabled && string.IsNullOrEmpty(miningConfig.Signer)) + set.OriginalSignerKey = nodeKeyManager.LoadSignerKey(); + IPAddress ipAddress = networkConfig.ExternalIp is not null ? IPAddress.Parse(networkConfig.ExternalIp) : IPAddress.Loopback; IEnode enode = set.Enode = new Enode(nodeKey.PublicKey, ipAddress, networkConfig.P2PPort); diff --git a/src/Nethermind/Nethermind.Init/Steps/StartBlockProducer.cs b/src/Nethermind/Nethermind.Init/Steps/StartBlockProducer.cs index e0041cd6e53..bd744143ee5 100644 --- a/src/Nethermind/Nethermind.Init/Steps/StartBlockProducer.cs +++ b/src/Nethermind/Nethermind.Init/Steps/StartBlockProducer.cs @@ -24,18 +24,21 @@ public StartBlockProducer(INethermindApi api) _api = api; } - public async Task Execute(CancellationToken _) + public Task Execute(CancellationToken _) { if (_api.BlockProductionPolicy!.ShouldStartBlockProduction() && _api.BlockProducer is not null) { if (_api.BlockTree is null) throw new StepDependencyException(nameof(_api.BlockTree)); + if (_api.BlockProducerRunner is null) throw new StepDependencyException(nameof(_api.BlockProducerRunner)); ILogger logger = _api.LogManager.GetClassLogger(); if (logger.IsInfo) logger.Info($"Starting {_api.SealEngineType} block producer & sealer"); - ProducedBlockSuggester suggester = new(_api.BlockTree, _api.BlockProducer); + ProducedBlockSuggester suggester = new(_api.BlockTree, _api.BlockProducerRunner); _api.DisposeStack.Push(suggester); - await _api.BlockProducer.Start(); + _api.BlockProducerRunner.Start(); } + + return Task.CompletedTask; } } } diff --git a/src/Nethermind/Nethermind.Init/Steps/StepInitializationException.cs b/src/Nethermind/Nethermind.Init/Steps/StepInitializationException.cs index f6fda099528..8fe0e62a4f3 100644 --- a/src/Nethermind/Nethermind.Init/Steps/StepInitializationException.cs +++ b/src/Nethermind/Nethermind.Init/Steps/StepInitializationException.cs @@ -2,6 +2,8 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; namespace Nethermind.Init.Steps { @@ -15,5 +17,12 @@ public StepDependencyException(string message) : base(message) { } + + public static void ThrowIfNull([NotNull] object? argument, [CallerArgumentExpression(nameof(argument))] string? paramName = null) + { + if (argument is not null) + return; + throw new StepDependencyException(paramName ?? ""); + } } } diff --git a/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs b/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs index 8abef2c22b8..998c379a216 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Benchmark/EthModuleBenchmarks.cs @@ -36,6 +36,7 @@ using BlockTree = Nethermind.Blockchain.BlockTree; using Nethermind.Blockchain.Synchronization; using Nethermind.Config; +using Nethermind.Facade.Simulate; using Nethermind.Synchronization.ParallelSync; namespace Nethermind.JsonRpc.Benchmark @@ -80,7 +81,8 @@ public void GlobalSetup() new SyncConfig(), LimboLogs.Instance); _blockhashProvider = new BlockhashProvider(blockTree, specProvider, stateProvider, LimboLogs.Instance); - _virtualMachine = new VirtualMachine(_blockhashProvider, specProvider, LimboLogs.Instance); + CodeInfoRepository codeInfoRepository = new(); + _virtualMachine = new VirtualMachine(_blockhashProvider, specProvider, codeInfoRepository, LimboLogs.Instance); Block genesisBlock = Build.A.Block.Genesis.TestObject; blockTree.SuggestBlock(genesisBlock); @@ -89,11 +91,11 @@ public void GlobalSetup() blockTree.SuggestBlock(block1); TransactionProcessor transactionProcessor - = new(MainnetSpecProvider.Instance, stateProvider, _virtualMachine, LimboLogs.Instance); + = new(MainnetSpecProvider.Instance, stateProvider, _virtualMachine, codeInfoRepository, LimboLogs.Instance); IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor = new BlockProcessor.BlockValidationTransactionsExecutor(transactionProcessor, stateProvider); BlockProcessor blockProcessor = new(specProvider, Always.Valid, new RewardCalculator(specProvider), transactionsExecutor, - stateProvider, NullReceiptStorage.Instance, NullWitnessCollector.Instance, new BlockhashStore(blockTree, specProvider, stateProvider), transactionProcessor, LimboLogs.Instance); + stateProvider, NullReceiptStorage.Instance, new BlockhashStore(blockTree, specProvider, stateProvider), transactionProcessor, LimboLogs.Instance); EthereumEcdsa ecdsa = new(specProvider.ChainId, LimboLogs.Instance); BlockchainProcessor blockchainProcessor = new( @@ -127,6 +129,12 @@ TransactionProcessor transactionProcessor new ReadOnlyBlockTree(blockTree), specProvider, LimboLogs.Instance), + new SimulateReadOnlyBlocksProcessingEnvFactory( + stateManager, + new ReadOnlyBlockTree(blockTree), + new ReadOnlyDbProvider(dbProvider, true), + specProvider, + LimboLogs.Instance), NullTxPool.Instance, NullReceiptStorage.Instance, NullFilterStore.Instance, @@ -158,7 +166,8 @@ TransactionProcessor transactionProcessor specProvider, gasPriceOracle, ethSyncingInfo, - feeHistoryOracle); + feeHistoryOracle, + new BlocksConfig().SecondsPerSlot); } [Benchmark] diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/ConsensusHelperTests.ReceiptsJsonRpcDataSource.cs b/src/Nethermind/Nethermind.JsonRpc.Test/ConsensusHelperTests.ReceiptsJsonRpcDataSource.cs index 081ebeddd34..fe635d2b871 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/ConsensusHelperTests.ReceiptsJsonRpcDataSource.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/ConsensusHelperTests.ReceiptsJsonRpcDataSource.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Threading.Tasks; using Nethermind.Core.Crypto; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.JsonRpc.Modules.Eth; using Nethermind.Serialization.Json; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Data/AccessListItemForRpcTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Data/AccessListItemForRpcTests.cs index 31345ac4e63..ae948e1d693 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Data/AccessListItemForRpcTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Data/AccessListItemForRpcTests.cs @@ -6,6 +6,7 @@ using Nethermind.Core; using Nethermind.Core.Eip2930; using Nethermind.Core.Test.Builders; +using Nethermind.Facade.Eth; using Nethermind.Int256; using Nethermind.JsonRpc.Data; using NUnit.Framework; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs index a9176491bdf..f34daa6963f 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Data/Eip2930Tests.cs @@ -8,6 +8,7 @@ using Nethermind.Core.Eip2930; using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; +using Nethermind.Facade.Eth; using Nethermind.Int256; using Nethermind.JsonRpc.Data; using Nethermind.Serialization.Json; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcProcessorTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcProcessorTests.cs index 56eeed59f67..77ea1ccbba1 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcProcessorTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcProcessorTests.cs @@ -208,7 +208,7 @@ public async Task Can_process_batch_request_with_some_params_missing() [Test] public async Task Can_process_batch_request_with_two_requests() { - IList result = await ProcessAsync("{\"id\":67,\"jsonrpc\":\"2.0\",\"method\":\"eth_getTransactionCount\",\"params\":[\"0x7f01d9b227593e033bf8d6fc86e634d27aa85568\",\"0x668c24\"]}{\"id\":68,\"jsonrpc\":\"2.0\",\"method\":\"eth_getTransactionCount\",\"params\":[\"0x7f01d9b227593e033bf8d6fc86e634d27aa85568\",\"0x668c24\"]}"); + IList result = await ProcessAsync("{\"id\":67,\"jsonrpc\":\"2.0\",\"method\":\"eth_getTransactionCount\",\"params\":[\"0x7f01d9b227593e033bf8d6fc86e634d27aa85568\",\"0x668c24\"]}\r\n{\"id\":68,\"jsonrpc\":\"2.0\",\"method\":\"eth_getTransactionCount\",\"params\":[\"0x7f01d9b227593e033bf8d6fc86e634d27aa85568\",\"0x668c24\"]}"); result.Should().HaveCount(2); result[0].Response.Should().NotBeNull(); result[0].BatchedResponses.Should().BeNull(); diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcServiceTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcServiceTests.cs index 6748030a5bf..00b2e77d735 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcServiceTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/JsonRpcServiceTests.cs @@ -16,6 +16,8 @@ using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; +using Nethermind.Facade.Eth; +using Nethermind.Facade.Proxy.Models.Simulate; using Nethermind.Int256; using Nethermind.JsonRpc.Data; using Nethermind.JsonRpc.Modules; @@ -38,12 +40,17 @@ public class JsonRpcServiceTests [SetUp] public void Initialize() { - Assembly jConfig = typeof(JsonRpcConfig).Assembly; _configurationProvider = new ConfigProvider(); _logManager = LimboLogs.Instance; _context = new JsonRpcContext(RpcEndpoint.Http); } + [TearDown] + public void TearDown() + { + _context?.Dispose(); + } + private IJsonRpcService _jsonRpcService = null!; private IConfigProvider _configurationProvider = null!; private ILogManager _logManager = null!; @@ -87,13 +94,27 @@ public void Eth_module_populates_size_when_returning_block_data() Assert.That((response?.Result as BlockForRpc)?.Size, Is.EqualTo(513L)); } + + [Test] + public void CanRunEthSimulateV1Empty() + { + SimulatePayload payload = new() { BlockStateCalls = new List>() }; + string serializedCall = new EthereumJsonSerializer().Serialize(payload); + IEthRpcModule ethRpcModule = Substitute.For(); + ethRpcModule.eth_simulateV1(payload).ReturnsForAnyArgs(_ => + ResultWrapper>.Success(Array.Empty())); + JsonRpcSuccessResponse? response = TestRequest(ethRpcModule, "eth_simulateV1", serializedCall) as JsonRpcSuccessResponse; + Assert.That(response?.Result, Is.EqualTo(Array.Empty())); + } + + [Test] public void CanHandleOptionalArguments() { EthereumJsonSerializer serializer = new(); string serialized = serializer.Serialize(new TransactionForRpc()); IEthRpcModule ethRpcModule = Substitute.For(); - ethRpcModule.eth_call(Arg.Any()).ReturnsForAnyArgs(x => ResultWrapper.Success("0x1")); + ethRpcModule.eth_call(Arg.Any()).ReturnsForAnyArgs(_ => ResultWrapper.Success("0x1")); JsonRpcSuccessResponse? response = TestRequest(ethRpcModule, "eth_call", serialized) as JsonRpcSuccessResponse; Assert.That(response?.Result, Is.EqualTo("0x1")); } diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/BoundedModulePoolTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/BoundedModulePoolTests.cs index 341c574c4dd..5a09f99127e 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/BoundedModulePoolTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/BoundedModulePoolTests.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; +using Nethermind.Config; using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; using Nethermind.Db; @@ -56,7 +57,8 @@ public Task Initialize() Substitute.For(), Substitute.For(), Substitute.For(), - Substitute.For()), + Substitute.For(), + new BlocksConfig().SecondsPerSlot), 1, 1000); return Task.CompletedTask; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugModuleTests.cs index 064fff4e886..e66d93d334f 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugModuleTests.cs @@ -19,6 +19,7 @@ using Nethermind.Db; using Nethermind.Evm.Tracing.GethStyle; using Nethermind.Evm.Tracing.GethStyle.Custom; +using Nethermind.Facade.Eth; using Nethermind.Int256; using Nethermind.JsonRpc.Data; using Nethermind.JsonRpc.Modules.DebugModule; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EstimateGas.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EstimateGas.cs index e40b064b796..0bc1a37fb09 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EstimateGas.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EstimateGas.cs @@ -10,6 +10,7 @@ using Nethermind.Core.Test.Builders; using Nethermind.Evm; using Nethermind.Evm.Tracing; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.Specs; using Nethermind.Specs.Forks; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EthCall.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EthCall.cs index cd7fb615ec8..ed73767fcf1 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EthCall.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.EthCall.cs @@ -8,6 +8,7 @@ using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; using Nethermind.Evm; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.Specs; using Nethermind.Specs.Forks; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs index 107e473b58b..b613c75e75c 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs @@ -22,6 +22,7 @@ using Nethermind.Evm; using Nethermind.Evm.Tracing; using Nethermind.Facade; +using Nethermind.Facade.Eth; using Nethermind.Facade.Filters; using Nethermind.Int256; using Nethermind.JsonRpc.Data; @@ -64,6 +65,7 @@ public async Task Eth_get_balance_default_block() } [Test] + [SetCulture("en-US")] public async Task Eth_get_eth_feeHistory() { using Context ctx = await Context.Create(); diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcSimulateTestsBase.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcSimulateTestsBase.cs new file mode 100644 index 00000000000..b3d887961c1 --- /dev/null +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcSimulateTestsBase.cs @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Nethermind.Abi; +using Nethermind.Blockchain.Contracts.Json; +using Nethermind.Blockchain.Find; +using Nethermind.Consensus; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; +using Nethermind.Crypto; +using Nethermind.Facade.Eth; +using Nethermind.Int256; +using Nethermind.JsonRpc.Data; +using Nethermind.Logging; +using Nethermind.Specs; +using Nethermind.Specs.Forks; +using Nethermind.TxPool; + +namespace Nethermind.JsonRpc.Test.Modules.Eth; + +public class EthRpcSimulateTestsBase +{ + public static Task CreateChain(IReleaseSpec? releaseSpec = null) + { + TestRpcBlockchain testMevRpcBlockchain = new(); + TestSpecProvider testSpecProvider = releaseSpec is not null + ? new TestSpecProvider(releaseSpec) + : new TestSpecProvider(London.Instance); + return TestRpcBlockchain.ForTest(testMevRpcBlockchain).Build(testSpecProvider); + } + + private static string GetEcRecoverContractJsonAbi(string name = "recover") + { + return $@" +[ + {{ + ""payable"": false, + ""inputs"": [ + {{ + ""internalType"": ""bytes32"", + ""name"": ""hash"", + ""type"": ""bytes32"" + }}, + {{ + ""internalType"": ""uint8"", + ""name"": ""v"", + ""type"": ""uint8"" + + }}, + {{ + ""internalType"": ""bytes32"", + ""name"": ""r"", + ""type"": ""bytes32"" + }}, + {{ + ""internalType"": ""bytes32"", + ""name"": ""s"", + ""type"": ""bytes32"" + }} + ], + ""name"": ""{name}"", + ""outputs"": [ + {{ + ""internalType"": ""address"", + ""name"": """", + ""type"": ""address"" + }} + ], + ""stateMutability"": ""pure"", + ""type"": ""function"" + }} +]"; + } + + public static byte[] GetTxData(TestRpcBlockchain chain, PrivateKey account, string name = "recover") + { + // Step 1: Hash the message + Hash256 messageHash = Keccak.Compute("Hello, world!"); + // Step 2: Sign the hash + Signature signature = chain.EthereumEcdsa.Sign(account, messageHash); + + //Check real address + return GenerateTransactionDataForEcRecover(messageHash, signature, name); + } + + public static async Task
DeployEcRecoverContract(TestRpcBlockchain chain, PrivateKey privateKey, string contractBytecode) + { + byte[] bytecode = Bytes.FromHexString(contractBytecode); + Transaction tx = new() + { + Value = UInt256.Zero, + Nonce = 0, + Data = bytecode, + GasLimit = 3_000_000, + SenderAddress = privateKey.Address, + To = null, + GasPrice = 20.GWei() + }; + + TxPoolSender txSender = new(chain.TxPool, + new TxSealer(new Signer(chain.SpecProvider.ChainId, privateKey, LimboLogs.Instance), chain.Timestamper), + chain.NonceManager, + chain.EthereumEcdsa); + + (Hash256 hash, AcceptTxResult? code) = await txSender.SendTransaction(tx, TxHandlingOptions.ManagedNonce | TxHandlingOptions.PersistentBroadcast); + + code?.Should().Be(AcceptTxResult.Accepted); + Transaction[] txs = chain.TxPool.GetPendingTransactions(); + await chain.AddBlock(true, txs); + + TxReceipt? createContractTxReceipt = null; + while (createContractTxReceipt is null) + { + await Task.Delay(100); + createContractTxReceipt = chain.Bridge.GetReceipt(hash); + } + + createContractTxReceipt.ContractAddress.Should().NotBeNull($"Contract transaction {tx.Hash!} was not deployed."); + return createContractTxReceipt.ContractAddress!; + } + + protected static byte[] GenerateTransactionDataForEcRecover(Hash256 keccak, Signature signature, string name = "recover") + { + AbiDefinition call = new AbiDefinitionParser().Parse(GetEcRecoverContractJsonAbi(name)); + AbiEncodingInfo functionInfo = call.GetFunction(name).GetCallInfo(); + return AbiEncoder.Instance.Encode(functionInfo.EncodingStyle, functionInfo.Signature, keccak, signature.V, signature.R, signature.S); + } + + private static Address? ParseEcRecoverAddress(byte[] data, string name = "recover") + { + AbiDefinition call = new AbiDefinitionParser().Parse(GetEcRecoverContractJsonAbi(name)); + AbiEncodingInfo functionInfo = call.GetFunction(name).GetReturnInfo(); + return AbiEncoder.Instance.Decode(functionInfo.EncodingStyle, functionInfo.Signature, data).FirstOrDefault() as Address; + } + + public static Address? EcRecoverCall(TestRpcBlockchain testRpcBlockchain, Address senderAddress, byte[] bytes, Address? toAddress = null) + { + SystemTransaction transaction = new() { Data = bytes, To = toAddress, SenderAddress = senderAddress }; + transaction.Hash = transaction.CalculateHash(); + TransactionForRpc transactionForRpc = new(transaction); + ResultWrapper mainChainResult = testRpcBlockchain.EthRpcModule.eth_call(transactionForRpc, BlockParameter.Pending); + return ParseEcRecoverAddress(Bytes.FromHexString(mainChainResult.Data)); + } +} diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthSimulateTestsPrecompilesWithRedirection.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthSimulateTestsPrecompilesWithRedirection.cs new file mode 100644 index 00000000000..33d80a28d30 --- /dev/null +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthSimulateTestsPrecompilesWithRedirection.cs @@ -0,0 +1,194 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Nethermind.Blockchain.Find; +using Nethermind.Config; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Test.Builders; +using Nethermind.Evm; +using Nethermind.Evm.Precompiles; +using Nethermind.Facade.Eth; +using Nethermind.Facade.Proxy.Models; +using Nethermind.Facade.Proxy.Models.Simulate; +using Nethermind.JsonRpc.Data; +using Nethermind.JsonRpc.Modules.Eth; +using NUnit.Framework; + +namespace Nethermind.JsonRpc.Test.Modules.Eth; + +public class EthSimulateTestsPrecompilesWithRedirection +{ + [Test] + public async Task Test_eth_simulate_create() + { + TestRpcBlockchain chain = await EthRpcSimulateTestsBase.CreateChain(); + + Transaction systemTransactionForModifiedVm = new() + { + SenderAddress = TestItem.AddressB, + Data = Bytes.FromHexString("0xee82ac5e0000000000000000000000000000000000000000000000000000000000000001"), + To = TestItem.AddressA, + GasLimit = 3_500_000, + GasPrice = 20.GWei() + }; + + TransactionForRpc transactionForRpc = new(systemTransactionForModifiedVm) { Nonce = null }; + + SimulatePayload payload = new() + { + BlockStateCalls = + [ + new() + { + StateOverrides = new Dictionary + { + { + TestItem.AddressA, + new AccountOverride + { + Code = Bytes.FromHexString("0x6080604052348015600f57600080fd5b506004361060285760003560e01c8063ee82ac5e14602d575b600080fd5b60436004803603810190603f91906098565b6057565b604051604e919060d7565b60405180910390f35b600081409050919050565b600080fd5b6000819050919050565b6078816067565b8114608257600080fd5b50565b6000813590506092816071565b92915050565b60006020828403121560ab5760aa6062565b5b600060b7848285016085565b91505092915050565b6000819050919050565b60d18160c0565b82525050565b600060208201905060ea600083018460ca565b9291505056fea2646970667358221220a4d7face162688805e99e86526524ac3dadfb01cc29366d0d68b70dadcf01afe64736f6c63430008120033") + } + }, + }, + Calls = [transactionForRpc] + } + ] + }; + + //will mock our GetCachedCodeInfo function - it shall be called 3 times if redirect is working, 2 times if not + SimulateTxExecutor executor = new(chain.Bridge, chain.BlockFinder, new JsonRpcConfig(), new BlocksConfig().SecondsPerSlot); + + ResultWrapper> result = executor.Execute(payload, BlockParameter.Latest); + + //Check results + byte[]? returnData = result.Data[0].Calls.First().ReturnData; + Assert.IsNotNull(returnData); + } + + + /// + /// This test verifies that a temporary forked blockchain can redirect precompiles + /// + [Test] + public async Task Test_eth_simulate_ecr_moved() + { + TestRpcBlockchain chain = await EthRpcSimulateTestsBase.CreateChain(); + //The following opcodes code is based on the following contract compiled: + /* + function ecrecover(bytes32 hash, uint8 v, bytes32 r, bytes32 s) public returns(address) + { + + address redirectedToAddress = 0x0000000000000000000000000000000000000666; + + assembly { + // Copy msg.data. We take full control of memory in this inline assembly + // block because it will not return to Solidity code. We overwrite the + // Solidity scratch pad at memory position 0. + calldatacopy(0, 0, calldatasize()) + + // Call the implementation. + // out and outsize are 0 because we don't know the size yet. + let result := delegatecall(gas(), redirectedToAddress, 0, calldatasize(), 0, 0) + + // Copy the returned data. + returndatacopy(0, 0, returndatasize()) + + return (0, returndatasize()) + } + } + */ + + byte[] code = Prepare.EvmCode + .JUMPDEST() + .PushData(Bytes.ZeroByte) + .Op(Instruction.DUP1) + .PushData(TestItem.AddressB.Bytes) + .Op(Instruction.SWAP1) + .Op(Instruction.POP) + .Op(Instruction.CALLDATASIZE) + .PushData(Bytes.ZeroByte) + .Op(Instruction.DUP1) + .Op(Instruction.CALLDATACOPY) + .PushData(Bytes.ZeroByte) + .Op(Instruction.DUP1) + .Op(Instruction.CALLDATASIZE) + .PushData(Bytes.ZeroByte) + .Op(Instruction.DUP5) + .Op(Instruction.GAS) + .Op(Instruction.DELEGATECALL) + .Op(Instruction.RETURNDATASIZE) + .PushData(Bytes.ZeroByte) + .Op(Instruction.DUP1) + .Op(Instruction.RETURNDATACOPY) + .Op(Instruction.RETURNDATASIZE) + .PushData(Bytes.ZeroByte) + .Op(Instruction.RETURN) + .Done; + + byte[] transactionData = EthRpcSimulateTestsBase.GetTxData(chain, TestItem.PrivateKeyA); + + Hash256 headHash = chain.BlockFinder.Head!.Hash!; + Address contractAddress = await EthRpcSimulateTestsBase.DeployEcRecoverContract(chain, TestItem.PrivateKeyB, EthSimulateTestsSimplePrecompiles.EcRecoverCallerContractBytecode); + + EthRpcSimulateTestsBase.EcRecoverCall(chain, TestItem.AddressB, transactionData, contractAddress); + + chain.BlockTree.UpdateMainChain(new List { chain.BlockFinder.Head! }, true, true); + chain.BlockTree.UpdateHeadBlock(chain.BlockFinder.Head!.Hash!); + + Assert.That(headHash != chain.BlockFinder.Head!.Hash!); + chain.State.StateRoot = chain.BlockFinder.Head!.StateRoot!; + + TransactionForRpc transactionForRpc = new(new Transaction + { + Data = transactionData, + To = contractAddress, + SenderAddress = TestItem.AddressA, + GasLimit = 3_500_000, + GasPrice = 20.GWei() + }) + { + Nonce = null + }; + + SimulatePayload payload = new() + { + BlockStateCalls = + [ + new() + { + StateOverrides = new Dictionary + { + { + EcRecoverPrecompile.Address, + new AccountOverride + { + Code = code, + MovePrecompileToAddress = TestItem.AddressB + } + } + }, + Calls = [transactionForRpc] + } + ] + }; + + //will mock our GetCachedCodeInfo function - it shall be called 3 times if redirect is working, 2 times if not + SimulateTxExecutor executor = new(chain.Bridge, chain.BlockFinder, new JsonRpcConfig(), new BlocksConfig().SecondsPerSlot); + + Debug.Assert(contractAddress is not null, nameof(contractAddress) + " is not null"); + Assert.IsTrue(chain.State.AccountExists(contractAddress)); + + ResultWrapper> result = executor.Execute(payload, BlockParameter.Latest); + + //Check results + byte[] addressBytes = result.Data[0].Calls[0].ReturnData!.SliceWithZeroPaddingEmptyOnError(12, 20); + Address resultingAddress = new(addressBytes); + Assert.That(resultingAddress, Is.EqualTo(TestItem.AddressA)); + } +} diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsBlocksAndTransactions.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsBlocksAndTransactions.cs new file mode 100644 index 00000000000..28a28e68803 --- /dev/null +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsBlocksAndTransactions.cs @@ -0,0 +1,281 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Nethermind.Blockchain.Find; +using Nethermind.Config; +using Nethermind.Core; +using Nethermind.Core.Extensions; +using Nethermind.Core.Test.Builders; +using Nethermind.Crypto; +using Nethermind.Facade.Eth; +using Nethermind.Facade.Proxy.Models; +using Nethermind.Facade.Proxy.Models.Simulate; +using Nethermind.Int256; +using Nethermind.JsonRpc.Data; +using Nethermind.JsonRpc.Modules.Eth; +using Nethermind.Serialization.Json; +using NUnit.Framework; +using ResultType = Nethermind.Facade.Proxy.Models.Simulate.ResultType; + +namespace Nethermind.JsonRpc.Test.Modules.Eth; + +public class EthSimulateTestsBlocksAndTransactions +{ + private static Transaction GetTransferTxData(UInt256 nonce, IEthereumEcdsa ethereumEcdsa, PrivateKey from, Address to, UInt256 amount) + { + Transaction tx = new() + { + Value = amount, + Nonce = nonce, + GasLimit = 50_000, + SenderAddress = from.Address, + To = to, + GasPrice = 20.GWei() + }; + + ethereumEcdsa.Sign(from, tx); + tx.Hash = tx.CalculateHash(); + return tx; + } + + [Test] + public async Task Test_eth_simulate_serialisation() + { + TestRpcBlockchain chain = await EthRpcSimulateTestsBase.CreateChain(); + + UInt256 nonceA = chain.State.GetNonce(TestItem.AddressA); + Transaction txToFail = GetTransferTxData(nonceA, chain.EthereumEcdsa, TestItem.PrivateKeyA, TestItem.AddressB, 10_000_000); + UInt256 nextNonceA = ++nonceA; + Transaction tx = GetTransferTxData(nextNonceA, chain.EthereumEcdsa, TestItem.PrivateKeyA, TestItem.AddressB, 4_000_000); + + SimulatePayload payload = new() + { + BlockStateCalls = + [ + new() + { + BlockOverrides = new BlockOverride { Number = 10 }, + Calls = [new TransactionForRpc(txToFail), new TransactionForRpc(tx)], + StateOverrides = new Dictionary + { + { TestItem.AddressA, new AccountOverride { Balance = Math.Max(420_000_004_000_001UL, 1_000_000_004_000_001UL) } } + } + } + ], + TraceTransfers = true, + Validation = true + }; + + //Force persistence of head block in main chain + chain.BlockTree.UpdateMainChain(new List { chain.BlockFinder.Head! }, true, true); + chain.BlockTree.UpdateHeadBlock(chain.BlockFinder.Head!.Hash!); + + //will mock our GetCachedCodeInfo function - it shall be called 3 times if redirect is working, 2 times if not + SimulateTxExecutor executor = new(chain.Bridge, chain.BlockFinder, new JsonRpcConfig(), new BlocksConfig().SecondsPerSlot); + ResultWrapper> result = executor.Execute(payload, BlockParameter.Latest); + IReadOnlyList data = result.Data; + Assert.That(data.Count, Is.EqualTo(7)); + + SimulateBlockResult blockResult = data.Last(); + blockResult.Calls.Select(c => c.Status).Should().BeEquivalentTo(new[] { (ulong)ResultType.Success, (ulong)ResultType.Success }); + + } + + + /// + /// This test verifies that a temporary forked blockchain can make transactions, blocks and report on them + /// We test on blocks before current head and after it, + /// Note that if we get blocks before head we set simulation start state to one of that first block + /// + [Test] + public async Task Test_eth_simulate_eth_moved() + { + TestRpcBlockchain chain = await EthRpcSimulateTestsBase.CreateChain(); + + UInt256 nonceA = chain.State.GetNonce(TestItem.AddressA); + Transaction txMainnetAtoB = GetTransferTxData(nonceA, chain.EthereumEcdsa, TestItem.PrivateKeyA, TestItem.AddressB, 1); + Transaction txAtoB1 = GetTransferTxData(nonceA + 1, chain.EthereumEcdsa, TestItem.PrivateKeyA, TestItem.AddressB, 1); + Transaction txAtoB2 = GetTransferTxData(nonceA + 2, chain.EthereumEcdsa, TestItem.PrivateKeyA, TestItem.AddressB, 1); + Transaction txAtoB3 = GetTransferTxData(nonceA + 3, chain.EthereumEcdsa, TestItem.PrivateKeyA, TestItem.AddressB, 1); + Transaction txAtoB4 = GetTransferTxData(nonceA + 4, chain.EthereumEcdsa, TestItem.PrivateKeyA, TestItem.AddressB, 1); + + SimulatePayload payload = new() + { + BlockStateCalls = new List> + { + new() + { + BlockOverrides = + new BlockOverride + { + Number = (ulong)chain.BlockFinder.Head!.Number+2, + GasLimit = 5_000_000, + FeeRecipient = TestItem.AddressC, + BaseFeePerGas = 0 + }, + Calls = new[] { new TransactionForRpc(txAtoB1), new TransactionForRpc(txAtoB2) } + }, + new() + { + BlockOverrides = + new BlockOverride + { + Number = (ulong)checked(chain.Bridge.HeadBlock.Number + 10), + GasLimit = 5_000_000, + FeeRecipient = TestItem.AddressC, + BaseFeePerGas = 0 + }, + Calls = new[] { new TransactionForRpc(txAtoB3), new TransactionForRpc(txAtoB4) } + } + }, + TraceTransfers = true + }; + + //Test that transfer tx works on mainchain + UInt256 before = chain.State.GetBalance(TestItem.AddressA); + await chain.AddBlock(true, txMainnetAtoB); + UInt256 after = chain.State.GetBalance(TestItem.AddressA); + Assert.Less(after, before); + + chain.Bridge.GetReceipt(txMainnetAtoB.Hash!); + + //Force persistancy of head block in main chain + chain.BlockTree.UpdateMainChain(new List { chain.BlockFinder.Head! }, true, true); + chain.BlockTree.UpdateHeadBlock(chain.BlockFinder.Head!.Hash!); + + //will mock our GetCachedCodeInfo function - it shall be called 3 times if redirect is working, 2 times if not + SimulateTxExecutor executor = new(chain.Bridge, chain.BlockFinder, new JsonRpcConfig(), new BlocksConfig().SecondsPerSlot); + ResultWrapper> result = + executor.Execute(payload, BlockParameter.Latest); + IReadOnlyList data = result.Data; + + Assert.That(data.Count, Is.EqualTo(9)); + + SimulateBlockResult blockResult = data[0]; + Assert.That(blockResult.Calls.Count(), Is.EqualTo(2)); + blockResult = data.Last(); + Assert.That(blockResult.Calls.Count(), Is.EqualTo(2)); + } + + /// + /// This test verifies that a temporary forked blockchain can make transactions, blocks and report on them + /// + [Test] + public async Task Test_eth_simulate_transactions_forced_fail() + { + TestRpcBlockchain chain = await EthRpcSimulateTestsBase.CreateChain(); + + UInt256 nonceA = chain.State.GetNonce(TestItem.AddressA); + + Transaction txMainnetAtoB = + GetTransferTxData(nonceA, chain.EthereumEcdsa, TestItem.PrivateKeyA, TestItem.AddressB, 1); + //shall be Ok + Transaction txAtoB1 = + GetTransferTxData(nonceA + 1, chain.EthereumEcdsa, TestItem.PrivateKeyA, TestItem.AddressB, 1); + + //shall fail + Transaction txAtoB2 = + GetTransferTxData(nonceA + 2, chain.EthereumEcdsa, TestItem.PrivateKeyA, TestItem.AddressB, UInt256.MaxValue); + TransactionForRpc transactionForRpc = new(txAtoB2) { Nonce = null }; + TransactionForRpc transactionForRpc2 = new(txAtoB1) { Nonce = null }; + SimulatePayload payload = new() + { + BlockStateCalls = new List> + { + new() + { + BlockOverrides = + new BlockOverride + { + Number = (ulong)checked(chain.Bridge.HeadBlock.Number + 10), + GasLimit = 5_000_000, + FeeRecipient = TestItem.AddressC, + BaseFeePerGas = 0 + }, + Calls = new[] { transactionForRpc2 } + }, + new() + { + BlockOverrides = + new BlockOverride + { + Number = 123, + GasLimit = 5_000_000, + FeeRecipient = TestItem.AddressC, + BaseFeePerGas = 0 + }, + Calls = new[] { transactionForRpc } + } + }, + TraceTransfers = true, + Validation = true + }; + + //Test that transfer tx works on mainchain + UInt256 before = chain.State.GetBalance(TestItem.AddressA); + await chain.AddBlock(true, txMainnetAtoB); + UInt256 after = chain.State.GetBalance(TestItem.AddressA); + Assert.Less(after, before); + + chain.Bridge.GetReceipt(txMainnetAtoB.Hash!); + + //Force persistancy of head block in main chain + chain.BlockTree.UpdateMainChain(new List { chain.BlockFinder.Head! }, true, true); + chain.BlockTree.UpdateHeadBlock(chain.BlockFinder.Head!.Hash!); + + //will mock our GetCachedCodeInfo function - it shall be called 3 times if redirect is working, 2 times if not + SimulateTxExecutor executor = new(chain.Bridge, chain.BlockFinder, new JsonRpcConfig(), new BlocksConfig().SecondsPerSlot); + + ResultWrapper> result = + executor.Execute(payload, BlockParameter.Latest); + Assert.IsTrue(result.Result!.Error!.Contains("higher than sender balance")); + } + + + [Test] + public async Task TestTransferLogsAddress() + { + EthereumJsonSerializer serializer = new(); + string input = """ + { + "traceTransfers": true, + "blockStateCalls": [ + { + "blockOverrides": { + "baseFeePerGas": "0xa" + }, + "stateOverrides": { + "0xc000000000000000000000000000000000000000": { + "balance": "0x35a4ece8" + } + }, + "calls": [ + { + "from": "0xc000000000000000000000000000000000000000", + "to": "0xc100000000000000000000000000000000000000", + "gas": "0x5208", + "maxFeePerGas": "0x14", + "maxPriorityFeePerGas": "0x1", + "maxFeePerBlobGas": "0x0", + "value": "0x65", + "nonce": "0x0", + "input": "0x" + } + ] + } + ] + } + """; + var payload = serializer.Deserialize>(input); + TestRpcBlockchain chain = await EthRpcSimulateTestsBase.CreateChain(); + Console.WriteLine("current test: simulateTransferOverBlockStateCalls"); + var result = chain.EthRpcModule.eth_simulateV1(payload!, BlockParameter.Latest); + var logs = result.Data.First().Calls.First().Logs.ToArray(); + Assert.That(logs.First().Address == new Address("0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE")); + } +} diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsHiveBase.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsHiveBase.cs new file mode 100644 index 00000000000..7c1b85e4325 --- /dev/null +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsHiveBase.cs @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Nethermind.Blockchain.Find; +using Nethermind.Facade.Eth; +using Nethermind.Facade.Proxy.Models.Simulate; +using Nethermind.JsonRpc.Data; +using Nethermind.Serialization.Json; +using NUnit.Framework; +using ResultType = Nethermind.Core.ResultType; + +namespace Nethermind.JsonRpc.Test.Modules.Eth; + +public class EthSimulateTestsHiveBase +{ + private static readonly object[] HiveTestCases = + { +new object[] {"multicall-add-more-non-defined-BlockStateCalls-than-fit-but-now-with-fit", "{\"blockStateCalls\": [{\"blockOverrides\": {\"number\": \"0xa\"}, \"stateOverrides\": {\"0xc100000000000000000000000000000000000000\": {\"code\": \"0x608060405234801561001057600080fd5b506000366060484641444543425a3a60014361002c919061009b565b406040516020016100469a99989796959493929190610138565b6040516020818303038152906040529050915050805190602001f35b6000819050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b60006100a682610062565b91506100b183610062565b92508282039050818111156100c9576100c861006c565b5b92915050565b6100d881610062565b82525050565b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610109826100de565b9050919050565b610119816100fe565b82525050565b6000819050919050565b6101328161011f565b82525050565b60006101408201905061014e600083018d6100cf565b61015b602083018c6100cf565b610168604083018b610110565b610175606083018a6100cf565b61018260808301896100cf565b61018f60a08301886100cf565b61019c60c08301876100cf565b6101a960e08301866100cf565b6101b76101008301856100cf565b6101c5610120830184610129565b9b9a505050505050505050505056fea26469706673582212205139ae3ba8d46d11c29815d001b725f9840c90e330884ed070958d5af4813d8764736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x\"}]}, {\"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x\"}]}, {\"blockOverrides\": {\"number\": \"0x14\"}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x\"}]}, {\"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x\"}]}]}"}, +new object[] {"multicall-basefee-too-low-without-validation-38012", "{\"blockStateCalls\": [{\"blockOverrides\": {\"baseFeePerGas\": \"0xa\"}, \"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x7d0\"}}, \"calls\": [{\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"maxFeePerGas\": \"0x0\", \"maxPriorityFeePerGas\": \"0x0\"}]}]}"}, +//new object[] {"multicall-block-override-reflected-in-contract-simple", "{\"blockStateCalls\": [{\"blockOverrides\": {\"number\": \"0x12a\", \"time\": \"0x64\"}}, {\"blockOverrides\": {\"number\": \"0x14\", \"time\": \"0x65\"}}, {\"blockOverrides\": {\"number\": \"0x15\", \"time\": \"0xc8\"}}]}"}, +//new object[] {"multicall-block-timestamps-incrementing", "{\"blockStateCalls\": [{\"blockOverrides\": {\"time\": \"0x12b\"}}, {\"blockOverrides\": {\"time\": \"0x12c\"}}]}"}, +new object[] {"multicall-blockhash-complex", "{\"blockStateCalls\": [{\"blockOverrides\": {\"number\": \"0xa\"}, \"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x1e8480\"}, \"0xc200000000000000000000000000000000000000\": {\"code\": \"0x6080604052348015600f57600080fd5b506004361060285760003560e01c8063ee82ac5e14602d575b600080fd5b60436004803603810190603f91906098565b6057565b604051604e919060d7565b60405180910390f35b600081409050919050565b600080fd5b6000819050919050565b6078816067565b8114608257600080fd5b50565b6000813590506092816071565b92915050565b60006020828403121560ab5760aa6062565b5b600060b7848285016085565b91505092915050565b6000819050919050565b60d18160c0565b82525050565b600060208201905060ea600083018460ca565b9291505056fea2646970667358221220a4d7face162688805e99e86526524ac3dadfb01cc29366d0d68b70dadcf01afe64736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0xee82ac5e0000000000000000000000000000000000000000000000000000000000000001\"}]}, {\"blockOverrides\": {\"number\": \"0x14\"}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0xee82ac5e000000000000000000000000000000000000000000000000000000000000000a\"}]}, {\"blockOverrides\": {\"number\": \"0x1e\"}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0xee82ac5e000000000000000000000000000000000000000000000000000000000000001d\"}]}]}"}, +new object[] {"multicall-blockhash-simple", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc200000000000000000000000000000000000000\": {\"code\": \"0x6080604052348015600f57600080fd5b506004361060285760003560e01c8063ee82ac5e14602d575b600080fd5b60436004803603810190603f91906098565b6057565b604051604e919060d7565b60405180910390f35b600081409050919050565b600080fd5b6000819050919050565b6078816067565b8114608257600080fd5b50565b6000813590506092816071565b92915050565b60006020828403121560ab5760aa6062565b5b600060b7848285016085565b91505092915050565b6000819050919050565b60d18160c0565b82525050565b600060208201905060ea600083018460ca565b9291505056fea2646970667358221220a4d7face162688805e99e86526524ac3dadfb01cc29366d0d68b70dadcf01afe64736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0xee82ac5e0000000000000000000000000000000000000000000000000000000000000001\"}]}]}"}, +new object[] {"multicall-blockhash-start-before-head", "{\"blockStateCalls\": [{\"blockOverrides\": {\"number\": \"0xa\"}, \"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x1e8480\"}, \"0xc200000000000000000000000000000000000000\": {\"code\": \"0x6080604052348015600f57600080fd5b506004361060285760003560e01c8063ee82ac5e14602d575b600080fd5b60436004803603810190603f91906098565b6057565b604051604e919060d7565b60405180910390f35b600081409050919050565b600080fd5b6000819050919050565b6078816067565b8114608257600080fd5b50565b6000813590506092816071565b92915050565b60006020828403121560ab5760aa6062565b5b600060b7848285016085565b91505092915050565b6000819050919050565b60d18160c0565b82525050565b600060208201905060ea600083018460ca565b9291505056fea2646970667358221220a4d7face162688805e99e86526524ac3dadfb01cc29366d0d68b70dadcf01afe64736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0xee82ac5e0000000000000000000000000000000000000000000000000000000000000001\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0xee82ac5e0000000000000000000000000000000000000000000000000000000000000002\"}]}, {\"blockOverrides\": {\"number\": \"0x14\"}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0xee82ac5e0000000000000000000000000000000000000000000000000000000000000013\"}]}]}"}, +new object[] {"multicall-check-that-balance-is-there-after-new-block", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x2710\"}, \"0xc200000000000000000000000000000000000000\": {\"code\": \"0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f8b2cb4f14610030575b600080fd5b61004a600480360381019061004591906100e4565b610060565b604051610057919061012a565b60405180910390f35b60008173ffffffffffffffffffffffffffffffffffffffff16319050919050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006100b182610086565b9050919050565b6100c1816100a6565b81146100cc57600080fd5b50565b6000813590506100de816100b8565b92915050565b6000602082840312156100fa576100f9610081565b5b6000610108848285016100cf565b91505092915050565b6000819050919050565b61012481610111565b82525050565b600060208201905061013f600083018461011b565b9291505056fea2646970667358221220172c443a163d8a43e018c339d1b749c312c94b6de22835953d960985daf228c764736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0xf8b2cb4f000000000000000000000000c000000000000000000000000000000000000000\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0xf8b2cb4f000000000000000000000000c100000000000000000000000000000000000000\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x3e8\"}]}, {\"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0xf8b2cb4f000000000000000000000000c000000000000000000000000000000000000000\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0xf8b2cb4f000000000000000000000000c100000000000000000000000000000000000000\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x3e8\"}]}]}"}, +new object[] {"multicall-contract-calls-itself", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"code\": \"0x608060405234801561001057600080fd5b506000366060484641444543425a3a60014361002c919061009b565b406040516020016100469a99989796959493929190610138565b6040516020818303038152906040529050915050805190602001f35b6000819050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b60006100a682610062565b91506100b183610062565b92508282039050818111156100c9576100c861006c565b5b92915050565b6100d881610062565b82525050565b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610109826100de565b9050919050565b610119816100fe565b82525050565b6000819050919050565b6101328161011f565b82525050565b60006101408201905061014e600083018d6100cf565b61015b602083018c6100cf565b610168604083018b610110565b610175606083018a6100cf565b61018260808301896100cf565b61018f60a08301886100cf565b61019c60c08301876100cf565b6101a960e08301866100cf565b6101b76101008301856100cf565b6101c5610120830184610129565b9b9a505050505050505050505056fea26469706673582212205139ae3ba8d46d11c29815d001b725f9840c90e330884ed070958d5af4813d8764736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc000000000000000000000000000000000000000\"}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-empty-calls-and-overrides-multicall", "{\"blockStateCalls\": [{\"stateOverrides\": {}, \"calls\": [{}]}, {\"stateOverrides\": {}, \"calls\": [{}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-eth-send-should-not-produce-logs-by-default", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x7d0\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x3e8\"}]}]}"}, +new object[] {"multicall-eth-send-should-not-produce-logs-on-revert", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x7d0\"}, \"0xc100000000000000000000000000000000000000\": {\"code\": \"0x608060405260006042576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401603990609d565b60405180910390fd5b005b600082825260208201905092915050565b7f416c7761797320726576657274696e6720636f6e747261637400000000000000600082015250565b600060896019836044565b91506092826055565b602082019050919050565b6000602082019050818103600083015260b481607e565b905091905056fea264697066735822122005cbbbc709291f66fadc17416c1b0ed4d72941840db11468a21b8e1a0362024c64736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x3e8\"}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-eth-send-should-produce-logs", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x7d0\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x3e8\"}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-eth-send-should-produce-more-logs-on-forward", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x7d0\"}, \"0xc100000000000000000000000000000000000000\": {\"code\": \"0x60806040526004361061001e5760003560e01c80634b64e49214610023575b600080fd5b61003d6004803603810190610038919061011f565b61003f565b005b60008173ffffffffffffffffffffffffffffffffffffffff166108fc349081150290604051600060405180830381858888f193505050509050806100b8576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016100af906101a9565b60405180910390fd5b5050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006100ec826100c1565b9050919050565b6100fc816100e1565b811461010757600080fd5b50565b600081359050610119816100f3565b92915050565b600060208284031215610135576101346100bc565b5b60006101438482850161010a565b91505092915050565b600082825260208201905092915050565b7f4661696c656420746f2073656e64204574686572000000000000000000000000600082015250565b600061019360148361014c565b915061019e8261015d565b602082019050919050565b600060208201905081810360008301526101c281610186565b905091905056fea2646970667358221220563acd6f5b8ad06a3faf5c27fddd0ecbc198408b99290ce50d15c2cf7043694964736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x3e8\", \"input\": \"0x4b64e4920000000000000000000000000000000000000000000000000000000000000100\"}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-eth-send-should-produce-no-logs-on-forward-revert", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x7d0\"}, \"0xc100000000000000000000000000000000000000\": {\"code\": \"0x60806040526004361061001e5760003560e01c80634b64e49214610023575b600080fd5b61003d6004803603810190610038919061011f565b61003f565b005b60008173ffffffffffffffffffffffffffffffffffffffff166108fc349081150290604051600060405180830381858888f193505050509050806100b8576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016100af906101a9565b60405180910390fd5b5050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006100ec826100c1565b9050919050565b6100fc816100e1565b811461010757600080fd5b50565b600081359050610119816100f3565b92915050565b600060208284031215610135576101346100bc565b5b60006101438482850161010a565b91505092915050565b600082825260208201905092915050565b7f4661696c656420746f2073656e64204574686572000000000000000000000000600082015250565b600061019360148361014c565b915061019e8261015d565b602082019050919050565b600060208201905081810360008301526101c281610186565b905091905056fea2646970667358221220563acd6f5b8ad06a3faf5c27fddd0ecbc198408b99290ce50d15c2cf7043694964736f6c63430008120033\"}, \"0xc200000000000000000000000000000000000000\": {\"code\": \"0x608060405260006042576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401603990609d565b60405180910390fd5b005b600082825260208201905092915050565b7f416c7761797320726576657274696e6720636f6e747261637400000000000000600082015250565b600060896019836044565b91506092826055565b602082019050919050565b6000602082019050818103600083015260b481607e565b905091905056fea264697066735822122005cbbbc709291f66fadc17416c1b0ed4d72941840db11468a21b8e1a0362024c64736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x3e8\", \"input\": \"0x4b64e492c200000000000000000000000000000000000000000000000000000000000000\"}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-get-block-properties", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc100000000000000000000000000000000000000\": {\"code\": \"0x608060405234801561001057600080fd5b506000366060484641444543425a3a60014361002c919061009b565b406040516020016100469a99989796959493929190610138565b6040516020818303038152906040529050915050805190602001f35b6000819050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b60006100a682610062565b91506100b183610062565b92508282039050818111156100c9576100c861006c565b5b92915050565b6100d881610062565b82525050565b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610109826100de565b9050919050565b610119816100fe565b82525050565b6000819050919050565b6101328161011f565b82525050565b60006101408201905061014e600083018d6100cf565b61015b602083018c6100cf565b610168604083018b610110565b610175606083018a6100cf565b61018260808301896100cf565b61018f60a08301886100cf565b61019c60c08301876100cf565b6101a960e08301866100cf565b6101b76101008301856100cf565b6101c5610120830184610129565b9b9a505050505050505050505056fea26469706673582212205139ae3ba8d46d11c29815d001b725f9840c90e330884ed070958d5af4813d8764736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x\"}]}]}"}, +new object[] {"multicall-instrict-gas-38013", "{\"blockStateCalls\": [{\"calls\": [{\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"gas\": \"0x0\"}]}]}"}, +new object[] {"multicall-logs", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc200000000000000000000000000000000000000\": {\"code\": \"0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff80600080a1600080f3\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0x6057361d0000000000000000000000000000000000000000000000000000000000000005\"}]}]}"}, +new object[] {"multicall-move-account-twice", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x3e8\"}, \"0xc200000000000000000000000000000000000000\": {\"balance\": \"0x7d0\"}, \"0xc300000000000000000000000000000000000000\": {\"balance\": \"0xbb8\"}, \"0xc400000000000000000000000000000000000000\": {\"code\": \"0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063f8b2cb4f14610030575b600080fd5b61004a600480360381019061004591906100e4565b610060565b604051610057919061012a565b60405180910390f35b60008173ffffffffffffffffffffffffffffffffffffffff16319050919050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006100b182610086565b9050919050565b6100c1816100a6565b81146100cc57600080fd5b50565b6000813590506100de816100b8565b92915050565b6000602082840312156100fa576100f9610081565b5b6000610108848285016100cf565b91505092915050565b6000819050919050565b61012481610111565b82525050565b600060208201905061013f600083018461011b565b9291505056fea2646970667358221220172c443a163d8a43e018c339d1b749c312c94b6de22835953d960985daf228c764736f6c63430008120033\"}}}, {\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0xbb8\", \"MovePrecompileToAddress\": \"0xc200000000000000000000000000000000000000\"}, \"0xc200000000000000000000000000000000000000\": {\"balance\": \"0xfa0\", \"MovePrecompileToAddress\": \"0xc300000000000000000000000000000000000000\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc400000000000000000000000000000000000000\", \"input\": \"0xf8b2cb4f000000000000000000000000c000000000000000000000000000000000000000\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc400000000000000000000000000000000000000\", \"input\": \"0xf8b2cb4f000000000000000000000000c200000000000000000000000000000000000000\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc400000000000000000000000000000000000000\", \"input\": \"0xf8b2cb4f000000000000000000000000c300000000000000000000000000000000000000\"}]}]}"}, +new object[] {"multicall-move-ecrecover-and-call", "{\"blockStateCalls\": [{\"calls\": [{\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000000001\", \"input\": \"0x4554480000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b45544800000000000000000000000000000000000000000000000000000000004554480000000000000000000000000000000000000000000000000000000000\"}, {\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000000001\", \"input\": \"0x1c8aff950685c2ed4bc3174f3472287b56d9517b9c948127319a09a7a36deac8000000000000000000000000000000000000000000000000000000000000001cb7cf302145348387b9e69fde82d8e634a0f8761e78da3bfa059efced97cbed0d2a66b69167cafe0ccfc726aec6ee393fea3cf0e4f3f9c394705e0f56d9bfe1c9\"}]}, {\"stateOverrides\": {\"0x0000000000000000000000000000000000000001\": {\"MovePrecompileToAddress\": \"0x0000000000000000000000000000000000123456\"}}, \"calls\": [{\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000123456\", \"input\": \"0x4554480000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b45544800000000000000000000000000000000000000000000000000000000004554480000000000000000000000000000000000000000000000000000000000\"}, {\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000123456\", \"input\": \"0x1c8aff950685c2ed4bc3174f3472287b56d9517b9c948127319a09a7a36deac8000000000000000000000000000000000000000000000000000000000000001cb7cf302145348387b9e69fde82d8e634a0f8761e78da3bfa059efced97cbed0d2a66b69167cafe0ccfc726aec6ee393fea3cf0e4f3f9c394705e0f56d9bfe1c9\"}]}]}"}, +new object[] {"multicall-move-to-address-itself-reference-38022", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x30d40\"}, \"0xc100000000000000000000000000000000000000\": {\"MovePrecompileToAddress\": \"0xc100000000000000000000000000000000000000\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x1\"}]}]}"}, +new object[] {"multicall-move-two-accounts-to-same-38023", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0x0000000000000000000000000000000000000001\": {\"MovePrecompileToAddress\": \"0xc200000000000000000000000000000000000000\"}, \"0x0000000000000000000000000000000000000002\": {\"MovePrecompileToAddress\": \"0xc200000000000000000000000000000000000000\"}}}]}"}, +new object[] {"multicall-move-two-non-precompiles-accounts-to-same", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0x0100000000000000000000000000000000000000\": {\"MovePrecompileToAddress\": \"0xc200000000000000000000000000000000000000\"}, \"0x0200000000000000000000000000000000000000\": {\"MovePrecompileToAddress\": \"0xc200000000000000000000000000000000000000\"}}}]}"}, +new object[] {"multicall-no-fields-call", "{\"blockStateCalls\": [{\"calls\": [{}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-only-from-to-transaction", "{\"blockStateCalls\": [{\"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\"}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-only-from-transaction", "{\"blockStateCalls\": [{\"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\"}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-override-address-twice-in-separate-BlockStateCalls", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x7d0\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x3e8\"}]}, {\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x7d0\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x3e8\"}]}], \"traceTransfers\": true}"}, +//new object[] {"multicall-override-all-in-BlockStateCalls", "{\"blockStateCalls\": [{\"blockOverrides\": {\"number\": \"0x3e9\", \"time\": \"0x3eb\", \"gasLimit\": \"0x3ec\", \"feeRecipient\": \"0xc200000000000000000000000000000000000000\", \"prevRandao\": \"0xc300000000000000000000000000000000000000000000000000000000000000\", \"baseFeePerGas\": \"0x3ef\"}}]}"}, +new object[] {"multicall-override-block-num", "{\"blockStateCalls\": [{\"blockOverrides\": {\"number\": \"0xb\"}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"input\": \"0x4360005260206000f3\"}]}, {\"blockOverrides\": {\"number\": \"0xc\"}, \"calls\": [{\"from\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x4360005260206000f3\"}]}]}"}, +new object[] {"multicall-override-ecrecover", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0x0000000000000000000000000000000000000001\": {\"code\": \"0x608060405234801561001057600080fd5b506004361061003a5760003560e01c806305fdbc81146101ee578063c00692601461020a5761003b565b5b600036606060008060008086868101906100559190610462565b93509350935093506000806000868686866040516020016100799493929190610520565b60405160208183030381529060405280519060200120815260200190815260200160002060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff169050600073ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff16036101bb576000806212345673ffffffffffffffffffffffffffffffffffffffff166127108b8b6040516101249291906105ad565b60006040518083038160008787f1925050503d8060008114610162576040519150601f19603f3d011682016040523d82523d6000602084013e610167565b606091505b5091509150816101ac576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016101a39061066f565b60405180910390fd5b809750505050505050506101e3565b806040516020016101cc9190610709565b604051602081830303815290604052955050505050505b915050805190602001f35b6102086004803603810190610203919061093a565b610226565b005b610224600480360381019061021f9190610983565b6102ec565b005b60005b81518110156102e8576102d5828281518110610248576102476109fe565b5b602002602001015160000151838381518110610267576102666109fe565b5b602002602001015160200151848481518110610286576102856109fe565b5b6020026020010151604001518585815181106102a5576102a46109fe565b5b6020026020010151606001518686815181106102c4576102c36109fe565b5b6020026020010151608001516102ec565b80806102e090610a66565b915050610229565b5050565b600073ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff160361035b576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161035290610afa565b60405180910390fd5b80600080878787876040516020016103769493929190610520565b60405160208183030381529060405280519060200120815260200190815260200160002060006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055505050505050565b6000604051905090565b600080fd5b600080fd5b6000819050919050565b610406816103f3565b811461041157600080fd5b50565b600081359050610423816103fd565b92915050565b600060ff82169050919050565b61043f81610429565b811461044a57600080fd5b50565b60008135905061045c81610436565b92915050565b6000806000806080858703121561047c5761047b6103e9565b5b600061048a87828801610414565b945050602061049b8782880161044d565b93505060406104ac87828801610414565b92505060606104bd87828801610414565b91505092959194509250565b6000819050919050565b6104e46104df826103f3565b6104c9565b82525050565b60008160f81b9050919050565b6000610502826104ea565b9050919050565b61051a61051582610429565b6104f7565b82525050565b600061052c82876104d3565b60208201915061053c8286610509565b60018201915061054c82856104d3565b60208201915061055c82846104d3565b60208201915081905095945050505050565b600081905092915050565b82818337600083830152505050565b6000610594838561056e565b93506105a1838584610579565b82840190509392505050565b60006105ba828486610588565b91508190509392505050565b600082825260208201905092915050565b7f6661696c656420746f2063616c6c206d6f7665642065637265636f766572206160008201527f742061646472657373203078303030303030303030303030303030303030303060208201527f3030303030303030303030303030313233343536000000000000000000000000604082015250565b60006106596054836105c6565b9150610664826105d7565b606082019050919050565b600060208201905081810360008301526106888161064c565b9050919050565b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006106ba8261068f565b9050919050565b60008160601b9050919050565b60006106d9826106c1565b9050919050565b60006106eb826106ce565b9050919050565b6107036106fe826106af565b6106e0565b82525050565b600061071582846106f2565b60148201915081905092915050565b600080fd5b6000601f19601f8301169050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b61077282610729565b810181811067ffffffffffffffff821117156107915761079061073a565b5b80604052505050565b60006107a46103df565b90506107b08282610769565b919050565b600067ffffffffffffffff8211156107d0576107cf61073a565b5b602082029050602081019050919050565b600080fd5b600080fd5b6107f4816106af565b81146107ff57600080fd5b50565b600081359050610811816107eb565b92915050565b600060a0828403121561082d5761082c6107e6565b5b61083760a061079a565b9050600061084784828501610414565b600083015250602061085b8482850161044d565b602083015250604061086f84828501610414565b604083015250606061088384828501610414565b606083015250608061089784828501610802565b60808301525092915050565b60006108b66108b1846107b5565b61079a565b90508083825260208201905060a084028301858111156108d9576108d86107e1565b5b835b8181101561090257806108ee8882610817565b84526020840193505060a0810190506108db565b5050509392505050565b600082601f83011261092157610920610724565b5b81356109318482602086016108a3565b91505092915050565b6000602082840312156109505761094f6103e9565b5b600082013567ffffffffffffffff81111561096e5761096d6103ee565b5b61097a8482850161090c565b91505092915050565b600080600080600060a0868803121561099f5761099e6103e9565b5b60006109ad88828901610414565b95505060206109be8882890161044d565b94505060406109cf88828901610414565b93505060606109e088828901610414565b92505060806109f188828901610802565b9150509295509295909350565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052603260045260246000fd5b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b6000819050919050565b6000610a7182610a5c565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8203610aa357610aa2610a2d565b5b600182019050919050565b7f72657475726e20616464726573732063616e6e6f742062652030783000000000600082015250565b6000610ae4601c836105c6565b9150610aef82610aae565b602082019050919050565b60006020820190508181036000830152610b1381610ad7565b905091905056fea2646970667358221220154f5b68ccfa5be744e7245765a3530dac4035052284a68b5dded1945b45075e64736f6c63430008120033\", \"MovePrecompileToAddress\": \"0x0000000000000000000000000000000000123456\"}, \"0xc100000000000000000000000000000000000000\": {\"balance\": \"0x30d40\"}}, \"calls\": [{\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000123456\", \"input\": \"0x4554480000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b45544800000000000000000000000000000000000000000000000000000000004554480000000000000000000000000000000000000000000000000000000000\"}, {\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000123456\", \"input\": \"0x1c8aff950685c2ed4bc3174f3472287b56d9517b9c948127319a09a7a36deac8000000000000000000000000000000000000000000000000000000000000001cb7cf302145348387b9e69fde82d8e634a0f8761e78da3bfa059efced97cbed0d2a66b69167cafe0ccfc726aec6ee393fea3cf0e4f3f9c394705e0f56d9bfe1c9\"}, {\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000000001\", \"input\": \"0xc00692604554480000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b45544800000000000000000000000000000000000000000000000000000000004554480000000000000000000000000000000000000000000000000000000000000000000000000000000000d8da6bf26964af9d7eed9e03e53415d37aa96045\"}, {\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000000001\", \"input\": \"0x4554480000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b45544800000000000000000000000000000000000000000000000000000000004554480000000000000000000000000000000000000000000000000000000000\"}, {\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000000001\", \"input\": \"0x1c8aff950685c2ed4bc3174f3472287b56d9517b9c948127319a09a7a36deac8000000000000000000000000000000000000000000000000000000000000001cb7cf302145348387b9e69fde82d8e634a0f8761e78da3bfa059efced97cbed0d2a66b69167cafe0ccfc726aec6ee393fea3cf0e4f3f9c394705e0f56d9bfe1c9\"}, {\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000000001\", \"input\": \"0x4554480000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b45544800000000000000000000000000000000000000000000000000000000004554490000000000000000000000000000000000000000000000000000000000\"}]}]}"}, +new object[] {"multicall-override-identity", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0x0000000000000000000000000000000000000004\": {\"code\": \"0x\", \"MovePrecompileToAddress\": \"0x0000000000000000000000000000000000123456\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000123456\", \"input\": \"0x1234\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000000004\", \"input\": \"0x1234\"}]}]}"}, +new object[] {"multicall-override-sha256", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0x0000000000000000000000000000000000000002\": {\"code\": \"0x\", \"MovePrecompileToAddress\": \"0x0000000000000000000000000000000000123456\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000123456\", \"input\": \"0x1234\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0x0000000000000000000000000000000000000002\", \"input\": \"0x1234\"}]}]}"}, +new object[] {"multicall-override-storage-slots", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x7d0\"}, \"0xc100000000000000000000000000000000000000\": {\"code\": \"0x608060405234801561001057600080fd5b506004361061004c5760003560e01c80630ff4c916146100515780633033413b1461008157806344e12f871461009f5780637b8d56e3146100bd575b600080fd5b61006b600480360381019061006691906101f6565b6100d9565b6040516100789190610232565b60405180910390f35b61008961013f565b6040516100969190610232565b60405180910390f35b6100a7610145565b6040516100b49190610232565b60405180910390f35b6100d760048036038101906100d2919061024d565b61014b565b005b60006002821061011e576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610115906102ea565b60405180910390fd5b6000820361012c5760005490505b6001820361013a5760015490505b919050565b60015481565b60005481565b6002821061018e576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610185906102ea565b60405180910390fd5b600082036101a257806000819055506101b7565b600182036101b657806001819055506101b7565b5b5050565b600080fd5b6000819050919050565b6101d3816101c0565b81146101de57600080fd5b50565b6000813590506101f0816101ca565b92915050565b60006020828403121561020c5761020b6101bb565b5b600061021a848285016101e1565b91505092915050565b61022c816101c0565b82525050565b60006020820190506102476000830184610223565b92915050565b60008060408385031215610264576102636101bb565b5b6000610272858286016101e1565b9250506020610283858286016101e1565b9150509250929050565b600082825260208201905092915050565b7f746f6f2062696720736c6f740000000000000000000000000000000000000000600082015250565b60006102d4600c8361028d565b91506102df8261029e565b602082019050919050565b60006020820190508181036000830152610303816102c7565b905091905056fea2646970667358221220ceea194bb66b5b9f52c83e5bf5a1989255de8cb7157838eff98f970c3a04cb3064736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x7b8d56e300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x7b8d56e300000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x0ff4c9160000000000000000000000000000000000000000000000000000000000000000\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x0ff4c9160000000000000000000000000000000000000000000000000000000000000001\"}]}, {\"stateOverrides\": {\"0xc100000000000000000000000000000000000000\": {\"stateDiff\": {\"0x0000000000000000000000000000000000000000000000000000000000000000\": \"0x1200000000000000000000000000000000000000000000000000000000000000\"}}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x0ff4c9160000000000000000000000000000000000000000000000000000000000000000\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x0ff4c9160000000000000000000000000000000000000000000000000000000000000001\"}]}, {\"stateOverrides\": {\"0xc100000000000000000000000000000000000000\": {\"state\": {\"0x0000000000000000000000000000000000000000000000000000000000000000\": \"0x1200000000000000000000000000000000000000000000000000000000000000\"}}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x0ff4c9160000000000000000000000000000000000000000000000000000000000000000\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x0ff4c9160000000000000000000000000000000000000000000000000000000000000001\"}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-precompile-is-sending-transaction", "{\"blockStateCalls\": [{\"calls\": [{\"from\": \"0x0000000000000000000000000000000000000004\", \"to\": \"0x0000000000000000000000000000000000000002\", \"input\": \"0x1234\"}]}]}"}, +new object[] {"multicall-run-gas-spending", "{\"blockStateCalls\": [{\"blockOverrides\": {\"gasLimit\": \"0x16e360\"}, \"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x1e8480\"}, \"0xc200000000000000000000000000000000000000\": {\"code\": \"0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063815b8ab414610030575b600080fd5b61004a600480360381019061004591906100b6565b61004c565b005b60005a90505b60011561007657815a826100669190610112565b106100715750610078565b610052565b505b50565b600080fd5b6000819050919050565b61009381610080565b811461009e57600080fd5b50565b6000813590506100b08161008a565b92915050565b6000602082840312156100cc576100cb61007b565b5b60006100da848285016100a1565b91505092915050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b600061011d82610080565b915061012883610080565b92508282039050818111156101405761013f6100e3565b5b9291505056fea2646970667358221220a659ba4db729a6ee4db02fcc5c1118db53246b0e5e686534fc9add6f2e93faec64736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0x815b8ab40000000000000000000000000000000000000000000000000000000000000000\"}]}, {\"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0x815b8ab40000000000000000000000000000000000000000000000000000000000000000\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0x815b8ab400000000000000000000000000000000000000000000000000000000000f4240\"}]}, {\"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0x815b8ab40000000000000000000000000000000000000000000000000000000000000000\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0x815b8ab400000000000000000000000000000000000000000000000000000000000f4240\"}]}]}"}, +new object[] {"multicall-run-out-of-gas-in-block-38015", "{\"blockStateCalls\": [{\"blockOverrides\": {\"gasLimit\": \"0x16e360\"}, \"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x1e8480\"}, \"0xc200000000000000000000000000000000000000\": {\"code\": \"0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063815b8ab414610030575b600080fd5b61004a600480360381019061004591906100b6565b61004c565b005b60005a90505b60011561007657815a826100669190610112565b106100715750610078565b610052565b505b50565b600080fd5b6000819050919050565b61009381610080565b811461009e57600080fd5b50565b6000813590506100b08161008a565b92915050565b6000602082840312156100cc576100cb61007b565b5b60006100da848285016100a1565b91505092915050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b600061011d82610080565b915061012883610080565b92508282039050818111156101405761013f6100e3565b5b9291505056fea2646970667358221220a659ba4db729a6ee4db02fcc5c1118db53246b0e5e686534fc9add6f2e93faec64736f6c63430008120033\"}}}, {\"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0x815b8ab400000000000000000000000000000000000000000000000000000000000f4240\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0x815b8ab400000000000000000000000000000000000000000000000000000000000f4240\"}]}]}"}, +new object[] {"multicall-self-destructing-state-override", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc200000000000000000000000000000000000000\": {\"code\": \"0x6080604052348015600f57600080fd5b506004361060285760003560e01c806383197ef014602d575b600080fd5b60336035565b005b600073ffffffffffffffffffffffffffffffffffffffff16fffea26469706673582212208e566fde20a17fff9658b9b1db37e27876fd8934ccf9b2aa308cabd37698681f64736f6c63430008120033\"}, \"0xc300000000000000000000000000000000000000\": {\"code\": \"0x73000000000000000000000000000000000000000030146080604052600436106100355760003560e01c8063dce4a4471461003a575b600080fd5b610054600480360381019061004f91906100f8565b61006a565b60405161006191906101b5565b60405180910390f35b6060813b6040519150601f19601f602083010116820160405280825280600060208401853c50919050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006100c58261009a565b9050919050565b6100d5816100ba565b81146100e057600080fd5b50565b6000813590506100f2816100cc565b92915050565b60006020828403121561010e5761010d610095565b5b600061011c848285016100e3565b91505092915050565b600081519050919050565b600082825260208201905092915050565b60005b8381101561015f578082015181840152602081019050610144565b60008484015250505050565b6000601f19601f8301169050919050565b600061018782610125565b6101918185610130565b93506101a1818560208601610141565b6101aa8161016b565b840191505092915050565b600060208201905081810360008301526101cf818461017c565b90509291505056fea26469706673582212206a5f0cd9f230619fa520fc4b9d4b518643258cad412f2fa33945ce528b4b895164736f6c63430008120033\"}}}, {\"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc300000000000000000000000000000000000000\", \"input\": \"0xdce4a447000000000000000000000000c200000000000000000000000000000000000000\"}]}, {\"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0x83197ef0\"}]}, {\"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc300000000000000000000000000000000000000\", \"input\": \"0xdce4a447000000000000000000000000c200000000000000000000000000000000000000\"}]}, {\"stateOverrides\": {\"0xc200000000000000000000000000000000000000\": {\"code\": \"0x6080604052348015600f57600080fd5b506004361060285760003560e01c806383197ef014602d575b600080fd5b60336035565b005b600073ffffffffffffffffffffffffffffffffffffffff16fffea26469706673582212208e566fde20a17fff9658b9b1db37e27876fd8934ccf9b2aa308cabd37698681f64736f6c63430008120033\"}}}, {\"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc300000000000000000000000000000000000000\", \"input\": \"0xdce4a447000000000000000000000000c200000000000000000000000000000000000000\"}]}]}"}, +new object[] {"multicall-self-destructive-contract-produces-logs", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc200000000000000000000000000000000000000\": {\"code\": \"0x6080604052348015600f57600080fd5b506004361060285760003560e01c806383197ef014602d575b600080fd5b60336035565b005b600073ffffffffffffffffffffffffffffffffffffffff16fffea26469706673582212208e566fde20a17fff9658b9b1db37e27876fd8934ccf9b2aa308cabd37698681f64736f6c63430008120033\", \"balance\": \"0x1e8480\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0x83197ef0\"}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-set-read-storage", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc200000000000000000000000000000000000000\": {\"code\": \"0x608060405234801561001057600080fd5b50600436106100365760003560e01c80632e64cec11461003b5780636057361d14610059575b600080fd5b610043610075565b60405161005091906100d9565b60405180910390f35b610073600480360381019061006e919061009d565b61007e565b005b60008054905090565b8060008190555050565b60008135905061009781610103565b92915050565b6000602082840312156100b3576100b26100fe565b5b60006100c184828501610088565b91505092915050565b6100d3816100f4565b82525050565b60006020820190506100ee60008301846100ca565b92915050565b6000819050919050565b600080fd5b61010c816100f4565b811461011757600080fd5b5056fea2646970667358221220404e37f487a89a932dca5e77faaf6ca2de3b991f93d230604b1b8daaef64766264736f6c63430008070033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0x6057361d0000000000000000000000000000000000000000000000000000000000000005\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"input\": \"0x2e64cec1\"}]}]}"}, +new object[] {"multicall-simple", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x3e8\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x3e8\"}, {\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"value\": \"0x3e8\"}]}]}"}, +new object[] {"multicall-simple-send-from-contract", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"code\": \"0x60806040526004361061001e5760003560e01c80634b64e49214610023575b600080fd5b61003d6004803603810190610038919061011f565b61003f565b005b60008173ffffffffffffffffffffffffffffffffffffffff166108fc349081150290604051600060405180830381858888f193505050509050806100b8576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016100af906101a9565b60405180910390fd5b5050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006100ec826100c1565b9050919050565b6100fc816100e1565b811461010757600080fd5b50565b600081359050610119816100f3565b92915050565b600060208284031215610135576101346100bc565b5b60006101438482850161010a565b91505092915050565b600082825260208201905092915050565b7f4661696c656420746f2073656e64204574686572000000000000000000000000600082015250565b600061019360148361014c565b915061019e8261015d565b602082019050919050565b600060208201905081810360008301526101c281610186565b905091905056fea2646970667358221220563acd6f5b8ad06a3faf5c27fddd0ecbc198408b99290ce50d15c2cf7043694964736f6c63430008120033\", \"balance\": \"0x3e8\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x3e8\"}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-simple-state-diff", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x7d0\"}, \"0xc100000000000000000000000000000000000000\": {\"code\": \"0x608060405234801561001057600080fd5b506004361061004c5760003560e01c80630ff4c916146100515780633033413b1461008157806344e12f871461009f5780637b8d56e3146100bd575b600080fd5b61006b600480360381019061006691906101f6565b6100d9565b6040516100789190610232565b60405180910390f35b61008961013f565b6040516100969190610232565b60405180910390f35b6100a7610145565b6040516100b49190610232565b60405180910390f35b6100d760048036038101906100d2919061024d565b61014b565b005b60006002821061011e576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610115906102ea565b60405180910390fd5b6000820361012c5760005490505b6001820361013a5760015490505b919050565b60015481565b60005481565b6002821061018e576040517f08c379a0000000000000000000000000000000000000000000000000000000008152600401610185906102ea565b60405180910390fd5b600082036101a257806000819055506101b7565b600182036101b657806001819055506101b7565b5b5050565b600080fd5b6000819050919050565b6101d3816101c0565b81146101de57600080fd5b50565b6000813590506101f0816101ca565b92915050565b60006020828403121561020c5761020b6101bb565b5b600061021a848285016101e1565b91505092915050565b61022c816101c0565b82525050565b60006020820190506102476000830184610223565b92915050565b60008060408385031215610264576102636101bb565b5b6000610272858286016101e1565b9250506020610283858286016101e1565b9150509250929050565b600082825260208201905092915050565b7f746f6f2062696720736c6f740000000000000000000000000000000000000000600082015250565b60006102d4600c8361028d565b91506102df8261029e565b602082019050919050565b60006020820190508181036000830152610303816102c7565b905091905056fea2646970667358221220ceea194bb66b5b9f52c83e5bf5a1989255de8cb7157838eff98f970c3a04cb3064736f6c63430008120033\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x7b8d56e300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x7b8d56e300000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002\"}]}, {\"stateOverrides\": {\"0xc100000000000000000000000000000000000000\": {\"state\": {\"0x0000000000000000000000000000000000000000000000000000000000000000\": \"0x1200000000000000000000000000000000000000000000000000000000000000\"}}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x0ff4c9160000000000000000000000000000000000000000000000000000000000000000\"}, {\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"input\": \"0x0ff4c9160000000000000000000000000000000000000000000000000000000000000001\"}]}], \"traceTransfers\": true}"}, +new object[] {"multicall-simple-with-validation-no-funds", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"balance\": \"0x3e8\"}}, \"calls\": [{\"from\": \"0xc000000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"value\": \"0x3e8\"}, {\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0xc200000000000000000000000000000000000000\", \"value\": \"0x3e8\"}]}]}"}, +new object[] {"multicall-transaction-too-high-nonce", "{\"blockStateCalls\": [{\"calls\": [{\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"nonce\": \"0x64\"}]}]}"}, +new object[] {"multicall-transaction-too-low-nonce-38010", "{\"blockStateCalls\": [{\"stateOverrides\": {\"0xc000000000000000000000000000000000000000\": {\"nonce\": \"0xa\"}}, \"calls\": [{\"from\": \"0xc100000000000000000000000000000000000000\", \"to\": \"0xc100000000000000000000000000000000000000\", \"nonce\": \"0x0\"}]}]}"}, +}; + + [Test, TestCaseSource(nameof(HiveTestCases)), Parallelizable(ParallelScope.All)] + public async Task TestsimulateHive(string name, string data) + { + EthereumJsonSerializer serializer = new(); + SimulatePayload? payload = serializer.Deserialize>(data); + TestRpcBlockchain chain = await EthRpcSimulateTestsBase.CreateChain(); + Console.WriteLine($"current test: {name}"); + ResultWrapper> result = + chain.EthRpcModule.eth_simulateV1(payload!, BlockParameter.Latest); + + Console.WriteLine(); + Assert.That(result.Result.ResultType, Is.EqualTo(ResultType.Success)); + Assert.IsNotNull(result.Data); + } +} diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsSimplePrecompiles.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsSimplePrecompiles.cs new file mode 100644 index 00000000000..b362a5439fd --- /dev/null +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/Simulate/EthSimulateTestsSimplePrecompiles.cs @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Test.Builders; +using Nethermind.Crypto; +using Nethermind.Evm; +using Nethermind.Evm.Precompiles; +using Nethermind.Facade.Proxy.Models; +using Nethermind.Facade.Proxy.Models.Simulate; +using Nethermind.Facade.Simulate; +using NUnit.Framework; + +namespace Nethermind.JsonRpc.Test.Modules.Eth; + +public class EthSimulateTestsSimplePrecompiles : EthRpcSimulateTestsBase +{ + /* Compiled contract + * Call example for TestItem.AddressA + * recover 0xb6e16d27ac5ab427a7f68900ac5559ce272dc6c37c82b3e052246c82244c50e4 28 0x7b8b1991eb44757bc688016d27940df8fb971d7c87f77a6bc4e938e3202c4403 0x7e9267b0aeaa82fa765361918f2d8abd9cdd86e64aa6f2b81d3c4e0b69a7b055 + * returns address: 0xb7705aE4c6F81B66cdB323C65f4E8133690fC099 + + pragma solidity ^0.8.7; + + contract EcrecoverProxy { + function recover(bytes32 hash, uint8 v, bytes32 r, bytes32 s) public pure returns (address) { + return ecrecover(hash, v, r, s); + } + } + */ + + //Taken from contract compiler output metadata + public const string EcRecoverCallerContractBytecode = + "608060405234801561001057600080fd5b5061028b806100206000396000f3fe608060405234801561001057600080fd5b506004361061002b5760003560e01c8063c2bf17b014610030575b600080fd5b61004a6004803603810190610045919061012f565b610060565b60405161005791906101d7565b60405180910390f35b6000600185858585604051600081526020016040526040516100859493929190610210565b6020604051602081039080840390855afa1580156100a7573d6000803e3d6000fd5b505050602060405103519050949350505050565b600080fd5b6000819050919050565b6100d3816100c0565b81146100de57600080fd5b50565b6000813590506100f0816100ca565b92915050565b600060ff82169050919050565b61010c816100f6565b811461011757600080fd5b50565b60008135905061012981610103565b92915050565b60008060008060808587031215610149576101486100bb565b5b6000610157878288016100e1565b94505060206101688782880161011a565b9350506040610179878288016100e1565b925050606061018a878288016100e1565b91505092959194509250565b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006101c182610196565b9050919050565b6101d1816101b6565b82525050565b60006020820190506101ec60008301846101c8565b92915050565b6101fb816100c0565b82525050565b61020a816100f6565b82525050565b600060808201905061022560008301876101f2565b6102326020830186610201565b61023f60408301856101f2565b61024c60608301846101f2565b9594505050505056fea26469706673582212204855668ab62273dde1249722b61c57ad057ef3d17384f21233e1b7bb309db7e464736f6c63430008120033"; + + + /// + /// This test verifies that a temporary forked blockchain can updates precompiles + /// + [Test] + public async Task Test_eth_simulate_erc() + { + TestRpcBlockchain chain = await CreateChain(); + + //Impose Opcode instead of EcRecoverPrecompile, it returns const TestItem.AddressE address + byte[] code = Prepare.EvmCode + .StoreDataInMemory(0, TestItem.AddressE + .ToString(false, false) + .PadLeft(64, '0')) + .PushData(Bytes.FromHexString("0x20")) + .PushData(Bytes.FromHexString("0x0")) + .Op(Instruction.RETURN).Done; + + // Step 1: Hash the message + Hash256 messageHash = Keccak.Compute("Hello, world!"); + // Step 2: Sign the hash + Signature signature = chain.EthereumEcdsa.Sign(TestItem.PrivateKeyA, messageHash); + + Address contractAddress = await DeployEcRecoverContract(chain, TestItem.PrivateKeyB, EcRecoverCallerContractBytecode); + byte[] transactionData = GenerateTransactionDataForEcRecover(messageHash, signature); + + SystemTransaction tx = new() + { + Data = transactionData, + To = contractAddress, + SenderAddress = TestItem.PublicKeyB.Address + }; + tx.Hash = tx.CalculateHash(); + + TransactionWithSourceDetails txDetails = new() + { + Transaction = tx, + HadGasLimitInRequest = false, + HadNonceInRequest = false + }; + + SimulatePayload payload = new() + { + BlockStateCalls = + [ + new BlockStateCall + { + StateOverrides = new Dictionary + { + { EcRecoverPrecompile.Address, new AccountOverride { Code = code } } + }, + Calls = [txDetails] + } + ], + TraceTransfers = true + }; + + SimulateOutput result = chain.Bridge.Simulate(chain.BlockFinder.Head?.Header!, payload, CancellationToken.None); + + byte[] addressBytes = result.Items[0].Calls[0].ReturnData!.SliceWithZeroPaddingEmptyOnError(12, 20); + Address resultingAddress = new(addressBytes); + Assert.That(resultingAddress, Is.EqualTo(TestItem.AddressE)); + + //Check that initial VM is intact + Address? mainChainRpcAddress = EcRecoverCall(chain, TestItem.AddressB, transactionData, contractAddress); + Assert.That(mainChainRpcAddress, Is.EqualTo(TestItem.AddressA)); + } +} diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/NetModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/NetModuleTests.cs index baadbcb415f..f652a3ad149 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/NetModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/NetModuleTests.cs @@ -57,7 +57,6 @@ public async Task NetVersionSuccessTest() Substitute.For(), Substitute.For(), syncConfig, - Substitute.For(), Substitute.For(), Substitute.For(), Substitute.For()); diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Proof/ProofRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Proof/ProofRpcModuleTests.cs index 6868f02c725..231725b4b37 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Proof/ProofRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Proof/ProofRpcModuleTests.cs @@ -29,6 +29,7 @@ using FluentAssertions; using Nethermind.Consensus.Processing; using Nethermind.Core.Buffers; +using Nethermind.Facade.Eth; using Nethermind.State.Tracing; using NSubstitute; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/RpcModuleProviderTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/RpcModuleProviderTests.cs index 980d8f169c2..707fff9b4e0 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/RpcModuleProviderTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/RpcModuleProviderTests.cs @@ -28,6 +28,12 @@ public void Initialize() _context = new JsonRpcContext(RpcEndpoint.Http); } + [TearDown] + public void TearDown() + { + _context?.Dispose(); + } + [Test] public void Module_provider_will_recognize_disabled_modules() { diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SingletonModulePoolTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SingletonModulePoolTests.cs index bc06fd5cb57..1ffb4b32cdb 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SingletonModulePoolTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SingletonModulePoolTests.cs @@ -24,6 +24,7 @@ using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Modules.Eth.FeeHistory; using Nethermind.JsonRpc.Modules.Eth.GasPrice; +using Nethermind.Config; namespace Nethermind.JsonRpc.Test.Modules { @@ -55,7 +56,8 @@ public Task Initialize() Substitute.For(), Substitute.For(), Substitute.For(), - Substitute.For()); + Substitute.For(), + new BlocksConfig().SecondsPerSlot); return Task.CompletedTask; } diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SubscribeModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SubscribeModuleTests.cs index 7e3248e927e..5134da7c20b 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SubscribeModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SubscribeModuleTests.cs @@ -137,7 +137,7 @@ private List GetLogsSubscriptionResult(Filter filter, BlockReplac _blockTree.BlockAddedToMain += Raise.EventWith(new object(), blockEventArgs); _receiptStorage.ReceiptsInserted += Raise.EventWith(new object(), blockEventArgs); - semaphoreSlim.Wait(TimeSpan.FromMilliseconds(100)); + semaphoreSlim.Wait(TimeSpan.FromMilliseconds(500)); subscriptionId = logsSubscription.Id; return jsonRpcResults; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TestRpcBlockchain.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TestRpcBlockchain.cs index c112755087c..6fba1d025d9 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TestRpcBlockchain.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TestRpcBlockchain.cs @@ -29,6 +29,8 @@ using Nethermind.Wallet; using Nethermind.Config; +using Nethermind.Db; +using Nethermind.Facade.Simulate; using Nethermind.Synchronization.ParallelSync; using NSubstitute; @@ -58,14 +60,9 @@ public class TestRpcBlockchain : TestBlockchain public static Builder ForTest(T blockchain) where T : TestRpcBlockchain => new(blockchain); - public class Builder where T : TestRpcBlockchain + public class Builder(T blockchain) where T : TestRpcBlockchain { - private readonly TestRpcBlockchain _blockchain; - - public Builder(T blockchain) - { - _blockchain = blockchain; - } + private readonly TestRpcBlockchain _blockchain = blockchain; public Builder WithBlockchainBridge(IBlockchainBridge blockchainBridge) { @@ -108,31 +105,41 @@ public Builder WithConfig(IJsonRpcConfig config) return this; } - public async Task Build(ISpecProvider? specProvider = null, UInt256? initialValues = null) - { - return (T)(await _blockchain.Build(specProvider, initialValues)); - } + public async Task Build( + ISpecProvider? specProvider = null, + UInt256? initialValues = null) => + (T)(await _blockchain.Build(specProvider, initialValues, true)); } - protected override async Task Build(ISpecProvider? specProvider = null, UInt256? initialValues = null, bool addBlockOnStart = true) + protected override async Task Build( + ISpecProvider? specProvider = null, + UInt256? initialValues = null, + bool addBlockOnStart = true) { specProvider ??= new TestSpecProvider(Berlin.Instance); - await base.Build(specProvider, initialValues); + await base.Build(specProvider, initialValues, addBlockOnStart); IFilterStore filterStore = new FilterStore(); IFilterManager filterManager = new FilterManager(filterStore, BlockProcessor, TxPool, LimboLogs.Instance); - + var dbProvider = new ReadOnlyDbProvider(DbProvider, false); + IReadOnlyBlockTree? roBlockTree = BlockTree!.AsReadOnly(); ReadOnlyTxProcessingEnv processingEnv = new( WorldStateManager, - new ReadOnlyBlockTree(BlockTree), + roBlockTree, + SpecProvider, + LimboLogs.Instance); + SimulateReadOnlyBlocksProcessingEnvFactory simulateProcessingEnvFactory = new SimulateReadOnlyBlocksProcessingEnvFactory( + WorldStateManager, + roBlockTree, + new ReadOnlyDbProvider(dbProvider, true), SpecProvider, LimboLogs.Instance); + BlocksConfig blocksConfig = new BlocksConfig(); ReceiptFinder ??= ReceiptStorage; - Bridge ??= new BlockchainBridge(processingEnv, TxPool, ReceiptFinder, filterStore, filterManager, EthereumEcdsa, Timestamper, LogFinder, SpecProvider, new BlocksConfig(), false); + Bridge ??= new BlockchainBridge(processingEnv, simulateProcessingEnvFactory, TxPool, ReceiptFinder, filterStore, filterManager, EthereumEcdsa, Timestamper, LogFinder, SpecProvider, blocksConfig, false); BlockFinder ??= BlockTree; GasPriceOracle ??= new GasPriceOracle(BlockFinder, SpecProvider, LogManager); - ITxSigner txSigner = new WalletTxSigner(TestWallet, specProvider.ChainId); TxSealer = new TxSealer(txSigner, Timestamper); TxSender ??= new TxPoolSender(TxPool, TxSealer, NonceManager, EthereumEcdsa ?? new EthereumEcdsa(specProvider.ChainId, LogManager)); @@ -153,7 +160,8 @@ protected override async Task Build(ISpecProvider? specProvider GasPriceOracle, new EthSyncingInfo(BlockTree, ReceiptStorage, syncConfig, new StaticSelector(SyncMode.All), Substitute.For(), LogManager), - FeeHistoryOracle); + FeeHistoryOracle, + blocksConfig.SecondsPerSlot); return this; } diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs index f6e285624ad..239fef3b75c 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Trace/ParityStyleTracerTests.cs @@ -63,8 +63,9 @@ public void Setup() _stateReader = new StateReader(trieStore, codeDb, LimboLogs.Instance); BlockhashProvider blockhashProvider = new(_blockTree, specProvider, stateProvider, LimboLogs.Instance); - VirtualMachine virtualMachine = new(blockhashProvider, specProvider, LimboLogs.Instance); - TransactionProcessor transactionProcessor = new(specProvider, stateProvider, virtualMachine, LimboLogs.Instance); + CodeInfoRepository codeInfoRepository = new(); + VirtualMachine virtualMachine = new(blockhashProvider, specProvider, codeInfoRepository, LimboLogs.Instance); + TransactionProcessor transactionProcessor = new(specProvider, stateProvider, virtualMachine, codeInfoRepository, LimboLogs.Instance); _poSSwitcher = Substitute.For(); BlockProcessor blockProcessor = new( @@ -74,7 +75,6 @@ public void Setup() new BlockProcessor.BlockValidationTransactionsExecutor(transactionProcessor, stateProvider), stateProvider, NullReceiptStorage.Instance, - NullWitnessCollector.Instance, new BlockhashStore(_blockTree, specProvider, stateProvider), transactionProcessor, LimboLogs.Instance); @@ -95,7 +95,7 @@ public void Setup() [Test] public void Can_trace_raw_parity_style() { - TraceRpcModule traceRpcModule = new(NullReceiptStorage.Instance, _tracer, _blockTree, _jsonRpcConfig, MainnetSpecProvider.Instance, LimboLogs.Instance, _stateReader); + TraceRpcModule traceRpcModule = new(NullReceiptStorage.Instance, _tracer, _blockTree, _jsonRpcConfig, _stateReader); ResultWrapper result = traceRpcModule.trace_rawTransaction(Bytes.FromHexString("f889808609184e72a00082271094000000000000000000000000000000000000000080a47f74657374320000000000000000000000000000000000000000000000000000006000571ca08a8bbf888cfa37bbf0bb965423625641fc956967b81d12e23709cead01446075a01ce999b56a8a88504be365442ea61239198e23d1fce7d00fcfc5cd3b44b7215f"), new[] { "trace" }); Assert.NotNull(result.Data); } @@ -103,7 +103,7 @@ public void Can_trace_raw_parity_style() [Test] public void Can_trace_raw_parity_style_berlin_tx() { - TraceRpcModule traceRpcModule = new(NullReceiptStorage.Instance, _tracer, _blockTree, _jsonRpcConfig, MainnetSpecProvider.Instance, LimboLogs.Instance, _stateReader); + TraceRpcModule traceRpcModule = new(NullReceiptStorage.Instance, _tracer, _blockTree, _jsonRpcConfig, _stateReader); ResultWrapper result = traceRpcModule.trace_rawTransaction(Bytes.FromHexString("01f85b821e8e8204d7847735940083030d408080853a60005500c080a0f43e70c79190701347517e283ef63753f6143a5225cbb500b14d98eadfb7616ba070893923d8a1fc97499f426524f9e82f8e0322dfac7c3d7e8a9eee515f0bcdc4"), new[] { "trace" }); Assert.NotNull(result.Data); } @@ -116,7 +116,7 @@ public void Should_return_correct_block_reward(bool isPostMerge) _blockTree!.SuggestBlock(block).Should().Be(AddBlockResult.Added); _poSSwitcher!.IsPostMerge(Arg.Any()).Returns(isPostMerge); - TraceRpcModule traceRpcModule = new(NullReceiptStorage.Instance, _tracer, _blockTree, _jsonRpcConfig, MainnetSpecProvider.Instance, LimboLogs.Instance, _stateReader); + TraceRpcModule traceRpcModule = new(NullReceiptStorage.Instance, _tracer, _blockTree, _jsonRpcConfig, _stateReader); ParityTxTraceFromStore[] result = traceRpcModule.trace_block(new BlockParameter(block.Number)).Data.ToArray(); if (isPostMerge) { diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TraceRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TraceRpcModuleTests.cs index 0e499183eb8..1fab4d08e2b 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TraceRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TraceRpcModuleTests.cs @@ -22,9 +22,12 @@ using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Tracing; using Nethermind.Consensus.Validators; +using Nethermind.Core.Crypto; using Nethermind.Db; using Nethermind.Evm; using Nethermind.Evm.Tracing.ParityStyle; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Facade.Eth; using Nethermind.Serialization.Json; using Nethermind.Specs.Forks; using Nethermind.Specs.Test; @@ -51,28 +54,33 @@ public async Task Build(ISpecProvider? specProvider = null, bool isAura = false) IReceiptFinder receiptFinder = new FullInfoReceiptFinder(Blockchain.ReceiptStorage, receiptsRecovery, Blockchain.BlockFinder); ReadOnlyTxProcessingEnv txProcessingEnv = new(Blockchain.WorldStateManager, Blockchain.BlockTree.AsReadOnly(), Blockchain.SpecProvider, Blockchain.LogManager); + IReadOnlyTxProcessingScope scope = txProcessingEnv.Build(Keccak.EmptyTreeHash); + RewardCalculator rewardCalculatorSource = new(Blockchain.SpecProvider); - IRewardCalculator rewardCalculator = rewardCalculatorSource.Get(txProcessingEnv.TransactionProcessor); + IRewardCalculator rewardCalculator = rewardCalculatorSource.Get(scope.TransactionProcessor); + + RpcBlockTransactionsExecutor rpcBlockTransactionsExecutor = new(scope.TransactionProcessor, scope.WorldState); + BlockProcessor.BlockValidationTransactionsExecutor executeBlockTransactionsExecutor = new(scope.TransactionProcessor, + scope.WorldState); - RpcBlockTransactionsExecutor rpcBlockTransactionsExecutor = new(txProcessingEnv.TransactionProcessor, txProcessingEnv.StateProvider); - BlockProcessor.BlockValidationTransactionsExecutor executeBlockTransactionsExecutor = new(txProcessingEnv.TransactionProcessor, - txProcessingEnv.StateProvider); ReadOnlyChainProcessingEnv CreateChainProcessingEnv(IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor) => new( - txProcessingEnv, + scope, Always.Valid, Blockchain.BlockPreprocessorStep, rewardCalculator, Blockchain.ReceiptStorage, Blockchain.SpecProvider, + Blockchain.BlockTree, + Blockchain.StateReader, Blockchain.LogManager, transactionsExecutor); ReadOnlyChainProcessingEnv traceProcessingEnv = CreateChainProcessingEnv(rpcBlockTransactionsExecutor); ReadOnlyChainProcessingEnv executeProcessingEnv = CreateChainProcessingEnv(executeBlockTransactionsExecutor); - Tracer tracer = new(txProcessingEnv.StateProvider, traceProcessingEnv.ChainProcessor, executeProcessingEnv.ChainProcessor); - TraceRpcModule = new TraceRpcModule(receiptFinder, tracer, Blockchain.BlockFinder, JsonRpcConfig, MainnetSpecProvider.Instance, LimboLogs.Instance, txProcessingEnv.StateReader); + Tracer tracer = new(scope.WorldState, traceProcessingEnv.ChainProcessor, executeProcessingEnv.ChainProcessor); + TraceRpcModule = new TraceRpcModule(receiptFinder, tracer, Blockchain.BlockFinder, JsonRpcConfig, txProcessingEnv.StateReader); for (int i = 1; i < 10; i++) { diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TransactionForRpcConverterTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TransactionForRpcConverterTests.cs index 9e1822b3e8e..6b6add97578 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TransactionForRpcConverterTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/TransactionForRpcConverterTests.cs @@ -5,6 +5,7 @@ using FluentAssertions; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.JsonRpc.Test.Data; using Nethermind.Serialization.Json; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/WitnessModuleRpcTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/WitnessModuleRpcTests.cs deleted file mode 100644 index 55791cb01ce..00000000000 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/WitnessModuleRpcTests.cs +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using System.Threading.Tasks; -using FluentAssertions; -using Nethermind.Blockchain; -using Nethermind.Blockchain.Find; -using Nethermind.Core; -using Nethermind.Core.Test.Builders; -using Nethermind.Crypto; -using Nethermind.Db; -using Nethermind.JsonRpc.Modules.Witness; -using Nethermind.Logging; -using Nethermind.State.Witnesses; -using NSubstitute; -using NUnit.Framework; - -namespace Nethermind.JsonRpc.Test.Modules -{ - public class WitnessModuleTests - { - private const string GetOneWitnessHashResponse = - "{\"jsonrpc\":\"2.0\",\"result\":[\"0xa2a9f03b9493046696099d27b2612b99497aa1f392ec966716ab393c715a5bb6\"],\"id\":67}"; - private const string BlockNotFoundResponse = - "{\"jsonrpc\":\"2.0\",\"error\":{\"code\":-32001,\"message\":\"Block not found\"},\"id\":67}"; - private const string WitnessNotFoundResponse = - "{\"jsonrpc\":\"2.0\",\"error\":{\"code\":-32002,\"message\":\"Witness unavailable\"},\"id\":67}"; - - private IBlockFinder _blockFinder = null!; - - private WitnessCollector _witnessRepository = null!; - private WitnessRpcModule _witnessRpcModule = null!; - - private Block _block = null!; - - - [SetUp] - public void Setup() - { - _block = Build.A.Block - .WithUncles(Build.A.BlockHeader.TestObject, Build.A.BlockHeader.TestObject).TestObject; - - _blockFinder = Substitute.For(); - _witnessRepository = new WitnessCollector(new MemDb(), LimboLogs.Instance); - _witnessRpcModule = new WitnessRpcModule(_witnessRepository, _blockFinder); - } - - [Test] - public async Task GetOneWitnessHash() - { - _blockFinder.FindHeader((BlockParameter)null!).ReturnsForAnyArgs(_block.Header); - _blockFinder.Head.Returns(_block); - - using IDisposable tracker = _witnessRepository.TrackOnThisThread(); - _witnessRepository.Add(_block.Hash!); - _witnessRepository.Persist(_block.Hash!); - - string serialized = await RpcTest.TestSerializedRequest(_witnessRpcModule, "get_witnesses", _block.CalculateHash().ToString()); - serialized.Should().Be(GetOneWitnessHashResponse); - } - - [Test] - public async Task BlockNotFound() - { - string serialized = await RpcTest.TestSerializedRequest(_witnessRpcModule, "get_witnesses", "0x583"); - serialized.Should().Be(BlockNotFoundResponse); - } - - [Test] - public async Task WitnessNotFound() - { - _blockFinder.FindHeader((BlockParameter)null!).ReturnsForAnyArgs(_block.Header); - _blockFinder.Head.Returns(_block); - - string serialized = await RpcTest.TestSerializedRequest(_witnessRpcModule, "get_witnesses", "0x1"); - serialized.Should().Be(WitnessNotFoundResponse); - - } - } -} diff --git a/src/Nethermind/Nethermind.JsonRpc.TraceStore.Test/TraceStoreRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.TraceStore.Test/TraceStoreRpcModuleTests.cs index dbe61f725b1..8c82d299e41 100644 --- a/src/Nethermind/Nethermind.JsonRpc.TraceStore.Test/TraceStoreRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.TraceStore.Test/TraceStoreRpcModuleTests.cs @@ -12,6 +12,7 @@ using Nethermind.Core.Test.Builders; using Nethermind.Db; using Nethermind.Evm.Tracing.ParityStyle; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.JsonRpc.Modules.Trace; using Nethermind.Logging; diff --git a/src/Nethermind/Nethermind.JsonRpc.TraceStore/TraceStoreRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc.TraceStore/TraceStoreRpcModule.cs index b01b95b265d..d9267e28112 100644 --- a/src/Nethermind/Nethermind.JsonRpc.TraceStore/TraceStoreRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc.TraceStore/TraceStoreRpcModule.cs @@ -7,6 +7,7 @@ using Nethermind.Core.Crypto; using Nethermind.Db; using Nethermind.Evm.Tracing.ParityStyle; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.JsonRpc.Modules; using Nethermind.JsonRpc.Modules.Trace; diff --git a/src/Nethermind/Nethermind.JsonRpc/Client/BasicJsonRpcClient.cs b/src/Nethermind/Nethermind.JsonRpc/Client/BasicJsonRpcClient.cs index 9570d9a4e2d..a75fca723c2 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Client/BasicJsonRpcClient.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Client/BasicJsonRpcClient.cs @@ -14,19 +14,22 @@ namespace Nethermind.JsonRpc.Client { - public class BasicJsonRpcClient : IJsonRpcClient + public class BasicJsonRpcClient : IJsonRpcClient, IDisposable { private readonly HttpClient _client; private readonly IJsonSerializer _jsonSerializer; private readonly ILogger _logger; - public BasicJsonRpcClient(Uri uri, IJsonSerializer jsonSerializer, ILogManager logManager) + public BasicJsonRpcClient(Uri uri, IJsonSerializer jsonSerializer, ILogManager logManager) : + this(uri, jsonSerializer, logManager, /*support long block traces better, default 100s might be too small*/ TimeSpan.FromMinutes(5)) + { } + public BasicJsonRpcClient(Uri uri, IJsonSerializer jsonSerializer, ILogManager logManager, TimeSpan timeout) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _jsonSerializer = jsonSerializer; _client = new HttpClient { BaseAddress = uri }; - _client.Timeout = TimeSpan.FromMinutes(5); // support long block traces better, default 100s might be too small + _client.Timeout = timeout; _client.DefaultRequestHeaders.Accept.Clear(); _client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); AddAuthorizationHeader(); @@ -58,21 +61,15 @@ public async Task Post(string method, params object[] parameters) return jsonResponse.Result; } - catch (NotSupportedException) + catch (Exception e) when + ( + e is not TaskCanceledException && + e is not HttpRequestException && + e is not NotImplementedException && + e is not NotSupportedException + ) { - throw; - } - catch (NotImplementedException) - { - throw; - } - catch (HttpRequestException) - { - throw; - } - catch (Exception) - { - throw new DataException($"Cannot deserialize {responseString}"); + throw new DataException($"Cannot deserialize {responseString}", e); } } @@ -105,5 +102,10 @@ private void AddAuthorizationHeader() private static string Base64Encode(string plainText) => Convert.ToBase64String(Encoding.UTF8.GetBytes(plainText)); + + public virtual void Dispose() + { + _client.Dispose(); + } } } diff --git a/src/Nethermind/Nethermind.JsonRpc/Data/AccessListForRpc.cs b/src/Nethermind/Nethermind.JsonRpc/Data/AccessListForRpc.cs index 84e569a4965..7c21797de61 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Data/AccessListForRpc.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Data/AccessListForRpc.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Generic; +using Nethermind.Facade.Eth; using Nethermind.Int256; namespace Nethermind.JsonRpc.Data diff --git a/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpcWithTraceTypes.cs b/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpcWithTraceTypes.cs index 061f0b02f11..5f5c98acaf1 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpcWithTraceTypes.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Data/TransactionForRpcWithTraceTypes.cs @@ -4,6 +4,7 @@ using System; using System.Text.Json; using System.Text.Json.Serialization; +using Nethermind.Facade.Eth; namespace Nethermind.JsonRpc.Data { diff --git a/src/Nethermind/Nethermind.JsonRpc/ErrorType.cs b/src/Nethermind/Nethermind.JsonRpc/ErrorCodes.cs similarity index 72% rename from src/Nethermind/Nethermind.JsonRpc/ErrorType.cs rename to src/Nethermind/Nethermind.JsonRpc/ErrorCodes.cs index 5af200cf91b..3990b33b775 100644 --- a/src/Nethermind/Nethermind.JsonRpc/ErrorType.cs +++ b/src/Nethermind/Nethermind.JsonRpc/ErrorCodes.cs @@ -86,5 +86,30 @@ public static class ErrorCodes /// Unknown block error ///
public const int UnknownBlockError = -39001; + + /// + /// Invalid RPC simulate call block number out of order + /// + public const int InvalidInputBlocksOutOfOrder = -38020; + + /// + /// Invalid RPC simulate call Block timestamp in sequence did not increase + /// + public const int BlockTimestampNotIncreased = -38021; + + /// + /// Invalid RPC simulate call containing too many blocks + /// + public const int InvalidInputTooManyBlocks = -38026; + + /// + /// Invalid RPC simulate call Not enough gas provided to pay for intrinsic gas for a transaction + /// + public const int InsufficientIntrinsicGas = -38013; + + /// + /// Invalid RPC simulate call transaction + /// + public const int InvalidTransaction = -38014; } } diff --git a/src/Nethermind/Nethermind.JsonRpc/IJsonRpcConfig.cs b/src/Nethermind/Nethermind.JsonRpc/IJsonRpcConfig.cs index c20dc47e593..e52d9d1e265 100644 --- a/src/Nethermind/Nethermind.JsonRpc/IJsonRpcConfig.cs +++ b/src/Nethermind/Nethermind.JsonRpc/IJsonRpcConfig.cs @@ -106,6 +106,12 @@ public interface IJsonRpcConfig : IConfig [ConfigItem(Description = "The max length of HTTP request body, in bytes.", DefaultValue = "30000000")] long? MaxRequestBodySize { get; set; } + + [ConfigItem( + Description = "The max number of logs per response. For method `eth_getLogs`. If 0 then no limit.", + DefaultValue = "20000")] + public int MaxLogsPerResponse { get; set; } + [ConfigItem( Description = """ The number of concurrent instances for non-sharable calls: @@ -155,6 +161,9 @@ public interface IJsonRpcConfig : IConfig [ConfigItem(Description = "The max batch size limit for batched JSON-RPC calls.", DefaultValue = "33554432")] long? MaxBatchResponseBodySize { get; set; } + [ConfigItem(Description = "The max blocks count limit for eth_simulate JSON-RPC calls.", DefaultValue = "256")] + long? MaxSimulateBlocksCap { get; set; } + [ConfigItem(Description = "The error margin used in eth_estimateGas expressed in basis points.", DefaultValue = "150")] int EstimateErrorMargin { get; set; } } diff --git a/src/Nethermind/Nethermind.JsonRpc/JsonRpcConfig.cs b/src/Nethermind/Nethermind.JsonRpc/JsonRpcConfig.cs index 1e68d3dbc52..26d3d32760e 100644 --- a/src/Nethermind/Nethermind.JsonRpc/JsonRpcConfig.cs +++ b/src/Nethermind/Nethermind.JsonRpc/JsonRpcConfig.cs @@ -37,6 +37,7 @@ public int WebSocketsPort public bool BufferResponses { get; set; } public string CallsFilterFilePath { get; set; } = "Data/jsonrpc.filter"; public long? MaxRequestBodySize { get; set; } = 30000000; + public int MaxLogsPerResponse { get; set; } = 20_000; public int? EthModuleConcurrentInstances { get; set; } = null; public string JwtSecretFile { get; set; } = "keystore/jwt-secret"; public bool UnsecureDevNoRpcAuthentication { get; set; } @@ -54,6 +55,7 @@ public int WebSocketsPort public string[] EngineEnabledModules { get; set; } = ModuleType.DefaultEngineModules.ToArray(); public int MaxBatchSize { get; set; } = 1024; public long? MaxBatchResponseBodySize { get; set; } = 32.MiB(); + public long? MaxSimulateBlocksCap { get; set; } = 256; public int EstimateErrorMargin { get; set; } = 150; }; }; diff --git a/src/Nethermind/Nethermind.JsonRpc/JsonRpcContext.cs b/src/Nethermind/Nethermind.JsonRpc/JsonRpcContext.cs index 9f130a4bee6..c145607924d 100644 --- a/src/Nethermind/Nethermind.JsonRpc/JsonRpcContext.cs +++ b/src/Nethermind/Nethermind.JsonRpc/JsonRpcContext.cs @@ -1,12 +1,16 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; +using System.Threading; using Nethermind.JsonRpc.Modules; namespace Nethermind.JsonRpc { - public class JsonRpcContext + public class JsonRpcContext : IDisposable { + public static AsyncLocal Current { get; private set; } = new(); + public static JsonRpcContext Http(JsonRpcUrl url) => new(RpcEndpoint.Http, url: url); public static JsonRpcContext WebSocket(JsonRpcUrl url) => new(RpcEndpoint.Ws, url: url); @@ -16,11 +20,19 @@ public JsonRpcContext(RpcEndpoint rpcEndpoint, IJsonRpcDuplexClient? duplexClien DuplexClient = duplexClient; Url = url; IsAuthenticated = Url?.IsAuthenticated == true || RpcEndpoint == RpcEndpoint.IPC; + Current.Value = this; } public RpcEndpoint RpcEndpoint { get; } public IJsonRpcDuplexClient? DuplexClient { get; } public JsonRpcUrl? Url { get; } public bool IsAuthenticated { get; } + public void Dispose() + { + if (Current.Value == this) + { + Current.Value = null; + } + } } } diff --git a/src/Nethermind/Nethermind.JsonRpc/JsonRpcProcessor.cs b/src/Nethermind/Nethermind.JsonRpc/JsonRpcProcessor.cs index 99f81d57751..bcc2a257cab 100644 --- a/src/Nethermind/Nethermind.JsonRpc/JsonRpcProcessor.cs +++ b/src/Nethermind/Nethermind.JsonRpc/JsonRpcProcessor.cs @@ -241,6 +241,8 @@ JsonRpcResult GetParsingError(string error, Exception? exception = null) yield return deserializationFailureResult.Value; break; } + + buffer = buffer.TrimStart(); } // Checks if the deserialization failed diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugBridge.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugBridge.cs index 788ce98e957..137939bcf6a 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugBridge.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugBridge.cs @@ -177,7 +177,7 @@ public IReadOnlyCollection GetBlockTrace(Rlp blockRlp, Cancella public byte[] GetBlockRlp(Hash256 blockHash) { BlockHeader? header = _blockTree.FindHeader(blockHash); - if (header == null) return null; + if (header is null) return null; return _blockStore.GetRaw(header.Number, blockHash); } diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugModuleFactory.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugModuleFactory.cs index 925d8e98bb4..3716de73a7d 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugModuleFactory.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugModuleFactory.cs @@ -12,6 +12,7 @@ using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Tracing; using Nethermind.Consensus.Validators; +using Nethermind.Core.Crypto; using Nethermind.Core.Specs; using Nethermind.Db; using Nethermind.Evm.TransactionProcessing; @@ -84,21 +85,25 @@ public override IDebugRpcModule Create() _specProvider, _logManager); - ChangeableTransactionProcessorAdapter transactionProcessorAdapter = new(txEnv.TransactionProcessor); - BlockProcessor.BlockValidationTransactionsExecutor transactionsExecutor = new(transactionProcessorAdapter, txEnv.StateProvider); + IReadOnlyTxProcessingScope scope = txEnv.Build(Keccak.EmptyTreeHash); + + ChangeableTransactionProcessorAdapter transactionProcessorAdapter = new(scope.TransactionProcessor); + BlockProcessor.BlockValidationTransactionsExecutor transactionsExecutor = new(transactionProcessorAdapter, scope.WorldState); ReadOnlyChainProcessingEnv chainProcessingEnv = new( - txEnv, + scope, _blockValidator, _recoveryStep, - _rewardCalculatorSource.Get(txEnv.TransactionProcessor), + _rewardCalculatorSource.Get(scope.TransactionProcessor), _receiptStorage, _specProvider, + _blockTree, + _worldStateManager.GlobalStateReader, _logManager, transactionsExecutor); GethStyleTracer tracer = new( chainProcessingEnv.ChainProcessor, - chainProcessingEnv.StateProvider, + scope.WorldState, _receiptStorage, _blockTree, _specProvider, diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugRpcModule.cs index 2c8813c7dbd..2de6e774830 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugRpcModule.cs @@ -18,6 +18,7 @@ using System.Collections.Generic; using Nethermind.JsonRpc.Modules.Eth; using Nethermind.Core.Specs; +using Nethermind.Facade.Eth; namespace Nethermind.JsonRpc.Modules.DebugModule; diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/IDebugRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/IDebugRpcModule.cs index 4a057892b70..6615e53f408 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/IDebugRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/IDebugRpcModule.cs @@ -6,6 +6,7 @@ using Nethermind.Blockchain.Find; using Nethermind.Core.Crypto; using Nethermind.Evm.Tracing.GethStyle; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.JsonRpc.Modules.Eth; using Nethermind.Synchronization.Reporting; diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/BadBlock.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/BadBlock.cs index 3844a4a20ea..de2af695562 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/BadBlock.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/BadBlock.cs @@ -5,6 +5,7 @@ using Nethermind.Core; using Nethermind.Serialization.Rlp; using Nethermind.Core.Crypto; +using Nethermind.Facade.Eth; namespace Nethermind.JsonRpc.Modules.Eth; diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthModuleFactory.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthModuleFactory.cs index 7f174d89081..6a8bc34d9b7 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthModuleFactory.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthModuleFactory.cs @@ -8,7 +8,6 @@ using Nethermind.Core.Specs; using Nethermind.Facade; using Nethermind.Facade.Eth; -using Nethermind.JsonRpc.Data; using Nethermind.JsonRpc.Modules.Eth.GasPrice; using Nethermind.JsonRpc.Modules.Eth.FeeHistory; using Nethermind.Logging; @@ -17,6 +16,7 @@ using Nethermind.Wallet; using System.Text.Json; using System.Text.Json.Serialization; +using Nethermind.Config; namespace Nethermind.JsonRpc.Modules.Eth { @@ -33,7 +33,8 @@ public class EthModuleFactory( IReceiptStorage receiptStorage, IGasPriceOracle gasPriceOracle, IEthSyncingInfo ethSyncingInfo, - IFeeHistoryOracle feeHistoryOracle) + IFeeHistoryOracle feeHistoryOracle, + ulong secondsPerSlot) : ModuleFactoryBase { private readonly IReadOnlyBlockTree _blockTree = blockTree.AsReadOnly(); @@ -65,7 +66,8 @@ public override IEthRpcModule Create() _specProvider, _gasPriceOracle, _ethSyncingInfo, - _feeHistoryOracle); + _feeHistoryOracle, + secondsPerSlot); } } } diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.TransactionExecutor.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.TransactionExecutor.cs index afc281f7f6d..6da17831925 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.TransactionExecutor.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.TransactionExecutor.cs @@ -11,110 +11,81 @@ using Nethermind.Core.Extensions; using Nethermind.Evm; using Nethermind.Facade; +using Nethermind.Facade.Eth; using Nethermind.Int256; using Nethermind.JsonRpc.Data; using Nethermind.Specs.Forks; namespace Nethermind.JsonRpc.Modules.Eth { + //General executor public partial class EthRpcModule { - private abstract class TxExecutor + // Single call executor + private abstract class TxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig) + : ExecutorBase(blockchainBridge, blockFinder, rpcConfig) { - protected readonly IBlockchainBridge _blockchainBridge; - private readonly IBlockFinder _blockFinder; - private readonly IJsonRpcConfig _rpcConfig; + private bool NoBaseFee { get; set; } - protected TxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig) - { - _blockchainBridge = blockchainBridge; - _blockFinder = blockFinder; - _rpcConfig = rpcConfig; - } + protected override Transaction Prepare(TransactionForRpc call) => call.ToTransaction(_blockchainBridge.GetChainId()); - public ResultWrapper ExecuteTx( - TransactionForRpc transactionCall, - BlockParameter? blockParameter) + protected override ResultWrapper Execute(BlockHeader header, Transaction tx, CancellationToken token) { - SearchResult searchResult = _blockFinder.SearchForHeader(blockParameter); - if (searchResult.IsError) - { - return ResultWrapper.Fail(searchResult); - } - - BlockHeader header = searchResult.Object; - if (!HasStateForBlock(_blockchainBridge, header)) - { - return ResultWrapper.Fail($"No state available for block {header.Hash}", - ErrorCodes.ResourceUnavailable); - } - BlockHeader clonedHeader = header.Clone(); - var noBaseFee = !ShouldSetBaseFee(transactionCall); - if (noBaseFee) + if (NoBaseFee) { clonedHeader.BaseFeePerGas = 0; } - - transactionCall.EnsureDefaults(_rpcConfig.GasCap); - - using CancellationTokenSource cancellationTokenSource = new(_rpcConfig.Timeout); - Transaction tx = transactionCall.ToTransaction(_blockchainBridge.GetChainId()); if (tx.IsContractCreation && tx.DataLength == 0) { - return ResultWrapper.Fail("Contract creation without any data provided.", - ErrorCodes.InvalidInput); + return ResultWrapper.Fail("Contract creation without any data provided.", ErrorCodes.InvalidInput); } - - return ExecuteTx(clonedHeader, tx, cancellationTokenSource.Token); + return ExecuteTx(clonedHeader, tx, token); } - private static bool ShouldSetBaseFee(TransactionForRpc t) + private static bool ShouldSetBaseFee(TransactionForRpc t) => + // x?.IsZero == false <=> x > 0 + t.GasPrice?.IsZero == false || t.MaxFeePerGas?.IsZero == false || t.MaxPriorityFeePerGas?.IsZero == false; + + public override ResultWrapper Execute( + TransactionForRpc transactionCall, + BlockParameter? blockParameter) { - return - t.GasPrice > 0 || t.MaxFeePerGas > 0 || t.MaxPriorityFeePerGas > 0; + NoBaseFee = !ShouldSetBaseFee(transactionCall); + transactionCall.EnsureDefaults(_rpcConfig.GasCap); + return base.Execute(transactionCall, blockParameter); } + public ResultWrapper ExecuteTx(TransactionForRpc transactionCall, BlockParameter? blockParameter) => Execute(transactionCall, blockParameter); + protected abstract ResultWrapper ExecuteTx(BlockHeader header, Transaction tx, CancellationToken token); - protected static ResultWrapper GetInputError(BlockchainBridge.CallOutput result) => + protected ResultWrapper GetInputError(CallOutput result) => ResultWrapper.Fail(result.Error, ErrorCodes.InvalidInput); } - private class CallTxExecutor : TxExecutor + private class CallTxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig) + : TxExecutor(blockchainBridge, blockFinder, rpcConfig) { - public CallTxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig) - : base(blockchainBridge, blockFinder, rpcConfig) - { - } - protected override ResultWrapper ExecuteTx(BlockHeader header, Transaction tx, CancellationToken token) { - BlockchainBridge.CallOutput result = _blockchainBridge.Call(header, tx, token); + CallOutput result = _blockchainBridge.Call(header, tx, token); - if (result.Error is null) - { - return ResultWrapper.Success(result.OutputData.ToHexString(true)); - } - - return result.InputError - ? GetInputError(result) - : ResultWrapper.Fail("VM execution error.", ErrorCodes.ExecutionError, result.Error); + return result.Error is null + ? ResultWrapper.Success(result.OutputData.ToHexString(true)) + : TryGetInputError(result) ?? ResultWrapper.Fail("VM execution error.", ErrorCodes.ExecutionError, result.Error); } + } - private class EstimateGasTxExecutor : TxExecutor + private class EstimateGasTxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig) + : TxExecutor(blockchainBridge, blockFinder, rpcConfig) { - private readonly int _errorMargin; - public EstimateGasTxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig) - : base(blockchainBridge, blockFinder, rpcConfig) - { - _errorMargin = rpcConfig.EstimateErrorMargin; - } + private readonly int _errorMargin = rpcConfig.EstimateErrorMargin; protected override ResultWrapper ExecuteTx(BlockHeader header, Transaction tx, CancellationToken token) { - BlockchainBridge.CallOutput result = _blockchainBridge.EstimateGas(header, tx, _errorMargin, token); + CallOutput result = _blockchainBridge.EstimateGas(header, tx, _errorMargin, token); if (result.Error is null) { @@ -127,19 +98,12 @@ public EstimateGasTxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder bl } } - private class CreateAccessListTxExecutor : TxExecutor + private class CreateAccessListTxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig, bool optimize) + : TxExecutor(blockchainBridge, blockFinder, rpcConfig) { - private readonly bool _optimize; - - public CreateAccessListTxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig, bool optimize) - : base(blockchainBridge, blockFinder, rpcConfig) - { - _optimize = optimize; - } - protected override ResultWrapper ExecuteTx(BlockHeader header, Transaction tx, CancellationToken token) { - BlockchainBridge.CallOutput result = _blockchainBridge.CreateAccessList(header, tx, token, _optimize); + CallOutput result = _blockchainBridge.CreateAccessList(header, tx, token, optimize); if (result.Error is null) { @@ -151,13 +115,13 @@ public CreateAccessListTxExecutor(IBlockchainBridge blockchainBridge, IBlockFind : ResultWrapper.Fail(result.Error, ErrorCodes.ExecutionError, new AccessListForRpc(GetResultAccessList(tx, result), GetResultGas(tx, result))); } - private static IEnumerable GetResultAccessList(Transaction tx, BlockchainBridge.CallOutput result) + private static IEnumerable GetResultAccessList(Transaction tx, CallOutput result) { AccessList? accessList = result.AccessList ?? tx.AccessList; return accessList is null ? Enumerable.Empty() : AccessListItemForRpc.FromAccessList(accessList); } - private static UInt256 GetResultGas(Transaction transaction, BlockchainBridge.CallOutput result) + private static UInt256 GetResultGas(Transaction transaction, CallOutput result) { long gas = result.GasSpent; if (result.AccessList is not null) diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs index 290f3c22ab8..6abec26637d 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs @@ -12,6 +12,7 @@ using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; using Nethermind.Core; +using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Core.Specs; @@ -19,6 +20,7 @@ using Nethermind.Facade; using Nethermind.Facade.Eth; using Nethermind.Facade.Filters; +using Nethermind.Facade.Proxy.Models.Simulate; using Nethermind.Int256; using Nethermind.JsonRpc.Data; using Nethermind.JsonRpc.Modules.Eth.FeeHistory; @@ -38,58 +40,43 @@ namespace Nethermind.JsonRpc.Modules.Eth; -public partial class EthRpcModule : IEthRpcModule +public partial class EthRpcModule( + IJsonRpcConfig rpcConfig, + IBlockchainBridge blockchainBridge, + IBlockFinder blockFinder, + IReceiptFinder receiptFinder, + IStateReader stateReader, + ITxPool txPool, + ITxSender txSender, + IWallet wallet, + ILogManager logManager, + ISpecProvider specProvider, + IGasPriceOracle gasPriceOracle, + IEthSyncingInfo ethSyncingInfo, + IFeeHistoryOracle feeHistoryOracle, + ulong? secondsPerSlot) : IEthRpcModule { - private readonly Encoding _messageEncoding = Encoding.UTF8; - private readonly IJsonRpcConfig _rpcConfig; - private readonly IBlockchainBridge _blockchainBridge; - private readonly IBlockFinder _blockFinder; - private readonly IReceiptFinder _receiptFinder; - private readonly IStateReader _stateReader; - private readonly ITxPool _txPoolBridge; - private readonly ITxSender _txSender; - private readonly IWallet _wallet; - private readonly ISpecProvider _specProvider; - private readonly ILogger _logger; - private readonly IGasPriceOracle _gasPriceOracle; - private readonly IEthSyncingInfo _ethSyncingInfo; - - private readonly IFeeHistoryOracle _feeHistoryOracle; + protected readonly Encoding _messageEncoding = Encoding.UTF8; + protected readonly IJsonRpcConfig _rpcConfig = rpcConfig ?? throw new ArgumentNullException(nameof(rpcConfig)); + protected readonly IBlockchainBridge _blockchainBridge = blockchainBridge ?? throw new ArgumentNullException(nameof(blockchainBridge)); + protected readonly IBlockFinder _blockFinder = blockFinder ?? throw new ArgumentNullException(nameof(blockFinder)); + protected readonly IReceiptFinder _receiptFinder = receiptFinder ?? throw new ArgumentNullException(nameof(receiptFinder)); + protected readonly IStateReader _stateReader = stateReader ?? throw new ArgumentNullException(nameof(stateReader)); + protected readonly ITxPool _txPoolBridge = txPool ?? throw new ArgumentNullException(nameof(txPool)); + protected readonly ITxSender _txSender = txSender ?? throw new ArgumentNullException(nameof(txSender)); + protected readonly IWallet _wallet = wallet ?? throw new ArgumentNullException(nameof(wallet)); + protected readonly ISpecProvider _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); + protected readonly ILogger _logger = logManager.GetClassLogger(); + protected readonly IGasPriceOracle _gasPriceOracle = gasPriceOracle ?? throw new ArgumentNullException(nameof(gasPriceOracle)); + protected readonly IEthSyncingInfo _ethSyncingInfo = ethSyncingInfo ?? throw new ArgumentNullException(nameof(ethSyncingInfo)); + protected readonly IFeeHistoryOracle _feeHistoryOracle = feeHistoryOracle ?? throw new ArgumentNullException(nameof(feeHistoryOracle)); + protected readonly ulong _secondsPerSlot = secondsPerSlot ?? throw new ArgumentNullException(nameof(secondsPerSlot)); + private static bool HasStateForBlock(IBlockchainBridge blockchainBridge, BlockHeader header) { return blockchainBridge.HasStateForRoot(header.StateRoot!); } - public EthRpcModule( - IJsonRpcConfig rpcConfig, - IBlockchainBridge blockchainBridge, - IBlockFinder blockFinder, - IReceiptFinder receiptFinder, - IStateReader stateReader, - ITxPool txPool, - ITxSender txSender, - IWallet wallet, - ILogManager logManager, - ISpecProvider specProvider, - IGasPriceOracle gasPriceOracle, - IEthSyncingInfo ethSyncingInfo, - IFeeHistoryOracle feeHistoryOracle) - { - _logger = logManager.GetClassLogger(); - _rpcConfig = rpcConfig ?? throw new ArgumentNullException(nameof(rpcConfig)); - _blockchainBridge = blockchainBridge ?? throw new ArgumentNullException(nameof(blockchainBridge)); - _blockFinder = blockFinder ?? throw new ArgumentNullException(nameof(blockFinder)); - _receiptFinder = receiptFinder ?? throw new ArgumentNullException(nameof(receiptFinder)); - _stateReader = stateReader ?? throw new ArgumentNullException(nameof(stateReader)); - _txPoolBridge = txPool ?? throw new ArgumentNullException(nameof(txPool)); - _txSender = txSender ?? throw new ArgumentNullException(nameof(txSender)); - _wallet = wallet ?? throw new ArgumentNullException(nameof(wallet)); - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); - _gasPriceOracle = gasPriceOracle ?? throw new ArgumentNullException(nameof(gasPriceOracle)); - _ethSyncingInfo = ethSyncingInfo ?? throw new ArgumentNullException(nameof(ethSyncingInfo)); - _feeHistoryOracle = feeHistoryOracle ?? throw new ArgumentNullException(nameof(feeHistoryOracle)); - } - public ResultWrapper eth_protocolVersion() { int highestVersion = P2PProtocolInfoProvider.GetHighestVersionOfEthProtocol(); @@ -193,7 +180,7 @@ public ResultWrapper eth_getStorageAt(Address address, UInt256 positionI return ResultWrapper.Success(storage.IsEmpty ? Bytes32.Zero.Unwrap() : storage!.PadLeft(32)); } - public Task> eth_getTransactionCount(Address address, BlockParameter blockParameter) + public Task> eth_getTransactionCount(Address address, BlockParameter? blockParameter) { if (blockParameter == BlockParameter.Pending) { @@ -233,11 +220,6 @@ public Task> eth_getTransactionCount(Address address, Blo : ResultWrapper.Success((UInt256)searchResult.Object!.Transactions.Length); } - public ResultWrapper eth_getBlockReceipts(BlockParameter blockParameter) - { - return _receiptFinder.GetBlockReceipts(blockParameter, _blockFinder, _specProvider); - } - public ResultWrapper eth_getUncleCountByBlockHash(Hash256 blockHash) { SearchResult searchResult = _blockFinder.SearchForBlock(new BlockParameter(blockHash)); @@ -295,14 +277,14 @@ public ResultWrapper eth_sign(Address addressData, byte[] message) return ResultWrapper.Success(sig.Bytes); } - public Task> eth_sendTransaction(TransactionForRpc rpcTx) + public virtual Task> eth_sendTransaction(TransactionForRpc rpcTx) { Transaction tx = rpcTx.ToTransactionWithDefaults(_blockchainBridge.GetChainId()); TxHandlingOptions options = rpcTx.Nonce is null ? TxHandlingOptions.ManagedNonce : TxHandlingOptions.None; return SendTx(tx, options); } - public async Task> eth_sendRawTransaction(byte[] transaction) + public virtual async Task> eth_sendRawTransaction(byte[] transaction) { try { @@ -343,7 +325,11 @@ public ResultWrapper eth_call(TransactionForRpc transactionCall, BlockPa new CallTxExecutor(_blockchainBridge, _blockFinder, _rpcConfig) .ExecuteTx(transactionCall, blockParameter); - public ResultWrapper eth_estimateGas(TransactionForRpc transactionCall, BlockParameter blockParameter) => + public ResultWrapper> eth_simulateV1(SimulatePayload payload, BlockParameter? blockParameter = null) => + new SimulateTxExecutor(_blockchainBridge, _blockFinder, _rpcConfig, _secondsPerSlot) + .Execute(payload, blockParameter); + + public ResultWrapper eth_estimateGas(TransactionForRpc transactionCall, BlockParameter? blockParameter) => new EstimateGasTxExecutor(_blockchainBridge, _blockFinder, _rpcConfig) .ExecuteTx(transactionCall, blockParameter); @@ -468,18 +454,6 @@ public ResultWrapper eth_getTransactionByBlockNumberAndIndex( return ResultWrapper.Success(transactionModel); } - public Task> eth_getTransactionReceipt(Hash256 txHash) - { - (TxReceipt? receipt, TxGasInfo? gasInfo, int logIndexStart) = _blockchainBridge.GetReceiptAndGasInfo(txHash); - if (receipt is null || gasInfo is null) - { - return Task.FromResult(ResultWrapper.Success(null)); - } - - if (_logger.IsTrace) _logger.Trace($"eth_getTransactionReceipt request {txHash}, result: {txHash}"); - return Task.FromResult(ResultWrapper.Success(new(txHash, receipt, gasInfo.Value, logIndexStart))); - } - public ResultWrapper eth_getUncleByBlockHashAndIndex(Hash256 blockHash, UInt256 positionIndex) { return GetUncle(new BlockParameter(blockHash), positionIndex); @@ -642,11 +616,28 @@ public ResultWrapper> eth_getLogs(Filter filter) try { - LogFilter logFilter = _blockchainBridge.GetFilter(filter.FromBlock, filter.ToBlock, - filter.Address, filter.Topics); + LogFilter logFilter = _blockchainBridge.GetFilter(filter.FromBlock, filter.ToBlock, filter.Address, filter.Topics); + IEnumerable filterLogs = _blockchainBridge.GetLogs(logFilter, fromBlock, toBlock, cancellationToken); - return ResultWrapper>.Success(GetLogs(filterLogs, cancellationTokenSource)); + ArrayPoolList logs = new(_rpcConfig.MaxLogsPerResponse); + + using (cancellationTokenSource) + { + foreach (FilterLog log in filterLogs) + { + logs.Add(log); + if (JsonRpcContext.Current.Value?.IsAuthenticated != true // not authenticated + && _rpcConfig.MaxLogsPerResponse != 0 // not unlimited + && logs.Count > _rpcConfig.MaxLogsPerResponse) + { + logs.Dispose(); + return ResultWrapper>.Fail($"Too many logs requested. Max logs per response is {_rpcConfig.MaxLogsPerResponse}.", ErrorCodes.LimitExceeded); + } + } + } + + return ResultWrapper>.Success(logs); } catch (ResourceNotFoundException exception) { @@ -730,4 +721,22 @@ private static ResultWrapper GetFailureResult(ResourceNotFound private ResultWrapper GetStateFailureResult(BlockHeader header) => ResultWrapper.Fail($"No state available for block {header.ToString(BlockHeader.Format.FullHashAndNumber)}", ErrorCodes.ResourceUnavailable, _ethSyncingInfo.SyncMode.HaveNotSyncedStateYet()); + + public ResultWrapper eth_getTransactionReceipt(Hash256 txHash) + { + (TxReceipt? receipt, TxGasInfo? gasInfo, int logIndexStart) = _blockchainBridge.GetReceiptAndGasInfo(txHash); + if (receipt is null || gasInfo is null) + { + return ResultWrapper.Success(null); + } + + if (_logger.IsTrace) _logger.Trace($"eth_getTransactionReceipt request {txHash}, result: {txHash}"); + return ResultWrapper.Success(new(txHash, receipt, gasInfo.Value, logIndexStart)); + } + + + public ResultWrapper eth_getBlockReceipts(BlockParameter blockParameter) + { + return _receiptFinder.GetBlockReceipts(blockParameter, _blockFinder, _specProvider); + } } diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/ExecutorBase.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/ExecutorBase.cs new file mode 100644 index 00000000000..2e04d10b455 --- /dev/null +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/ExecutorBase.cs @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Threading; +using Nethermind.Blockchain.Find; +using Nethermind.Core; +using Nethermind.Facade; + +namespace Nethermind.JsonRpc.Modules.Eth; + +public abstract class ExecutorBase +{ + protected readonly IBlockchainBridge _blockchainBridge; + protected readonly IBlockFinder _blockFinder; + protected readonly IJsonRpcConfig _rpcConfig; + + protected ExecutorBase(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig) + { + _blockchainBridge = blockchainBridge; + _blockFinder = blockFinder; + _rpcConfig = rpcConfig; + } + + public virtual ResultWrapper Execute( + TRequest call, + BlockParameter? blockParameter) + { + SearchResult searchResult = _blockFinder.SearchForHeader(blockParameter); + if (searchResult.IsError) return ResultWrapper.Fail(searchResult); + + BlockHeader header = searchResult.Object; + if (!_blockchainBridge.HasStateForBlock(header!)) + return ResultWrapper.Fail($"No state available for block {header.Hash}", + ErrorCodes.ResourceUnavailable); + + using CancellationTokenSource cancellationTokenSource = new(_rpcConfig.Timeout); + TProcessing? toProcess = Prepare(call); + return Execute(header.Clone(), toProcess, cancellationTokenSource.Token); + } + + protected abstract TProcessing Prepare(TRequest call); + + protected abstract ResultWrapper Execute(BlockHeader header, TProcessing tx, CancellationToken token); + + protected ResultWrapper? TryGetInputError(CallOutput result) + { + return result.InputError ? ResultWrapper.Fail(result.Error!, ErrorCodes.InvalidInput) : null; + } +} diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/FeeHistory/FeeHistoryOracle.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/FeeHistory/FeeHistoryOracle.cs index f1ee8593978..6d319ab4f9c 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/FeeHistory/FeeHistoryOracle.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/FeeHistory/FeeHistoryOracle.cs @@ -52,6 +52,7 @@ private void OnBlockAddedToMain(object? sender, BlockReplacementEventArgs e) if (ShouldCache(e.Block)) { SaveHistorySearchInfo(e.Block); + TryRunCleanup(); } }); } @@ -190,6 +191,14 @@ public ResultWrapper GetFeeHistory( historyInfo = info.ParentHash is null ? null : GetHistorySearchInfo(info.ParentHash, info.BlockNumber - 1); } + TryRunCleanup(); + + return ResultWrapper.Success(new(oldestBlockNumber, baseFeePerGas, + gasUsedRatio, baseFeePerBlobGas, blobGasUsedRatio, rewards)); + } + + private void TryRunCleanup() + { long headNumber = _blockTree.Head?.Number ?? 0; long lastCleanupHeadBlockNumber = _lastCleanupHeadBlockNumber; if (lastCleanupHeadBlockNumber != headNumber @@ -199,9 +208,6 @@ public ResultWrapper GetFeeHistory( { _cleanupTask = Task.Run(CleanupCache); } - - return ResultWrapper.Success(new(oldestBlockNumber, baseFeePerGas, - gasUsedRatio, baseFeePerBlobGas, blobGasUsedRatio, rewards)); } private void CleanupCache() diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/IEthRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/IEthRpcModule.cs index c02596d76db..89c52debbda 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/IEthRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/IEthRpcModule.cs @@ -8,6 +8,9 @@ using Nethermind.Core.Crypto; using Nethermind.Facade.Eth; using Nethermind.Facade.Filters; +using Nethermind.Facade.Proxy.Models.Simulate; +using Nethermind.Facade.Proxy.Models; +using Nethermind.Facade.Proxy; using Nethermind.Int256; using Nethermind.JsonRpc.Data; using Nethermind.State.Proofs; @@ -81,19 +84,19 @@ public interface IEthRpcModule : IRpcModule Description = "Returns account balance", IsSharable = true, ExampleResponse = "0x6c8ae945bfe6e")] - Task> eth_getBalance([JsonRpcParameter(ExampleValue = "[\"0x78467cada5f1883e79fcf0f3ebfa50abeec8c820\"]")] Address address, BlockParameter blockParameter = null); + Task> eth_getBalance([JsonRpcParameter(ExampleValue = "[\"0x78467cada5f1883e79fcf0f3ebfa50abeec8c820\"]")] Address address, BlockParameter? blockParameter = null); [JsonRpcMethod(IsImplemented = true, Description = "Returns storage data at address. storage_index", IsSharable = true, ExampleResponse = "0x")] - ResultWrapper eth_getStorageAt([JsonRpcParameter(ExampleValue = "[\"0x000000000000000000000000c666d239cbda32aa7ebca894b6dc598ddb881285\",\"0x2\"]")] Address address, UInt256 positionIndex, BlockParameter blockParameter = null); + ResultWrapper eth_getStorageAt([JsonRpcParameter(ExampleValue = "[\"0x000000000000000000000000c666d239cbda32aa7ebca894b6dc598ddb881285\",\"0x2\"]")] Address address, UInt256 positionIndex, BlockParameter? blockParameter = null); [JsonRpcMethod(IsImplemented = true, Description = "Returns account nonce (number of trnsactions from the account since genesis) at the given block number", IsSharable = true, ExampleResponse = "0x3e")] - Task> eth_getTransactionCount([JsonRpcParameter(ExampleValue = "[\"0xae3ed7a6ccdddf2914133d0669b5f02ff6fa8ad2\"]")] Address address, BlockParameter blockParameter = null); + Task> eth_getTransactionCount([JsonRpcParameter(ExampleValue = "[\"0xae3ed7a6ccdddf2914133d0669b5f02ff6fa8ad2\"]")] Address address, BlockParameter? blockParameter = null); [JsonRpcMethod(IsImplemented = true, Description = "Returns number of transactions in the block block hash", @@ -111,7 +114,7 @@ public interface IEthRpcModule : IRpcModule [JsonRpcMethod(Description = "Get receipts from all transactions from particular block, more efficient than fetching the receipts one-by-one.", IsImplemented = true, ExampleResponse = "{\"jsonrpc\":\"2.0\",\"result\":[{\"transactionHash\":\"0x681c2b6f99e37fd6fe6046db8b51ec3460d699cacd6a376143fd5842ac50621f\",\"transactionIndex\":\"0x0\",\"blockHash\":\"0x29f141925d2d8e357ae5b6040c97aa12d7ac6dfcbe2b20e7b616d8907ac8e1f3\",\"blockNumber\":\"0x3\",\"cumulativeGasUsed\":\"0x5208\",\"gasUsed\":\"0x5208\",\"effectiveGasPrice\":\"0x1\",\"from\":\"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099\",\"to\":\"0x942921b14f1b1c385cd7e0cc2ef7abe5598c8358\",\"contractAddress\":null,\"logs\":[],\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"status\":\"0x1\",\"type\":\"0x0\"},{\"transactionHash\":\"0x7126cf20a0ad8bd51634837d9049615c34c1bff5e1a54e5663f7e23109bff48b\",\"transactionIndex\":\"0x1\",\"blockHash\":\"0x29f141925d2d8e357ae5b6040c97aa12d7ac6dfcbe2b20e7b616d8907ac8e1f3\",\"blockNumber\":\"0x3\",\"cumulativeGasUsed\":\"0xa410\",\"gasUsed\":\"0x5208\",\"effectiveGasPrice\":\"0x1\",\"from\":\"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099\",\"to\":\"0x942921b14f1b1c385cd7e0cc2ef7abe5598c8358\",\"contractAddress\":null,\"logs\":[],\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"status\":\"0x1\",\"type\":\"0x0\"}],\"id\":67}")] - ResultWrapper eth_getBlockReceipts([JsonRpcParameter(ExampleValue = "latest")] BlockParameter blockParameter); + ResultWrapper eth_getBlockReceipts([JsonRpcParameter(ExampleValue = "latest")] BlockParameter blockParameter); [JsonRpcMethod(IsImplemented = true, Description = "Returns number of uncles in the block by block hash", @@ -126,7 +129,7 @@ public interface IEthRpcModule : IRpcModule ResultWrapper eth_getUncleCountByBlockNumber([JsonRpcParameter(ExampleValue = "[\"5127400\"]")] BlockParameter blockParameter); [JsonRpcMethod(IsImplemented = true, Description = "Returns account code at given address and block", IsSharable = true)] - ResultWrapper eth_getCode(Address address, BlockParameter blockParameter = null); + ResultWrapper eth_getCode(Address address, BlockParameter? blockParameter = null); [JsonRpcMethod(IsImplemented = false, Description = "Signs a transaction", IsSharable = true)] ResultWrapper eth_sign(Address addressData, byte[] message); @@ -150,6 +153,13 @@ public interface IEthRpcModule : IRpcModule ExampleResponse = "0x")] ResultWrapper eth_call([JsonRpcParameter(ExampleValue = "[{\"from\":\"0x0001020304050607080910111213141516171819\",\"gasPrice\":\"0x100000\", \"data\": \"0x70a082310000000000000000000000006c1f09f6271fbe133db38db9c9280307f5d22160\", \"to\": \"0x0d8775f648430679a709e98d2b0cb6250d2887ef\"}]")] TransactionForRpc transactionCall, BlockParameter? blockParameter = null); + [JsonRpcMethod(IsImplemented = true, + Description = "Executes a simulation across multiple blocks (does not create a transaction or block)", + IsSharable = false, + ExampleResponse = "0x")] + ResultWrapper> eth_simulateV1([JsonRpcParameter(ExampleValue = "{\"blockStateCalls\":[{\"stateOverrides\":{\"0x0000000000000000000000000000000000000001\":{\"code\":\"0x608060405234801561001057600080fd5b506004361061003a5760003560e01c806305fdbc81146101ee578063c00692601461020a5761003b565b5b600036606060008060008086868101906100559190610462565b93509350935093506000806000868686866040516020016100799493929190610520565b60405160208183030381529060405280519060200120815260200190815260200160002060009054906101000a900473ffffffffffffffffffffffffffffffffffffffff169050600073ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff16036101bb576000806212345673ffffffffffffffffffffffffffffffffffffffff166127108b8b6040516101249291906105ad565b60006040518083038160008787f1925050503d8060008114610162576040519150601f19603f3d011682016040523d82523d6000602084013e610167565b606091505b5091509150816101ac576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004016101a39061066f565b60405180910390fd5b809750505050505050506101e3565b806040516020016101cc9190610709565b604051602081830303815290604052955050505050505b915050805190602001f35b6102086004803603810190610203919061093a565b610226565b005b610224600480360381019061021f9190610983565b6102ec565b005b60005b81518110156102e8576102d5828281518110610248576102476109fe565b5b602002602001015160000151838381518110610267576102666109fe565b5b602002602001015160200151848481518110610286576102856109fe565b5b6020026020010151604001518585815181106102a5576102a46109fe565b5b6020026020010151606001518686815181106102c4576102c36109fe565b5b6020026020010151608001516102ec565b80806102e090610a66565b915050610229565b5050565b600073ffffffffffffffffffffffffffffffffffffffff168173ffffffffffffffffffffffffffffffffffffffff160361035b576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040161035290610afa565b60405180910390fd5b80600080878787876040516020016103769493929190610520565b60405160208183030381529060405280519060200120815260200190815260200160002060006101000a81548173ffffffffffffffffffffffffffffffffffffffff021916908373ffffffffffffffffffffffffffffffffffffffff1602179055505050505050565b6000604051905090565b600080fd5b600080fd5b6000819050919050565b610406816103f3565b811461041157600080fd5b50565b600081359050610423816103fd565b92915050565b600060ff82169050919050565b61043f81610429565b811461044a57600080fd5b50565b60008135905061045c81610436565b92915050565b6000806000806080858703121561047c5761047b6103e9565b5b600061048a87828801610414565b945050602061049b8782880161044d565b93505060406104ac87828801610414565b92505060606104bd87828801610414565b91505092959194509250565b6000819050919050565b6104e46104df826103f3565b6104c9565b82525050565b60008160f81b9050919050565b6000610502826104ea565b9050919050565b61051a61051582610429565b6104f7565b82525050565b600061052c82876104d3565b60208201915061053c8286610509565b60018201915061054c82856104d3565b60208201915061055c82846104d3565b60208201915081905095945050505050565b600081905092915050565b82818337600083830152505050565b6000610594838561056e565b93506105a1838584610579565b82840190509392505050565b60006105ba828486610588565b91508190509392505050565b600082825260208201905092915050565b7f6661696c656420746f2063616c6c206d6f7665642065637265636f766572206160008201527f742061646472657373203078303030303030303030303030303030303030303060208201527f3030303030303030303030303030313233343536000000000000000000000000604082015250565b60006106596054836105c6565b9150610664826105d7565b606082019050919050565b600060208201905081810360008301526106888161064c565b9050919050565b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b60006106ba8261068f565b9050919050565b60008160601b9050919050565b60006106d9826106c1565b9050919050565b60006106eb826106ce565b9050919050565b6107036106fe826106af565b6106e0565b82525050565b600061071582846106f2565b60148201915081905092915050565b600080fd5b6000601f19601f8301169050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b61077282610729565b810181811067ffffffffffffffff821117156107915761079061073a565b5b80604052505050565b60006107a46103df565b90506107b08282610769565b919050565b600067ffffffffffffffff8211156107d0576107cf61073a565b5b602082029050602081019050919050565b600080fd5b600080fd5b6107f4816106af565b81146107ff57600080fd5b50565b600081359050610811816107eb565b92915050565b600060a0828403121561082d5761082c6107e6565b5b61083760a061079a565b9050600061084784828501610414565b600083015250602061085b8482850161044d565b602083015250604061086f84828501610414565b604083015250606061088384828501610414565b606083015250608061089784828501610802565b60808301525092915050565b60006108b66108b1846107b5565b61079a565b90508083825260208201905060a084028301858111156108d9576108d86107e1565b5b835b8181101561090257806108ee8882610817565b84526020840193505060a0810190506108db565b5050509392505050565b600082601f83011261092157610920610724565b5b81356109318482602086016108a3565b91505092915050565b6000602082840312156109505761094f6103e9565b5b600082013567ffffffffffffffff81111561096e5761096d6103ee565b5b61097a8482850161090c565b91505092915050565b600080600080600060a0868803121561099f5761099e6103e9565b5b60006109ad88828901610414565b95505060206109be8882890161044d565b94505060406109cf88828901610414565b93505060606109e088828901610414565b92505060806109f188828901610802565b9150509295509295909350565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052603260045260246000fd5b7f4e487b7100000000000000000000000000000000000000000000000000000000600052601160045260246000fd5b6000819050919050565b6000610a7182610a5c565b91507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8203610aa357610aa2610a2d565b5b600182019050919050565b7f72657475726e20616464726573732063616e6e6f742062652030783000000000600082015250565b6000610ae4601c836105c6565b9150610aef82610aae565b602082019050919050565b60006020820190508181036000830152610b1381610ad7565b905091905056fea2646970667358221220154f5b68ccfa5be744e7245765a3530dac4035052284a68b5dded1945b45075e64736f6c63430008120033\",\"MovePrecompileToAddress\":\"0x0000000000000000000000000000000000123456\"},\"0xc100000000000000000000000000000000000000\":{\"balance\":\"0x30d40\"}},\"calls\":[{\"from\":\"0xc100000000000000000000000000000000000000\",\"to\":\"0x0000000000000000000000000000000000123456\",\"input\":\"0x4554480000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b45544800000000000000000000000000000000000000000000000000000000004554480000000000000000000000000000000000000000000000000000000000\"},{\"from\":\"0xc100000000000000000000000000000000000000\",\"to\":\"0x0000000000000000000000000000000000123456\",\"input\":\"0x1c8aff950685c2ed4bc3174f3472287b56d9517b9c948127319a09a7a36deac8000000000000000000000000000000000000000000000000000000000000001cb7cf302145348387b9e69fde82d8e634a0f8761e78da3bfa059efced97cbed0d2a66b69167cafe0ccfc726aec6ee393fea3cf0e4f3f9c394705e0f56d9bfe1c9\"},{\"from\":\"0xc100000000000000000000000000000000000000\",\"to\":\"0x0000000000000000000000000000000000000001\",\"input\":\"0xc00692604554480000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b45544800000000000000000000000000000000000000000000000000000000004554480000000000000000000000000000000000000000000000000000000000000000000000000000000000d8da6bf26964af9d7eed9e03e53415d37aa96045\"},{\"from\":\"0xc100000000000000000000000000000000000000\",\"to\":\"0x0000000000000000000000000000000000000001\",\"input\":\"0x4554480000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b45544800000000000000000000000000000000000000000000000000000000004554480000000000000000000000000000000000000000000000000000000000\"},{\"from\":\"0xc100000000000000000000000000000000000000\",\"to\":\"0x0000000000000000000000000000000000000001\",\"input\":\"0x1c8aff950685c2ed4bc3174f3472287b56d9517b9c948127319a09a7a36deac8000000000000000000000000000000000000000000000000000000000000001cb7cf302145348387b9e69fde82d8e634a0f8761e78da3bfa059efced97cbed0d2a66b69167cafe0ccfc726aec6ee393fea3cf0e4f3f9c394705e0f56d9bfe1c9\"},{\"from\":\"0xc100000000000000000000000000000000000000\",\"to\":\"0x0000000000000000000000000000000000000001\",\"input\":\"0x4554480000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b45544800000000000000000000000000000000000000000000000000000000004554490000000000000000000000000000000000000000000000000000000000\"}]}]}")] SimulatePayload payload, + BlockParameter? blockParameter = null); + [JsonRpcMethod(IsImplemented = true, Description = "Executes a tx call and returns gas used (does not create a transaction)", IsSharable = false, @@ -212,7 +222,7 @@ ResultWrapper eth_getTransactionByBlockNumberAndIndex( Description = "Retrieves a transaction receipt by tx hash", IsSharable = true, ExampleResponse = "{\"transactionHash\":\"0x80757153e93d1b475e203406727b62a501187f63e23b8fa999279e219ee3be71\",\"transactionIndex\":\"0x7\",\"blockHash\":\"0x42def051b21038905cd2a2bc28d460a94df2249466847f0e1bcb4be4eb21891a\",\"blockNumber\":\"0x4e3f39\",\"cumulativeGasUsed\":\"0x62c9d\",\"gasUsed\":\"0xe384\",\"effectiveGasPrice\":\"0x12a05f200\",\"from\":\"0x0afe0a94415e8974052e7e6cfab19ee1c2ef4f69\",\"to\":\"0x19e8c84d4943e58b035626b064cfc76ee13ee6cb\",\"contractAddress\":null,\"logs\":[{\"removed\":false,\"logIndex\":\"0x0\",\"transactionIndex\":\"0x7\",\"transactionHash\":\"0x80757153e93d1b475e203406727b62a501187f63e23b8fa999279e219ee3be71\",\"blockHash\":\"0x42def051b21038905cd2a2bc28d460a94df2249466847f0e1bcb4be4eb21891a\",\"blockNumber\":\"0x4e3f39\",\"address\":\"0x2ac3c1d3e24b45c6c310534bc2dd84b5ed576335\",\"data\":\"0x0000000000000000000000000000000000000000000000000000000000000000\",\"topics\":[\"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef\",\"0x00000000000000000000000019e8c84d4943e58b035626b064cfc76ee13ee6cb\",\"0x00000000000000000000000028078300a459a9e136f872285654cdc74463041e\"]},{\"removed\":false,\"logIndex\":\"0x1\",\"transactionIndex\":\"0x7\",\"transactionHash\":\"0x80757153e93d1b475e203406727b62a501187f63e23b8fa999279e219ee3be71\",\"blockHash\":\"0x42def051b21038905cd2a2bc28d460a94df2249466847f0e1bcb4be4eb21891a\",\"blockNumber\":\"0x4e3f39\",\"address\":\"0x19e8c84d4943e58b035626b064cfc76ee13ee6cb\",\"data\":\"0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007735940000000000000000000000000000000000000000000000000000000000000000000\",\"topics\":[\"0x950494fc3642fae5221b6c32e0e45765c95ebb382a04a71b160db0843e74c99f\",\"0x0000000000000000000000000afe0a94415e8974052e7e6cfab19ee1c2ef4f69\",\"0x00000000000000000000000028078300a459a9e136f872285654cdc74463041e\",\"0x0000000000000000000000000afe0a94415e8974052e7e6cfab19ee1c2ef4f69\"]}],\"logsBloom\":\"0x00000000000000000000000000000000000000000000000020000000000000800000000000000000000400000000000000000000000000000000000000002000000000000000000000000008000000000000000000000000000000000000000000000002002000000000000000000000000000000000000000000812000000000000000000000000000001000000000000000000000008000400008000000000000000000000000000000000000000000000000000000000800000000000000000000002000000000000000000000000000000000000100000000000000000002000000000000000000000000010000000000000000000000400000000020000\",\"status\":\"0x1\",\"type\":\"0x0\"}")] - Task> eth_getTransactionReceipt([JsonRpcParameter(ExampleValue = "[\"0x80757153e93d1b475e203406727b62a501187f63e23b8fa999279e219ee3be71\"]")] Hash256 txHashData); + ResultWrapper eth_getTransactionReceipt([JsonRpcParameter(ExampleValue = "[\"0x80757153e93d1b475e203406727b62a501187f63e23b8fa999279e219ee3be71\"]")] Hash256 txHashData); [JsonRpcMethod(IsImplemented = true, Description = "Retrieves an uncle block header by block hash and uncle index", @@ -267,6 +277,6 @@ ResultWrapper eth_getTransactionByBlockNumberAndIndex( ResultWrapper eth_getProof([JsonRpcParameter(ExampleValue = "[\"0x7F0d15C7FAae65896648C8273B6d7E43f58Fa842\",[ \"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421\" ],\"latest\"]")] Address accountAddress, UInt256[] hashRate, BlockParameter blockParameter); [JsonRpcMethod(IsImplemented = true, Description = "Retrieves Accounts via Address and Blocknumber", IsSharable = true)] - ResultWrapper eth_getAccount([JsonRpcParameter(ExampleValue = "[\"0xaa00000000000000000000000000000000000000\", \"latest\"]")] Address accountAddress, BlockParameter blockParameter = null); + ResultWrapper eth_getAccount([JsonRpcParameter(ExampleValue = "[\"0xaa00000000000000000000000000000000000000\", \"latest\"]")] Address accountAddress, BlockParameter? blockParameter = null); } } diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/SimulateTxExecutor.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/SimulateTxExecutor.cs new file mode 100644 index 00000000000..898f13ae6b2 --- /dev/null +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/SimulateTxExecutor.cs @@ -0,0 +1,217 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Nethermind.Blockchain.Find; +using Nethermind.Config; +using Nethermind.Core; +using Nethermind.Facade; +using Nethermind.Facade.Eth; +using Nethermind.Facade.Proxy.Models.Simulate; +using Nethermind.Facade.Simulate; +using Nethermind.JsonRpc.Data; + +namespace Nethermind.JsonRpc.Modules.Eth; + +public class SimulateTxExecutor(IBlockchainBridge blockchainBridge, IBlockFinder blockFinder, IJsonRpcConfig rpcConfig, ulong? secondsPerSlot = null) + : ExecutorBase, SimulatePayload, + SimulatePayload>(blockchainBridge, blockFinder, rpcConfig) +{ + private readonly long _blocksLimit = rpcConfig.MaxSimulateBlocksCap ?? 256; + private long _gasCapBudget = rpcConfig.GasCap ?? long.MaxValue; + + protected override SimulatePayload Prepare(SimulatePayload call) + { + SimulatePayload result = new() + { + TraceTransfers = call.TraceTransfers, + Validation = call.Validation, + BlockStateCalls = call.BlockStateCalls?.Select(blockStateCall => + { + if (blockStateCall.BlockOverrides?.GasLimit is not null) + { + blockStateCall.BlockOverrides.GasLimit = (ulong)Math.Min((long)blockStateCall.BlockOverrides.GasLimit!.Value, _gasCapBudget); + } + + return new BlockStateCall + { + BlockOverrides = blockStateCall.BlockOverrides, + StateOverrides = blockStateCall.StateOverrides, + Calls = blockStateCall.Calls?.Select(callTransactionModel => + { + if (callTransactionModel.Type == TxType.Legacy) + { + callTransactionModel.Type = TxType.EIP1559; + } + + bool hadGasLimitInRequest = callTransactionModel.Gas.HasValue; + bool hadNonceInRequest = callTransactionModel.Nonce.HasValue; + callTransactionModel.EnsureDefaults(_gasCapBudget); + _gasCapBudget -= callTransactionModel.Gas!.Value; + + Transaction tx = callTransactionModel.ToTransaction(_blockchainBridge.GetChainId()); + + TransactionWithSourceDetails? result = new() + { + HadGasLimitInRequest = hadGasLimitInRequest, + HadNonceInRequest = hadNonceInRequest, + Transaction = tx + }; + + return result; + }).ToArray() + }; + }).ToList() + }; + + return result; + } + + public override ResultWrapper> Execute( + SimulatePayload call, + BlockParameter? blockParameter) + { + if (call.BlockStateCalls is null) + return ResultWrapper>.Fail("Must contain BlockStateCalls", ErrorCodes.InvalidParams); + + if (call.BlockStateCalls!.Count > _rpcConfig.MaxSimulateBlocksCap) + return ResultWrapper>.Fail( + $"This node is configured to support only {_rpcConfig.MaxSimulateBlocksCap} blocks", ErrorCodes.InvalidInputTooManyBlocks); + + SearchResult searchResult = _blockFinder.SearchForBlock(blockParameter); + + if (searchResult.IsError || searchResult.Object is null) + return ResultWrapper>.Fail(searchResult); + + BlockHeader header = searchResult.Object.Header; + + if (!_blockchainBridge.HasStateForBlock(header!)) + return ResultWrapper>.Fail($"No state available for block {header.Hash}", + ErrorCodes.ResourceUnavailable); + + if (call.BlockStateCalls?.Count > _blocksLimit) + return ResultWrapper>.Fail( + $"Too many blocks provided, node is configured to simulate up to {_blocksLimit} while {call.BlockStateCalls?.Count} were given", + ErrorCodes.InvalidParams); + + secondsPerSlot ??= new BlocksConfig().SecondsPerSlot; + + if (call.BlockStateCalls is not null) + { + long lastBlockNumber = -1; + ulong lastBlockTime = 0; + + foreach (BlockStateCall? blockToSimulate in call.BlockStateCalls) + { + ulong givenNumber = blockToSimulate.BlockOverrides?.Number ?? + (lastBlockNumber == -1 ? (ulong)header.Number + 1 : (ulong)lastBlockNumber + 1); + + if (givenNumber > long.MaxValue) + return ResultWrapper>.Fail( + $"Block number too big {givenNumber}!", ErrorCodes.InvalidParams); + + if (givenNumber < (ulong)header.Number) + return ResultWrapper>.Fail( + $"Block number out of order {givenNumber} is < than given base number of {header.Number}!", ErrorCodes.InvalidInputBlocksOutOfOrder); + + long given = (long)givenNumber; + if (given > lastBlockNumber) + { + lastBlockNumber = given; + } + else + { + return ResultWrapper>.Fail( + $"Block number out of order {givenNumber}!", ErrorCodes.InvalidInputBlocksOutOfOrder); + } + + blockToSimulate.BlockOverrides ??= new BlockOverride(); + blockToSimulate.BlockOverrides.Number = givenNumber; + + ulong givenTime = blockToSimulate.BlockOverrides.Time ?? + (lastBlockTime == 0 + ? header.Timestamp + secondsPerSlot.Value + : lastBlockTime + secondsPerSlot.Value); + + if (givenTime < header.Timestamp) + return ResultWrapper>.Fail( + $"Block timestamp out of order {givenTime} is < than given base timestamp of {header.Timestamp}!", ErrorCodes.BlockTimestampNotIncreased); + + if (givenTime > lastBlockTime) + { + lastBlockTime = givenTime; + } + else + { + return ResultWrapper>.Fail( + $"Block timestamp out of order {givenTime}!", ErrorCodes.BlockTimestampNotIncreased); + } + + blockToSimulate.BlockOverrides.Time = givenTime; + } + + long minBlockNumber = Math.Min( + call.BlockStateCalls.Min(b => (long)(b.BlockOverrides?.Number ?? ulong.MaxValue)), + header.Number + 1); + + long maxBlockNumber = Math.Max( + call.BlockStateCalls.Max(b => (long)(b.BlockOverrides?.Number ?? ulong.MinValue)), + minBlockNumber); + + HashSet existingBlockNumbers = + [ + .. call.BlockStateCalls.Select(b => (long)(b.BlockOverrides?.Number ?? ulong.MinValue)) + ]; + + List> completeBlockStateCalls = call.BlockStateCalls; + + for (long blockNumber = minBlockNumber; blockNumber <= maxBlockNumber; blockNumber++) + { + if (!existingBlockNumbers.Contains(blockNumber)) + { + completeBlockStateCalls.Add(new BlockStateCall + { + BlockOverrides = new BlockOverride { Number = (ulong)blockNumber }, + StateOverrides = null, + Calls = Array.Empty() + }); + } + } + + call.BlockStateCalls.Sort((b1, b2) => b1.BlockOverrides!.Number!.Value.CompareTo(b2.BlockOverrides!.Number!.Value)); + } + + using CancellationTokenSource cancellationTokenSource = new(_rpcConfig.Timeout); //TODO remove! + SimulatePayload toProcess = Prepare(call); + return Execute(header.Clone(), toProcess, cancellationTokenSource.Token); + } + + protected override ResultWrapper> Execute(BlockHeader header, + SimulatePayload tx, CancellationToken token) + { + SimulateOutput results = _blockchainBridge.Simulate(header, tx, token); + + if (results.Error is not null && (results.Error.Contains("invalid transaction") + || results.Error.Contains("InsufficientBalanceException") + )) + results.ErrorCode = ErrorCodes.InvalidTransaction; + + if (results.Error is not null && results.Error.Contains("InvalidBlockException")) + results.ErrorCode = ErrorCodes.InvalidParams; + + + if (results.Error is not null && results.Error.Contains("below intrinsic gas")) + results.ErrorCode = ErrorCodes.InsufficientIntrinsicGas; + + + + return results.Error is null + ? ResultWrapper>.Success(results.Items) + : results.ErrorCode is not null + ? ResultWrapper>.Fail(results.Error!, results.ErrorCode!.Value, results.Items) + : ResultWrapper>.Fail(results.Error, results.Items); + } +} diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/ModuleType.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/ModuleType.cs index 27aed311c00..6d53f38714a 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/ModuleType.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/ModuleType.cs @@ -30,7 +30,6 @@ public static class ModuleType public const string Vault = nameof(Vault); public const string Deposit = nameof(Deposit); public const string Health = nameof(Health); - public const string Witness = nameof(Witness); public const string AccountAbstraction = nameof(AccountAbstraction); public const string Rpc = nameof(Rpc); @@ -59,7 +58,6 @@ public static class ModuleType Vault, Deposit, Health, - Witness, AccountAbstraction, Rpc, }; diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Personal/IPersonalRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Personal/IPersonalRpcModule.cs index f255d904da1..3ca1db260a6 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Personal/IPersonalRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Personal/IPersonalRpcModule.cs @@ -3,6 +3,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; namespace Nethermind.JsonRpc.Modules.Personal diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Personal/PersonalRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Personal/PersonalRpcModule.cs index 1070270a9d5..1116810731d 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Personal/PersonalRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Personal/PersonalRpcModule.cs @@ -7,6 +7,7 @@ using Nethermind.Core.Attributes; using Nethermind.Core.Crypto; using Nethermind.Crypto; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.KeyStore; using Nethermind.Wallet; diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/IProofRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/IProofRpcModule.cs index a7b8e9a450d..8e2123e5533 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/IProofRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/IProofRpcModule.cs @@ -3,6 +3,7 @@ using Nethermind.Blockchain.Find; using Nethermind.Core.Crypto; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; namespace Nethermind.JsonRpc.Modules.Proof diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofModuleFactory.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofModuleFactory.cs index 0aeaeb2e336..cd6b142126f 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofModuleFactory.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofModuleFactory.cs @@ -10,8 +10,10 @@ using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Tracing; using Nethermind.Consensus.Validators; +using Nethermind.Core.Crypto; using Nethermind.Core.Specs; using Nethermind.Db; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; using Nethermind.State; using Nethermind.Trie.Pruning; @@ -48,13 +50,24 @@ public override IProofRpcModule Create() ReadOnlyTxProcessingEnv txProcessingEnv = new( _worldStateManager, _blockTree, _specProvider, _logManager); - RpcBlockTransactionsExecutor traceExecutor = new(txProcessingEnv.TransactionProcessor, txProcessingEnv.StateProvider); + IReadOnlyTxProcessingScope scope = txProcessingEnv.Build(Keccak.EmptyTreeHash); + + RpcBlockTransactionsExecutor traceExecutor = new(scope.TransactionProcessor, scope.WorldState); ReadOnlyChainProcessingEnv chainProcessingEnv = new( - txProcessingEnv, Always.Valid, _recoveryStep, NoBlockRewards.Instance, new InMemoryReceiptStorage(), _specProvider, _logManager, traceExecutor); + scope, + Always.Valid, + _recoveryStep, + NoBlockRewards.Instance, + new InMemoryReceiptStorage(), + _specProvider, + _blockTree, + _worldStateManager.GlobalStateReader, + _logManager, + traceExecutor); Tracer tracer = new( - txProcessingEnv.StateProvider, + scope.WorldState, chainProcessingEnv.ChainProcessor, chainProcessingEnv.ChainProcessor); diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofRpcModule.cs index d4885d1c608..e79f6fdd106 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/ProofRpcModule.cs @@ -14,6 +14,7 @@ using Nethermind.Evm; using Nethermind.Evm.Tracing; using Nethermind.Evm.Tracing.Proofs; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.Logging; using Nethermind.Serialization.Rlp; @@ -32,6 +33,7 @@ public class ProofRpcModule : IProofRpcModule private readonly IReceiptFinder _receiptFinder; private readonly ISpecProvider _specProvider; private readonly HeaderDecoder _headerDecoder = new(); + private static readonly IRlpStreamDecoder _receiptDecoder = Rlp.GetStreamDecoder(); public ProofRpcModule( ITracer tracer, @@ -205,7 +207,7 @@ private static byte[][] BuildTxProofs(Transaction[] txs, IReleaseSpec releaseSpe private byte[][] BuildReceiptProofs(BlockHeader blockHeader, TxReceipt[] receipts, int index) { - return ReceiptTrie.CalculateReceiptProofs(_specProvider.GetSpec(blockHeader), receipts, index, ReceiptMessageDecoder.Instance); + return ReceiptTrie.CalculateReceiptProofs(_specProvider.GetSpec(blockHeader), receipts, index, _receiptDecoder); } } } diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/TransactionWithProof.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/TransactionWithProof.cs index 4294cfb4ee6..3069c957849 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/TransactionWithProof.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Proof/TransactionWithProof.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; namespace Nethermind.JsonRpc.Modules.Proof diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/RpcModuleProvider.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/RpcModuleProvider.cs index 25811ed3d4d..a4c11d06a26 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/RpcModuleProvider.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/RpcModuleProvider.cs @@ -139,7 +139,12 @@ public void Return(string methodName, IRpcModule rpcModule) private static IDictionary GetMethodDict(Type type) { - var methods = type.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly); + BindingFlags methodFlags = BindingFlags.Public | BindingFlags.Instance | BindingFlags.DeclaredOnly; + + IEnumerable methods = type.GetMethods(methodFlags) + .Concat(type.GetInterfaces().SelectMany(i => i.GetMethods(methodFlags))) + .DistinctBy(x => x.Name); + return methods.ToDictionary( x => x.Name.Trim(), x => diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Subscribe/NewHeadSubscription.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Subscribe/NewHeadSubscription.cs index 11a414c1bf3..ed9fccd2511 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Subscribe/NewHeadSubscription.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Subscribe/NewHeadSubscription.cs @@ -5,6 +5,7 @@ using Nethermind.Blockchain; using Nethermind.Core; using Nethermind.Core.Specs; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Modules.Eth; using Nethermind.Logging; diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Subscribe/NewPendingTransactionsSubscription.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Subscribe/NewPendingTransactionsSubscription.cs index fe900bc6181..7ec4b75d77d 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Subscribe/NewPendingTransactionsSubscription.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Subscribe/NewPendingTransactionsSubscription.cs @@ -3,6 +3,7 @@ using System; using System.Threading.Tasks; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.JsonRpc.Modules.Eth; using Nethermind.Logging; diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/ITraceRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/ITraceRpcModule.cs index d3b06c6675e..dae0f773487 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/ITraceRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/ITraceRpcModule.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using Nethermind.Blockchain.Find; using Nethermind.Core.Crypto; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; namespace Nethermind.JsonRpc.Modules.Trace diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceModuleFactory.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceModuleFactory.cs index 9d34f031e44..1ec63377195 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceModuleFactory.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceModuleFactory.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; - using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; using Nethermind.Consensus; @@ -11,77 +10,69 @@ using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Tracing; using Nethermind.Consensus.Validators; +using Nethermind.Core.Crypto; using Nethermind.Core.Specs; using Nethermind.Db; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; using Nethermind.State; using Nethermind.Trie.Pruning; -namespace Nethermind.JsonRpc.Modules.Trace -{ - public class TraceModuleFactory : ModuleFactoryBase - { - private readonly IWorldStateManager _worldStateManager; - private readonly IReadOnlyBlockTree _blockTree; - private readonly IJsonRpcConfig _jsonRpcConfig; - private readonly IReceiptStorage _receiptStorage; - private readonly ISpecProvider _specProvider; - private readonly ILogManager _logManager; - private readonly IBlockPreprocessorStep _recoveryStep; - private readonly IRewardCalculatorSource _rewardCalculatorSource; - private readonly IPoSSwitcher _poSSwitcher; - - public TraceModuleFactory( - IWorldStateManager worldStateManager, - IBlockTree blockTree, - IJsonRpcConfig jsonRpcConfig, - IBlockPreprocessorStep recoveryStep, - IRewardCalculatorSource rewardCalculatorSource, - IReceiptStorage receiptFinder, - ISpecProvider specProvider, - IPoSSwitcher poSSwitcher, - ILogManager logManager) - { - _worldStateManager = worldStateManager; - _blockTree = blockTree.AsReadOnly(); - _jsonRpcConfig = jsonRpcConfig ?? throw new ArgumentNullException(nameof(jsonRpcConfig)); - _recoveryStep = recoveryStep ?? throw new ArgumentNullException(nameof(recoveryStep)); - _rewardCalculatorSource = rewardCalculatorSource ?? throw new ArgumentNullException(nameof(rewardCalculatorSource)); - _receiptStorage = receiptFinder ?? throw new ArgumentNullException(nameof(receiptFinder)); - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); - _poSSwitcher = poSSwitcher ?? throw new ArgumentNullException(nameof(poSSwitcher)); - _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); - logManager.GetClassLogger(); - } - - public override ITraceRpcModule Create() - { - ReadOnlyTxProcessingEnv txProcessingEnv = - new(_worldStateManager, _blockTree, _specProvider, _logManager); +namespace Nethermind.JsonRpc.Modules.Trace; - IRewardCalculator rewardCalculator = - new MergeRpcRewardCalculator(_rewardCalculatorSource.Get(txProcessingEnv.TransactionProcessor), - _poSSwitcher); +public class TraceModuleFactory( + IWorldStateManager worldStateManager, + IBlockTree blockTree, + IJsonRpcConfig jsonRpcConfig, + IBlockPreprocessorStep recoveryStep, + IRewardCalculatorSource rewardCalculatorSource, + IReceiptStorage receiptFinder, + ISpecProvider specProvider, + IPoSSwitcher poSSwitcher, + ILogManager logManager) : ModuleFactoryBase +{ + protected readonly IWorldStateManager _worldStateManager = worldStateManager; + protected readonly IReadOnlyBlockTree _blockTree = blockTree.AsReadOnly(); + protected readonly IJsonRpcConfig _jsonRpcConfig = jsonRpcConfig ?? throw new ArgumentNullException(nameof(jsonRpcConfig)); + protected readonly IReceiptStorage _receiptStorage = receiptFinder ?? throw new ArgumentNullException(nameof(receiptFinder)); + protected readonly ISpecProvider _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); + protected readonly ILogManager _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); + protected readonly IBlockPreprocessorStep _recoveryStep = recoveryStep ?? throw new ArgumentNullException(nameof(recoveryStep)); + protected readonly IRewardCalculatorSource _rewardCalculatorSource = rewardCalculatorSource ?? throw new ArgumentNullException(nameof(rewardCalculatorSource)); + protected readonly IPoSSwitcher _poSSwitcher = poSSwitcher ?? throw new ArgumentNullException(nameof(poSSwitcher)); - RpcBlockTransactionsExecutor rpcBlockTransactionsExecutor = new(txProcessingEnv.TransactionProcessor, txProcessingEnv.StateProvider); - BlockProcessor.BlockValidationTransactionsExecutor executeBlockTransactionsExecutor = new(txProcessingEnv.TransactionProcessor, txProcessingEnv.StateProvider); + protected virtual ReadOnlyTxProcessingEnv CreateTxProcessingEnv() => new(_worldStateManager, _blockTree, _specProvider, _logManager); - ReadOnlyChainProcessingEnv CreateChainProcessingEnv(IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor) => new( - txProcessingEnv, + protected virtual ReadOnlyChainProcessingEnv CreateChainProcessingEnv(IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor, IReadOnlyTxProcessingScope scope, IRewardCalculator rewardCalculator) => new( + scope, Always.Valid, _recoveryStep, rewardCalculator, _receiptStorage, _specProvider, + _blockTree, + _worldStateManager.GlobalStateReader, _logManager, transactionsExecutor); - ReadOnlyChainProcessingEnv traceProcessingEnv = CreateChainProcessingEnv(rpcBlockTransactionsExecutor); - ReadOnlyChainProcessingEnv executeProcessingEnv = CreateChainProcessingEnv(executeBlockTransactionsExecutor); + public override ITraceRpcModule Create() + { + ReadOnlyTxProcessingEnv txProcessingEnv = CreateTxProcessingEnv(); + IReadOnlyTxProcessingScope scope = txProcessingEnv.Build(Keccak.EmptyTreeHash); + + IRewardCalculator rewardCalculator = + new MergeRpcRewardCalculator(_rewardCalculatorSource.Get(scope.TransactionProcessor), + _poSSwitcher); + + RpcBlockTransactionsExecutor rpcBlockTransactionsExecutor = new(scope.TransactionProcessor, scope.WorldState); + BlockProcessor.BlockValidationTransactionsExecutor executeBlockTransactionsExecutor = new(scope.TransactionProcessor, scope.WorldState); - Tracer tracer = new(txProcessingEnv.StateProvider, traceProcessingEnv.ChainProcessor, executeProcessingEnv.ChainProcessor); + ReadOnlyChainProcessingEnv traceProcessingEnv = CreateChainProcessingEnv(rpcBlockTransactionsExecutor, scope, rewardCalculator); + ReadOnlyChainProcessingEnv executeProcessingEnv = CreateChainProcessingEnv(executeBlockTransactionsExecutor, scope, rewardCalculator); - return new TraceRpcModule(_receiptStorage, tracer, _blockTree, _jsonRpcConfig, _specProvider, _logManager, txProcessingEnv.StateReader); - } + Tracer tracer = new(scope.WorldState, traceProcessingEnv.ChainProcessor, executeProcessingEnv.ChainProcessor); + + return new TraceRpcModule(_receiptStorage, tracer, _blockTree, _jsonRpcConfig, txProcessingEnv.StateReader); } + } diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceRpcModule.cs index 903fb1b1bfc..3ee10c8bcb3 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Trace/TraceRpcModule.cs @@ -15,6 +15,7 @@ using Nethermind.Evm.Tracing; using Nethermind.Evm.Tracing.ParityStyle; using Nethermind.Facade; +using Nethermind.Facade.Eth; using Nethermind.Int256; using Nethermind.JsonRpc.Data; using Nethermind.Logging; @@ -39,24 +40,19 @@ public class TraceRpcModule : ITraceRpcModule private readonly IBlockFinder _blockFinder; private readonly TxDecoder _txDecoder = new(); private readonly IJsonRpcConfig _jsonRpcConfig; - private readonly ILogManager _logManager; - private readonly ILogger _logger; - private readonly ISpecProvider _specProvider; private readonly TimeSpan _cancellationTokenTimeout; private readonly IStateReader _stateReader; - public TraceRpcModule(IReceiptFinder? receiptFinder, ITracer? tracer, IBlockFinder? blockFinder, IJsonRpcConfig? jsonRpcConfig, ISpecProvider? specProvider, ILogManager? logManager, IStateReader stateReader) + public TraceRpcModule(IReceiptFinder? receiptFinder, ITracer? tracer, IBlockFinder? blockFinder, IJsonRpcConfig? jsonRpcConfig, IStateReader stateReader) { _receiptFinder = receiptFinder ?? throw new ArgumentNullException(nameof(receiptFinder)); _tracer = tracer ?? throw new ArgumentNullException(nameof(tracer)); _blockFinder = blockFinder ?? throw new ArgumentNullException(nameof(blockFinder)); _jsonRpcConfig = jsonRpcConfig ?? throw new ArgumentNullException(nameof(jsonRpcConfig)); - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); - _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); _stateReader = stateReader ?? throw new ArgumentNullException(nameof(stateReader)); - _logger = logManager.GetClassLogger(); _cancellationTokenTimeout = TimeSpan.FromMilliseconds(_jsonRpcConfig.Timeout); } + public static ParityTraceTypes GetParityTypes(string[] types) => types.Select(s => FastEnum.Parse(s, true)).Aggregate((t1, t2) => t1 | t2); diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/TxPool/TransactionPoolContent.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/TxPool/TransactionPoolContent.cs index 5d0b6c0f3ba..2040cc403e8 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/TxPool/TransactionPoolContent.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/TxPool/TransactionPoolContent.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using Nethermind.Core; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.TxPool; diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Witness/IWitnessRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Witness/IWitnessRpcModule.cs deleted file mode 100644 index 6f954d00ce3..00000000000 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Witness/IWitnessRpcModule.cs +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System.Threading.Tasks; -using Nethermind.Blockchain.Find; -using Nethermind.Core.Crypto; - -namespace Nethermind.JsonRpc.Modules.Witness -{ - [RpcModule(ModuleType.Witness)] - public interface IWitnessRpcModule : IRpcModule - { - [JsonRpcMethod(Description = "Return witness of Block provided", - ResponseDescription = "Table of hashes of state nodes that were read during block processing", - ExampleResponse = - "\"0x1\"", - IsImplemented = true)] - Task> get_witnesses([JsonRpcParameter(Description = "Block to get witness", - ExampleValue = "{\"jsonrpc\":\"2.0\",\"result\":[\"0xa2a9f03b9493046696099d27b2612b99497aa1f392ec966716ab393c715a5bb6\"],\"id\":67}")] - BlockParameter blockParameter); - } -} diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Witness/WitnessRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Witness/WitnessRpcModule.cs deleted file mode 100644 index 9438b40f0ec..00000000000 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Witness/WitnessRpcModule.cs +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System.Threading.Tasks; -using Nethermind.Blockchain.Find; -using Nethermind.Core; -using Nethermind.Core.Crypto; -using Nethermind.State; - -namespace Nethermind.JsonRpc.Modules.Witness -{ - public class WitnessRpcModule : IWitnessRpcModule - { - private readonly IBlockFinder _blockFinder; - private readonly IWitnessRepository _witnessRepository; - - public WitnessRpcModule(IWitnessRepository witnessRepository, IBlockFinder finder) - { - _witnessRepository = witnessRepository; - _blockFinder = finder; - } - - public Task> get_witnesses(BlockParameter blockParameter) - { - SearchResult searchResult = _blockFinder.SearchForHeader(blockParameter); - if (searchResult.Object is null) - { - return Task.FromResult(ResultWrapper.Fail("Block not found", ErrorCodes.ResourceNotFound)); - } - - Hash256 hash = searchResult.Object.Hash; - Hash256[] result = _witnessRepository.Load(hash!); - return result is null ? Task.FromResult(ResultWrapper.Fail("Witness unavailable", ErrorCodes.ResourceUnavailable)) : Task.FromResult(ResultWrapper.Success(result)); - } - } -} diff --git a/src/Nethermind/Nethermind.JsonRpc/WebSockets/JsonRpcSocketsClient.cs b/src/Nethermind/Nethermind.JsonRpc/WebSockets/JsonRpcSocketsClient.cs index 1ea4d983058..1176e351741 100644 --- a/src/Nethermind/Nethermind.JsonRpc/WebSockets/JsonRpcSocketsClient.cs +++ b/src/Nethermind/Nethermind.JsonRpc/WebSockets/JsonRpcSocketsClient.cs @@ -48,6 +48,7 @@ public override void Dispose() { base.Dispose(); _sendSemaphore.Dispose(); + _jsonRpcContext.Dispose(); Closed?.Invoke(this, EventArgs.Empty); } diff --git a/src/Nethermind/Nethermind.Merge.AuRa.Test/AuRaMergeEngineModuleTests.cs b/src/Nethermind/Nethermind.Merge.AuRa.Test/AuRaMergeEngineModuleTests.cs index 94e728d5d8b..cb62721f8bc 100644 --- a/src/Nethermind/Nethermind.Merge.AuRa.Test/AuRaMergeEngineModuleTests.cs +++ b/src/Nethermind/Nethermind.Merge.AuRa.Test/AuRaMergeEngineModuleTests.cs @@ -142,7 +142,6 @@ protected override IBlockProcessor CreateBlockProcessor() new BlockProcessor.BlockValidationTransactionsExecutor(TxProcessor, State), State, ReceiptStorage, - NullWitnessCollector.Instance, new BlockhashStore(BlockTree, SpecProvider, State), TxProcessor, LogManager, @@ -186,12 +185,11 @@ protected override IBlockProducer CreateTestBlockProducer(TxPoolTxSource txPoolT BlockProducerEnv blockProducerEnv = blockProducerEnvFactory.Create(); - PostMergeBlockProducer postMergeBlockProducer = blockProducerFactory.Create(blockProducerEnv, BlockProductionTrigger); + PostMergeBlockProducer postMergeBlockProducer = blockProducerFactory.Create(blockProducerEnv); PostMergeBlockProducer = postMergeBlockProducer; PayloadPreparationService ??= new PayloadPreparationService( postMergeBlockProducer, - new BlockImprovementContextFactory(BlockProductionTrigger, TimeSpan.FromSeconds(MergeConfig.SecondsPerSlot) - ), + new BlockImprovementContextFactory(PostMergeBlockProducer, TimeSpan.FromSeconds(MergeConfig.SecondsPerSlot)), TimerFactory.Default, LogManager, TimeSpan.FromSeconds(MergeConfig.SecondsPerSlot), @@ -204,7 +202,6 @@ protected override IBlockProducer CreateTestBlockProducer(TxPoolTxSource txPoolT AuRaBlockProducer preMergeBlockProducer = new( txPoolTxSource, blockProducerEnvFactory.Create().ChainProcessor, - BlockProductionTrigger, State, sealer, BlockTree, diff --git a/src/Nethermind/Nethermind.Merge.AuRa/AuRaMergeBlockProducerEnvFactory.cs b/src/Nethermind/Nethermind.Merge.AuRa/AuRaMergeBlockProducerEnvFactory.cs index df1e0c2956f..128b6a6098c 100644 --- a/src/Nethermind/Nethermind.Merge.AuRa/AuRaMergeBlockProducerEnvFactory.cs +++ b/src/Nethermind/Nethermind.Merge.AuRa/AuRaMergeBlockProducerEnvFactory.cs @@ -12,6 +12,7 @@ using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Validators; using Nethermind.Core.Specs; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; using Nethermind.Merge.AuRa.Withdrawals; using Nethermind.State; @@ -56,7 +57,7 @@ public AuRaMergeBlockProducerEnvFactory( } protected override BlockProcessor CreateBlockProcessor( - ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv, + IReadOnlyTxProcessingScope readOnlyTxProcessingEnv, ISpecProvider specProvider, IBlockValidator blockValidator, IRewardCalculatorSource rewardCalculatorSource, @@ -71,7 +72,7 @@ protected override BlockProcessor CreateBlockProcessor( blockValidator, rewardCalculatorSource.Get(readOnlyTxProcessingEnv.TransactionProcessor), TransactionsExecutorFactory.Create(readOnlyTxProcessingEnv), - readOnlyTxProcessingEnv.StateProvider, + readOnlyTxProcessingEnv.WorldState, receiptStorage, logManager, _blockTree, @@ -92,6 +93,6 @@ protected override TxPoolTxSource CreateTxPoolTxSource( ITransactionComparerProvider transactionComparerProvider, ILogManager logManager) { - return new StartBlockProducerAuRa(_auraApi).CreateTxPoolTxSource(processingEnv); + return new StartBlockProducerAuRa(_auraApi).CreateTxPoolTxSource(); } } diff --git a/src/Nethermind/Nethermind.Merge.AuRa/AuRaMergePlugin.cs b/src/Nethermind/Nethermind.Merge.AuRa/AuRaMergePlugin.cs index 2b82d8febbc..ac61be06612 100644 --- a/src/Nethermind/Nethermind.Merge.AuRa/AuRaMergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.AuRa/AuRaMergePlugin.cs @@ -46,7 +46,7 @@ originalFilter is MinGasPriceContractTxFilter ? originalFilter } } - public override Task InitBlockProducer(IBlockProducerFactory consensusPlugin, IBlockProductionTrigger blockProductionTrigger, ITxSource? txSource) + public override IBlockProducer InitBlockProducer(IBlockProducerFactory consensusPlugin, ITxSource? txSource) { _api.BlockProducerEnvFactory = new AuRaMergeBlockProducerEnvFactory( (AuRaNethermindApi)_api, @@ -62,7 +62,7 @@ public override Task InitBlockProducer(IBlockProducerFactory con _api.Config(), _api.LogManager); - return base.InitBlockProducer(consensusPlugin, blockProductionTrigger, txSource); + return base.InitBlockProducer(consensusPlugin, txSource); } protected override PostMergeBlockProducerFactory CreateBlockProducerFactory() diff --git a/src/Nethermind/Nethermind.Merge.AuRa/AuRaPostMergeBlockProducerFactory.cs b/src/Nethermind/Nethermind.Merge.AuRa/AuRaPostMergeBlockProducerFactory.cs index edacc0bf1e0..6222cdac7a1 100644 --- a/src/Nethermind/Nethermind.Merge.AuRa/AuRaPostMergeBlockProducerFactory.cs +++ b/src/Nethermind/Nethermind.Merge.AuRa/AuRaPostMergeBlockProducerFactory.cs @@ -33,7 +33,6 @@ public AuRaPostMergeBlockProducerFactory( public override PostMergeBlockProducer Create( BlockProducerEnv producerEnv, - IBlockProductionTrigger blockProductionTrigger, ITxSource? txSource = null) { TargetAdjustedGasLimitCalculator targetAdjustedGasLimitCalculator = @@ -43,7 +42,6 @@ public override PostMergeBlockProducer Create( txSource ?? producerEnv.TxSource, producerEnv.ChainProcessor, producerEnv.BlockTree, - blockProductionTrigger, producerEnv.ReadOnlyStateProvider, _gasLimitCalculator ?? targetAdjustedGasLimitCalculator, _sealEngine, diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/BlobBundleExtensions.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/BlobBundleExtensions.cs new file mode 100644 index 00000000000..90e4ea0545f --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/BlobBundleExtensions.cs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Crypto; +using Nethermind.Merge.Plugin.Data; + +namespace Nethermind.Merge.Plugin.Test; + +internal static class BlobBundleExtensions +{ + public static byte[][] GetBlobVersionedHashes(this BlobsBundleV1 blobsBundle) + { + byte[][] hashes = new byte[blobsBundle.Commitments.Length][]; + + for (var i = 0; i < blobsBundle.Commitments.Length; i++) + { + hashes[i] = new byte[KzgPolynomialCommitments.BytesPerBlobVersionedHash]; + KzgPolynomialCommitments.TryComputeCommitmentHashV1(blobsBundle.Commitments[i], hashes[i]); + } + + return hashes; + } +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.DelayBlockImprovementContextFactory.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.DelayBlockImprovementContextFactory.cs index b6fce670c01..81e20cc6f87 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.DelayBlockImprovementContextFactory.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.DelayBlockImprovementContextFactory.cs @@ -4,6 +4,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Nethermind.Consensus; using Nethermind.Consensus.Producers; using Nethermind.Core; using Nethermind.Core.Extensions; @@ -17,19 +18,19 @@ public partial class EngineModuleTests { private class DelayBlockImprovementContextFactory : IBlockImprovementContextFactory { - private readonly IManualBlockProductionTrigger _productionTrigger; + private readonly IBlockProducer _blockProducer; private readonly TimeSpan _timeout; private readonly TimeSpan _delay; - public DelayBlockImprovementContextFactory(IManualBlockProductionTrigger productionTrigger, TimeSpan timeout, TimeSpan delay) + public DelayBlockImprovementContextFactory(IBlockProducer blockProducer, TimeSpan timeout, TimeSpan delay) { - _productionTrigger = productionTrigger; + _blockProducer = blockProducer; _timeout = timeout; _delay = delay; } public IBlockImprovementContext StartBlockImprovementContext(Block currentBestBlock, BlockHeader parentHeader, PayloadAttributes payloadAttributes, DateTimeOffset startDateTime) => - new DelayBlockImprovementContext(currentBestBlock, _productionTrigger, _timeout, parentHeader, payloadAttributes, _delay, startDateTime); + new DelayBlockImprovementContext(currentBestBlock, _blockProducer, _timeout, parentHeader, payloadAttributes, _delay, startDateTime); } private class DelayBlockImprovementContext : IBlockImprovementContext @@ -37,7 +38,7 @@ private class DelayBlockImprovementContext : IBlockImprovementContext private CancellationTokenSource? _cancellationTokenSource; public DelayBlockImprovementContext(Block currentBestBlock, - IManualBlockProductionTrigger blockProductionTrigger, + IBlockProducer blockProducer, TimeSpan timeout, BlockHeader parentHeader, PayloadAttributes payloadAttributes, @@ -47,17 +48,17 @@ public DelayBlockImprovementContext(Block currentBestBlock, _cancellationTokenSource = new CancellationTokenSource(timeout); CurrentBestBlock = currentBestBlock; StartDateTime = startDateTime; - ImprovementTask = BuildBlock(blockProductionTrigger, parentHeader, payloadAttributes, delay, _cancellationTokenSource.Token); + ImprovementTask = BuildBlock(blockProducer, parentHeader, payloadAttributes, delay, _cancellationTokenSource.Token); } private async Task BuildBlock( - IManualBlockProductionTrigger blockProductionTrigger, + IBlockProducer blockProducer, BlockHeader parentHeader, PayloadAttributes payloadAttributes, TimeSpan delay, CancellationToken cancellationToken) { - Block? block = await blockProductionTrigger.BuildBlock(parentHeader, cancellationToken, NullBlockTracer.Instance, payloadAttributes); + Block? block = await blockProducer.BuildBlock(parentHeader, NullBlockTracer.Instance, payloadAttributes, cancellationToken); await Task.Delay(delay, cancellationToken); if (block is not null) { diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.HelperFunctions.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.HelperFunctions.cs index e4b77d1270e..16801b77544 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.HelperFunctions.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.HelperFunctions.cs @@ -48,17 +48,30 @@ private void AssertExecutionStatusChanged(IBlockFinder blockFinder, Hash256 head private Transaction[] BuildTransactions(MergeTestBlockchain chain, Hash256 parentHash, PrivateKey from, Address to, uint count, int value, out AccountStruct accountFrom, out BlockHeader parentHeader, int blobCountPerTx = 0) { - Transaction BuildTransaction(uint index, AccountStruct senderAccount) => - Build.A.Transaction.WithNonce(senderAccount.Nonce + index) + Transaction BuildTransaction(uint index, AccountStruct senderAccount) + { + TransactionBuilder builder = Build.A.Transaction + .WithNonce(senderAccount.Nonce + index) .WithTimestamp(Timestamper.UnixTime.Seconds) .WithTo(to) .WithValue(value.GWei()) .WithGasPrice(1.GWei()) .WithChainId(chain.SpecProvider.ChainId) - .WithSenderAddress(from.Address) - .WithShardBlobTxTypeAndFields(blobCountPerTx) + .WithSenderAddress(from.Address); + + if (blobCountPerTx != 0) + { + builder = builder.WithShardBlobTxTypeAndFields(blobCountPerTx); + } + else + { + builder = builder.WithType(TxType.EIP1559); + } + + return builder .WithMaxFeePerGasIfSupports1559(1.GWei()) .SignedAndResolved(from).TestObject; + } parentHeader = chain.BlockTree.FindHeader(parentHash, BlockTreeLookupOptions.None)!; chain.StateReader.TryGetAccount(parentHeader.StateRoot!, from.Address, out AccountStruct account); diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.PayloadProduction.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.PayloadProduction.cs index d5e2cd080a4..3f4ee0c4213 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.PayloadProduction.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.PayloadProduction.cs @@ -62,7 +62,7 @@ public async Task getPayloadV1_should_return_error_if_called_after_cleanup_timer MergeConfig mergeConfig = new() { SecondsPerSlot = 1, TerminalTotalDifficulty = "0" }; TimeSpan timePerSlot = TimeSpan.FromMilliseconds(10); using MergeTestBlockchain chain = await CreateBlockchainWithImprovementContext( - chain => new BlockImprovementContextFactory(chain.BlockProductionTrigger, TimeSpan.FromSeconds(1)), + chain => new BlockImprovementContextFactory(chain.PostMergeBlockProducer!, TimeSpan.FromSeconds(1)), timePerSlot, mergeConfig); IEngineRpcModule rpc = CreateEngineModule(chain); @@ -134,7 +134,7 @@ public static IEnumerable WaitTestCases public async Task getPayloadV1_waits_for_block_production(TimeSpan delay) { using MergeTestBlockchain chain = await CreateBlockchainWithImprovementContext( - chain => new DelayBlockImprovementContextFactory(chain.BlockProductionTrigger, TimeSpan.FromSeconds(10), delay), + chain => new DelayBlockImprovementContextFactory(chain.PostMergeBlockProducer!, TimeSpan.FromSeconds(10), delay), TimeSpan.FromSeconds(10)); IEngineRpcModule rpc = CreateEngineModule(chain); @@ -298,7 +298,7 @@ public async Task getPayloadV1_picks_transactions_from_pool_constantly_improving TimeSpan delay = TimeSpan.FromMilliseconds(10); TimeSpan timePerSlot = 50 * delay; using MergeTestBlockchain chain = await CreateBlockchainWithImprovementContext( - chain => new StoringBlockImprovementContextFactory(new BlockImprovementContextFactory(chain.BlockProductionTrigger, TimeSpan.FromSeconds(chain.MergeConfig.SecondsPerSlot))), + chain => new StoringBlockImprovementContextFactory(new BlockImprovementContextFactory(chain.PostMergeBlockProducer!, TimeSpan.FromSeconds(chain.MergeConfig.SecondsPerSlot))), timePerSlot, delay: delay); StoringBlockImprovementContextFactory improvementContextFactory = (StoringBlockImprovementContextFactory)chain.BlockImprovementContextFactory; @@ -352,7 +352,7 @@ public async Task TestTwoTransaction_SameContract_WithBlockImprovement() TimeSpan delay = TimeSpan.FromMilliseconds(10); TimeSpan timePerSlot = 50 * delay; - StoringBlockImprovementContextFactory improvementContextFactory = new(new BlockImprovementContextFactory(chain.BlockProductionTrigger, TimeSpan.FromSeconds(chain.MergeConfig.SecondsPerSlot))); + StoringBlockImprovementContextFactory improvementContextFactory = new(new BlockImprovementContextFactory(chain.PostMergeBlockProducer!, TimeSpan.FromSeconds(chain.MergeConfig.SecondsPerSlot))); ConfigureBlockchainWithImprovementContextFactory(chain, improvementContextFactory, timePerSlot, delay); IEngineRpcModule rpc = CreateEngineModule(chain); @@ -389,7 +389,7 @@ public async Task getPayloadV1_doesnt_wait_for_improvement_when_block_is_not_emp TimeSpan timePerSlot = 50 * delay; using MergeTestBlockchain chain = await CreateBlockchainWithImprovementContext( chain => new StoringBlockImprovementContextFactory(new DelayBlockImprovementContextFactory( - chain.BlockProductionTrigger, TimeSpan.FromSeconds(chain.MergeConfig.SecondsPerSlot), 3 * delay)), + chain.PostMergeBlockProducer!, TimeSpan.FromSeconds(chain.MergeConfig.SecondsPerSlot), 3 * delay)), timePerSlot, delay: delay); StoringBlockImprovementContextFactory improvementContextFactory = (StoringBlockImprovementContextFactory)chain.BlockImprovementContextFactory; @@ -431,7 +431,7 @@ public async Task Cannot_build_invalid_block_with_the_branch() TimeSpan delay = TimeSpan.FromMilliseconds(10); TimeSpan timePerSlot = 4 * delay; ConfigureBlockchainWithImprovementContextFactory(chain, - new StoringBlockImprovementContextFactory(new BlockImprovementContextFactory(chain.BlockProductionTrigger, TimeSpan.FromSeconds(chain.MergeConfig.SecondsPerSlot))), + new StoringBlockImprovementContextFactory(new BlockImprovementContextFactory(chain.PostMergeBlockProducer!, TimeSpan.FromSeconds(chain.MergeConfig.SecondsPerSlot))), timePerSlot, delay); IEngineRpcModule rpc = CreateEngineModule(chain); @@ -511,7 +511,7 @@ public async Task Cannot_produce_bad_blocks() TimeSpan delay = TimeSpan.FromMilliseconds(10); TimeSpan timePerSlot = 4 * delay; using MergeTestBlockchain chain = await CreateBlockchainWithImprovementContext( - (chain) => new StoringBlockImprovementContextFactory(new BlockImprovementContextFactory(chain.BlockProductionTrigger, TimeSpan.FromSeconds(chain.MergeConfig.SecondsPerSlot))), + (chain) => new StoringBlockImprovementContextFactory(new BlockImprovementContextFactory(chain.PostMergeBlockProducer!, TimeSpan.FromSeconds(chain.MergeConfig.SecondsPerSlot))), timePerSlot, delay: delay); IEngineRpcModule rpc = CreateEngineModule(chain); diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.RelayBuilder.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.RelayBuilder.cs index 3132ef31e06..ba29dd47b27 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.RelayBuilder.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.RelayBuilder.cs @@ -48,7 +48,7 @@ public async Task forkchoiceUpdatedV1_should_communicate_with_boost_relay() }; }); - BoostBlockImprovementContextFactory improvementContextFactory = new(chain.BlockProductionTrigger, TimeSpan.FromSeconds(5), boostRelay, chain.StateReader); + BoostBlockImprovementContextFactory improvementContextFactory = new(chain.PostMergeBlockProducer!, TimeSpan.FromSeconds(5), boostRelay, chain.StateReader); TimeSpan timePerSlot = TimeSpan.FromSeconds(10); chain.PayloadPreparationService = new PayloadPreparationService( chain.PostMergeBlockProducer!, @@ -151,7 +151,7 @@ public virtual async Task forkchoiceUpdatedV1_should_communicate_with_boost_rela DefaultHttpClient defaultHttpClient = new(mockHttp.ToHttpClient(), serializer, chain.LogManager, 1, 100); BoostRelay boostRelay = new(defaultHttpClient, relayUrl); - BoostBlockImprovementContextFactory improvementContextFactory = new(chain.BlockProductionTrigger, TimeSpan.FromSeconds(5000), boostRelay, chain.StateReader); + BoostBlockImprovementContextFactory improvementContextFactory = new(chain.PostMergeBlockProducer!, TimeSpan.FromSeconds(5000), boostRelay, chain.StateReader); TimeSpan timePerSlot = TimeSpan.FromSeconds(1000); chain.PayloadPreparationService = new PayloadPreparationService( chain.PostMergeBlockProducer!, @@ -194,11 +194,11 @@ public async Task forkchoiceUpdatedV1_should_ignore_gas_limit([Values(false, tru boostRelay.GetPayloadAttributes(Arg.Any(), Arg.Any()) .Returns(c => (BoostPayloadAttributes)c.Arg()); - improvementContextFactory = new BoostBlockImprovementContextFactory(chain.BlockProductionTrigger, TimeSpan.FromSeconds(5), boostRelay, chain.StateReader); + improvementContextFactory = new BoostBlockImprovementContextFactory(chain.PostMergeBlockProducer!, TimeSpan.FromSeconds(5), boostRelay, chain.StateReader); } else { - improvementContextFactory = new BlockImprovementContextFactory(chain.BlockProductionTrigger, TimeSpan.FromSeconds(5)); + improvementContextFactory = new BlockImprovementContextFactory(chain.PostMergeBlockProducer!, TimeSpan.FromSeconds(5)); } TimeSpan timePerSlot = TimeSpan.FromSeconds(10); diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs index e83116afd87..cb82da27f3c 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs @@ -226,12 +226,11 @@ protected override IBlockProducer CreateTestBlockProducer(TxPoolTxSource txPoolT ConsensusRequestsProcessor); BlockProducerEnv blockProducerEnv = blockProducerEnvFactory.Create(); - PostMergeBlockProducer? postMergeBlockProducer = blockProducerFactory.Create( - blockProducerEnv, BlockProductionTrigger); + PostMergeBlockProducer? postMergeBlockProducer = blockProducerFactory.Create(blockProducerEnv); PostMergeBlockProducer = postMergeBlockProducer; PayloadPreparationService ??= new PayloadPreparationService( postMergeBlockProducer, - new BlockImprovementContextFactory(BlockProductionTrigger, TimeSpan.FromSeconds(MergeConfig.SecondsPerSlot)), + new BlockImprovementContextFactory(PostMergeBlockProducer, TimeSpan.FromSeconds(MergeConfig.SecondsPerSlot)), TimerFactory.Default, LogManager, TimeSpan.FromSeconds(MergeConfig.SecondsPerSlot), @@ -252,7 +251,6 @@ protected override IBlockProcessor CreateBlockProcessor() new BlockProcessor.BlockValidationTransactionsExecutor(TxProcessor, State), State, ReceiptStorage, - NullWitnessCollector.Instance, new BlockhashStore(BlockTree, SpecProvider, State), TxProcessor, LogManager, diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs index 847c6b5a0b9..0e6d59ef663 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V1.cs @@ -20,6 +20,7 @@ using Nethermind.Core.Test.Builders; using Nethermind.Crypto; using Nethermind.Evm; +using Nethermind.Facade.Eth; using Nethermind.HealthChecks; using Nethermind.Int256; using Nethermind.JsonRpc; @@ -1033,7 +1034,7 @@ public async Task executePayloadV1_processes_passed_transactions(bool moveHead) executePayloadRequest.GasUsed = GasCostOf.Transaction * count; executePayloadRequest.StateRoot = new Hash256("0x3d2e3ced6da0d1e94e65894dc091190480f045647610ef614e1cab4241ca66e0"); - executePayloadRequest.ReceiptsRoot = new Hash256("0xc538d36ed1acf6c28187110a2de3e5df707d6d38982f436eb0db7a623f9dc2cd"); + executePayloadRequest.ReceiptsRoot = new Hash256("0xb34a29e4a30ab5d32fdbc0292a97ac1cf1028c085f538dec2d91d91c6d0b0562"); TryCalculateHash(executePayloadRequest, out Hash256? hash); executePayloadRequest.BlockHash = hash; ResultWrapper result = await rpc.engine_newPayloadV1(executePayloadRequest); @@ -1071,7 +1072,7 @@ public async Task executePayloadV1_transactions_produce_receipts() executionPayload.StateRoot = new Hash256("0x3d2e3ced6da0d1e94e65894dc091190480f045647610ef614e1cab4241ca66e0"); executionPayload.ReceiptsRoot = - new Hash256("0xc538d36ed1acf6c28187110a2de3e5df707d6d38982f436eb0db7a623f9dc2cd"); + new Hash256("0xb34a29e4a30ab5d32fdbc0292a97ac1cf1028c085f538dec2d91d91c6d0b0562"); TryCalculateHash(executionPayload, out Hash256 hash); executionPayload.BlockHash = hash; ResultWrapper result = await rpc.engine_newPayloadV1(executionPayload); diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs index fb2d7fbf1d7..53736b24f91 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.V3.cs @@ -476,6 +476,39 @@ public async Task ForkChoiceUpdated_should_return_unsupported_fork_but_change_la }); } + [Test] + public async Task ForkChoiceUpdated_should_return_valid_for_previous_blocks_without_state_synced() + { + static void MarkAsUnprocessed(MergeTestBlockchain chain, int blockNumber) + { + ChainLevelInfo lvl = chain.ChainLevelInfoRepository.LoadLevel(blockNumber)!; + foreach (BlockInfo info in lvl.BlockInfos) + { + info.WasProcessed = false; + } + chain.ChainLevelInfoRepository.PersistLevel(blockNumber, lvl); + } + + const int BlockCount = 10; + const int SyncingBlockNumber = 5; + + MergeTestBlockchain chain = await CreateBlockchain(releaseSpec: Cancun.Instance); + IEngineRpcModule rpcModule = CreateEngineModule(chain, null, TimeSpan.FromDays(1)); + + for (var i = 1; i < BlockCount; i++) + { + await AddNewBlockV3(rpcModule, chain, 1); + } + + Hash256 syncingBlockHash = chain.BlockTree.FindBlock(SyncingBlockNumber)!.Hash!; + MarkAsUnprocessed(chain, SyncingBlockNumber); + + ResultWrapper res2 = await rpcModule.engine_forkchoiceUpdatedV3( + new ForkchoiceStateV1(syncingBlockHash, syncingBlockHash, syncingBlockHash), null); + + Assert.That(res2.Data.PayloadStatus.Status, Is.EqualTo(PayloadStatus.Valid)); + } + public static IEnumerable ForkchoiceUpdatedV3DeclinedTestCaseSource { get @@ -628,15 +661,40 @@ public static IEnumerable CancunFieldsTestSource } } - private async Task SendNewBlockV3(IEngineRpcModule rpc, MergeTestBlockchain chain, Withdrawal[]? withdrawals) + private async Task AddNewBlockV3(IEngineRpcModule rpcModule, MergeTestBlockchain chain, int transactionCount = 0) { - ExecutionPayloadV3 executionPayload = CreateBlockRequestV3( - chain, CreateParentBlockRequestOnHead(chain.BlockTree), TestItem.AddressD, withdrawals, 0, 0, parentBeaconBlockRoot: TestItem.KeccakE); - ResultWrapper executePayloadResult = await rpc.engine_newPayloadV3(executionPayload, Array.Empty(), executionPayload.ParentBeaconBlockRoot); + Transaction[] txs = BuildTransactions(chain, chain.BlockTree.Head!.Hash!, TestItem.PrivateKeyA, TestItem.AddressB, (uint)transactionCount, 0, out _, out _, 0); + chain.AddTransactions(txs); + + PayloadAttributes payloadAttributes = new() + { + Timestamp = chain.BlockTree.Head!.Timestamp + 1, + PrevRandao = TestItem.KeccakH, + SuggestedFeeRecipient = TestItem.AddressF, + Withdrawals = [], + ParentBeaconBlockRoot = TestItem.KeccakE + }; + Hash256 currentHeadHash = chain.BlockTree.HeadHash; + ForkchoiceStateV1 forkchoiceState = new(currentHeadHash, currentHeadHash, currentHeadHash); + + using SemaphoreSlim blockImprovementLock = new(0); + EventHandler onBlockImprovedHandler = (_, _) => blockImprovementLock.Release(1); + chain.PayloadPreparationService!.BlockImproved += onBlockImprovedHandler; + + string payloadId = (await rpcModule.engine_forkchoiceUpdatedV3(forkchoiceState, payloadAttributes)).Data.PayloadId!; + + await blockImprovementLock.WaitAsync(10000); + chain.PayloadPreparationService!.BlockImproved -= onBlockImprovedHandler; - executePayloadResult.Data.Status.Should().Be(PayloadStatus.Valid); + ResultWrapper payloadResult = await rpcModule.engine_getPayloadV3(Bytes.FromHexString(payloadId)); + Assert.That(payloadResult.Result, Is.EqualTo(Result.Success)); + Assert.That(payloadResult.Data, Is.Not.Null); - return executionPayload; + GetPayloadV3Result payload = payloadResult.Data; + await rpcModule.engine_newPayloadV3(payload.ExecutionPayload, payload.BlobsBundle.GetBlobVersionedHashes(), TestItem.KeccakE); + + ForkchoiceStateV1 newForkchoiceState = new(payload.ExecutionPayload.BlockHash, payload.ExecutionPayload.BlockHash, payload.ExecutionPayload.BlockHash); + await rpcModule.engine_forkchoiceUpdatedV3(newForkchoiceState, null); } private async Task<(IEngineRpcModule, string?, Transaction[], MergeTestBlockchain chain)> BuildAndGetPayloadV3Result( @@ -644,21 +702,18 @@ private async Task SendNewBlockV3(IEngineRpcModule rpc, MergeT { MergeTestBlockchain chain = await CreateBlockchain(releaseSpec: spec, null); IEngineRpcModule rpcModule = CreateEngineModule(chain, null, TimeSpan.FromDays(1)); - Transaction[] txs = Array.Empty(); + Transaction[] txs = []; + + using SemaphoreSlim blockImprovementLock = new(0); + EventHandler onBlockImprovedHandler = (_, _) => blockImprovementLock.Release(1); + chain.PayloadPreparationService!.BlockImproved += onBlockImprovedHandler; + + Hash256 currentHeadHash = chain.BlockTree.HeadHash; if (transactionCount is not 0) { - using SemaphoreSlim blockImprovementLock = new(0); - - ExecutionPayload executionPayload1 = await SendNewBlockV3(rpcModule, chain, Array.Empty()); - txs = BuildTransactions(chain, executionPayload1.BlockHash, TestItem.PrivateKeyA, TestItem.AddressB, (uint)transactionCount, 0, out _, out _, 1); + txs = BuildTransactions(chain, currentHeadHash, TestItem.PrivateKeyA, TestItem.AddressB, (uint)transactionCount, 0, out _, out _, 1); chain.AddTransactions(txs); - - EventHandler onBlockImprovedHandler = (_, _) => blockImprovementLock.Release(1); - - chain.PayloadPreparationService!.BlockImproved += onBlockImprovedHandler; - await blockImprovementLock.WaitAsync(10000); - chain.PayloadPreparationService!.BlockImproved -= onBlockImprovedHandler; } PayloadAttributes payloadAttributes = new() @@ -669,11 +724,19 @@ private async Task SendNewBlockV3(IEngineRpcModule rpc, MergeT Withdrawals = [TestItem.WithdrawalA_1Eth], ParentBeaconBlockRoot = spec.IsBeaconBlockRootAvailable ? TestItem.KeccakE : null }; - Hash256 currentHeadHash = chain.BlockTree.HeadHash; + ForkchoiceStateV1 forkchoiceState = new(currentHeadHash, currentHeadHash, currentHeadHash); + string? payloadId = spec.IsBeaconBlockRootAvailable ? rpcModule.engine_forkchoiceUpdatedV3(forkchoiceState, payloadAttributes).Result?.Data?.PayloadId : rpcModule.engine_forkchoiceUpdatedV2(forkchoiceState, payloadAttributes).Result?.Data?.PayloadId; + + if (transactionCount is not 0) + { + await blockImprovementLock.WaitAsync(10000); + } + chain.PayloadPreparationService!.BlockImproved -= onBlockImprovedHandler; + return (rpcModule, payloadId, txs, chain); } } diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/ExternalRpcIntegrationTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/ExternalRpcIntegrationTests.cs index 194e413c785..23b20746b5b 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/ExternalRpcIntegrationTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/ExternalRpcIntegrationTests.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; +using Nethermind.Facade.Eth; using Nethermind.Int256; using Nethermind.JsonRpc.Modules.Eth; using Nethermind.Overseer.Test.JsonRpc; diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/MergePluginTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/MergePluginTests.cs index e8f45b69687..ae09a209b5c 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/MergePluginTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/MergePluginTests.cs @@ -96,7 +96,7 @@ public void Init_merge_plugin_does_not_throw_exception(bool enabled) Assert.DoesNotThrowAsync(async () => await _plugin.Init(_context)); Assert.DoesNotThrowAsync(async () => await _plugin.InitNetworkProtocol()); Assert.DoesNotThrowAsync(async () => await _plugin.InitSynchronization()); - Assert.DoesNotThrowAsync(async () => await _plugin.InitBlockProducer(_consensusPlugin!, NeverProduceTrigger.Instance, null)); + Assert.DoesNotThrow(() => _plugin.InitBlockProducer(_consensusPlugin!, null)); Assert.DoesNotThrowAsync(async () => await _plugin.InitRpcModules()); Assert.DoesNotThrowAsync(async () => await _plugin.DisposeAsync()); } @@ -111,7 +111,7 @@ public async Task Initializes_correctly() ISyncConfig syncConfig = _context.Config(); Assert.IsTrue(syncConfig.NetworkingEnabled); Assert.IsTrue(_context.GossipPolicy.CanGossipBlocks); - await _plugin.InitBlockProducer(_consensusPlugin!, NeverProduceTrigger.Instance, null); + _plugin.InitBlockProducer(_consensusPlugin!, null); Assert.IsInstanceOf(_context.BlockProducer); await _plugin.InitRpcModules(); _context.RpcModuleProvider!.Received().Register(Arg.Is>(m => m is SingletonModulePool)); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/BlockImprovementContext.cs b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/BlockImprovementContext.cs index 059ad02f220..eeef3c2f39f 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/BlockImprovementContext.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/BlockImprovementContext.cs @@ -4,6 +4,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Nethermind.Consensus; using Nethermind.Consensus.Producers; using Nethermind.Core; using Nethermind.Core.Extensions; @@ -18,7 +19,7 @@ public class BlockImprovementContext : IBlockImprovementContext private readonly FeesTracer _feesTracer = new(); public BlockImprovementContext(Block currentBestBlock, - IManualBlockProductionTrigger blockProductionTrigger, + IBlockProducer blockProducer, TimeSpan timeout, BlockHeader parentHeader, PayloadAttributes payloadAttributes, @@ -27,8 +28,8 @@ public BlockImprovementContext(Block currentBestBlock, _cancellationTokenSource = new CancellationTokenSource(timeout); CurrentBestBlock = currentBestBlock; StartDateTime = startDateTime; - ImprovementTask = blockProductionTrigger - .BuildBlock(parentHeader, _cancellationTokenSource.Token, _feesTracer, payloadAttributes) + ImprovementTask = blockProducer + .BuildBlock(parentHeader, _feesTracer, payloadAttributes, _cancellationTokenSource.Token) .ContinueWith(SetCurrentBestBlock, _cancellationTokenSource.Token); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/BlockImprovementContextFactory.cs b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/BlockImprovementContextFactory.cs index 9540077a323..df2aa910e15 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/BlockImprovementContextFactory.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/BlockImprovementContextFactory.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using Nethermind.Consensus; using Nethermind.Consensus.Producers; using Nethermind.Core; @@ -9,12 +10,12 @@ namespace Nethermind.Merge.Plugin.BlockProduction; public class BlockImprovementContextFactory : IBlockImprovementContextFactory { - private readonly IManualBlockProductionTrigger _blockProductionTrigger; + private readonly IBlockProducer _blockProducer; private readonly TimeSpan _timeout; - public BlockImprovementContextFactory(IManualBlockProductionTrigger blockProductionTrigger, TimeSpan timeout) + public BlockImprovementContextFactory(IBlockProducer blockProducer, TimeSpan timeout) { - _blockProductionTrigger = blockProductionTrigger; + _blockProducer = blockProducer; _timeout = timeout; } @@ -23,5 +24,5 @@ public IBlockImprovementContext StartBlockImprovementContext( BlockHeader parentHeader, PayloadAttributes payloadAttributes, DateTimeOffset startDateTime) => - new BlockImprovementContext(currentBestBlock, _blockProductionTrigger, _timeout, parentHeader, payloadAttributes, startDateTime); + new BlockImprovementContext(currentBestBlock, _blockProducer, _timeout, parentHeader, payloadAttributes, startDateTime); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/Boost/BoostBlockImprovementContext.cs b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/Boost/BoostBlockImprovementContext.cs index 97059251647..e90d8d8401b 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/Boost/BoostBlockImprovementContext.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/Boost/BoostBlockImprovementContext.cs @@ -4,6 +4,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Nethermind.Consensus; using Nethermind.Consensus.Producers; using Nethermind.Core; using Nethermind.Core.Extensions; @@ -22,7 +23,7 @@ public class BoostBlockImprovementContext : IBlockImprovementContext private CancellationTokenSource? _cancellationTokenSource; public BoostBlockImprovementContext(Block currentBestBlock, - IManualBlockProductionTrigger blockProductionTrigger, + IBlockProducer blockProducer, TimeSpan timeout, BlockHeader parentHeader, PayloadAttributes payloadAttributes, @@ -35,11 +36,11 @@ public BoostBlockImprovementContext(Block currentBestBlock, _cancellationTokenSource = new CancellationTokenSource(timeout); CurrentBestBlock = currentBestBlock; StartDateTime = startDateTime; - ImprovementTask = StartImprovingBlock(blockProductionTrigger, parentHeader, payloadAttributes, _cancellationTokenSource.Token); + ImprovementTask = StartImprovingBlock(blockProducer, parentHeader, payloadAttributes, _cancellationTokenSource.Token); } private async Task StartImprovingBlock( - IManualBlockProductionTrigger blockProductionTrigger, + IBlockProducer blockProducer, BlockHeader parentHeader, PayloadAttributes payloadAttributes, CancellationToken cancellationToken) @@ -48,7 +49,7 @@ public BoostBlockImprovementContext(Block currentBestBlock, payloadAttributes = await _boostRelay.GetPayloadAttributes(payloadAttributes, cancellationToken); _stateReader.TryGetAccount(parentHeader.StateRoot!, payloadAttributes.SuggestedFeeRecipient, out AccountStruct account); UInt256 balanceBefore = account.Balance; - Block? block = await blockProductionTrigger.BuildBlock(parentHeader, cancellationToken, _feesTracer, payloadAttributes); + Block? block = await blockProducer.BuildBlock(parentHeader, _feesTracer, payloadAttributes, cancellationToken); if (block is not null) { CurrentBestBlock = block; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/Boost/BoostBlockImprovementContextFactory.cs b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/Boost/BoostBlockImprovementContextFactory.cs index 06c6bf79956..fcd85831112 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/Boost/BoostBlockImprovementContextFactory.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/Boost/BoostBlockImprovementContextFactory.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using Nethermind.Consensus; using Nethermind.Consensus.Producers; using Nethermind.Core; using Nethermind.State; @@ -10,14 +11,14 @@ namespace Nethermind.Merge.Plugin.BlockProduction.Boost; public class BoostBlockImprovementContextFactory : IBlockImprovementContextFactory { - private readonly IManualBlockProductionTrigger _blockProductionTrigger; + private readonly IBlockProducer _blockProducer; private readonly TimeSpan _timeout; private readonly IBoostRelay _boostRelay; private readonly IStateReader _stateReader; - public BoostBlockImprovementContextFactory(IManualBlockProductionTrigger blockProductionTrigger, TimeSpan timeout, IBoostRelay boostRelay, IStateReader stateReader) + public BoostBlockImprovementContextFactory(IBlockProducer blockProducer, TimeSpan timeout, IBoostRelay boostRelay, IStateReader stateReader) { - _blockProductionTrigger = blockProductionTrigger; + _blockProducer = blockProducer; _timeout = timeout; _boostRelay = boostRelay; _stateReader = stateReader; @@ -28,5 +29,5 @@ public IBlockImprovementContext StartBlockImprovementContext( BlockHeader parentHeader, PayloadAttributes payloadAttributes, DateTimeOffset startDateTime) => - new BoostBlockImprovementContext(currentBestBlock, _blockProductionTrigger, _timeout, parentHeader, payloadAttributes, _boostRelay, _stateReader, startDateTime); + new BoostBlockImprovementContext(currentBestBlock, _blockProducer, _timeout, parentHeader, payloadAttributes, _boostRelay, _stateReader, startDateTime); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/MergeBlockProducer.cs b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/MergeBlockProducer.cs index ab662a624be..cb3592a36ba 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/MergeBlockProducer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/MergeBlockProducer.cs @@ -2,9 +2,12 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Threading; using System.Threading.Tasks; using Nethermind.Consensus; +using Nethermind.Consensus.Producers; using Nethermind.Core; +using Nethermind.Evm.Tracing; namespace Nethermind.Merge.Plugin.BlockProduction; @@ -20,45 +23,13 @@ public MergeBlockProducer(IBlockProducer? preMergeProducer, IBlockProducer? post _preMergeProducer = preMergeProducer; _eth2BlockProducer = postMergeBlockProducer ?? throw new ArgumentNullException(nameof(postMergeBlockProducer)); _poSSwitcher = poSSwitcher ?? throw new ArgumentNullException(nameof(poSSwitcher)); - _poSSwitcher.TerminalBlockReached += OnSwitchHappened; - if (HasPreMergeProducer) - _preMergeProducer!.BlockProduced += OnBlockProduced; - - postMergeBlockProducer.BlockProduced += OnBlockProduced; - } - - private void OnBlockProduced(object? sender, BlockEventArgs e) - { - BlockProduced?.Invoke(this, e); - } - - private void OnSwitchHappened(object? sender, EventArgs e) - { - _preMergeProducer?.StopAsync(); } - public async Task Start() - { - await _eth2BlockProducer.Start(); - if (_poSSwitcher.HasEverReachedTerminalBlock() == false && HasPreMergeProducer) - { - await _preMergeProducer!.Start(); - } - } - - public async Task StopAsync() - { - await _eth2BlockProducer.StopAsync(); - if (_poSSwitcher.HasEverReachedTerminalBlock() && HasPreMergeProducer) - await _preMergeProducer!.StopAsync(); - } - - public bool IsProducingBlocks(ulong? maxProducingInterval) + public Task BuildBlock(BlockHeader? parentHeader, IBlockTracer? blockTracer = null, + PayloadAttributes? payloadAttributes = null, CancellationToken? token = null) { return _poSSwitcher.HasEverReachedTerminalBlock() || HasPreMergeProducer == false - ? _eth2BlockProducer.IsProducingBlocks(maxProducingInterval) - : _preMergeProducer!.IsProducingBlocks(maxProducingInterval); + ? _eth2BlockProducer.BuildBlock(parentHeader, blockTracer, payloadAttributes, token) + : _preMergeProducer!.BuildBlock(parentHeader, blockTracer, payloadAttributes, token); } - - public event EventHandler? BlockProduced; } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/MergeBlockProducerRunner.cs b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/MergeBlockProducerRunner.cs new file mode 100644 index 00000000000..5521a1c339c --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/MergeBlockProducerRunner.cs @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading.Tasks; +using Nethermind.Consensus; +using Nethermind.Core; + +namespace Nethermind.Merge.Plugin.BlockProduction; + +public class MergeBlockProducerRunner : IBlockProducerRunner +{ + private readonly IBlockProducerRunner? _preMergeProducer; + private readonly IBlockProducerRunner _eth2BlockProducer; + private readonly IPoSSwitcher _poSSwitcher; + private bool HasPreMergeProducer => _preMergeProducer is not null; + + public MergeBlockProducerRunner(IBlockProducerRunner? preMergeProducer, IBlockProducerRunner? postMergeBlockProducer, IPoSSwitcher? poSSwitcher) + { + _preMergeProducer = preMergeProducer; + _eth2BlockProducer = postMergeBlockProducer ?? throw new ArgumentNullException(nameof(postMergeBlockProducer)); + _poSSwitcher = poSSwitcher ?? throw new ArgumentNullException(nameof(poSSwitcher)); + _poSSwitcher.TerminalBlockReached += OnSwitchHappened; + if (HasPreMergeProducer) + _preMergeProducer!.BlockProduced += OnBlockProduced; + + postMergeBlockProducer.BlockProduced += OnBlockProduced; + } + + private void OnBlockProduced(object? sender, BlockEventArgs e) + { + BlockProduced?.Invoke(this, e); + } + + private void OnSwitchHappened(object? sender, EventArgs e) + { + _preMergeProducer?.StopAsync(); + } + + public void Start() + { + _eth2BlockProducer.Start(); + if (_poSSwitcher.HasEverReachedTerminalBlock() == false && HasPreMergeProducer) + { + _preMergeProducer!.Start(); + } + } + + public async Task StopAsync() + { + await _eth2BlockProducer.StopAsync(); + if (_poSSwitcher.HasEverReachedTerminalBlock() && HasPreMergeProducer) + await _preMergeProducer!.StopAsync(); + } + + public bool IsProducingBlocks(ulong? maxProducingInterval) + { + return _poSSwitcher.HasEverReachedTerminalBlock() || HasPreMergeProducer == false + ? _eth2BlockProducer.IsProducingBlocks(maxProducingInterval) + : _preMergeProducer!.IsProducingBlocks(maxProducingInterval); + } + + public event EventHandler? BlockProduced; +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/PostMergeBlockProducer.cs b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/PostMergeBlockProducer.cs index dc0d94753ae..16b69b24a1f 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/PostMergeBlockProducer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/PostMergeBlockProducer.cs @@ -23,7 +23,6 @@ public PostMergeBlockProducer( ITxSource txSource, IBlockchainProcessor processor, IBlockTree blockTree, - IBlockProductionTrigger blockProductionTrigger, IWorldState stateProvider, IGasLimitCalculator gasLimitCalculator, ISealEngine sealEngine, @@ -36,7 +35,6 @@ public PostMergeBlockProducer( processor, sealEngine, blockTree, - blockProductionTrigger, stateProvider, gasLimitCalculator, timestamper, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/PostMergeBlockProducerFactory.cs b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/PostMergeBlockProducerFactory.cs index a9e203829bd..4e1e1317629 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/PostMergeBlockProducerFactory.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/BlockProduction/PostMergeBlockProducerFactory.cs @@ -3,7 +3,6 @@ using Nethermind.Config; using Nethermind.Consensus; -using Nethermind.Consensus.Producers; using Nethermind.Consensus.Transactions; using Nethermind.Core; using Nethermind.Core.Specs; @@ -38,7 +37,6 @@ public PostMergeBlockProducerFactory( public virtual PostMergeBlockProducer Create( BlockProducerEnv producerEnv, - IBlockProductionTrigger blockProductionTrigger, ITxSource? txSource = null) { @@ -46,7 +44,6 @@ public virtual PostMergeBlockProducer Create( txSource ?? producerEnv.TxSource, producerEnv.ChainProcessor, producerEnv.BlockTree, - blockProductionTrigger, producerEnv.ReadOnlyStateProvider, _gasLimitCalculator ?? new TargetAdjustedGasLimitCalculator(_specProvider, _blocksConfig), _sealEngine, diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/ClientVersionV1.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/ClientVersionV1.cs index dafad038918..e2108102c3f 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Data/ClientVersionV1.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/ClientVersionV1.cs @@ -16,7 +16,9 @@ public ClientVersionV1() Code = ProductInfo.ClientCode; Name = ProductInfo.Name; Version = ProductInfo.Version; - Commit = ProductInfo.Commit; + Commit = ProductInfo.Commit.Length < 8 + ? string.Empty.PadLeft(8, '0') + : ProductInfo.Commit[..8]; } public string Code { get; } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Data/NewPayloadV1Result.cs b/src/Nethermind/Nethermind.Merge.Plugin/Data/NewPayloadV1Result.cs index 502123b1732..ad97093144f 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Data/NewPayloadV1Result.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Data/NewPayloadV1Result.cs @@ -12,9 +12,7 @@ namespace Nethermind.Merge.Plugin.Data; /// public static class NewPayloadV1Result { - public static ResultWrapper Syncing = ResultWrapper.Success(PayloadStatusV1.Syncing); - - public static ResultWrapper Accepted = ResultWrapper.Success(PayloadStatusV1.Accepted); + public static readonly ResultWrapper Syncing = ResultWrapper.Success(PayloadStatusV1.Syncing); public static ResultWrapper Invalid(string validationError) { diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/ForkchoiceUpdatedHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/ForkchoiceUpdatedHandler.cs index 3360272a11c..94df1f54b71 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/ForkchoiceUpdatedHandler.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/ForkchoiceUpdatedHandler.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using Nethermind.Blockchain; using Nethermind.Blockchain.Find; -using Nethermind.Blockchain.Synchronization; using Nethermind.Consensus; using Nethermind.Consensus.Processing; using Nethermind.Consensus.Producers; @@ -22,9 +21,7 @@ using Nethermind.Merge.Plugin.Data; using Nethermind.Merge.Plugin.InvalidChainTracker; using Nethermind.Merge.Plugin.Synchronization; -using Nethermind.Synchronization; using Nethermind.Synchronization.Peers; -using Nethermind.Synchronization.Peers.AllocationStrategies; namespace Nethermind.Merge.Plugin.Handlers; @@ -98,7 +95,11 @@ public async Task> Handle(ForkchoiceSta private async Task?> ApplyForkchoiceUpdate(Block? newHeadBlock, ForkchoiceStateV1 forkchoiceState, PayloadAttributes? payloadAttributes) { - using var handle = Thread.CurrentThread.BoostPriority(); + // if a head is unknown we are syncing + bool isDefinitelySyncing = newHeadBlock is null; + using ThreadExtensions.Disposable handle = isDefinitelySyncing ? + default : // Don't boost priority if we are definitely syncing + Thread.CurrentThread.BoostPriority(); if (_invalidChainTracker.IsOnKnownInvalidChain(forkchoiceState.HeadBlockHash, out Hash256? lastValidHash)) { @@ -106,7 +107,7 @@ public async Task> Handle(ForkchoiceSta return ForkchoiceUpdatedV1Result.Invalid(lastValidHash); } - if (newHeadBlock is null) // if a head is unknown we are syncing + if (isDefinitelySyncing) { string simpleRequestStr = payloadAttributes is null ? forkchoiceState.ToString() : $"{forkchoiceState} {payloadAttributes}"; if (_logger.IsInfo) _logger.Info($"Received {simpleRequestStr}"); @@ -134,7 +135,7 @@ public async Task> Handle(ForkchoiceSta return ForkchoiceUpdatedV1Result.Syncing; } - BlockInfo? blockInfo = _blockTree.GetInfo(newHeadBlock.Number, newHeadBlock.GetOrCalculateHash()).Info; + BlockInfo? blockInfo = _blockTree.GetInfo(newHeadBlock!.Number, newHeadBlock.GetOrCalculateHash()).Info; BlockHeader? safeBlockHeader = ValidateBlockHash(forkchoiceState.SafeBlockHash, out string? safeBlockErrorMsg); BlockHeader? finalizedHeader = ValidateBlockHash(forkchoiceState.FinalizedBlockHash, out string? finalizationErrorMsg); string requestStr = payloadAttributes is null @@ -151,6 +152,12 @@ public async Task> Handle(ForkchoiceSta if (!blockInfo.WasProcessed) { + if (_blockTree.IsOnMainChainBehindHead(newHeadBlock)) + { + if (_logger.IsInfo) _logger.Info($"Valid. ForkChoiceUpdated ignored - already in canonical chain. Request: {requestStr}."); + return ForkchoiceUpdatedV1Result.Valid(null, forkchoiceState.HeadBlockHash); + } + BlockHeader? blockParent = _blockTree.FindHeader(newHeadBlock.ParentHash!, blockNumber: newHeadBlock.Number - 1); if (blockParent is null) { diff --git a/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Cancun.cs b/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Cancun.cs index 715cd1b44e7..1510791e28b 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Cancun.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Cancun.cs @@ -14,7 +14,7 @@ public partial interface IEngineRpcModule : IRpcModule { [JsonRpcMethod( - Description = "Verifies the payload according to the execution environment rules and returns the verification status and hash of the last valid block.", + Description = "Applies fork choice and starts building a new block if payload attributes are present.", IsSharable = true, IsImplemented = true)] Task> engine_forkchoiceUpdatedV3(ForkchoiceStateV1 forkchoiceState, PayloadAttributes? payloadAttributes = null); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Paris.cs b/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Paris.cs index 49d5f898b03..327a93d99cf 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Paris.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Paris.cs @@ -18,7 +18,7 @@ public partial interface IEngineRpcModule : IRpcModule ResultWrapper engine_exchangeTransitionConfigurationV1(TransitionConfigurationV1 beaconTransitionConfiguration); [JsonRpcMethod( - Description = "Verifies the payload according to the execution environment rules and returns the verification status and hash of the last valid block.", + Description = "Applies fork choice and starts building a new block if payload attributes are present.", IsSharable = true, IsImplemented = true)] Task> engine_forkchoiceUpdatedV1(ForkchoiceStateV1 forkchoiceState, PayloadAttributes? payloadAttributes = null); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Shanghai.cs b/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Shanghai.cs index 38060d6beea..c1b63e3d301 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Shanghai.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/IEngineRpcModule.Shanghai.cs @@ -14,7 +14,7 @@ namespace Nethermind.Merge.Plugin; public partial interface IEngineRpcModule : IRpcModule { [JsonRpcMethod( - Description = "Verifies the payload according to the execution environment rules and returns the verification status and hash of the last valid block.", + Description = "Applies fork choice and starts building a new block if payload attributes are present.", IsSharable = true, IsImplemented = true)] Task> engine_forkchoiceUpdatedV2(ForkchoiceStateV1 forkchoiceState, PayloadAttributes? payloadAttributes = null); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergeGossipPolicy.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergeGossipPolicy.cs index 91fa59badba..31f676a02dd 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergeGossipPolicy.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergeGossipPolicy.cs @@ -36,7 +36,7 @@ public MergeGossipPolicy( // According to spec (https://github.com/ethereum/EIPs/blob/d896145678bd65d3eafd8749690c1b5228875c39/EIPS/eip-3675.md#network) // We MUST discard NewBlock/NewBlockHash messages after receiving FIRST_FINALIZED_BLOCK. public bool ShouldDiscardBlocks => _poSSwitcher.TransitionFinished || - (_blockCacheService.FinalizedHash is not null && _blockCacheService.FinalizedHash != Keccak.Zero); /* _blockCacheService.FinalizedHash != null && _blockCacheService.FinalizedHash != Keccak.Zero + (_blockCacheService.FinalizedHash is not null && _blockCacheService.FinalizedHash != Keccak.Zero); /* _blockCacheService.FinalizedHash is not null && _blockCacheService.FinalizedHash != Keccak.Zero This condition was added for edge case situation. We started beacon sync, and we hadn't reached transition yet. If CL sent us non zero finalization hash, it would mean that network reached transition. However, in edge case situation (verified by merge hive tests), our node needs to be reorged to PoW again, so we can't add this condition _blockCacheService.FinalizedHash != Keccak.Zero diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.BlockProducer.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.BlockProducer.cs index 38138684d89..2ad0d4f412e 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.BlockProducer.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.BlockProducer.cs @@ -3,9 +3,7 @@ using System; using System.Threading.Tasks; -using Nethermind.Api.Extensions; using Nethermind.Consensus; -using Nethermind.Consensus.Producers; using Nethermind.Consensus.Transactions; using Nethermind.Core; using Nethermind.Merge.Plugin.BlockProduction; @@ -16,13 +14,12 @@ namespace Nethermind.Merge.Plugin public partial class MergePlugin { protected PostMergeBlockProducer _postMergeBlockProducer = null!; - protected IManualBlockProductionTrigger? _blockProductionTrigger = null; protected ManualTimestamper? _manualTimestamper; protected virtual PostMergeBlockProducerFactory CreateBlockProducerFactory() => new(_api.SpecProvider!, _api.SealEngine, _manualTimestamper!, _blocksConfig, _api.LogManager); - public virtual async Task InitBlockProducer(IBlockProducerFactory baseBlockProducerFactory, IBlockProductionTrigger blockProductionTrigger, ITxSource? txSource) + public virtual IBlockProducer InitBlockProducer(IBlockProducerFactory baseBlockProducerFactory, ITxSource? txSource) { if (MergeEnabled) { @@ -45,25 +42,33 @@ public virtual async Task InitBlockProducer(IBlockProducerFactor if (_logger.IsInfo) _logger.Info("Starting Merge block producer & sealer"); IBlockProducer? blockProducer = _mergeBlockProductionPolicy.ShouldInitPreMergeBlockProduction() - ? await baseBlockProducerFactory.InitBlockProducer(blockProductionTrigger, txSource) + ? baseBlockProducerFactory.InitBlockProducer(txSource) : null; _manualTimestamper ??= new ManualTimestamper(); - _blockProductionTrigger = new BuildBlocksWhenRequested(); BlockProducerEnv blockProducerEnv = _api.BlockProducerEnvFactory.Create(); _api.SealEngine = new MergeSealEngine(_api.SealEngine, _poSSwitcher, _api.SealValidator, _api.LogManager); _api.Sealer = _api.SealEngine; - _postMergeBlockProducer = CreateBlockProducerFactory().Create( - blockProducerEnv, - _blockProductionTrigger - ); - + _postMergeBlockProducer = CreateBlockProducerFactory().Create(blockProducerEnv); _api.BlockProducer = new MergeBlockProducer(blockProducer, _postMergeBlockProducer, _poSSwitcher); } return _api.BlockProducer!; } + public IBlockProducerRunner InitBlockProducerRunner(IBlockProducerRunner baseRunner) + { + if (MergeEnabled) + { + // The trigger can be different, so need to stop the old block production runner at this point. + StandardBlockProducerRunner postMergeRunner = new StandardBlockProducerRunner( + _api.ManualBlockProductionTrigger, _api.BlockTree!, _api.BlockProducer!); + return new MergeBlockProducerRunner(baseRunner, postMergeRunner, _poSSwitcher); + } + + return baseRunner; + } + // this looks redundant but Enabled actually comes from IConsensusWrapperPlugin // while MergeEnabled comes from merge config public bool Enabled => MergeEnabled; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 88621ba0177..e5bbb6d07ce 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -284,12 +284,8 @@ public Task InitRpcModules() if (_api.StateReader is null) throw new ArgumentNullException(nameof(_api.StateReader)); if (_beaconPivot is null) throw new ArgumentNullException(nameof(_beaconPivot)); if (_beaconSync is null) throw new ArgumentNullException(nameof(_beaconSync)); - if (_blockProductionTrigger is null) throw new ArgumentNullException(nameof(_blockProductionTrigger)); if (_peerRefresher is null) throw new ArgumentNullException(nameof(_peerRefresher)); - - if (_postMergeBlockProducer is null) throw new ArgumentNullException(nameof(_postMergeBlockProducer)); - if (_blockProductionTrigger is null) throw new ArgumentNullException(nameof(_blockProductionTrigger)); // ToDo: ugly temporary hack to not receive engine API messages before end of processing of all blocks after restart. Then we will wait 5s more to ensure everything is processed while (!_api.BlockProcessingQueue.IsEmpty) @@ -301,13 +297,13 @@ public Task InitRpcModules() IBlockImprovementContextFactory improvementContextFactory; if (string.IsNullOrEmpty(_mergeConfig.BuilderRelayUrl)) { - improvementContextFactory = new BlockImprovementContextFactory(_blockProductionTrigger, TimeSpan.FromSeconds(_blocksConfig.SecondsPerSlot)); + improvementContextFactory = new BlockImprovementContextFactory(_api.BlockProducer!, TimeSpan.FromSeconds(_blocksConfig.SecondsPerSlot)); } else { DefaultHttpClient httpClient = new(new HttpClient(), _api.EthereumJsonSerializer, _api.LogManager, retryDelayMilliseconds: 100); IBoostRelay boostRelay = new BoostRelay(httpClient, _mergeConfig.BuilderRelayUrl); - BoostBlockImprovementContextFactory boostBlockImprovementContextFactory = new(_blockProductionTrigger, TimeSpan.FromSeconds(_blocksConfig.SecondsPerSlot), boostRelay, _api.StateReader); + BoostBlockImprovementContextFactory boostBlockImprovementContextFactory = new(_api.BlockProducer!, TimeSpan.FromSeconds(_blocksConfig.SecondsPerSlot), boostRelay, _api.StateReader); improvementContextFactory = boostBlockImprovementContextFactory; } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs index 3173a05fe16..e808719cf13 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs @@ -227,9 +227,9 @@ bool HasMoreToSync(out BlockHeader[]? headers, out int headersToRequest) if (shouldProcess) { - // An edge case when we've got state already, but still downloading blocks before it. - // We cannot process such blocks, but still we are requested to process them via blocksRequest.Options - // So we'are detecting it and chaing from processing to receipts downloading + // An edge case where we already have the state but are still downloading preceding blocks. + // We cannot process such blocks, but we are still requested to process them via blocksRequest.Options. + // Therefore, we detect this situation and switch from processing to receipts downloading. bool headIsGenesis = _blockTree.Head?.IsGenesis ?? false; bool toBeProcessedHasNoProcessedParent = currentBlock.Number > (bestProcessedBlock + 1); bool isFastSyncTransition = headIsGenesis && toBeProcessedHasNoProcessedParent; diff --git a/src/Nethermind/Nethermind.Merkleization/Merkleizer.cs b/src/Nethermind/Nethermind.Merkleization/Merkleizer.cs index 6fa128c9fbf..1713b34da38 100644 --- a/src/Nethermind/Nethermind.Merkleization/Merkleizer.cs +++ b/src/Nethermind/Nethermind.Merkleization/Merkleizer.cs @@ -444,7 +444,7 @@ public void FeedBitlist(BitArray bitArray, ulong maximumBitlistLength) //{ // if (value.Root is null) // { - // if (value.Item == null) + // if (value.Item is null) // { // return; // } diff --git a/src/Nethermind/Nethermind.Mev.Test/MevPluginTests.cs b/src/Nethermind/Nethermind.Mev.Test/MevPluginTests.cs index 64d05280d73..f9a4def7e3f 100644 --- a/src/Nethermind/Nethermind.Mev.Test/MevPluginTests.cs +++ b/src/Nethermind/Nethermind.Mev.Test/MevPluginTests.cs @@ -49,11 +49,11 @@ public async Task Can_initialize_block_producer() await plugin.InitRpcModules(); IConsensusPlugin consensusPlugin = Substitute.For(); - consensusPlugin.InitBlockProducer(NeverProduceTrigger.Instance, null).Returns(Substitute.For()); + consensusPlugin.InitBlockProducer(null).Returns(Substitute.For()); - Task blockProducer = plugin.InitBlockProducer(consensusPlugin, NeverProduceTrigger.Instance, null); + IBlockProducer blockProducer = plugin.InitBlockProducer(consensusPlugin, null); - blockProducer.Result.Should().BeOfType(typeof(MevBlockProducer)); + blockProducer.Should().BeOfType(typeof(MevBlockProducer)); } } } diff --git a/src/Nethermind/Nethermind.Mev.Test/MevRpcModuleTests.TestMevRpcBlockchain.cs b/src/Nethermind/Nethermind.Mev.Test/MevRpcModuleTests.TestMevRpcBlockchain.cs index 66f85dd2140..8d0f2bdc5f3 100644 --- a/src/Nethermind/Nethermind.Mev.Test/MevRpcModuleTests.TestMevRpcBlockchain.cs +++ b/src/Nethermind/Nethermind.Mev.Test/MevRpcModuleTests.TestMevRpcBlockchain.cs @@ -109,42 +109,22 @@ protected override IBlockProducer CreateTestBlockProducer(TxPoolTxSource txPoolT new MevBlockProducerTransactionsExecutorFactory(SpecProvider, LogManager) }; - PostMergeBlockProducer CreatePostMergeBlockProducer(IBlockProductionTrigger blockProductionTrigger, - ITxSource? txSource = null) + PostMergeBlockProducer CreatePostMergeBlockProducer(ITxSource? txSource = null) { BlockProducerEnv blockProducerEnv = blockProducerEnvFactory.Create(txSource); return new PostMergeBlockProducerFactory(SpecProvider, SealEngine, Timestamper, blocksConfig, - LogManager).Create( - blockProducerEnv, blockProductionTrigger, txSource); + LogManager).Create(blockProducerEnv, txSource); } MevBlockProducer.MevBlockProducerInfo CreateProducer(int bundleLimit = 0, ITxSource? additionalTxSource = null) { - // TODO: this could be simplified a lot of the parent was not retrieved, not sure why do we need the parent here - bool BundleLimitTriggerCondition(BlockProductionEventArgs e) - { - // TODO: why do we need this parent? later we use only the current block number - BlockHeader? parent = BlockTree.GetProducedBlockParent(e.ParentHeader); - if (parent is not null) - { - // ToDo resolved conflict parent.Timestamp? - IEnumerable bundles = BundlePool.GetBundles(parent.Number + 1, parent.Timestamp); - return bundles.Count() >= bundleLimit; - } - - return false; - } - - IManualBlockProductionTrigger manualTrigger = new BuildBlocksWhenRequested(); - IBlockProductionTrigger trigger = manualTrigger; - if (bundleLimit != 0) - { - trigger = new TriggerWithCondition(manualTrigger, BundleLimitTriggerCondition); - } - - IBlockProducer producer = CreatePostMergeBlockProducer(trigger, additionalTxSource); - return new MevBlockProducer.MevBlockProducerInfo(producer, manualTrigger, new BeneficiaryTracer()); + IBlockProductionCondition condition = bundleLimit == 0 + ? AlwaysOkBlockProductionCondition.Instance + : new TestBundleLimitBlockProductionTrigger(BlockTree, BundlePool, bundleLimit); + + IBlockProducer producer = CreatePostMergeBlockProducer(additionalTxSource); + return new MevBlockProducer.MevBlockProducerInfo(producer, condition, new BeneficiaryTracer()); } int megabundleProducerCount = _relayAddresses.Length != 0 ? 1 : 0; @@ -172,11 +152,17 @@ bool BundleLimitTriggerCondition(BlockProductionEventArgs e) blockProducers.Add(bundleProducer); } - MevBlockProducer blockProducer = new MevBlockProducer(BlockProductionTrigger, LogManager, blockProducers.ToArray()); - blockProducer.BlockProduced += OnBlockProduced; + MevBlockProducer blockProducer = new MevBlockProducer(LogManager, blockProducers.ToArray()); return blockProducer; } + protected override IBlockProducerRunner CreateBlockProducerRunner() + { + IBlockProducerRunner baseRunner = base.CreateBlockProducerRunner(); + baseRunner.BlockProduced += OnBlockProduced; + return baseRunner; + } + private void OnBlockProduced(object? sender, BlockEventArgs e) { BlockTree.SuggestBlock(e.Block, BlockTreeSuggestOptions.ForceDontSetAsMain); @@ -201,7 +187,6 @@ protected override BlockProcessor CreateBlockProcessor() new BlockProcessor.BlockValidationTransactionsExecutor(TxProcessor, State), State, ReceiptStorage, - NullWitnessCollector.Instance, new BlockhashStore(BlockTree, SpecProvider, State), TxProcessor, LogManager); @@ -268,5 +253,28 @@ public MevBundle SendBundle(int blockNumber, params BundleTransaction[] txs) return new MevBundle(blockNumber, txs); } } + + private class TestBundleLimitBlockProductionTrigger( + IBlockTree blockTree, + IBundlePool bundlePool, + int bundleLimit + ) : IBlockProductionCondition + { + + // TODO: this could be simplified a lot of the parent was not retrieved, not sure why do we need the parent here + public bool CanProduce(BlockHeader parentHeader) + { + // TODO: why do we need this parent? later we use only the current block number + BlockHeader? parent = blockTree.GetProducedBlockParent(parentHeader); + if (parent is not null) + { + // ToDo resolved conflict parent.Timestamp? + IEnumerable bundles = bundlePool.GetBundles(parent.Number + 1, parent.Timestamp); + return bundles.Count() >= bundleLimit; + } + + return false; + } + } } } diff --git a/src/Nethermind/Nethermind.Mev/Execution/TracerFactory.cs b/src/Nethermind/Nethermind.Mev/Execution/TracerFactory.cs index 7fd9f25a611..d3687188d85 100644 --- a/src/Nethermind/Nethermind.Mev/Execution/TracerFactory.cs +++ b/src/Nethermind/Nethermind.Mev/Execution/TracerFactory.cs @@ -8,8 +8,10 @@ using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Tracing; using Nethermind.Consensus.Validators; +using Nethermind.Core.Crypto; using Nethermind.Core.Specs; using Nethermind.Db; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; using Nethermind.State; using Nethermind.Trie.Pruning; @@ -46,13 +48,23 @@ public ITracer Create() ReadOnlyTxProcessingEnv txProcessingEnv = new( _worldStateManager, _blockTree, _specProvider, _logManager); + IReadOnlyTxProcessingScope scope = txProcessingEnv.Build(Keccak.EmptyTreeHash); + ReadOnlyChainProcessingEnv chainProcessingEnv = new( - txProcessingEnv, Always.Valid, _recoveryStep, NoBlockRewards.Instance, new InMemoryReceiptStorage(), _specProvider, _logManager); + scope, + Always.Valid, + _recoveryStep, + NoBlockRewards.Instance, + new InMemoryReceiptStorage(), + _specProvider, + _blockTree, + _worldStateManager.GlobalStateReader, + _logManager); - return CreateTracer(txProcessingEnv, chainProcessingEnv); + return CreateTracer(scope, chainProcessingEnv); } - protected virtual ITracer CreateTracer(ReadOnlyTxProcessingEnv txProcessingEnv, ReadOnlyChainProcessingEnv chainProcessingEnv) => - new Tracer(txProcessingEnv.StateProvider, chainProcessingEnv.ChainProcessor, chainProcessingEnv.ChainProcessor, _processingOptions); + protected virtual ITracer CreateTracer(IReadOnlyTxProcessingScope scope, ReadOnlyChainProcessingEnv chainProcessingEnv) => + new Tracer(scope.WorldState, chainProcessingEnv.ChainProcessor, chainProcessingEnv.ChainProcessor, _processingOptions); } } diff --git a/src/Nethermind/Nethermind.Mev/Execution/TxBundleExecutor.cs b/src/Nethermind/Nethermind.Mev/Execution/TxBundleExecutor.cs index 17688769cdc..9791875b94c 100644 --- a/src/Nethermind/Nethermind.Mev/Execution/TxBundleExecutor.cs +++ b/src/Nethermind/Nethermind.Mev/Execution/TxBundleExecutor.cs @@ -68,7 +68,7 @@ private Block BuildBlock(MevBundle bundle, BlockHeader parent, ulong? timestamp) protected abstract TBlockTracer CreateBlockTracer(MevBundle mevBundle); - protected ResultWrapper GetInputError(BlockchainBridge.CallOutput result) => + protected ResultWrapper GetInputError(CallOutput result) => ResultWrapper.Fail(result.Error ?? string.Empty, ErrorCodes.InvalidInput); diff --git a/src/Nethermind/Nethermind.Mev/MevBlockProducer.cs b/src/Nethermind/Nethermind.Mev/MevBlockProducer.cs index b91295929db..9c99e6b0edc 100644 --- a/src/Nethermind/Nethermind.Mev/MevBlockProducer.cs +++ b/src/Nethermind/Nethermind.Mev/MevBlockProducer.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Collections.Generic; using Nethermind.Consensus; using Nethermind.Consensus.Producers; @@ -14,10 +15,9 @@ namespace Nethermind.Mev public class MevBlockProducer : MultipleBlockProducer { public MevBlockProducer( - IBlockProductionTrigger blockProductionTrigger, ILogManager logManager, params MevBlockProducerInfo[] blockProducers) - : base(blockProductionTrigger, new MevBestBlockPicker(), logManager, blockProducers) + : base(new MevBestBlockPicker(), logManager, blockProducers) { } @@ -48,18 +48,19 @@ private class MevBestBlockPicker : IBestBlockPicker public class MevBlockProducerInfo : IBlockProducerInfo { public IBlockProducer BlockProducer { get; } - public IManualBlockProductionTrigger BlockProductionTrigger { get; } + public IBlockProductionCondition Condition { get; } public IBlockTracer BlockTracer => BeneficiaryTracer; public BeneficiaryTracer BeneficiaryTracer { get; } public MevBlockProducerInfo( IBlockProducer blockProducer, - IManualBlockProductionTrigger blockProductionTrigger, + IBlockProductionCondition checkCondition, BeneficiaryTracer beneficiaryTracer) { BlockProducer = blockProducer; - BlockProductionTrigger = blockProductionTrigger; + Condition = checkCondition; BeneficiaryTracer = beneficiaryTracer; } + } } } diff --git a/src/Nethermind/Nethermind.Mev/MevBlockProducerTransactionsExecutorFactory.cs b/src/Nethermind/Nethermind.Mev/MevBlockProducerTransactionsExecutorFactory.cs index ddfb4d90f37..110b303f61b 100644 --- a/src/Nethermind/Nethermind.Mev/MevBlockProducerTransactionsExecutorFactory.cs +++ b/src/Nethermind/Nethermind.Mev/MevBlockProducerTransactionsExecutorFactory.cs @@ -4,6 +4,7 @@ using Nethermind.Consensus.Processing; using Nethermind.Consensus.Producers; using Nethermind.Core.Specs; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; namespace Nethermind.Mev @@ -19,7 +20,7 @@ public MevBlockProducerTransactionsExecutorFactory(ISpecProvider specProvider, I _logManager = logManager; } - public IBlockProcessor.IBlockTransactionsExecutor Create(ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv) => + public IBlockProcessor.IBlockTransactionsExecutor Create(IReadOnlyTxProcessingScope readOnlyTxProcessingEnv) => new MevBlockProductionTransactionsExecutor(readOnlyTxProcessingEnv, _specProvider, _logManager); } } diff --git a/src/Nethermind/Nethermind.Mev/MevBlockProductionTransactionsExecutor.cs b/src/Nethermind/Nethermind.Mev/MevBlockProductionTransactionsExecutor.cs index a12e997e876..5391651f7fa 100644 --- a/src/Nethermind/Nethermind.Mev/MevBlockProductionTransactionsExecutor.cs +++ b/src/Nethermind/Nethermind.Mev/MevBlockProductionTransactionsExecutor.cs @@ -27,12 +27,12 @@ public class MevBlockProductionTransactionsExecutor : BlockProcessor.BlockProduc private readonly IWorldState _stateProvider; public MevBlockProductionTransactionsExecutor( - ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv, + IReadOnlyTxProcessingScope readOnlyTxProcessingEnv, ISpecProvider specProvider, ILogManager logManager) : this( readOnlyTxProcessingEnv.TransactionProcessor, - readOnlyTxProcessingEnv.StateProvider, + readOnlyTxProcessingEnv.WorldState, specProvider, logManager) { diff --git a/src/Nethermind/Nethermind.Mev/MevPlugin.cs b/src/Nethermind/Nethermind.Mev/MevPlugin.cs index 19fe7901cd2..1eba1f786b2 100644 --- a/src/Nethermind/Nethermind.Mev/MevPlugin.cs +++ b/src/Nethermind/Nethermind.Mev/MevPlugin.cs @@ -133,7 +133,7 @@ public Task InitRpcModules() return Task.CompletedTask; } - public async Task InitBlockProducer(IBlockProducerFactory consensusPlugin, IBlockProductionTrigger blockProductionTrigger, ITxSource? txSource) + public IBlockProducer InitBlockProducer(IBlockProducerFactory consensusPlugin, ITxSource? txSource) { if (!Enabled) { @@ -147,57 +147,62 @@ public async Task InitBlockProducer(IBlockProducerFactory consen new(_mevConfig.MaxMergedBundles + megabundleProducerCount + 1); // Add non-mev block - MevBlockProducer.MevBlockProducerInfo standardProducer = await CreateProducer(consensusPlugin); + MevBlockProducer.MevBlockProducerInfo standardProducer = CreateProducer(consensusPlugin); blockProducers.Add(standardProducer); // Try blocks with all bundle numbers <= MaxMergedBundles for (int bundleLimit = 1; bundleLimit <= _mevConfig.MaxMergedBundles; bundleLimit++) { BundleSelector bundleSelector = new(BundlePool, bundleLimit); - MevBlockProducer.MevBlockProducerInfo bundleProducer = await CreateProducer(consensusPlugin, bundleLimit, new BundleTxSource(bundleSelector, _nethermindApi.Timestamper)); + MevBlockProducer.MevBlockProducerInfo bundleProducer = CreateProducer(consensusPlugin, bundleLimit, new BundleTxSource(bundleSelector, _nethermindApi.Timestamper)); blockProducers.Add(bundleProducer); } if (megabundleProducerCount > 0) { MegabundleSelector megabundleSelector = new(BundlePool); - MevBlockProducer.MevBlockProducerInfo bundleProducer = await CreateProducer(consensusPlugin, 0, new BundleTxSource(megabundleSelector, _nethermindApi.Timestamper)); + MevBlockProducer.MevBlockProducerInfo bundleProducer = CreateProducer(consensusPlugin, 0, new BundleTxSource(megabundleSelector, _nethermindApi.Timestamper)); blockProducers.Add(bundleProducer); } - return new MevBlockProducer(blockProductionTrigger, _nethermindApi.LogManager, blockProducers.ToArray()); + return new MevBlockProducer(_nethermindApi.LogManager, blockProducers.ToArray()); } - private async Task CreateProducer( + private MevBlockProducer.MevBlockProducerInfo CreateProducer( IBlockProducerFactory consensusPlugin, int bundleLimit = 0, ITxSource? additionalTxSource = null) { - bool BundleLimitTriggerCondition(BlockProductionEventArgs e) + IBlockProductionCondition condition = bundleLimit == 0 ? + AlwaysOkBlockProductionCondition.Instance : + new BundleLimitBlockProductionTrigger(_nethermindApi.BlockTree!, _nethermindApi.Timestamper!, BundlePool, bundleLimit); + + IBlockProducer producer = consensusPlugin.InitBlockProducer(additionalTxSource); + return new MevBlockProducer.MevBlockProducerInfo(producer, condition, new BeneficiaryTracer()); + } + + public bool Enabled => _mevConfig.Enabled; + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + private class BundleLimitBlockProductionTrigger( + IBlockTree blockTree, + ITimestamper timestamper, + IBundlePool bundlePool, + int bundleLimit + ) : IBlockProductionCondition + { + public bool CanProduce(BlockHeader parentHeader) { // TODO: why we are checking parent and not the currently produced block...? - BlockHeader? parent = _nethermindApi.BlockTree!.GetProducedBlockParent(e.ParentHeader); + BlockHeader? parent = blockTree!.GetProducedBlockParent(parentHeader); if (parent is not null) { - IEnumerable bundles = BundlePool.GetBundles(parent, _nethermindApi.Timestamper); + IEnumerable bundles = bundlePool.GetBundles(parent, timestamper); return bundles.Count() >= bundleLimit; } return false; } - - IManualBlockProductionTrigger manualTrigger = new BuildBlocksWhenRequested(); - IBlockProductionTrigger trigger = manualTrigger; - if (bundleLimit != 0) - { - trigger = new TriggerWithCondition(manualTrigger, BundleLimitTriggerCondition); - } - - IBlockProducer producer = await consensusPlugin.InitBlockProducer(trigger, additionalTxSource); - return new MevBlockProducer.MevBlockProducerInfo(producer, manualTrigger, new BeneficiaryTracer()); } - - public bool Enabled => _mevConfig.Enabled; - - public ValueTask DisposeAsync() => ValueTask.CompletedTask; } diff --git a/src/Nethermind/Nethermind.Network.Contract/P2P/Protocol.cs b/src/Nethermind/Nethermind.Network.Contract/P2P/Protocol.cs index 50b36599d7b..4175cc638a8 100644 --- a/src/Nethermind/Nethermind.Network.Contract/P2P/Protocol.cs +++ b/src/Nethermind/Nethermind.Network.Contract/P2P/Protocol.cs @@ -30,10 +30,6 @@ public static class Protocol /// public const string Bzz = "bzz"; /// - /// Lightweight Clients - /// - public const string Les = "les"; - /// /// Parity Warp Sync /// public const string Par = "par"; @@ -42,10 +38,6 @@ public static class Protocol ///
public const string Ndm = "ndm"; /// - /// Witness - /// - public const string Wit = "wit"; - /// /// Account Abstraction /// public const string AA = "aa"; diff --git a/src/Nethermind/Nethermind.Network.Discovery/DiscoveryApp.cs b/src/Nethermind/Nethermind.Network.Discovery/DiscoveryApp.cs index 2cf631d6e3b..0a447340ad3 100644 --- a/src/Nethermind/Nethermind.Network.Discovery/DiscoveryApp.cs +++ b/src/Nethermind/Nethermind.Network.Discovery/DiscoveryApp.cs @@ -145,14 +145,15 @@ private void InitializeUdpChannel() .Handler(new ActionChannelInitializer(InitializeChannel)); } - _bindingTask = bootstrap.BindAsync(IPAddress.Parse(_networkConfig.LocalIp!), _networkConfig.DiscoveryPort) + IPAddress ip = IPAddress.Parse(_networkConfig.LocalIp!); + _bindingTask = bootstrap.BindAsync(ip, _networkConfig.DiscoveryPort) .ContinueWith( t => { if (t.IsFaulted) { - _logger.Error("Error when establishing discovery connection", t.Exception); + _logger.Error($"Error when establishing discovery connection on Address: {ip}({_networkConfig.LocalIp}:{_networkConfig.DiscoveryPort})", t.Exception); } return _channel = t.Result; diff --git a/src/Nethermind/Nethermind.Network.Discovery/RoutingTable/NodeBucket.cs b/src/Nethermind/Nethermind.Network.Discovery/RoutingTable/NodeBucket.cs index e0a5445a4fa..6ce661d0546 100644 --- a/src/Nethermind/Nethermind.Network.Discovery/RoutingTable/NodeBucket.cs +++ b/src/Nethermind/Nethermind.Network.Discovery/RoutingTable/NodeBucket.cs @@ -51,9 +51,11 @@ public struct BondedItemsEnumerator : IEnumerator, IEnumerable this; diff --git a/src/Nethermind/Nethermind.Network.Discovery/RoutingTable/NodeTable.cs b/src/Nethermind/Nethermind.Network.Discovery/RoutingTable/NodeTable.cs index fbd597224ac..4dc4eae50d7 100644 --- a/src/Nethermind/Nethermind.Network.Discovery/RoutingTable/NodeTable.cs +++ b/src/Nethermind/Nethermind.Network.Discovery/RoutingTable/NodeTable.cs @@ -105,32 +105,25 @@ public ClosestNodesEnumerator(NodeBucket[] buckets, int bucketSize) public bool MoveNext() { - try + while (_count < _bucketSize) { - while (_count < _bucketSize) + if (!_enumeratorSet || !_itemEnumerator.MoveNext()) { - if (!_enumeratorSet || !_itemEnumerator.MoveNext()) + _itemEnumerator.Dispose(); + _bucketIndex++; + if (_bucketIndex >= _buckets.Length) { - _itemEnumerator.Dispose(); - _bucketIndex++; - if (_bucketIndex >= _buckets.Length) - { - return false; - } - - _itemEnumerator = _buckets[_bucketIndex].BondedItems.GetEnumerator(); - _enumeratorSet = true; - continue; + return false; } - Current = _itemEnumerator.Current.Node!; - _count++; - return true; + _itemEnumerator = _buckets[_bucketIndex].BondedItems.GetEnumerator(); + _enumeratorSet = true; + continue; } - } - finally - { - _itemEnumerator.Dispose(); + + Current = _itemEnumerator.Current.Node!; + _count++; + return true; } return false; @@ -138,7 +131,7 @@ public bool MoveNext() void IEnumerator.Reset() => throw new NotSupportedException(); - public void Dispose() { } + public void Dispose() => _itemEnumerator.Dispose(); public ClosestNodesEnumerator GetEnumerator() => this; @@ -171,7 +164,7 @@ public ClosestNodesFromNodeEnumerator(NodeBucket[] buckets, byte[] targetNodeId, { foreach (var item in bucket.BondedItems) { - if (item.Node != null && item.Node.IdHash != idHash) + if (item.Node is not null && item.Node.IdHash != idHash) { _sortedNodes.Add(item.Node); } diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/P2PProtocolHandlerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/P2PProtocolHandlerTests.cs index b216c17ae05..9209d9d192b 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/P2PProtocolHandlerTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/P2PProtocolHandlerTests.cs @@ -85,10 +85,9 @@ public void On_init_sends_a_hello_message() public void On_init_sends_a_hello_message_with_capabilities() { P2PProtocolHandler p2PProtocolHandler = CreateSession(); - p2PProtocolHandler.AddSupportedCapability(new Capability(Protocol.Wit, 0)); p2PProtocolHandler.Init(); - string[] expectedCapabilities = { "eth66", "eth67", "eth68", "nodedata1", "wit0" }; + string[] expectedCapabilities = ["eth66", "eth67", "eth68", "nodedata1"]; _session.Received(1).DeliverMessage( Arg.Is(m => m.Capabilities.Select(c => c.ToString()).SequenceEqual(expectedCapabilities))); } @@ -97,7 +96,6 @@ public void On_init_sends_a_hello_message_with_capabilities() public void On_hello_with_no_matching_capability() { P2PProtocolHandler p2PProtocolHandler = CreateSession(); - p2PProtocolHandler.AddSupportedCapability(new Capability(Protocol.Wit, 66)); using HelloMessage message = new() { diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/AnnounceMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/AnnounceMessageSerializerTests.cs deleted file mode 100644 index ab1384d86dc..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/AnnounceMessageSerializerTests.cs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Crypto; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Les -{ - [TestFixture] - public class AnnounceMessageSerializerTests - { - [Test] - public void RoundTripWithRequiredData() - { - using AnnounceMessage announceMessage = new(); - announceMessage.HeadHash = Keccak.Compute("1"); - announceMessage.HeadBlockNo = 4; - announceMessage.TotalDifficulty = 131200; - announceMessage.ReorgDepth = 0; - - AnnounceMessageSerializer serializer = new(); - - SerializerTester.TestZero(serializer, announceMessage, "e8a0c89efdaa54c0f20c7adf612882df0950f5a951637e0307cdcb4c672f298b8bc6048302008080c0"); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/BlockBodiesSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/BlockBodiesSerializerTests.cs deleted file mode 100644 index 38723f845dc..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/BlockBodiesSerializerTests.cs +++ /dev/null @@ -1,34 +0,0 @@ -/// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core; -using Nethermind.Core.Test.Builders; -using Nethermind.Crypto; -using Nethermind.Logging; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using Nethermind.Specs; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Les -{ - [TestFixture] - public class BlockBodiesSerializerTests - { - [Test] - public void RoundTrip() - { - BlockHeader header = Build.A.BlockHeader.TestObject; - Address to = Build.An.Address.FromNumber(1).TestObject; - Transaction tx = Build.A.Transaction.WithTo(to).SignedAndResolved(new EthereumEcdsa(MainnetSpecProvider.Instance.ChainId, LimboLogs.Instance), TestItem.PrivateKeyA).TestObject; - tx.SenderAddress = null; - var ethMessage = new Network.P2P.Subprotocols.Eth.V62.Messages.BlockBodiesMessage(); - ethMessage.Bodies = new(new[] { new BlockBody(new[] { tx }, new[] { header }) }); - - using BlockBodiesMessage message = new(ethMessage, 1, 1000); - - BlockBodiesMessageSerializer serializer = new(); - - SerializerTester.TestZero(serializer, message); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/BlockHeadersMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/BlockHeadersMessageSerializerTests.cs deleted file mode 100644 index 538ad03f5b6..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/BlockHeadersMessageSerializerTests.cs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core; -using Nethermind.Core.Collections; -using Nethermind.Core.Test.Builders; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Les -{ - [TestFixture] - public class BlockHeadersMessageSerializerTests - { - [Test] - public void RoundTrip() - { - var ethMessage = new Network.P2P.Subprotocols.Eth.V62.Messages.BlockHeadersMessage(); - ethMessage.BlockHeaders = new ArrayPoolList(1) { Build.A.BlockHeader.TestObject }; - using BlockHeadersMessage message = new(ethMessage, 2, 3000); - - BlockHeadersMessageSerializer serializer = new(); - - SerializerTester.TestZero(serializer, message); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/ContractCodesMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/ContractCodesMessageSerializerTests.cs deleted file mode 100644 index 0c0e4427a73..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/ContractCodesMessageSerializerTests.cs +++ /dev/null @@ -1,25 +0,0 @@ -/// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Collections; -using Nethermind.Core.Test.Builders; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Les -{ - [TestFixture] - public class ContractCodesMessageSerializerTests - { - [Test] - public void RoundTrip() - { - ArrayPoolList data = new(3) { TestItem.KeccakA.BytesToArray(), TestItem.KeccakB.BytesToArray(), TestItem.KeccakC.BytesToArray() }; - using ContractCodesMessage message = new(data, 13452, 134); - - ContractCodesMessageSerializer serializer = new(); - - SerializerTester.TestZero(serializer, message); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetBlockBodiesMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetBlockBodiesMessageSerializerTests.cs deleted file mode 100644 index 444d47a032b..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetBlockBodiesMessageSerializerTests.cs +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Crypto; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Les -{ - [TestFixture] - public class GetBlockBodiesMessageSerializerTests - { - [Test] - public void RoundTrip() - { - var ethMessage = new Network.P2P.Subprotocols.Eth.V62.Messages.GetBlockBodiesMessage(Keccak.OfAnEmptySequenceRlp, Keccak.Zero, Keccak.EmptyTreeHash); - var message = new GetBlockBodiesMessage(ethMessage, 1); - - GetBlockBodiesMessageSerializer serializer = new(); - - SerializerTester.TestZero(serializer, message, "f86601f863a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347a00000000000000000000000000000000000000000000000000000000000000000a056e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetBlockHeadersMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetBlockHeadersMessageSerializerTests.cs deleted file mode 100644 index bbef9b11c24..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetBlockHeadersMessageSerializerTests.cs +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Crypto; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Les -{ - [TestFixture] - public class GetBlockHeadersMessageSerializerTests - { - [Test] - public void RoundTripWithHash() - { - var ethMessage = new Network.P2P.Subprotocols.Eth.V62.Messages.GetBlockHeadersMessage(); - ethMessage.StartBlockHash = Keccak.Compute("1"); - ethMessage.MaxHeaders = 10; - ethMessage.Skip = 2; - ethMessage.Reverse = 0; - - var message = new GetBlockHeadersMessage(ethMessage, 2); - - GetBlockHeadersMessageSerializer serializer = new(); - - SerializerTester.TestZero(serializer, message, "e602e4a0c89efdaa54c0f20c7adf612882df0950f5a951637e0307cdcb4c672f298b8bc60a0280"); - } - - [Test] - public void RoundTripWithNumber() - { - var ethMessage = new Network.P2P.Subprotocols.Eth.V62.Messages.GetBlockHeadersMessage(); - ethMessage.StartBlockNumber = 1; - ethMessage.MaxHeaders = 10; - ethMessage.Skip = 2; - ethMessage.Reverse = 0; - - var message = new GetBlockHeadersMessage(ethMessage, 2); - - GetBlockHeadersMessageSerializer serializer = new(); - - SerializerTester.TestZero(serializer, message, "c602c4010a0280"); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetContractCodesMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetContractCodesMessageSerializerTests.cs deleted file mode 100644 index dd88c9e3d90..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetContractCodesMessageSerializerTests.cs +++ /dev/null @@ -1,30 +0,0 @@ -/// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Test.Builders; -using Nethermind.Network.P2P.Subprotocols.Les; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Les -{ - [TestFixture] - public class GetContractCodesMessageSerializerTests - { - [Test] - public void RoundTrip() - { - CodeRequest[] requests = new CodeRequest[] - { - new(TestItem.KeccakA, TestItem.KeccakB), - new(TestItem.KeccakC, TestItem.KeccakD), - }; - - using GetContractCodesMessage message = new(requests, 774); - - GetContractCodesMessageSerializer serializer = new(); - - SerializerTester.TestZero(serializer, message); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetHelperTrieProofsMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetHelperTrieProofsMessageSerializerTests.cs deleted file mode 100644 index f58ba84a90f..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetHelperTrieProofsMessageSerializerTests.cs +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Test.Builders; -using Nethermind.Network.P2P.Subprotocols.Les; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Les -{ - [TestFixture] - public class GetHelperTrieProofsMessageSerializerTests - { - [Test] - public void RoundTrip() - { - HelperTrieRequest[] requests = new HelperTrieRequest[] - { - new(HelperTrieType.CHT, 177, TestItem.RandomDataA, 2, 1), - new(HelperTrieType.BloomBits, 77, TestItem.RandomDataB, 4, 0), - }; - using GetHelperTrieProofsMessage message = new(); - message.RequestId = 100; - message.Requests = requests; - - GetHelperTrieProofsMessageSerializer serializer = new(); - - SerializerTester.TestZero(serializer, message); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetReceiptsMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetReceiptsMessageSerializerTests.cs deleted file mode 100644 index eb07234cd78..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/GetReceiptsMessageSerializerTests.cs +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Crypto; -using Nethermind.Core.Extensions; -using Nethermind.Core.Test.Builders; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Les -{ - [TestFixture] - public class GetReceiptsMessageSerializerTests - { - [Test] - public void RoundTrip() - { - Hash256[] hashes = { TestItem.KeccakA, TestItem.KeccakB, TestItem.KeccakC }; - var ethMessage = new Network.P2P.Subprotocols.Eth.V63.Messages.GetReceiptsMessage(hashes.ToPooledList()); - - using GetReceiptsMessage getReceiptsMessage = new(ethMessage, 1); - - GetReceiptsMessageSerializer serializer = new(); - - SerializerTester.TestZero(serializer, getReceiptsMessage); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/HelperTrieProofsMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/HelperTrieProofsMessageSerializerTests.cs deleted file mode 100644 index 0386738a5c2..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/HelperTrieProofsMessageSerializerTests.cs +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Test.Builders; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using Nethermind.Serialization.Rlp; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Les -{ - [TestFixture] - public class HelperTrieProofsMessageSerializerTests - { - [Test] - public void RoundTrip() - { - byte[][] proofs = new byte[][] - { - TestItem.KeccakA.BytesToArray(), - TestItem.KeccakB.BytesToArray(), - TestItem.KeccakC.BytesToArray(), - TestItem.KeccakD.BytesToArray(), - TestItem.KeccakE.BytesToArray(), - TestItem.KeccakF.BytesToArray(), - }; - byte[][] auxData = new byte[][] - { - TestItem.KeccakG.BytesToArray(), - TestItem.KeccakH.BytesToArray(), - Rlp.Encode(Build.A.BlockHeader.TestObject).Bytes, - }; - var message = new HelperTrieProofsMessage(proofs, auxData, 324, 734); - - HelperTrieProofsMessageSerializer serializer = new(); - - SerializerTester.TestZero(serializer, message); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/ReceiptsMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/ReceiptsMessageSerializerTests.cs deleted file mode 100644 index 614e8d840c9..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/ReceiptsMessageSerializerTests.cs +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core; -using Nethermind.Core.Extensions; -using Nethermind.Specs; -using Nethermind.Core.Test.Builders; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Les -{ - [TestFixture] - public class ReceiptsMessageSerializerTests - { - [Test] - public void RoundTrip() - { - TxReceipt[][] data = { new[] { Build.A.Receipt.WithAllFieldsFilled.TestObject, Build.A.Receipt.WithAllFieldsFilled.WithBlockNumber(0).TestObject }, new[] { Build.A.Receipt.WithAllFieldsFilled.TestObject, Build.A.Receipt.WithAllFieldsFilled.TestObject }, new[] { Build.A.Receipt.WithAllFieldsFilled.WithTxType(TxType.AccessList).TestObject, Build.A.Receipt.WithAllFieldsFilled.TestObject } }; - using Network.P2P.Subprotocols.Eth.V63.Messages.ReceiptsMessage ethMessage = new(data.ToPooledList()); - using ReceiptsMessage receiptsMessage = new(ethMessage, 1, 2000); - - ReceiptsMessageSerializer serializer = new(MainnetSpecProvider.Instance); - - // Eth.ReceiptsMessageSerializer intentionally excludes fields when deserializing. - // I think it's probably best to not copy the test logic checking for this here. - byte[] bytes = serializer.Serialize(receiptsMessage); - using ReceiptsMessage deserialized = serializer.Deserialize(bytes); - - Assert.That(deserialized.RequestId, Is.EqualTo(receiptsMessage.RequestId), "RequestId"); - Assert.That(deserialized.BufferValue, Is.EqualTo(receiptsMessage.BufferValue), "BufferValue"); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/StatusMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/StatusMessageSerializerTests.cs deleted file mode 100644 index 2b8652d7a6a..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Les/StatusMessageSerializerTests.cs +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Crypto; -using Nethermind.Network.P2P.Subprotocols.Les; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Les -{ - [TestFixture] - public class StatusMessageSerializerTests - { - [Test] - public void RoundTripWithAllData() - { - using StatusMessage statusMessage = new(); - statusMessage.ProtocolVersion = 3; - statusMessage.NetworkId = 1; - statusMessage.TotalDifficulty = 131200; - statusMessage.BestHash = Keccak.Compute("1"); - statusMessage.HeadBlockNo = 4; - statusMessage.GenesisHash = Keccak.Compute("0"); - statusMessage.AnnounceType = 1; - statusMessage.ServeHeaders = true; - statusMessage.ServeChainSince = 0; - statusMessage.ServeRecentChain = 1000; - statusMessage.ServeStateSince = 1; - statusMessage.ServeRecentState = 500; - statusMessage.TxRelay = true; - statusMessage.BufferLimit = 1000; - statusMessage.MaximumRechargeRate = 100; - statusMessage.MaximumRequestCosts = CostTracker.DefaultRequestCostTable; - - StatusMessageSerializer serializer = new(); - - SerializerTester.TestZero(serializer, statusMessage, "f90176d18f70726f746f636f6c56657273696f6e03cb896e6574776f726b496401cb8668656164546483020080ea886865616448617368a0c89efdaa54c0f20c7adf612882df0950f5a951637e0307cdcb4c672f298b8bc6c987686561644e756d04ed8b67656e6573697348617368a0044852b2a670ade5407e78fb2863c51de9fcb96542a07186fe3aeda6bb8a116dce8c616e6e6f756e63655479706501ce8c736572766548656164657273c0d18f7365727665436861696e53696e636580d4907365727665526563656e74436861696e8203e8d18f7365727665537461746553696e636501d4907365727665526563656e7453746174658201f4c987747852656c6179c0d28e666c6f77436f6e74726f6c2f424c8203e8d18f666c6f77436f6e74726f6c2f4d525264f84c8f666c6f77436f6e74726f6c2f4d5243f83ac802830249f0827530c60480830aae60c60680830f4240c60a808306ddd0c60f80830927c0c61180830f4240c613808306ddd0c614808303d090"); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Wit/BlockWitnessHashesMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Wit/BlockWitnessHashesMessageSerializerTests.cs deleted file mode 100644 index 26eb269a095..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Wit/BlockWitnessHashesMessageSerializerTests.cs +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using FluentAssertions; -using Nethermind.Core.Crypto; -using Nethermind.Core.Test.Builders; -using Nethermind.Network.P2P.Subprotocols.Wit.Messages; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Wit -{ - [TestFixture, Parallelizable(ParallelScope.All)] - public class BlockWitnessHashesMessageSerializerTests - { - [Test] - public void Can_handle_zero() - { - BlockWitnessHashesMessageSerializer serializer = new(); - using BlockWitnessHashesMessage message = new(1, Array.Empty()); - SerializerTester.TestZero(serializer, message); - } - - [Test] - public void Can_handle_one() - { - BlockWitnessHashesMessageSerializer serializer = new(); - using BlockWitnessHashesMessage message = new(1, new[] { Keccak.Zero }); - SerializerTester.TestZero(serializer, message); - } - - [Test] - public void Can_handle_many() - { - BlockWitnessHashesMessageSerializer serializer = new(); - using BlockWitnessHashesMessage message = new(1, TestItem.Keccaks); - SerializerTester.TestZero(serializer, message); - } - - [Test] - public void Can_handle_null() - { - BlockWitnessHashesMessageSerializer serializer = new(); - using BlockWitnessHashesMessage message = new(1, null); - byte[] serialized = serializer.Serialize(message); - serialized[0].Should().Be(194); - serializer.Deserialize(serialized); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Wit/GetBlockWitnessHashesMessageSerializerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Wit/GetBlockWitnessHashesMessageSerializerTests.cs deleted file mode 100644 index 25fdf425b45..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Wit/GetBlockWitnessHashesMessageSerializerTests.cs +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using FluentAssertions; -using Nethermind.Core.Crypto; -using Nethermind.Core.Extensions; -using Nethermind.Network.P2P.Subprotocols.Wit.Messages; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Wit -{ - [TestFixture, Parallelizable(ParallelScope.All)] - public class MessageTests - { - [Test] - public void Message_code_is_correct_in_request() - { - new GetBlockWitnessHashesMessage(1, Keccak.Zero).PacketType.Should().Be(1); - } - - [Test] - public void Message_code_is_correct_in_response() - { - new BlockWitnessHashesMessage(1, null).PacketType.Should().Be(2); - } - } - - [TestFixture, Parallelizable(ParallelScope.All)] - public class GetBlockWitnessHashesMessageSerializerTests - { - [Test] - public void Roundtrip_init() - { - GetBlockWitnessHashesMessageSerializer serializer = new(); - using GetBlockWitnessHashesMessage message = new(1, Keccak.Zero); - SerializerTester.TestZero(serializer, message); - } - - [Test] - public void Can_handle_null() - { - GetBlockWitnessHashesMessageSerializer serializer = new(); - using GetBlockWitnessHashesMessage message = new(1, null); - SerializerTester.TestZero(serializer, message); - } - - [Test] - public void Can_deserialize_trinity() - { - GetBlockWitnessHashesMessageSerializer serializer = new(); - var trinityBytes = Bytes.FromHexString("0xea880ea29ca8028d7edea04bf6040124107de018c753ff2a9e464ca13e9d099c45df6a48ddbf436ce30c83"); - var buffer = ByteBufferUtil.DefaultAllocator.Buffer(trinityBytes.Length); - buffer.WriteBytes(trinityBytes); - using GetBlockWitnessHashesMessage msg = - ((IZeroMessageSerializer)serializer).Deserialize(buffer); - } - } -} diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Wit/WitProtocolHandlerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Wit/WitProtocolHandlerTests.cs deleted file mode 100644 index fe339024c82..00000000000 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Wit/WitProtocolHandlerTests.cs +++ /dev/null @@ -1,163 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System.Linq; -using System.Net; -using System.Reflection; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Nethermind.Core.Crypto; -using Nethermind.Core.Test.Builders; -using Nethermind.Logging; -using Nethermind.Network.P2P; -using Nethermind.Network.P2P.Subprotocols.Wit; -using Nethermind.Network.P2P.Subprotocols.Wit.Messages; -using Nethermind.Network.Rlpx; -using Nethermind.Stats; -using Nethermind.Stats.Model; -using Nethermind.Synchronization; -using NSubstitute; -using NUnit.Framework; - -namespace Nethermind.Network.Test.P2P.Subprotocols.Wit -{ - [TestFixture, Parallelizable(ParallelScope.All)] - public class WitProtocolHandlerTests - { - [Test] - public void Protocol_code_is_wit() - { - Context context = new(); - context.WitProtocolHandler.ProtocolCode.Should().Be("wit"); - } - - [Test] - public void Protocol_version_is_zero() - { - Context context = new(); - context.WitProtocolHandler.ProtocolVersion.Should().Be(0); - } - - [Test] - public void Message_space_should_be_correct() - { - int maxCode = typeof(WitMessageCode) - .GetFields(BindingFlags.Static | BindingFlags.Public) - .Where(f => f.FieldType == typeof(int)) - .Select(f => (int)f.GetValue(null)!).Max(); - - Context context = new(); - context.WitProtocolHandler.MessageIdSpaceSize.Should().Be(maxCode + 1); - } - - [Test] - public void Name_should_be_wit0() - { - Context context = new(); - context.WitProtocolHandler.Name.Should().Be("wit0"); - } - - [Test] - public void Init_does_nothing() - { - Context context = new(); - context.WitProtocolHandler.Init(); - } - - [Test] - public void Can_disconnect() - { - Context context = new(); - context.WitProtocolHandler.DisconnectProtocol(DisconnectReason.Other, "just because"); - context.WitProtocolHandler.Dispose(); - } - - [Test] - public void Init_timeout_should_timeout_eth() - { - Context context = new(); - bool initialized = false; - context.WitProtocolHandler.ProtocolInitialized += - (s, e) => initialized = true; - context.WitProtocolHandler.Init(); - initialized.Should().BeTrue(); - } - - [Test] - public void Can_handle_request_for_an_empty_witness() - { - Context context = new(); - context.WitProtocolHandler.Init(); - - using GetBlockWitnessHashesMessage msg = new(5, Keccak.Zero); - GetBlockWitnessHashesMessageSerializer serializer = new(); - var serialized = serializer.Serialize(msg); - - context.WitProtocolHandler.HandleMessage(new Packet("wit", WitMessageCode.GetBlockWitnessHashes, serialized)); - context.SyncServer.Received().GetBlockWitnessHashes(Keccak.Zero); - } - - [Test] - public void Can_handle_request_for_a_non_empty_witness() - { - Context context = new(); - context.SyncServer.GetBlockWitnessHashes(Keccak.Zero) - .Returns(new[] { TestItem.KeccakA, TestItem.KeccakB }); - - context.WitProtocolHandler.Init(); - - using GetBlockWitnessHashesMessage msg = new(5, Keccak.Zero); - GetBlockWitnessHashesMessageSerializer serializer = new(); - var serialized = serializer.Serialize(msg); - - context.WitProtocolHandler.HandleMessage(new Packet("wit", WitMessageCode.GetBlockWitnessHashes, serialized)); - context.Session.Received().DeliverMessage( - Arg.Is(msg => msg.Hashes.Length == 2)); - } - - [Test] - public async Task Can_request_non_empty_witness() - { - Context context = new(); - using BlockWitnessHashesMessage msg = new(5, new[] { TestItem.KeccakA, TestItem.KeccakB }); - BlockWitnessHashesMessageSerializer serializer = new(); - var serialized = serializer.Serialize(msg); - - context.WitProtocolHandler.Init(); - - - context.Session - // check if you did not enable the Init -> send message diag - .When(s => s.DeliverMessage( - Arg.Is(msg => msg.BlockHash == TestItem.KeccakA))) - .Do(_ => context.WitProtocolHandler.HandleMessage(new Packet("wit", WitMessageCode.BlockWitnessHashes, serialized))); - Task result = context.WitProtocolHandler.GetBlockWitnessHashes(TestItem.KeccakA, CancellationToken.None); - await result; - result.Status.Should().Be(TaskStatus.RanToCompletion); - } - - private class Context - { - public ISession Session { get; } - public ISyncServer SyncServer { get; } - public WitProtocolHandler WitProtocolHandler { get; } - - public Context() - { - MessageSerializationService serializationService = new(); - serializationService.Register(typeof(GetBlockWitnessHashesMessage).Assembly); - Session = Substitute.For(); - Session.Node.Returns(new Node(TestItem.PublicKeyA, new IPEndPoint(IPAddress.Loopback, 30303))); - INodeStatsManager nodeStats = Substitute.For(); - SyncServer = Substitute.For(); - WitProtocolHandler = new WitProtocolHandler( - Session, - serializationService, - nodeStats, - SyncServer, - LimboLogs.Instance); - } - } - } -} diff --git a/src/Nethermind/Nethermind.Network/Metrics.cs b/src/Nethermind/Nethermind.Network/Metrics.cs index c8d7f1c84fa..01b41f57889 100644 --- a/src/Nethermind/Nethermind.Network/Metrics.cs +++ b/src/Nethermind/Nethermind.Network/Metrics.cs @@ -1,25 +1,11 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; using System.Collections.Concurrent; -using System.Collections.Frozen; -using System.Collections.Generic; using System.ComponentModel; -using System.Linq; -using System.Reflection; using System.Runtime.Serialization; using Nethermind.Core.Attributes; using Nethermind.Network.P2P; -using Nethermind.Network.P2P.Subprotocols.Eth.V62; -using Nethermind.Network.P2P.Subprotocols.Eth.V63; -using Nethermind.Network.P2P.Subprotocols.Eth.V65; -using Nethermind.Network.P2P.Subprotocols.Eth.V66; -using Nethermind.Network.P2P.Subprotocols.Eth.V68; -using Nethermind.Network.P2P.Subprotocols.Les; -using Nethermind.Network.P2P.Subprotocols.NodeData; -using Nethermind.Network.P2P.Subprotocols.Snap; -using Nethermind.Network.P2P.Subprotocols.Wit; using Nethermind.Stats.Model; namespace Nethermind.Network diff --git a/src/Nethermind/Nethermind.Network/P2P/P2PMessageKey.cs b/src/Nethermind/Nethermind.Network/P2P/P2PMessageKey.cs index 17bcacfb843..8166c0cb8aa 100644 --- a/src/Nethermind/Nethermind.Network/P2P/P2PMessageKey.cs +++ b/src/Nethermind/Nethermind.Network/P2P/P2PMessageKey.cs @@ -12,10 +12,8 @@ using Nethermind.Network.P2P.Subprotocols.Eth.V65; using Nethermind.Network.P2P.Subprotocols.Eth.V66; using Nethermind.Network.P2P.Subprotocols.Eth.V68; -using Nethermind.Network.P2P.Subprotocols.Les; using Nethermind.Network.P2P.Subprotocols.NodeData; using Nethermind.Network.P2P.Subprotocols.Snap; -using Nethermind.Network.P2P.Subprotocols.Wit; namespace Nethermind.Network.P2P; @@ -33,8 +31,6 @@ public record struct P2PMessageKey(VersionedProtocol Protocol, int PacketType) : .Concat(FromMessageCodeClass(Contract.P2P.Protocol.Eth, typeof(Eth68MessageCode))) .Concat(FromMessageCodeClass(Contract.P2P.Protocol.NodeData, typeof(NodeDataMessageCode))) - .Concat(FromMessageCodeClass(Contract.P2P.Protocol.Wit, typeof(WitMessageCode))) - .Concat(FromMessageCodeClass(Contract.P2P.Protocol.Les, typeof(LesMessageCode))) .Concat(FromMessageCodeClass(Contract.P2P.Protocol.Snap, typeof(SnapMessageCode))) diff --git a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/SyncPeerProtocolHandlerBase.cs b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/SyncPeerProtocolHandlerBase.cs index f65e44e8122..bc5ebe0ef08 100644 --- a/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/SyncPeerProtocolHandlerBase.cs +++ b/src/Nethermind/Nethermind.Network/P2P/ProtocolHandlers/SyncPeerProtocolHandlerBase.cs @@ -74,7 +74,8 @@ public abstract class SyncPeerProtocolHandlerBase : ZeroProtocolHandlerBase, ISy initialRequestSize: 4 ); - protected LruKeyCache NotifiedTransactions { get; } = new(2 * MemoryAllowance.MemPoolSize, "notifiedTransactions"); + protected LruKeyCacheLowObject? _notifiedTransactions; + protected LruKeyCacheLowObject NotifiedTransactions => _notifiedTransactions ??= new(2 * MemoryAllowance.MemPoolSize, "notifiedTransactions"); protected SyncPeerProtocolHandlerBase(ISession session, IMessageSerializationService serializer, diff --git a/src/Nethermind/Nethermind.Network/P2P/Session.cs b/src/Nethermind/Nethermind.Network/P2P/Session.cs index e29e2df39e8..f092fd41ce2 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Session.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Session.cs @@ -674,7 +674,7 @@ private void RecordOutgoingMessageMetric(T message, int size) where T : P2PMe private void RecordIncomingMessageMetric(string protocol, int packetType, int size) { - if (protocol == null) return; + if (protocol is null) return; byte version = _protocols.TryGetValue(protocol, out IProtocolHandler? handler) ? handler!.ProtocolVersion : (byte)0; diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs index 05c9805e9d0..b9036c0285b 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs @@ -19,8 +19,7 @@ public class PooledTxsRequestor : IPooledTxsRequestor private readonly ITxPool _txPool; private readonly ITxPoolConfig _txPoolConfig; - private readonly LruKeyCache _pendingHashes = new(MemoryAllowance.TxHashCacheSize, - Math.Min(1024 * 16, MemoryAllowance.TxHashCacheSize), "pending tx hashes"); + private readonly LruKeyCacheLowObject _pendingHashes = new(MemoryAllowance.TxHashCacheSize, "pending tx hashes"); public PooledTxsRequestor(ITxPool txPool, ITxPoolConfig txPoolConfig) { diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Eth62ProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Eth62ProtocolHandler.cs index 783c85e47d9..8c531355cc1 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Eth62ProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V62/Eth62ProtocolHandler.cs @@ -33,7 +33,8 @@ public class Eth62ProtocolHandler : SyncPeerProtocolHandlerBase, IZeroProtocolHa protected readonly ITxPool _txPool; private readonly IGossipPolicy _gossipPolicy; private readonly ITxGossipPolicy _txGossipPolicy; - private readonly LruKeyCache _lastBlockNotificationCache = new(10, "LastBlockNotificationCache"); + private LruKeyCache? _lastBlockNotificationCache; + private LruKeyCache LastBlockNotificationCache => _lastBlockNotificationCache ??= new(10, "LastBlockNotificationCache"); public Eth62ProtocolHandler(ISession session, IMessageSerializationService serializer, @@ -338,7 +339,7 @@ public override void NotifyOfNewBlock(Block block, SendBlockMode mode) return; } - if (_lastBlockNotificationCache.Set(block.Hash)) + if (LastBlockNotificationCache.Set(block.Hash)) { switch (mode) { diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/ReceiptsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/ReceiptsMessageSerializer.cs index 63b4dbe5940..d6fb02e485d 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/ReceiptsMessageSerializer.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V63/Messages/ReceiptsMessageSerializer.cs @@ -14,7 +14,7 @@ namespace Nethermind.Network.P2P.Subprotocols.Eth.V63.Messages public class ReceiptsMessageSerializer : IZeroInnerMessageSerializer { private readonly ISpecProvider _specProvider; - private readonly ReceiptMessageDecoder _decoder = new(); + private static readonly IRlpStreamDecoder _decoder = Rlp.GetStreamDecoder(); public ReceiptsMessageSerializer(ISpecProvider specProvider) { @@ -50,8 +50,6 @@ public void Serialize(IByteBuffer byteBuffer, ReceiptsMessage message) _decoder.Encode(stream, txReceipt, _specProvider.GetReceiptSpec(txReceipt.BlockNumber).IsEip658Enabled ? RlpBehaviors.Eip658Receipts : RlpBehaviors.None); } - - } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/CodeRequest.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/CodeRequest.cs deleted file mode 100644 index 778928450b3..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/CodeRequest.cs +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Crypto; - -namespace Nethermind.Network.P2P.Subprotocols.Les -{ - public class CodeRequest - { - public Hash256 BlockHash; - public Hash256 AccountKey; - - public CodeRequest() - { - } - - public CodeRequest(Hash256 blockHash, Hash256 accountKey) - { - BlockHash = blockHash; - AccountKey = accountKey; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/CostTracker.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/CostTracker.cs deleted file mode 100644 index 576383f449f..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/CostTracker.cs +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -namespace Nethermind.Network.P2P.Subprotocols.Les -{ - class CostTracker - { - // These are the initial defaults from geth. - // It probably doesn't make sense to define them here in the finished implementation, since the client will want to use the values supplied by the server - - // TODO: Benchmark finished implementation and update based on our actual serve times. - // TODO: Implement cost scaling to account for users with different capabilities - https://github.com/ethereum/go-ethereum/blob/01d92531ee0993c0e6e5efe877a1242bfd808626/les/costtracker.go#L437 - // TODO: Implement multiple cost lists, so it can be limited based on the minimum of available bandwidth, cpu time, etc. - https://github.com/ethereum/go-ethereum/blob/01d92531ee0993c0e6e5efe877a1242bfd808626/les/costtracker.go#L186 - // TODO: This might be better as a dictionary - public static RequestCostItem[] DefaultRequestCostTable = new RequestCostItem[] - { - new(LesMessageCode.GetBlockHeaders, 150_000, 30_000), - new(LesMessageCode.GetBlockBodies, 0, 700_000), - new(LesMessageCode.GetReceipts, 0, 1_000_000), - new(LesMessageCode.GetContractCodes, 0, 450_000), - new(LesMessageCode.GetProofsV2, 0, 600_000), - new(LesMessageCode.GetHelperTrieProofs, 0, 1_000_000), - new(LesMessageCode.SendTxV2, 0, 450_000), - new(LesMessageCode.GetTxStatus, 0, 250_000) - }; - - } - -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/HelperTrieRequest.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/HelperTrieRequest.cs deleted file mode 100644 index 594f208600f..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/HelperTrieRequest.cs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -namespace Nethermind.Network.P2P.Subprotocols.Les -{ - public class HelperTrieRequest - { - public HelperTrieType SubType; - public long SectionIndex; - public byte[] Key; - public long FromLevel; - public int AuxiliaryData; - - public HelperTrieRequest() - { - } - - public HelperTrieRequest(HelperTrieType subType, long sectionIndex, byte[] key, long fromLevel, int auxiliaryData) - { - SubType = subType; - SectionIndex = sectionIndex; - Key = key; - FromLevel = fromLevel; - AuxiliaryData = auxiliaryData; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/HelperTrieType.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/HelperTrieType.cs deleted file mode 100644 index 3095020682f..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/HelperTrieType.cs +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -namespace Nethermind.Network.P2P.Subprotocols.Les -{ - public enum HelperTrieType - { - CHT = 0, - BloomBits = 1, - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/LesAnnounceType.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/LesAnnounceType.cs deleted file mode 100644 index 69fedcd3027..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/LesAnnounceType.cs +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -namespace Nethermind.Network.P2P.Subprotocols.Les -{ - public enum LesAnnounceType - { - None = 0x00, - Simple = 0x01, - Signed = 0x02 - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/LesMessageCode.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/LesMessageCode.cs deleted file mode 100644 index ccf4812d5ad..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/LesMessageCode.cs +++ /dev/null @@ -1,33 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -namespace Nethermind.Network.P2P.Subprotocols.Les -{ - public static class LesMessageCode - { - public const int Status = 0x00; - public const int Announce = 0x01; - public const int GetBlockHeaders = 0x02; - public const int BlockHeaders = 0x03; - public const int GetBlockBodies = 0x04; - public const int BlockBodies = 0x05; - public const int GetReceipts = 0x06; - public const int Receipts = 0x07; - public const int GetProofs = 0x08; // deprecated - public const int Proofs = 0x09; // deprecated - public const int GetContractCodes = 0x0a; - public const int ContractCodes = 0x0b; - public const int SendTx = 0x0c; // deprecated - public const int GetHeaderProofs = 0x0d; // deprecated - public const int HeaderProofs = 0x0e; // deprecated - public const int GetProofsV2 = 0x0f; - public const int ProofsV2 = 0x10; - public const int GetHelperTrieProofs = 0x11; - public const int HelperTrieProofs = 0x12; - public const int SendTxV2 = 0x13; - public const int GetTxStatus = 0x14; - public const int TxStatus = 0x15; - public const int Stop = 0x16; - public const int Resume = 0x17; - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/LesProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/LesProtocolHandler.cs deleted file mode 100644 index d1d71d1f40a..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/LesProtocolHandler.cs +++ /dev/null @@ -1,314 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Nethermind.Blockchain; -using Nethermind.Blockchain.Synchronization; -using Nethermind.Consensus.Scheduler; -using Nethermind.Core; -using Nethermind.Core.Collections; -using Nethermind.Core.Crypto; -using Nethermind.Core.Extensions; -using Nethermind.Int256; -using Nethermind.Logging; -using Nethermind.Network.Contract.P2P; -using Nethermind.Network.P2P.EventArg; -using Nethermind.Network.P2P.ProtocolHandlers; -using Nethermind.Network.P2P.Subprotocols.Les.Messages; -using Nethermind.Network.Rlpx; -using Nethermind.Serialization.Rlp; -using Nethermind.Stats; -using Nethermind.Synchronization; -using Nethermind.Synchronization.FastSync; -using CancellationToken = System.Threading.CancellationToken; - -namespace Nethermind.Network.P2P.Subprotocols.Les -{ - public class LesProtocolHandler : SyncPeerProtocolHandlerBase, ISyncPeer - { - public override string Name => "les3"; - public override bool IncludeInTxPool => false; - - public LesProtocolHandler( - ISession session, - IMessageSerializationService serializer, - INodeStatsManager statsManager, - ISyncServer syncServer, - IBackgroundTaskScheduler backgroundTaskScheduler, - ILogManager logManager) : base(session, serializer, statsManager, syncServer, backgroundTaskScheduler, logManager) - { - _lastSentBlock = SyncServer.Head; - } - - public override void Init() - { - if (Logger.IsTrace) Logger.Trace($"{ProtocolCode} v{ProtocolVersion} subprotocol initializing with {Session.Node:c}"); - if (SyncServer.Head is null) - { - throw new InvalidOperationException($"Cannot initialize {ProtocolCode} v{ProtocolVersion} protocol without the head block set"); - } - - BlockHeader head = SyncServer.Head; - StatusMessage statusMessage = new() - { - ProtocolVersion = ProtocolVersion, - NetworkId = (UInt256)SyncServer.NetworkId, - TotalDifficulty = head.TotalDifficulty ?? head.Difficulty, - BestHash = head.Hash, - HeadBlockNo = head.Number, - GenesisHash = SyncServer.Genesis.Hash, - - // TODO - implement config option for these - ServeHeaders = true, - ServeChainSince = 0x00, - //if (config.recentchain is not null) - // ServeRecentChain = Config.recentchain - ServeStateSince = 0x00, - //if (Config.serverecentstate is not null) - // ServeRecentState = Config.RecentState - TxRelay = true, - // TODO - should allow setting to infinite - BufferLimit = int.MaxValue, - MaximumRechargeRate = int.MaxValue - }; - Send(statusMessage); - - CheckProtocolInitTimeout().ContinueWith(x => - { - if (x.IsFaulted && Logger.IsError) - { - Logger.Error("Error during lesProtocol handler timeout logic", x.Exception); - } - }); - } - - - public override byte ProtocolVersion => 3; - - public override string ProtocolCode => Protocol.Les; - - public override int MessageIdSpaceSize => 23; - - protected override TimeSpan InitTimeout => Timeouts.Les3Status; - - public LesAnnounceType RequestedAnnounceType; - - public override event EventHandler ProtocolInitialized; - public override event EventHandler SubprotocolRequested - { - add { } - remove { } - } - - bool _statusReceived; - public override void HandleMessage(ZeroPacket message) - { - if (message.PacketType != LesMessageCode.Status && !_statusReceived) - { - throw new SubprotocolException($"No {nameof(StatusMessage)} received prior to communication with {Session.Node:c}."); - } - - int size = message.Content.ReadableBytes; - - switch (message.PacketType) - { - case LesMessageCode.Status: - { - using StatusMessage statusMessage = Deserialize(message.Content); - if (NetworkDiagTracer.IsEnabled) NetworkDiagTracer.ReportIncomingMessage(Session.Node.Address, Name, statusMessage.ToString(), size); - Handle(statusMessage); - break; - } - case LesMessageCode.GetBlockHeaders: - GetBlockHeadersMessage getBlockHeadersMessage = Deserialize(message.Content); - if (NetworkDiagTracer.IsEnabled) NetworkDiagTracer.ReportIncomingMessage(Session.Node.Address, Name, getBlockHeadersMessage.ToString(), size); - BackgroundTaskScheduler.ScheduleSyncServe(getBlockHeadersMessage, Handle); - break; - case LesMessageCode.GetBlockBodies: - GetBlockBodiesMessage getBlockBodiesMessage = Deserialize(message.Content); - if (NetworkDiagTracer.IsEnabled) NetworkDiagTracer.ReportIncomingMessage(Session.Node.Address, Name, getBlockBodiesMessage.ToString(), size); - BackgroundTaskScheduler.ScheduleSyncServe(getBlockBodiesMessage, Handle); - break; - case LesMessageCode.GetReceipts: - GetReceiptsMessage getReceiptsMessage = Deserialize(message.Content); - if (NetworkDiagTracer.IsEnabled) NetworkDiagTracer.ReportIncomingMessage(Session.Node.Address, Name, getReceiptsMessage.ToString(), size); - BackgroundTaskScheduler.ScheduleSyncServe(getReceiptsMessage, Handle); - break; - case LesMessageCode.GetContractCodes: - GetContractCodesMessage getContractCodesMessage = Deserialize(message.Content); - if (NetworkDiagTracer.IsEnabled) NetworkDiagTracer.ReportIncomingMessage(Session.Node.Address, Name, getContractCodesMessage.ToString(), size); - BackgroundTaskScheduler.ScheduleSyncServe(getContractCodesMessage, Handle); - break; - case LesMessageCode.GetHelperTrieProofs: - GetHelperTrieProofsMessage getHelperTrieProofsMessage = Deserialize(message.Content); - if (NetworkDiagTracer.IsEnabled) NetworkDiagTracer.ReportIncomingMessage(Session.Node.Address, Name, getHelperTrieProofsMessage.ToString(), size); - BackgroundTaskScheduler.ScheduleSyncServe(getHelperTrieProofsMessage, Handle); - break; - } - } - - public void Handle(StatusMessage status) - { - // set defaults - if (!status.AnnounceType.HasValue) status.AnnounceType = 1; - - if (_statusReceived) - { - throw new SubprotocolException($"{nameof(StatusMessage)} has already been received in the past"); - } - - _statusReceived = true; - if (Logger.IsTrace) - Logger.Trace($"LES received status from {Session.Node:c} with" + - Environment.NewLine + $" prot version\t{status.ProtocolVersion}" + - Environment.NewLine + $" network ID\t{status.NetworkId}," + - Environment.NewLine + $" genesis hash\t{status.GenesisHash}," + - Environment.NewLine + $" best hash\t{status.BestHash}," + - Environment.NewLine + $" head blockno\t{status.HeadBlockNo}," + - Environment.NewLine + $" difficulty\t{status.TotalDifficulty}" + - Environment.NewLine + $" announce type\t{status.AnnounceType}" + - Environment.NewLine + $" serve headers\t{status.ServeHeaders}" + - Environment.NewLine + $" serve chain since\t{status.ServeChainSince}" + - Environment.NewLine + $" serve recent chain\t{status.ServeRecentChain}" + - Environment.NewLine + $" serve state since\t{status.ServeStateSince}" + - Environment.NewLine + $" serve recent state\t{status.ServeRecentState}" + - Environment.NewLine + $" transaction relay\t{status.TxRelay}" + - Environment.NewLine + $" buffer limit\t{status.BufferLimit}" + - Environment.NewLine + $" max recharge\t{status.MaximumRechargeRate}"); - // todo - log max request costs table - - _remoteHeadBlockHash = status.BestHash; - - ReceivedProtocolInitMsg(status); - SyncPeerProtocolInitializedEventArgs eventArgs = new(this) - { - NetworkId = (ulong)status.NetworkId, - BestHash = status.BestHash, - GenesisHash = status.GenesisHash, - Protocol = status.Protocol, - ProtocolVersion = status.ProtocolVersion, - TotalDifficulty = status.TotalDifficulty - }; - - TotalDifficulty = status.TotalDifficulty; - RequestedAnnounceType = (LesAnnounceType)status.AnnounceType.Value; - if (RequestedAnnounceType == LesAnnounceType.Signed) throw new NotImplementedException("Signed announcements are not yet supported."); - - ProtocolInitialized?.Invoke(this, eventArgs); - } - - public async Task Handle(GetBlockHeadersMessage getBlockHeaders, CancellationToken cancellationToken) - { - using var message = getBlockHeaders; - Eth.V62.Messages.BlockHeadersMessage ethBlockHeadersMessage = await FulfillBlockHeadersRequest(message.EthMessage, cancellationToken); - // todo - implement cost tracking - return new BlockHeadersMessage(ethBlockHeadersMessage, message.RequestId, int.MaxValue); - } - - public async Task Handle(GetBlockBodiesMessage getBlockBodies, CancellationToken cancellationToken) - { - using var message = getBlockBodies; - Eth.V62.Messages.BlockBodiesMessage ethBlockBodiesMessage = await FulfillBlockBodiesRequest(message.EthMessage, cancellationToken); - // todo - implement cost tracking - return new BlockBodiesMessage(ethBlockBodiesMessage, message.RequestId, int.MaxValue); - } - - public async Task Handle(GetReceiptsMessage getReceipts, CancellationToken cancellationToken) - { - using var message = getReceipts; - Eth.V63.Messages.ReceiptsMessage ethReceiptsMessage = await FulfillReceiptsRequest(message.EthMessage, cancellationToken); - // todo - implement cost tracking - return new ReceiptsMessage(ethReceiptsMessage, message.RequestId, int.MaxValue); - } - - public Task Handle(GetContractCodesMessage getContractCodes, CancellationToken cancellationToken) - { - using var message = getContractCodes; - var codes = SyncServer.GetNodeData(message.RequestAddresses, cancellationToken, NodeDataType.Code); - // todo - implement cost tracking - return Task.FromResult(new ContractCodesMessage(codes, message.RequestId, int.MaxValue)); - } - - public Task Handle(GetHelperTrieProofsMessage getHelperTrieProofs, CancellationToken cancellationToken) - { - using var message = getHelperTrieProofs; - List proofNodes = new(); - List auxData = new(); - - for (int requestNo = 0; requestNo < message.Requests.Length; requestNo++) - { - var request = message.Requests[requestNo]; - switch (request.SubType) - { - case HelperTrieType.CHT: - GetCHTData(request, proofNodes, auxData); - break; - case HelperTrieType.BloomBits: - throw new SubprotocolException("bloom bits trie not yet supported"); - } - } - - return Task.FromResult(new HelperTrieProofsMessage(proofNodes.Distinct().ToArray(), auxData.ToArray(), message.RequestId, int.MaxValue)); - } - - public void GetCHTData(HelperTrieRequest request, List proofNodes, List auxData) - { - var cht = SyncServer.GetCHT(); - // todo - enum? - if (request.AuxiliaryData == 1) - { - auxData.Add(cht.RootHash.BytesToArray()); - return; - } - else if (request.AuxiliaryData == 2) - { - (Hash256 hash, _) = cht.Get(request.Key); - using IOwnedReadOnlyList headerResult = SyncServer.FindHeaders(hash, 1, 0, false); - if (headerResult.Count != 1) throw new SubprotocolException($"Unable to find header for block {request.Key.WithoutLeadingZeros().ToArray().ToLongFromBigEndianByteArrayWithoutLeadingZeros()} for GetHelperProofs response."); - auxData.Add(Rlp.Encode(headerResult[0]).Bytes); - } - proofNodes.AddRange(Synchronization.LesSync.CanonicalHashTrie.BuildProof(request.Key, request.SectionIndex, request.FromLevel)); - } - - private BlockHeader _lastSentBlock; - - public override void NotifyOfNewBlock(Block block, SendBlockMode mode) - { - if (RequestedAnnounceType == LesAnnounceType.None) return; - if (!block.TotalDifficulty.HasValue) - { - throw new InvalidOperationException($"Trying to send a block {block.Hash} with null total difficulty"); - } - - if (block.TotalDifficulty <= _lastSentBlock.TotalDifficulty) return; - - AnnounceMessage announceMessage = new(); - announceMessage.HeadHash = block.Hash; - announceMessage.HeadBlockNo = block.Number; - announceMessage.TotalDifficulty = block.TotalDifficulty.Value; - if (_lastSentBlock is null || block.ParentHash == _lastSentBlock.Hash) - announceMessage.ReorgDepth = 0; - else - { - BlockHeader firstCommonAncestor = SyncServer.FindLowestCommonAncestor(block.Header, _lastSentBlock); - if (firstCommonAncestor is null) - throw new SubprotocolException($"Unable to send announcement to LES peer - No common ancestor found between {block.Header} and {_lastSentBlock}"); - announceMessage.ReorgDepth = _lastSentBlock.Number - firstCommonAncestor.Number; - } - - _lastSentBlock = block.Header; - - Send(announceMessage); - } - - Task ISyncPeer.GetHeadBlockHeader(Hash256? hash, CancellationToken token) - { - return Task.FromResult(_lastSentBlock); - } - - protected override void OnDisposed() { } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/LesProtocolInitializedEventArgs.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/LesProtocolInitializedEventArgs.cs deleted file mode 100644 index 32d73f020de..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/LesProtocolInitializedEventArgs.cs +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System.Numerics; -using Nethermind.Core.Crypto; -using Nethermind.Network.P2P.EventArg; - -namespace Nethermind.Network.P2P.Subprotocols.Les -{ - // todo - I don't think we actually need all of these. prune later. - public class LesProtocolInitializedEventArgs : ProtocolInitializedEventArgs - { - public string Protocol { get; set; } - public byte ProtocolVersion { get; set; } - public long ChainId { get; set; } - public BigInteger TotalDifficulty { get; set; } - public Hash256 BestHash { get; set; } - public long HeadBlockNo { get; set; } - public Hash256 GenesisHash { get; set; } - public byte AnnounceType { get; set; } - public bool ServeHeaders { get; set; } - public long? ServeChainSince { get; set; } - public long? ServeRecentChain { get; set; } - public long? ServeStateSince { get; set; } - public long? ServeRecentState { get; set; } - public bool TxRelay { get; set; } - public int? BufferLimit { get; set; } - public int? MaximumRechargeRate { get; set; } - public RequestCostItem[] MaximumRequestCosts { get; set; } - public LesProtocolInitializedEventArgs(LesProtocolHandler protocolHandler) : base(protocolHandler) - { - - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/AnnounceMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/AnnounceMessage.cs deleted file mode 100644 index b4081912c5d..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/AnnounceMessage.cs +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Crypto; -using Nethermind.Int256; -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class AnnounceMessage : P2PMessage - { - public override int PacketType { get; } = LesMessageCode.Announce; - public override string Protocol => Contract.P2P.Protocol.Les; - public Hash256 HeadHash; - public long HeadBlockNo; - public UInt256 TotalDifficulty; - public long ReorgDepth; - // todo - add optional items - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/AnnounceMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/AnnounceMessageSerializer.cs deleted file mode 100644 index d91b2bf3905..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/AnnounceMessageSerializer.cs +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class AnnounceMessageSerializer : IZeroMessageSerializer - { - public void Serialize(IByteBuffer byteBuffer, AnnounceMessage message) - { - int length = GetLength(message, out int contentLength); - byteBuffer.EnsureWritable(length); - NettyRlpStream rlpStream = new(byteBuffer); - - rlpStream.StartSequence(contentLength); - rlpStream.Encode(message.HeadHash); - rlpStream.Encode(message.HeadBlockNo); - rlpStream.Encode(message.TotalDifficulty); - rlpStream.Encode(message.ReorgDepth); - rlpStream.Encode(Rlp.OfEmptySequence); - } - - public AnnounceMessage Deserialize(IByteBuffer byteBuffer) - { - RlpStream rlpStream = new NettyRlpStream(byteBuffer); - return Deserialize(rlpStream); - } - - private static int GetLength(AnnounceMessage message, out int contentLength) - { - contentLength = - Rlp.LengthOf(message.HeadHash) + - Rlp.LengthOf(message.HeadBlockNo) + - Rlp.LengthOf(message.TotalDifficulty) + - Rlp.LengthOf(message.ReorgDepth) + - Rlp.OfEmptySequence.Length; - - return Rlp.LengthOfSequence(contentLength); - } - - private static AnnounceMessage Deserialize(RlpStream rlpStream) - { - AnnounceMessage announceMessage = new(); - rlpStream.ReadSequenceLength(); - announceMessage.HeadHash = rlpStream.DecodeKeccak(); - announceMessage.HeadBlockNo = rlpStream.DecodeLong(); - announceMessage.TotalDifficulty = rlpStream.DecodeUInt256(); - announceMessage.ReorgDepth = rlpStream.DecodeLong(); - rlpStream.ReadSequenceLength(); - return announceMessage; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/BlockBodiesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/BlockBodiesMessage.cs deleted file mode 100644 index 39faff48984..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/BlockBodiesMessage.cs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class BlockBodiesMessage : P2PMessage - { - public override int PacketType { get; } = LesMessageCode.BlockBodies; - public override string Protocol { get; } = Contract.P2P.Protocol.Les; - public Eth.V62.Messages.BlockBodiesMessage EthMessage { get; set; } - public long RequestId { get; set; } - public int BufferValue { get; set; } - - public BlockBodiesMessage() - { - } - - public BlockBodiesMessage(Eth.V62.Messages.BlockBodiesMessage ethMessage, long requestId, int bufferValue) - { - EthMessage = ethMessage; - RequestId = requestId; - BufferValue = bufferValue; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/BlockBodiesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/BlockBodiesMessageSerializer.cs deleted file mode 100644 index 3211a9705d0..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/BlockBodiesMessageSerializer.cs +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class BlockBodiesMessageSerializer : IZeroMessageSerializer - { - private readonly Eth.V62.Messages.BlockBodiesMessageSerializer _baseDeserializer = new(); - - public void Serialize(IByteBuffer byteBuffer, BlockBodiesMessage message) - { - Eth.V62.Messages.BlockBodiesMessageSerializer ethSerializer = new(); - int ethMessageTotalLength = ethSerializer.GetLength(message.EthMessage, out _); - int contentLength = Rlp.LengthOf(message.RequestId) + Rlp.LengthOf(message.BufferValue) + ethMessageTotalLength; - - int totalLength = Rlp.LengthOfSequence(contentLength); - - RlpStream rlpStream = new NettyRlpStream(byteBuffer); - byteBuffer.EnsureWritable(totalLength); - - rlpStream.StartSequence(contentLength); - rlpStream.Encode(message.RequestId); - rlpStream.Encode(message.BufferValue); - ethSerializer.Serialize(byteBuffer, message.EthMessage); - } - - public BlockBodiesMessage Deserialize(IByteBuffer byteBuffer) - { - NettyRlpStream rlpStream = new(byteBuffer); - BlockBodiesMessage blockBodiesMessage = new(); - rlpStream.ReadSequenceLength(); - blockBodiesMessage.RequestId = rlpStream.DecodeLong(); - blockBodiesMessage.BufferValue = rlpStream.DecodeInt(); - blockBodiesMessage.EthMessage = _baseDeserializer.Deserialize(byteBuffer); - return blockBodiesMessage; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/BlockHeadersMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/BlockHeadersMessage.cs deleted file mode 100644 index 06d65d96b49..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/BlockHeadersMessage.cs +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class BlockHeadersMessage : P2PMessage - { - public override int PacketType { get; } = LesMessageCode.BlockHeaders; - public override string Protocol { get; } = Contract.P2P.Protocol.Les; - public Eth.V62.Messages.BlockHeadersMessage EthMessage { get; set; } - public long RequestId { get; set; } - public int BufferValue { get; set; } - - public BlockHeadersMessage() - { - } - public BlockHeadersMessage(Eth.V62.Messages.BlockHeadersMessage ethMessage, long requestId, int bufferValue) - { - EthMessage = ethMessage; - RequestId = requestId; - BufferValue = bufferValue; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/BlockHeadersMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/BlockHeadersMessageSerializer.cs deleted file mode 100644 index 7a80cba9989..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/BlockHeadersMessageSerializer.cs +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class BlockHeadersMessageSerializer : IZeroMessageSerializer - { - public void Serialize(IByteBuffer byteBuffer, BlockHeadersMessage message) - { - Eth.V62.Messages.BlockHeadersMessageSerializer ethSerializer = new(); - Rlp ethMessage = new(ethSerializer.Serialize(message.EthMessage)); - int contentLength = - Rlp.LengthOf(message.RequestId) + - Rlp.LengthOf(message.BufferValue) + - ethMessage.Length; - - int totalLength = Rlp.LengthOfSequence(contentLength); - - RlpStream rlpStream = new NettyRlpStream(byteBuffer); - byteBuffer.EnsureWritable(totalLength); - - rlpStream.StartSequence(contentLength); - rlpStream.Encode(message.RequestId); - rlpStream.Encode(message.BufferValue); - rlpStream.Encode(ethMessage); - } - - public BlockHeadersMessage Deserialize(IByteBuffer byteBuffer) - { - NettyRlpStream rlpStream = new(byteBuffer); - return Deserialize(rlpStream); - } - - private static BlockHeadersMessage Deserialize(RlpStream rlpStream) - { - BlockHeadersMessage blockHeadersMessage = new(); - rlpStream.ReadSequenceLength(); - blockHeadersMessage.RequestId = rlpStream.DecodeLong(); - blockHeadersMessage.BufferValue = rlpStream.DecodeInt(); - blockHeadersMessage.EthMessage = Eth.V62.Messages.BlockHeadersMessageSerializer.Deserialize(rlpStream); - return blockHeadersMessage; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/ContractCodesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/ContractCodesMessage.cs deleted file mode 100644 index 3c01e93dbfd..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/ContractCodesMessage.cs +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Collections; -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class ContractCodesMessage : P2PMessage - { - public override int PacketType { get; } = LesMessageCode.ContractCodes; - public override string Protocol { get; } = Contract.P2P.Protocol.Les; - public long RequestId; - public int BufferValue; - public IOwnedReadOnlyList Codes; - - public ContractCodesMessage() - { - } - - public ContractCodesMessage(IOwnedReadOnlyList codes, long requestId, int bufferValue) - { - Codes = codes; - RequestId = requestId; - BufferValue = bufferValue; - } - - public override void Dispose() - { - base.Dispose(); - Codes.Dispose(); - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/ContractCodesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/ContractCodesMessageSerializer.cs deleted file mode 100644 index 274d453311d..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/ContractCodesMessageSerializer.cs +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class ContractCodesMessageSerializer : IZeroMessageSerializer - { - public void Serialize(IByteBuffer byteBuffer, ContractCodesMessage message) - { - int innerLength = 0; - for (int i = 0; i < message.Codes.Count; i++) - { - innerLength += Rlp.LengthOf(message.Codes[i]); - } - int contentLength = - Rlp.LengthOf(message.RequestId) + - Rlp.LengthOf(message.BufferValue) + - Rlp.LengthOfSequence(innerLength); - - int totalLength = Rlp.LengthOfSequence(contentLength); - - RlpStream rlpStream = new NettyRlpStream(byteBuffer); - byteBuffer.EnsureWritable(totalLength); - - rlpStream.StartSequence(contentLength); - rlpStream.Encode(message.RequestId); - rlpStream.Encode(message.BufferValue); - rlpStream.StartSequence(innerLength); - for (int i = 0; i < message.Codes.Count; i++) - { - rlpStream.Encode(message.Codes[i]); - } - } - - public ContractCodesMessage Deserialize(IByteBuffer byteBuffer) - { - NettyRlpStream rlpStream = new(byteBuffer); - return Deserialize(rlpStream); - } - - public static ContractCodesMessage Deserialize(RlpStream rlpStream) - { - ContractCodesMessage contractCodesMessage = new(); - rlpStream.ReadSequenceLength(); - contractCodesMessage.RequestId = rlpStream.DecodeLong(); - contractCodesMessage.BufferValue = rlpStream.DecodeInt(); - contractCodesMessage.Codes = rlpStream.DecodeArrayPoolList(stream => stream.DecodeByteArray()); - return contractCodesMessage; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetBlockBodiesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetBlockBodiesMessage.cs deleted file mode 100644 index 31fd5323bd6..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetBlockBodiesMessage.cs +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class GetBlockBodiesMessage : P2PMessage - { - public override int PacketType { get; } = LesMessageCode.GetBlockBodies; - public override string Protocol { get; } = Contract.P2P.Protocol.Les; - public long RequestId; - public Eth.V62.Messages.GetBlockBodiesMessage EthMessage; - - public GetBlockBodiesMessage() - { - } - - public GetBlockBodiesMessage(Eth.V62.Messages.GetBlockBodiesMessage ethMessage, long requestId) - { - EthMessage = ethMessage; - RequestId = requestId; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetBlockBodiesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetBlockBodiesMessageSerializer.cs deleted file mode 100644 index 75d9a1a54f2..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetBlockBodiesMessageSerializer.cs +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class GetBlockBodiesMessageSerializer : IZeroMessageSerializer - { - public void Serialize(IByteBuffer byteBuffer, GetBlockBodiesMessage message) - { - Eth.V62.Messages.GetBlockBodiesMessageSerializer ethSerializer = new(); - Rlp ethMessage = new(ethSerializer.Serialize(message.EthMessage)); - int contentLength = Rlp.LengthOf(message.RequestId) + ethMessage.Length; - - int totalLength = Rlp.LengthOfSequence(contentLength); - - RlpStream rlpStream = new NettyRlpStream(byteBuffer); - byteBuffer.EnsureWritable(totalLength); - - rlpStream.StartSequence(contentLength); - rlpStream.Encode(message.RequestId); - rlpStream.Encode(ethMessage); - } - - public GetBlockBodiesMessage Deserialize(IByteBuffer byteBuffer) - { - NettyRlpStream rlpStream = new(byteBuffer); - return Deserialize(rlpStream); - } - - private static GetBlockBodiesMessage Deserialize(RlpStream rlpStream) - { - GetBlockBodiesMessage getBlockBodiesMessage = new(); - rlpStream.ReadSequenceLength(); - getBlockBodiesMessage.RequestId = rlpStream.DecodeLong(); - getBlockBodiesMessage.EthMessage = Eth.V62.Messages.GetBlockBodiesMessageSerializer.Deserialize(rlpStream); - return getBlockBodiesMessage; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetBlockHeadersMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetBlockHeadersMessage.cs deleted file mode 100644 index ba7ba020278..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetBlockHeadersMessage.cs +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class GetBlockHeadersMessage : P2PMessage - { - public override int PacketType { get; } = LesMessageCode.GetBlockHeaders; - public override string Protocol { get; } = Contract.P2P.Protocol.Les; - public long RequestId; - public Eth.V62.Messages.GetBlockHeadersMessage EthMessage; - - public GetBlockHeadersMessage() - { - } - - public GetBlockHeadersMessage(Eth.V62.Messages.GetBlockHeadersMessage ethMessage, long requestId) - { - EthMessage = ethMessage; - RequestId = requestId; - } - - public override void Dispose() - { - base.Dispose(); - EthMessage?.Dispose(); - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetBlockHeadersMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetBlockHeadersMessageSerializer.cs deleted file mode 100644 index e522b890213..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetBlockHeadersMessageSerializer.cs +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class GetBlockHeadersMessageSerializer : IZeroMessageSerializer - { - public void Serialize(IByteBuffer byteBuffer, GetBlockHeadersMessage message) - { - Eth.V62.Messages.GetBlockHeadersMessageSerializer ethSerializer = new(); - Rlp ethMessage = new(ethSerializer.Serialize(message.EthMessage)); - int contentLength = Rlp.LengthOf(message.RequestId) + ethMessage.Length; - - int totalLength = Rlp.LengthOfSequence(contentLength); - - RlpStream rlpStream = new NettyRlpStream(byteBuffer); - byteBuffer.EnsureWritable(totalLength); - - rlpStream.StartSequence(contentLength); - rlpStream.Encode(message.RequestId); - rlpStream.Encode(ethMessage); - } - - public GetBlockHeadersMessage Deserialize(IByteBuffer byteBuffer) - { - NettyRlpStream rlpStream = new(byteBuffer); - return Deserialize(rlpStream); - } - - private static GetBlockHeadersMessage Deserialize(RlpStream rlpStream) - { - GetBlockHeadersMessage getBlockHeadersMessage = new(); - rlpStream.ReadSequenceLength(); - getBlockHeadersMessage.RequestId = rlpStream.DecodeLong(); - getBlockHeadersMessage.EthMessage = Eth.V62.Messages.GetBlockHeadersMessageSerializer.Deserialize(rlpStream); - return getBlockHeadersMessage; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetContractCodesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetContractCodesMessage.cs deleted file mode 100644 index 8e157088bc3..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetContractCodesMessage.cs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System.Linq; -using Nethermind.Core.Crypto; -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class GetContractCodesMessage : P2PMessage - { - public override int PacketType { get; } = LesMessageCode.GetContractCodes; - public override string Protocol { get; } = Contract.P2P.Protocol.Les; - public long RequestId; - public CodeRequest[] Requests; - - public Hash256[] RequestAddresses => - Requests.Select(request => request.AccountKey).ToArray(); - - public GetContractCodesMessage() - { - } - - public GetContractCodesMessage(CodeRequest[] requests, long requestId) - { - Requests = requests; - RequestId = requestId; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetContractCodesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetContractCodesMessageSerializer.cs deleted file mode 100644 index 125a8ae9b4b..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetContractCodesMessageSerializer.cs +++ /dev/null @@ -1,62 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Core.Crypto; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class GetContractCodesMessageSerializer : IZeroMessageSerializer - { - public void Serialize(IByteBuffer byteBuffer, GetContractCodesMessage message) - { - // note: If there are any changes to how a hash is encoded, this will break (compression?) - // calling LengthOf for each hash would be more resistant to future changes, if we think there will be any - int requestLength = Rlp.LengthOf(Keccak.OfAnEmptyString) * 2; - int allRequestsLength = Rlp.LengthOfSequence(requestLength) * message.Requests.Length; - int contentLength = - Rlp.LengthOf(message.RequestId) + - Rlp.LengthOfSequence(allRequestsLength); - - int totalLength = Rlp.LengthOfSequence(contentLength); - - RlpStream rlpStream = new NettyRlpStream(byteBuffer); - byteBuffer.EnsureWritable(totalLength); - - rlpStream.StartSequence(contentLength); - rlpStream.Encode(message.RequestId); - - rlpStream.StartSequence(allRequestsLength); - foreach (CodeRequest request in message.Requests) - { - rlpStream.StartSequence(requestLength); - rlpStream.Encode(request.BlockHash); - rlpStream.Encode(request.AccountKey); - } - } - - public GetContractCodesMessage Deserialize(IByteBuffer byteBuffer) - { - NettyRlpStream rlpStream = new(byteBuffer); - return Deserialize(rlpStream); - } - - public static GetContractCodesMessage Deserialize(RlpStream rlpStream) - { - GetContractCodesMessage getContractCodesMessage = new(); - rlpStream.ReadSequenceLength(); - getContractCodesMessage.RequestId = rlpStream.DecodeLong(); - getContractCodesMessage.Requests = rlpStream.DecodeArray(stream => - { - CodeRequest request = new(); - stream.ReadSequenceLength(); - request.BlockHash = stream.DecodeKeccak(); - request.AccountKey = stream.DecodeKeccak(); - return request; - }); - - return getContractCodesMessage; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetHelperTrieProofsMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetHelperTrieProofsMessage.cs deleted file mode 100644 index 6b12d29e277..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetHelperTrieProofsMessage.cs +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class GetHelperTrieProofsMessage : P2PMessage - { - public override int PacketType { get; } = LesMessageCode.GetHelperTrieProofs; - public override string Protocol { get; } = Contract.P2P.Protocol.Les; - public long RequestId; - public HelperTrieRequest[] Requests; - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetHelperTrieProofsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetHelperTrieProofsMessageSerializer.cs deleted file mode 100644 index af4c3756c8d..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetHelperTrieProofsMessageSerializer.cs +++ /dev/null @@ -1,75 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class GetHelperTrieProofsMessageSerializer : IZeroMessageSerializer - { - public void Serialize(IByteBuffer byteBuffer, GetHelperTrieProofsMessage message) - { - int innerLength = 0; - foreach (var request in message.Requests) - { - innerLength += Rlp.LengthOfSequence(GetRequestLength(request)); - } - int contentLength = Rlp.LengthOf(message.RequestId) + - Rlp.LengthOfSequence(innerLength); - - int totalLength = Rlp.LengthOfSequence(contentLength); - - RlpStream rlpStream = new NettyRlpStream(byteBuffer); - byteBuffer.EnsureWritable(totalLength); - - rlpStream.StartSequence(contentLength); - rlpStream.Encode(message.RequestId); - rlpStream.StartSequence(innerLength); - foreach (var request in message.Requests) - { - rlpStream.StartSequence(GetRequestLength(request)); - rlpStream.Encode((int)request.SubType); - rlpStream.Encode(request.SectionIndex); - rlpStream.Encode(request.Key); - rlpStream.Encode(request.FromLevel); - rlpStream.Encode(request.AuxiliaryData); - } - } - - private static int GetRequestLength(HelperTrieRequest request) - { - return - Rlp.LengthOf((int)request.SubType) + - Rlp.LengthOf(request.SectionIndex) + - Rlp.LengthOf(request.Key) + - Rlp.LengthOf(request.FromLevel) + - Rlp.LengthOf(request.AuxiliaryData); - } - - public GetHelperTrieProofsMessage Deserialize(IByteBuffer byteBuffer) - { - NettyRlpStream rlpStream = new(byteBuffer); - return Deserialize(rlpStream); - } - - public static GetHelperTrieProofsMessage Deserialize(RlpStream rlpStream) - { - GetHelperTrieProofsMessage message = new(); - rlpStream.ReadSequenceLength(); - message.RequestId = rlpStream.DecodeLong(); - message.Requests = rlpStream.DecodeArray(stream => - { - HelperTrieRequest request = new(); - stream.ReadSequenceLength(); - request.SubType = (HelperTrieType)stream.DecodeInt(); - request.SectionIndex = stream.DecodeLong(); - request.Key = stream.DecodeByteArray(); - request.FromLevel = stream.DecodeLong(); - request.AuxiliaryData = stream.DecodeInt(); - return request; - }); - return message; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetReceiptsMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetReceiptsMessage.cs deleted file mode 100644 index 1fcccd3af48..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetReceiptsMessage.cs +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class GetReceiptsMessage : P2PMessage - { - public override int PacketType { get; } = LesMessageCode.GetReceipts; - public override string Protocol { get; } = Contract.P2P.Protocol.Les; - public long RequestId; - public Eth.V63.Messages.GetReceiptsMessage EthMessage; - - public GetReceiptsMessage() - { - } - - public GetReceiptsMessage(Eth.V63.Messages.GetReceiptsMessage ethMessage, long requestId) - { - EthMessage = ethMessage; - RequestId = requestId; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetReceiptsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetReceiptsMessageSerializer.cs deleted file mode 100644 index e8d0f3decad..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/GetReceiptsMessageSerializer.cs +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class GetReceiptsMessageSerializer : IZeroMessageSerializer - { - public void Serialize(IByteBuffer byteBuffer, GetReceiptsMessage message) - { - Eth.V63.Messages.GetReceiptsMessageSerializer ethSerializer = new(); - Rlp ethMessage = new(ethSerializer.Serialize(message.EthMessage)); - int contentLength = Rlp.LengthOf(message.RequestId) + ethMessage.Length; - - int totalLength = Rlp.LengthOfSequence(contentLength); - - RlpStream rlpStream = new NettyRlpStream(byteBuffer); - byteBuffer.EnsureWritable(totalLength); - - rlpStream.StartSequence(contentLength); - rlpStream.Encode(message.RequestId); - rlpStream.Encode(ethMessage); - } - - public GetReceiptsMessage Deserialize(IByteBuffer byteBuffer) - { - NettyRlpStream rlpStream = new(byteBuffer); - return Deserialize(rlpStream); - } - - public static GetReceiptsMessage Deserialize(RlpStream rlpStream) - { - GetReceiptsMessage getReceiptsMessage = new(); - rlpStream.ReadSequenceLength(); - getReceiptsMessage.RequestId = rlpStream.DecodeLong(); - getReceiptsMessage.EthMessage = Eth.V63.Messages.GetReceiptsMessageSerializer.Deserialize(rlpStream); - return getReceiptsMessage; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/HelperTrieProofsMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/HelperTrieProofsMessage.cs deleted file mode 100644 index 961644f654d..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/HelperTrieProofsMessage.cs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class HelperTrieProofsMessage : P2PMessage - { - public override int PacketType { get; } = LesMessageCode.HelperTrieProofs; - public override string Protocol { get; } = Contract.P2P.Protocol.Les; - public long RequestId; - public int BufferValue; - - public byte[][] ProofNodes; - public byte[][] AuxiliaryData; - - public HelperTrieProofsMessage() - { - } - - public HelperTrieProofsMessage(byte[][] proofNodes, byte[][] auxiliaryData, long requestId, int bufferValue) - { - ProofNodes = proofNodes; - AuxiliaryData = auxiliaryData; - RequestId = requestId; - BufferValue = bufferValue; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/HelperTrieProofsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/HelperTrieProofsMessageSerializer.cs deleted file mode 100644 index 14a11ba6d70..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/HelperTrieProofsMessageSerializer.cs +++ /dev/null @@ -1,72 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Core.Crypto; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class HelperTrieProofsMessageSerializer : IZeroMessageSerializer - { - public void Serialize(IByteBuffer byteBuffer, HelperTrieProofsMessage message) - { - Hash256[] proofNodesKeccak = new Hash256[message.ProofNodes.Length]; - int proofNodesContentLength = 0; - for (int i = 0; i < message.ProofNodes.Length; i++) - { - proofNodesKeccak[i] = new Hash256(message.ProofNodes[i]); - proofNodesContentLength += Rlp.LengthOf(proofNodesKeccak[i]); - } - - int tempAuxContentLength = 0; - for (int i = 0; i < message.AuxiliaryData.Length; i++) - { - tempAuxContentLength += Rlp.LengthOf(message.AuxiliaryData[i]); - } - - int innerContentLength = Rlp.LengthOfSequence(proofNodesContentLength) + Rlp.LengthOfSequence(tempAuxContentLength); - int contentLength = - Rlp.LengthOf(message.RequestId) + - Rlp.LengthOf(message.BufferValue) + - Rlp.LengthOfSequence(innerContentLength); - int totalLength = Rlp.LengthOfSequence(contentLength); - - RlpStream rlpStream = new NettyRlpStream(byteBuffer); - byteBuffer.EnsureWritable(totalLength); - - rlpStream.StartSequence(contentLength); - rlpStream.Encode(message.RequestId); - rlpStream.Encode(message.BufferValue); - rlpStream.StartSequence(innerContentLength); - rlpStream.StartSequence(proofNodesContentLength); - for (int i = 0; i < message.ProofNodes.Length; i++) - { - rlpStream.Encode(proofNodesKeccak[i]); - } - rlpStream.StartSequence(tempAuxContentLength); - for (int i = 0; i < message.AuxiliaryData.Length; i++) - { - rlpStream.Encode(message.AuxiliaryData[i]); - } - } - - public HelperTrieProofsMessage Deserialize(IByteBuffer byteBuffer) - { - NettyRlpStream rlpStream = new(byteBuffer); - return Deserialize(rlpStream); - } - - public static HelperTrieProofsMessage Deserialize(RlpStream rlpStream) - { - HelperTrieProofsMessage message = new(); - rlpStream.ReadSequenceLength(); - message.RequestId = rlpStream.DecodeLong(); - message.BufferValue = rlpStream.DecodeInt(); - rlpStream.ReadSequenceLength(); - message.ProofNodes = rlpStream.DecodeArray(stream => stream.DecodeByteArray()); - message.AuxiliaryData = rlpStream.DecodeArray(stream => stream.DecodeByteArray()); - return message; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/ReceiptsMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/ReceiptsMessage.cs deleted file mode 100644 index e7dbc49ddac..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/ReceiptsMessage.cs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class ReceiptsMessage : P2PMessage - { - public override int PacketType { get; } = LesMessageCode.Receipts; - public override string Protocol { get; } = Contract.P2P.Protocol.Les; - public long RequestId; - public int BufferValue; - public Eth.V63.Messages.ReceiptsMessage EthMessage; - - public ReceiptsMessage() - { - } - - public ReceiptsMessage(Eth.V63.Messages.ReceiptsMessage ethMessage, long requestId, int bufferValue) - { - EthMessage = ethMessage; - RequestId = requestId; - BufferValue = bufferValue; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/ReceiptsMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/ReceiptsMessageSerializer.cs deleted file mode 100644 index 8504c7df3b2..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/ReceiptsMessageSerializer.cs +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using DotNetty.Buffers; -using Nethermind.Core.Specs; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class ReceiptsMessageSerializer : IZeroMessageSerializer - { - private readonly ISpecProvider _specProvider; - - public ReceiptsMessageSerializer(ISpecProvider specProvider) - { - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); - } - - public void Serialize(IByteBuffer byteBuffer, ReceiptsMessage message) - { - Eth.V63.Messages.ReceiptsMessageSerializer ethSerializer = new(_specProvider); - Rlp ethMessage = new(ethSerializer.Serialize(message.EthMessage)); - int contentLength = Rlp.LengthOf(message.RequestId) + Rlp.LengthOf(message.BufferValue) + ethMessage.Length; - - int totalLength = Rlp.LengthOfSequence(contentLength); - - RlpStream rlpStream = new NettyRlpStream(byteBuffer); - byteBuffer.EnsureWritable(totalLength); - - rlpStream.StartSequence(contentLength); - rlpStream.Encode(message.RequestId); - rlpStream.Encode(message.BufferValue); - rlpStream.Encode(ethMessage); - } - - public ReceiptsMessage Deserialize(IByteBuffer byteBuffer) - { - NettyRlpStream rlpStream = new(byteBuffer); - return Deserialize(rlpStream); - } - - public ReceiptsMessage Deserialize(RlpStream rlpStream) - { - ReceiptsMessage receiptsMessage = new(); - Eth.V63.Messages.ReceiptsMessageSerializer ethSerializer = new(_specProvider); - - rlpStream.ReadSequenceLength(); - receiptsMessage.RequestId = rlpStream.DecodeLong(); - receiptsMessage.BufferValue = rlpStream.DecodeInt(); - receiptsMessage.EthMessage = ethSerializer.Deserialize(rlpStream); - return receiptsMessage; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/StatusMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/StatusMessage.cs deleted file mode 100644 index a3f4c79d97c..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/StatusMessage.cs +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Crypto; -using Nethermind.Int256; -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class StatusMessage : P2PMessage - { - public static class KeyNames - { - public const string ProtocolVersion = "protocolVersion"; - public const string NetworkId = "networkId"; - public const string TotalDifficulty = "headTd"; - public const string BestHash = "headHash"; - public const string HeadBlockNo = "headNum"; - public const string GenesisHash = "genesisHash"; - public const string AnnounceType = "announceType"; - public const string ServeHeaders = "serveHeaders"; - public const string ServeChainSince = "serveChainSince"; - public const string ServeRecentChain = "serveRecentChain"; - public const string ServeStateSince = "serveStateSince"; - public const string ServeRecentState = "serveRecentState"; - public const string TxRelay = "txRelay"; - public const string BufferLimit = "flowControl/BL"; - public const string MaximumRechargeRate = "flowControl/MRR"; - public const string MaximumRequestCosts = "flowControl/MRC"; - } - - public override int PacketType { get; } = LesMessageCode.Status; - public override string Protocol => Contract.P2P.Protocol.Les; - public byte ProtocolVersion { get; set; } - public UInt256 NetworkId { get; set; } - public UInt256 TotalDifficulty { get; set; } - public Hash256 BestHash { get; set; } - public long HeadBlockNo { get; set; } - public Hash256 GenesisHash { get; set; } - #region optional - public byte? AnnounceType { get; set; } // sent from client only - public bool ServeHeaders { get; set; } - public long? ServeChainSince { get; set; } - public long? ServeRecentChain { get; set; } - public long? ServeStateSince { get; set; } - public long? ServeRecentState { get; set; } - public bool TxRelay { get; set; } - public int? BufferLimit { get; set; } - public int? MaximumRechargeRate { get; set; } - // These are the initial defaults from geth. - // It probably doesn't make sense to define them here in the finished implementation, since the client will want to use the values supplied by the server - - // TODO: Benchmark finished implementation and update based on our actual serve times. - // TODO: Implement cost scaling to account for users with different capabilities - https://github.com/ethereum/go-ethereum/blob/01d92531ee0993c0e6e5efe877a1242bfd808626/les/costtracker.go#L437 - // TODO: Implement multiple cost lists, so it can be limited based on the minimum of available bandwidth, cpu time, etc. - https://github.com/ethereum/go-ethereum/blob/01d92531ee0993c0e6e5efe877a1242bfd808626/les/costtracker.go#L186 - // TODO: This might be better as a dictionary - public RequestCostItem[] MaximumRequestCosts = CostTracker.DefaultRequestCostTable; - #endregion - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/StatusMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/StatusMessageSerializer.cs deleted file mode 100644 index 279233a99a9..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/Messages/StatusMessageSerializer.cs +++ /dev/null @@ -1,304 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Les.Messages -{ - public class StatusMessageSerializer : IZeroMessageSerializer - { - public void Serialize(IByteBuffer byteBuffer, StatusMessage message) - { - NettyRlpStream rlpStream = new(byteBuffer); - - #region Find Lengths - int totalContentLength = 0; - int protocolVersionLength = Rlp.LengthOf(StatusMessage.KeyNames.ProtocolVersion) + Rlp.LengthOf(message.ProtocolVersion); - totalContentLength += Rlp.LengthOfSequence(protocolVersionLength); - - int networkIdLength = Rlp.LengthOf(StatusMessage.KeyNames.NetworkId) + Rlp.LengthOf(message.NetworkId); - totalContentLength += Rlp.LengthOfSequence(networkIdLength); - - int headTdLength = Rlp.LengthOf(StatusMessage.KeyNames.TotalDifficulty) + Rlp.LengthOf(message.TotalDifficulty); - totalContentLength += Rlp.LengthOfSequence(headTdLength); - - int headHashLength = Rlp.LengthOf(StatusMessage.KeyNames.BestHash) + Rlp.LengthOf(message.BestHash); - totalContentLength += Rlp.LengthOfSequence(headHashLength); - - int headNumLength = Rlp.LengthOf(StatusMessage.KeyNames.HeadBlockNo) + Rlp.LengthOf(message.HeadBlockNo); - totalContentLength += Rlp.LengthOfSequence(headNumLength); - - int genesisHashLength = Rlp.LengthOf(StatusMessage.KeyNames.GenesisHash) + Rlp.LengthOf(message.GenesisHash); - totalContentLength += Rlp.LengthOfSequence(genesisHashLength); - - int announceTypeLength = 0; - if (message.AnnounceType.HasValue) - { - announceTypeLength = Rlp.LengthOf(StatusMessage.KeyNames.AnnounceType) + Rlp.LengthOf(message.AnnounceType.Value); - totalContentLength += Rlp.LengthOfSequence(announceTypeLength); - } - - int serveHeadersLength = 0; - if (message.ServeHeaders) - { - serveHeadersLength = Rlp.LengthOf(StatusMessage.KeyNames.ServeHeaders) + Rlp.OfEmptySequence.Length; - totalContentLength += Rlp.LengthOfSequence(serveHeadersLength); - } - - int serveChainSinceLength = 0; - if (message.ServeChainSince.HasValue) - { - serveChainSinceLength = Rlp.LengthOf(StatusMessage.KeyNames.ServeChainSince) + Rlp.LengthOf(message.ServeChainSince.Value); - totalContentLength += Rlp.LengthOfSequence(serveChainSinceLength); - } - - int serveRecentChainLength = 0; - if (message.ServeRecentChain.HasValue) - { - serveRecentChainLength = Rlp.LengthOf(StatusMessage.KeyNames.ServeRecentChain) + Rlp.LengthOf(message.ServeRecentChain.Value); - totalContentLength += Rlp.LengthOfSequence(serveRecentChainLength); - } - - int serveStateSinceLength = 0; - if (message.ServeStateSince.HasValue) - { - serveStateSinceLength = Rlp.LengthOf(StatusMessage.KeyNames.ServeStateSince) + Rlp.LengthOf(message.ServeStateSince.Value); - totalContentLength += Rlp.LengthOfSequence(serveStateSinceLength); - } - - int serveRecentStateLength = 0; - if (message.ServeRecentState.HasValue) - { - serveRecentStateLength = Rlp.LengthOf(StatusMessage.KeyNames.ServeRecentState) + Rlp.LengthOf(message.ServeRecentState.Value); - totalContentLength += Rlp.LengthOfSequence(serveRecentStateLength); - } - - int txRelayLength = 0; - if (message.TxRelay) - { - txRelayLength = Rlp.LengthOf(StatusMessage.KeyNames.TxRelay) + Rlp.OfEmptySequence.Length; - totalContentLength += Rlp.LengthOfSequence(txRelayLength); - } - - int bufferLimitLength = 0; - if (message.BufferLimit.HasValue) - { - bufferLimitLength = Rlp.LengthOf(StatusMessage.KeyNames.BufferLimit) + Rlp.LengthOf(message.BufferLimit.Value); - totalContentLength += Rlp.LengthOfSequence(bufferLimitLength); - } - - int maxRechargeRateLength = 0; - if (message.MaximumRechargeRate.HasValue) - { - maxRechargeRateLength = Rlp.LengthOf(StatusMessage.KeyNames.MaximumRechargeRate) + Rlp.LengthOf(message.MaximumRechargeRate.Value); - totalContentLength += Rlp.LengthOfSequence(maxRechargeRateLength); - } - - int maxRequestCostsLength = 0; - int innerCostListLength = 0; - if (message.MaximumRequestCosts is not null) - { - // todo - what's the best way to do this? Calculating the length twice is definitely less than ideal. - // Maybe build RLP for them here, and append bytes below? - maxRequestCostsLength += Rlp.LengthOf(StatusMessage.KeyNames.MaximumRequestCosts); - foreach (var item in message.MaximumRequestCosts) - { - int costContentLength = Rlp.LengthOf(item.MessageCode) + Rlp.LengthOf(item.BaseCost) + Rlp.LengthOf(item.RequestCost); - innerCostListLength += Rlp.LengthOfSequence(costContentLength); - } - maxRequestCostsLength += Rlp.LengthOfSequence(innerCostListLength); - totalContentLength += Rlp.LengthOfSequence(maxRequestCostsLength); - } - #endregion - - #region Encode Values - int totalLength = Rlp.LengthOfSequence(totalContentLength); - byteBuffer.EnsureWritable(totalLength); - rlpStream.StartSequence(totalContentLength); - - rlpStream.StartSequence(protocolVersionLength); - rlpStream.Encode(StatusMessage.KeyNames.ProtocolVersion); - rlpStream.Encode(message.ProtocolVersion); - - rlpStream.StartSequence(networkIdLength); - rlpStream.Encode(StatusMessage.KeyNames.NetworkId); - rlpStream.Encode(message.NetworkId); - - rlpStream.StartSequence(headTdLength); - rlpStream.Encode(StatusMessage.KeyNames.TotalDifficulty); - rlpStream.Encode(message.TotalDifficulty); - - rlpStream.StartSequence(headHashLength); - rlpStream.Encode(StatusMessage.KeyNames.BestHash); - rlpStream.Encode(message.BestHash); - - rlpStream.StartSequence(headNumLength); - rlpStream.Encode(StatusMessage.KeyNames.HeadBlockNo); - rlpStream.Encode(message.HeadBlockNo); - - rlpStream.StartSequence(genesisHashLength); - rlpStream.Encode(StatusMessage.KeyNames.GenesisHash); - rlpStream.Encode(message.GenesisHash); - - if (message.AnnounceType.HasValue) - { - rlpStream.StartSequence(announceTypeLength); - rlpStream.Encode(StatusMessage.KeyNames.AnnounceType); - rlpStream.Encode(message.AnnounceType.Value); - } - - if (message.ServeHeaders) - { - rlpStream.StartSequence(serveHeadersLength); - rlpStream.Encode(StatusMessage.KeyNames.ServeHeaders); - rlpStream.Encode(Rlp.OfEmptySequence); - } - - if (message.ServeChainSince.HasValue) - { - rlpStream.StartSequence(serveChainSinceLength); - rlpStream.Encode(StatusMessage.KeyNames.ServeChainSince); - rlpStream.Encode(message.ServeChainSince.Value); - } - - if (message.ServeRecentChain.HasValue) - { - rlpStream.StartSequence(serveRecentChainLength); - rlpStream.Encode(StatusMessage.KeyNames.ServeRecentChain); - rlpStream.Encode(message.ServeRecentChain.Value); - } - - if (message.ServeStateSince.HasValue) - { - rlpStream.StartSequence(serveStateSinceLength); - rlpStream.Encode(StatusMessage.KeyNames.ServeStateSince); - rlpStream.Encode(message.ServeStateSince.Value); - } - - if (message.ServeRecentState.HasValue) - { - rlpStream.StartSequence(serveRecentStateLength); - rlpStream.Encode(StatusMessage.KeyNames.ServeRecentState); - rlpStream.Encode(message.ServeRecentState.Value); - } - - if (message.TxRelay) - { - rlpStream.StartSequence(txRelayLength); - rlpStream.Encode(StatusMessage.KeyNames.TxRelay); - rlpStream.Encode(Rlp.OfEmptySequence); - } - - if (message.BufferLimit.HasValue) - { - rlpStream.StartSequence(bufferLimitLength); - rlpStream.Encode(StatusMessage.KeyNames.BufferLimit); - rlpStream.Encode(message.BufferLimit.Value); - } - - if (message.MaximumRechargeRate.HasValue) - { - rlpStream.StartSequence(maxRechargeRateLength); - rlpStream.Encode(StatusMessage.KeyNames.MaximumRechargeRate); - rlpStream.Encode(message.MaximumRechargeRate.Value); - } - - if (message.MaximumRequestCosts is not null) - { - rlpStream.StartSequence(maxRequestCostsLength); - rlpStream.Encode(StatusMessage.KeyNames.MaximumRequestCosts); - rlpStream.StartSequence(innerCostListLength); - foreach (var item in message.MaximumRequestCosts) - { - int length = Rlp.LengthOf(item.MessageCode) + Rlp.LengthOf(item.BaseCost) + Rlp.LengthOf(item.RequestCost); - rlpStream.StartSequence(length); - rlpStream.Encode(item.MessageCode); - rlpStream.Encode(item.BaseCost); - rlpStream.Encode(item.RequestCost); - } - } - #endregion - } - - public StatusMessage Deserialize(IByteBuffer byteBuffer) - { - RlpStream rlpStream = new NettyRlpStream(byteBuffer); - return Deserialize(rlpStream); - } - - private static StatusMessage Deserialize(RlpStream rlpStream) - { - StatusMessage statusMessage = new(); - (int prefixLength, int contentLength) = rlpStream.PeekPrefixAndContentLength(); - var totalLength = contentLength; - rlpStream.Position += prefixLength; - var readLength = prefixLength; - while (totalLength > readLength) - { - (prefixLength, contentLength) = rlpStream.PeekPrefixAndContentLength(); - readLength += prefixLength + contentLength; - rlpStream.Position += prefixLength; - string key = rlpStream.DecodeString(); - switch (key) - { - case StatusMessage.KeyNames.ProtocolVersion: - statusMessage.ProtocolVersion = rlpStream.DecodeByte(); - break; - case StatusMessage.KeyNames.NetworkId: - statusMessage.NetworkId = rlpStream.DecodeUInt256(); - break; - case StatusMessage.KeyNames.TotalDifficulty: - statusMessage.TotalDifficulty = rlpStream.DecodeUInt256(); - break; - case StatusMessage.KeyNames.BestHash: - statusMessage.BestHash = rlpStream.DecodeKeccak(); - break; - case StatusMessage.KeyNames.HeadBlockNo: - statusMessage.HeadBlockNo = rlpStream.DecodeLong(); - break; - case StatusMessage.KeyNames.GenesisHash: - statusMessage.GenesisHash = rlpStream.DecodeKeccak(); - break; - case StatusMessage.KeyNames.AnnounceType: - statusMessage.AnnounceType = rlpStream.DecodeByte(); - break; - case StatusMessage.KeyNames.ServeHeaders: - statusMessage.ServeHeaders = true; - rlpStream.SkipItem(); - break; - case StatusMessage.KeyNames.ServeChainSince: - statusMessage.ServeChainSince = rlpStream.DecodeLong(); - break; - case StatusMessage.KeyNames.ServeRecentChain: - statusMessage.ServeRecentChain = rlpStream.DecodeLong(); - break; - case StatusMessage.KeyNames.ServeStateSince: - statusMessage.ServeStateSince = rlpStream.DecodeLong(); - break; - case StatusMessage.KeyNames.ServeRecentState: - statusMessage.ServeRecentState = rlpStream.DecodeLong(); - break; - case StatusMessage.KeyNames.TxRelay: - statusMessage.TxRelay = true; - rlpStream.SkipItem(); - break; - case StatusMessage.KeyNames.BufferLimit: - statusMessage.BufferLimit = rlpStream.DecodeInt(); - break; - case StatusMessage.KeyNames.MaximumRechargeRate: - statusMessage.MaximumRechargeRate = rlpStream.DecodeInt(); - break; - case StatusMessage.KeyNames.MaximumRequestCosts: - // todo - default: - // Ignore unknown keys - rlpStream.Position = readLength; - break; - } - } - - return statusMessage; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/RequestCost.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/RequestCost.cs deleted file mode 100644 index d9aa502fac8..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Les/RequestCost.cs +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -namespace Nethermind.Network.P2P.Subprotocols.Les -{ - public class RequestCostItem - { - public int MessageCode; - public int BaseCost; - public int RequestCost; - public RequestCostItem(int messageCode, int baseCost, int requestCost) - { - MessageCode = messageCode; - BaseCost = baseCost; - RequestCost = requestCost; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/Messages/BlockWitnessHashesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/Messages/BlockWitnessHashesMessage.cs deleted file mode 100644 index a6092bd0859..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/Messages/BlockWitnessHashesMessage.cs +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Crypto; -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Wit.Messages -{ - public class BlockWitnessHashesMessage(long requestId, Hash256[] hashes) : P2PMessage - { - public override int PacketType { get; } = WitMessageCode.BlockWitnessHashes; - - public override string Protocol { get; } = "wit"; - - public long RequestId { get; } = requestId; - - public Hash256[] Hashes { get; } = hashes; - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/Messages/BlockWitnessHashesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/Messages/BlockWitnessHashesMessageSerializer.cs deleted file mode 100644 index 23f99bfa29d..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/Messages/BlockWitnessHashesMessageSerializer.cs +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Core.Crypto; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Wit.Messages -{ - public class BlockWitnessHashesMessageSerializer : IZeroInnerMessageSerializer - { - public void Serialize(IByteBuffer byteBuffer, BlockWitnessHashesMessage message) - { - NettyRlpStream nettyRlpStream = new(byteBuffer); - - int contentLength = GetLength(message, out int totalLength); - - byteBuffer.EnsureWritable(totalLength); - nettyRlpStream.StartSequence(contentLength); - nettyRlpStream.Encode(message.RequestId); - if (message.Hashes is null) - { - nettyRlpStream.EncodeNullObject(); - } - else - { - int hashesContentLength = message.Hashes?.Length * Rlp.LengthOfKeccakRlp ?? 0; - nettyRlpStream.StartSequence(hashesContentLength); - foreach (Hash256 keccak in message.Hashes) - { - nettyRlpStream.Encode(keccak); - } - } - } - - public int GetLength(BlockWitnessHashesMessage message, out int contentLength) - { - if (message.Hashes is null) - { - contentLength = Rlp.OfEmptySequence.Length; - } - else - { - int hashesContentLength = message.Hashes?.Length * Rlp.LengthOfKeccakRlp ?? 0; - contentLength = Rlp.LengthOfSequence(hashesContentLength) + Rlp.LengthOf(message.RequestId); - - } - return Rlp.LengthOfSequence(contentLength); - } - - public BlockWitnessHashesMessage Deserialize(IByteBuffer byteBuffer) - { - NettyRlpStream rlpStream = new(byteBuffer); - rlpStream.ReadSequenceLength(); - long requestId = rlpStream.DecodeLong(); - int sequenceLength = rlpStream.ReadSequenceLength(); - Hash256[] hashes = new Hash256[sequenceLength / Rlp.LengthOfKeccakRlp]; - for (int i = 0; i < hashes.Length; i++) - { - hashes[i] = rlpStream.DecodeKeccak(); - } - - return new BlockWitnessHashesMessage(requestId, hashes); - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/Messages/GetBlockWitnessHashesMessage.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/Messages/GetBlockWitnessHashesMessage.cs deleted file mode 100644 index 007ef042ba6..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/Messages/GetBlockWitnessHashesMessage.cs +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Crypto; -using Nethermind.Network.P2P.Messages; - -namespace Nethermind.Network.P2P.Subprotocols.Wit.Messages -{ - public class GetBlockWitnessHashesMessage : P2PMessage - { - public override int PacketType { get; } = WitMessageCode.GetBlockWitnessHashes; - public override string Protocol { get; } = "wit"; - - public long RequestId { get; set; } - public Hash256 BlockHash { get; set; } - - public GetBlockWitnessHashesMessage(long requestId, Hash256 blockHash) - { - RequestId = requestId; - BlockHash = blockHash; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/Messages/GetBlockWitnessHashesMessageSerializer.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/Messages/GetBlockWitnessHashesMessageSerializer.cs deleted file mode 100644 index 132c69efbfb..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/Messages/GetBlockWitnessHashesMessageSerializer.cs +++ /dev/null @@ -1,37 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using DotNetty.Buffers; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Network.P2P.Subprotocols.Wit.Messages -{ - public class GetBlockWitnessHashesMessageSerializer : IZeroInnerMessageSerializer - { - public void Serialize(IByteBuffer byteBuffer, GetBlockWitnessHashesMessage message) - { - NettyRlpStream nettyRlpStream = new(byteBuffer); - int totalLength = GetLength(message, out int contentLength); - byteBuffer.EnsureWritable(totalLength, true); - nettyRlpStream.StartSequence(contentLength); - nettyRlpStream.Encode(message.RequestId); - nettyRlpStream.Encode(message.BlockHash); - } - - public int GetLength(GetBlockWitnessHashesMessage message, out int contentLength) - { - contentLength = Rlp.LengthOf(message.RequestId) - + (message.BlockHash is null ? 1 : Rlp.LengthOfKeccakRlp); - return Rlp.LengthOfSequence(contentLength) + Rlp.LengthOf(message.RequestId) + Rlp.LengthOf(message.BlockHash); - } - - public GetBlockWitnessHashesMessage Deserialize(IByteBuffer byteBuffer) - { - NettyRlpStream rlpStream = new(byteBuffer); - rlpStream.ReadSequenceLength(); - long requestId = rlpStream.DecodeLong(); - var hash = rlpStream.DecodeKeccak(); - return new GetBlockWitnessHashesMessage(requestId, hash); - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/WitMessageCode.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/WitMessageCode.cs deleted file mode 100644 index 631ac4d53d5..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/WitMessageCode.cs +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -namespace Nethermind.Network.P2P.Subprotocols.Wit -{ - public static class WitMessageCode - { - /// - /// Not used, reserved. - /// - public const int Status = 0x00; - public const int GetBlockWitnessHashes = 0x01; - public const int BlockWitnessHashes = 0x02; - - public static string GetDescription(int code) - { - return code switch - { - GetBlockWitnessHashes => nameof(GetBlockWitnessHashes), - BlockWitnessHashes => nameof(BlockWitnessHashes), - _ => $"Unknown({code.ToString()})" - }; - } - } -} diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/WitProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/WitProtocolHandler.cs deleted file mode 100644 index 586b9044eb1..00000000000 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Wit/WitProtocolHandler.cs +++ /dev/null @@ -1,153 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using System.Threading; -using System.Threading.Tasks; -using Nethermind.Blockchain.Synchronization; -using Nethermind.Core.Crypto; -using Nethermind.Logging; -using Nethermind.Network.Contract.P2P; -using Nethermind.Network.P2P.EventArg; -using Nethermind.Network.P2P.ProtocolHandlers; -using Nethermind.Network.P2P.Subprotocols.Wit.Messages; -using Nethermind.Network.Rlpx; -using Nethermind.Stats; -using Nethermind.Stats.Model; -using Nethermind.Synchronization; - -namespace Nethermind.Network.P2P.Subprotocols.Wit -{ - public class WitProtocolHandler : ZeroProtocolHandlerBase, IWitnessPeer - { - private readonly ISyncServer _syncServer; - - private readonly MessageQueue _witnessRequests; - - public WitProtocolHandler(ISession session, - IMessageSerializationService serializer, - INodeStatsManager nodeStats, - ISyncServer syncServer, - ILogManager logManager) : base(session, nodeStats, serializer, logManager) - { - _syncServer = syncServer ?? throw new ArgumentNullException(nameof(syncServer)); - _witnessRequests = new MessageQueue(Send); - } - - public override byte ProtocolVersion => 0; - - public override string ProtocolCode => Protocol.Wit; - - public override int MessageIdSpaceSize => 3; - - public override string Name => "wit0"; - - protected override TimeSpan InitTimeout => Timeouts.Eth; - - public override event EventHandler ProtocolInitialized; - - public override event EventHandler SubprotocolRequested - { - add { } - remove { } - } - - public override void Init() - { - ProtocolInitialized?.Invoke(this, new ProtocolInitializedEventArgs(this)); - // GetBlockWitnessHashes(Keccak.Zero, CancellationToken.None); - } - - public override void HandleMessage(ZeroPacket message) - { - int size = message.Content.ReadableBytes; - int packetType = message.PacketType; - switch (packetType) - { - case WitMessageCode.GetBlockWitnessHashes: - { - using GetBlockWitnessHashesMessage requestMsg = Deserialize(message.Content); - ReportIn(requestMsg, size); - Handle(requestMsg); - break; - } - case WitMessageCode.BlockWitnessHashes: - { - BlockWitnessHashesMessage responseMsg = Deserialize(message.Content); - ReportIn(responseMsg, size); - Handle(responseMsg, size); - break; - } - } - } - - private void Handle(GetBlockWitnessHashesMessage requestMsg) - { - Hash256[] hashes = _syncServer.GetBlockWitnessHashes(requestMsg.BlockHash); - BlockWitnessHashesMessage msg = new(requestMsg.RequestId, hashes); - Send(msg); - } - - private void Handle(BlockWitnessHashesMessage responseMsg, long size) - { - _witnessRequests.Handle(responseMsg.Hashes, size); - } - - private static long _requestId; - - public async Task GetBlockWitnessHashes(Hash256 blockHash, CancellationToken token) - { - long requestId = Interlocked.Increment(ref _requestId); - GetBlockWitnessHashesMessage msg = new(requestId, blockHash); - - if (Logger.IsTrace) Logger.Trace( - $"{Counter:D5} {nameof(WitMessageCode.GetBlockWitnessHashes)} to {Session}"); - Hash256[] witnessHashes = await SendRequest(msg, token); - return witnessHashes; - } - - private async Task SendRequest(GetBlockWitnessHashesMessage message, CancellationToken token) - { - if (Logger.IsTrace) - { - Logger.Trace($"Sending block witness hashes request: {message.BlockHash}"); - } - - Request request = new(message); - _witnessRequests.Send(request); - - Task task = request.CompletionSource.Task; - using CancellationTokenSource delayCancellation = new(); - using CancellationTokenSource compositeCancellation = CancellationTokenSource.CreateLinkedTokenSource(token, delayCancellation.Token); - Task firstTask = await Task.WhenAny(task, Task.Delay(Timeouts.Eth, compositeCancellation.Token)); - if (firstTask.IsCanceled) - { - token.ThrowIfCancellationRequested(); - } - - if (firstTask == task) - { - delayCancellation.Cancel(); - return task.Result; - } - - throw new TimeoutException($"{Session} Request timeout in {nameof(GetBlockWitnessHashes)} for {message.BlockHash}"); - } - - #region Cleanup - - private int _isDisposed; - - public override void DisconnectProtocol(DisconnectReason disconnectReason, string details) - { - Dispose(); - } - - public override void Dispose() - { - if (Interlocked.Exchange(ref _isDisposed, 1) == 0) { } - } - - #endregion - } -} diff --git a/src/Nethermind/Nethermind.Network/ProtocolValidator.cs b/src/Nethermind/Nethermind.Network/ProtocolValidator.cs index 782347ba20c..1b26c75d553 100644 --- a/src/Nethermind/Nethermind.Network/ProtocolValidator.cs +++ b/src/Nethermind/Nethermind.Network/ProtocolValidator.cs @@ -35,7 +35,7 @@ public bool DisconnectOnInvalid(string protocol, ISession session, ProtocolIniti return protocol switch { Protocol.P2P => ValidateP2PProtocol(session, eventArgs), - Protocol.Eth or Protocol.Les => ValidateEthProtocol(session, eventArgs), + Protocol.Eth => ValidateEthProtocol(session, eventArgs), _ => true, }; } diff --git a/src/Nethermind/Nethermind.Network/ProtocolsManager.cs b/src/Nethermind/Nethermind.Network/ProtocolsManager.cs index bd2e15e2def..236acec43ed 100644 --- a/src/Nethermind/Nethermind.Network/ProtocolsManager.cs +++ b/src/Nethermind/Nethermind.Network/ProtocolsManager.cs @@ -20,10 +20,8 @@ using Nethermind.Network.P2P.Subprotocols.Eth.V66; using Nethermind.Network.P2P.Subprotocols.Eth.V67; using Nethermind.Network.P2P.Subprotocols.Eth.V68; -using Nethermind.Network.P2P.Subprotocols.Les; using Nethermind.Network.P2P.Subprotocols.NodeData; using Nethermind.Network.P2P.Subprotocols.Snap; -using Nethermind.Network.P2P.Subprotocols.Wit; using Nethermind.Network.Rlpx; using Nethermind.Stats; using Nethermind.Stats.Model; @@ -236,24 +234,6 @@ private IDictionary> GetProtocolFa }; InitSatelliteProtocol(session, handler); - return handler; - }, - [Protocol.Wit] = (session, version) => - { - var handler = version switch - { - 0 => new WitProtocolHandler(session, _serializer, _stats, _syncServer, _logManager), - _ => throw new NotSupportedException($"{Protocol.Wit}.{version} is not supported.") - }; - InitSatelliteProtocol(session, handler); - - return handler; - }, - [Protocol.Les] = (session, version) => - { - LesProtocolHandler handler = new(session, _serializer, _stats, _syncServer, _backgroundTaskScheduler, _logManager); - InitSyncPeerProtocol(session, handler); - return handler; } }; diff --git a/src/Nethermind/Nethermind.Network/Rlpx/FrameCipher.cs b/src/Nethermind/Nethermind.Network/Rlpx/FrameCipher.cs index 7d5822f8d75..10cb64da1a9 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/FrameCipher.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/FrameCipher.cs @@ -2,46 +2,41 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Diagnostics; - -using Nethermind.Crypto; - using Org.BouncyCastle.Crypto; -using Org.BouncyCastle.Crypto.Engines; using Org.BouncyCastle.Crypto.Modes; using Org.BouncyCastle.Crypto.Parameters; using Org.BouncyCastle.Security; -namespace Nethermind.Network.Rlpx +namespace Nethermind.Network.Rlpx; + +public class FrameCipher : IFrameCipher { - public class FrameCipher : IFrameCipher - { - private const int BlockSize = 16; - private const int KeySize = 32; + private const int BlockSize = 16; + private const int KeySize = 32; - private readonly IBufferedCipher _decryptionCipher; - private readonly IBufferedCipher _encryptionCipher; + private readonly IBufferedCipher _decryptionCipher; + private readonly IBufferedCipher _encryptionCipher; - public FrameCipher(byte[] aesKey) - { - IBlockCipher aes = AesEngineX86Intrinsic.IsSupported ? new AesEngineX86Intrinsic() : new AesEngine(); + public FrameCipher(byte[] aesKey) + { + IBlockCipher aes = AesUtilities.CreateEngine(); - Debug.Assert(aesKey.Length == KeySize, $"AES key expected to be {KeySize} bytes long"); + Debug.Assert(aesKey.Length == KeySize, $"AES key expected to be {KeySize} bytes long"); - _encryptionCipher = new BufferedBlockCipher(new SicBlockCipher(aes)); - _encryptionCipher.Init(true, new ParametersWithIV(ParameterUtilities.CreateKeyParameter("AES", aesKey), new byte[BlockSize])); + _encryptionCipher = new BufferedBlockCipher(new SicBlockCipher(aes)); + _encryptionCipher.Init(true, new ParametersWithIV(ParameterUtilities.CreateKeyParameter("AES", aesKey), new byte[BlockSize])); - _decryptionCipher = new BufferedBlockCipher(new SicBlockCipher(aes)); - _decryptionCipher.Init(false, new ParametersWithIV(ParameterUtilities.CreateKeyParameter("AES", aesKey), new byte[BlockSize])); - } + _decryptionCipher = new BufferedBlockCipher(new SicBlockCipher(aes)); + _decryptionCipher.Init(false, new ParametersWithIV(ParameterUtilities.CreateKeyParameter("AES", aesKey), new byte[BlockSize])); + } - public void Encrypt(byte[] input, int offset, int length, byte[] output, int outputOffset) - { - _encryptionCipher.ProcessBytes(input, offset, length, output, outputOffset); - } + public void Encrypt(byte[] input, int offset, int length, byte[] output, int outputOffset) + { + _encryptionCipher.ProcessBytes(input, offset, length, output, outputOffset); + } - public void Decrypt(byte[] input, int offset, int length, byte[] output, int outputOffset) - { - _decryptionCipher.ProcessBytes(input, offset, length, output, outputOffset); - } + public void Decrypt(byte[] input, int offset, int length, byte[] output, int outputOffset) + { + _decryptionCipher.ProcessBytes(input, offset, length, output, outputOffset); } } diff --git a/src/Nethermind/Nethermind.Network/Rlpx/FrameMacProcessor.cs b/src/Nethermind/Nethermind.Network/Rlpx/FrameMacProcessor.cs index 990637126e2..f4d9ef42f57 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/FrameMacProcessor.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/FrameMacProcessor.cs @@ -4,219 +4,215 @@ using System; using System.IO; using Nethermind.Core.Crypto; -using Nethermind.Crypto; - using Org.BouncyCastle.Crypto; -using Org.BouncyCastle.Crypto.Engines; using Org.BouncyCastle.Crypto.Parameters; -namespace Nethermind.Network.Rlpx +namespace Nethermind.Network.Rlpx; + +/// +/// partially adapted from ethereumJ +/// +public sealed class FrameMacProcessor : IFrameMacProcessor { - /// - /// partially adapted from ethereumJ - /// - public sealed class FrameMacProcessor : IFrameMacProcessor + private readonly PublicKey _remoteNodeId; + private readonly KeccakHash _egressMac; + private readonly KeccakHash _ingressMac; + private readonly KeccakHash _egressMacCopy; + private readonly KeccakHash _ingressMacCopy; + private readonly IBlockCipher _aesEngine; + private readonly byte[] _macSecret; + + public FrameMacProcessor(PublicKey remoteNodeId, EncryptionSecrets secrets) { - private readonly PublicKey _remoteNodeId; - private readonly KeccakHash _egressMac; - private readonly KeccakHash _ingressMac; - private readonly KeccakHash _egressMacCopy; - private readonly KeccakHash _ingressMacCopy; - private readonly IBlockCipher _aesEngine; - private readonly byte[] _macSecret; - - public FrameMacProcessor(PublicKey remoteNodeId, EncryptionSecrets secrets) - { - _remoteNodeId = remoteNodeId; - _macSecret = secrets.MacSecret; - _egressMac = secrets.EgressMac; - _egressMacCopy = _egressMac.Copy(); - _ingressMac = secrets.IngressMac; - _ingressMacCopy = _ingressMac.Copy(); - _aesEngine = MakeMacCipher(); - _checkMacBuffer = new byte[_ingressMac.HashSize]; - _addMacBuffer = new byte[_ingressMac.HashSize]; - _ingressAesBlockBuffer = new byte[_ingressMac.HashSize]; - _egressAesBlockBuffer = new byte[_ingressMac.HashSize]; - } + _remoteNodeId = remoteNodeId; + _macSecret = secrets.MacSecret; + _egressMac = secrets.EgressMac; + _egressMacCopy = _egressMac.Copy(); + _ingressMac = secrets.IngressMac; + _ingressMacCopy = _ingressMac.Copy(); + _aesEngine = MakeMacCipher(); + _checkMacBuffer = new byte[_ingressMac.HashSize]; + _addMacBuffer = new byte[_ingressMac.HashSize]; + _ingressAesBlockBuffer = new byte[_ingressMac.HashSize]; + _egressAesBlockBuffer = new byte[_ingressMac.HashSize]; + } - private IBlockCipher MakeMacCipher() + private IBlockCipher MakeMacCipher() + { + IBlockCipher aesFastEngine = AesUtilities.CreateEngine(); + aesFastEngine.Init(true, new KeyParameter(_macSecret)); + return aesFastEngine; + } + + public void AddMac(byte[] input, int offset, int length, bool isHeader) + { + if (isHeader) { - IBlockCipher aesFastEngine = AesEngineX86Intrinsic.IsSupported ? new AesEngineX86Intrinsic() : new AesEngine(); - aesFastEngine.Init(true, new KeyParameter(_macSecret)); - return aesFastEngine; + input.AsSpan(0, 32).CopyTo(_addMacBuffer); + UpdateMac(_egressMac, _egressMacCopy, _addMacBuffer, offset, input, offset + length, true); // TODO: confirm header is seed } - - public void AddMac(byte[] input, int offset, int length, bool isHeader) + else { - if (isHeader) - { - input.AsSpan(0, 32).CopyTo(_addMacBuffer); - UpdateMac(_egressMac, _egressMacCopy, _addMacBuffer, offset, input, offset + length, true); // TODO: confirm header is seed - } - else - { - _egressMac.Update(input.AsSpan(offset, length)); + _egressMac.Update(input.AsSpan(offset, length)); - // frame-mac: right128 of egress-mac.update(aes(mac-secret,egress-mac) ^ right128(egress-mac.update(frame-ciphertext).digest)) - DoFinalNoReset(_egressMac, _egressMacCopy, _addMacBuffer); // frame MAC seed - UpdateMac(_egressMac, _egressMacCopy, _addMacBuffer, 0, input, offset + length, true); - } + // frame-mac: right128 of egress-mac.update(aes(mac-secret,egress-mac) ^ right128(egress-mac.update(frame-ciphertext).digest)) + DoFinalNoReset(_egressMac, _egressMacCopy, _addMacBuffer); // frame MAC seed + UpdateMac(_egressMac, _egressMacCopy, _addMacBuffer, 0, input, offset + length, true); } + } + + public void UpdateEgressMac(byte[] input) + { + _egressMac.Update(input); + } - public void UpdateEgressMac(byte[] input) + public void UpdateIngressMac(byte[] input, bool isHeader) + { + if (isHeader) { - _egressMac.Update(input); + input.AsSpan().CopyTo(_checkMacBuffer.AsSpan(0, 16)); } - - public void UpdateIngressMac(byte[] input, bool isHeader) + else { - if (isHeader) - { - input.AsSpan().CopyTo(_checkMacBuffer.AsSpan(0, 16)); - } - else - { - _ingressMac.Update(input); - } + _ingressMac.Update(input); } + } + + public void CalculateMac(byte[] output) + { + DoFinalNoReset(_egressMac, _egressMacCopy, _addMacBuffer); // frame MAC seed + UpdateMac(_egressMac, _egressMacCopy, _addMacBuffer, 0, output, 0, true); + } - public void CalculateMac(byte[] output) + public void AddMac(byte[] input, int offset, int length, byte[] output, int outputOffset, bool isHeader) + { + if (isHeader) { - DoFinalNoReset(_egressMac, _egressMacCopy, _addMacBuffer); // frame MAC seed - UpdateMac(_egressMac, _egressMacCopy, _addMacBuffer, 0, output, 0, true); + input.AsSpan(0, 16).CopyTo(_addMacBuffer); + UpdateMac(_egressMac, _egressMacCopy, _addMacBuffer, offset, output, outputOffset, true); // TODO: confirm header is seed } - - public void AddMac(byte[] input, int offset, int length, byte[] output, int outputOffset, bool isHeader) + else { - if (isHeader) - { - input.AsSpan(0, 16).CopyTo(_addMacBuffer); - UpdateMac(_egressMac, _egressMacCopy, _addMacBuffer, offset, output, outputOffset, true); // TODO: confirm header is seed - } - else - { - _egressMac.Update(input.AsSpan(offset, length)); + _egressMac.Update(input.AsSpan(offset, length)); - // frame-mac: right128 of egress-mac.update(aes(mac-secret,egress-mac) ^ right128(egress-mac.update(frame-ciphertext).digest)) - DoFinalNoReset(_egressMac, _egressMacCopy, _addMacBuffer); // frame MAC seed - UpdateMac(_egressMac, _egressMacCopy, _addMacBuffer, 0, output, outputOffset, true); - } + // frame-mac: right128 of egress-mac.update(aes(mac-secret,egress-mac) ^ right128(egress-mac.update(frame-ciphertext).digest)) + DoFinalNoReset(_egressMac, _egressMacCopy, _addMacBuffer); // frame MAC seed + UpdateMac(_egressMac, _egressMacCopy, _addMacBuffer, 0, output, outputOffset, true); } + } - private readonly byte[] _addMacBuffer; - private readonly byte[] _checkMacBuffer; - private readonly byte[] _ingressAesBlockBuffer; - private readonly byte[] _egressAesBlockBuffer; + private readonly byte[] _addMacBuffer; + private readonly byte[] _checkMacBuffer; + private readonly byte[] _ingressAesBlockBuffer; + private readonly byte[] _egressAesBlockBuffer; - public bool CheckMac(byte[] mac, bool isHeader) + public bool CheckMac(byte[] mac, bool isHeader) + { + if (!isHeader) { - if (!isHeader) - { - DoFinalNoReset(_ingressMac, _ingressMacCopy, _checkMacBuffer); // frame MAC seed - } + DoFinalNoReset(_ingressMac, _ingressMacCopy, _checkMacBuffer); // frame MAC seed + } - byte[] aesBlock = _ingressAesBlockBuffer; - DoFinalNoReset(_ingressMac, _ingressMacCopy, aesBlock); + byte[] aesBlock = _ingressAesBlockBuffer; + DoFinalNoReset(_ingressMac, _ingressMacCopy, aesBlock); - _aesEngine.ProcessBlock(aesBlock, 0, aesBlock, 0); + _aesEngine.ProcessBlock(aesBlock, 0, aesBlock, 0); - // Note that although the mac digest size is 32 bytes, we only use 16 bytes in the computation - int length = 16; - for (int i = 0; i < length; i++) - { - aesBlock[i] ^= _checkMacBuffer[i]; - } + // Note that although the mac digest size is 32 bytes, we only use 16 bytes in the computation + int length = 16; + for (int i = 0; i < length; i++) + { + aesBlock[i] ^= _checkMacBuffer[i]; + } - _ingressMac.Update(aesBlock.AsSpan(0, length)); - byte[] result = _checkMacBuffer; - DoFinalNoReset(_ingressMac, _ingressMacCopy, result); + _ingressMac.Update(aesBlock.AsSpan(0, length)); + byte[] result = _checkMacBuffer; + DoFinalNoReset(_ingressMac, _ingressMacCopy, result); - bool isMacSame = true; - for (int i = 0; i < length; i++) + bool isMacSame = true; + for (int i = 0; i < length; i++) + { + if (mac[i] != result[i]) { - if (mac[i] != result[i]) - { - isMacSame = false; - break; - } + isMacSame = false; + break; } - - return isMacSame; } - public void CheckMac(byte[] input, int offset, int length, bool isHeader) + return isMacSame; + } + + public void CheckMac(byte[] input, int offset, int length, bool isHeader) + { + if (isHeader) { - if (isHeader) - { - input.AsSpan(0, 32).CopyTo(_checkMacBuffer); - UpdateMac(_ingressMac, _ingressMacCopy, _checkMacBuffer, offset, input, offset + length, false); - } - else - { - _ingressMac.Update(input.AsSpan(offset, length)); + input.AsSpan(0, 32).CopyTo(_checkMacBuffer); + UpdateMac(_ingressMac, _ingressMacCopy, _checkMacBuffer, offset, input, offset + length, false); + } + else + { + _ingressMac.Update(input.AsSpan(offset, length)); - // frame-mac: right128 of egress-mac.update(aes(mac-secret,egress-mac) ^ right128(egress-mac.update(frame-ciphertext).digest)) - DoFinalNoReset(_ingressMac, _ingressMacCopy, _checkMacBuffer); // frame MAC seed - UpdateMac(_ingressMac, _ingressMacCopy, _checkMacBuffer, 0, input, offset + length, false); - } + // frame-mac: right128 of egress-mac.update(aes(mac-secret,egress-mac) ^ right128(egress-mac.update(frame-ciphertext).digest)) + DoFinalNoReset(_ingressMac, _ingressMacCopy, _checkMacBuffer); // frame MAC seed + UpdateMac(_ingressMac, _ingressMacCopy, _checkMacBuffer, 0, input, offset + length, false); } + } - /// - /// adapted from ethereumJ - /// - private void UpdateMac(KeccakHash mac, KeccakHash macCopy, byte[] seed, int offset, byte[] output, int outOffset, bool egress) - { - byte[] aesBlock = egress ? _egressAesBlockBuffer : _ingressAesBlockBuffer; - DoFinalNoReset(mac, macCopy, aesBlock); + /// + /// adapted from ethereumJ + /// + private void UpdateMac(KeccakHash mac, KeccakHash macCopy, byte[] seed, int offset, byte[] output, int outOffset, bool egress) + { + byte[] aesBlock = egress ? _egressAesBlockBuffer : _ingressAesBlockBuffer; + DoFinalNoReset(mac, macCopy, aesBlock); - _aesEngine.ProcessBlock(aesBlock, 0, aesBlock, 0); + _aesEngine.ProcessBlock(aesBlock, 0, aesBlock, 0); - // Note that although the mac digest size is 32 bytes, we only use 16 bytes in the computation - int length = 16; - for (int i = 0; i < length; i++) - { - aesBlock[i] ^= seed[i + offset]; - } + // Note that although the mac digest size is 32 bytes, we only use 16 bytes in the computation + int length = 16; + for (int i = 0; i < length; i++) + { + aesBlock[i] ^= seed[i + offset]; + } - mac.Update(aesBlock.AsSpan(0, length)); - byte[] result = seed; - DoFinalNoReset(mac, macCopy, result); + mac.Update(aesBlock.AsSpan(0, length)); + byte[] result = seed; + DoFinalNoReset(mac, macCopy, result); - if (egress) - { - Array.Copy(result, 0, output, outOffset, length); - } - else + if (egress) + { + Array.Copy(result, 0, output, outOffset, length); + } + else + { + bool isMacSame = true; + for (int i = 0; i < length; i++) { - bool isMacSame = true; - for (int i = 0; i < length; i++) + if (output[i + outOffset] != result[i]) { - if (output[i + outOffset] != result[i]) - { - isMacSame = false; - break; - } + isMacSame = false; + break; } + } - if (!isMacSame) - { - throw new IOException($"MAC mismatch from {_remoteNodeId}"); - } + if (!isMacSame) + { + throw new IOException($"MAC mismatch from {_remoteNodeId}"); } } + } - private static void DoFinalNoReset(KeccakHash mac, KeccakHash macCopy, byte[] output) - { - macCopy.ResetTo(mac); - macCopy.UpdateFinalTo(output); - } + private static void DoFinalNoReset(KeccakHash mac, KeccakHash macCopy, byte[] output) + { + macCopy.ResetTo(mac); + macCopy.UpdateFinalTo(output); + } - public void Dispose() - { - _egressMacCopy.Reset(); - _ingressMacCopy.Reset(); - } + public void Dispose() + { + _egressMacCopy.Reset(); + _ingressMacCopy.Reset(); } } diff --git a/src/Nethermind/Nethermind.Network/Rlpx/RlpxHost.cs b/src/Nethermind/Nethermind.Network/Rlpx/RlpxHost.cs index 7e9305057df..32bafe6001a 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/RlpxHost.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/RlpxHost.cs @@ -20,7 +20,6 @@ using Nethermind.Network.P2P.EventArg; using Nethermind.Network.Rlpx.Handshake; using Nethermind.Stats.Model; -using ILogger = Nethermind.Logging.InterfaceLogger; using LogLevel = DotNetty.Handlers.Logging.LogLevel; namespace Nethermind.Network.Rlpx @@ -105,8 +104,13 @@ public async Task Init() try { - _bossGroup = new MultithreadEventLoopGroup(); - _workerGroup = new MultithreadEventLoopGroup(); + // Default is LogicalCoreCount * 2 + // - so with two groups and 32 logical cores, we would have 128 threads + // Max at 8 threads per group for 16 threads total + // Min of 2 threads per group for 4 threads total + var threads = Math.Clamp(Environment.ProcessorCount / 2, min: 2, max: 8); + _bossGroup = new MultithreadEventLoopGroup(threads); + _workerGroup = new MultithreadEventLoopGroup(threads); ServerBootstrap bootstrap = new(); bootstrap diff --git a/src/Nethermind/Nethermind.Optimism.Test/GasCostTests.cs b/src/Nethermind/Nethermind.Optimism.Test/GasCostTests.cs new file mode 100644 index 00000000000..677bebec6b8 --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism.Test/GasCostTests.cs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections; +using Nethermind.Int256; +using NUnit.Framework; + +namespace Nethermind.Optimism.Test; + +public class GasCostTests +{ + [TestCaseSource(nameof(FjordL1CostCalculationTestCases))] + public UInt256 Fjord_l1cost_should_match(UInt256 fastLzSize, UInt256 l1BaseFee, UInt256 blobBaseFee, UInt256 l1BaseFeeScalar, UInt256 l1BlobBaseFeeScalar) => + OPL1CostHelper.ComputeL1CostFjord(fastLzSize, l1BaseFee, blobBaseFee, l1BaseFeeScalar, l1BlobBaseFeeScalar); + + public static IEnumerable FjordL1CostCalculationTestCases + { + get + { + static TestCaseData MakeTestCase(string testCase, ulong result, ulong fastLzSize, ulong l1BaseFee, ulong blobBaseFee, ulong l1BaseFeeScalar, ulong l1BlobBaseFeeScalar) + { + return new TestCaseData(new UInt256(fastLzSize), new UInt256(l1BaseFee), new UInt256(blobBaseFee), new UInt256(l1BaseFeeScalar), new UInt256(l1BlobBaseFeeScalar)) + { + ExpectedResult = new UInt256(result), + TestName = testCase + }; + } + + yield return MakeTestCase("Low compressed size", 3203000, 50, 1000000000, 10000000, 2, 3); + yield return MakeTestCase("Below minimal #1", 3203000, 150, 1000000000, 10000000, 2, 3); + yield return MakeTestCase("Below minimal #2", 3203000, 170, 1000000000, 10000000, 2, 3); + yield return MakeTestCase("Above minimal #1", 3217602, 171, 1000000000, 10000000, 2, 3); + yield return MakeTestCase("Above minimal #2", 3994602, 200, 1000000000, 10000000, 2, 3); + yield return MakeTestCase("Regular block #1", 2883950646753, 1044, 28549556977, 1, 7600, 862000); + } + } +} diff --git a/src/Nethermind/Nethermind.Optimism.Test/Nethermind.Optimism.Test.csproj b/src/Nethermind/Nethermind.Optimism.Test/Nethermind.Optimism.Test.csproj new file mode 100644 index 00000000000..b2d4b0a3d49 --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism.Test/Nethermind.Optimism.Test.csproj @@ -0,0 +1,27 @@ + + + + enable + $(NoWarn);NUnit1032 + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + diff --git a/src/Nethermind/Nethermind.Optimism.Test/ReceiptDecoderTests.cs b/src/Nethermind/Nethermind.Optimism.Test/ReceiptDecoderTests.cs new file mode 100644 index 00000000000..37e1a5e4df7 --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism.Test/ReceiptDecoderTests.cs @@ -0,0 +1,140 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections; +using Nethermind.Core.Extensions; +using Nethermind.Serialization.Rlp; +using NUnit.Framework; + +namespace Nethermind.Optimism.Test; + +public class ReceiptDecoderTests +{ + [TestCaseSource(nameof(DepositTxReceiptsSerializationTestCases))] + public void Test_tx_network_form_receipts_properly_encoded_for_trie(byte[] rlp, bool includesNonce, bool includesVersion, bool shouldIncludeNonceAndVersionForTxTrie) + { + static OptimismTxReceipt TestNetworkEncodingRoundTrip(byte[] rlp, bool includesNonce, bool includesVersion) + { + OptimismReceiptMessageDecoder decoder = new(); + OptimismTxReceipt decodedReceipt = decoder.Decode(new RlpStream(rlp), RlpBehaviors.SkipTypedWrapping); + + RlpStream encodedRlp = new(decoder.GetLength(decodedReceipt, RlpBehaviors.SkipTypedWrapping)); + decoder.Encode(encodedRlp, decodedReceipt, RlpBehaviors.SkipTypedWrapping); + + Assert.Multiple(() => + { + Assert.That(decodedReceipt.DepositNonce, includesNonce ? Is.Not.Null : Is.Null); + Assert.That(decodedReceipt.DepositReceiptVersion, includesVersion ? Is.Not.Null : Is.Null); + Assert.That(rlp, Is.EqualTo(encodedRlp.Data.ToArray())); + }); + + return decodedReceipt; + } + + static OptimismTxReceipt TestStorageEncodingRoundTrip(OptimismTxReceipt decodedReceipt, bool includesNonce, bool includesVersion) + { + OptimismCompactReceiptStorageDecoder decoder = new(); + + RlpStream encodedRlp = new(decoder.GetLength(decodedReceipt, RlpBehaviors.SkipTypedWrapping)); + decoder.Encode(encodedRlp, decodedReceipt, RlpBehaviors.SkipTypedWrapping); + encodedRlp.Position = 0; + + OptimismTxReceipt decodedStorageReceipt = decoder.Decode(encodedRlp, RlpBehaviors.SkipTypedWrapping); + + Assert.Multiple(() => + { + Assert.That(decodedStorageReceipt.DepositNonce, includesNonce ? Is.Not.Null : Is.Null); + Assert.That(decodedStorageReceipt.DepositReceiptVersion, includesVersion ? Is.Not.Null : Is.Null); + }); + + Rlp.ValueDecoderContext valueDecoderCtx = new(encodedRlp.Data); + decodedStorageReceipt = decoder.Decode(ref valueDecoderCtx, RlpBehaviors.SkipTypedWrapping); + + Assert.Multiple(() => + { + Assert.That(decodedStorageReceipt.DepositNonce, includesNonce ? Is.Not.Null : Is.Null); + Assert.That(decodedStorageReceipt.DepositReceiptVersion, includesVersion ? Is.Not.Null : Is.Null); + }); + + return decodedReceipt; + } + + static void TestTrieEncoding(OptimismTxReceipt decodedReceipt, bool shouldIncludeNonceAndVersionForTxTrie) + { + OptimismReceiptTrieDecoder trieDecoder = new(); + RlpStream encodedTrieRlp = new(trieDecoder.GetLength(decodedReceipt, RlpBehaviors.SkipTypedWrapping)); + + trieDecoder.Encode(encodedTrieRlp, decodedReceipt, RlpBehaviors.SkipTypedWrapping); + encodedTrieRlp.Position = 0; + + OptimismTxReceipt decodedTrieReceipt = trieDecoder.Decode(encodedTrieRlp, RlpBehaviors.SkipTypedWrapping); + + Assert.Multiple(() => + { + Assert.That(decodedTrieReceipt.DepositNonce, shouldIncludeNonceAndVersionForTxTrie ? Is.Not.Null : Is.Null); + Assert.That(decodedTrieReceipt.DepositReceiptVersion, shouldIncludeNonceAndVersionForTxTrie ? Is.Not.Null : Is.Null); + }); + } + + OptimismTxReceipt decodedReceipt = TestNetworkEncodingRoundTrip(rlp, includesNonce, includesVersion); + TestStorageEncodingRoundTrip(decodedReceipt, includesNonce, includesVersion); + TestTrieEncoding(decodedReceipt, shouldIncludeNonceAndVersionForTxTrie); + } + + + public static IEnumerable DepositTxReceiptsSerializationTestCases + { + get + { + yield return new TestCaseData( + Bytes.FromHexString("7ef901090182f9f5b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c080"), + true, + false, + false + ) + { + TestName = "1st OP Sepolia block receipt" + }; + + yield return new TestCaseData( + Bytes.FromHexString("0x7ef9010c0182b729b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c0830154f4"), + true, + false, + false + ) + { + TestName = "Regolith receipt" + }; + + yield return new TestCaseData( + Bytes.FromHexString("0x7ef903660183023676b9010000000000000000000000000000000000000000000000000000000000001000000000000000000080000000000000000001000000000000000000000000000204000200000004000000000000000000000000000000000000000000000000000100000000020000400000000000020800000000000000000000000008000000000000000000004000400000020000000001800000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000100000000000002100000000000000000000020001000100000040000000000000000000000000000000000000000000008000000f9025af9011d944200000000000000000000000000000000000010f884a0b0444523268717a02698be47d0803aa7468c00acbed2f8bd93a0459cde61dd89a00000000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000deaddeaddeaddeaddeaddeaddeaddeaddead0000a000000000000000000000000072fb15f502af58765015972a85f2c58551ef3fa1b88000000000000000000000000072fb15f502af58765015972a85f2c58551ef3fa1000000000000000000000000000000000000000000000000016345785d8a000000000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000000000f8dc944200000000000000000000000000000000000010f863a031b2166ff604fc5672ea5df08a78081d2bc6d746cadce880747f3643d819e83da000000000000000000000000072fb15f502af58765015972a85f2c58551ef3fa1a000000000000000000000000072fb15f502af58765015972a85f2c58551ef3fa1b860000000000000000000000000000000000000000000000000016345785d8a000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000000f85a944200000000000000000000000000000000000007f842a04641df4a962071e12719d8c8c8e5ac7fc4d97b927346a3d7a335b1f7517e133ca0c056e47e441542720e5a953ab9fcf1cc3de86fb1d3078293fb9708e6e77816938080"), + true, + false, + false + ) + { + TestName = "Regolith receipt 2" + }; + + yield return new TestCaseData( + Bytes.FromHexString("0xf901090183011711b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c0"), + false, + false, + false + ) + { + TestName = "Regolith receipt of a regular tx" + }; + + yield return new TestCaseData( + Bytes.FromHexString("7ef9010d0182ab7bb9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c083b2557501"), + true, + true, + true + ) + { + TestName = "Canyon receipt" + }; + } + } +} diff --git a/src/Nethermind/Nethermind.Optimism/Create2DeployerContractRewriter.cs b/src/Nethermind/Nethermind.Optimism/Create2DeployerContractRewriter.cs index 3510481d6a2..fc7a1714d5a 100644 --- a/src/Nethermind/Nethermind.Optimism/Create2DeployerContractRewriter.cs +++ b/src/Nethermind/Nethermind.Optimism/Create2DeployerContractRewriter.cs @@ -11,13 +11,13 @@ namespace Nethermind.Optimism; public class Create2DeployerContractRewriter { - private readonly IOPConfigHelper _opConfigHelper; + private readonly IOptimismSpecHelper _opSpecHelper; private readonly ISpecProvider _specProvider; private readonly IBlockTree _blockTree; - public Create2DeployerContractRewriter(IOPConfigHelper opConfigHelper, ISpecProvider specProvider, IBlockTree blockTree) + public Create2DeployerContractRewriter(IOptimismSpecHelper opSpecHelper, ISpecProvider specProvider, IBlockTree blockTree) { - _opConfigHelper = opConfigHelper; + _opSpecHelper = opSpecHelper; _specProvider = specProvider; _blockTree = blockTree; } @@ -26,9 +26,9 @@ public void RewriteContract(BlockHeader header, IWorldState worldState) { IReleaseSpec spec = _specProvider.GetSpec(header); BlockHeader? parent = _blockTree.FindParent(header, BlockTreeLookupOptions.None)?.Header; - if ((parent is null || !_opConfigHelper.IsCanyon(parent)) && _opConfigHelper.IsCanyon(header)) + if ((parent is null || !_opSpecHelper.IsCanyon(parent)) && _opSpecHelper.IsCanyon(header)) { - worldState.InsertCode(_opConfigHelper.Create2DeployerAddress!, _opConfigHelper.Create2DeployerCode, spec); + worldState.InsertCode(_opSpecHelper.Create2DeployerAddress!, _opSpecHelper.Create2DeployerCode, spec); } } } diff --git a/src/Nethermind/Nethermind.Optimism/DepositTxExtensions.cs b/src/Nethermind/Nethermind.Optimism/DepositTxExtensions.cs index 05ba056f307..5c908bb730d 100644 --- a/src/Nethermind/Nethermind.Optimism/DepositTxExtensions.cs +++ b/src/Nethermind/Nethermind.Optimism/DepositTxExtensions.cs @@ -2,14 +2,11 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Core; -using Nethermind.Int256; -using Nethermind.State; namespace Nethermind.Optimism; public static class DepositTxExtensions { - public static bool IsDeposit(this Transaction tx) { return tx.Type == TxType.DepositTx; diff --git a/src/Nethermind/Nethermind.Optimism/IOptimismConfig.cs b/src/Nethermind/Nethermind.Optimism/IOptimismConfig.cs new file mode 100644 index 00000000000..014780a369b --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/IOptimismConfig.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Config; + +namespace Nethermind.Optimism; + +public interface IOptimismConfig : IConfig +{ + [ConfigItem(Description = "Sequencer address", DefaultValue = "null")] + string? SequencerUrl { get; set; } +} diff --git a/src/Nethermind/Nethermind.Optimism/IOPConfigHelper.cs b/src/Nethermind/Nethermind.Optimism/IOptimismSpecHelper.cs similarity index 77% rename from src/Nethermind/Nethermind.Optimism/IOPConfigHelper.cs rename to src/Nethermind/Nethermind.Optimism/IOptimismSpecHelper.cs index e59e591d4b4..5874e01fa86 100644 --- a/src/Nethermind/Nethermind.Optimism/IOPConfigHelper.cs +++ b/src/Nethermind/Nethermind.Optimism/IOptimismSpecHelper.cs @@ -5,13 +5,15 @@ namespace Nethermind.Optimism; -public interface IOPConfigHelper +public interface IOptimismSpecHelper { Address L1FeeReceiver { get; } bool IsBedrock(BlockHeader header); bool IsRegolith(BlockHeader header); bool IsCanyon(BlockHeader header); + bool IsEcotone(BlockHeader header); + bool IsFjord(BlockHeader header); Address? Create2DeployerAddress { get; } byte[]? Create2DeployerCode { get; } } diff --git a/src/Nethermind/Nethermind.Optimism/InitializeBlockProducerOptimism.cs b/src/Nethermind/Nethermind.Optimism/InitializeBlockProducerOptimism.cs index 3de030e80ce..06111a6445e 100644 --- a/src/Nethermind/Nethermind.Optimism/InitializeBlockProducerOptimism.cs +++ b/src/Nethermind/Nethermind.Optimism/InitializeBlockProducerOptimism.cs @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.Threading.Tasks; using Nethermind.Api; using Nethermind.Config; using Nethermind.Consensus; @@ -19,7 +18,7 @@ public InitializeBlockProducerOptimism(OptimismNethermindApi api) : base(api) _api = api; } - protected override Task BuildProducer() + protected override IBlockProducer BuildProducer() { if (_api.DbProvider is null) throw new StepDependencyException(nameof(_api.DbProvider)); if (_api.BlockTree is null) throw new StepDependencyException(nameof(_api.BlockTree)); @@ -35,7 +34,6 @@ protected override Task BuildProducer() _api.BlockProducerEnvFactory = new OptimismBlockProducerEnvFactory( _api.WorldStateManager, - _api.ChainSpec, _api.BlockTree, _api.SpecProvider, _api.BlockValidator, @@ -57,7 +55,6 @@ protected override Task BuildProducer() producerEnv.TxSource, producerEnv.ChainProcessor, producerEnv.BlockTree, - _api.ManualBlockProductionTrigger, producerEnv.ReadOnlyStateProvider, _api.GasLimitCalculator, NullSealEngine.Instance, @@ -66,6 +63,6 @@ protected override Task BuildProducer() _api.LogManager, _api.Config()); - return Task.FromResult(_api.BlockProducer); + return _api.BlockProducer; } } diff --git a/src/Nethermind/Nethermind.Optimism/InitializeBlockchainOptimism.cs b/src/Nethermind/Nethermind.Optimism/InitializeBlockchainOptimism.cs index 20e79c79e11..26e15ffeee6 100644 --- a/src/Nethermind/Nethermind.Optimism/InitializeBlockchainOptimism.cs +++ b/src/Nethermind/Nethermind.Optimism/InitializeBlockchainOptimism.cs @@ -6,9 +6,11 @@ using Nethermind.Blockchain.Blocks; using Nethermind.Blockchain.Services; using Nethermind.Config; +using Nethermind.Consensus.AuRa.Withdrawals; using Nethermind.Consensus.Processing; using Nethermind.Consensus.Producers; using Nethermind.Consensus.Validators; +using Nethermind.Consensus.Withdrawals; using Nethermind.Evm; using Nethermind.Evm.TransactionProcessing; using Nethermind.Init.Steps; @@ -40,8 +42,10 @@ protected override ITransactionProcessor CreateTransactionProcessor() if (_api.SpecProvider is null) throw new StepDependencyException(nameof(_api.SpecProvider)); if (_api.SpecHelper is null) throw new StepDependencyException(nameof(_api.SpecHelper)); if (_api.L1CostHelper is null) throw new StepDependencyException(nameof(_api.L1CostHelper)); + if (_api.WorldState is null) throw new StepDependencyException(nameof(_api.WorldState)); - VirtualMachine virtualMachine = CreateVirtualMachine(); + CodeInfoRepository codeInfoRepository = new(); + VirtualMachine virtualMachine = CreateVirtualMachine(codeInfoRepository); return new OptimismTransactionProcessor( _api.SpecProvider, @@ -49,7 +53,8 @@ protected override ITransactionProcessor CreateTransactionProcessor() virtualMachine, _api.LogManager, _api.L1CostHelper, - _api.SpecHelper + _api.SpecHelper, + codeInfoRepository ); } @@ -102,12 +107,12 @@ protected override BlockProcessor CreateBlockProcessor() new BlockProcessor.BlockValidationTransactionsExecutor(_api.TransactionProcessor, _api.WorldState), _api.WorldState, _api.ReceiptStorage, - _api.WitnessCollector, new BlockhashStore(_api.BlockTree, _api.SpecProvider, _api.WorldState), _api.LogManager, _api.SpecHelper, _api.TransactionProcessor, - contractRewriter); + contractRewriter, + new BlockProductionWithdrawalProcessor(new NullWithdrawalProcessor())); } protected override IUnclesValidator CreateUnclesValidator() => Always.Valid; diff --git a/src/Nethermind/Nethermind.Optimism/L1BlockGasInfo.cs b/src/Nethermind/Nethermind.Optimism/L1BlockGasInfo.cs new file mode 100644 index 00000000000..e27a2f6efaf --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/L1BlockGasInfo.cs @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Int256; +using System; +using System.Linq; + +namespace Nethermind.Optimism; + +public readonly struct L1TxGasInfo(UInt256? l1Fee, UInt256? l1GasPrice, UInt256? l1GasUsed, string? l1FeeScalar) +{ + public UInt256? L1Fee { get; } = l1Fee; + public UInt256? L1GasPrice { get; } = l1GasPrice; + public UInt256? L1GasUsed { get; } = l1GasUsed; + public string? L1FeeScalar { get; } = l1FeeScalar; +} + +public readonly struct L1BlockGasInfo +{ + private readonly UInt256? _l1GasPrice; + private readonly UInt256 _l1BlobBaseFee; + private readonly UInt256 _l1BaseFeeScalar; + private readonly UInt256 _l1BlobBaseFeeScalar; + private readonly UInt256 _l1BaseFee; + private readonly UInt256 _overhead; + private readonly UInt256 _feeScalar; + private readonly string? _feeScalarDecimal; + private readonly bool _isFjord; + private readonly bool _isEcotone; + private readonly bool _isRegolith; + + private static readonly byte[] BedrockL1AttributesSelector = [0x01, 0x5d, 0x8e, 0xb9]; + private readonly IOptimismSpecHelper _specHelper; + + public L1BlockGasInfo(Block block, IOptimismSpecHelper specHelper) + { + _specHelper = specHelper; + + if (block is not null && block.Transactions.Length > 0) + { + Transaction depositTx = block.Transactions[0]; + if (depositTx.Data is null || depositTx.Data.Value.Length < 4) + { + return; + } + + Memory data = depositTx.Data.Value; + + _isFjord = _specHelper.IsFjord(block.Header); + + if (_isFjord || (_isEcotone = (_specHelper.IsEcotone(block.Header) && !data[0..4].Span.SequenceEqual(BedrockL1AttributesSelector)))) + { + if (data.Length != 164) + { + return; + } + + _l1GasPrice = new(data[36..68].Span, true); + _l1BlobBaseFee = new(data[68..100].Span, true); + _l1BaseFeeScalar = new(data[4..8].Span, true); + _l1BlobBaseFeeScalar = new(data[8..12].Span, true); + } + else + { + _isRegolith = true; + if (data.Length < 4 + 32 * 8) + { + return; + } + + _l1GasPrice = new(data[(4 + 32 * 2)..(4 + 32 * 3)].Span, true); + _l1BaseFee = new(data[(4 + 32 * 2)..(4 + 32 * 3)].Span, true); + _overhead = new(data[(4 + 32 * 6)..(4 + 32 * 7)].Span, true); + _feeScalar = new UInt256(data[(4 + 32 * 7)..(4 + 32 * 8)].Span, true); + _feeScalarDecimal = (((ulong)_feeScalar) / 1_000_000m).ToString(); + } + } + } + + public readonly L1TxGasInfo GetTxGasInfo(Transaction tx) + { + UInt256? l1Fee = null; + UInt256? l1GasUsed = null; + + if (_l1GasPrice is not null) + { + if (_isFjord) + { + UInt256 fastLzSize = OPL1CostHelper.ComputeFlzCompressLen(tx); + l1Fee = OPL1CostHelper.ComputeL1CostFjord(fastLzSize, _l1GasPrice.Value, _l1BlobBaseFee, _l1BaseFeeScalar, _l1BlobBaseFeeScalar); + } + else if (_isEcotone) + { + l1GasUsed = OPL1CostHelper.ComputeDataGas(tx, _isRegolith); + l1Fee = OPL1CostHelper.ComputeL1CostEcotone(l1GasUsed.Value, _l1GasPrice.Value, _l1BlobBaseFee, _l1BaseFeeScalar, _l1BlobBaseFeeScalar); + } + else + { + l1GasUsed = OPL1CostHelper.ComputeDataGas(tx, _isRegolith) + _overhead; + l1Fee = OPL1CostHelper.ComputeL1CostPreEcotone(l1GasUsed.Value, _l1BaseFee, _feeScalar); + } + } + + return new L1TxGasInfo(l1Fee, _l1GasPrice, l1GasUsed, _feeScalarDecimal); + } +} diff --git a/src/Nethermind/Nethermind.Optimism/Nethermind.Optimism.csproj b/src/Nethermind/Nethermind.Optimism/Nethermind.Optimism.csproj index c39c5320974..53cf806f73d 100644 --- a/src/Nethermind/Nethermind.Optimism/Nethermind.Optimism.csproj +++ b/src/Nethermind/Nethermind.Optimism/Nethermind.Optimism.csproj @@ -2,6 +2,7 @@ enable + true diff --git a/src/Nethermind/Nethermind.Optimism/OPConfigHelper.cs b/src/Nethermind/Nethermind.Optimism/OPConfigHelper.cs index 3902d88b024..c404456c8aa 100644 --- a/src/Nethermind/Nethermind.Optimism/OPConfigHelper.cs +++ b/src/Nethermind/Nethermind.Optimism/OPConfigHelper.cs @@ -6,23 +6,15 @@ namespace Nethermind.Optimism; -public class OPSpecHelper : IOPConfigHelper +public class OptimismSpecHelper(OptimismParameters parameters) : IOptimismSpecHelper { - private readonly ulong _regolithTimestamp; - private readonly long _bedrockBlockNumber; - private readonly ulong? _canyonTimestamp; + private readonly long _bedrockBlockNumber = parameters.BedrockBlockNumber; + private readonly ulong _regolithTimestamp = parameters.RegolithTimestamp; + private readonly ulong? _canyonTimestamp = parameters.CanyonTimestamp; + private readonly ulong? _ecotoneTimestamp = parameters.EcotoneTimestamp; + private readonly ulong? _fjordTimestamp = parameters.FjordTimestamp; - public Address L1FeeReceiver { get; init; } - - public OPSpecHelper(OptimismParameters parameters) - { - _regolithTimestamp = parameters.RegolithTimestamp; - _bedrockBlockNumber = parameters.BedrockBlockNumber; - _canyonTimestamp = parameters.CanyonTimestamp; - L1FeeReceiver = parameters.L1FeeRecipient; - Create2DeployerCode = parameters.Create2DeployerCode; - Create2DeployerAddress = parameters.Create2DeployerAddress; - } + public Address L1FeeReceiver { get; init; } = parameters.L1FeeRecipient; public bool IsRegolith(BlockHeader header) { @@ -36,9 +28,19 @@ public bool IsBedrock(BlockHeader header) public bool IsCanyon(BlockHeader header) { - return header.Timestamp >= (_canyonTimestamp ?? long.MaxValue); + return header.Timestamp >= _canyonTimestamp; + } + + public bool IsEcotone(BlockHeader header) + { + return header.Timestamp >= _ecotoneTimestamp; + } + + public bool IsFjord(BlockHeader header) + { + return header.Timestamp >= _fjordTimestamp; } - public Address? Create2DeployerAddress { get; } - public byte[]? Create2DeployerCode { get; } + public Address? Create2DeployerAddress { get; } = parameters.Create2DeployerAddress; + public byte[]? Create2DeployerCode { get; } = parameters.Create2DeployerCode; } diff --git a/src/Nethermind/Nethermind.Optimism/OPL1CostHelper.cs b/src/Nethermind/Nethermind.Optimism/OPL1CostHelper.cs index bc6451fd25a..ef26b270c49 100644 --- a/src/Nethermind/Nethermind.Optimism/OPL1CostHelper.cs +++ b/src/Nethermind/Nethermind.Optimism/OPL1CostHelper.cs @@ -2,7 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers; using System.Linq; +using System.Runtime.CompilerServices; using Nethermind.Core; using Nethermind.Evm; using Nethermind.Int256; @@ -11,48 +13,241 @@ namespace Nethermind.Optimism; -public class OPL1CostHelper : IL1CostHelper +public class OPL1CostHelper(IOptimismSpecHelper opSpecHelper, Address l1BlockAddr) : IL1CostHelper { - private readonly IOPConfigHelper _opConfigHelper; + private readonly IOptimismSpecHelper _opSpecHelper = opSpecHelper; - private readonly StorageCell _l1BaseFeeSlot; - private readonly StorageCell _overheadSlot; - private readonly StorageCell _scalarSlot; + private readonly StorageCell _l1BaseFeeSlot = new(l1BlockAddr, new UInt256(1)); + private readonly StorageCell _overheadSlot = new(l1BlockAddr, new UInt256(5)); + private readonly StorageCell _scalarSlot = new(l1BlockAddr, new UInt256(6)); - public OPL1CostHelper(IOPConfigHelper opConfigHelper, Address l1BlockAddr) - { - _opConfigHelper = opConfigHelper; + private static readonly UInt256 BasicDivisor = 1_000_000; - _l1BaseFeeSlot = new StorageCell(l1BlockAddr, new UInt256(1)); - _overheadSlot = new StorageCell(l1BlockAddr, new UInt256(5)); - _scalarSlot = new StorageCell(l1BlockAddr, new UInt256(6)); - } + // Ecotone + private readonly StorageCell _blobBaseFeeSlot = new(l1BlockAddr, new UInt256(7)); + private readonly StorageCell _baseFeeScalarSlot = new(l1BlockAddr, new UInt256(3)); + + private static readonly UInt256 PrecisionMultiplier = 16; + private static readonly UInt256 PrecisionDivisor = PrecisionMultiplier * BasicDivisor; + + + // Fjord + private static readonly UInt256 L1CostInterceptNeg = 42_585_600; + private static readonly UInt256 L1CostFastlzCoef = 836_500; + private static readonly UInt256 MinTransactionSizeScaled = 100 * 1_000_000; + private static readonly UInt256 FjordDivisor = 1_000_000_000_000; + + [SkipLocalsInit] public UInt256 ComputeL1Cost(Transaction tx, BlockHeader header, IWorldState worldState) { if (tx.IsDeposit()) return UInt256.Zero; - long dataGas = ComputeDataGas(tx, header); - if (dataGas == 0) + UInt256 l1BaseFee = new(worldState.Get(_l1BaseFeeSlot), true); + + if (_opSpecHelper.IsFjord(header)) + { + UInt256 blobBaseFee = new(worldState.Get(_blobBaseFeeSlot), true); + + ReadOnlySpan scalarData = worldState.Get(_baseFeeScalarSlot); + + const int baseFeeFieldsStart = 16; + const int fieldSize = sizeof(uint); + + int l1BaseFeeScalarStart = scalarData.Length > baseFeeFieldsStart ? scalarData.Length - baseFeeFieldsStart : 0; + int l1BaseFeeScalarEnd = l1BaseFeeScalarStart + (scalarData.Length >= baseFeeFieldsStart ? fieldSize : fieldSize - baseFeeFieldsStart + scalarData.Length); + UInt256 l1BaseFeeScalar = new(scalarData[l1BaseFeeScalarStart..l1BaseFeeScalarEnd], true); + UInt256 l1BlobBaseFeeScalar = new(scalarData[l1BaseFeeScalarEnd..(l1BaseFeeScalarEnd + fieldSize)], true); + + uint fastLzSize = ComputeFlzCompressLen(tx); + + return ComputeL1CostFjord(fastLzSize, l1BaseFee, blobBaseFee, l1BaseFeeScalar, l1BlobBaseFeeScalar); + } + + UInt256 dataGas = ComputeDataGas(tx, _opSpecHelper.IsRegolith(header)); + + if (dataGas.IsZero) return UInt256.Zero; - UInt256 l1BaseFee = new(worldState.Get(_l1BaseFeeSlot), true); - UInt256 overhead = new(worldState.Get(_overheadSlot), true); - UInt256 scalar = new(worldState.Get(_scalarSlot), true); + if (_opSpecHelper.IsEcotone(header)) + { + UInt256 blobBaseFee = new(worldState.Get(_blobBaseFeeSlot), true); + + ReadOnlySpan scalarData = worldState.Get(_baseFeeScalarSlot); + + const int baseFeeFieldsStart = 16; + const int fieldSize = sizeof(uint); + + int l1BaseFeeScalarStart = scalarData.Length > baseFeeFieldsStart ? scalarData.Length - baseFeeFieldsStart : 0; + int l1BaseFeeScalarEnd = l1BaseFeeScalarStart + (scalarData.Length >= baseFeeFieldsStart ? fieldSize : fieldSize - baseFeeFieldsStart + scalarData.Length); + UInt256 l1BaseFeeScalar = new(scalarData[l1BaseFeeScalarStart..l1BaseFeeScalarEnd], true); + UInt256 l1BlobBaseFeeScalar = new(scalarData[l1BaseFeeScalarEnd..(l1BaseFeeScalarEnd + fieldSize)], true); - return ((UInt256)dataGas + overhead) * l1BaseFee * scalar / 1_000_000; + return ComputeL1CostEcotone(dataGas, l1BaseFee, blobBaseFee, l1BaseFeeScalar, l1BlobBaseFeeScalar); + } + else + { + UInt256 overhead = new(worldState.Get(_overheadSlot), true); + UInt256 feeScalar = new(worldState.Get(_scalarSlot), true); + + return ComputeL1CostPreEcotone(dataGas + overhead, l1BaseFee, feeScalar); + } } - private long ComputeDataGas(Transaction tx, BlockHeader header) + [SkipLocalsInit] + public static UInt256 ComputeDataGas(Transaction tx, bool isRegolith) { byte[] encoded = Rlp.Encode(tx, RlpBehaviors.SkipTypedWrapping).Bytes; long zeroCount = encoded.Count(b => b == 0); long nonZeroCount = encoded.Length - zeroCount; // Add pre-EIP-3529 overhead - nonZeroCount += _opConfigHelper.IsRegolith(header) ? 0 : OptimismConstants.PreRegolithNonZeroCountOverhead; + nonZeroCount += isRegolith ? 0 : OptimismConstants.PreRegolithNonZeroCountOverhead; + + return (ulong)(zeroCount * GasCostOf.TxDataZero + nonZeroCount * GasCostOf.TxDataNonZeroEip2028); + } + + // Fjord L1 formula: + // l1FeeScaled = baseFeeScalar * l1BaseFee * 16 + blobFeeScalar * l1BlobBaseFee + // estimatedSize = max(minTransactionSize, intercept + fastlzCoef * fastlzSize) + // l1Cost = estimatedSize * l1FeeScaled / 1e12 + public static UInt256 ComputeL1CostFjord(UInt256 fastLzSize, UInt256 l1BaseFee, UInt256 blobBaseFee, UInt256 l1BaseFeeScalar, UInt256 l1BlobBaseFeeScalar) + { + UInt256 l1FeeScaled = l1BaseFeeScalar * l1BaseFee * PrecisionMultiplier + l1BlobBaseFeeScalar * blobBaseFee; + UInt256 fastLzCost = L1CostFastlzCoef * fastLzSize; + + if (fastLzCost < L1CostInterceptNeg) + { + fastLzCost = 0; + } + else + { + fastLzCost -= L1CostInterceptNeg; + } + + var estimatedSize = UInt256.Max(MinTransactionSizeScaled, fastLzCost); + return estimatedSize * l1FeeScaled / FjordDivisor; + } + + // Ecotone formula: (dataGas) * (16 * l1BaseFee * l1BaseFeeScalar + l1BlobBaseFee*l1BlobBaseFeeScalar) / 16e6 + public static UInt256 ComputeL1CostEcotone(UInt256 dataGas, UInt256 l1BaseFee, UInt256 blobBaseFee, UInt256 l1BaseFeeScalar, UInt256 l1BlobBaseFeeScalar) + { + return dataGas * (PrecisionMultiplier * l1BaseFee * l1BaseFeeScalar + blobBaseFee * l1BlobBaseFeeScalar) / PrecisionDivisor; + } + + // Pre-Ecotone formula: (dataGas + overhead) * l1BaseFee * scalar / 1e6 + public static UInt256 ComputeL1CostPreEcotone(UInt256 dataGasWithOverhead, UInt256 l1BaseFee, UInt256 feeScalar) + { + return dataGasWithOverhead * l1BaseFee * feeScalar / BasicDivisor; + } + + // Based on: + // https://github.com/ethereum-optimism/op-geth/blob/7c2819836018bfe0ca07c4e4955754834ffad4e0/core/types/rollup_cost.go + // https://github.com/Vectorized/solady/blob/5315d937d79b335c668896d7533ac603adac5315/js/solady.js + [SkipLocalsInit] + public static uint ComputeFlzCompressLen(Transaction tx) + { + byte[] encoded = Rlp.Encode(tx, RlpBehaviors.SkipTypedWrapping).Bytes; + + [SkipLocalsInit] + static uint FlzCompressLen(byte[] data) + { + uint n = 0; + uint[] ht = ArrayPool.Shared.Rent(8192); + try + { + uint u24(uint i) => data[i] | ((uint)data[i + 1] << 8) | ((uint)data[i + 2] << 16); + uint cmp(uint p, uint q, uint e) + { + uint l = 0; + for (e -= q; l < e; l++) + { + if (data[p + (int)l] != data[q + (int)l]) + { + e = 0; + } + } + return l; + } + void literals(uint r) + { + n += 0x21 * (r / 0x20); + r %= 0x20; + if (r != 0) + { + n += r + 1; + } + } + void match(uint l) + { + l--; + n += 3 * (l / 262); + if (l % 262 >= 6) + { + n += 3; + } + else + { + n += 2; + } + } + uint hash(uint v) => ((2654435769 * v) >> 19) & 0x1fff; + uint setNextHash(uint ip) + { + ht[hash(u24(ip))] = ip; + return ip + 1; + } + uint a = 0; + uint ipLimit = (uint)data.Length - 13; + if (data.Length < 13) + { + ipLimit = 0; + } + for (uint ip = a + 2; ip < ipLimit;) + { + uint d; + uint r; + for (; ; ) + { + uint s = u24(ip); + uint h = hash(s); + r = ht[h]; + ht[h] = ip; + d = ip - r; + if (ip >= ipLimit) + { + break; + } + ip++; + if (d <= 0x1fff && s == u24(r)) + { + break; + } + } + if (ip >= ipLimit) + { + break; + } + ip--; + if (ip > a) + { + literals(ip - a); + } + uint l = cmp(r + 3, ip + 3, ipLimit + 9); + match(l); + ip = setNextHash(setNextHash(ip + l)); + a = ip; + } + literals((uint)data.Length - a); + return n; + } + finally + { + ArrayPool.Shared.Return(ht); + } + } - return zeroCount * GasCostOf.TxDataZero + nonZeroCount * GasCostOf.TxDataNonZeroEip2028; + return FlzCompressLen(encoded); } } diff --git a/src/Nethermind/Nethermind.Optimism/OptimismBlockProcessor.cs b/src/Nethermind/Nethermind.Optimism/OptimismBlockProcessor.cs index 05c3071ca4d..21100981178 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismBlockProcessor.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismBlockProcessor.cs @@ -2,9 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using Nethermind.Blockchain; using Nethermind.Blockchain.Blocks; -using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; using Nethermind.Consensus.Processing; using Nethermind.Consensus.Rewards; @@ -12,7 +10,6 @@ using Nethermind.Consensus.Withdrawals; using Nethermind.Core; using Nethermind.Core.Specs; -using Nethermind.Evm; using Nethermind.Evm.Tracing; using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; @@ -31,20 +28,18 @@ public OptimismBlockProcessor( IBlockProcessor.IBlockTransactionsExecutor? blockTransactionsExecutor, IWorldState? stateProvider, IReceiptStorage? receiptStorage, - IWitnessCollector? witnessCollector, IBlockhashStore? blockhashStore, ILogManager? logManager, - IOPConfigHelper opConfigHelper, + IOptimismSpecHelper opSpecHelper, ITransactionProcessor txProcessor, Create2DeployerContractRewriter contractRewriter, IWithdrawalProcessor? withdrawalProcessor = null) : base(specProvider, blockValidator, rewardCalculator, blockTransactionsExecutor, - stateProvider, receiptStorage, witnessCollector, blockhashStore, txProcessor, logManager, withdrawalProcessor, - receiptsRootCalculator: OptimismReceiptsRootCalculator.Instance) + stateProvider, receiptStorage, blockhashStore, txProcessor, logManager, withdrawalProcessor, receiptsRootCalculator: ReceiptsRootCalculator.Instance) { ArgumentNullException.ThrowIfNull(stateProvider); _contractRewriter = contractRewriter; - ReceiptsTracer = new OptimismBlockReceiptTracer(opConfigHelper, stateProvider); + ReceiptsTracer = new OptimismBlockReceiptTracer(opSpecHelper, stateProvider); } protected override TxReceipt[] ProcessBlock(Block block, IBlockTracer blockTracer, ProcessingOptions options) diff --git a/src/Nethermind/Nethermind.Optimism/OptimismBlockProducerEnvFactory.cs b/src/Nethermind/Nethermind.Optimism/OptimismBlockProducerEnvFactory.cs index 84c6a5afde9..7acc08e17ab 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismBlockProducerEnvFactory.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismBlockProducerEnvFactory.cs @@ -13,54 +13,42 @@ using Nethermind.Consensus.Validators; using Nethermind.Consensus.Withdrawals; using Nethermind.Core.Specs; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; -using Nethermind.Specs.ChainSpecStyle; using Nethermind.State; using Nethermind.TxPool; namespace Nethermind.Optimism; -public class OptimismBlockProducerEnvFactory : BlockProducerEnvFactory +public class OptimismBlockProducerEnvFactory( + IWorldStateManager worldStateManager, + IBlockTree blockTree, + ISpecProvider specProvider, + IBlockValidator blockValidator, + IRewardCalculatorSource rewardCalculatorSource, + IReceiptStorage receiptStorage, + IBlockPreprocessorStep blockPreprocessorStep, + ITxPool txPool, + ITransactionComparerProvider transactionComparerProvider, + IBlocksConfig blocksConfig, + OptimismSpecHelper specHelper, + OPL1CostHelper l1CostHelper, + ILogManager logManager) : BlockProducerEnvFactory( + worldStateManager, + blockTree, + specProvider, + blockValidator, + rewardCalculatorSource, + receiptStorage, + blockPreprocessorStep, + txPool, + transactionComparerProvider, + blocksConfig, + logManager) { - private readonly ChainSpec _chainSpec; - private readonly OPSpecHelper _specHelper; - private readonly OPL1CostHelper _l1CostHelper; - - public OptimismBlockProducerEnvFactory( - IWorldStateManager worldStateManager, - ChainSpec chainSpec, - IBlockTree blockTree, - ISpecProvider specProvider, - IBlockValidator blockValidator, - IRewardCalculatorSource rewardCalculatorSource, - IReceiptStorage receiptStorage, - IBlockPreprocessorStep blockPreprocessorStep, - ITxPool txPool, - ITransactionComparerProvider transactionComparerProvider, - IBlocksConfig blocksConfig, - OPSpecHelper specHelper, - OPL1CostHelper l1CostHelper, - ILogManager logManager) : base(worldStateManager, - blockTree, specProvider, blockValidator, - rewardCalculatorSource, receiptStorage, blockPreprocessorStep, - txPool, transactionComparerProvider, blocksConfig, logManager) - { - _specHelper = specHelper; - _l1CostHelper = l1CostHelper; - _chainSpec = chainSpec; - TransactionsExecutorFactory = new OptimismTransactionsExecutorFactory(specProvider, logManager); - } - protected override ReadOnlyTxProcessingEnv CreateReadonlyTxProcessingEnv(IWorldStateManager worldStateManager, - ReadOnlyBlockTree readOnlyBlockTree) - { - ReadOnlyTxProcessingEnv result = new(worldStateManager, - readOnlyBlockTree, _specProvider, _logManager); - result.TransactionProcessor = - new OptimismTransactionProcessor(_specProvider, result.StateProvider, result.Machine, _logManager, _l1CostHelper, _specHelper); - - return result; - } + ReadOnlyBlockTree readOnlyBlockTree) => + new OptimismReadOnlyTxProcessingEnv(worldStateManager, readOnlyBlockTree, _specProvider, _logManager, l1CostHelper, specHelper); protected override ITxSource CreateTxSourceForProducer(ITxSource? additionalTxSource, ReadOnlyTxProcessingEnv processingEnv, @@ -74,7 +62,7 @@ protected override ITxSource CreateTxSourceForProducer(ITxSource? additionalTxSo } protected override BlockProcessor CreateBlockProcessor( - ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv, + IReadOnlyTxProcessingScope readOnlyTxProcessingEnv, ISpecProvider specProvider, IBlockValidator blockValidator, IRewardCalculatorSource rewardCalculatorSource, @@ -86,14 +74,13 @@ protected override BlockProcessor CreateBlockProcessor( blockValidator, rewardCalculatorSource.Get(readOnlyTxProcessingEnv.TransactionProcessor), TransactionsExecutorFactory.Create(readOnlyTxProcessingEnv), - readOnlyTxProcessingEnv.StateProvider, + readOnlyTxProcessingEnv.WorldState, receiptStorage, - NullWitnessCollector.Instance, - new BlockhashStore(_blockTree, specProvider, readOnlyTxProcessingEnv.StateProvider), + new BlockhashStore(_blockTree, specProvider, readOnlyTxProcessingEnv.WorldState), logManager, - _specHelper, + specHelper, readOnlyTxProcessingEnv.TransactionProcessor, - new Create2DeployerContractRewriter(_specHelper, _specProvider, _blockTree), - new BlockProductionWithdrawalProcessor(new WithdrawalProcessor(readOnlyTxProcessingEnv.StateProvider, logManager))); + new Create2DeployerContractRewriter(specHelper, _specProvider, _blockTree), + new BlockProductionWithdrawalProcessor(new WithdrawalProcessor(readOnlyTxProcessingEnv.WorldState, logManager))); } } diff --git a/src/Nethermind/Nethermind.Optimism/OptimismBlockReceiptTracer.cs b/src/Nethermind/Nethermind.Optimism/OptimismBlockReceiptTracer.cs index 7f43df08c32..aab105b9090 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismBlockReceiptTracer.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismBlockReceiptTracer.cs @@ -11,12 +11,12 @@ namespace Nethermind.Optimism; public class OptimismBlockReceiptTracer : BlockReceiptsTracer { - private readonly IOPConfigHelper _opConfigHelper; + private readonly IOptimismSpecHelper _opSpecHelper; private readonly IWorldState _worldState; - public OptimismBlockReceiptTracer(IOPConfigHelper opConfigHelper, IWorldState worldState) + public OptimismBlockReceiptTracer(IOptimismSpecHelper opSpecHelper, IWorldState worldState) { - _opConfigHelper = opConfigHelper; + _opSpecHelper = opSpecHelper; _worldState = worldState; } @@ -35,7 +35,7 @@ public OptimismBlockReceiptTracer(IOPConfigHelper opConfigHelper, IWorldState wo { depositNonce--; } - if (_opConfigHelper.IsCanyon(header)) + if (_opSpecHelper.IsCanyon(header)) { version = 1; } diff --git a/src/Nethermind/Nethermind.Optimism/OptimismConfig.cs b/src/Nethermind/Nethermind.Optimism/OptimismConfig.cs new file mode 100644 index 00000000000..0903acfb3ce --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/OptimismConfig.cs @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Optimism; + +public class OptimismConfig : IOptimismConfig +{ + public string? SequencerUrl { get; set; } = null; +} diff --git a/src/Nethermind/Nethermind.Optimism/OptimismNethermindApi.cs b/src/Nethermind/Nethermind.Optimism/OptimismNethermindApi.cs index 9f39d63b1ff..8e1e8f4002d 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismNethermindApi.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismNethermindApi.cs @@ -22,5 +22,5 @@ public OptimismNethermindApi( public IInvalidChainTracker? InvalidChainTracker { get; set; } public OPL1CostHelper? L1CostHelper { get; set; } - public OPSpecHelper? SpecHelper { get; set; } + public OptimismSpecHelper? SpecHelper { get; set; } } diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPayloadPreparationService.cs b/src/Nethermind/Nethermind.Optimism/OptimismPayloadPreparationService.cs index 71187bba01a..35fd2e3abb2 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPayloadPreparationService.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPayloadPreparationService.cs @@ -8,6 +8,7 @@ using Nethermind.Int256; using Nethermind.Logging; using Nethermind.Merge.Plugin.BlockProduction; +using Nethermind.Optimism.Rpc; namespace Nethermind.Optimism; diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPayloadTxSource.cs b/src/Nethermind/Nethermind.Optimism/OptimismPayloadTxSource.cs index 77f3c9ca35f..53b23ea2695 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPayloadTxSource.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPayloadTxSource.cs @@ -6,6 +6,7 @@ using Nethermind.Consensus.Producers; using Nethermind.Consensus.Transactions; using Nethermind.Core; +using Nethermind.Optimism.Rpc; namespace Nethermind.Optimism; diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs index cba1f2a1566..4a51af48b79 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPlugin.cs @@ -25,6 +25,8 @@ using Nethermind.HealthChecks; using Nethermind.Serialization.Json; using Nethermind.Specs.ChainSpecStyle; +using Nethermind.Serialization.Rlp; +using Nethermind.Optimism.Rpc; namespace Nethermind.Optimism; @@ -54,17 +56,16 @@ public class OptimismPlugin : IConsensusPlugin, ISynchronizationPlugin, IInitial public IBlockProductionTrigger DefaultBlockProductionTrigger => NeverProduceTrigger.Instance; - public Task InitBlockProducer(IBlockProductionTrigger? blockProductionTrigger = null, - ITxSource? additionalTxSource = null) + public IBlockProducer InitBlockProducer(ITxSource? additionalTxSource = null) { - if (blockProductionTrigger is not null || additionalTxSource is not null) + if (additionalTxSource is not null) throw new ArgumentException( - "Optimism does not support custom block production trigger or additional tx source"); + "Optimism does not support additional tx source"); ArgumentNullException.ThrowIfNull(_api); ArgumentNullException.ThrowIfNull(_api.BlockProducer); - return Task.FromResult(_api.BlockProducer); + return _api.BlockProducer; } #endregion @@ -73,6 +74,14 @@ public INethermindApi CreateApi(IConfigProvider configProvider, IJsonSerializer ILogManager logManager, ChainSpec chainSpec) => new OptimismNethermindApi(configProvider, jsonSerializer, logManager, chainSpec); + public void InitRlpDecoders(INethermindApi api) + { + if (ShouldRunSteps(api)) + { + Rlp.RegisterDecoders(typeof(OptimismReceiptMessageDecoder).Assembly, true); + } + } + public Task Init(INethermindApi api) { if (!ShouldRunSteps(api)) @@ -109,8 +118,6 @@ public Task Init(INethermindApi api) return Task.CompletedTask; } - public Task InitNetworkProtocol() => Task.CompletedTask; - public Task InitSynchronization() { if (_api is null || !ShouldRunSteps(_api)) @@ -200,7 +207,7 @@ public async Task InitRpcModules() await Task.Delay(5000); BlockImprovementContextFactory improvementContextFactory = new( - _api.ManualBlockProductionTrigger, + _api.BlockProducer, TimeSpan.FromSeconds(_blocksConfig.SecondsPerSlot)); OptimismPayloadPreparationService payloadPreparationService = new( @@ -266,6 +273,14 @@ public async Task InitRpcModules() if (_logger.IsInfo) _logger.Info("Optimism Engine Module has been enabled"); } + public IBlockProducerRunner CreateBlockProducerRunner() + { + return new StandardBlockProducerRunner( + DefaultBlockProductionTrigger, + _api!.BlockTree!, + _api.BlockProducer!); + } + public ValueTask DisposeAsync() => ValueTask.CompletedTask; public bool MustInitialize => true; diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPostMergeBlockProducer.cs b/src/Nethermind/Nethermind.Optimism/OptimismPostMergeBlockProducer.cs index 4b100aa48e3..93df8a49f9c 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPostMergeBlockProducer.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismPostMergeBlockProducer.cs @@ -13,6 +13,7 @@ using Nethermind.Core.Specs; using Nethermind.Logging; using Nethermind.Merge.Plugin.BlockProduction; +using Nethermind.Optimism.Rpc; using Nethermind.State; namespace Nethermind.Optimism; @@ -26,7 +27,6 @@ public OptimismPostMergeBlockProducer( ITxSource txPoolTxSource, IBlockchainProcessor processor, IBlockTree blockTree, - IBlockProductionTrigger blockProductionTrigger, IWorldState stateProvider, IGasLimitCalculator gasLimitCalculator, ISealEngine sealEngine, @@ -38,7 +38,6 @@ public OptimismPostMergeBlockProducer( payloadAttrsTxSource.Then(txPoolTxSource), processor, blockTree, - blockProductionTrigger, stateProvider, gasLimitCalculator, sealEngine, diff --git a/src/Nethermind/Nethermind.Optimism/OptimismReadOnlyTxProcessingEnv.cs b/src/Nethermind/Nethermind.Optimism/OptimismReadOnlyTxProcessingEnv.cs new file mode 100644 index 00000000000..499a0c9883f --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/OptimismReadOnlyTxProcessingEnv.cs @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Blockchain; +using Nethermind.Consensus.Processing; +using Nethermind.Core.Specs; +using Nethermind.Evm; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Logging; +using Nethermind.State; +using System; + +namespace Nethermind.Optimism; + +public class OptimismReadOnlyTxProcessingEnv( + IWorldStateManager worldStateManager, + IReadOnlyBlockTree readOnlyBlockTree, + ISpecProvider specProvider, + ILogManager logManager, + IL1CostHelper l1CostHelper, + IOptimismSpecHelper opSpecHelper, + IWorldState? worldStateToWarmUp = null) : ReadOnlyTxProcessingEnv( + worldStateManager, + readOnlyBlockTree, + specProvider, + logManager, + worldStateToWarmUp + ) +{ + protected override TransactionProcessor CreateTransactionProcessor() + { + ArgumentNullException.ThrowIfNull(LogManager); + + BlockhashProvider blockhashProvider = new(BlockTree, SpecProvider, StateProvider, LogManager); + VirtualMachine virtualMachine = new(blockhashProvider, SpecProvider, CodeInfoRepository, LogManager); + return new OptimismTransactionProcessor(SpecProvider, StateProvider, virtualMachine, LogManager, l1CostHelper, opSpecHelper, CodeInfoRepository); + } +} diff --git a/src/Nethermind/Nethermind.Optimism/OptimismReceiptDecoder.cs b/src/Nethermind/Nethermind.Optimism/OptimismReceiptMessageDecoder.cs similarity index 67% rename from src/Nethermind/Nethermind.Optimism/OptimismReceiptDecoder.cs rename to src/Nethermind/Nethermind.Optimism/OptimismReceiptMessageDecoder.cs index 03ade6f79ba..186fc2375d3 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismReceiptDecoder.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismReceiptMessageDecoder.cs @@ -8,9 +8,13 @@ namespace Nethermind.Optimism; -public class OptimismReceiptDecoder : IRlpStreamDecoder +[Rlp.Decoder(RlpDecoderKey.Trie)] +public class OptimismReceiptTrieDecoder() : OptimismReceiptMessageDecoder(true) { } + +[Rlp.Decoder] +public class OptimismReceiptMessageDecoder(bool isEncodedForTrie = false) : IRlpStreamDecoder, IRlpStreamDecoder { - public static readonly OptimismReceiptDecoder Instance = new(); + private readonly bool _isEncodedForTrie = isEncodedForTrie; public OptimismTxReceipt Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { @@ -21,7 +25,8 @@ public OptimismTxReceipt Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = txReceipt.TxType = (TxType)rlpStream.ReadByte(); } - _ = rlpStream.ReadSequenceLength(); + int lastCheck = rlpStream.ReadSequenceLength() + rlpStream.Position; + byte[] firstItem = rlpStream.DecodeByteArray(); if (firstItem.Length == 1 && (firstItem[0] == 0 || firstItem[0] == 1)) { @@ -41,9 +46,9 @@ public OptimismTxReceipt Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = txReceipt.Bloom = rlpStream.DecodeBloom(); - int lastCheck = rlpStream.ReadSequenceLength() + rlpStream.Position; + int logEntriesCheck = rlpStream.ReadSequenceLength() + rlpStream.Position; - int numberOfReceipts = rlpStream.PeekNumberOfItemsRemaining(lastCheck); + int numberOfReceipts = rlpStream.PeekNumberOfItemsRemaining(logEntriesCheck); LogEntry[] entries = new LogEntry[numberOfReceipts]; for (int i = 0; i < numberOfReceipts; i++) { @@ -51,16 +56,23 @@ public OptimismTxReceipt Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = } txReceipt.Logs = entries; - if (txReceipt.TxType == TxType.DepositTx && lastCheck > rlpStream.Position) + if (lastCheck > rlpStream.Position) { - txReceipt.DepositNonce = rlpStream.DecodeUlong(); - txReceipt.DepositReceiptVersion = rlpStream.DecodeUlong(); + if (txReceipt.TxType == TxType.DepositTx && lastCheck > rlpStream.Position) + { + txReceipt.DepositNonce = rlpStream.DecodeUlong(); + + if (lastCheck > rlpStream.Position) + { + txReceipt.DepositReceiptVersion = rlpStream.DecodeUlong(); + } + } } return txReceipt; } - private static (int Total, int Logs) GetContentLength(OptimismTxReceipt item, RlpBehaviors rlpBehaviors) + private (int Total, int Logs) GetContentLength(OptimismTxReceipt item, RlpBehaviors rlpBehaviors) { if (item is null) { @@ -83,10 +95,15 @@ private static (int Total, int Logs) GetContentLength(OptimismTxReceipt item, Rl : Rlp.LengthOf(item.PostTransactionState); } - if (item.TxType == TxType.DepositTx && item.DepositReceiptVersion is not null) + if (item.TxType == TxType.DepositTx && item.DepositNonce is not null && + (item.DepositReceiptVersion is not null || !_isEncodedForTrie)) { - contentLength += Rlp.LengthOf(item.DepositNonce ?? 0); - contentLength += Rlp.LengthOf(item.DepositReceiptVersion.Value); + contentLength += Rlp.LengthOf(item.DepositNonce); + + if (item.DepositReceiptVersion is not null) + { + contentLength += Rlp.LengthOf(item.DepositReceiptVersion.Value); + } } return (contentLength, logsLength); @@ -156,10 +173,30 @@ public void Encode(RlpStream rlpStream, OptimismTxReceipt item, RlpBehaviors rlp rlpStream.Encode(item.Logs[i]); } - if (item.TxType == TxType.DepositTx && item.DepositReceiptVersion is not null) + if (item.TxType == TxType.DepositTx && item.DepositNonce is not null && + (item.DepositReceiptVersion is not null || !_isEncodedForTrie)) { - rlpStream.Encode(item.DepositNonce!.Value); - rlpStream.Encode(item.DepositReceiptVersion.Value); + rlpStream.Encode(item.DepositNonce.Value); + + if (item.DepositReceiptVersion is not null) + { + rlpStream.Encode(item.DepositReceiptVersion.Value); + } } } + + TxReceipt IRlpStreamDecoder.Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors) + { + return Decode(rlpStream, rlpBehaviors); + } + + public void Encode(RlpStream stream, TxReceipt item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + Encode(stream, (OptimismTxReceipt)item, rlpBehaviors); + } + + public int GetLength(TxReceipt item, RlpBehaviors rlpBehaviors) + { + return GetLength((OptimismTxReceipt)item, rlpBehaviors); + } } diff --git a/src/Nethermind/Nethermind.Optimism/OptimismReceiptStorageDecoder.cs b/src/Nethermind/Nethermind.Optimism/OptimismReceiptStorageDecoder.cs new file mode 100644 index 00000000000..38d3a9071b0 --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/OptimismReceiptStorageDecoder.cs @@ -0,0 +1,336 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; +using Nethermind.Core; +using Nethermind.Serialization.Rlp; +using static Nethermind.Serialization.Rlp.Rlp; + +namespace Nethermind.Optimism; + +[Decoder(RlpDecoderKey.Storage)] +public class OptimismCompactReceiptStorageDecoder : + IRlpStreamDecoder, IRlpValueDecoder, IRlpObjectDecoder, IReceiptRefDecoder, + IRlpStreamDecoder, IRlpValueDecoder, IRlpObjectDecoder +{ + public OptimismTxReceipt Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + if (rlpStream.IsNextItemNull()) + { + rlpStream.ReadByte(); + return null!; + } + + OptimismTxReceipt txReceipt = new(); + int lastCheck = rlpStream.ReadSequenceLength() + rlpStream.Position; + + byte[] firstItem = rlpStream.DecodeByteArray(); + if (firstItem.Length == 1) + { + txReceipt.StatusCode = firstItem[0]; + } + else + { + txReceipt.PostTransactionState = firstItem.Length == 0 ? null : new Hash256(firstItem); + } + + txReceipt.Sender = rlpStream.DecodeAddress(); + txReceipt.GasUsedTotal = (long)rlpStream.DecodeUBigInt(); + + int sequenceLength = rlpStream.ReadSequenceLength(); + int logEntriesCheck = sequenceLength + rlpStream.Position; + using ArrayPoolList logEntries = new(sequenceLength * 2 / LengthOfAddressRlp); + + while (rlpStream.Position < logEntriesCheck) + { + logEntries.Add(CompactLogEntryDecoder.Decode(rlpStream, RlpBehaviors.AllowExtraBytes)!); + } + + txReceipt.Logs = [.. logEntries]; + + if (lastCheck > rlpStream.Position) + { + int remainingItems = rlpStream.PeekNumberOfItemsRemaining(lastCheck); + if (remainingItems > 0) + { + txReceipt.DepositNonce = rlpStream.DecodeUlong(); + } + + if (remainingItems > 1) + { + txReceipt.DepositReceiptVersion = rlpStream.DecodeUlong(); + } + } + + bool allowExtraBytes = (rlpBehaviors & RlpBehaviors.AllowExtraBytes) != 0; + if (!allowExtraBytes) + { + rlpStream.Check(lastCheck); + } + + txReceipt.Bloom = new Bloom(txReceipt.Logs); + + return txReceipt; + } + + public OptimismTxReceipt Decode(ref ValueDecoderContext decoderContext, + RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + if (decoderContext.IsNextItemNull()) + { + decoderContext.ReadByte(); + return null!; + } + + OptimismTxReceipt txReceipt = new(); + int lastCheck = decoderContext.ReadSequenceLength() + decoderContext.Position; + + byte[] firstItem = decoderContext.DecodeByteArray(); + if (firstItem.Length == 1) + { + txReceipt.StatusCode = firstItem[0]; + } + else + { + txReceipt.PostTransactionState = firstItem.Length == 0 ? null : new Hash256(firstItem); + } + + txReceipt.Sender = decoderContext.DecodeAddress(); + txReceipt.GasUsedTotal = (long)decoderContext.DecodeUBigInt(); + + int sequenceLength = decoderContext.ReadSequenceLength(); + int logEntriesCheck = sequenceLength + decoderContext.Position; + + // Don't know the size exactly, I'll just assume its just an address and add some margin + using ArrayPoolList logEntries = new(sequenceLength * 2 / LengthOfAddressRlp); + while (decoderContext.Position < logEntriesCheck) + { + logEntries.Add(CompactLogEntryDecoder.Decode(ref decoderContext, RlpBehaviors.AllowExtraBytes)!); + } + + txReceipt.Logs = [.. logEntries]; + + if (lastCheck > decoderContext.Position) + { + int remainingItems = decoderContext.PeekNumberOfItemsRemaining(lastCheck); + if (remainingItems > 0) + { + txReceipt.DepositNonce = decoderContext.DecodeULong(); + } + + if (remainingItems > 1) + { + txReceipt.DepositReceiptVersion = decoderContext.DecodeULong(); + } + } + + bool allowExtraBytes = (rlpBehaviors & RlpBehaviors.AllowExtraBytes) != 0; + if (!allowExtraBytes) + { + decoderContext.Check(lastCheck); + } + + txReceipt.Bloom = new Bloom(txReceipt.Logs); + + return txReceipt; + } + + public void DecodeStructRef(scoped ref ValueDecoderContext decoderContext, RlpBehaviors rlpBehaviors, + out TxReceiptStructRef item) + { + // Note: This method runs at 2.5 million times/sec on my machine + item = new TxReceiptStructRef(); + + if (decoderContext.IsNextItemNull()) + { + decoderContext.ReadByte(); + return; + } + + int lastCheck = decoderContext.ReadSequenceLength() + decoderContext.Position; + + ReadOnlySpan firstItem = decoderContext.DecodeByteArraySpan(); + if (firstItem.Length == 1) + { + item.StatusCode = firstItem[0]; + } + else + { + item.PostTransactionState = + firstItem.Length == 0 ? new Hash256StructRef() : new Hash256StructRef(firstItem); + } + + decoderContext.DecodeAddressStructRef(out item.Sender); + item.GasUsedTotal = (long)decoderContext.DecodeUBigInt(); + + (int PrefixLength, int ContentLength) peekPrefixAndContentLength = + decoderContext.PeekPrefixAndContentLength(); + int logsBytes = peekPrefixAndContentLength.ContentLength + peekPrefixAndContentLength.PrefixLength; + item.LogsRlp = decoderContext.Data.Slice(decoderContext.Position, logsBytes); + + if (lastCheck > decoderContext.Position) + { + int remainingItems = decoderContext.PeekNumberOfItemsRemaining(lastCheck); + + if (remainingItems > 1) + { + decoderContext.SkipItem(); + } + + if (remainingItems > 2) + { + decoderContext.SkipItem(); + } + } + + decoderContext.SkipItem(); + } + + public void DecodeLogEntryStructRef(scoped ref ValueDecoderContext decoderContext, RlpBehaviors none, + out LogEntryStructRef current) + { + CompactLogEntryDecoder.DecodeLogEntryStructRef(ref decoderContext, none, out current); + } + + public Hash256[] DecodeTopics(ValueDecoderContext valueDecoderContext) + { + return CompactLogEntryDecoder.DecodeTopics(valueDecoderContext); + } + + // Refstruct decode does not generate bloom + public bool CanDecodeBloom => false; + + public Rlp Encode(OptimismTxReceipt? item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + RlpStream rlpStream = new(GetLength(item!, rlpBehaviors)); + Encode(rlpStream, item, rlpBehaviors); + return new Rlp(rlpStream.Data.ToArray()!); + } + + public void Encode(RlpStream rlpStream, OptimismTxReceipt? item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + if (item is null) + { + rlpStream.EncodeNullObject(); + return; + } + + (int totalContentLength, int logsLength) = GetContentLength(item, rlpBehaviors); + + bool isEip658receipts = (rlpBehaviors & RlpBehaviors.Eip658Receipts) == RlpBehaviors.Eip658Receipts; + + // Note: Any byte saved here is about 3GB on mainnet. + rlpStream.StartSequence(totalContentLength); + if (isEip658receipts) + { + rlpStream.Encode(item.StatusCode); + } + else + { + rlpStream.Encode(item.PostTransactionState); + } + + rlpStream.Encode(item.Sender); + rlpStream.Encode(item.GasUsedTotal); + + rlpStream.StartSequence(logsLength); + + LogEntry[] logs = item.Logs ?? Array.Empty(); + for (int i = 0; i < logs.Length; i++) + { + CompactLogEntryDecoder.Encode(rlpStream, logs[i]); + } + + if (item.TxType == TxType.DepositTx && item.DepositNonce is not null) + { + rlpStream.Encode(item.DepositNonce.Value); + + if (item.DepositReceiptVersion is not null) + { + rlpStream.Encode(item.DepositReceiptVersion.Value); + } + } + } + + private static (int Total, int Logs) GetContentLength(OptimismTxReceipt? item, RlpBehaviors rlpBehaviors) + { + int contentLength = 0; + if (item is null) + { + return (contentLength, 0); + } + + bool isEip658Receipts = (rlpBehaviors & RlpBehaviors.Eip658Receipts) == RlpBehaviors.Eip658Receipts; + if (isEip658Receipts) + { + contentLength += LengthOf(item.StatusCode); + } + else + { + contentLength += LengthOf(item.PostTransactionState); + } + + contentLength += LengthOf(item.Sender); + contentLength += LengthOf(item.GasUsedTotal); + + int logsLength = GetLogsLength(item); + contentLength += LengthOfSequence(logsLength); + + if (item.TxType == TxType.DepositTx && item.DepositNonce is not null) + { + contentLength += LengthOf(item.DepositNonce); + + if (item.DepositReceiptVersion is not null) + { + contentLength += LengthOf(item.DepositReceiptVersion.Value); + } + } + + return (contentLength, logsLength); + } + + private static int GetLogsLength(OptimismTxReceipt item) + { + int logsLength = 0; + LogEntry[] logs = item.Logs ?? Array.Empty(); + for (int i = 0; i < logs.Length; i++) + { + logsLength += CompactLogEntryDecoder.Instance.GetLength(logs[i]); + } + + return logsLength; + } + + public int GetLength(OptimismTxReceipt item, RlpBehaviors rlpBehaviors) + { + (int Total, int Logs) length = GetContentLength(item, rlpBehaviors); + return LengthOfSequence(length.Total); + } + + TxReceipt IRlpStreamDecoder.Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors) + { + return Decode(rlpStream, rlpBehaviors); ; + } + + public void Encode(RlpStream stream, TxReceipt item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + Encode(stream, (OptimismTxReceipt)item, rlpBehaviors); + } + + public int GetLength(TxReceipt item, RlpBehaviors rlpBehaviors) + { + return GetLength((OptimismTxReceipt)item, rlpBehaviors); + } + + TxReceipt IRlpValueDecoder.Decode(ref ValueDecoderContext decoderContext, RlpBehaviors rlpBehaviors) + { + return Decode(ref decoderContext, rlpBehaviors); + } + + public Rlp Encode(TxReceipt? item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) + { + return Encode((OptimismTxReceipt?)item, rlpBehaviors); + } +} diff --git a/src/Nethermind/Nethermind.Optimism/OptimismReceiptsRootCalculator.cs b/src/Nethermind/Nethermind.Optimism/OptimismReceiptsRootCalculator.cs deleted file mode 100644 index 244cce79966..00000000000 --- a/src/Nethermind/Nethermind.Optimism/OptimismReceiptsRootCalculator.cs +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System.Linq; -using Nethermind.Blockchain.Receipts; -using Nethermind.Core; -using Nethermind.Core.Crypto; -using Nethermind.Core.Specs; -using Nethermind.State.Proofs; - -namespace Nethermind.Optimism; - -public class OptimismReceiptsRootCalculator : IReceiptsRootCalculator -{ - public static readonly OptimismReceiptsRootCalculator Instance = new(); - - public Hash256 GetReceiptsRoot(TxReceipt[] receipts, IReceiptSpec spec, Hash256? suggestedRoot) - { - Hash256 SkipStateAndStatusReceiptsRoot() - { - receipts.SetSkipStateAndStatusInRlp(true); - try - { - return ReceiptTrie.CalculateRoot(spec, receipts.Cast().ToArray(), OptimismReceiptDecoder.Instance); - } - finally - { - receipts.SetSkipStateAndStatusInRlp(false); - } - } - - Hash256 receiptsRoot = ReceiptTrie.CalculateRoot(spec, receipts.Cast().ToArray(), OptimismReceiptDecoder.Instance); - if (!spec.ValidateReceipts && receiptsRoot != suggestedRoot) - { - var skipStateAndStatusReceiptsRoot = SkipStateAndStatusReceiptsRoot(); - if (skipStateAndStatusReceiptsRoot == suggestedRoot) - { - return skipStateAndStatusReceiptsRoot; - } - } - - return receiptsRoot; - } -} diff --git a/src/Nethermind/Nethermind.Optimism/OptimismTransactionProcessor.cs b/src/Nethermind/Nethermind.Optimism/OptimismTransactionProcessor.cs index 4322fa58f32..aa71d1fae45 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismTransactionProcessor.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismTransactionProcessor.cs @@ -13,23 +13,16 @@ namespace Nethermind.Optimism; -public class OptimismTransactionProcessor : TransactionProcessor +public class OptimismTransactionProcessor( + ISpecProvider specProvider, + IWorldState worldState, + IVirtualMachine virtualMachine, + ILogManager logManager, + IL1CostHelper l1CostHelper, + IOptimismSpecHelper opSpecHelper, + ICodeInfoRepository? codeInfoRepository + ) : TransactionProcessor(specProvider, worldState, virtualMachine, codeInfoRepository, logManager) { - private readonly IL1CostHelper _l1CostHelper; - private readonly IOPConfigHelper _opConfigHelper; - - public OptimismTransactionProcessor( - ISpecProvider? specProvider, - IWorldState? worldState, - IVirtualMachine? virtualMachine, - ILogManager? logManager, - IL1CostHelper l1CostHelper, - IOPConfigHelper opConfigHelper) : base(specProvider, worldState, virtualMachine, logManager) - { - _l1CostHelper = l1CostHelper; - _opConfigHelper = opConfigHelper; - } - private UInt256? _currentTxL1Cost; protected override TransactionResult Execute(Transaction tx, in BlockExecutionContext blCtx, ITxTracer tracer, ExecutionOptions opts) @@ -108,7 +101,7 @@ protected override TransactionResult BuyGas(Transaction tx, BlockHeader header, return "insufficient sender balance"; } - UInt256 l1Cost = _currentTxL1Cost ??= _l1CostHelper.ComputeL1Cost(tx, header, WorldState); + UInt256 l1Cost = _currentTxL1Cost ??= l1CostHelper.ComputeL1Cost(tx, header, WorldState); if (UInt256.SubtractUnderflow(balanceLeft, l1Cost, out balanceLeft)) { TraceLogInvalidTx(tx, $"INSUFFICIENT_SENDER_BALANCE: ({tx.SenderAddress})_BALANCE = {senderBalance}"); @@ -158,10 +151,10 @@ protected override void PayFees(Transaction tx, BlockHeader header, IReleaseSpec // Skip coinbase payments for deposit tx in Regolith base.PayFees(tx, header, spec, tracer, substate, spentGas, premiumPerGas, statusCode); - if (_opConfigHelper.IsBedrock(header)) + if (opSpecHelper.IsBedrock(header)) { - UInt256 l1Cost = _currentTxL1Cost ??= _l1CostHelper.ComputeL1Cost(tx, header, WorldState); - WorldState.AddToBalanceAndCreateIfNotExists(_opConfigHelper.L1FeeReceiver, l1Cost, spec); + UInt256 l1Cost = _currentTxL1Cost ??= l1CostHelper.ComputeL1Cost(tx, header, WorldState); + WorldState.AddToBalanceAndCreateIfNotExists(opSpecHelper.L1FeeReceiver, l1Cost, spec); } } } @@ -171,7 +164,7 @@ protected override long Refund(Transaction tx, BlockHeader header, IReleaseSpec { // if deposit: skip refunds, skip tipping coinbase // Regolith changes this behaviour to report the actual gasUsed instead of always reporting all gas used. - if (tx.IsDeposit() && !_opConfigHelper.IsRegolith(header)) + if (tx.IsDeposit() && !opSpecHelper.IsRegolith(header)) { // Record deposits as using all their gas // System Transactions are special & are not recorded as using any gas (anywhere) diff --git a/src/Nethermind/Nethermind.Optimism/OptimismTransactionsExecutorFactory.cs b/src/Nethermind/Nethermind.Optimism/OptimismTransactionsExecutorFactory.cs index d7485be9bd4..951d01542a8 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismTransactionsExecutorFactory.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismTransactionsExecutorFactory.cs @@ -4,6 +4,7 @@ using Nethermind.Consensus.Processing; using Nethermind.Consensus.Producers; using Nethermind.Core.Specs; +using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; namespace Nethermind.Optimism; @@ -19,10 +20,10 @@ public OptimismTransactionsExecutorFactory(ISpecProvider specProvider, ILogManag _logManager = logManager; } - public IBlockProcessor.IBlockTransactionsExecutor Create(ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv) + public IBlockProcessor.IBlockTransactionsExecutor Create(IReadOnlyTxProcessingScope readOnlyTxProcessingEnv) { return new BlockProcessor.BlockProductionTransactionsExecutor(readOnlyTxProcessingEnv.TransactionProcessor, - readOnlyTxProcessingEnv.StateProvider, new OptimismBlockProductionTransactionPicker(_specProvider), + readOnlyTxProcessingEnv.WorldState, new OptimismBlockProductionTransactionPicker(_specProvider), _logManager); } } diff --git a/src/Nethermind/Nethermind.Optimism/OptimismTxPoolTxSource.cs b/src/Nethermind/Nethermind.Optimism/OptimismTxPoolTxSource.cs index b91f085b892..7334e9448b4 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismTxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismTxPoolTxSource.cs @@ -6,6 +6,7 @@ using Nethermind.Consensus.Producers; using Nethermind.Consensus.Transactions; using Nethermind.Core; +using Nethermind.Optimism.Rpc; namespace Nethermind.Optimism; diff --git a/src/Nethermind/Nethermind.Optimism/ReadOnlyChainProcessingEnv.cs b/src/Nethermind/Nethermind.Optimism/ReadOnlyChainProcessingEnv.cs new file mode 100644 index 00000000000..9321df272de --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/ReadOnlyChainProcessingEnv.cs @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Blockchain; +using Nethermind.Blockchain.Blocks; +using Nethermind.Blockchain.Receipts; +using Nethermind.Consensus.Processing; +using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Validators; +using Nethermind.Consensus.Withdrawals; +using Nethermind.Core.Specs; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.Logging; +using Nethermind.State; + +namespace Nethermind.Optimism; + +/// +/// Not thread safe. +/// +public class OptimismReadOnlyChainProcessingEnv( + IReadOnlyTxProcessingScope txEnv, + IBlockValidator blockValidator, + IBlockPreprocessorStep recoveryStep, + IRewardCalculator rewardCalculator, + IReceiptStorage receiptStorage, + ISpecProvider specProvider, + IBlockTree blockTree, + IStateReader stateReader, + ILogManager logManager, + IOptimismSpecHelper opSpecHelper, + Create2DeployerContractRewriter contractRewriter, + IWithdrawalProcessor? withdrawalProcessor, + IBlockProcessor.IBlockTransactionsExecutor? blockTransactionsExecutor = null) : ReadOnlyChainProcessingEnv( + txEnv, + blockValidator, + recoveryStep, + rewardCalculator, + receiptStorage, + specProvider, + blockTree, + stateReader, + logManager, + blockTransactionsExecutor) +{ + + protected override IBlockProcessor CreateBlockProcessor( + IReadOnlyTxProcessingScope scope, + IBlockTree blockTree, + IBlockValidator blockValidator, + IRewardCalculator rewardCalculator, + IReceiptStorage receiptStorage, + ISpecProvider specProvider, + ILogManager logManager, + IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor + ) + { + return new OptimismBlockProcessor( + specProvider, + blockValidator, + rewardCalculator, + transactionsExecutor, + scope.WorldState, + receiptStorage, + new BlockhashStore(blockTree, specProvider, scope.WorldState), + logManager, + opSpecHelper, + scope.TransactionProcessor, + contractRewriter, + withdrawalProcessor); + } +} diff --git a/src/Nethermind/Nethermind.Optimism/IOptimismEngineRpcModule.cs b/src/Nethermind/Nethermind.Optimism/Rpc/IOptimismEngineRpcModule.cs similarity index 99% rename from src/Nethermind/Nethermind.Optimism/IOptimismEngineRpcModule.cs rename to src/Nethermind/Nethermind.Optimism/Rpc/IOptimismEngineRpcModule.cs index 08b1ee5f25f..a4fff0ee437 100644 --- a/src/Nethermind/Nethermind.Optimism/IOptimismEngineRpcModule.cs +++ b/src/Nethermind/Nethermind.Optimism/Rpc/IOptimismEngineRpcModule.cs @@ -7,7 +7,7 @@ using Nethermind.JsonRpc.Modules; using Nethermind.Merge.Plugin.Data; -namespace Nethermind.Optimism; +namespace Nethermind.Optimism.Rpc; [RpcModule(ModuleType.Engine)] diff --git a/src/Nethermind/Nethermind.Optimism/Rpc/IOptimismEthRpcModule.cs b/src/Nethermind/Nethermind.Optimism/Rpc/IOptimismEthRpcModule.cs new file mode 100644 index 00000000000..963fbe27a4b --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/Rpc/IOptimismEthRpcModule.cs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.JsonRpc.Modules.Eth; +using Nethermind.JsonRpc.Modules; +using Nethermind.Core.Crypto; +using Nethermind.JsonRpc; +using Nethermind.Blockchain.Find; + +namespace Nethermind.Optimism.Rpc; + +[RpcModule(ModuleType.Eth)] +public interface IOptimismEthRpcModule : IEthRpcModule +{ + [JsonRpcMethod(Description = "Get receipts from all transactions from particular block, more efficient than fetching the receipts one-by-one.", + IsImplemented = true, + ExampleResponse = "{\"jsonrpc\":\"2.0\",\"result\":[{\"transactionHash\":\"0x681c2b6f99e37fd6fe6046db8b51ec3460d699cacd6a376143fd5842ac50621f\",\"transactionIndex\":\"0x0\",\"blockHash\":\"0x29f141925d2d8e357ae5b6040c97aa12d7ac6dfcbe2b20e7b616d8907ac8e1f3\",\"blockNumber\":\"0x3\",\"cumulativeGasUsed\":\"0x5208\",\"gasUsed\":\"0x5208\",\"effectiveGasPrice\":\"0x1\",\"from\":\"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099\",\"to\":\"0x942921b14f1b1c385cd7e0cc2ef7abe5598c8358\",\"contractAddress\":null,\"logs\":[],\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"status\":\"0x1\",\"type\":\"0x0\"},{\"transactionHash\":\"0x7126cf20a0ad8bd51634837d9049615c34c1bff5e1a54e5663f7e23109bff48b\",\"transactionIndex\":\"0x1\",\"blockHash\":\"0x29f141925d2d8e357ae5b6040c97aa12d7ac6dfcbe2b20e7b616d8907ac8e1f3\",\"blockNumber\":\"0x3\",\"cumulativeGasUsed\":\"0xa410\",\"gasUsed\":\"0x5208\",\"effectiveGasPrice\":\"0x1\",\"from\":\"0xb7705ae4c6f81b66cdb323c65f4e8133690fc099\",\"to\":\"0x942921b14f1b1c385cd7e0cc2ef7abe5598c8358\",\"contractAddress\":null,\"logs\":[],\"logsBloom\":\"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000\",\"status\":\"0x1\",\"type\":\"0x0\"}],\"id\":67}")] + new ResultWrapper eth_getBlockReceipts([JsonRpcParameter(ExampleValue = "latest")] BlockParameter blockParameter); + + + [JsonRpcMethod(IsImplemented = true, + Description = "Retrieves a transaction receipt by tx hash", + IsSharable = true, + ExampleResponse = "{\"transactionHash\":\"0x80757153e93d1b475e203406727b62a501187f63e23b8fa999279e219ee3be71\",\"transactionIndex\":\"0x7\",\"blockHash\":\"0x42def051b21038905cd2a2bc28d460a94df2249466847f0e1bcb4be4eb21891a\",\"blockNumber\":\"0x4e3f39\",\"cumulativeGasUsed\":\"0x62c9d\",\"gasUsed\":\"0xe384\",\"effectiveGasPrice\":\"0x12a05f200\",\"from\":\"0x0afe0a94415e8974052e7e6cfab19ee1c2ef4f69\",\"to\":\"0x19e8c84d4943e58b035626b064cfc76ee13ee6cb\",\"contractAddress\":null,\"logs\":[{\"removed\":false,\"logIndex\":\"0x0\",\"transactionIndex\":\"0x7\",\"transactionHash\":\"0x80757153e93d1b475e203406727b62a501187f63e23b8fa999279e219ee3be71\",\"blockHash\":\"0x42def051b21038905cd2a2bc28d460a94df2249466847f0e1bcb4be4eb21891a\",\"blockNumber\":\"0x4e3f39\",\"address\":\"0x2ac3c1d3e24b45c6c310534bc2dd84b5ed576335\",\"data\":\"0x0000000000000000000000000000000000000000000000000000000000000000\",\"topics\":[\"0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef\",\"0x00000000000000000000000019e8c84d4943e58b035626b064cfc76ee13ee6cb\",\"0x00000000000000000000000028078300a459a9e136f872285654cdc74463041e\"]},{\"removed\":false,\"logIndex\":\"0x1\",\"transactionIndex\":\"0x7\",\"transactionHash\":\"0x80757153e93d1b475e203406727b62a501187f63e23b8fa999279e219ee3be71\",\"blockHash\":\"0x42def051b21038905cd2a2bc28d460a94df2249466847f0e1bcb4be4eb21891a\",\"blockNumber\":\"0x4e3f39\",\"address\":\"0x19e8c84d4943e58b035626b064cfc76ee13ee6cb\",\"data\":\"0x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007735940000000000000000000000000000000000000000000000000000000000000000000\",\"topics\":[\"0x950494fc3642fae5221b6c32e0e45765c95ebb382a04a71b160db0843e74c99f\",\"0x0000000000000000000000000afe0a94415e8974052e7e6cfab19ee1c2ef4f69\",\"0x00000000000000000000000028078300a459a9e136f872285654cdc74463041e\",\"0x0000000000000000000000000afe0a94415e8974052e7e6cfab19ee1c2ef4f69\"]}],\"logsBloom\":\"0x00000000000000000000000000000000000000000000000020000000000000800000000000000000000400000000000000000000000000000000000000002000000000000000000000000008000000000000000000000000000000000000000000000002002000000000000000000000000000000000000000000812000000000000000000000000000001000000000000000000000008000400008000000000000000000000000000000000000000000000000000000000800000000000000000000002000000000000000000000000000000000000100000000000000000002000000000000000000000000010000000000000000000000400000000020000\",\"status\":\"0x1\",\"type\":\"0x0\"}")] + new ResultWrapper eth_getTransactionReceipt([JsonRpcParameter(ExampleValue = "[\"0x80757153e93d1b475e203406727b62a501187f63e23b8fa999279e219ee3be71\"]")] Hash256 txHashData); +} diff --git a/src/Nethermind/Nethermind.Optimism/OptimismEngineRpcModule.cs b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismEngineRpcModule.cs similarity index 98% rename from src/Nethermind/Nethermind.Optimism/OptimismEngineRpcModule.cs rename to src/Nethermind/Nethermind.Optimism/Rpc/OptimismEngineRpcModule.cs index 79e063eb4b6..92aba891d7b 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismEngineRpcModule.cs +++ b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismEngineRpcModule.cs @@ -7,7 +7,7 @@ using Nethermind.Merge.Plugin; using Nethermind.Merge.Plugin.Data; -namespace Nethermind.Optimism; +namespace Nethermind.Optimism.Rpc; public class OptimismEngineRpcModule : IOptimismEngineRpcModule { diff --git a/src/Nethermind/Nethermind.Optimism/Rpc/OptimismEthModuleFactory.cs b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismEthModuleFactory.cs new file mode 100644 index 00000000000..9604bbdc78b --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismEthModuleFactory.cs @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Blockchain.Receipts; +using Nethermind.Core.Specs; +using Nethermind.Facade; +using Nethermind.Facade.Eth; +using Nethermind.JsonRpc.Modules.Eth.GasPrice; +using Nethermind.JsonRpc.Modules.Eth.FeeHistory; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.TxPool; +using Nethermind.Wallet; +using Nethermind.JsonRpc; +using Nethermind.JsonRpc.Modules; +using Nethermind.Blockchain.Find; +using Nethermind.Core; +using Nethermind.Crypto; +using Nethermind.JsonRpc.Client; + +namespace Nethermind.Optimism.Rpc; + +public class OptimismEthModuleFactory( + IJsonRpcConfig rpcConfig, + IBlockchainBridgeFactory blockchainBridgeFactory, + IBlockFinder blockFinder, + IReceiptFinder receiptFinder, + IStateReader stateReader, + ITxPool txPool, + ITxSender txSender, + IWallet wallet, + ILogManager logManager, + ISpecProvider specProvider, + IGasPriceOracle gasPriceOracle, + IEthSyncingInfo ethSyncingInfo, + IFeeHistoryOracle feeHistoryOracle, + ulong? secondsPerSlot, + + IJsonRpcClient? sequencerRpcClient, + IAccountStateProvider accountStateProvider, + IEthereumEcdsa ecdsa, + ITxSealer sealer, + IOptimismSpecHelper opSpecHelper + ) + : ModuleFactoryBase +{ + private readonly ILogManager _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); + private readonly IStateReader _stateReader = stateReader ?? throw new ArgumentNullException(nameof(stateReader)); + private readonly IBlockchainBridgeFactory _blockchainBridgeFactory = blockchainBridgeFactory ?? throw new ArgumentNullException(nameof(blockchainBridgeFactory)); + private readonly ITxPool _txPool = txPool ?? throw new ArgumentNullException(nameof(txPool)); + private readonly ITxSender _txSender = txSender ?? throw new ArgumentNullException(nameof(txSender)); + private readonly IWallet _wallet = wallet ?? throw new ArgumentNullException(nameof(wallet)); + private readonly IJsonRpcConfig _rpcConfig = rpcConfig ?? throw new ArgumentNullException(nameof(rpcConfig)); + private readonly ISpecProvider _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); + private readonly IGasPriceOracle _gasPriceOracle = gasPriceOracle ?? throw new ArgumentNullException(nameof(gasPriceOracle)); + private readonly IEthSyncingInfo _ethSyncingInfo = ethSyncingInfo ?? throw new ArgumentNullException(nameof(ethSyncingInfo)); + private readonly IFeeHistoryOracle _feeHistoryOracle = feeHistoryOracle ?? throw new ArgumentNullException(nameof(feeHistoryOracle)); + private readonly IJsonRpcClient? _sequencerRpcClient = sequencerRpcClient ?? throw new ArgumentNullException(nameof(sequencerRpcClient)); + private readonly IAccountStateProvider _accountStateProvider = accountStateProvider ?? throw new ArgumentNullException(nameof(accountStateProvider)); + private readonly IEthereumEcdsa _ecdsa = ecdsa ?? throw new ArgumentNullException(nameof(ecdsa)); + private readonly ITxSealer _sealer = sealer ?? throw new ArgumentNullException(nameof(sealer)); + private readonly IBlockFinder _blockFinder = blockFinder ?? throw new ArgumentNullException(nameof(blockFinder)); + private readonly IReceiptFinder _receiptFinder = receiptFinder ?? throw new ArgumentNullException(nameof(receiptFinder)); + private readonly IOptimismSpecHelper _opSpecHelper = opSpecHelper ?? throw new ArgumentNullException(nameof(opSpecHelper)); + + public override IOptimismEthRpcModule Create() + { + return new OptimismEthRpcModule( + _rpcConfig, + _blockchainBridgeFactory.CreateBlockchainBridge(), + _blockFinder, + _receiptFinder, + _stateReader, + _txPool, + _txSender, + _wallet, + _logManager, + _specProvider, + _gasPriceOracle, + _ethSyncingInfo, + _feeHistoryOracle, + secondsPerSlot, + + _sequencerRpcClient, + _accountStateProvider, + _ecdsa, + _sealer, + _opSpecHelper + ); + } +} diff --git a/src/Nethermind/Nethermind.Optimism/Rpc/OptimismEthRpcModule.cs b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismEthRpcModule.cs new file mode 100644 index 00000000000..27154fdf0e1 --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismEthRpcModule.cs @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Linq; +using System.Threading.Tasks; +using Nethermind.Blockchain.Find; +using Nethermind.Blockchain.Receipts; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Crypto; +using Nethermind.Evm; +using Nethermind.Facade; +using Nethermind.Facade.Eth; +using Nethermind.JsonRpc; +using Nethermind.JsonRpc.Client; +using Nethermind.JsonRpc.Data; +using Nethermind.JsonRpc.Modules; +using Nethermind.JsonRpc.Modules.Eth; +using Nethermind.JsonRpc.Modules.Eth.FeeHistory; +using Nethermind.JsonRpc.Modules.Eth.GasPrice; +using Nethermind.Logging; +using Nethermind.Serialization.Rlp; +using Nethermind.State; +using Nethermind.TxPool; +using Nethermind.Wallet; + +namespace Nethermind.Optimism.Rpc; + +public class OptimismEthRpcModule : EthRpcModule, IOptimismEthRpcModule +{ + private readonly IJsonRpcClient? _sequencerRpcClient; + private readonly IAccountStateProvider _accountStateProvider; + private readonly IEthereumEcdsa _ecdsa; + private readonly ITxSealer _sealer; + private readonly IOptimismSpecHelper _opSpecHelper; + + public OptimismEthRpcModule( + IJsonRpcConfig rpcConfig, + IBlockchainBridge blockchainBridge, + IBlockFinder blockFinder, + IReceiptFinder receiptFinder, + IStateReader stateReader, + ITxPool txPool, + ITxSender txSender, + IWallet wallet, + ILogManager logManager, + ISpecProvider specProvider, + IGasPriceOracle gasPriceOracle, + IEthSyncingInfo ethSyncingInfo, + IFeeHistoryOracle feeHistoryOracle, + ulong? secondsPerSlot, + + IJsonRpcClient? sequencerRpcClient, + IAccountStateProvider accountStateProvider, + IEthereumEcdsa ecdsa, + ITxSealer sealer, + IOptimismSpecHelper opSpecHelper) : base( + rpcConfig, + blockchainBridge, + blockFinder, + receiptFinder, + stateReader, + txPool, + txSender, + wallet, + logManager, + specProvider, + gasPriceOracle, + ethSyncingInfo, + feeHistoryOracle, + secondsPerSlot) + { + _sequencerRpcClient = sequencerRpcClient; + _accountStateProvider = accountStateProvider; + _ecdsa = ecdsa; + _sealer = sealer; + _opSpecHelper = opSpecHelper; + } + + public new ResultWrapper eth_getBlockReceipts(BlockParameter blockParameter) + { + static ResultWrapper GetBlockReceipts(IReceiptFinder receiptFinder, BlockParameter blockParameter, IBlockFinder blockFinder, ISpecProvider specProvider, IOptimismSpecHelper opSpecHelper) + { + SearchResult searchResult = blockFinder.SearchForBlock(blockParameter); + if (searchResult.IsError) + { + return ResultWrapper.Success(null); + } + + Block? block = searchResult.Object!; + OptimismTxReceipt[] receipts = receiptFinder.Get(block).Cast().ToArray() ?? new OptimismTxReceipt[block.Transactions.Length]; + bool isEip1559Enabled = specProvider.GetSpec(block.Header).IsEip1559Enabled; + + L1BlockGasInfo l1BlockGasInfo = new(block, opSpecHelper); + + OptimismReceiptForRpc[]? result = [.. receipts + .Zip(block.Transactions, (r, t) => + { + return new OptimismReceiptForRpc(t.Hash!, r, t.GetGasInfo(isEip1559Enabled, block.Header), l1BlockGasInfo.GetTxGasInfo(t), receipts.GetBlockLogFirstIndex(r.Index)); + })]; + return ResultWrapper.Success(result); + } + + return GetBlockReceipts(_receiptFinder, blockParameter, _blockFinder, _specProvider, _opSpecHelper); + } + + public override async Task> eth_sendTransaction(TransactionForRpc rpcTx) + { + Transaction tx = rpcTx.ToTransactionWithDefaults(_blockchainBridge.GetChainId()); + tx.SenderAddress ??= _ecdsa.RecoverAddress(tx); + + if (tx.SenderAddress is null) + { + return ResultWrapper.Fail("Failed to recover sender"); + } + + if (rpcTx.Nonce is null) + { + tx.Nonce = _accountStateProvider.GetNonce(tx.SenderAddress); + } + + await _sealer.Seal(tx, TxHandlingOptions.None); + + return await eth_sendRawTransaction(Rlp.Encode(tx, RlpBehaviors.SkipTypedWrapping).Bytes); + } + + public override async Task> eth_sendRawTransaction(byte[] transaction) + { + if (_sequencerRpcClient is null) + { + return ResultWrapper.Fail("No sequencer url in the config"); + } + Hash256? result = await _sequencerRpcClient.Post(nameof(eth_sendRawTransaction), transaction); + if (result is null) + { + return ResultWrapper.Fail("Failed to forward transaction"); + } + return ResultWrapper.Success(result); + } + + public new ResultWrapper eth_getTransactionReceipt(Hash256 txHash) + { + (TxReceipt? receipt, TxGasInfo? gasInfo, int logIndexStart) = _blockchainBridge.GetReceiptAndGasInfo(txHash); + if (receipt is null || gasInfo is null) + { + return ResultWrapper.Success(null); + } + + SearchResult foundBlock = _blockFinder.SearchForBlock(new(receipt.BlockHash!)); + if (foundBlock.Object is null) + { + return ResultWrapper.Success(null); + } + + Block block = foundBlock.Object; + + L1BlockGasInfo l1GasInfo = new(block, _opSpecHelper); + return ResultWrapper.Success( + new(txHash, (OptimismTxReceipt)receipt, gasInfo.Value, l1GasInfo.GetTxGasInfo(block.Transactions.First(tx => tx.Hash == txHash)), logIndexStart)); + } +} diff --git a/src/Nethermind/Nethermind.Optimism/OptimismPayloadAttributes.cs b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismPayloadAttributes.cs similarity index 97% rename from src/Nethermind/Nethermind.Optimism/OptimismPayloadAttributes.cs rename to src/Nethermind/Nethermind.Optimism/Rpc/OptimismPayloadAttributes.cs index f1727be019c..8db3156fcae 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismPayloadAttributes.cs +++ b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismPayloadAttributes.cs @@ -12,7 +12,7 @@ using Nethermind.Core.Specs; using Nethermind.Serialization.Rlp; -namespace Nethermind.Optimism; +namespace Nethermind.Optimism.Rpc; public class OptimismPayloadAttributes : PayloadAttributes { @@ -69,7 +69,7 @@ protected override int ComputePayloadIdMembersSize() => protected override int WritePayloadIdMembers(BlockHeader parentHeader, Span inputSpan) { - int offset = base.WritePayloadIdMembers(parentHeader, inputSpan); + var offset = base.WritePayloadIdMembers(parentHeader, inputSpan); inputSpan[offset] = NoTxPool ? (byte)1 : (byte)0; offset += 1; @@ -122,9 +122,7 @@ public override string ToString() .Append($"{nameof(Transactions)}: {Transactions?.Length ?? 0}"); if (Withdrawals is not null) - { sb.Append($", {nameof(Withdrawals)} count: {Withdrawals.Length}"); - } sb.Append('}'); return sb.ToString(); diff --git a/src/Nethermind/Nethermind.Optimism/Rpc/OptimismReceiptForRpc.cs b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismReceiptForRpc.cs new file mode 100644 index 00000000000..d902059872b --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismReceiptForRpc.cs @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; +using Nethermind.Evm; +using Nethermind.Int256; +using Nethermind.JsonRpc.Data; +using System.Text.Json.Serialization; + +namespace Nethermind.Optimism.Rpc; + +public class OptimismReceiptForRpc : ReceiptForRpc +{ + public OptimismReceiptForRpc(Hash256 txHash, OptimismTxReceipt receipt, TxGasInfo gasInfo, L1TxGasInfo l1GasInfo, int logIndexStart = 0) : base( + txHash, receipt, gasInfo, logIndexStart) + { + if (receipt.TxType == Core.TxType.DepositTx) + { + DepositNonce = receipt.DepositNonce; + DepositReceiptVersion = receipt.DepositReceiptVersion; + } + else + { + L1Fee = l1GasInfo.L1Fee; + L1GasUsed = l1GasInfo.L1GasUsed; + L1GasPrice = l1GasInfo.L1GasPrice; + L1FeeScalar = l1GasInfo.L1FeeScalar; + } + } + + // DepositTx related fields + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public UInt256? DepositNonce; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public UInt256? DepositReceiptVersion; + + // Regular tx fields + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public UInt256? L1Fee; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public UInt256? L1GasPrice; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public UInt256? L1GasUsed; + + // Pre-ecotone field of a regular tx fields + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? L1FeeScalar; +} diff --git a/src/Nethermind/Nethermind.Optimism/Rpc/OptimismTraceModuleFactory.cs b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismTraceModuleFactory.cs new file mode 100644 index 00000000000..8a1b5da4d21 --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/Rpc/OptimismTraceModuleFactory.cs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Blockchain; +using Nethermind.Blockchain.Receipts; +using Nethermind.Consensus; +using Nethermind.Consensus.Processing; +using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Validators; +using Nethermind.Consensus.Withdrawals; +using Nethermind.Core.Specs; +using Nethermind.Evm.TransactionProcessing; +using Nethermind.JsonRpc; +using Nethermind.JsonRpc.Modules.Trace; +using Nethermind.Logging; +using Nethermind.State; + +namespace Nethermind.Optimism.Rpc; + +public class OptimismTraceModuleFactory( + IWorldStateManager worldStateManager, + IBlockTree blockTree, + IJsonRpcConfig jsonRpcConfig, + IBlockPreprocessorStep recoveryStep, + IRewardCalculatorSource rewardCalculatorSource, + IReceiptStorage receiptFinder, + ISpecProvider specProvider, + IPoSSwitcher poSSwitcher, + ILogManager logManager, + IL1CostHelper l1CostHelper, + IOptimismSpecHelper opSpecHelper, + Create2DeployerContractRewriter contractRewriter, + IWithdrawalProcessor withdrawalProcessor) : TraceModuleFactory( + worldStateManager, + blockTree, + jsonRpcConfig, + recoveryStep, + rewardCalculatorSource, + receiptFinder, + specProvider, + poSSwitcher, + logManager) +{ + protected override ReadOnlyTxProcessingEnv CreateTxProcessingEnv() => + new OptimismReadOnlyTxProcessingEnv(_worldStateManager, _blockTree, _specProvider, _logManager, l1CostHelper, opSpecHelper); + + protected override ReadOnlyChainProcessingEnv CreateChainProcessingEnv(IBlockProcessor.IBlockTransactionsExecutor transactionsExecutor, IReadOnlyTxProcessingScope scope, IRewardCalculator rewardCalculator) => new OptimismReadOnlyChainProcessingEnv( + scope, + Always.Valid, + _recoveryStep, + rewardCalculator, + _receiptStorage, + _specProvider, + _blockTree, + _worldStateManager.GlobalStateReader, + _logManager, + opSpecHelper, + contractRewriter, + withdrawalProcessor, + transactionsExecutor); +} diff --git a/src/Nethermind/Nethermind.Optimism/Rpc/RegisterOptimismRpcModules.cs b/src/Nethermind/Nethermind.Optimism/Rpc/RegisterOptimismRpcModules.cs new file mode 100644 index 00000000000..2b509d129c0 --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/Rpc/RegisterOptimismRpcModules.cs @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Api; +using Nethermind.Blockchain; +using Nethermind.Config; +using Nethermind.Consensus.AuRa.Withdrawals; +using Nethermind.Consensus.Withdrawals; +using Nethermind.Init.Steps; +using Nethermind.JsonRpc; +using Nethermind.JsonRpc.Client; +using Nethermind.JsonRpc.Modules; +using Nethermind.JsonRpc.Modules.Eth; +using Nethermind.JsonRpc.Modules.Eth.FeeHistory; +using Nethermind.Logging; +using Nethermind.TxPool; +using Nethermind.Wallet; + +namespace Nethermind.Optimism.Rpc; + +public class RegisterOptimismRpcModules : RegisterRpcModules +{ + private readonly OptimismNethermindApi _api; + private readonly ILogger _logger; + private readonly IOptimismConfig _config; + private readonly IJsonRpcConfig _jsonRpcConfig; + + public RegisterOptimismRpcModules(INethermindApi api) : base(api) + { + _api = (OptimismNethermindApi)api; + _config = _api.Config(); + _logger = _api.LogManager.GetClassLogger(); + _jsonRpcConfig = _api.Config(); + } + + protected override void RegisterEthRpcModule(IRpcModuleProvider rpcModuleProvider) + { + StepDependencyException.ThrowIfNull(_api.BlockTree); + StepDependencyException.ThrowIfNull(_api.ReceiptStorage); + StepDependencyException.ThrowIfNull(_api.StateReader); + StepDependencyException.ThrowIfNull(_api.TxPool); + StepDependencyException.ThrowIfNull(_api.TxSender); + StepDependencyException.ThrowIfNull(_api.Wallet); + StepDependencyException.ThrowIfNull(_api.EthSyncingInfo); + StepDependencyException.ThrowIfNull(_api.GasPriceOracle); + StepDependencyException.ThrowIfNull(_api.SpecHelper); + StepDependencyException.ThrowIfNull(_api.SpecProvider); + StepDependencyException.ThrowIfNull(_api.WorldState); + StepDependencyException.ThrowIfNull(_api.EthereumEcdsa); + StepDependencyException.ThrowIfNull(_api.Sealer); + + if (_config.SequencerUrl is null && _logger.IsWarn) + { + _logger.Warn($"SequencerUrl is not set."); + } + + BasicJsonRpcClient? sequencerJsonRpcClient = _config.SequencerUrl is null + ? null + : new(new Uri(_config.SequencerUrl), _api.EthereumJsonSerializer, _api.LogManager); + ModuleFactoryBase ethModuleFactory = CreateEthModuleFactory(); + + ITxSigner txSigner = new WalletTxSigner(_api.Wallet, _api.SpecProvider.ChainId); + TxSealer sealer = new(txSigner, _api.Timestamper); + + var feeHistoryOracle = new FeeHistoryOracle(_api.BlockTree, _api.ReceiptStorage, _api.SpecProvider); + _api.DisposeStack.Push(feeHistoryOracle); + + ModuleFactoryBase optimismEthModuleFactory = new OptimismEthModuleFactory( + _jsonRpcConfig, + _api, + _api.BlockTree.AsReadOnly(), + _api.ReceiptStorage, + _api.StateReader, + _api.TxPool, + _api.TxSender, + _api.Wallet, + _api.LogManager, + _api.SpecProvider, + _api.GasPriceOracle, + _api.EthSyncingInfo, + feeHistoryOracle, + _api.ConfigProvider.GetConfig().SecondsPerSlot, + + sequencerJsonRpcClient, + _api.WorldState, + _api.EthereumEcdsa, + sealer, + _api.SpecHelper); + + rpcModuleProvider.RegisterBounded(optimismEthModuleFactory, + _jsonRpcConfig.EthModuleConcurrentInstances ?? Environment.ProcessorCount, _jsonRpcConfig.Timeout); + } + + protected override void RegisterTraceRpcModule(IRpcModuleProvider rpcModuleProvider) + { + StepDependencyException.ThrowIfNull(_api.WorldStateManager); + StepDependencyException.ThrowIfNull(_api.BlockTree); + StepDependencyException.ThrowIfNull(_api.ReceiptStorage); + StepDependencyException.ThrowIfNull(_api.RewardCalculatorSource); + StepDependencyException.ThrowIfNull(_api.SpecProvider); + StepDependencyException.ThrowIfNull(_api.WorldState); + StepDependencyException.ThrowIfNull(_api.L1CostHelper); + StepDependencyException.ThrowIfNull(_api.SpecHelper); + + OptimismTraceModuleFactory traceModuleFactory = new( + _api.WorldStateManager, + _api.BlockTree, + _jsonRpcConfig, + _api.BlockPreprocessor, + _api.RewardCalculatorSource, + _api.ReceiptStorage, + _api.SpecProvider, + _api.PoSSwitcher, + _api.LogManager, + _api.L1CostHelper, + _api.SpecHelper, + new Create2DeployerContractRewriter(_api.SpecHelper, _api.SpecProvider, _api.BlockTree), + new BlockProductionWithdrawalProcessor(new NullWithdrawalProcessor())); + + rpcModuleProvider.RegisterBoundedByCpuCount(traceModuleFactory, _jsonRpcConfig.Timeout); + } +} diff --git a/src/Nethermind/Nethermind.Overseer.Test/CliqueTests.cs b/src/Nethermind/Nethermind.Overseer.Test/CliqueTests.cs index 5088be5b61f..bec343e4e35 100644 --- a/src/Nethermind/Nethermind.Overseer.Test/CliqueTests.cs +++ b/src/Nethermind/Nethermind.Overseer.Test/CliqueTests.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using Nethermind.Core.Extensions; using Nethermind.Crypto; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.Overseer.Test.Framework; using NUnit.Framework; diff --git a/src/Nethermind/Nethermind.Overseer.Test/Framework/CliqueContext.cs b/src/Nethermind/Nethermind.Overseer.Test/Framework/CliqueContext.cs index 8affec1b268..9d22da0ee84 100644 --- a/src/Nethermind/Nethermind.Overseer.Test/Framework/CliqueContext.cs +++ b/src/Nethermind/Nethermind.Overseer.Test/Framework/CliqueContext.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Core; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.Overseer.Test.JsonRpc; diff --git a/src/Nethermind/Nethermind.PerfTest/Program.cs b/src/Nethermind/Nethermind.PerfTest/Program.cs index 03e39859f9e..a387f2cd2b5 100644 --- a/src/Nethermind/Nethermind.PerfTest/Program.cs +++ b/src/Nethermind/Nethermind.PerfTest/Program.cs @@ -329,12 +329,12 @@ private static async Task RunBenchmarkBlocks() foreach ((Address address, ChainSpecAllocation allocation) in chainSpec.Allocations) { stateProvider.CreateAccount(address, allocation.Balance); - if (allocation.Code != null) + if (allocation.Code is not null) { stateProvider.InsertCode(address, allocation.Code, specProvider.GenesisSpec); } - if (allocation.Constructor != null) + if (allocation.Constructor is not null) { Transaction constructorTransaction = new SystemTransaction() { diff --git a/src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs b/src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs index 4224aca2f0d..73c0c925c45 100644 --- a/src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs +++ b/src/Nethermind/Nethermind.Runner.Test/Ethereum/ContextWithMocks.cs @@ -119,7 +119,6 @@ public static NethermindApi ContextWithMocks() SyncProgressResolver = Substitute.For(), BetterPeerStrategy = Substitute.For(), ReceiptMonitor = Substitute.For(), - WitnessRepository = Substitute.For(), BadBlocksStore = Substitute.For() }; diff --git a/src/Nethermind/Nethermind.Runner/Ethereum/Api/ApiBuilder.cs b/src/Nethermind/Nethermind.Runner/Ethereum/Api/ApiBuilder.cs index 217cb5fbd6c..2d097c4a769 100644 --- a/src/Nethermind/Nethermind.Runner/Ethereum/Api/ApiBuilder.cs +++ b/src/Nethermind/Nethermind.Runner/Ethereum/Api/ApiBuilder.cs @@ -11,6 +11,7 @@ using Nethermind.Config; using Nethermind.Consensus; using Nethermind.Core; +using Nethermind.Facade.Eth; using Nethermind.JsonRpc.Data; using Nethermind.Logging; using Nethermind.Serialization.Json; diff --git a/src/Nethermind/Nethermind.Runner/Ethereum/EthereumRunner.cs b/src/Nethermind/Nethermind.Runner/Ethereum/EthereumRunner.cs index e445b7320ab..5e79b03e1cb 100644 --- a/src/Nethermind/Nethermind.Runner/Ethereum/EthereumRunner.cs +++ b/src/Nethermind/Nethermind.Runner/Ethereum/EthereumRunner.cs @@ -57,7 +57,7 @@ public async Task StopAsync() Stop(() => _api.SessionMonitor?.Stop(), "Stopping session monitor"); Stop(() => _api.SyncModeSelector?.Stop(), "Stopping session sync mode selector"); Task discoveryStopTask = Stop(() => _api.DiscoveryApp?.StopAsync(), "Stopping discovery app"); - Task blockProducerTask = Stop(() => _api.BlockProducer?.StopAsync(), "Stopping block producer"); + Task blockProducerTask = Stop(() => _api.BlockProducerRunner?.StopAsync(), "Stopping block producer"); Task syncPeerPoolTask = Stop(() => _api.SyncPeerPool?.StopAsync(), "Stopping sync peer pool"); Task peerPoolTask = Stop(() => _api.PeerPool?.StopAsync(), "Stopping peer pool"); Task peerManagerTask = Stop(() => _api.PeerManager?.StopAsync(), "Stopping peer manager"); diff --git a/src/Nethermind/Nethermind.Runner/JsonRpc/Startup.cs b/src/Nethermind/Nethermind.Runner/JsonRpc/Startup.cs index 6625abe7e9a..5c6c24f6156 100644 --- a/src/Nethermind/Nethermind.Runner/JsonRpc/Startup.cs +++ b/src/Nethermind/Nethermind.Runner/JsonRpc/Startup.cs @@ -160,7 +160,7 @@ await PushErrorResponse(StatusCodes.Status403Forbidden, ErrorCodes.InvalidReques CountingPipeReader request = new(ctx.Request.BodyReader); try { - JsonRpcContext jsonRpcContext = JsonRpcContext.Http(jsonRpcUrl); + using JsonRpcContext jsonRpcContext = JsonRpcContext.Http(jsonRpcUrl); await foreach (JsonRpcResult result in jsonRpcProcessor.ProcessAsync(request, jsonRpcContext)) { using (result) @@ -215,7 +215,7 @@ await PushErrorResponse(StatusCodes.Status403Forbidden, ErrorCodes.InvalidReques { jsonSerializer.Serialize(resultWriter, result.Response); } - + await resultWriter.CompleteAsync(); if (stream is not null) { ctx.Response.ContentLength = resultWriter.WrittenCount; diff --git a/src/Nethermind/Nethermind.Runner/Nethermind.Runner.csproj b/src/Nethermind/Nethermind.Runner/Nethermind.Runner.csproj index c033683318c..a3cc9a0f65b 100644 --- a/src/Nethermind/Nethermind.Runner/Nethermind.Runner.csproj +++ b/src/Nethermind/Nethermind.Runner/Nethermind.Runner.csproj @@ -10,7 +10,7 @@ true true true - 1 + 0 @@ -21,6 +21,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + @@ -38,6 +42,7 @@ + @@ -86,11 +91,11 @@ - - + + - - + + @@ -105,25 +110,4 @@ - - - - - - $(Commit.Substring(0, 8)) - - - - <_Parameter1>Commit - <_Parameter2>$(Commit) - - - - - - - - - - diff --git a/src/Nethermind/Nethermind.Runner/Properties/launchSettings.json b/src/Nethermind/Nethermind.Runner/Properties/launchSettings.json index 2c5c4f7df6d..da27c06e51c 100644 --- a/src/Nethermind/Nethermind.Runner/Properties/launchSettings.json +++ b/src/Nethermind/Nethermind.Runner/Properties/launchSettings.json @@ -56,23 +56,30 @@ "ASPNETCORE_ENVIRONMENT": "Development" } }, - "OP Goerli": { + "OP Mainnet": { "commandName": "Project", - "commandLineArgs": "-c op-goerli -dd %NETHERMIND_DATA_DIR% --JsonRpc.UnsecureDevNoRpcAuthentication=true", + "commandLineArgs": "-c op-mainnet -dd %NETHERMIND_DATA_DIR%", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "OP Sepolia": { + "commandName": "Project", + "commandLineArgs": "-c op-sepolia -dd %NETHERMIND_DATA_DIR%", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, "Base Mainnet": { "commandName": "Project", - "commandLineArgs": "-c base-mainnet -dd %NETHERMIND_DATA_DIR% --JsonRpc.UnsecureDevNoRpcAuthentication=true", + "commandLineArgs": "-c base-mainnet -dd %NETHERMIND_DATA_DIR%", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } }, - "Base Goerli": { + "Base Sepolia": { "commandName": "Project", - "commandLineArgs": "-c base-goerli -dd %NETHERMIND_DATA_DIR% --JsonRpc.UnsecureDevNoRpcAuthentication=true", + "commandLineArgs": "-c base-sepolia -dd %NETHERMIND_DATA_DIR%", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } @@ -100,7 +107,7 @@ }, "Mainnet": { "commandName": "Project", - "commandLineArgs": "-c mainnet -dd .data", + "commandLineArgs": "-c mainnet -dd %NETHERMIND_DATA_DIR%", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } diff --git a/src/Nethermind/Nethermind.Runner/configs/base-mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/base-mainnet.cfg index c7485d7dbd7..359c438be29 100644 --- a/src/Nethermind/Nethermind.Runner/configs/base-mainnet.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/base-mainnet.cfg @@ -10,11 +10,35 @@ "BlobsSupport": "Disabled" }, "Sync": { - "NetworkingEnabled": false + "FastSync": true, + "SnapSync": true, + "AncientBodiesBarrier": 105235063, + "AncientReceiptsBarrier": 105235063, + "FastSyncCatchUpHeightDelta": "10000000000", + "PivotNumber": 14239812, + "PivotHash": "0x335a694a721f223125c0f494f44c7525ef1503a664b227a7b3478fbf8c4175b7", + "PivotTotalDifficulty": "0", + "MaxAttemptsToUpdatePivot": 0 + }, + "Discovery": { + "Discv5Enabled": true }, "JsonRpc": { "Enabled": true, "Port": 8545, "EnginePort": 8551 + }, + "Pruning": { + "PruningBoundary": 1024 + }, + "Blocks": { + "SecondsPerSlot": 2, + "TargetBlockGasLimit": 60000000 + }, + "Merge": { + "Enabled": true + }, + "Optimism": { + "SequencerUrl": "https://mainnet-sequencer.base.org" } } diff --git a/src/Nethermind/Nethermind.Runner/configs/base-mainnet_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/base-mainnet_archive.cfg index 2124907388b..a694d5ca33f 100644 --- a/src/Nethermind/Nethermind.Runner/configs/base-mainnet_archive.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/base-mainnet_archive.cfg @@ -19,5 +19,8 @@ "Enabled": true, "Port": 8545, "EnginePort": 8551 + }, + "Optimism": { + "SequencerUrl": "https://mainnet-sequencer.base.org" } } diff --git a/src/Nethermind/Nethermind.Runner/configs/base-sepolia.cfg b/src/Nethermind/Nethermind.Runner/configs/base-sepolia.cfg index 574cee32bf4..c20d5897040 100644 --- a/src/Nethermind/Nethermind.Runner/configs/base-sepolia.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/base-sepolia.cfg @@ -10,11 +10,33 @@ "BlobsSupport": "Disabled" }, "Sync": { - "NetworkingEnabled": false + "FastSync": true, + "SnapSync": true, + "FastSyncCatchUpHeightDelta": "10000000000", + "PivotNumber": 9750377, + "PivotHash": "0x71ce74cb650b24f7fc473934e97c41f3cabe6b094ac368d9d7a733f0cb514599", + "PivotTotalDifficulty": "0", + "MaxAttemptsToUpdatePivot": 0 + }, + "Discovery": { + "Discv5Enabled": true }, "JsonRpc": { "Enabled": true, "Port": 8545, "EnginePort": 8551 + }, + "Pruning": { + "PruningBoundary": 1024 + }, + "Blocks": { + "SecondsPerSlot": 2, + "TargetBlockGasLimit": 45000000 + }, + "Merge": { + "Enabled": true + }, + "Optimism": { + "SequencerUrl": "https://sepolia-sequencer.base.org" } } diff --git a/src/Nethermind/Nethermind.Runner/configs/base-sepolia_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/base-sepolia_archive.cfg index 8e0bc623da3..49828a0b09f 100644 --- a/src/Nethermind/Nethermind.Runner/configs/base-sepolia_archive.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/base-sepolia_archive.cfg @@ -19,5 +19,8 @@ "Enabled": true, "Port": 8545, "EnginePort": 8551 + }, + "Optimism": { + "SequencerUrl": "https://sepolia-sequencer.base.org" } } diff --git a/src/Nethermind/Nethermind.Runner/configs/chiado.cfg b/src/Nethermind/Nethermind.Runner/configs/chiado.cfg index d5f56dcdac7..18726ba0af2 100644 --- a/src/Nethermind/Nethermind.Runner/configs/chiado.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/chiado.cfg @@ -15,8 +15,8 @@ }, "Sync": { "FastSync": true, - "PivotNumber": 9600000, - "PivotHash": "0xc2eb45a42ee83ee2784b016d2808466b754bce896eedfde5856b5ac1865a436c", + "PivotNumber": 10200000, + "PivotHash": "0x66705e2ef5f4edcd627afe1e512903778c780b185c9b3dc92e6934e295683afd", "PivotTotalDifficulty": "231708131825107706987652208063906496124457284", "FastSyncCatchUpHeightDelta": "10000000000", "UseGethLimitsInFastBlocks": false diff --git a/src/Nethermind/Nethermind.Runner/configs/energyweb.cfg b/src/Nethermind/Nethermind.Runner/configs/energyweb.cfg index b3f7396e5ab..e54f328f45e 100644 --- a/src/Nethermind/Nethermind.Runner/configs/energyweb.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/energyweb.cfg @@ -11,9 +11,9 @@ }, "Sync": { "FastSync": true, - "PivotNumber": 29760000, - "PivotHash": "0x863b083c2d9d8adaec5b0591e8c57e13e12c1675ad091cfe8cad589c993f55ed", - "PivotTotalDifficulty": "10126803239567128672670028317169421972557970934", + "PivotNumber": 30320000, + "PivotHash": "0x7e06ec6e44fa1a5fdbc25aa33758451465083ef73e8935d7174d2aa3122578a7", + "PivotTotalDifficulty": "10317361365042854212209518097331212170972166406", "FastBlocks": true, "UseGethLimitsInFastBlocks": false, "FastSyncCatchUpHeightDelta": 10000000000 diff --git a/src/Nethermind/Nethermind.Runner/configs/exosama.cfg b/src/Nethermind/Nethermind.Runner/configs/exosama.cfg index 8caac79e843..782b1cb621d 100644 --- a/src/Nethermind/Nethermind.Runner/configs/exosama.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/exosama.cfg @@ -11,9 +11,9 @@ }, "Sync": { "FastSync": true, - "PivotNumber": 9680000, - "PivotHash": "0xd8ffbe65bc320f6d874b2b08ff578d00f9d8bca29d6dbb0d892a21466d548c2d", - "PivotTotalDifficulty": "3293933311794684326325466199939516286541573286", + "PivotNumber": 10290000, + "PivotHash": "0x55a1b682599b65ba2712393c130e25a3992ac32cf278ce6785d772067f695657", + "PivotTotalDifficulty": "3501505555616456789038124710472894895528513286", "FastBlocks": true, "UseGethLimitsInFastBlocks": false, "FastSyncCatchUpHeightDelta": 10000000000 diff --git a/src/Nethermind/Nethermind.Runner/configs/gnosis.cfg b/src/Nethermind/Nethermind.Runner/configs/gnosis.cfg index 52b382e60f9..aae981b34a7 100644 --- a/src/Nethermind/Nethermind.Runner/configs/gnosis.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/gnosis.cfg @@ -12,8 +12,8 @@ }, "Sync": { "FastSync": true, - "PivotNumber": 33770000, - "PivotHash": "0xf164c80064fdd455a5ead5c5b72b8e926c174ccca7794559309bc4984a2abed9", + "PivotNumber": 34350000, + "PivotHash": "0x456b1aa69a62a859343be8f1a9b607d482a58d4e80d82cb7c53f24bcc5b352e9", "PivotTotalDifficulty": "8626000110427538733349499292577475819600160930", "FastBlocks": true, "UseGethLimitsInFastBlocks": false, diff --git a/src/Nethermind/Nethermind.Runner/configs/joc-mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/joc-mainnet.cfg index 9d21954c287..c9b1a620d9f 100644 --- a/src/Nethermind/Nethermind.Runner/configs/joc-mainnet.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/joc-mainnet.cfg @@ -12,9 +12,9 @@ "FastSync": true, "SnapSync": true, "FastBlocks": true, - "PivotNumber": 11050000, - "PivotHash": "0x1ebcdd7500ccca9b887bf6b39679b6ec15d7c7f47c03517b8a7b57aee29f3d09", - "PivotTotalDifficulty": "22076851" + "PivotNumber": 11650000, + "PivotHash": "0x7a9089d0e6a3f43b5b0321f7aa4da8ec37e4bc1220d1b7cb5b22dd058fa2dd03", + "PivotTotalDifficulty": "23276561" }, "Metrics": { "NodeName": "JOC-Mainnet" diff --git a/src/Nethermind/Nethermind.Runner/configs/joc-testnet.cfg b/src/Nethermind/Nethermind.Runner/configs/joc-testnet.cfg index 47d8d3f3037..d13d50c4a2f 100644 --- a/src/Nethermind/Nethermind.Runner/configs/joc-testnet.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/joc-testnet.cfg @@ -11,9 +11,9 @@ "Sync": { "FastSync": true, "SnapSync": true, - "PivotNumber": 4650000, - "PivotHash": "0xed3919f210421fd7e2679210f0d99aca5ba5f9528e4b82f6cb0fba0ece51cd51", - "PivotTotalDifficulty": "9299443" + "PivotNumber": 5260000, + "PivotHash": "0xfeedf52b4e93509a31e933f04b5115d1395ebe46cde5105bff44cd7ce60e71b9", + "PivotTotalDifficulty": "10295113" }, "Metrics": { "NodeName": "JOC-Testnet" diff --git a/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg index fdcf0daa35d..e33b0a33265 100644 --- a/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg @@ -9,8 +9,8 @@ "Sync": { "FastSync": true, "SnapSync": true, - "PivotNumber": 19799000, - "PivotHash": "0x62bf6817b7c009c7e26348d2b059c8972e424049888fe4534f5cad6dd4582fd2", + "PivotNumber": 20049000, + "PivotHash": "0x7e1be28e7effe28715bee8a1bc9991ea966979b84e62432a32c543dcf8e45624", "PivotTotalDifficulty": "58750003716598352816469", "FastBlocks": true, "FastSyncCatchUpHeightDelta": "10000000000" diff --git a/src/Nethermind/Nethermind.Runner/configs/op-mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/op-mainnet.cfg index abd95bf7a81..d8f2a8fcdae 100644 --- a/src/Nethermind/Nethermind.Runner/configs/op-mainnet.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/op-mainnet.cfg @@ -11,19 +11,41 @@ "BlobsSupport": "Disabled" }, "Sync": { - "NetworkingEnabled": false, + "FastSync": true, + "SnapSync": true, "AncientBodiesBarrier": 105235063, - "AncientReceiptsBarrier": 105235063 + "AncientReceiptsBarrier": 105235063, + "FastSyncCatchUpHeightDelta": "10000000000", + "PivotNumber": 119834660, + "PivotHash": "0xcdab6080b3846cf67b50935604deccbdf0e27434fa7b454ccf2195051ecd7a10", + "PivotTotalDifficulty": "0", + "MaxAttemptsToUpdatePivot": 0 + }, + "Discovery": { + "Discv5Enabled": true }, "JsonRpc": { "Enabled": true, "Port": 8545, "EnginePort": 8551 }, + "Pruning": { + "PruningBoundary": 1024 + }, + "Blocks": { + "SecondsPerSlot": 2, + "TargetBlockGasLimit": 30000000 + }, + "Merge": { + "Enabled": true + }, "Snapshot": { "Enabled": true, "DownloadUrl": "http://optimism-snapshot.nethermind.io/op-mainnet-genesis-v2.zip", "SnapshotFileName": "op-mainnet-genesis-v2.zip", "Checksum": "0x9b08eeb974c46f257c8bca812b119849590d92975ac86e4215d0d5647c26b147" + }, + "Optimism": { + "SequencerUrl": "https://mainnet-sequencer.optimism.io" } } diff --git a/src/Nethermind/Nethermind.Runner/configs/op-mainnet_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/op-mainnet_archive.cfg index 0d93e309fad..00a493c0081 100644 --- a/src/Nethermind/Nethermind.Runner/configs/op-mainnet_archive.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/op-mainnet_archive.cfg @@ -28,5 +28,8 @@ "DownloadUrl": "http://optimism-snapshot.nethermind.io/op-mainnet-genesis-v1.zip", "SnapshotFileName": "op-mainnet-genesis-v1.zip", "Checksum": "0xd7e15b26175c4c924acf75c5790e75d5eaa044977ca8e1904dc62d5d0769eba3" + }, + "Optimism": { + "SequencerUrl": "https://mainnet-sequencer.optimism.io" } } diff --git a/src/Nethermind/Nethermind.Runner/configs/op-sepolia.cfg b/src/Nethermind/Nethermind.Runner/configs/op-sepolia.cfg index cb2be7b62f2..e2fc13f67b8 100644 --- a/src/Nethermind/Nethermind.Runner/configs/op-sepolia.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/op-sepolia.cfg @@ -10,11 +10,33 @@ "BlobsSupport": "Disabled" }, "Sync": { - "NetworkingEnabled": false + "FastSync": true, + "SnapSync": true, + "FastSyncCatchUpHeightDelta": "10000000000", + "PivotNumber": 11338054, + "PivotHash": "0xf4307e8fd758db460d57b87bd910fa932feb928932c99faa85edf5a588ede049", + "PivotTotalDifficulty": "0", + "MaxAttemptsToUpdatePivot": 0 + }, + "Discovery": { + "Discv5Enabled": true }, "JsonRpc": { "Enabled": true, "Port": 8545, "EnginePort": 8551 + }, + "Pruning": { + "PruningBoundary": 1024 + }, + "Blocks": { + "SecondsPerSlot": 2, + "TargetBlockGasLimit": 30000000 + }, + "Merge": { + "Enabled": true + }, + "Optimism": { + "SequencerUrl": "https://sepolia-sequencer.optimism.io" } } diff --git a/src/Nethermind/Nethermind.Runner/configs/op-sepolia_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/op-sepolia_archive.cfg index 9d8da6803b2..47240200d7a 100644 --- a/src/Nethermind/Nethermind.Runner/configs/op-sepolia_archive.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/op-sepolia_archive.cfg @@ -19,5 +19,8 @@ "Enabled": true, "Port": 8545, "EnginePort": 8551 + }, + "Optimism": { + "SequencerUrl": "https://sepolia-sequencer.optimism.io" } } diff --git a/src/Nethermind/Nethermind.Runner/configs/sepolia.cfg b/src/Nethermind/Nethermind.Runner/configs/sepolia.cfg index 3786e2e498d..e9aff51d60b 100644 --- a/src/Nethermind/Nethermind.Runner/configs/sepolia.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/sepolia.cfg @@ -17,8 +17,8 @@ "FastSync": true, "SnapSync": true, "UseGethLimitsInFastBlocks": true, - "PivotNumber": 5837000, - "PivotHash": "0x1b2a4df93c9d2bd6b0482572250fbe6c199dec78fc09cb0d9fc61380bd66d92c", + "PivotNumber": 6067000, + "PivotHash": "0xec18eaec616ea4cec9cd09ab19eee43cfea9fe03addcae2a8eb4868b0fe64c36", "PivotTotalDifficulty": "17000018015853232", "FastSyncCatchUpHeightDelta": "10000000000" }, diff --git a/src/Nethermind/Nethermind.Runner/configs/volta.cfg b/src/Nethermind/Nethermind.Runner/configs/volta.cfg index 3c636c5b61e..837c380940e 100644 --- a/src/Nethermind/Nethermind.Runner/configs/volta.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/volta.cfg @@ -15,9 +15,9 @@ }, "Sync": { "FastSync": true, - "PivotNumber": 27630000, - "PivotHash": "0xc9a3f35992dfd2bc4f27066a4b4b753a90276bc6d75f10a5010316c8cb7c50c8", - "PivotTotalDifficulty": "9402001798025529745493040403339755682158823836", + "PivotNumber": 28070000, + "PivotHash": "0x40256b527c12040e809347751dfa67d896f292a123e912d85dc4d6039f778b33", + "PivotTotalDifficulty": "9551726039470742669416925230609733695198421023", "UseGethLimitsInFastBlocks": false, "FastSyncCatchUpHeightDelta": 10000000000 }, diff --git a/src/Nethermind/Nethermind.Serialization.Json/StreamPipeWriter.cs b/src/Nethermind/Nethermind.Serialization.Json/StreamPipeWriter.cs index da1ecd2a3fb..89f802e849f 100644 --- a/src/Nethermind/Nethermind.Serialization.Json/StreamPipeWriter.cs +++ b/src/Nethermind/Nethermind.Serialization.Json/StreamPipeWriter.cs @@ -18,7 +18,8 @@ namespace Nethermind.Serialization.Json; public interface ICountingBufferWriter : IBufferWriter { - public long WrittenCount { get; } + long WrittenCount { get; } + ValueTask CompleteAsync(Exception? exception = null); } public sealed class CountingPipeWriter : ICountingBufferWriter @@ -42,6 +43,12 @@ public void Advance(int count) public Memory GetMemory(int sizeHint = 0) => _writer.GetMemory(sizeHint); public Span GetSpan(int sizeHint = 0) => _writer.GetSpan(sizeHint); + + public ValueTask CompleteAsync(Exception? exception = null) + { + return _writer.CompleteAsync(); + } + } public sealed class CountingStreamPipeWriter : PipeWriter, ICountingBufferWriter @@ -84,11 +91,6 @@ public CountingStreamPipeWriter(Stream writingStream, StreamPipeWriterOptions? o { ThrowHelper.ThrowArgumentNullException_WritingStream(); } - if (options is null) - { - ThrowHelper.ThrowArgumentNullException_Options(); - } - InnerStream = writingStream; _minimumBufferSize = options?.MinimumBufferSize ?? 4096; _pool = options?.Pool == MemoryPool.Shared ? null : options?.Pool; diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/CompactReceiptStorageDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/CompactReceiptStorageDecoder.cs index 25e8ff25576..aa2ddcca370 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/CompactReceiptStorageDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/CompactReceiptStorageDecoder.cs @@ -6,12 +6,13 @@ using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using static Nethermind.Serialization.Rlp.Rlp; #pragma warning disable 618 namespace Nethermind.Serialization.Rlp { - [Rlp.SkipGlobalRegistration] + [Decoder(RlpDecoderKey.Storage)] public class CompactReceiptStorageDecoder : IRlpStreamDecoder, IRlpValueDecoder, IRlpObjectDecoder, IReceiptRefDecoder { public static readonly CompactReceiptStorageDecoder Instance = new(); diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptArrayStorageDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptArrayStorageDecoder.cs index 731ed2c33a7..f2d7c8051e1 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptArrayStorageDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptArrayStorageDecoder.cs @@ -8,19 +8,17 @@ namespace Nethermind.Serialization.Rlp; [Rlp.SkipGlobalRegistration] -public class ReceiptArrayStorageDecoder : IRlpStreamDecoder +public class ReceiptArrayStorageDecoder(bool compactEncoding = true) : IRlpStreamDecoder { public static readonly ReceiptArrayStorageDecoder Instance = new(); - private readonly ReceiptStorageDecoder StorageDecoder = ReceiptStorageDecoder.Instance; - private readonly CompactReceiptStorageDecoder CompactReceiptStorageDecoder = CompactReceiptStorageDecoder.Instance; - public const int CompactEncoding = 127; - private readonly bool _useCompactEncoding = true; + private static readonly IRlpStreamDecoder Decoder = Rlp.GetStreamDecoder(RlpDecoderKey.LegacyStorage); + private static readonly IRlpValueDecoder ValueDecoder = Rlp.GetValueDecoder(RlpDecoderKey.LegacyStorage); - public ReceiptArrayStorageDecoder(bool compactEncoding = true) - { - _useCompactEncoding = compactEncoding; - } + private static readonly IRlpStreamDecoder CompactDecoder = Rlp.GetStreamDecoder(RlpDecoderKey.Storage); + private static readonly IRlpValueDecoder CompactValueDecoder = Rlp.GetValueDecoder(RlpDecoderKey.Storage); + + public const int CompactEncoding = 127; public int GetLength(TxReceipt[] items, RlpBehaviors rlpBehaviors) { @@ -30,7 +28,7 @@ public int GetLength(TxReceipt[] items, RlpBehaviors rlpBehaviors) } int bufferLength = Rlp.LengthOfSequence(GetContentLength(items, rlpBehaviors)); - if (_useCompactEncoding && (rlpBehaviors & RlpBehaviors.Storage) != 0) + if (compactEncoding && (rlpBehaviors & RlpBehaviors.Storage) != 0) { bufferLength++; } @@ -39,12 +37,12 @@ public int GetLength(TxReceipt[] items, RlpBehaviors rlpBehaviors) private int GetContentLength(TxReceipt[] items, RlpBehaviors rlpBehaviors) { - if (_useCompactEncoding && (rlpBehaviors & RlpBehaviors.Storage) != 0) + if (compactEncoding && (rlpBehaviors & RlpBehaviors.Storage) != 0) { int totalLength = 0; for (int i = 0; i < items.Length; i++) { - totalLength += CompactReceiptStorageDecoder.GetLength(items[i], rlpBehaviors); + totalLength += CompactDecoder.GetLength(items[i], rlpBehaviors); } return totalLength; @@ -54,7 +52,7 @@ private int GetContentLength(TxReceipt[] items, RlpBehaviors rlpBehaviors) int totalLength = 0; for (int i = 0; i < items.Length; i++) { - totalLength += StorageDecoder.GetLength(items[i], rlpBehaviors); + totalLength += Decoder.GetLength(items[i], rlpBehaviors); } return totalLength; @@ -66,11 +64,11 @@ public TxReceipt[] Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBe if (rlpStream.PeekByte() == CompactEncoding) { rlpStream.ReadByte(); - return CompactReceiptStorageDecoder.DecodeArray(rlpStream, RlpBehaviors.Storage); + return CompactDecoder.DecodeArray(rlpStream, RlpBehaviors.Storage); } else { - return StorageDecoder.DecodeArray(rlpStream, RlpBehaviors.Storage); + return Decoder.DecodeArray(rlpStream, RlpBehaviors.Storage); } } @@ -82,7 +80,7 @@ public void Encode(RlpStream stream, TxReceipt[] items, RlpBehaviors rlpBehavior return; } - if (_useCompactEncoding && (rlpBehaviors & RlpBehaviors.Storage) != 0) + if (compactEncoding && (rlpBehaviors & RlpBehaviors.Storage) != 0) { int totalLength = GetContentLength(items, rlpBehaviors); stream.WriteByte(CompactEncoding); @@ -90,7 +88,7 @@ public void Encode(RlpStream stream, TxReceipt[] items, RlpBehaviors rlpBehavior for (int i = 0; i < items.Length; i++) { - CompactReceiptStorageDecoder.Encode(stream, items[i], rlpBehaviors); + CompactDecoder.Encode(stream, items[i], rlpBehaviors); } } else @@ -100,7 +98,7 @@ public void Encode(RlpStream stream, TxReceipt[] items, RlpBehaviors rlpBehavior for (int i = 0; i < items.Length; i++) { - StorageDecoder.Encode(stream, items[i], rlpBehaviors); + Decoder.Encode(stream, items[i], rlpBehaviors); } } } @@ -115,19 +113,19 @@ public TxReceipt[] Decode(in Span receiptsData) if (receiptsData.Length > 0 && receiptsData[0] == CompactEncoding) { var decoderContext = new Rlp.ValueDecoderContext(receiptsData[1..]); - return CompactReceiptStorageDecoder.DecodeArray(ref decoderContext, RlpBehaviors.Storage); + return CompactValueDecoder.DecodeArray(ref decoderContext, RlpBehaviors.Storage); } else { var decoderContext = new Rlp.ValueDecoderContext(receiptsData); try { - return StorageDecoder.DecodeArray(ref decoderContext, RlpBehaviors.Storage); + return ValueDecoder.DecodeArray(ref decoderContext, RlpBehaviors.Storage); } catch (RlpException) { decoderContext.Position = 0; - return StorageDecoder.DecodeArray(ref decoderContext); + return ValueDecoder.DecodeArray(ref decoderContext); } } } @@ -137,14 +135,14 @@ public TxReceipt DeserializeReceiptObsolete(Hash256 hash, Span receiptData var context = new Rlp.ValueDecoderContext(receiptData); try { - var receipt = StorageDecoder.Decode(ref context, RlpBehaviors.Storage); + var receipt = ValueDecoder.Decode(ref context, RlpBehaviors.Storage); receipt.TxHash = hash; return receipt; } catch (RlpException) { context.Position = 0; - var receipt = StorageDecoder.Decode(ref context); + var receipt = ValueDecoder.Decode(ref context); receipt.TxHash = hash; return receipt; } @@ -155,13 +153,13 @@ public static bool IsCompactEncoding(Span receiptsData) return receiptsData.Length > 0 && receiptsData[0] == CompactEncoding; } - public static IReceiptRefDecoder GetRefDecoder(Span receiptsData) + public IReceiptRefDecoder GetRefDecoder(Span receiptsData) { if (IsCompactEncoding(receiptsData)) { - return CompactReceiptStorageDecoder.Instance; + return (IReceiptRefDecoder)CompactValueDecoder; } - return ReceiptStorageDecoder.Instance; + return (IReceiptRefDecoder)ValueDecoder; } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptMessageDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptMessageDecoder.cs index 2453a91d1db..9cba7dab049 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptMessageDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptMessageDecoder.cs @@ -7,15 +7,10 @@ namespace Nethermind.Serialization.Rlp { + [Rlp.Decoder(RlpDecoderKey.Default)] + [Rlp.Decoder(RlpDecoderKey.Trie)] public class ReceiptMessageDecoder : IRlpStreamDecoder { - public static readonly ReceiptMessageDecoder Instance = new(); - - static ReceiptMessageDecoder() - { - Rlp.RegisterDecoder(typeof(TxReceipt), new ReceiptMessageDecoder()); - } - public TxReceipt Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { if (rlpStream.IsNextItemNull()) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptStorageDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptStorageDecoder.cs index f9a9c4bfccd..6bd8a18c5aa 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptStorageDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptStorageDecoder.cs @@ -10,18 +10,22 @@ namespace Nethermind.Serialization.Rlp { + [Rlp.Decoder(RlpDecoderKey.LegacyStorage)] public class ReceiptStorageDecoder : IRlpStreamDecoder, IRlpValueDecoder, IRlpObjectDecoder, IReceiptRefDecoder { private readonly bool _supportTxHash; private const byte MarkTxHashByte = 255; - public static readonly ReceiptStorageDecoder Instance = new(); - public ReceiptStorageDecoder(bool supportTxHash = true) { _supportTxHash = supportTxHash; } + // Used by Rlp decoders discovery + public ReceiptStorageDecoder() : this(true) + { + } + public TxReceipt? Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { if (rlpStream.IsNextItemNull()) diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs index 923cb54f74f..9a55db5e4bd 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/Rlp.cs @@ -75,30 +75,18 @@ public Rlp(byte[] bytes) public int Length => Bytes.Length; - private static Dictionary _decoderBuilder = new(); - private static FrozenDictionary? _decoders; - public static FrozenDictionary Decoders => _decoders ??= _decoderBuilder.ToFrozenDictionary(); + private static Dictionary _decoderBuilder = new(); + private static FrozenDictionary? _decoders; + public static FrozenDictionary Decoders => _decoders ??= _decoderBuilder.ToFrozenDictionary(); - public struct TypeAsKey(Type key) : IEquatable + public static void RegisterDecoder(RlpDecoderKey key, IRlpDecoder decoder) { - private readonly Type _key = key; - public Type Value => _key; - - public static implicit operator Type(TypeAsKey key) => key._key; - public static implicit operator TypeAsKey(Type key) => new(key); - - public bool Equals(TypeAsKey other) => _key.Equals(other._key); - public override int GetHashCode() => _key.GetHashCode(); - } - - public static void RegisterDecoder(Type type, IRlpDecoder decoder) - { - _decoderBuilder[type] = decoder; + _decoderBuilder[key] = decoder; // Mark FrozenDictionary as null to force re-creation _decoders = null; } - public static void RegisterDecoders(Assembly assembly) + public static void RegisterDecoders(Assembly assembly, bool canOverrideExistingDecoders = false) { foreach (Type? type in assembly.GetExportedTypes()) { @@ -123,16 +111,41 @@ public static void RegisterDecoders(Assembly assembly) Type? interfaceGenericDefinition = implementedInterface.GetGenericTypeDefinition(); if (interfaceGenericDefinition == typeof(IRlpDecoder<>).GetGenericTypeDefinition()) { - ConstructorInfo? constructor = type.GetConstructor(Type.EmptyTypes); - if (constructor is null) + bool isSetForAnyAttribute = false; + IRlpDecoder? instance = null; + + foreach (DecoderAttribute rlpDecoderAttr in type.GetCustomAttributes()) { - continue; + RlpDecoderKey key = new(implementedInterface.GenericTypeArguments[0], rlpDecoderAttr.Key); + AddEncoder(key); + + isSetForAnyAttribute = true; } - Type key = implementedInterface.GenericTypeArguments[0]; - if (!_decoderBuilder.ContainsKey(key)) + if (!isSetForAnyAttribute) { - _decoderBuilder[key] = (IRlpDecoder)Activator.CreateInstance(type); + AddEncoder(new(implementedInterface.GenericTypeArguments[0])); + } + + void AddEncoder(RlpDecoderKey key) + { + if (!_decoderBuilder.ContainsKey(key) || canOverrideExistingDecoders) + { + try + { + _decoderBuilder[key] = instance ??= (IRlpDecoder)(type.GetConstructor(Type.EmptyTypes) is not null ? + Activator.CreateInstance(type) : + Activator.CreateInstance(type, BindingFlags.CreateInstance | BindingFlags.OptionalParamBinding, null, [Type.Missing], null)); + } + catch (Exception) + { + throw new ArgumentException($"Unable to set decoder for {key}, because {type} decoder has no suitable constructor."); + } + } + else + { + throw new InvalidOperationException($"Unable to override decoder for {key}, because the following decoder is already set: {_decoderBuilder[key]}."); + } } } } @@ -234,10 +247,9 @@ internal static ArrayPoolList ByteSpanToArrayPool(ReadOnlySpan span) return span.ToPooledList(); } - public static IRlpValueDecoder? GetValueDecoder() => Decoders.TryGetValue(typeof(T), out IRlpDecoder value) ? value as IRlpValueDecoder : null; - public static IRlpStreamDecoder? GetStreamDecoder() => Decoders.TryGetValue(typeof(T), out IRlpDecoder value) ? value as IRlpStreamDecoder : null; - public static IRlpObjectDecoder? GetObjectDecoder() => Decoders.TryGetValue(typeof(T), out IRlpDecoder value) ? value as IRlpObjectDecoder : null; - public static IRlpDecoder? GetDecoder() => Decoders.TryGetValue(typeof(T), out IRlpDecoder value) ? value as IRlpDecoder : null; + public static IRlpValueDecoder? GetValueDecoder(string key = RlpDecoderKey.Default) => Decoders.TryGetValue(new(typeof(T), key), out IRlpDecoder value) ? value as IRlpValueDecoder : null; + public static IRlpStreamDecoder? GetStreamDecoder(string key = RlpDecoderKey.Default) => Decoders.TryGetValue(new(typeof(T), key), out IRlpDecoder value) ? value as IRlpStreamDecoder : null; + public static IRlpObjectDecoder GetObjectDecoder(string key = RlpDecoderKey.Default) => Decoders.GetValueOrDefault(new(typeof(T), key)) as IRlpObjectDecoder ?? throw new RlpException($"{nameof(Rlp)} does not support encoding {typeof(T).Name}"); public static T Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { @@ -284,8 +296,7 @@ public static Rlp Encode(T item, RlpBehaviors behaviors = RlpBehaviors.None) return new Rlp(stream.Data.ToArray()); } - IRlpObjectDecoder? rlpDecoder = GetObjectDecoder(); - return rlpDecoder is not null ? rlpDecoder.Encode(item, behaviors) : throw new RlpException($"{nameof(Rlp)} does not support encoding {typeof(T).Name}"); + return GetObjectDecoder().Encode(item, behaviors); } public static Rlp Encode(T[]? items, RlpBehaviors behaviors = RlpBehaviors.None) @@ -304,13 +315,7 @@ public static Rlp Encode(T[]? items, RlpBehaviors behaviors = RlpBehaviors.No return new Rlp(stream.Data.ToArray()); } - IRlpObjectDecoder rlpDecoder = GetObjectDecoder(); - if (rlpDecoder is not null) - { - return rlpDecoder.Encode(items, behaviors); - } - - throw new RlpException($"{nameof(Rlp)} does not support encoding {typeof(T).Name}"); + return GetObjectDecoder().Encode(items, behaviors); } public static Rlp Encode(int[] integers) @@ -422,8 +427,8 @@ public static int Encode(Span buffer, int position, Hash256 hash) ThrowArgumentOutOfRangeException(); } - Unsafe.Add(ref MemoryMarshal.GetReference(buffer), position) = 160; - Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(buffer), position + 1)) = hash.ValueHash256; + Unsafe.Add(ref MemoryMarshal.GetReference(buffer), (nuint)position) = 160; + Unsafe.As(ref Unsafe.Add(ref MemoryMarshal.GetReference(buffer), (nuint)position + 1)) = hash.ValueHash256; return newPosition; [DoesNotReturn] @@ -1867,8 +1872,50 @@ public static int LengthOf(BlockInfo item) return BlockInfoDecoder.Instance.GetLength(item, RlpBehaviors.None); } + [AttributeUsage(AttributeTargets.Class)] public class SkipGlobalRegistration : Attribute { } + + /// + /// Optional attribute for RLP decoders. + /// + /// Optional custom key that helps to have more than one decoder for the given type. + [AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = false)] + public sealed class DecoderAttribute(string key = RlpDecoderKey.Default) : Attribute + { + public string Key { get; } = key; + } + } + + public readonly struct RlpDecoderKey(Type type, string key = RlpDecoderKey.Default) : IEquatable + { + public const string Default = "default"; + public const string Storage = "storage"; + public const string LegacyStorage = "legacy-storage"; + public const string Trie = "trie"; + + private readonly Type _type = type; + private readonly string _key = key; + public Type Type => _type; + public string Key => _key; + + public static implicit operator Type(RlpDecoderKey key) => key._type; + public static implicit operator RlpDecoderKey(Type key) => new(key); + + public bool Equals(RlpDecoderKey other) => _type.Equals(other._type) && _key.Equals(other._key); + + public override int GetHashCode() => HashCode.Combine(_type, _key); + + public override bool Equals(object obj) => obj is RlpDecoderKey key && Equals(key); + + public static bool operator ==(RlpDecoderKey left, RlpDecoderKey right) => left.Equals(right); + + public static bool operator !=(RlpDecoderKey left, RlpDecoderKey right) => !(left == right); + + public override string ToString() + { + return $"({Type.Name},{Key})"; + } } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs b/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs index 23bea01afa1..d1bdeddd804 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs @@ -26,7 +26,6 @@ public class RlpStream private static readonly BlockDecoder _blockDecoder = new(); private static readonly BlockInfoDecoder _blockInfoDecoder = new(); private static readonly TxDecoder _txDecoder = new(); - private static readonly ReceiptMessageDecoder _receiptDecoder = new(); private static readonly WithdrawalDecoder _withdrawalDecoder = new(); private static readonly ConsensusRequestDecoder _requestsDecoder = new(); private static readonly LogEntryDecoder _logEntryDecoder = LogEntryDecoder.Instance; @@ -72,11 +71,6 @@ public void Encode(Transaction value, RlpBehaviors rlpBehaviors = RlpBehaviors.N _txDecoder.Encode(this, value, rlpBehaviors); } - public void Encode(TxReceipt value) - { - _receiptDecoder.Encode(this, value); - } - public void Encode(Withdrawal value) => _withdrawalDecoder.Encode(this, value); public void Encode(ConsensusRequest value) => _requestsDecoder.Encode(this, value); diff --git a/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs b/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs index dc212ca3b55..5948b1b57ba 100644 --- a/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs +++ b/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs @@ -776,7 +776,7 @@ void TestTransitions(ForkActivation activation, Action changes) TestTransitions((ForkActivation)20280L, r => { r.IsEip2028Enabled = true; }); TestTransitions((ForkActivation)22000L, r => { r.IsEip2200Enabled = true; }); TestTransitions((ForkActivation)23000L, r => { r.IsEip1283Enabled = r.IsEip1344Enabled = true; }); - TestTransitions((ForkActivation)24000L, r => { r.IsEip2315Enabled = r.ValidateChainId = r.ValidateReceipts = true; }); + TestTransitions((ForkActivation)24000L, r => { r.ValidateChainId = r.ValidateReceipts = true; }); TestTransitions((ForkActivation)29290L, r => { r.IsEip2929Enabled = r.IsEip2565Enabled = true; }); TestTransitions((ForkActivation)29300L, r => { r.IsEip2930Enabled = true; }); TestTransitions((ForkActivation)31980L, r => { r.IsEip3198Enabled = true; }); diff --git a/src/Nethermind/Nethermind.Specs.Test/GoerliSpecProviderTests.cs b/src/Nethermind/Nethermind.Specs.Test/GoerliSpecProviderTests.cs index 29af41d1b07..31414ad9405 100644 --- a/src/Nethermind/Nethermind.Specs.Test/GoerliSpecProviderTests.cs +++ b/src/Nethermind/Nethermind.Specs.Test/GoerliSpecProviderTests.cs @@ -16,7 +16,6 @@ public class GoerliSpecProviderTests [TestCase(4_460_644, true)] public void Berlin_eips(long blockNumber, bool isEnabled) { - _specProvider.GetSpec((ForkActivation)blockNumber).IsEip2315Enabled.Should().Be(false); _specProvider.GetSpec((ForkActivation)blockNumber).IsEip2537Enabled.Should().Be(false); _specProvider.GetSpec((ForkActivation)blockNumber).IsEip2565Enabled.Should().Be(isEnabled); _specProvider.GetSpec((ForkActivation)blockNumber).IsEip2929Enabled.Should().Be(isEnabled); diff --git a/src/Nethermind/Nethermind.Specs.Test/MainnetSpecProviderTests.cs b/src/Nethermind/Nethermind.Specs.Test/MainnetSpecProviderTests.cs index 9809cdb69fa..20b6fbb1f51 100644 --- a/src/Nethermind/Nethermind.Specs.Test/MainnetSpecProviderTests.cs +++ b/src/Nethermind/Nethermind.Specs.Test/MainnetSpecProviderTests.cs @@ -17,7 +17,6 @@ public class MainnetSpecProviderTests [TestCase(12_244_000, true)] public void Berlin_eips(long blockNumber, bool isEnabled) { - _specProvider.GetSpec((ForkActivation)blockNumber).IsEip2315Enabled.Should().Be(false); _specProvider.GetSpec((ForkActivation)blockNumber).IsEip2537Enabled.Should().Be(false); _specProvider.GetSpec((ForkActivation)blockNumber).IsEip2565Enabled.Should().Be(isEnabled); _specProvider.GetSpec((ForkActivation)blockNumber).IsEip2929Enabled.Should().Be(isEnabled); diff --git a/src/Nethermind/Nethermind.Specs.Test/OverridableReleaseSpec.cs b/src/Nethermind/Nethermind.Specs.Test/OverridableReleaseSpec.cs index ada18d648c4..8568c1f16b9 100644 --- a/src/Nethermind/Nethermind.Specs.Test/OverridableReleaseSpec.cs +++ b/src/Nethermind/Nethermind.Specs.Test/OverridableReleaseSpec.cs @@ -97,9 +97,7 @@ public OverridableReleaseSpec(IReleaseSpec spec) public bool IsEip2200Enabled => _spec.IsEip2200Enabled; - public bool IsEip2315Enabled => _spec.IsEip2315Enabled; - - public bool IsEip2537Enabled => _spec.IsEip2315Enabled; + public bool IsEip2537Enabled => _spec.IsEip2537Enabled; public bool IsEip2565Enabled => _spec.IsEip2565Enabled; diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecBasedSpecProvider.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecBasedSpecProvider.cs index 9698f7f594b..52b8ba84dd3 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecBasedSpecProvider.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecBasedSpecProvider.cs @@ -195,7 +195,6 @@ private static ReleaseSpec CreateReleaseSpec(ChainSpec chainSpec, long releaseSt releaseSpec.IsEip2200Enabled = (chainSpec.Parameters.Eip2200Transition ?? long.MaxValue) <= releaseStartBlock || (chainSpec.Parameters.Eip1706Transition ?? long.MaxValue) <= releaseStartBlock && releaseSpec.IsEip1283Enabled; releaseSpec.IsEip1559Enabled = (chainSpec.Parameters.Eip1559Transition ?? long.MaxValue) <= releaseStartBlock; releaseSpec.Eip1559TransitionBlock = chainSpec.Parameters.Eip1559Transition ?? long.MaxValue; - releaseSpec.IsEip2315Enabled = (chainSpec.Parameters.Eip2315Transition ?? long.MaxValue) <= releaseStartBlock; releaseSpec.IsEip2537Enabled = (chainSpec.Parameters.Eip2537Transition ?? long.MaxValue) <= releaseStartBlock || (chainSpec.Parameters.Eip2537TransitionTimestamp ?? ulong.MaxValue) <= releaseStartTimestamp; releaseSpec.IsEip2565Enabled = (chainSpec.Parameters.Eip2565Transition ?? long.MaxValue) <= releaseStartBlock; diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecLoader.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecLoader.cs index 734dcac3729..12eefe04059 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecLoader.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecLoader.cs @@ -343,6 +343,9 @@ static AuRaParameters.Validator LoadValidator(ChainSpecJson.AuRaValidatorJson va RegolithTimestamp = chainSpecJson.Engine.Optimism.RegolithTimestamp, BedrockBlockNumber = chainSpecJson.Engine.Optimism.BedrockBlockNumber, CanyonTimestamp = chainSpecJson.Engine.Optimism.CanyonTimestamp, + EcotoneTimestamp = chainSpecJson.Engine.Optimism.EcotoneTimestamp, + FjordTimestamp = chainSpecJson.Engine.Optimism.FjordTimestamp, + L1FeeRecipient = chainSpecJson.Engine.Optimism.L1FeeRecipient, L1BlockAddress = chainSpecJson.Engine.Optimism.L1BlockAddress, CanyonBaseFeeChangeDenominator = chainSpecJson.Engine.Optimism.CanyonBaseFeeChangeDenominator, diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecJson.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecJson.cs index d819732c34f..4600e1f9777 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecJson.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecJson.cs @@ -173,6 +173,8 @@ internal class OptimismEngineJson public ulong RegolithTimestamp => Params.RegolithTimestamp; public long BedrockBlockNumber => Params.BedrockBlockNumber; public ulong? CanyonTimestamp => Params.CanyonTimestamp; + public ulong? EcotoneTimestamp => Params.EcotoneTimestamp; + public ulong? FjordTimestamp => Params.FjordTimestamp; public Address L1FeeRecipient => Params.L1FeeRecipient; public Address L1BlockAddress => Params.L1BlockAddress; public UInt256 CanyonBaseFeeChangeDenominator => Params.CanyonBaseFeeChangeDenominator; @@ -186,6 +188,8 @@ internal class OptimismEngineParamsJson public ulong RegolithTimestamp { get; set; } public long BedrockBlockNumber { get; set; } public ulong? CanyonTimestamp { get; set; } + public ulong? EcotoneTimestamp { get; set; } + public ulong? FjordTimestamp { get; set; } public Address L1FeeRecipient { get; set; } public Address L1BlockAddress { get; set; } public UInt256 CanyonBaseFeeChangeDenominator { get; set; } diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/OptimismParameters.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/OptimismParameters.cs index b2439224257..b457040ebcf 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/OptimismParameters.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/OptimismParameters.cs @@ -14,6 +14,10 @@ public class OptimismParameters public ulong? CanyonTimestamp { get; set; } + public ulong? EcotoneTimestamp { get; set; } + + public ulong? FjordTimestamp { get; set; } + public Address L1FeeRecipient { get; set; } public Address L1BlockAddress { get; set; } diff --git a/src/Nethermind/Nethermind.Specs/ReleaseSpec.cs b/src/Nethermind/Nethermind.Specs/ReleaseSpec.cs index 923f995afab..66799e40167 100644 --- a/src/Nethermind/Nethermind.Specs/ReleaseSpec.cs +++ b/src/Nethermind/Nethermind.Specs/ReleaseSpec.cs @@ -47,7 +47,6 @@ public class ReleaseSpec : IReleaseSpec public bool IsEip1108Enabled { get; set; } public bool IsEip1884Enabled { get; set; } public bool IsEip2200Enabled { get; set; } - public bool IsEip2315Enabled { get; set; } public bool IsEip2537Enabled { get; set; } public bool IsEip2565Enabled { get; set; } public bool IsEip2929Enabled { get; set; } diff --git a/src/Nethermind/Nethermind.Specs/SystemTransactionReleaseSpec.cs b/src/Nethermind/Nethermind.Specs/SystemTransactionReleaseSpec.cs index d450e5621dd..c5785219823 100644 --- a/src/Nethermind/Nethermind.Specs/SystemTransactionReleaseSpec.cs +++ b/src/Nethermind/Nethermind.Specs/SystemTransactionReleaseSpec.cs @@ -93,9 +93,7 @@ public SystemTransactionReleaseSpec(IReleaseSpec spec) public bool IsEip2200Enabled => _spec.IsEip2200Enabled; - public bool IsEip2315Enabled => _spec.IsEip2315Enabled; - - public bool IsEip2537Enabled => _spec.IsEip2315Enabled; + public bool IsEip2537Enabled => _spec.IsEip2537Enabled; public bool IsEip2565Enabled => _spec.IsEip2565Enabled; diff --git a/src/Nethermind/Nethermind.State.Test/StateProviderTests.cs b/src/Nethermind/Nethermind.State.Test/StateProviderTests.cs index cb2b6dab056..8197dae5530 100644 --- a/src/Nethermind/Nethermind.State.Test/StateProviderTests.cs +++ b/src/Nethermind/Nethermind.State.Test/StateProviderTests.cs @@ -176,9 +176,9 @@ public void Keep_in_cache() [Test] public void Restore_in_the_middle() { - byte[] code = new byte[] { 1 }; + byte[] code = [1]; - WorldState provider = new(new TrieStore(new MemDb(), Logger), _codeDb, Logger); + IWorldState provider = new WorldState(new TrieStore(new MemDb(), Logger), _codeDb, Logger); provider.CreateAccount(_address1, 1); provider.AddToBalance(_address1, 1, Frontier.Instance); provider.IncrementNonce(_address1); diff --git a/src/Nethermind/Nethermind.State.Test/Witnesses/NullWitnessCollectorTests.cs b/src/Nethermind/Nethermind.State.Test/Witnesses/NullWitnessCollectorTests.cs deleted file mode 100644 index fb82cff588d..00000000000 --- a/src/Nethermind/Nethermind.State.Test/Witnesses/NullWitnessCollectorTests.cs +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using FluentAssertions; -using Nethermind.Core.Crypto; -using Nethermind.State; -using NUnit.Framework; - -namespace Nethermind.Store.Test.Witnesses -{ - [TestFixture, Parallelizable(ParallelScope.All)] - public class NullWitnessCollectorTests - { - [Test] - public void Cannot_call_add() - { - Assert.Throws( - () => NullWitnessCollector.Instance.Add(Keccak.Zero)); - } - - [Test] - public void Collected_is_empty() - { - NullWitnessCollector.Instance.Collected.Should().HaveCount(0); - } - - [Test] - public void Reset_does_nothing() - { - NullWitnessCollector.Instance.Reset(); - NullWitnessCollector.Instance.Reset(); - } - - [Test] - public void Persist_does_nothing() - { - NullWitnessCollector.Instance.Persist(Keccak.Zero); - } - - [Test] - public void Load_throws() - { - Assert.Throws( - () => NullWitnessCollector.Instance.Load(Keccak.Zero)); - } - } -} diff --git a/src/Nethermind/Nethermind.State.Test/Witnesses/WitnessCollectorTests.cs b/src/Nethermind/Nethermind.State.Test/Witnesses/WitnessCollectorTests.cs deleted file mode 100644 index a95eebe7ed4..00000000000 --- a/src/Nethermind/Nethermind.State.Test/Witnesses/WitnessCollectorTests.cs +++ /dev/null @@ -1,193 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using FluentAssertions; -using Nethermind.Core; -using Nethermind.Core.Crypto; -using Nethermind.Core.Test.Builders; -using Nethermind.Db; -using Nethermind.Logging; -using Nethermind.State.Witnesses; -using NUnit.Framework; - -namespace Nethermind.Store.Test.Witnesses -{ - [TestFixture, Parallelizable(ParallelScope.All)] - public class WitnessCollectorTests - { - [Test] - public void Collects_each_cache_once() - { - WitnessCollector witnessCollector = new(new MemDb(), LimboLogs.Instance); - - using IDisposable tracker = witnessCollector.TrackOnThisThread(); - witnessCollector.Add(Keccak.Zero); - witnessCollector.Add(Keccak.Zero); - - witnessCollector.Collected.Should().HaveCount(1); - } - - [Test] - public void Can_collect_many() - { - WitnessCollector witnessCollector = new(new MemDb(), LimboLogs.Instance); - - using IDisposable tracker = witnessCollector.TrackOnThisThread(); - witnessCollector.Add(TestItem.KeccakA); - witnessCollector.Add(TestItem.KeccakB); - - witnessCollector.Collected.Should().HaveCount(2); - } - - [Test] - public void Can_reset() - { - WitnessCollector witnessCollector = new(new MemDb(), LimboLogs.Instance); - - using IDisposable tracker = witnessCollector.TrackOnThisThread(); - witnessCollector.Add(TestItem.KeccakA); - witnessCollector.Add(TestItem.KeccakB); - witnessCollector.Reset(); - - witnessCollector.Collected.Should().HaveCount(0); - } - - [Test] - public void Can_collect_after_reset() - { - WitnessCollector witnessCollector = new(new MemDb(), LimboLogs.Instance); - - using IDisposable tracker = witnessCollector.TrackOnThisThread(); - witnessCollector.Add(TestItem.KeccakA); - witnessCollector.Add(TestItem.KeccakB); - witnessCollector.Reset(); - witnessCollector.Add(TestItem.KeccakC); - - witnessCollector.Collected.Should().HaveCount(1); - } - - [Test] - public void Collects_what_it_should_collect() - { - WitnessCollector witnessCollector = new(new MemDb(), LimboLogs.Instance); - - using IDisposable tracker = witnessCollector.TrackOnThisThread(); - witnessCollector.Add(TestItem.KeccakA); - witnessCollector.Add(TestItem.KeccakB); - - witnessCollector.Collected.Should().Contain(TestItem.KeccakA); - witnessCollector.Collected.Should().Contain(TestItem.KeccakB); - } - - [Test] - public void Can_reset_empty() - { - WitnessCollector witnessCollector = new(new MemDb(), LimboLogs.Instance); - - using IDisposable tracker = witnessCollector.TrackOnThisThread(); - witnessCollector.Reset(); - - witnessCollector.Collected.Should().HaveCount(0); - } - - [Test] - public void Can_reset_empty_many_times() - { - WitnessCollector witnessCollector = new(new MemDb(), LimboLogs.Instance); - - using IDisposable tracker = witnessCollector.TrackOnThisThread(); - witnessCollector.Reset(); - witnessCollector.Reset(); - witnessCollector.Reset(); - - witnessCollector.Collected.Should().HaveCount(0); - } - - [Test] - public void Can_reset_non_empty_many_times() - { - WitnessCollector witnessCollector = new(new MemDb(), LimboLogs.Instance); - - using IDisposable tracker = witnessCollector.TrackOnThisThread(); - witnessCollector.Add(TestItem.KeccakA); - witnessCollector.Reset(); - witnessCollector.Add(TestItem.KeccakA); - witnessCollector.Reset(); - witnessCollector.Add(TestItem.KeccakA); - witnessCollector.Reset(); - - witnessCollector.Collected.Should().HaveCount(0); - } - - [Test] - public void Can_persist_empty() - { - IKeyValueStore keyValueStore = new MemDb(); - - WitnessCollector witnessCollector = new(keyValueStore, LimboLogs.Instance); - witnessCollector.Persist(Keccak.Zero); - - var witness = keyValueStore[Keccak.Zero.Bytes]; - witness.Should().BeNull(); - } - - [Test] - public void Can_persist_more() - { - IKeyValueStore keyValueStore = new MemDb(); - WitnessCollector witnessCollector = new(keyValueStore, LimboLogs.Instance); - - using IDisposable tracker = witnessCollector.TrackOnThisThread(); - witnessCollector.Add(TestItem.KeccakA); - witnessCollector.Add(TestItem.KeccakB); - witnessCollector.Persist(Keccak.Zero); - - var witness = keyValueStore[Keccak.Zero.Bytes]; - witness.Length.Should().Be(64); - } - - [Test] - public void Can_persist_and_load() - { - IKeyValueStore keyValueStore = new MemDb(); - WitnessCollector witnessCollector = new(keyValueStore, LimboLogs.Instance); - - using IDisposable tracker = witnessCollector.TrackOnThisThread(); - witnessCollector.Add(TestItem.KeccakA); - witnessCollector.Add(TestItem.KeccakB); - witnessCollector.Persist(Keccak.Zero); - - var witness = witnessCollector.Load(Keccak.Zero); - witness.Should().HaveCount(2); - } - - [Test] - public void Can_load_missing() - { - IKeyValueStore keyValueStore = new MemDb(); - WitnessCollector witnessCollector = new(keyValueStore, LimboLogs.Instance); - var witness = witnessCollector.Load(Keccak.Zero); - witness.Should().BeNull(); - } - - [Test] - public void Can_read_beyond_cache() - { - IKeyValueStore keyValueStore = new MemDb(); - WitnessCollector witnessCollector = new(keyValueStore, LimboLogs.Instance); - - using IDisposable tracker = witnessCollector.TrackOnThisThread(); - for (int i = 0; i < 255; i++) - { - witnessCollector.Add(TestItem.Keccaks[i]); - witnessCollector.Persist(TestItem.Keccaks[i]); - } - - witnessCollector.Persist(TestItem.KeccakA); - witnessCollector.Persist(TestItem.KeccakB); - - witnessCollector.Load(TestItem.Keccaks[0]); - } - } -} diff --git a/src/Nethermind/Nethermind.State.Test/Witnesses/WitnessingStoreTests.cs b/src/Nethermind/Nethermind.State.Test/Witnesses/WitnessingStoreTests.cs deleted file mode 100644 index cb8c183e1f1..00000000000 --- a/src/Nethermind/Nethermind.State.Test/Witnesses/WitnessingStoreTests.cs +++ /dev/null @@ -1,130 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using FluentAssertions; -using Nethermind.Core.Extensions; -using Nethermind.Core.Test; -using Nethermind.Core.Test.Builders; -using Nethermind.Db; -using Nethermind.Logging; -using Nethermind.State; -using Nethermind.State.Witnesses; -using Nethermind.Trie; -using NUnit.Framework; - -namespace Nethermind.Store.Test.Witnesses -{ - [TestFixture, Parallelizable(ParallelScope.All)] - public class WitnessingStoreTests - { - [Test] - public void Collects_on_reads() - { - Context context = new(); - context.Wrapped.ReadFunc = (key) => Value1; - - using IDisposable tracker = context.WitnessCollector.TrackOnThisThread(); - _ = context.Database[Key1]; - - context.WitnessCollector.Collected.Should().HaveCount(1); - } - - [Test] - public void Does_not_collect_if_no_tracking() - { - Context context = new(); - context.Wrapped.ReadFunc = (key) => Value1; - _ = context.Database[Key1]; - context.WitnessCollector.Collected.Should().HaveCount(0); - } - - [Test] - public void Collects_on_reads_2() - { - Context context = new(); - context.Wrapped[Key1] = Value1; - context.Wrapped[Key2] = Value2; - context.Wrapped[Key3] = Value3; - - using IDisposable tracker = context.WitnessCollector.TrackOnThisThread(); - _ = context.Database[Key1]; - _ = context.Database[Key2]; - _ = context.Database[Key3]; - - context.WitnessCollector.Collected.Should().HaveCount(3); - - context.WitnessCollector.Reset(); - _ = context.Database[Key1]; - _ = context.Database[Key2]; - _ = context.Database[Key3]; - - context.WitnessCollector.Collected.Should().HaveCount(3); - } - - [Test] - public void Collects_on_reads_and_previously_populated() - { - Context context = new(); - - using IDisposable tracker = context.WitnessCollector.TrackOnThisThread(); - context.Database[Key1] = Value1; - context.Database[Key2] = Value1; - context.Database[Key3] = Value1; - context.WitnessCollector.Collected.Should().HaveCount(0); - _ = context.Database[Key1]; - _ = context.Database[Key2]; - _ = context.Database[Key3]; - - context.WitnessCollector.Collected.Should().HaveCount(3); - } - - [Test] - public void Does_not_collect_on_writes() - { - Context context = new(); - context.Database[Key1] = Value1; - context.WitnessCollector.Collected.Should().HaveCount(0); - } - - [TestCase(0)] - [TestCase(1)] - [TestCase(31)] - [TestCase(33)] - public void Only_works_with_32_bytes_keys(int keyLength) - { - Context context = new(); - context.Wrapped.ReadFunc = (key) => Bytes.Empty; - - Assert.Throws( - () => _ = context.Database[new byte[keyLength]]); - } - - private class Context - { - public TestMemDb Wrapped { get; } = new TestMemDb(); - - public WitnessingStore Database { get; } - - public IWitnessCollector WitnessCollector { get; } - - public Context() - { - WitnessCollector = new WitnessCollector(new MemDb(), LimboLogs.Instance); - Database = new WitnessingStore(Wrapped, WitnessCollector); - } - } - - private static readonly byte[] Key1 = TestItem.KeccakA.BytesToArray(); - - private static readonly byte[] Key2 = TestItem.KeccakB.BytesToArray(); - - private static readonly byte[] Key3 = TestItem.KeccakC.BytesToArray(); - - private static readonly byte[] Value1 = { 1 }; - - private static readonly byte[] Value2 = { 2 }; - - private static readonly byte[] Value3 = { 3 }; - } -} diff --git a/src/Nethermind/Nethermind.State/IPreBlockCaches.cs b/src/Nethermind/Nethermind.State/IPreBlockCaches.cs new file mode 100644 index 00000000000..0ab88f7acd9 --- /dev/null +++ b/src/Nethermind/Nethermind.State/IPreBlockCaches.cs @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.State; + +public interface IPreBlockCaches +{ + PreBlockCaches Caches { get; } +} diff --git a/src/Nethermind/Nethermind.State/IWitnessCollector.cs b/src/Nethermind/Nethermind.State/IWitnessCollector.cs deleted file mode 100644 index 91461853ced..00000000000 --- a/src/Nethermind/Nethermind.State/IWitnessCollector.cs +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using System.Collections.Generic; -using Nethermind.Core.Crypto; - -namespace Nethermind.State -{ - /// - /// Collects witnesses during block processing, allows to persist them - /// - public interface IWitnessCollector - { - IReadOnlyCollection Collected { get; } - - void Add(Hash256 hash); - - void Reset(); - - void Persist(Hash256 blockHash); - - IDisposable TrackOnThisThread(); - } -} diff --git a/src/Nethermind/Nethermind.State/IWitnessRepository.cs b/src/Nethermind/Nethermind.State/IWitnessRepository.cs deleted file mode 100644 index d7a2e166992..00000000000 --- a/src/Nethermind/Nethermind.State/IWitnessRepository.cs +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.Core.Crypto; - -namespace Nethermind.State -{ - /// - /// Allows to access persisted witnesses - /// - /// - /// Witnesses can be pruned (deleted) to decrease space that is used - /// - public interface IWitnessRepository - { - Hash256[]? Load(Hash256 blockHash); - - void Delete(Hash256 blockHash); - } -} diff --git a/src/Nethermind/Nethermind.State/IWorldState.cs b/src/Nethermind/Nethermind.State/IWorldState.cs index 7fa18dc0bc8..e90d806dc41 100644 --- a/src/Nethermind/Nethermind.State/IWorldState.cs +++ b/src/Nethermind/Nethermind.State/IWorldState.cs @@ -3,7 +3,9 @@ using System; using Nethermind.Core; +using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using Nethermind.Core.Eip2930; using Nethermind.Core.Specs; using Nethermind.Int256; using Nethermind.State.Tracing; @@ -53,7 +55,7 @@ public interface IWorldState : IJournal, IReadOnlyStateProvider /// /// Reset all storage /// - void Reset(); + void Reset(bool resizeCollections = false); /// /// Creates a restartable snapshot. @@ -67,7 +69,8 @@ public interface IWorldState : IJournal, IReadOnlyStateProvider Snapshot TakeSnapshot(bool newTransactionStart = false); Snapshot IJournal.TakeSnapshot() => TakeSnapshot(); - + void WarmUp(AccessList? accessList); + void WarmUp(Address address); /// /// Clear all storage at specified address /// @@ -93,21 +96,22 @@ public interface IWorldState : IJournal, IReadOnlyStateProvider void UpdateStorageRoot(Address address, Hash256 storageRoot); - void IncrementNonce(Address address); + void IncrementNonce(Address address, UInt256 delta); + + void DecrementNonce(Address address, UInt256 delta); + + void IncrementNonce(Address address) => IncrementNonce(address, UInt256.One); - void DecrementNonce(Address address); + void DecrementNonce(Address address) => DecrementNonce(address, UInt256.One); /* snapshots */ - void Commit(IReleaseSpec releaseSpec, bool isGenesis = false); + void Commit(IReleaseSpec releaseSpec, bool isGenesis = false, bool commitStorageRoots = true); - void Commit(IReleaseSpec releaseSpec, IWorldStateTracer? traver, bool isGenesis = false); + void Commit(IReleaseSpec releaseSpec, IWorldStateTracer? tracer, bool isGenesis = false, bool commitStorageRoots = true); void CommitTree(long blockNumber); + ArrayPoolList? GetAccountChanges(); - /// - /// For witness - /// - /// - void TouchCode(in ValueHash256 codeHash); + bool ClearCache() => false; } diff --git a/src/Nethermind/Nethermind.State/IWorldStateManager.cs b/src/Nethermind/Nethermind.State/IWorldStateManager.cs index 26a01d09f67..ef853ddbf51 100644 --- a/src/Nethermind/Nethermind.State/IWorldStateManager.cs +++ b/src/Nethermind/Nethermind.State/IWorldStateManager.cs @@ -10,12 +10,14 @@ public interface IWorldStateManager { IWorldState GlobalWorldState { get; } IStateReader GlobalStateReader { get; } + IReadOnlyTrieStore TrieStore { get; } /// /// Used by read only tasks that need to execute blocks. /// + /// Specify a world state to warm up by the returned world state. /// - IWorldState CreateResettableWorldState(); + IWorldState CreateResettableWorldState(IWorldState? forWarmup = null); event EventHandler? ReorgBoundaryReached; } diff --git a/src/Nethermind/Nethermind.State/NullWitnessCollector.cs b/src/Nethermind/Nethermind.State/NullWitnessCollector.cs deleted file mode 100644 index 9239c877a00..00000000000 --- a/src/Nethermind/Nethermind.State/NullWitnessCollector.cs +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using System.Collections.Generic; -using Nethermind.Core.Crypto; - -namespace Nethermind.State -{ - public class NullWitnessCollector : IWitnessCollector, IWitnessRepository - { - private NullWitnessCollector() { } - - public static NullWitnessCollector Instance { get; } = new(); - - public IReadOnlyCollection Collected => Array.Empty(); - - public void Add(Hash256 hash) - { - throw new InvalidOperationException( - $"{nameof(NullWitnessCollector)} is not expected to receive {nameof(Add)} calls."); - } - - public void Reset() { } - - public void Persist(Hash256 blockHash) { } - - class EmptyDisposable : IDisposable - { - public void Dispose() - { - } - } - - public IDisposable TrackOnThisThread() { return new EmptyDisposable(); } - - public Hash256[]? Load(Hash256 blockHash) - { - throw new InvalidOperationException( - $"{nameof(NullWitnessCollector)} is not expected to receive {nameof(Load)} calls."); - } - - public void Delete(Hash256 blockHash) - { - throw new InvalidOperationException( - $"{nameof(NullWitnessCollector)} is not expected to receive {nameof(Delete)} calls."); - } - } -} diff --git a/src/Nethermind/Nethermind.State/OverlayWorldStateManager.cs b/src/Nethermind/Nethermind.State/OverlayWorldStateManager.cs new file mode 100644 index 00000000000..a4dda8fdd65 --- /dev/null +++ b/src/Nethermind/Nethermind.State/OverlayWorldStateManager.cs @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.Trie; +using Nethermind.Trie.Pruning; + +namespace Nethermind.State; + +public class OverlayWorldStateManager( + IReadOnlyDbProvider dbProvider, + OverlayTrieStore overlayTrieStore, + ILogManager? logManager) + : IWorldStateManager +{ + private readonly IDb _codeDb = dbProvider.GetDb(DbNames.Code); + + private readonly StateReader _reader = new(overlayTrieStore, dbProvider.GetDb(DbNames.Code), logManager); + + private readonly WorldState _state = new(overlayTrieStore, dbProvider.GetDb(DbNames.Code), logManager); + + public IWorldState GlobalWorldState => _state; + + public IStateReader GlobalStateReader => _reader; + + public IReadOnlyTrieStore TrieStore { get; } = overlayTrieStore.AsReadOnly(); + + public IWorldState CreateResettableWorldState(IWorldState? forWarmup = null) + { + PreBlockCaches? preBlockCaches = (forWarmup as IPreBlockCaches)?.Caches; + return preBlockCaches is not null + ? new WorldState( + new PreCachedTrieStore(overlayTrieStore, preBlockCaches.RlpCache), + _codeDb, + logManager, + preBlockCaches) + : new WorldState( + overlayTrieStore, + _codeDb, + logManager); + } + + public event EventHandler? ReorgBoundaryReached + { + add => overlayTrieStore.ReorgBoundaryReached += value; + remove => overlayTrieStore.ReorgBoundaryReached -= value; + } +} diff --git a/src/Nethermind/Nethermind.State/PartialStorageProviderBase.cs b/src/Nethermind/Nethermind.State/PartialStorageProviderBase.cs index e3e4bc676c8..ec0e284d260 100644 --- a/src/Nethermind/Nethermind.State/PartialStorageProviderBase.cs +++ b/src/Nethermind/Nethermind.State/PartialStorageProviderBase.cs @@ -145,9 +145,9 @@ public void Restore(int snapshot) /// /// Commit persistent storage /// - public void Commit() + public void Commit(bool commitStorageRoots = true) { - Commit(NullStateTracer.Instance); + Commit(NullStateTracer.Instance, commitStorageRoots); } protected readonly struct ChangeTrace @@ -172,7 +172,7 @@ public ChangeTrace(byte[]? after) /// Commit persistent storage /// /// State tracer - public void Commit(IStorageTracer tracer) + public void Commit(IStorageTracer tracer, bool commitStorageRoots = true) { if (_currentPosition == Snapshot.EmptyPosition) { @@ -182,6 +182,16 @@ public void Commit(IStorageTracer tracer) { CommitCore(tracer); } + + if (commitStorageRoots) + { + CommitStorageRoots(); + } + } + + protected virtual void CommitStorageRoots() + { + // Commit storage roots } /// @@ -199,7 +209,7 @@ protected virtual void CommitCore(IStorageTracer tracer) /// /// Reset the storage state /// - public virtual void Reset() + public virtual void Reset(bool resizeCollections = true) { if (_logger.IsTrace) _logger.Trace("Resetting storage"); diff --git a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs index 820655d53b8..a43b892466b 100644 --- a/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs +++ b/src/Nethermind/Nethermind.State/PersistentStorageProvider.cs @@ -2,29 +2,35 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Threading.Tasks; using Nethermind.Core; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Core.Resettables; +using Nethermind.Int256; using Nethermind.Logging; using Nethermind.State.Tracing; using Nethermind.Trie.Pruning; namespace Nethermind.State { + using Nethermind.Core.Cpu; /// /// Manages persistent storage allowing for snapshotting and restoring /// Persists data to ITrieStore /// - internal class PersistentStorageProvider : PartialStorageProviderBase + internal sealed class PersistentStorageProvider : PartialStorageProviderBase { private readonly ITrieStore _trieStore; private readonly StateProvider _stateProvider; private readonly ILogManager? _logManager; internal readonly IStorageTreeFactory _storageTreeFactory; private readonly ResettableDictionary _storages = new(); + private readonly HashSet _toUpdateRoots = new(); /// /// EIP-1283 @@ -32,14 +38,31 @@ internal class PersistentStorageProvider : PartialStorageProviderBase private readonly ResettableDictionary _originalValues = new(); private readonly ResettableHashSet _committedThisRound = new(); + private readonly Dictionary> _blockCache = new(4_096); + private readonly ConcurrentDictionary? _preBlockCache; + private readonly Func _loadFromTree; - public PersistentStorageProvider(ITrieStore? trieStore, StateProvider? stateProvider, ILogManager? logManager, IStorageTreeFactory? storageTreeFactory = null) - : base(logManager) + /// + /// Manages persistent storage allowing for snapshotting and restoring + /// Persists data to ITrieStore + /// + public PersistentStorageProvider(ITrieStore? trieStore, + StateProvider? stateProvider, + ILogManager? logManager, + IStorageTreeFactory? storageTreeFactory = null, + ConcurrentDictionary? preBlockCache = null) : base(logManager) { _trieStore = trieStore ?? throw new ArgumentNullException(nameof(trieStore)); _stateProvider = stateProvider ?? throw new ArgumentNullException(nameof(stateProvider)); _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager)); _storageTreeFactory = storageTreeFactory ?? new StorageTreeFactory(); + _preBlockCache = preBlockCache; + _loadFromTree = storageCell => + { + StorageTree tree = GetOrCreateStorage(storageCell.Address); + Db.Metrics.IncrementStorageTreeReads(); + return !storageCell.IsHash ? tree.Get(storageCell.Index) : tree.GetArray(storageCell.Hash.Bytes); + }; } public Hash256 StateRoot { get; set; } = null!; @@ -47,12 +70,14 @@ public PersistentStorageProvider(ITrieStore? trieStore, StateProvider? stateProv /// /// Reset the storage state /// - public override void Reset() + public override void Reset(bool resizeCollections = true) { base.Reset(); - _storages.Reset(); + _blockCache.Clear(); + _storages.Reset(resizeCollections); _originalValues.Clear(); _committedThisRound.Clear(); + _toUpdateRoots.Clear(); } /// @@ -179,16 +204,17 @@ protected override void CommitCore(IStorageTracer tracer) } } - // TODO: it seems that we are unnecessarily recalculating root hashes all the time in storage? foreach (Address address in toUpdateRoots) { // since the accounts could be empty accounts that are removing (EIP-158) if (_stateProvider.AccountExists(address)) { - Hash256 root = RecalculateRootHash(address); - - // _logger.Warn($"Recalculating storage root {address}->{root} ({toUpdateRoots.Count})"); - _stateProvider.UpdateStorageRoot(address, root); + _toUpdateRoots.Add(address); + } + else + { + _toUpdateRoots.Remove(address); + _storages.Remove(address); } } @@ -202,6 +228,68 @@ protected override void CommitCore(IStorageTracer tracer) } } + protected override void CommitStorageRoots() + { + if (_toUpdateRoots.Count == 0) + { + return; + } + + // Is overhead of parallel foreach worth it? + if (_toUpdateRoots.Count <= 4) + { + UpdateRootHashesSingleThread(); + } + else + { + UpdateRootHashesMultiThread(); + } + + void UpdateRootHashesSingleThread() + { + foreach (KeyValuePair kvp in _storages) + { + if (!_toUpdateRoots.Contains(kvp.Key)) + { + // Wasn't updated don't recalculate + continue; + } + + StorageTree storageTree = kvp.Value; + storageTree.UpdateRootHash(canBeParallel: true); + _stateProvider.UpdateStorageRoot(address: kvp.Key, storageTree.RootHash); + } + } + + void UpdateRootHashesMultiThread() + { + // We can recalculate the roots in parallel as they are all independent tries + Parallel.ForEach(_storages, RuntimeInformation.ParallelOptionsLogicalCores, kvp => + { + if (!_toUpdateRoots.Contains(kvp.Key)) + { + // Wasn't updated don't recalculate + return; + } + StorageTree storageTree = kvp.Value; + storageTree.UpdateRootHash(canBeParallel: false); + }); + + // Update the storage roots in the main thread non in parallel + foreach (KeyValuePair kvp in _storages) + { + if (!_toUpdateRoots.Contains(kvp.Key)) + { + continue; + } + + // Update the storage root for the Account + _stateProvider.UpdateStorageRoot(address: kvp.Key, kvp.Value.RootHash); + } + + } + } + private void SaveToTree(HashSet
toUpdateRoots, Change change) { if (_originalValues.TryGetValue(change.StorageCell, out byte[] initialValue) && @@ -215,6 +303,14 @@ private void SaveToTree(HashSet
toUpdateRoots, Change change) Db.Metrics.StorageTreeWrites++; toUpdateRoots.Add(change.StorageCell.Address); tree.Set(change.StorageCell.Index, change.Value); + + ref Dictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, change.StorageCell.Address, out bool exists); + if (!exists) + { + dict = new Dictionary(); + } + + dict[change.StorageCell.Index] = change.Value; } /// @@ -223,16 +319,19 @@ private void SaveToTree(HashSet
toUpdateRoots, Change change) /// Current block number public void CommitTrees(long blockNumber) { - // _logger.Warn($"Storage block commit {blockNumber}"); foreach (KeyValuePair storage in _storages) { + if (!_toUpdateRoots.Contains(storage.Key)) + { + continue; + } storage.Value.Commit(blockNumber); } - // TODO: maybe I could update storage roots only now? - + _toUpdateRoots.Clear(); // only needed here as there is no control over cached storage size otherwise _storages.Reset(); + _preBlockCache?.Clear(); } private StorageTree GetOrCreateStorage(Address address) @@ -241,26 +340,53 @@ private StorageTree GetOrCreateStorage(Address address) if (!exists) { value = _storageTreeFactory.Create(address, _trieStore.GetTrieStore(address.ToAccountPath), _stateProvider.GetStorageRoot(address), StateRoot, _logManager); - return value; } return value; } + public void WarmUp(in StorageCell storageCell, bool isEmpty) + { + if (isEmpty) + { + _preBlockCache[storageCell] = Array.Empty(); + } + else + { + LoadFromTree(in storageCell); + } + } + private ReadOnlySpan LoadFromTree(in StorageCell storageCell) { - StorageTree tree = GetOrCreateStorage(storageCell.Address); + ref Dictionary? dict = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, storageCell.Address, out bool exists); + if (!exists) + { + dict = new Dictionary(); + } - Db.Metrics.StorageTreeReads++; + ref byte[]? value = ref CollectionsMarshal.GetValueRefOrAddDefault(dict, storageCell.Index, out exists); + if (!exists) + { + long priorReads = Db.Metrics.ThreadLocalStorageTreeReads; - if (!storageCell.IsHash) + value = _preBlockCache is not null + ? _preBlockCache.GetOrAdd(storageCell, _loadFromTree) + : _loadFromTree(storageCell); + + if (Db.Metrics.ThreadLocalStorageTreeReads == priorReads) + { + // Read from Concurrent Cache + Db.Metrics.IncrementStorageTreeCache(); + } + } + else { - byte[] value = tree.Get(storageCell.Index); - PushToRegistryOnly(storageCell, value); - return value; + Db.Metrics.IncrementStorageTreeCache(); } - return tree.Get(storageCell.Hash.Bytes); + if (!storageCell.IsHash) PushToRegistryOnly(storageCell, value); + return value; } private void PushToRegistryOnly(in StorageCell cell, byte[] value) @@ -301,10 +427,14 @@ public override void ClearStorage(Address address) { base.ClearStorage(address); + // Bit heavy-handed, but we need to clear all the cache for that address + _blockCache.Remove(address); + // here it is important to make sure that we will not reuse the same tree when the contract is revived // by means of CREATE 2 - notice that the cached trie may carry information about items that were not // touched in this block, hence were not zeroed above // TODO: how does it work with pruning? + _toUpdateRoots.Remove(address); _storages[address] = new StorageTree(_trieStore.GetTrieStore(address.ToAccountPath), Keccak.EmptyTreeHash, _logManager); } diff --git a/src/Nethermind/Nethermind.State/PreBlockCaches.cs b/src/Nethermind/Nethermind.State/PreBlockCaches.cs new file mode 100644 index 00000000000..bffbd550740 --- /dev/null +++ b/src/Nethermind/Nethermind.State/PreBlockCaches.cs @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Concurrent; +using Nethermind.Core; +using Nethermind.Trie; + +namespace Nethermind.State; + +public class PreBlockCaches +{ + public ConcurrentDictionary StorageCache { get; } = new(Environment.ProcessorCount * 2, 4096 * 4); + public ConcurrentDictionary StateCache { get; } = new(Environment.ProcessorCount * 2, 4096 * 4); + public ConcurrentDictionary RlpCache { get; } = new(Environment.ProcessorCount * 2, 4096 * 4); + + public bool Clear() + { + bool isDirty = StorageCache.Count > 0 || StateCache.Count > 0 || RlpCache.Count > 0; + if (isDirty) + { + StorageCache.Clear(); + StateCache.Clear(); + RlpCache.Clear(); + } + + return isDirty; + } +} diff --git a/src/Nethermind/Nethermind.State/Proofs/ReceiptTrie.cs b/src/Nethermind/Nethermind.State/Proofs/ReceiptTrie.cs index cad6d77c749..e29c2a4f00e 100644 --- a/src/Nethermind/Nethermind.State/Proofs/ReceiptTrie.cs +++ b/src/Nethermind/Nethermind.State/Proofs/ReceiptTrie.cs @@ -20,13 +20,13 @@ public class ReceiptTrie : PatriciaTrie private readonly IRlpStreamDecoder _decoder; /// /// The transaction receipts to build the trie of. - public ReceiptTrie(IReceiptSpec spec, TReceipt[] receipts, IRlpStreamDecoder decoder, bool canBuildProof = false, ICappedArrayPool? bufferPool = null) + public ReceiptTrie(IReceiptSpec spec, TReceipt[] receipts, IRlpStreamDecoder trieDecoder, bool canBuildProof = false, ICappedArrayPool? bufferPool = null) : base(null, canBuildProof, bufferPool: bufferPool) { ArgumentNullException.ThrowIfNull(spec); ArgumentNullException.ThrowIfNull(receipts); - ArgumentNullException.ThrowIfNull(decoder); - _decoder = decoder; + ArgumentNullException.ThrowIfNull(trieDecoder); + _decoder = trieDecoder; if (receipts.Length > 0) { @@ -37,8 +37,7 @@ public ReceiptTrie(IReceiptSpec spec, TReceipt[] receipts, IRlpStreamDecoder public class ReadOnlyWorldStateManager : IWorldStateManager { - private readonly IReadOnlyDbProvider _readOnlyDbProvider; - private readonly IReadOnlyTrieStore? _readOnlyTrieStore; + private readonly IReadOnlyTrieStore _readOnlyTrieStore; private readonly ILogManager _logManager; - private readonly IDbProvider _dbProvider; private readonly ReadOnlyDb _codeDb; public ReadOnlyWorldStateManager( IDbProvider dbProvider, - IReadOnlyTrieStore? readOnlyTrieStore, + IReadOnlyTrieStore readOnlyTrieStore, ILogManager logManager ) { _readOnlyTrieStore = readOnlyTrieStore; - _dbProvider = dbProvider; _logManager = logManager; - _readOnlyDbProvider = _dbProvider.AsReadOnly(false); - _codeDb = _readOnlyDbProvider.GetDb(DbNames.Code).AsReadOnly(true); + IReadOnlyDbProvider readOnlyDbProvider = dbProvider.AsReadOnly(false); + _codeDb = readOnlyDbProvider.GetDb(DbNames.Code).AsReadOnly(true); GlobalStateReader = new StateReader(_readOnlyTrieStore, _codeDb, _logManager); } @@ -39,9 +36,21 @@ ILogManager logManager public IStateReader GlobalStateReader { get; } - public IWorldState CreateResettableWorldState() + public IReadOnlyTrieStore TrieStore => _readOnlyTrieStore; + + public IWorldState CreateResettableWorldState(IWorldState? forWarmup = null) { - return new WorldState(_readOnlyTrieStore, _codeDb, _logManager); + PreBlockCaches? preBlockCaches = (forWarmup as IPreBlockCaches)?.Caches; + return preBlockCaches is not null + ? new WorldState( + new PreCachedTrieStore(_readOnlyTrieStore, preBlockCaches.RlpCache), + _codeDb, + _logManager, + preBlockCaches) + : new WorldState( + _readOnlyTrieStore, + _codeDb, + _logManager); } public virtual event EventHandler? ReorgBoundaryReached diff --git a/src/Nethermind/Nethermind.State/StateProvider.cs b/src/Nethermind/Nethermind.State/StateProvider.cs index a24a647191a..2fc99f3ce0e 100644 --- a/src/Nethermind/Nethermind.State/StateProvider.cs +++ b/src/Nethermind/Nethermind.State/StateProvider.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Runtime.InteropServices; using Nethermind.Core; @@ -10,11 +11,9 @@ using Nethermind.Core.Crypto; using Nethermind.Core.Resettables; using Nethermind.Core.Specs; -using Nethermind.Db; using Nethermind.Int256; using Nethermind.Logging; using Nethermind.State.Tracing; -using Nethermind.State.Witnesses; using Nethermind.Trie; using Nethermind.Trie.Pruning; using Metrics = Nethermind.Db.Metrics; @@ -24,14 +23,16 @@ namespace Nethermind.State internal class StateProvider { private const int StartCapacity = Resettable.StartCapacity; - private readonly ResettableDictionary> _intraBlockCache = new(); + private readonly ResettableDictionary> _intraTxCache = new(); private readonly ResettableHashSet _committedThisRound = new(); private readonly HashSet _nullAccountReads = new(); // Only guarding against hot duplicates so filter doesn't need to be too big // Note: // False negatives are fine as they will just result in a overwrite set // False positives would be problematic as the code _must_ be persisted - private readonly LruKeyCache _codeInsertFilter = new(2048, "Code Insert Filter"); + private readonly LruKeyCacheNonConcurrent _codeInsertFilter = new(1_024, "Code Insert Filter"); + private readonly Dictionary _blockCache = new(4_096); + private readonly ConcurrentDictionary? _preBlockCache; private readonly List _keptInCache = new(); private readonly ILogger _logger; @@ -40,13 +41,8 @@ internal class StateProvider private int _capacity = StartCapacity; private Change?[] _changes = new Change?[StartCapacity]; private int _currentPosition = Resettable.EmptyPosition; - - public StateProvider(IScopedTrieStore? trieStore, IKeyValueStore? codeDb, ILogManager? logManager, StateTree? stateTree = null) - { - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _codeDb = codeDb ?? throw new ArgumentNullException(nameof(codeDb)); - _tree = stateTree ?? new StateTree(trieStore, logManager); - } + internal readonly StateTree _tree; + private readonly Func _getStateFromTrie; public void Accept(ITreeVisitor? visitor, Hash256? stateRoot, VisitingOptions? visitingOptions = null) { @@ -78,44 +74,24 @@ public Hash256 StateRoot set => _tree.RootHash = value; } - internal readonly StateTree _tree; - public bool IsContract(Address address) { Account? account = GetThroughCache(address); - if (account is null) - { - return false; - } - - return account.IsContract; + return account is not null && account.IsContract; } - public bool AccountExists(Address address) - { - if (_intraBlockCache.TryGetValue(address, out Stack value)) - { - return _changes[value.Peek()]!.ChangeType != ChangeType.Delete; - } - - return GetAndAddToCache(address) is not null; - } + public bool AccountExists(Address address) => + _intraTxCache.TryGetValue(address, out Stack value) + ? _changes[value.Peek()]!.ChangeType != ChangeType.Delete + : GetAndAddToCache(address) is not null; public bool IsEmptyAccount(Address address) { Account? account = GetThroughCache(address); - if (account is null) - { - throw new InvalidOperationException($"Account {address} is null when checking if empty"); - } - - return account.IsEmpty; + return account?.IsEmpty ?? throw new InvalidOperationException($"Account {address} is null when checking if empty"); } - public Account GetAccount(Address address) - { - return GetThroughCache(address) ?? Account.TotallyEmpty; - } + public Account GetAccount(Address address) => GetThroughCache(address) ?? Account.TotallyEmpty; public bool IsDeadAccount(Address address) { @@ -132,12 +108,7 @@ public UInt256 GetNonce(Address address) public Hash256 GetStorageRoot(Address address) { Account? account = GetThroughCache(address); - if (account is null) - { - throw new InvalidOperationException($"Account {address} is null when accessing storage root"); - } - - return account.StorageRoot; + return account is null ? throw new InvalidOperationException($"Account {address} is null when accessing storage root") : account.StorageRoot; } public UInt256 GetBalance(Address address) @@ -284,7 +255,7 @@ public void UpdateStorageRoot(Address address, Hash256 storageRoot) } } - public void IncrementNonce(Address address) + public void IncrementNonce(Address address, UInt256 delta) { _needsStateRootUpdate = true; Account? account = GetThroughCache(address); @@ -293,12 +264,12 @@ public void IncrementNonce(Address address) throw new InvalidOperationException($"Account {address} is null when incrementing nonce"); } - Account changedAccount = account.WithChangedNonce(account.Nonce + 1); + Account changedAccount = account.WithChangedNonce(account.Nonce + delta); if (_logger.IsTrace) _logger.Trace($" Update {address} N {account.Nonce} -> {changedAccount.Nonce}"); PushUpdate(address, changedAccount); } - public void DecrementNonce(Address address) + public void DecrementNonce(Address address, UInt256 delta) { _needsStateRootUpdate = true; Account? account = GetThroughCache(address); @@ -307,19 +278,11 @@ public void DecrementNonce(Address address) throw new InvalidOperationException($"Account {address} is null when decrementing nonce."); } - Account changedAccount = account.WithChangedNonce(account.Nonce - 1); + Account changedAccount = account.WithChangedNonce(account.Nonce - delta); if (_logger.IsTrace) _logger.Trace($" Update {address} N {account.Nonce} -> {changedAccount.Nonce}"); PushUpdate(address, changedAccount); } - public void TouchCode(in ValueHash256 codeHash) - { - if (_codeDb is WitnessingStore witnessingStore) - { - witnessingStore.Touch(codeHash.Bytes); - } - } - public Hash256 GetCodeHash(Address address) { Account? account = GetThroughCache(address); @@ -377,7 +340,7 @@ public void Restore(int snapshot) for (int i = 0; i < _currentPosition - snapshot; i++) { Change change = _changes[_currentPosition - i]; - Stack stack = _intraBlockCache[change!.Address]; + Stack stack = _intraTxCache[change!.Address]; if (stack.Count == 1) { if (change.ChangeType == ChangeType.JustCache) @@ -403,7 +366,7 @@ public void Restore(int snapshot) if (stack.Count == 0) { - _intraBlockCache.Remove(change.Address); + _intraTxCache.Remove(change.Address); } } @@ -412,7 +375,7 @@ public void Restore(int snapshot) { _currentPosition++; _changes[_currentPosition] = kept; - _intraBlockCache[kept.Address].Push(_currentPosition); + _intraTxCache[kept.Address].Push(_currentPosition); } _keptInCache.Clear(); @@ -428,8 +391,10 @@ public void CreateAccount(Address address, in UInt256 balance, in UInt256 nonce public void CreateAccountIfNotExists(Address address, in UInt256 balance, in UInt256 nonce = default) { - if (AccountExists(address)) return; - CreateAccount(address, balance, nonce); + if (!AccountExists(address)) + { + CreateAccount(address, balance, nonce); + } } public void AddToBalanceAndCreateIfNotExists(Address address, in UInt256 balance, IReleaseSpec spec) @@ -518,7 +483,7 @@ public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer stateTracer, bool continue; } - Stack stack = _intraBlockCache[change.Address]; + Stack stack = _intraTxCache[change.Address]; int forAssertion = stack.Pop(); if (forAssertion != _currentPosition - i) { @@ -613,7 +578,7 @@ public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer stateTracer, bool Resettable.Reset(ref _changes, ref _capacity, ref _currentPosition, StartCapacity); _committedThisRound.Reset(); _nullAccountReads.Clear(); - _intraBlockCache.Reset(); + _intraTxCache.Reset(); if (isTracing) { @@ -679,15 +644,54 @@ private void ReportChanges(IStateTracer stateTracer, Dictionary? preBlockCache = null) + { + _preBlockCache = preBlockCache; + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _codeDb = codeDb ?? throw new ArgumentNullException(nameof(codeDb)); + _tree = stateTree ?? new StateTree(trieStore, logManager); + _getStateFromTrie = address => + { + Metrics.IncrementStateTreeReads(); + return _tree.Get(address); + }; + } + + public bool WarmUp(Address address) + { + return GetState(address) is not null; + } + private Account? GetState(Address address) { - Metrics.StateTreeReads++; - Account? account = _tree.Get(address); + AddressAsKey addressAsKey = address; + ref Account? account = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, addressAsKey, out bool exists); + if (!exists) + { + long priorReads = Metrics.ThreadLocalStateTreeReads; + account = _preBlockCache is not null + ? _preBlockCache.GetOrAdd(addressAsKey, _getStateFromTrie) + : _getStateFromTrie(addressAsKey); + + if (Metrics.ThreadLocalStateTreeReads == priorReads) + { + Metrics.IncrementStateTreeCacheHits(); + } + } + else + { + Metrics.IncrementStateTreeCacheHits(); + } return account; } private void SetState(Address address, Account? account) { + _blockCache[address] = account; _needsStateRootUpdate = true; Metrics.StateTreeWrites++; _tree.Set(address, account); @@ -713,7 +717,7 @@ private void SetState(Address address, Account? account) private Account? GetThroughCache(Address address) { - if (_intraBlockCache.TryGetValue(address, out Stack value)) + if (_intraTxCache.TryGetValue(address, out Stack value)) { return _changes[value.Peek()]!.Account; } @@ -772,7 +776,7 @@ private void IncrementChangePosition() private Stack SetupCache(Address address) { - ref Stack? value = ref _intraBlockCache.GetValueRefOrAddDefault(address, out bool exists); + ref Stack? value = ref _intraTxCache.GetValueRefOrAddDefault(address, out bool exists); if (!exists) { value = new Stack(); @@ -804,11 +808,30 @@ public Change(ChangeType type, Address address, Account? account) public Account? Account { get; } } - public void Reset() + public ArrayPoolList? ChangedAddresses() + { + int count = _blockCache.Count; + if (count == 0) + { + return null; + } + else + { + ArrayPoolList addresses = new(count); + foreach (AddressAsKey address in _blockCache.Keys) + { + addresses.Add(address); + } + return addresses; + } + } + + public void Reset(bool resizeCollections = true) { if (_logger.IsTrace) _logger.Trace("Clearing state provider caches"); - _intraBlockCache.Reset(); - _committedThisRound.Reset(); + _blockCache.Clear(); + _intraTxCache.Reset(resizeCollections); + _committedThisRound.Reset(resizeCollections); _nullAccountReads.Clear(); _currentPosition = Resettable.EmptyPosition; Array.Clear(_changes, 0, _changes.Length); @@ -823,6 +846,7 @@ public void CommitTree(long blockNumber) } _tree.Commit(blockNumber); + _preBlockCache?.Clear(); } public static void CommitBranch() diff --git a/src/Nethermind/Nethermind.State/StateReader.cs b/src/Nethermind/Nethermind.State/StateReader.cs index a978013ba34..77558d564c2 100644 --- a/src/Nethermind/Nethermind.State/StateReader.cs +++ b/src/Nethermind/Nethermind.State/StateReader.cs @@ -34,7 +34,7 @@ public ReadOnlySpan GetStorage(Hash256 stateRoot, Address address, in UInt return Bytes.ZeroByte.Span; } - Metrics.StorageTreeReads++; + Metrics.StorageReaderReads++; StorageTree storage = new StorageTree(_trieStore.GetTrieStore(address.ToAccountPath), Keccak.EmptyTreeHash, _logManager); return storage.Get(index, new Hash256(storageRoot)); @@ -68,7 +68,7 @@ private bool TryGetState(Hash256 stateRoot, Address address, out AccountStruct a return false; } - Metrics.StateTreeReads++; + Metrics.IncrementStateReaderReads(); return _state.TryGetStruct(address, out account, stateRoot); } } diff --git a/src/Nethermind/Nethermind.State/StorageTree.cs b/src/Nethermind/Nethermind.State/StorageTree.cs index 6b9a520e5c4..472441c9e3b 100644 --- a/src/Nethermind/Nethermind.State/StorageTree.cs +++ b/src/Nethermind/Nethermind.State/StorageTree.cs @@ -19,7 +19,7 @@ public class StorageTree : PatriciaTree { private const int LookupSize = 1024; private static readonly FrozenDictionary Lookup = CreateLookup(); - private static readonly byte[] _emptyBytes = [0]; + public static readonly byte[] EmptyBytes = [0]; private static FrozenDictionary CreateLookup() { @@ -59,28 +59,29 @@ public byte[] Get(in UInt256 index, Hash256? storageRoot = null) { if (index < LookupSize) { - return Get(Lookup[index], storageRoot).ToArray(); + return GetArray(Lookup[index], storageRoot); } Span key = stackalloc byte[32]; ComputeKey(index, ref key); - return Get(key, storageRoot).ToArray(); - + return GetArray(key, storageRoot); } - public override ReadOnlySpan Get(ReadOnlySpan rawKey, Hash256? rootHash = null) + public byte[] GetArray(ReadOnlySpan rawKey, Hash256? rootHash = null) { ReadOnlySpan value = base.Get(rawKey, rootHash); if (value.IsEmpty) { - return _emptyBytes; + return EmptyBytes; } Rlp.ValueDecoderContext rlp = value.AsRlpValueContext(); return rlp.DecodeByteArray(); } + public override ReadOnlySpan Get(ReadOnlySpan rawKey, Hash256? rootHash = null) => GetArray(rawKey, rootHash); + [SkipLocalsInit] public void Set(in UInt256 index, byte[] value) { diff --git a/src/Nethermind/Nethermind.State/TransientStorageProvider.cs b/src/Nethermind/Nethermind.State/TransientStorageProvider.cs index 3e2177fdf33..4c0ec3efaef 100644 --- a/src/Nethermind/Nethermind.State/TransientStorageProvider.cs +++ b/src/Nethermind/Nethermind.State/TransientStorageProvider.cs @@ -11,7 +11,7 @@ namespace Nethermind.State /// EIP-1153 provides a transient store for contracts that doesn't persist /// storage across calls. Reverts will rollback any transient state changes. ///
- internal class TransientStorageProvider : PartialStorageProviderBase + internal sealed class TransientStorageProvider : PartialStorageProviderBase { public TransientStorageProvider(ILogManager? logManager) : base(logManager) { } diff --git a/src/Nethermind/Nethermind.State/Witnesses/WitnessCollector.cs b/src/Nethermind/Nethermind.State/Witnesses/WitnessCollector.cs deleted file mode 100644 index 86f0ceeb7a9..00000000000 --- a/src/Nethermind/Nethermind.State/Witnesses/WitnessCollector.cs +++ /dev/null @@ -1,130 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using System.Collections.Generic; -using System.Linq; -using Nethermind.Core; -using Nethermind.Core.Caching; -using Nethermind.Core.Crypto; -using Nethermind.Core.Resettables; -using Nethermind.Logging; - -namespace Nethermind.State.Witnesses -{ - /// - /// - /// - public class WitnessCollector : IWitnessCollector, IWitnessRepository - { - [ThreadStatic] - private static bool _collectWitness; - - private readonly LruCache _witnessCache = new(256, "Witnesses"); - - public IReadOnlyCollection Collected => _collected; - - public WitnessCollector(IKeyValueStore? keyValueStore, ILogManager? logManager) - { - _keyValueStore = keyValueStore ?? throw new ArgumentNullException(nameof(keyValueStore)); - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - } - - public void Add(Hash256 hash) - { - if (!_collectWitness) - { - return; - } - _collected.Add(hash); - } - - public void Reset() - { - _collected.Reset(); - } - - public void Persist(Hash256 blockHash) - { - if (_logger.IsDebug) _logger.Debug($"Persisting {blockHash} witness ({_collected.Count})"); - - - if (_collected.Count > 0) - { - Hash256[] collected = _collected.ToArray(); - byte[] witness = new byte[collected.Length * Hash256.Size]; - Span witnessSpan = witness; - - int i = 0; - for (var index = 0; index < collected.Length; index++) - { - Hash256 keccak = collected[index]; - keccak.Bytes.CopyTo(witnessSpan.Slice(i * Hash256.Size, Hash256.Size)); - i++; - } - - _keyValueStore[blockHash.Bytes] = witness; - _witnessCache.Set(blockHash, collected); - } - else - { - _witnessCache.Set(blockHash, Array.Empty()); - } - } - - class WitnessCollectorTrackingScope : IDisposable - { - public WitnessCollectorTrackingScope() => _collectWitness = true; - public void Dispose() => _collectWitness = false; - } - - public IDisposable TrackOnThisThread() => new WitnessCollectorTrackingScope(); - - public Hash256[]? Load(Hash256 blockHash) - { - if (_witnessCache.TryGet(blockHash, out Hash256[]? witness)) - { - if (_logger.IsTrace) _logger.Trace($"Loading cached witness for {blockHash} ({witness!.Length})"); - } - else // not cached - { - byte[]? witnessData = _keyValueStore[blockHash.Bytes]; - if (witnessData is null) - { - if (_logger.IsTrace) _logger.Trace($"Missing witness for {blockHash}"); - witness = null; - } - else // missing from the DB - { - Span witnessDataSpan = witnessData.AsSpan(); - int itemCount = witnessData.Length / Hash256.Size; - if (_logger.IsTrace) _logger.Trace($"Loading non-cached witness for {blockHash} ({itemCount})"); - - Hash256[] writableWitness = new Hash256[itemCount]; - for (int i = 0; i < itemCount; i++) - { - byte[] keccakBytes = witnessDataSpan.Slice(i * Hash256.Size, Hash256.Size).ToArray(); - writableWitness[i] = new Hash256(keccakBytes); - } - - _witnessCache.Set(blockHash, writableWitness); - witness = writableWitness; - } - } - - return witness; - } - - public void Delete(Hash256 blockHash) - { - _witnessCache.Delete(blockHash); - _keyValueStore[blockHash.Bytes] = null; - } - - private readonly ResettableHashSet _collected = new(); - - private readonly IKeyValueStore _keyValueStore; - - private readonly ILogger _logger; - } -} diff --git a/src/Nethermind/Nethermind.State/Witnesses/WitnessingStore.cs b/src/Nethermind/Nethermind.State/Witnesses/WitnessingStore.cs deleted file mode 100644 index e52af604339..00000000000 --- a/src/Nethermind/Nethermind.State/Witnesses/WitnessingStore.cs +++ /dev/null @@ -1,65 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using Nethermind.Core; -using Nethermind.Core.Crypto; - -namespace Nethermind.State.Witnesses -{ - public static class KeyValueStoreWithBatchingExtensions - { - public static IKeyValueStoreWithBatching WitnessedBy( - this IKeyValueStoreWithBatching @this, - IWitnessCollector witnessCollector) - { - return witnessCollector == NullWitnessCollector.Instance - ? @this - : new WitnessingStore(@this, witnessCollector); - } - } - - public class WitnessingStore : IKeyValueStoreWithBatching - { - private readonly IKeyValueStoreWithBatching _wrapped; - private readonly IWitnessCollector _witnessCollector; - - public WitnessingStore(IKeyValueStoreWithBatching? wrapped, IWitnessCollector? witnessCollector) - { - _wrapped = wrapped ?? throw new ArgumentNullException(nameof(wrapped)); - _witnessCollector = witnessCollector ?? throw new ArgumentNullException(nameof(witnessCollector)); - } - - public byte[]? this[ReadOnlySpan key] - { - get => Get(key); - set => Set(key, value); - } - - public byte[]? Get(ReadOnlySpan key, ReadFlags flags = ReadFlags.None) - { - if (key.Length != 32) - { - throw new NotSupportedException($"{nameof(WitnessingStore)} requires 32 bytes long keys."); - } - - Touch(key); - return _wrapped.Get(key, flags); - } - - public void Set(ReadOnlySpan key, byte[]? value, WriteFlags flags = WriteFlags.None) - { - _wrapped.Set(key, value, flags); - } - - public IWriteBatch StartWriteBatch() - { - return _wrapped.StartWriteBatch(); - } - - public void Touch(ReadOnlySpan key) - { - _witnessCollector.Add(new Hash256(key)); - } - } -} diff --git a/src/Nethermind/Nethermind.State/WorldState.cs b/src/Nethermind/Nethermind.State/WorldState.cs index 1300bcf6671..71933b6cc12 100644 --- a/src/Nethermind/Nethermind.State/WorldState.cs +++ b/src/Nethermind/Nethermind.State/WorldState.cs @@ -2,9 +2,12 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Collections.Generic; using System.Runtime.CompilerServices; using Nethermind.Core; +using Nethermind.Core.Collections; using Nethermind.Core.Crypto; +using Nethermind.Core.Eip2930; using Nethermind.Core.Specs; using Nethermind.Int256; using Nethermind.Logging; @@ -22,12 +25,13 @@ namespace Nethermind.State { - public class WorldState : IWorldState + public class WorldState : IWorldState, IPreBlockCaches { internal readonly StateProvider _stateProvider; internal readonly PersistentStorageProvider _persistentStorageProvider; private readonly TransientStorageProvider _transientStorageProvider; private readonly ITrieStore _trieStore; + private PreBlockCaches? PreBlockCaches { get; } public Hash256 StateRoot { @@ -39,19 +43,29 @@ public Hash256 StateRoot } } - public WorldState(ITrieStore? trieStore, IKeyValueStore? codeDb, ILogManager? logManager) + public WorldState(ITrieStore trieStore, IKeyValueStore? codeDb, ILogManager? logManager) + : this(trieStore, codeDb, logManager, null, null) { + } + + internal WorldState( + ITrieStore trieStore, + IKeyValueStore? codeDb, + ILogManager? logManager, + StateTree? stateTree = null, + IStorageTreeFactory? storageTreeFactory = null, + PreBlockCaches? preBlockCaches = null) + { + PreBlockCaches = preBlockCaches; _trieStore = trieStore; - _stateProvider = new StateProvider(trieStore.GetTrieStore(null), codeDb, logManager); - _persistentStorageProvider = new PersistentStorageProvider(trieStore, _stateProvider, logManager); + _stateProvider = new StateProvider(trieStore.GetTrieStore(null), codeDb, logManager, stateTree, PreBlockCaches?.StateCache); + _persistentStorageProvider = new PersistentStorageProvider(trieStore, _stateProvider, logManager, storageTreeFactory, PreBlockCaches?.StorageCache); _transientStorageProvider = new TransientStorageProvider(logManager); } - internal WorldState(ITrieStore? trieStore, IKeyValueStore? codeDb, ILogManager? logManager, StateTree stateTree, IStorageTreeFactory storageTreeFactory) + public WorldState(ITrieStore trieStore, IKeyValueStore? codeDb, ILogManager? logManager, PreBlockCaches? preBlockCaches) + : this(trieStore, codeDb, logManager, null, preBlockCaches: preBlockCaches) { - _stateProvider = new StateProvider(trieStore.GetTrieStore(null), codeDb, logManager, stateTree); - _persistentStorageProvider = new PersistentStorageProvider(trieStore, _stateProvider, logManager, storageTreeFactory); - _transientStorageProvider = new TransientStorageProvider(logManager); } public Account GetAccount(Address address) @@ -90,13 +104,28 @@ public void SetTransientState(in StorageCell storageCell, byte[] newValue) { _transientStorageProvider.Set(storageCell, newValue); } - public void Reset() + public void Reset(bool resizeCollections = false) { - _stateProvider.Reset(); - _persistentStorageProvider.Reset(); - _transientStorageProvider.Reset(); + _stateProvider.Reset(resizeCollections); + _persistentStorageProvider.Reset(resizeCollections); + _transientStorageProvider.Reset(resizeCollections); + } + public void WarmUp(AccessList? accessList) + { + if (accessList?.IsEmpty == false) + { + foreach ((Address address, AccessList.StorageKeysEnumerable storages) in accessList) + { + bool exists = _stateProvider.WarmUp(address); + foreach (UInt256 storage in storages) + { + _persistentStorageProvider.WarmUp(new StorageCell(address, storage), isEmpty: !exists); + } + } + } } + public void WarmUp(Address address) => _stateProvider.WarmUp(address); public void ClearStorage(Address address) { _persistentStorageProvider.ClearStorage(address); @@ -134,13 +163,13 @@ public void UpdateStorageRoot(Address address, Hash256 storageRoot) { _stateProvider.UpdateStorageRoot(address, storageRoot); } - public void IncrementNonce(Address address) + public void IncrementNonce(Address address, UInt256 delta) { - _stateProvider.IncrementNonce(address); + _stateProvider.IncrementNonce(address, delta); } - public void DecrementNonce(Address address) + public void DecrementNonce(Address address, UInt256 delta) { - _stateProvider.DecrementNonce(address); + _stateProvider.DecrementNonce(address, delta); } public void CommitTree(long blockNumber) @@ -150,11 +179,6 @@ public void CommitTree(long blockNumber) _persistentStorageProvider.StateRoot = _stateProvider.StateRoot; } - public void TouchCode(in ValueHash256 codeHash) - { - _stateProvider.TouchCode(codeHash); - } - public UInt256 GetNonce(Address address) => _stateProvider.GetNonce(address); public UInt256 GetBalance(Address address) => _stateProvider.GetBalance(address); @@ -196,16 +220,16 @@ public bool HasStateForRoot(Hash256 stateRoot) return _trieStore.HasRoot(stateRoot); } - public void Commit(IReleaseSpec releaseSpec, bool isGenesis = false) + public void Commit(IReleaseSpec releaseSpec, bool isGenesis = false, bool commitStorageRoots = true) { - _persistentStorageProvider.Commit(); - _transientStorageProvider.Commit(); + _persistentStorageProvider.Commit(commitStorageRoots); + _transientStorageProvider.Commit(commitStorageRoots); _stateProvider.Commit(releaseSpec, isGenesis); } - public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer tracer, bool isGenesis = false) + public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer tracer, bool isGenesis = false, bool commitStorageRoots = true) { - _persistentStorageProvider.Commit(tracer); - _transientStorageProvider.Commit(tracer); + _persistentStorageProvider.Commit(tracer, commitStorageRoots); + _transientStorageProvider.Commit(tracer, commitStorageRoots); _stateProvider.Commit(releaseSpec, tracer, isGenesis); } @@ -230,6 +254,7 @@ internal void Restore(int state, int persistantStorage, int transientStorage) Restore(new Snapshot(state, new Snapshot.Storage(persistantStorage, transientStorage))); } + // Needed for benchmarks internal void SetNonce(Address address, in UInt256 nonce) { _stateProvider.SetNonce(address, nonce); @@ -239,5 +264,11 @@ public void CreateAccountIfNotExists(Address address, in UInt256 balance, in UIn { _stateProvider.CreateAccountIfNotExists(address, balance, nonce); } + + ArrayPoolList? IWorldState.GetAccountChanges() => _stateProvider.ChangedAddresses(); + + PreBlockCaches? IPreBlockCaches.Caches => PreBlockCaches; + + public bool ClearCache() => PreBlockCaches?.Clear() == true; } } diff --git a/src/Nethermind/Nethermind.State/WorldStateManager.cs b/src/Nethermind/Nethermind.State/WorldStateManager.cs index cfb570a8daa..807db3ab54f 100644 --- a/src/Nethermind/Nethermind.State/WorldStateManager.cs +++ b/src/Nethermind/Nethermind.State/WorldStateManager.cs @@ -8,27 +8,18 @@ namespace Nethermind.State; -public class WorldStateManager : ReadOnlyWorldStateManager +public class WorldStateManager( + IWorldState worldState, + ITrieStore trieStore, + IDbProvider dbProvider, + ILogManager logManager) + : ReadOnlyWorldStateManager(dbProvider, trieStore.AsReadOnly(), logManager) { - private readonly IWorldState _worldState; - private readonly ITrieStore _trieStore; - - public WorldStateManager( - IWorldState worldState, - ITrieStore trieStore, - IDbProvider dbProvider, - ILogManager logManager - ) : base(dbProvider, trieStore.AsReadOnly(), logManager) - { - _worldState = worldState; - _trieStore = trieStore; - } - - public override IWorldState GlobalWorldState => _worldState; + public override IWorldState GlobalWorldState => worldState; public override event EventHandler? ReorgBoundaryReached { - add => _trieStore.ReorgBoundaryReached += value; - remove => _trieStore.ReorgBoundaryReached -= value; + add => trieStore.ReorgBoundaryReached += value; + remove => trieStore.ReorgBoundaryReached -= value; } } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs index 709607c9a41..113a6f3e641 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.cs @@ -7,7 +7,6 @@ using System.Threading; using System.Threading.Tasks; using FluentAssertions; -using Microsoft.ClearScript.JavaScript; using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; @@ -1361,7 +1360,7 @@ Block BuildBlockForHeader(BlockHeader header, int txSeed) _headers[blockHashes[i]].ReceiptsRoot = flags.HasFlag(Response.IncorrectReceiptRoot) ? Keccak.EmptyTreeHash - : ReceiptTrie.CalculateRoot(MainnetSpecProvider.Instance.GetSpec((ForkActivation)_headers[blockHashes[i]].Number), receipts[i], ReceiptMessageDecoder.Instance); + : ReceiptTrie.CalculateRoot(MainnetSpecProvider.Instance.GetSpec((ForkActivation)_headers[blockHashes[i]].Number), receipts[i], Rlp.GetStreamDecoder()!); } using ReceiptsMessage message = new(receipts.ToPooledList()); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/FastHeadersSyncTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/FastHeadersSyncTests.cs index 9a1d45b7af3..adeab5f2552 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/FastHeadersSyncTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/FastHeadersSyncTests.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Reactive.Linq; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; @@ -147,7 +148,7 @@ public async Task Can_prepare_several_request_and_ignore_request_from_previous_s syncReport.HeadersInQueue.Returns(new MeasuredProgress()); BlockHeader pivot = remoteBlockTree.FindHeader(500, BlockTreeLookupOptions.None)!; - ResettableHeaderSyncFeed feed = new( + using ResettableHeaderSyncFeed feed = new( blockTree: blockTree, syncPeerPool: Substitute.For(), syncConfig: new SyncConfig @@ -169,7 +170,7 @@ void FulfillBatch(HeadersSyncBatch batch) false)!; } - await feed.PrepareRequest(); + using HeadersSyncBatch? r = await feed.PrepareRequest(); using HeadersSyncBatch batch1 = (await feed.PrepareRequest())!; FulfillBatch(batch1); @@ -194,7 +195,7 @@ public async Task Will_dispatch_when_only_partially_processed_dependency() syncReport.HeadersInQueue.Returns(new MeasuredProgress()); BlockHeader pivot = remoteBlockTree.FindHeader(2000, BlockTreeLookupOptions.None)!; - HeadersSyncFeed feed = new( + using HeadersSyncFeed feed = new( blockTree: blockTree, syncPeerPool: Substitute.For(), syncConfig: new SyncConfig @@ -248,6 +249,7 @@ void FulfillBatch(HeadersSyncBatch batch) // New batch would be at end of batch 5 (batch 6). newBatch.EndNumber.Should().Be(batches[^1].StartNumber - 1); + batches.DisposeItems(); } [Test] @@ -380,6 +382,86 @@ public async Task Can_resume_downloading_from_parent_of_lowest_inserted_header() result!.EndNumber.Should().Be(499); } + //Missing headers in the start is not allowed + [TestCase(0, 1, 1, true, false)] + [TestCase(0, 1, 1, false, true)] + //Missing headers in the start is not allowed + [TestCase(0, 2, 1, true, false)] + [TestCase(0, 2, 1, false, true)] + //Missing headers in the start is not allowed + [TestCase(0, 2, 191, true, false)] + [TestCase(0, 2, 191, false, true)] + //Gaps are not allowed + [TestCase(1, 1, 1, true, false)] + [TestCase(1, 1, 1, true, true)] + [TestCase(187, 5, 1, false, false)] + [TestCase(187, 5, 1, false, true)] + [TestCase(191, 1, 1, false, false)] + [TestCase(191, 1, 1, false, true)] + [TestCase(190, 1, 1, true, false)] + [TestCase(190, 1, 1, true, true)] + [TestCase(80, 1, 1, true, false)] + [TestCase(80, 1, 1, true, true)] + //All empty reponse + [TestCase(0, 192, 1, false, false)] + //All null reponse + [TestCase(0, 192, 1, false, true)] + public async Task Can_insert_all_good_headers_from_dependent_batch_with_missing_or_null_headers(int nullIndex, int count, int increment, bool shouldReport, bool useNulls) + { + var peerChain = Build.A.BlockTree().OfChainLength(1000).TestObject; + var syncConfig = new SyncConfig { FastSync = true, PivotNumber = "1000", PivotHash = Keccak.Zero.ToString(), PivotTotalDifficulty = "1000" }; + + IBlockTree localBlockTree = Build.A.BlockTree(peerChain.FindBlock(0, BlockTreeLookupOptions.None)!, null).WithSyncConfig(syncConfig).TestObject; + const int lowestInserted = 999; + localBlockTree.Insert(peerChain.Head!, BlockTreeInsertBlockOptions.SaveHeader); + + ISyncReport report = Substitute.For(); + report.HeadersInQueue.Returns(new MeasuredProgress()); + report.FastBlocksHeaders.Returns(new MeasuredProgress()); + + ISyncPeerPool syncPeerPool = Substitute.For(); + using HeadersSyncFeed feed = new(localBlockTree, syncPeerPool, syncConfig, report, LimboLogs.Instance); + feed.InitializeFeed(); + using HeadersSyncBatch? firstBatch = await feed.PrepareRequest(); + using HeadersSyncBatch? dependentBatch = await feed.PrepareRequest(); + dependentBatch!.ResponseSourcePeer = new PeerInfo(Substitute.For()); + + void FillBatch(HeadersSyncBatch batch, long start, bool applyNulls) + { + int c = count; + List list = Enumerable.Range((int)start, batch.RequestSize) + .Select(i => peerChain.FindBlock(i, BlockTreeLookupOptions.None)!.Header) + .ToList(); + if (applyNulls) + for (int i = nullIndex; 0 < c; i += increment) + { + list[i] = null; + c--; + } + if (!useNulls) + list = list.Where(h => h is not null).ToList(); + batch.Response = list.ToPooledList(); + } + + FillBatch(firstBatch!, lowestInserted - firstBatch!.RequestSize, false); + FillBatch(dependentBatch, lowestInserted - dependentBatch.RequestSize * 2, true); + long targetHeaderInDependentBatch = dependentBatch.StartNumber; + + feed.HandleResponse(dependentBatch); + feed.HandleResponse(firstBatch); + + using HeadersSyncBatch? thirdbatch = await feed.PrepareRequest(); + FillBatch(thirdbatch!, thirdbatch!.StartNumber, false); + feed.HandleResponse(thirdbatch); + using HeadersSyncBatch? fourthbatch = await feed.PrepareRequest(); + FillBatch(fourthbatch!, fourthbatch!.StartNumber, false); + feed.HandleResponse(fourthbatch); + using HeadersSyncBatch? fifthbatch = await feed.PrepareRequest(); + + Assert.That(localBlockTree.LowestInsertedHeader!.Number, Is.LessThanOrEqualTo(targetHeaderInDependentBatch)); + syncPeerPool.Received(shouldReport ? 1 : 0).ReportBreachOfProtocol(Arg.Any(), Arg.Any(), Arg.Any()); + } + [Test] public async Task Will_never_lose_batch_on_invalid_batch() { diff --git a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs index d956adea689..d5e3adcbeae 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/OldStyleFullSynchronizerTests.cs @@ -22,7 +22,6 @@ using Nethermind.Specs; using Nethermind.Specs.ChainSpecStyle; using Nethermind.State; -using Nethermind.State.Witnesses; using Nethermind.Stats; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.Peers; @@ -97,7 +96,6 @@ public async Task Setup() _pool, _synchronizer.SyncModeSelector, quickConfig, - new WitnessCollector(new MemDb(), LimboLogs.Instance), Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/PeerInfoTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/PeerInfoTests.cs index 26fc35ab373..b7b77093564 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/PeerInfoTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/PeerInfoTests.cs @@ -15,7 +15,6 @@ namespace Nethermind.Synchronization.Test [TestFixture(AllocationContexts.Headers)] [TestFixture(AllocationContexts.Bodies)] [TestFixture(AllocationContexts.State)] - [TestFixture(AllocationContexts.Witness)] [TestFixture(AllocationContexts.All)] [Parallelizable(ParallelScope.All)] public class PeerInfoTests diff --git a/src/Nethermind/Nethermind.Synchronization.Test/RangeQueryVisitorTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/RangeQueryVisitorTests.cs index 496d6f6ce96..1c2bd7be51f 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/RangeQueryVisitorTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/RangeQueryVisitorTests.cs @@ -31,6 +31,8 @@ using Nethermind.Trie; using Nethermind.Trie.Pruning; using NUnit.Framework; +using Org.BouncyCastle.Utilities; +using Bytes = Nethermind.Core.Extensions.Bytes; namespace Nethermind.Synchronization.Test; @@ -136,6 +138,68 @@ public void RangeFetchPartialLimit() leafCollector.Leafs.Count.Should().Be(3); } + + [Test] + public void RangeFetchPartialLimit_FarProof() + { + + string[] paths = + [ + "0x1110000000000000000000000000000000000000000000000000000000000000", + "0x1120000000000000000000000000000000000000000000000000000000000000", + "0x1130000000000000000000000000000000000000000000000000000000000000", + // Query here 0x114... + "0x1210000000000000000000000000000000000000000000000000000000000000", + "0x1220000000000000000000000000000000000000000000000000000000000000", + "0x1230000000000000000000000000000000000000000000000000000000000000", + // Until here 0x1235.. + "0x1310000000000000000000000000000000000000000000000000000000000000", + "0x1320000000000000000000000000000000000000000000000000000000000000", + ]; + + var stateTree = new StateTree(); + var random = new Random(0); + foreach (var path in paths) + { + stateTree.Set(new Hash256(path), TestItem.GenerateRandomAccount(random)); + } + stateTree.Commit(0); + + var startHash = new Hash256("0x1140000000000000000000000000000000000000000000000000000000000000"); + var limitHash = new Hash256("0x1235000000000000000000000000000000000000000000000000000000000000"); + + RlpCollector leafCollector = new(); + using RangeQueryVisitor visitor = new(startHash, limitHash, leafCollector); + stateTree.Accept(visitor, stateTree.RootHash, CreateVisitingOptions()); + + leafCollector.Leafs.Count.Should().Be(4); + + using ArrayPoolList proofs = visitor.GetProofs(); + proofs.Count.Should().Be(6); // Need to make sure `0x11` is included + + var proofHashes = proofs.Select((rlp) => Keccak.Compute(rlp)).ToHashSet(); + foreach (Hash256 proofHash in proofHashes) + { + Console.Out.WriteLine(proofHash); + } + + string[] proofHashStrs = + [ + "0x35811c17fd5e33e75276677e27e3fe39653403a4d0df4a2f94af40ac265a4a6f", + "0xfd6d9e748837908d14fca3ddf76d06c3f74196543f3d05d8fa0b6d6726037f51", + "0xde44831292ba34a2a31566004549c1681dbe3a4042f265be60a9fff3643a3112", + "0x665b6b070a219250b89d36feeb07ae350bae619e8660598f9ec98176b19c5d07", + "0x07b17db6a32be868e9940568db8b1011c7679e642c2db0237a5a7ebdaadb0e6e", + "0xfbd8c8f3cd78599b87fd3ab0cfe127c5b2d488cb913aeec5b80230668fed45c8" + ]; + + foreach (var proofHashStr in proofHashStrs) + { + proofHashes.Contains(new Hash256(Bytes.FromHexString(proofHashStr))).Should().BeTrue(); + } + } + + [Test] public void StorageRangeFetchVisitor() { diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapProviderTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapProviderTests.cs new file mode 100644 index 00000000000..58e0a808b06 --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapProviderTests.cs @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Blockchain; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.State.Snap; +using Nethermind.Synchronization.SnapSync; +using Nethermind.Trie; +using NSubstitute; +using NUnit.Framework; +using System; +using System.Collections.Generic; + +namespace Nethermind.Synchronization.Test.SnapSync; + +[TestFixture] +public class SnapProviderTests +{ + + [Test] + public void AddAccountRange_AccountListIsEmpty_ThrowArgumentException() + { + MemDb db = new(); + IDbProvider dbProvider = new DbProvider(); + dbProvider.RegisterDb(DbNames.State, db); + using ProgressTracker progressTracker = new(Substitute.For(), dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + dbProvider.RegisterDb(DbNames.Code, new MemDb()); + SnapProvider sut = new(progressTracker, dbProvider.CodeDb, new NodeStorage(dbProvider.StateDb), LimboLogs.Instance); + + Assert.That( + () => sut.AddAccountRange( + 0, + Keccak.Zero, + Keccak.Zero, + Array.Empty(), + Array.Empty().AsReadOnly()), Throws.ArgumentException); + } + + + [Test] + public void AddAccountRange_ResponseHasEmptyListOfAccountsAndOneProof_ReturnsExpiredRootHash() + { + MemDb db = new(); + IDbProvider dbProvider = new DbProvider(); + dbProvider.RegisterDb(DbNames.State, db); + using ProgressTracker progressTracker = new(Substitute.For(), dbProvider.GetDb(DbNames.State), LimboLogs.Instance); + dbProvider.RegisterDb(DbNames.Code, new MemDb()); + AccountRange accountRange = new(Keccak.Zero, Keccak.Zero, Keccak.MaxValue); + using AccountsAndProofs accountsAndProofs = new(); + accountsAndProofs.PathAndAccounts = new List().ToPooledList(); + accountsAndProofs.Proofs = new List { new byte[] { 0x0 } }.ToPooledList(); + + SnapProvider sut = new(progressTracker, dbProvider.CodeDb, new NodeStorage(dbProvider.StateDb), LimboLogs.Instance); + + sut.AddAccountRange(accountRange, accountsAndProofs).Should().Be(AddRangeResult.ExpiredRootHash); + } + +} diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapServerTest.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapServerTest.cs index 50585fec605..3907b0237c5 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapServerTest.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/SnapServerTest.cs @@ -72,6 +72,52 @@ public void TestGetAccountRange() proofs.Dispose(); } + [Test] + public void TestGetAccountRange_InvalidRange() + { + Context context = CreateContext(); + TestItem.Tree.FillStateTreeWithTestAccounts(context.Tree); + + (IOwnedReadOnlyList accounts, IOwnedReadOnlyList proofs) = + context.Server.GetAccountRanges(context.Tree.RootHash, Keccak.MaxValue, Keccak.Zero, 4000, CancellationToken.None); + + accounts.Count.Should().Be(0); + accounts.Dispose(); + proofs.Dispose(); + } + + [Test] + public void TestGetTrieNode_Root() + { + Context context = CreateContext(); + TestItem.Tree.FillStateTreeWithTestAccounts(context.Tree); + + using IOwnedReadOnlyList result = context.Server.GetTrieNodes([ + new PathGroup() + { + Group = [[]] + } + ], context.Tree.RootHash, default)!; + + result.Count.Should().Be(1); + } + + [Test] + public void TestGetTrieNode_Storage_Root() + { + Context context = CreateContext(); + TestItem.Tree.FillStateTreeWithTestAccounts(context.Tree); + + using IOwnedReadOnlyList result = context.Server.GetTrieNodes([ + new PathGroup() + { + Group = [TestItem.Tree.AccountsWithPaths[0].Path.Bytes.ToArray(), []] + } + ], context.Tree.RootHash, default)!; + + result.Count.Should().Be(1); + } + [Test] public void TestNoState() { diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncPeersReportTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncPeersReportTests.cs index e3336c2ff6a..9bde37b91c3 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncPeersReportTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncPeersReportTests.cs @@ -165,8 +165,8 @@ public void PeerFormatIsCorrect() "== Header ==" + Environment.NewLine + "===[Active][Sleep ][Peer(ProtocolVersion/Head/Host:Port/Direction)][Transfer Speeds (L/H/B/R/N/S) ][Client Info (Name/Version/Operating System/Language) ]" + Environment.NewLine + "--------------------------------------------------------------------------------------------------------------------------------------------------------------" + Environment.NewLine + - " [HBRNSW][ ][Peer|eth99| 9999| 127.0.0.1: 3030| Out][ | | | | | ][]" + Environment.NewLine + - " [ ][HBRNSW][Peer|eth99| 9999| 127.0.0.1: 3030| In][ | | | | | ][]"; + " [HBRNS ][ ][Peer|eth99| 9999| 127.0.0.1: 3030| Out][ | | | | | ][]" + Environment.NewLine + + " [ ][HBRNS ][Peer|eth99| 9999| 127.0.0.1: 3030| In][ | | | | | ][]"; SyncPeersReport report = new(syncPeerPool, Substitute.For(), NoErrorLimboLogs.Instance); string reportStr = report.MakeReportForPeers(peers, "== Header =="); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs index adce701b001..28a8b87709e 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs @@ -102,7 +102,6 @@ public void Can_accept_new_valid_blocks(bool sealOk, bool validationOk, bool acc ctx.PeerPool, StaticSelector.Full, new SyncConfig(), - NullWitnessCollector.Instance, Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -145,7 +144,6 @@ public void Can_accept_blocks_that_are_fine() ctx.PeerPool, StaticSelector.Full, new SyncConfig(), - NullWitnessCollector.Instance, Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -214,7 +212,6 @@ public void Terminal_block_with_lower_td_should_not_change_best_suggested_but_sh ctx.PeerPool, StaticSelector.Full, new SyncConfig(), - NullWitnessCollector.Instance, Policy.FullGossip, testSpecProvider, LimboLogs.Instance); @@ -438,7 +435,6 @@ private Context CreateMergeContext(int blockTreeChainLength, UInt256 ttd) ctx.PeerPool, StaticSelector.Full, new SyncConfig(), - NullWitnessCollector.Instance, Policy.FullGossip, testSpecProvider, LimboLogs.Instance); @@ -479,7 +475,6 @@ public void Will_not_reject_block_with_bad_total_diff_but_will_reset_diff_to_nul ctx.PeerPool, StaticSelector.Full, new SyncConfig(), - NullWitnessCollector.Instance, Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -512,7 +507,6 @@ public void Rejects_new_old_blocks() ctx.PeerPool, StaticSelector.Full, new SyncConfig(), - NullWitnessCollector.Instance, Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -540,7 +534,6 @@ public async Task Broadcast_NewBlock_on_arrival() ctx.PeerPool, StaticSelector.Full, new SyncConfig(), - NullWitnessCollector.Instance, Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -577,7 +570,6 @@ public async Task Skip_known_block() ctx.PeerPool, StaticSelector.Full, new SyncConfig(), - NullWitnessCollector.Instance, Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -619,7 +611,6 @@ public async Task Broadcast_NewBlock_on_arrival_to_sqrt_of_peers([Values(1, 2, 3 ctx.PeerPool, StaticSelector.Full, new SyncConfig(), - NullWitnessCollector.Instance, Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -658,7 +649,6 @@ public void GetNodeData_returns_cached_trie_nodes() ctx.PeerPool, StaticSelector.Full, new SyncConfig(), - NullWitnessCollector.Instance, Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); @@ -693,7 +683,6 @@ public Context() PeerPool, selector, new SyncConfig(), - NullWitnessCollector.Instance, Policy.FullGossip, MainnetSpecProvider.Instance, LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index f55b12c2eca..e31072c5562 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -228,12 +228,13 @@ private class SyncTestContext public IWorldState StateProvider { get; set; } = null!; public DevBlockProducer? BlockProducer { get; set; } + public IBlockProducerRunner? BlockProducerRunner { get; set; } public ConsoleAsyncLogger? Logger { get; set; } public async Task StopAsync() { await (BlockchainProcessor?.StopAsync() ?? Task.CompletedTask); - await (BlockProducer?.StopAsync() ?? Task.CompletedTask); + await (BlockProducerRunner?.StopAsync() ?? Task.CompletedTask); await (PeerPool?.StopAsync() ?? Task.CompletedTask); await (Synchronizer?.StopAsync() ?? Task.CompletedTask); Logger?.Flush(); @@ -275,7 +276,8 @@ private SyncTestContext CreateSyncManager(int index) logManager, transactionComparerProvider.GetDefaultComparer()); BlockhashProvider blockhashProvider = new(tree, specProvider, stateProvider, LimboLogs.Instance); - VirtualMachine virtualMachine = new(blockhashProvider, specProvider, logManager); + CodeInfoRepository codeInfoRepository = new(); + VirtualMachine virtualMachine = new(blockhashProvider, specProvider, codeInfoRepository, logManager); Always sealValidator = Always.Valid; HeaderValidator headerValidator = new(tree, sealValidator, specProvider, logManager); @@ -290,7 +292,7 @@ private SyncTestContext CreateSyncManager(int index) RewardCalculator rewardCalculator = new(specProvider); TransactionProcessor txProcessor = - new(specProvider, stateProvider, virtualMachine, logManager); + new(specProvider, stateProvider, virtualMachine, codeInfoRepository, logManager); BlockProcessor blockProcessor = new( specProvider, @@ -299,7 +301,6 @@ private SyncTestContext CreateSyncManager(int index) new BlockProcessor.BlockValidationTransactionsExecutor(txProcessor, stateProvider), stateProvider, receiptStorage, - NullWitnessCollector.Instance, new BlockhashStore(tree, specProvider, stateProvider), txProcessor, logManager); @@ -313,8 +314,8 @@ private SyncTestContext CreateSyncManager(int index) SyncPeerPool syncPeerPool = new(tree, nodeStatsManager, new TotalDifficultyBetterPeerStrategy(LimboLogs.Instance), logManager, 25); WorldState devState = new(trieStore, codeDb, logManager); - VirtualMachine devEvm = new(blockhashProvider, specProvider, logManager); - TransactionProcessor devTxProcessor = new(specProvider, devState, devEvm, logManager); + VirtualMachine devEvm = new(blockhashProvider, specProvider, codeInfoRepository, logManager); + TransactionProcessor devTxProcessor = new(specProvider, devState, devEvm, codeInfoRepository, logManager); BlockProcessor devBlockProcessor = new( specProvider, @@ -323,7 +324,6 @@ private SyncTestContext CreateSyncManager(int index) new BlockProcessor.BlockProductionTransactionsExecutor(devTxProcessor, devState, specProvider, logManager), devState, receiptStorage, - NullWitnessCollector.Instance, new BlockhashStore(tree, specProvider, devState), devTxProcessor, logManager); @@ -341,12 +341,16 @@ private SyncTestContext CreateSyncManager(int index) devChainProcessor, stateProvider, tree, - new BuildBlocksRegularly(TimeSpan.FromMilliseconds(50)).IfPoolIsNotEmpty(txPool), Timestamper.Default, specProvider, new BlocksConfig(), logManager); + StandardBlockProducerRunner runner = new( + new BuildBlocksRegularly(TimeSpan.FromMilliseconds(50)).IfPoolIsNotEmpty(txPool), + tree, + producer); + TotalDifficultyBetterPeerStrategy bestPeerStrategy = new(LimboLogs.Instance); Pivot pivot = new(syncConfig); BlockDownloaderFactory blockDownloaderFactory = new( @@ -383,7 +387,6 @@ private SyncTestContext CreateSyncManager(int index) syncPeerPool, selector, syncConfig, - NullWitnessCollector.Instance, Policy.FullGossip, MainnetSpecProvider.Instance, logManager); @@ -394,7 +397,7 @@ private SyncTestContext CreateSyncManager(int index) if (index == 0) { _genesis = Build.A.Block.Genesis.WithStateRoot(stateProvider.StateRoot).TestObject; - producer.Start(); + runner.Start(); } syncPeerPool.Start(); @@ -416,6 +419,7 @@ private SyncTestContext CreateSyncManager(int index) context.SyncServer = syncServer; context.Tree = tree; context.BlockProducer = producer; + context.BlockProducerRunner = runner; context.TxPool = txPool; context.Logger = logger; return context; diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index 1d1aba2c6d9..c2f2e08128e 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -32,7 +32,6 @@ using Nethermind.Merge.Plugin.Test; using Nethermind.Specs.ChainSpecStyle; using Nethermind.State; -using Nethermind.State.Witnesses; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; @@ -413,7 +412,6 @@ ISyncConfig GetSyncConfig() => SyncPeerPool, Synchronizer.SyncModeSelector, syncConfig, - new WitnessCollector(new MemDb(), LimboLogs.Instance), Policy.FullGossip, MainnetSpecProvider.Instance, _logManager); @@ -437,7 +435,7 @@ public SyncingContext BlockIsKnown() return this; } - private const int DynamicTimeout = 5000; + private const int DynamicTimeout = 10000; public SyncingContext BestSuggestedHeaderIs(BlockHeader header) { @@ -975,8 +973,7 @@ await When.Syncing .StopAsync(); } - private const int Moment = 50; - private const int WaitTime = 500; + private const int WaitTime = 1500; private static bool IsMerge(SynchronizerType synchronizerType) => synchronizerType switch diff --git a/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloadContext.cs b/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloadContext.cs index ff8e7090c87..12ce99de7f1 100644 --- a/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloadContext.cs +++ b/src/Nethermind/Nethermind.Synchronization/Blocks/BlockDownloadContext.cs @@ -21,6 +21,7 @@ public class BlockDownloadContext private readonly PeerInfo _syncPeer; private bool _downloadReceipts; private readonly IReceiptsRecovery _receiptsRecovery; + private static readonly IRlpStreamDecoder _receiptDecoder = Rlp.GetStreamDecoder(); public BlockDownloadContext(ISpecProvider specProvider, PeerInfo syncPeer, IReadOnlyList headers, bool downloadReceipts, IReceiptsRecovery receiptsRecovery) @@ -126,7 +127,7 @@ public Block GetBlockByRequestIdx(int index) private void ValidateReceipts(Block block, TxReceipt[] blockReceipts) { - Hash256 receiptsRoot = ReceiptTrie.CalculateRoot(_specProvider.GetSpec(block.Header), blockReceipts, ReceiptMessageDecoder.Instance); + Hash256 receiptsRoot = ReceiptTrie.CalculateRoot(_specProvider.GetSpec(block.Header), blockReceipts, _receiptDecoder); if (receiptsRoot != block.ReceiptsRoot) { diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/FastHeadersSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/FastHeadersSyncFeed.cs index a5b175233c7..b5c5d2798d4 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/FastHeadersSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/FastHeadersSyncFeed.cs @@ -4,7 +4,9 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; +using System.Runtime.CompilerServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -137,14 +139,7 @@ private ulong CalculateMemoryInQueue() ulong amount = 0; while (enumerator.MoveNext()) { - var responses = enumerator.Current.Value.Response; - if (responses is not null) - { - foreach (var response in responses) - { - amount += (ulong)MemorySizeEstimator.EstimateSize(response); - } - } + amount += (ulong)enumerator.Current.Value?.ResponseSizeEstimate; } // Stop gap method to reduce allocations from non-struct enumerator @@ -250,6 +245,7 @@ protected virtual void FinishAndCleanUp() protected void ClearDependencies() { + _dependencies.Values.DisposeItems(); _dependencies.Clear(); MarkDirty(); } @@ -259,7 +255,9 @@ protected virtual void PostFinishCleanUp() HeadersSyncProgressReport.Update(_pivotNumber); HeadersSyncProgressReport.MarkEnd(); ClearDependencies(); // there may be some dependencies from wrong branches + _pending.DisposeItems(); _pending.Clear(); // there may be pending wrong branches + _sent.DisposeItems(); _sent.Clear(); // we my still be waiting for some bad branches HeadersSyncQueueReport.Update(0L); HeadersSyncQueueReport.MarkEnd(); @@ -270,14 +268,17 @@ private void HandleDependentBatches(CancellationToken cancellationToken) long? lowest = LowestInsertedBlockHeader?.Number; long processedBatchCount = 0; const long maxBatchToProcess = 4; - while (lowest.HasValue && processedBatchCount < maxBatchToProcess && _dependencies.TryRemove(lowest.Value - 1, out HeadersSyncBatch? dependentBatch)) + while (lowest.HasValue && processedBatchCount < maxBatchToProcess && _dependencies.TryRemove(lowest.Value - 1, out HeadersSyncBatch dependentBatch)) { - MarkDirty(); - InsertHeaders(dependentBatch!); - lowest = LowestInsertedBlockHeader?.Number; - cancellationToken.ThrowIfCancellationRequested(); + using (dependentBatch) + { + MarkDirty(); + InsertHeaders(dependentBatch); + lowest = LowestInsertedBlockHeader?.Number; + cancellationToken.ThrowIfCancellationRequested(); - processedBatchCount++; + processedBatchCount++; + } } } @@ -449,7 +450,7 @@ private static HeadersSyncBatch BuildLeftFiller(HeadersSyncBatch batch, int left private static HeadersSyncBatch BuildDependentBatch(HeadersSyncBatch batch, long addedLast, long addedEarliest) { HeadersSyncBatch dependentBatch = new(); - dependentBatch.StartNumber = batch.StartNumber; + dependentBatch.StartNumber = addedEarliest; int count = (int)(addedLast - addedEarliest + 1); dependentBatch.RequestSize = count; dependentBatch.MinNumber = batch.MinNumber; @@ -567,26 +568,30 @@ protected virtual int InsertHeaders(HeadersSyncBatch batch) _pending.Enqueue(batch); throw new InvalidOperationException($"Only one header dependency expected ({batch})"); } - + long lastNumber = -1; for (int j = 0; j < batch.Response.Count; j++) { BlockHeader? current = batch.Response[j]; if (current is not null) { + if (lastNumber != -1 && lastNumber < current.Number - 1) + { + //There is a gap in this response, + //so we save the whole batch for now, + //and let the next PrepareRequest() handle the disconnect + addedEarliest = batch.StartNumber; + addedLast = batch.EndNumber; + break; + } addedEarliest = Math.Min(addedEarliest, current.Number); addedLast = Math.Max(addedLast, current.Number); - } - else - { - break; + lastNumber = current.Number; } } - HeadersSyncBatch dependentBatch = BuildDependentBatch(batch, addedLast, addedEarliest); _dependencies[header.Number] = dependentBatch; MarkDirty(); if (_logger.IsDebug) _logger.Debug($"{batch} -> DEPENDENCY {dependentBatch}"); - // but we cannot do anything with it yet break; } @@ -636,6 +641,7 @@ protected virtual int InsertHeaders(HeadersSyncBatch batch) { if (added <= 0) { + batch.Response?.Dispose(); batch.Response = null; _pending.Enqueue(batch); } @@ -709,5 +715,17 @@ protected void SetExpectedNextHeaderToParent(BlockHeader header) _nextHeaderHash = header.ParentHash!; _nextHeaderDiff = (header.TotalDifficulty ?? 0) - header.Difficulty; } + private bool _disposed = false; + public override void Dispose() + { + if (!_disposed) + { + _sent.DisposeItems(); + _pending.DisposeItems(); + _dependencies.Values.DisposeItems(); + base.Dispose(); + _disposed = true; + } + } } } diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/HeadersSyncBatch.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/HeadersSyncBatch.cs index 9a5a8345023..aed3eabf42c 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/HeadersSyncBatch.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/HeadersSyncBatch.cs @@ -11,7 +11,27 @@ public class HeadersSyncBatch : FastBlocksBatch public long StartNumber { get; set; } public long EndNumber => StartNumber + RequestSize - 1; public int RequestSize { get; set; } - public IOwnedReadOnlyList? Response { get; set; } + public long ResponseSizeEstimate { get; private set; } + + private IOwnedReadOnlyList? _response; + public IOwnedReadOnlyList? Response + { + get => _response; + set + { + ResponseSizeEstimate = 0; + if (value is not null) + { + long size = 0; + foreach (BlockHeader response in value) + { + size += MemorySizeEstimator.EstimateSize(response); + } + ResponseSizeEstimate = size; + } + _response = value; + } + } public override string ToString() { diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncItem.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncItem.cs index c6d58b64389..5795bdd7458 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncItem.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/StateSyncItem.cs @@ -3,7 +3,9 @@ using System; using System.Diagnostics; -using Nethermind.Core; +using System.Numerics; +using System.Runtime.InteropServices; + using Nethermind.Core.Crypto; using Nethermind.Trie; @@ -57,6 +59,35 @@ public StateSyncItem(Hash256 hash, byte[]? accountPathNibbles, byte[]? pathNibbl private NodeKey? _key = null; public NodeKey Key => _key ??= new(Address, Path, Hash); - public record NodeKey(Hash256? Address, TreePath? Path, Hash256 Hash); + [StructLayout(LayoutKind.Auto)] + public readonly struct NodeKey(Hash256? address, TreePath? path, Hash256 hash) : IEquatable + { + private readonly ValueHash256 Address = address ?? default; + private readonly TreePath? Path = path; + private readonly ValueHash256 Hash = hash; + + public readonly bool Equals(NodeKey other) + => Address == other.Address && Path == other.Path && Hash == other.Hash; + + public override bool Equals(object obj) + => obj is NodeKey && Equals((NodeKey)obj); + + public override int GetHashCode() + { + uint hash0 = (uint)hash.GetHashCode(); + ulong hash1 = ((ulong)(uint)(address?.GetHashCode() ?? 1) << 32) | (ulong)(uint)(Path?.GetHashCode() ?? 2); + return (int)BitOperations.Crc32C(hash0, hash1); + } + + public static bool operator ==(in NodeKey left, in NodeKey right) + { + return left.Equals(right); + } + + public static bool operator !=(in NodeKey left, in NodeKey right) + { + return !(left == right); + } + } } } diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs index 4f53ea7dea0..4cc449cb47b 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs @@ -67,7 +67,7 @@ public class TreeSync private readonly ConcurrentDictionary _pendingRequests = new(); private Dictionary> _dependencies = new(); private readonly LruKeyCache _alreadySavedNode = new(AlreadySavedCapacity, "saved nodes"); - private readonly LruKeyCache _alreadySavedCode = new(AlreadySavedCapacity, "saved nodes"); + private readonly LruKeyCache _alreadySavedCode = new(AlreadySavedCapacity, "saved nodes"); private BranchProgress _branchProgress; private int _hintsToResetRoot; diff --git a/src/Nethermind/Nethermind.Synchronization/ISyncServer.cs b/src/Nethermind/Nethermind.Synchronization/ISyncServer.cs index a7228c0e224..3a14b7b96f9 100644 --- a/src/Nethermind/Nethermind.Synchronization/ISyncServer.cs +++ b/src/Nethermind/Nethermind.Synchronization/ISyncServer.cs @@ -10,7 +10,6 @@ using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Synchronization.FastSync; -using Nethermind.Synchronization.LesSync; namespace Nethermind.Synchronization { @@ -21,9 +20,6 @@ public interface ISyncServer : IDisposable void StopNotifyingPeersAboutNewBlocks(); TxReceipt[] GetReceipts(Hash256 blockHashes); Block? Find(Hash256 hash); - BlockHeader FindLowestCommonAncestor(BlockHeader firstDescendant, BlockHeader secondDescendant); - public Task BuildCHT(); - public CanonicalHashTrie? GetCHT(); Hash256? FindHash(long number); IOwnedReadOnlyList FindHeaders(Hash256 hash, int numberOfBlocks, int skip, bool reverse); IOwnedReadOnlyList GetNodeData(IReadOnlyList keys, CancellationToken cancellationToken, NodeDataType includedTypes = NodeDataType.Code | NodeDataType.State); @@ -31,6 +27,5 @@ public interface ISyncServer : IDisposable ulong NetworkId { get; } BlockHeader Genesis { get; } BlockHeader? Head { get; } - Hash256[]? GetBlockWitnessHashes(Hash256 blockHash); } } diff --git a/src/Nethermind/Nethermind.Synchronization/LesSync/CanonicalHashTrie.cs b/src/Nethermind/Nethermind.Synchronization/LesSync/CanonicalHashTrie.cs deleted file mode 100644 index 73374859419..00000000000 --- a/src/Nethermind/Nethermind.Synchronization/LesSync/CanonicalHashTrie.cs +++ /dev/null @@ -1,142 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; -using Nethermind.Core; -using Nethermind.Core.Crypto; -using Nethermind.Core.Extensions; -using Nethermind.Logging; -using Nethermind.Int256; -using Nethermind.Serialization.Rlp; -using Nethermind.Trie; - -namespace Nethermind.Synchronization.LesSync -{ - public class CanonicalHashTrie : PatriciaTree - { - private static readonly ChtDecoder _decoder = new(); - public static readonly int SectionSize = 32768; // 2**15 - - private static readonly byte[] MaxSectionKey = Encoding.ASCII.GetBytes("MaxSection"); - - public CanonicalHashTrie(IKeyValueStoreWithBatching db) - : base(db, GetMaxRootHash(db), true, true, NullLogManager.Instance) - { - } - - public CanonicalHashTrie(IKeyValueStoreWithBatching db, Hash256 rootHash) - : base(db, rootHash, true, true, NullLogManager.Instance) - { - } - - public void CommitSectionIndex(long sectionIndex) - { - StoreRootHash(sectionIndex); - } - - public static long GetMaxSectionIndex() - { - //return GetMaxSectionIndex(_keyValueStore); - return -1; - } - - public static long GetSectionFromBlockNo(long blockNo) => (blockNo / SectionSize) - 1L; - - public static byte[][] BuildProof(long blockNo, long sectionIndex, long fromLevel) - { - return BuildProof(GetKey(blockNo), sectionIndex, fromLevel); - } - - public static byte[][] BuildProof(byte[] key, long sectionIndex, long fromLevel) - { - ChtProofCollector proofCollector = new(key, fromLevel); - //Accept(proofCollector, GetRootHash(sectionIndex), false); - return proofCollector.BuildResult(); - } - - private void StoreRootHash(long sectionIndex) - { - UpdateRootHash(); - //_keyValueStore[GetRootHashKey(sectionIndex)] = RootHash.Bytes; - //if (GetMaxSectionIndex(_keyValueStore) < sectionIndex) - //{ - // SetMaxSectionIndex(sectionIndex); - //} - } - - private static long GetMaxSectionIndex(IKeyValueStore db) - { - byte[]? storeValue = null; - try - { - storeValue = db[MaxSectionKey]; - } - catch (KeyNotFoundException) { } - return storeValue?.ToLongFromBigEndianByteArrayWithoutLeadingZeros() ?? -1L; - } - - private static Hash256 GetRootHash(IKeyValueStore db, long sectionIndex) - { - byte[]? hash = db[GetRootHashKey(sectionIndex)]; - return hash is null ? EmptyTreeHash : new Hash256(hash); - } - - private static Hash256 GetMaxRootHash(IKeyValueStore db) - { - long maxSection = GetMaxSectionIndex(db); - return maxSection == 0L ? EmptyTreeHash : GetRootHash(db, maxSection); - } - - public void Set(BlockHeader header) - { - Set(GetKey(header), GetValue(header)); - } - - public (Hash256?, UInt256) Get(long key) - { - return Get(GetKey(key)); - } - - public (Hash256?, UInt256) Get(Span key) - { - ReadOnlySpan val = base.Get(key); - if (val.IsEmpty) - { - throw new InvalidDataException("Missing CHT data"); - } - - return _decoder.Decode(val); - } - - private static byte[] GetKey(BlockHeader header) - { - return GetKey(header.Number); - } - - private static byte[] GetKey(long key) - { - return key.ToBigEndianByteArrayWithoutLeadingZeros().PadLeft(8); - } - - private static byte[] GetRootHashKey(long key) - { - return Bytes.Concat(Encoding.ASCII.GetBytes("RootHash"), GetKey(key)); - } - - private static Rlp GetValue(BlockHeader header) - { - if (!header.TotalDifficulty.HasValue) - { - throw new ArgumentException("Trying to use a header with a null total difficulty in LES Canonical Hash Trie"); - } - - (Hash256? Hash, UInt256 Value) item = (header.Hash, header.TotalDifficulty.Value); - RlpStream stream = new(_decoder.GetLength(item, RlpBehaviors.None)); - _decoder.Encode(stream, item); - return new Rlp(stream.Data.ToArray()); - } - } -} diff --git a/src/Nethermind/Nethermind.Synchronization/LesSync/ChtDecoder.cs b/src/Nethermind/Nethermind.Synchronization/LesSync/ChtDecoder.cs deleted file mode 100644 index 7fbfbb7e82e..00000000000 --- a/src/Nethermind/Nethermind.Synchronization/LesSync/ChtDecoder.cs +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System; -using Nethermind.Core.Crypto; -using Nethermind.Int256; -using Nethermind.Serialization.Rlp; - -namespace Nethermind.Synchronization.LesSync -{ - public class ChtDecoder : IRlpStreamDecoder<(Hash256?, UInt256)>, IRlpValueDecoder<(Hash256?, UInt256)> - { - public (Hash256?, UInt256) Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBehaviors.None) - { - if (rlpStream.IsNextItemNull()) - { - rlpStream.ReadByte(); - return (null, 0); - } - - rlpStream.ReadSequenceLength(); - Hash256 hash = rlpStream.DecodeKeccak(); - UInt256 totalDifficulty = rlpStream.DecodeUInt256(); - return (hash, totalDifficulty); - } - - public void Encode(RlpStream stream, (Hash256?, UInt256) item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) - { - (Hash256? hash, UInt256 totalDifficulty) = item; - int contentLength = GetContentLength(item, RlpBehaviors.None); - stream.StartSequence(contentLength); - stream.Encode(hash); - stream.Encode(totalDifficulty); - } - - public (Hash256?, UInt256) Decode(byte[] bytes) - { - return Decode(new RlpStream(bytes)); - } - - public Rlp Encode((Hash256?, UInt256) item, RlpBehaviors rlpBehaviors = RlpBehaviors.None) - { - throw new NotImplementedException(); - } - - public int GetLength((Hash256?, UInt256) item, RlpBehaviors rlpBehaviors) - { - return Rlp.LengthOfSequence(GetContentLength(item, rlpBehaviors)); - } - - private static int GetContentLength((Hash256?, UInt256) item, RlpBehaviors rlpBehaviors) - { - (Hash256? hash, UInt256 totalDifficulty) = item; - return Rlp.LengthOf(hash) + Rlp.LengthOf(totalDifficulty); - } - - public (Hash256?, UInt256) Decode(ref Rlp.ValueDecoderContext decoderContext, RlpBehaviors rlpBehaviors = RlpBehaviors.None) - { - if (decoderContext.IsNextItemNull()) - { - decoderContext.ReadByte(); - return (null, 0); - } - - decoderContext.ReadSequenceLength(); - Hash256 hash = decoderContext.DecodeKeccak(); - UInt256 totalDifficulty = decoderContext.DecodeUInt256(); - return (hash, totalDifficulty); - } - } -} diff --git a/src/Nethermind/Nethermind.Synchronization/LesSync/ChtProofCollector.cs b/src/Nethermind/Nethermind.Synchronization/LesSync/ChtProofCollector.cs deleted file mode 100644 index 910c5a1473e..00000000000 --- a/src/Nethermind/Nethermind.Synchronization/LesSync/ChtProofCollector.cs +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using Nethermind.State.Proofs; -using Nethermind.Trie; - -namespace Nethermind.Synchronization.LesSync -{ - class ChtProofCollector : ProofCollector - { - readonly long _fromLevel; - long _level; - public ChtProofCollector(byte[] key, long fromLevel) : base(key) - { - _fromLevel = fromLevel; - _level = 0; - } - - protected override void AddProofBits(TrieNode node) - { - if (_level < _fromLevel) - { - _level++; - } - else - { - base.AddProofBits(node); - } - } - } -} diff --git a/src/Nethermind/Nethermind.Synchronization/Metrics.cs b/src/Nethermind/Nethermind.Synchronization/Metrics.cs index ac3f91ee690..68ffc7f8012 100644 --- a/src/Nethermind/Nethermind.Synchronization/Metrics.cs +++ b/src/Nethermind/Nethermind.Synchronization/Metrics.cs @@ -71,14 +71,6 @@ public static class Metrics [Description("State branch progress (percentage of completed branches at second level).")] public static long StateBranchProgress; - [GaugeMetric] - [Description("Requests sent for processing by the witness state sync")] - public static long WitnessStateRequests; - - [GaugeMetric] - [Description("Requests sent for processing by the witness block sync")] - public static long WitnessBlockRequests; - [GaugeMetric] [Description("Sync time in seconds")] public static long SyncTime; diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs index c573ef21289..c33e126b4f3 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs @@ -199,7 +199,7 @@ public void Update() best.IsInFastSync = ShouldBeInFastSyncMode(best); best.IsInStateSync = ShouldBeInStateSyncMode(best); best.IsInStateNodes = ShouldBeInStateNodesMode(best); - best.IsInSnapRanges = ShouldBeBeInSnapRangesPhase(best); + best.IsInSnapRanges = ShouldBeInSnapRangesPhase(best); best.IsInFastHeaders = ShouldBeInFastHeadersMode(best); best.IsInFastBodies = ShouldBeInFastBodiesMode(best); best.IsInFastReceipts = ShouldBeInFastReceiptsMode(best); @@ -338,15 +338,16 @@ private bool ShouldBeInWaitingForBlockMode(Snapshot best) private bool ShouldBeInBeaconHeaders(bool shouldBeInUpdatingPivot) { bool shouldBeInBeaconHeaders = _beaconSyncStrategy.ShouldBeInBeaconHeaders(); + bool shouldBeNotInUpdatingPivot = !shouldBeInUpdatingPivot; bool result = shouldBeInBeaconHeaders && - !shouldBeInUpdatingPivot; + shouldBeNotInUpdatingPivot; if (_logger.IsTrace) { LogDetailedSyncModeChecks("BEACON HEADERS", (nameof(shouldBeInBeaconHeaders), shouldBeInBeaconHeaders), - (nameof(shouldBeInUpdatingPivot), shouldBeInUpdatingPivot)); + (nameof(shouldBeNotInUpdatingPivot), shouldBeNotInUpdatingPivot)); } return result; @@ -613,7 +614,7 @@ private bool ShouldBeInStateSyncMode(Snapshot best) (nameof(notInUpdatingPivot), notInUpdatingPivot), (nameof(hasFastSyncBeenActive), hasFastSyncBeenActive), (nameof(hasAnyPostPivotPeer), hasAnyPostPivotPeer), - (nameof(notInFastSync), notInFastSync), + ($"{nameof(notInFastSync)}||{nameof(stickyStateNodes)}", notInFastSync || stickyStateNodes), (nameof(stateNotDownloadedYet), stateNotDownloadedYet), (nameof(notInAStickyFullSync), notInAStickyFullSync), (nameof(notHasJustStartedFullSync), notHasJustStartedFullSync), @@ -642,7 +643,7 @@ private bool ShouldBeInStateNodesMode(Snapshot best) return result; } - private bool ShouldBeBeInSnapRangesPhase(Snapshot best) + private bool ShouldBeInSnapRangesPhase(Snapshot best) { bool isInStateSync = best.IsInStateSync; bool isCloseToHead = best.TargetBlock >= best.Header && (best.TargetBlock - best.Header) < Constants.MaxDistanceFromHead; diff --git a/src/Nethermind/Nethermind.Synchronization/Peers/AllocationContexts.cs b/src/Nethermind/Nethermind.Synchronization/Peers/AllocationContexts.cs index 8361c2ffd65..0ed45626de6 100644 --- a/src/Nethermind/Nethermind.Synchronization/Peers/AllocationContexts.cs +++ b/src/Nethermind/Nethermind.Synchronization/Peers/AllocationContexts.cs @@ -14,8 +14,7 @@ public enum AllocationContexts Receipts = 4, Blocks = 7, State = 8, - Witness = 16, - Snap = 32, - All = Headers | Bodies | Receipts | Blocks | State | Witness | Snap + Snap = 16, + All = Headers | Bodies | Receipts | Blocks | State | Snap } } diff --git a/src/Nethermind/Nethermind.Synchronization/Peers/PeerInfo.cs b/src/Nethermind/Nethermind.Synchronization/Peers/PeerInfo.cs index de64e089455..3b6c7a8a409 100644 --- a/src/Nethermind/Nethermind.Synchronization/Peers/PeerInfo.cs +++ b/src/Nethermind/Nethermind.Synchronization/Peers/PeerInfo.cs @@ -155,9 +155,9 @@ private static void ResolveWeaknessChecks(ref int weakness, AllocationContexts s private static string BuildContextString(AllocationContexts contexts) { - return $"{((contexts & AllocationContexts.Headers) == AllocationContexts.Headers ? "H" : " ")}{((contexts & AllocationContexts.Bodies) == AllocationContexts.Bodies ? "B" : " ")}{((contexts & AllocationContexts.Receipts) == AllocationContexts.Receipts ? "R" : " ")}{((contexts & AllocationContexts.State) == AllocationContexts.State ? "N" : " ")}{((contexts & AllocationContexts.Snap) == AllocationContexts.Snap ? "S" : " ")}{((contexts & AllocationContexts.Witness) == AllocationContexts.Witness ? "W" : " ")}"; + return $"{((contexts & AllocationContexts.Headers) == AllocationContexts.Headers ? "H" : " ")}{((contexts & AllocationContexts.Bodies) == AllocationContexts.Bodies ? "B" : " ")}{((contexts & AllocationContexts.Receipts) == AllocationContexts.Receipts ? "R" : " ")}{((contexts & AllocationContexts.State) == AllocationContexts.State ? "N" : " ")}{((contexts & AllocationContexts.Snap) == AllocationContexts.Snap ? "S" : " ")}"; } - public override string ToString() => $"[{BuildContextString(AllocatedContexts)}][{BuildContextString(SleepingContexts)}]{SyncPeer}"; + public override string ToString() => $"[{BuildContextString(AllocatedContexts)} ][{BuildContextString(SleepingContexts)} ]{SyncPeer}"; } } diff --git a/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeersReport.cs b/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeersReport.cs index 39f179ffbe0..3ed69a9aef1 100644 --- a/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeersReport.cs +++ b/src/Nethermind/Nethermind.Synchronization/Peers/SyncPeersReport.cs @@ -135,7 +135,6 @@ static void CountContexts(AllocationContexts contexts, ref PeersContextCounts co contextCounts.Receipts += contexts.HasFlag(AllocationContexts.Receipts) ? 1 : 0; contextCounts.Blocks += contexts.HasFlag(AllocationContexts.Blocks) ? 1 : 0; contextCounts.State += contexts.HasFlag(AllocationContexts.State) ? 1 : 0; - contextCounts.Witness += contexts.HasFlag(AllocationContexts.Witness) ? 1 : 0; contextCounts.Snap += contexts.HasFlag(AllocationContexts.Snap) ? 1 : 0; } } @@ -240,7 +239,6 @@ private struct PeersContextCounts public int Receipts { get; set; } public int Blocks { get; set; } public int State { get; set; } - public int Witness { get; set; } public int Snap { get; set; } public int Total { get; set; } @@ -259,7 +257,6 @@ public readonly void AppendTo(StringBuilder sb, string allText) if (Receipts > 0) AddComma(sb, ref added).Append(Receipts).Append(" Receipts"); if (Blocks > 0) AddComma(sb, ref added).Append(Blocks).Append(" Blocks"); if (State > 0) AddComma(sb, ref added).Append(State).Append(" State"); - if (Witness > 0) AddComma(sb, ref added).Append(Witness).Append(" Witness"); if (Snap > 0) AddComma(sb, ref added).Append(Snap).Append(" Snap"); static StringBuilder AddComma(StringBuilder sb, ref bool itemAdded) diff --git a/src/Nethermind/Nethermind.Synchronization/Reporting/SyncReport.cs b/src/Nethermind/Nethermind.Synchronization/Reporting/SyncReport.cs index f51748f0b49..f088c7b1430 100644 --- a/src/Nethermind/Nethermind.Synchronization/Reporting/SyncReport.cs +++ b/src/Nethermind/Nethermind.Synchronization/Reporting/SyncReport.cs @@ -134,6 +134,7 @@ private static string Pad(long value, int length) private string _paddedAmountOfOldReceiptsToDownload; private long _amountOfBodiesToDownload; private long _amountOfReceiptsToDownload; + private uint _nodeInfoType; private void SetPaddedPivots() { @@ -175,8 +176,14 @@ private void WriteSyncReport() { if (_reportId % PeerCountFrequency == 0) { - _logger.Info(_syncPeersReport.MakeSummaryReportForPeers(_syncPeerPool.InitializedPeers, $"Peers | with best block: {_syncPeerPool.InitializedPeersCount} | all: {_syncPeerPool.PeerCount}")); - _logger.Info(_syncPeersReport.MakeDiversityReportForPeers(_syncPeerPool.InitializedPeers, $"Peers | node diversity : ")); + if (_nodeInfoType++ % 2 == 0) + { + _logger.Info(_syncPeersReport.MakeSummaryReportForPeers(_syncPeerPool.InitializedPeers, $"Peers | with best block: {_syncPeerPool.InitializedPeersCount} | all: {_syncPeerPool.PeerCount}")); + } + else + { + _logger.Info(_syncPeersReport.MakeDiversityReportForPeers(_syncPeerPool.InitializedPeers, $"Peers | node diversity : ")); + } } } diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index aa74716db65..772c227be98 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -31,7 +31,7 @@ public class SnapProvider : ISnapProvider private readonly ProgressTracker _progressTracker; // This is actually close to 97% effective. - private readonly LruKeyCache _codeExistKeyCache = new(1024 * 16, ""); + private readonly LruKeyCacheLowObject _codeExistKeyCache = new(1024 * 16, ""); public SnapProvider(ProgressTracker progressTracker, IDb codeDb, INodeStorage nodeStorage, ILogManager logManager) { @@ -51,7 +51,7 @@ public AddRangeResult AddAccountRange(AccountRange request, AccountsAndProofs re { AddRangeResult result; - if (response.PathAndAccounts.Count == 0 && response.Proofs.Count == 0) + if (response.PathAndAccounts.Count == 0) { _logger.Trace($"SNAP - GetAccountRange - requested expired RootHash:{request.RootHash}"); @@ -59,7 +59,13 @@ public AddRangeResult AddAccountRange(AccountRange request, AccountsAndProofs re } else { - result = AddAccountRange(request.BlockNumber.Value, request.RootHash, request.StartingHash, response.PathAndAccounts, response.Proofs, hashLimit: request.LimitHash); + result = AddAccountRange( + request.BlockNumber.Value, + request.RootHash, + request.StartingHash, + response.PathAndAccounts, + response.Proofs, + hashLimit: request.LimitHash); if (result == AddRangeResult.OK) { @@ -73,8 +79,16 @@ public AddRangeResult AddAccountRange(AccountRange request, AccountsAndProofs re return result; } - public AddRangeResult AddAccountRange(long blockNumber, in ValueHash256 expectedRootHash, in ValueHash256 startingHash, IReadOnlyList accounts, IReadOnlyList proofs = null, in ValueHash256? hashLimit = null!) + public AddRangeResult AddAccountRange( + long blockNumber, + in ValueHash256 expectedRootHash, + in ValueHash256 startingHash, + IReadOnlyList accounts, + IReadOnlyList proofs = null, + in ValueHash256? hashLimit = null!) { + if (accounts.Count == 0) + throw new ArgumentException("Cannot be empty.", nameof(accounts)); ITrieStore store = _trieStorePool.Get(); try { diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs index a3aaf5efa10..ea9dfc1f70a 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProviderHelper.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; @@ -31,7 +32,8 @@ public static (AddRangeResult result, bool moreChildrenToRight, List sortedBoundaryList, bool moreChildrenToRight) = diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapServer.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapServer.cs index ffdba2f1863..e18b9e1e36e 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapServer.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapServer.cs @@ -104,20 +104,20 @@ private bool IsRootMissing(in ValueHash256 stateRoot) default: try { - Hash256 storagePath = new Hash256(requestedPath[0]); + Hash256 storagePath = new Hash256( + requestedPath[0].Length == Hash256.Size + ? requestedPath[0] + : requestedPath[0].PadRight(Hash256.Size)); Account? account = GetAccountByPath(tree, rootHash, requestedPath[0]); if (account is not null) { Hash256? storageRoot = account.StorageRoot; - if (!storageRoot.Bytes.SequenceEqual(Keccak.EmptyTreeHash.Bytes)) - { - StorageTree sTree = new(_store.GetTrieStore(storagePath), storageRoot, _logManager); + StorageTree sTree = new(_store.GetTrieStore(storagePath), storageRoot, _logManager); - for (int reqStorage = 1; reqStorage < requestedPath.Length; reqStorage++) - { - byte[]? sRlp = sTree.GetNodeByPath(Nibbles.CompactToHexEncode(requestedPath[reqStorage])); - response.Add(sRlp); - } + for (int reqStorage = 1; reqStorage < requestedPath.Length; reqStorage++) + { + byte[]? sRlp = sTree.GetNodeByPath(Nibbles.CompactToHexEncode(requestedPath[reqStorage])); + response.Add(sRlp); } } } diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncDownloader.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncDownloader.cs index 28690a51013..32a6a0392dd 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncDownloader.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapSyncDownloader.cs @@ -52,8 +52,6 @@ public async Task Dispatch(PeerInfo peerInfo, SnapSyncBatch batch, CancellationT Logger.Error($"DEBUG/ERROR Error after dispatching the snap sync request. Request: {batch}", e); } } - - await Task.CompletedTask; } } } diff --git a/src/Nethermind/Nethermind.Synchronization/SyncServer.cs b/src/Nethermind/Nethermind.Synchronization/SyncServer.cs index bf00a1ad097..30ed536806e 100644 --- a/src/Nethermind/Nethermind.Synchronization/SyncServer.cs +++ b/src/Nethermind/Nethermind.Synchronization/SyncServer.cs @@ -22,9 +22,7 @@ using Nethermind.Core.Specs; using Nethermind.Int256; using Nethermind.Logging; -using Nethermind.State; using Nethermind.Synchronization.FastSync; -using Nethermind.Synchronization.LesSync; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; @@ -44,10 +42,8 @@ public class SyncServer : ISyncServer private readonly ISealValidator _sealValidator; private readonly IReadOnlyKeyValueStore _stateDb; private readonly IReadOnlyKeyValueStore _codeDb; - private readonly IWitnessRepository _witnessRepository; private readonly IGossipPolicy _gossipPolicy; private readonly ISpecProvider _specProvider; - private readonly CanonicalHashTrie? _cht; private bool _gossipStopped = false; private readonly Random _broadcastRandomizer = new(); @@ -67,14 +63,11 @@ public SyncServer( ISyncPeerPool pool, ISyncModeSelector syncModeSelector, ISyncConfig syncConfig, - IWitnessRepository? witnessRepository, IGossipPolicy gossipPolicy, ISpecProvider specProvider, - ILogManager logManager, - CanonicalHashTrie? cht = null) + ILogManager logManager) { ISyncConfig config = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); - _witnessRepository = witnessRepository ?? throw new ArgumentNullException(nameof(witnessRepository)); _gossipPolicy = gossipPolicy ?? throw new ArgumentNullException(nameof(gossipPolicy)); _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); _pool = pool ?? throw new ArgumentNullException(nameof(pool)); @@ -86,7 +79,6 @@ public SyncServer( _receiptFinder = receiptFinder ?? throw new ArgumentNullException(nameof(receiptFinder)); _blockValidator = blockValidator ?? throw new ArgumentNullException(nameof(blockValidator)); _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _cht = cht; _pivotNumber = config.PivotNumberParsed; _pivotHash = new Hash256(config.PivotHash ?? Keccak.Zero.ToString()); @@ -118,15 +110,7 @@ public BlockHeader? Head } } - public Hash256[]? GetBlockWitnessHashes(Hash256 blockHash) - { - return _witnessRepository.Load(blockHash); - } - - public int GetPeerCount() - { - return _pool.PeerCount; - } + public int GetPeerCount() => _pool.PeerCount; private readonly Guid _sealValidatorUserGuid = Guid.NewGuid(); @@ -403,11 +387,6 @@ public IOwnedReadOnlyList FindHeaders(Hash256 hash, int numberOfBlo return values; } - public BlockHeader FindLowestCommonAncestor(BlockHeader firstDescendant, BlockHeader secondDescendant) - { - return _blockTree.FindLowestCommonAncestor(firstDescendant, secondDescendant, Sync.MaxReorgLength); - } - public Block Find(Hash256 hash) => _blockTree.FindBlock(hash, BlockTreeLookupOptions.TotalDifficultyNotNeeded | BlockTreeLookupOptions.ExcludeTxHashes); public Hash256? FindHash(long number) @@ -466,49 +445,5 @@ public void Dispose() { StopNotifyingPeersAboutNewBlocks(); } - - private readonly object _chtLock = new(); - - // TODO - Cancellation token? - // TODO - not a fan of this function name - CatchUpCHT, AddMissingCHTBlocks, ...? - public Task BuildCHT() - { - return Task.CompletedTask; // removing LES code - -#pragma warning disable 162 - return Task.Run(() => - { - lock (_chtLock) - { - if (_cht is null) - { - throw new InvalidAsynchronousStateException("CHT reference is null when building CHT."); - } - - // Note: The spec says this should be 2048, but I don't think we'd ever want it to be higher than the max reorg depth we allow. - long maxSection = - CanonicalHashTrie.GetSectionFromBlockNo(_blockTree.FindLatestHeader().Number - - Sync.MaxReorgLength); - long maxKnownSection = CanonicalHashTrie.GetMaxSectionIndex(); - - for (long section = (maxKnownSection + 1); section <= maxSection; section++) - { - long sectionStart = section * CanonicalHashTrie.SectionSize; - for (int blockOffset = 0; blockOffset < CanonicalHashTrie.SectionSize; blockOffset++) - { - _cht.Set(_blockTree.FindHeader(sectionStart + blockOffset)); - } - - _cht.Commit(section); - } - } - }); -#pragma warning restore 162 - } - - public CanonicalHashTrie? GetCHT() - { - return _cht; - } } } diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index eb48d007cac..614ae1e50ea 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -457,7 +457,9 @@ public void Dispose() _syncReport.Dispose(); _fastSyncFeed?.Dispose(); _stateSyncFeed?.Dispose(); + _stateSyncFeed = null; SnapSyncFeed?.Dispose(); + _snapSyncFeed = null; _fullSyncFeed?.Dispose(); HeadersSyncFeed?.Dispose(); BodiesSyncFeed?.Dispose(); diff --git a/src/Nethermind/Nethermind.Synchronization/Trie/HealingWorldState.cs b/src/Nethermind/Nethermind.Synchronization/Trie/HealingWorldState.cs index 6ee9035a88f..3f66fc0de63 100644 --- a/src/Nethermind/Nethermind.Synchronization/Trie/HealingWorldState.cs +++ b/src/Nethermind/Nethermind.Synchronization/Trie/HealingWorldState.cs @@ -9,13 +9,9 @@ namespace Nethermind.Synchronization.Trie; -public class HealingWorldState : WorldState +public class HealingWorldState(ITrieStore trieStore, IKeyValueStore? codeDb, ILogManager? logManager, PreBlockCaches? preBlockCaches = null) + : WorldState(trieStore, codeDb, logManager, new HealingStateTree(trieStore, logManager), new HealingStorageTreeFactory(), preBlockCaches) { - public HealingWorldState(ITrieStore? trieStore, IKeyValueStore? codeDb, ILogManager? logManager) - : base(trieStore, codeDb, logManager, new HealingStateTree(trieStore, logManager), new HealingStorageTreeFactory()) - { - } - public void InitializeNetwork(ITrieNodeRecovery recovery) { StateProviderTree.InitializeNetwork(recovery); diff --git a/src/Nethermind/Nethermind.Test.Runner/BlockchainTestsRunner.cs b/src/Nethermind/Nethermind.Test.Runner/BlockchainTestsRunner.cs index ee23932b43e..efb3b8eeb22 100644 --- a/src/Nethermind/Nethermind.Test.Runner/BlockchainTestsRunner.cs +++ b/src/Nethermind/Nethermind.Test.Runner/BlockchainTestsRunner.cs @@ -35,7 +35,7 @@ public async Task> RunTestsAsync() Setup(); Console.Write($"{test,-120} "); - if (test.LoadFailure != null) + if (test.LoadFailure is not null) { WriteRed(test.LoadFailure); testResults.Add(new EthereumTestResult(test.Name, test.LoadFailure)); diff --git a/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs b/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs index 2b50fb04dcd..d127da5f076 100644 --- a/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs @@ -14,7 +14,6 @@ using Nethermind.Logging; using Nethermind.Serialization.Rlp; using Nethermind.State; -using Nethermind.State.Witnesses; using Nethermind.Trie.Pruning; using NSubstitute; using NUnit.Framework; @@ -682,30 +681,6 @@ public void Will_combine_same_storage() fullTrieStore.IsNodeCached(new Hash256(Nibbles.ToBytes(storage1Nib)), TreePath.Empty, storage1.Keccak).Should().BeTrue(); } - [Test] - public void ReadOnly_store_doesnt_change_witness() - { - TrieNode node = new(NodeType.Leaf); - Account account = new(1, 1, TestItem.KeccakA, Keccak.OfAnEmptyString); - node.Value = _accountDecoder.Encode(account).Bytes; - node.Key = Bytes.FromHexString("abc"); - TreePath emptyPath = TreePath.Empty; - node.ResolveKey(NullTrieNodeResolver.Instance, ref emptyPath, true); - - MemDb originalStore = new MemDb(); - WitnessCollector witnessCollector = new WitnessCollector(new MemDb(), LimboLogs.Instance); - IKeyValueStoreWithBatching store = originalStore.WitnessedBy(witnessCollector); - using TrieStore fullTrieStore = CreateTrieStore(pruningStrategy: new TestPruningStrategy(false), kvStore: store); - IScopedTrieStore trieStore = fullTrieStore.GetTrieStore(null); - trieStore.CommitNode(0, new NodeCommitInfo(node, TreePath.Empty)); - trieStore.FinishBlockCommit(TrieType.State, 0, node); - - IReadOnlyTrieStore readOnlyTrieStore = fullTrieStore.AsReadOnly(new NodeStorage(originalStore)); - readOnlyTrieStore.LoadRlp(null, TreePath.Empty, node.Keccak); - - witnessCollector.Collected.Should().BeEmpty(); - } - [TestCase(true)] [TestCase(false, Explicit = true)] public async Task Read_only_trie_store_is_allowing_many_thread_to_work_with_the_same_node(bool beThreadSafe) diff --git a/src/Nethermind/Nethermind.Trie.Test/TrieTests.cs b/src/Nethermind/Nethermind.Trie.Test/TrieTests.cs index 47953a99e84..0d9e05c1e1f 100644 --- a/src/Nethermind/Nethermind.Trie.Test/TrieTests.cs +++ b/src/Nethermind/Nethermind.Trie.Test/TrieTests.cs @@ -1013,7 +1013,7 @@ public void Fuzz_accounts_with_storage( MemDb memDb = new(); using TrieStore trieStore = new(memDb, Prune.WhenCacheReaches(1.MB()), Persist.IfBlockOlderThan(lookupLimit), _logManager); - WorldState stateProvider = new(trieStore, new MemDb(), _logManager); + WorldState stateProvider = new WorldState(trieStore, new MemDb(), _logManager); Account[] accounts = new Account[accountsCount]; Address[] addresses = new Address[accountsCount]; @@ -1062,7 +1062,7 @@ public void Fuzz_accounts_with_storage( address, existing.Balance - account.Balance, MuirGlacier.Instance); } - stateProvider.IncrementNonce(address); + stateProvider.IncrementNonce(address, UInt256.One); } byte[] storage = new byte[1]; diff --git a/src/Nethermind/Nethermind.Trie/INodeStorage.cs b/src/Nethermind/Nethermind.Trie/INodeStorage.cs index 0deb378ec00..91f8abc08d6 100644 --- a/src/Nethermind/Nethermind.Trie/INodeStorage.cs +++ b/src/Nethermind/Nethermind.Trie/INodeStorage.cs @@ -31,6 +31,7 @@ public interface INodeStorage /// Used by StateSync to make sure values are flushed. ///
void Flush(); + void Compact(); public enum KeyScheme { diff --git a/src/Nethermind/Nethermind.Trie/NodeStorage.cs b/src/Nethermind/Nethermind.Trie/NodeStorage.cs index c15dc6f2f38..066b5c19fb8 100644 --- a/src/Nethermind/Nethermind.Trie/NodeStorage.cs +++ b/src/Nethermind/Nethermind.Trie/NodeStorage.cs @@ -119,11 +119,11 @@ private static Span GetHashBasedStoragePath(Span pathSpan, in ValueH if (Scheme == INodeStorage.KeyScheme.HalfPath && (readFlags & ReadFlags.HintReadAhead) != 0) { - if (address == null && path.Length > TopStateBoundary) + if (address is null && path.Length > TopStateBoundary) { readFlags |= ReadFlags.HintReadAhead2; } - else if (address != null) + else if (address is not null) { readFlags |= ReadFlags.HintReadAhead3; } @@ -192,6 +192,14 @@ public void Flush() } } + public void Compact() + { + if (_keyValueStore is IDb db) + { + db.Compact(); + } + } + private class WriteBatch : INodeStorage.WriteBatch { private readonly IWriteBatch _writeBatch; diff --git a/src/Nethermind/Nethermind.Trie/NodeStorageFactory.cs b/src/Nethermind/Nethermind.Trie/NodeStorageFactory.cs index 579ad4be2af..113dbe5caa8 100644 --- a/src/Nethermind/Nethermind.Trie/NodeStorageFactory.cs +++ b/src/Nethermind/Nethermind.Trie/NodeStorageFactory.cs @@ -27,7 +27,7 @@ public NodeStorageFactory(INodeStorage.KeyScheme preferredKeyScheme, ILogManager public void DetectCurrentKeySchemeFrom(IDb mainStateDb) { _currentKeyScheme = DetectKeyScheme(mainStateDb); - if (_currentKeyScheme == null) + if (_currentKeyScheme is null) { _logger.Info("No current state db key scheme."); } diff --git a/src/Nethermind/Nethermind.Trie/PatriciaTree.cs b/src/Nethermind/Nethermind.Trie/PatriciaTree.cs index 1e2f0ea6d21..d66e5a56d85 100644 --- a/src/Nethermind/Nethermind.Trie/PatriciaTree.cs +++ b/src/Nethermind/Nethermind.Trie/PatriciaTree.cs @@ -14,6 +14,7 @@ using System.Threading.Tasks; using Nethermind.Core; using Nethermind.Core.Buffers; +using Nethermind.Core.Cpu; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Logging; @@ -227,7 +228,7 @@ private void Commit(NodeCommitInfo nodeCommitInfo, bool skipSelf = false) if (nodesToCommit.Count >= 4) { ClearExceptions(); - Parallel.For(0, nodesToCommit.Count, i => + Parallel.For(0, nodesToCommit.Count, RuntimeInformation.ParallelOptionsLogicalCores, i => { try { @@ -345,10 +346,10 @@ void TraceSkipInlineNode(TrieNode node) } } - public void UpdateRootHash() + public void UpdateRootHash(bool canBeParallel = true) { TreePath path = TreePath.Empty; - RootRef?.ResolveKey(TrieStore, ref path, isRoot: true, bufferPool: _bufferPool); + RootRef?.ResolveKey(TrieStore, ref path, isRoot: true, bufferPool: _bufferPool, canBeParallel); SetRootHash(RootRef?.Keccak ?? EmptyTreeHash, false); } @@ -380,7 +381,7 @@ public virtual ReadOnlySpan Get(ReadOnlySpan rawKey, Hash256? rootHa Nibbles.BytesToNibbleBytes(rawKey, nibbles); TreePath updatePathTreePath = TreePath.Empty; // Only used on update. - ref readonly CappedArray result = ref Run(nibbles, nibblesCount, ref updatePathTreePath, in CappedArray.Empty, isUpdate: false, startRootHash: rootHash); + ref readonly CappedArray result = ref Run(ref updatePathTreePath, in CappedArray.Empty, nibbles, isUpdate: false, startRootHash: rootHash); if (array is not null) ArrayPool.Shared.Return(array); return result.AsSpan(); @@ -397,9 +398,8 @@ public virtual ReadOnlySpan Get(ReadOnlySpan rawKey, Hash256? rootHa { try { - int nibblesCount = nibbles.Length; TreePath updatePathTreePath = TreePath.Empty; // Only used on update. - CappedArray result = Run(nibbles, nibblesCount, ref updatePathTreePath, Array.Empty(), false, startRootHash: rootHash, + CappedArray result = Run(ref updatePathTreePath, in CappedArray.Empty, nibbles, false, startRootHash: rootHash, isNodeRead: true); return result.ToArray() ?? Array.Empty(); } @@ -423,7 +423,7 @@ public virtual ReadOnlySpan Get(ReadOnlySpan rawKey, Hash256? rootHa [..nibblesCount]; // Slice to exact size; Nibbles.BytesToNibbleBytes(rawKey, nibbles); TreePath updatePathTreePath = TreePath.Empty; // Only used on update. - CappedArray result = Run(nibbles, nibblesCount, ref updatePathTreePath, Array.Empty(), false, startRootHash: rootHash, + CappedArray result = Run(ref updatePathTreePath, in CappedArray.Empty, nibbles, false, startRootHash: rootHash, isNodeRead: true); if (array is not null) ArrayPool.Shared.Return(array); return result.ToArray() ?? Array.Empty(); @@ -506,7 +506,7 @@ public virtual void Set(ReadOnlySpan rawKey, in CappedArray value) // lazy stack cleaning after the previous update ClearNodeStack(); TreePath updatePathTreePath = TreePath.FromPath(rawKey); // Only used on update. - Run(nibbles, nibblesCount, ref updatePathTreePath, in value, isUpdate: true); + Run(ref updatePathTreePath, in value, nibbles, isUpdate: true); if (array is not null) ArrayPool.Shared.Return(array); } @@ -531,27 +531,28 @@ static void ThrowNonConcurrentWrites() [DebuggerStepThrough] public void Set(ReadOnlySpan rawKey, Rlp? value) { - Set(rawKey, value is null ? Array.Empty() : value.Bytes); + if (value is null) + { + Set(rawKey, in CappedArray.Empty); + } + else + { + CappedArray valueBytes = new(value.Bytes); + Set(rawKey, in valueBytes); + } } private ref readonly CappedArray Run( - Span updatePath, - int nibblesCount, ref TreePath updatePathTreePath, in CappedArray updateValue, + Span updatePath, bool isUpdate, bool ignoreMissingDelete = true, Hash256? startRootHash = null, bool isNodeRead = false) { -#if DEBUG - if (nibblesCount != updatePath.Length) - { - throw new Exception("Does it ever happen?"); - } -#endif TraverseContext traverseContext = - new(updatePath[..nibblesCount], ref updatePathTreePath, updateValue, isUpdate, ignoreMissingDelete, isNodeRead: isNodeRead); + new(updatePath, ref updatePathTreePath, updateValue, isUpdate, ignoreMissingDelete, isNodeRead: isNodeRead); if (startRootHash is not null) { @@ -571,7 +572,7 @@ ref TraverseBranches(startNode, ref startingPath, traverseContext) : if (traverseContext.UpdateValue.IsNotNull) { if (_logger.IsTrace) TraceNewLeaf(in traverseContext); - byte[] key = updatePath[..nibblesCount].ToArray(); + byte[] key = updatePath.ToArray(); RootRef = TrieNodeFactory.CreateLeaf(key, in traverseContext.UpdateValue); } @@ -1356,7 +1357,7 @@ Hash256 DecodeStorageRoot(Hash256 root, Hash256 address) ? new TrieNodeResolverWithReadFlags(TrieStore, flags) : TrieStore; - if (storageAddr != null) + if (storageAddr is not null) { resolver = resolver.GetStorageTrieNodeResolver(storageAddr); } diff --git a/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs b/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs new file mode 100644 index 00000000000..564d9167547 --- /dev/null +++ b/src/Nethermind/Nethermind.Trie/PreCachedTrieStore.cs @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Concurrent; +using System.Numerics; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Trie.Pruning; + +namespace Nethermind.Trie; + +public class PreCachedTrieStore : ITrieStore +{ + private readonly ITrieStore _inner; + private readonly ConcurrentDictionary _preBlockCache; + private readonly Func _loadRlp; + private readonly Func _tryLoadRlp; + + public PreCachedTrieStore(ITrieStore inner, + ConcurrentDictionary preBlockCache) + { + _inner = inner; + _preBlockCache = preBlockCache; + + // Capture the delegate once for default path to avoid the allocation of the lambda per call + _loadRlp = (NodeKey key) => _inner.LoadRlp(key.Address, in key.Path, key.Hash, flags: ReadFlags.None); + _tryLoadRlp = (NodeKey key) => _inner.TryLoadRlp(key.Address, in key.Path, key.Hash, flags: ReadFlags.None); + } + + public void Dispose() + { + _inner.Dispose(); + } + + public void CommitNode(long blockNumber, Hash256? address, in NodeCommitInfo nodeCommitInfo, WriteFlags writeFlags = WriteFlags.None) + { + _inner.CommitNode(blockNumber, address, in nodeCommitInfo, writeFlags); + } + + public void FinishBlockCommit(TrieType trieType, long blockNumber, Hash256? address, TrieNode? root, WriteFlags writeFlags = WriteFlags.None) + { + _inner.FinishBlockCommit(trieType, blockNumber, address, root, writeFlags); + _preBlockCache.Clear(); + } + + public bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) + { + byte[]? rlp = _preBlockCache.GetOrAdd(new(address, in path, in keccak), + key => _inner.TryLoadRlp(key.Address, in key.Path, key.Hash)); + + return rlp is not null; + } + + public IReadOnlyTrieStore AsReadOnly(INodeStorage? keyValueStore = null) => _inner.AsReadOnly(keyValueStore); + + public event EventHandler? ReorgBoundaryReached + { + add => _inner.ReorgBoundaryReached += value; + remove => _inner.ReorgBoundaryReached -= value; + } + + public IReadOnlyKeyValueStore TrieNodeRlpStore => _inner.TrieNodeRlpStore; + + public void Set(Hash256? address, in TreePath path, in ValueHash256 keccak, byte[] rlp) + { + _preBlockCache[new(address, in path, in keccak)] = rlp; + _inner.Set(address, in path, in keccak, rlp); + } + + public bool HasRoot(Hash256 stateRoot) => _inner.HasRoot(stateRoot); + + public IScopedTrieStore GetTrieStore(Hash256? address) => new ScopedTrieStore(this, address); + + public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256 hash) => _inner.FindCachedOrUnknown(address, in path, hash); + + public byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => + _preBlockCache.GetOrAdd(new(address, in path, hash), + flags == ReadFlags.None ? _loadRlp : + key => _inner.LoadRlp(key.Address, in key.Path, key.Hash, flags)); + + public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => + _preBlockCache.GetOrAdd(new(address, in path, hash), + flags == ReadFlags.None ? _tryLoadRlp : + key => _inner.TryLoadRlp(key.Address, in key.Path, key.Hash, flags)); + + public INodeStorage.KeyScheme Scheme => _inner.Scheme; +} + +public class NodeKey : IEquatable +{ + public readonly Hash256? Address; + public readonly TreePath Path; + public readonly Hash256 Hash; + + public NodeKey(Hash256? address, in TreePath path, in ValueHash256 hash) + { + Address = address; + Path = path; + Hash = hash.ToCommitment(); + } + + public NodeKey(Hash256? address, in TreePath path, Hash256 hash) + { + Address = address; + Path = path; + Hash = hash; + } + + public bool Equals(NodeKey? other) => + other is not null && Address == other.Address && Path.Equals(in other.Path) && Hash.Equals(other.Hash); + + public override bool Equals(object? obj) => obj is NodeKey key && Equals(key); + + public override int GetHashCode() + { + uint hashCode0 = (uint)Hash.GetHashCode(); + ulong hashCode1 = ((ulong)(uint)Path.GetHashCode() << 32) | (uint)(Address?.GetHashCode() ?? 1); + return (int)BitOperations.Crc32C(hashCode0, hashCode1); + } +} diff --git a/src/Nethermind/Nethermind.Trie/Pruning/BlockCommitPackage.cs b/src/Nethermind/Nethermind.Trie/Pruning/BlockCommitPackage.cs index 0faf5ac6634..3ea7daf397a 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/BlockCommitPackage.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/BlockCommitPackage.cs @@ -9,7 +9,7 @@ internal class BlockCommitSet public TrieNode? Root { get; private set; } - public bool IsSealed => Root != null; + public bool IsSealed => Root is not null; public BlockCommitSet(long blockNumber) { diff --git a/src/Nethermind/Nethermind.Trie/Pruning/OverlayTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/OverlayTrieStore.cs new file mode 100644 index 00000000000..50977e6ec5d --- /dev/null +++ b/src/Nethermind/Nethermind.Trie/Pruning/OverlayTrieStore.cs @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Logging; + +namespace Nethermind.Trie.Pruning; + +public class OverlayTrieStore(IKeyValueStoreWithBatching? keyValueStore, IReadOnlyTrieStore store, ILogManager? logManager) : TrieStore(keyValueStore, logManager) +{ + public override bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) => + base.IsPersisted(address, in path, in keccak) || store.IsPersisted(address, in path, in keccak); + + public override TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256? hash) + { + TrieNode node = base.FindCachedOrUnknown(address, in path, hash); + return node.NodeType == NodeType.Unknown ? store.FindCachedOrUnknown(address, in path, hash) : node; + } + + public override byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => + base.TryLoadRlp(address, in path, hash, flags) ?? store.LoadRlp(address, in path, hash, flags); + + public override byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => + base.TryLoadRlp(address, in path, hash, flags) ?? store.TryLoadRlp(address, in path, hash, flags); +} diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TinyTreePath.cs b/src/Nethermind/Nethermind.Trie/Pruning/TinyTreePath.cs index a7ef7f5865d..de1d3bcc3bc 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TinyTreePath.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TinyTreePath.cs @@ -11,6 +11,7 @@ namespace Nethermind.Trie; /// /// Like TreePath, but tiny. Fit in 8 byte, like a long. Can only represent 14 nibble. /// +[StructLayout(LayoutKind.Auto)] public readonly struct TinyTreePath : IEquatable { public const int MaxNibbleLength = 14; @@ -41,6 +42,17 @@ public TreePath ToTreePath() } public bool Equals(TinyTreePath other) => _data == other._data; + public bool Equals(in TinyTreePath other) => _data == other._data; public override bool Equals(object? obj) => obj is TinyTreePath other && Equals(other); public override int GetHashCode() => _data.GetHashCode(); + + public static bool operator ==(in TinyTreePath left, in TinyTreePath right) + { + return left.Equals(in right); + } + + public static bool operator !=(in TinyTreePath left, in TinyTreePath right) + { + return !(left == right); + } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TreePath.cs b/src/Nethermind/Nethermind.Trie/Pruning/TreePath.cs index 5ed20c3a582..ba768f5844a 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TreePath.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TreePath.cs @@ -22,7 +22,7 @@ namespace Nethermind.Trie; /// [Todo("check if its worth it to change the length to byte, or if it actually make things slower.")] [Todo("check if its worth it to not clear byte during TruncateMut, but will need proper comparator, span copy, etc.")] -public struct TreePath +public struct TreePath : IEquatable { public const int MemorySize = 36; public ValueHash256 Path; @@ -259,9 +259,11 @@ public readonly bool Equals(in TreePath other) return Length == other.Length && Path.Equals(in other.Path); } + public readonly bool Equals(TreePath other) => Equals(in other); + public readonly override bool Equals(object? obj) { - return obj is TreePath other && Equals(other); + return obj is TreePath other && Equals(in other); } public readonly override int GetHashCode() diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs index 99576941087..a30a83402a9 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs @@ -21,6 +21,7 @@ namespace Nethermind.Trie.Pruning { + using Nethermind.Core.Cpu; /// /// Trie store helps to manage trie commits block by block. /// If persistence and pruning are needed they have a chance to execute their behaviour on commits. @@ -88,7 +89,7 @@ public TrieNode FromCachedRlpOrUnknown(in Key key) // we returning a copy to avoid multithreaded access trieNode = new TrieNode(NodeType.Unknown, key.Keccak, trieNode.FullRlp); - trieNode.ResolveNode(_trieStore.GetTrieStore(key.Address), key.Path); + trieNode.ResolveNode(_trieStore.GetTrieStore(key.AddressAsHash256), key.Path); trieNode.Keccak = key.Keccak; Metrics.LoadedFromCacheNodesCount++; @@ -108,8 +109,11 @@ void Trace(TrieNode trieNode) } } - private readonly ConcurrentDictionary _byKeyObjectCache = new(); - private readonly ConcurrentDictionary _byHashObjectCache = new(); + private static readonly int _concurrencyLevel = HashHelpers.GetPrime(Environment.ProcessorCount * 4); + private static readonly int _initialBuckets = HashHelpers.GetPrime(Math.Max(31, Environment.ProcessorCount * 16)); + + private readonly ConcurrentDictionary _byKeyObjectCache = new(_concurrencyLevel, _initialBuckets); + private readonly ConcurrentDictionary _byHashObjectCache = new(_concurrencyLevel, _initialBuckets); public bool IsNodeCached(in Key key) { @@ -211,13 +215,20 @@ public void Clear() internal readonly struct Key : IEquatable { internal const long MemoryUsage = 8 + 36 + 8; // (address (probably shared), path, keccak pointer (shared with TrieNode)) - public Hash256? Address { get; } + public readonly ValueHash256 Address; + public Hash256? AddressAsHash256 => Address == default ? null : Address.ToCommitment(); // Direct member rather than property for large struct, so members are called directly, // rather than struct copy through the property. Could also return a ref through property. public readonly TreePath Path; public Hash256 Keccak { get; } public Key(Hash256? address, in TreePath path, Hash256 keccak) + { + Address = address ?? default; + Path = path; + Keccak = keccak; + } + public Key(in ValueHash256 address, in TreePath path, Hash256 keccak) { Address = address; Path = path; @@ -227,13 +238,7 @@ public Key(Hash256? address, in TreePath path, Hash256 keccak) [SkipLocalsInit] public override int GetHashCode() { - Hash256? address = Address; - var addressHash = 0; - if (address is not null) - { - addressHash = address.ValueHash256.GetHashCode(); - } - + var addressHash = Address != default ? Address.GetHashCode() : 1; return Keccak.ValueHash256.GetChainedHashCode((uint)Path.GetHashCode()) ^ addressHash; } @@ -276,11 +281,11 @@ public readonly void Dispose() // Track some of the persisted path hash. Used to be able to remove keys when it is replaced. // If null, disable removing key. - private LruCache? _pastPathHash; + private LruCacheLowObject? _pastPathHash; // Track ALL of the recently re-committed persisted nodes. This is so that we don't accidentally remove // recommitted persisted nodes (which will not get re-persisted). - private ConcurrentDictionary _persistedLastSeens = new(); + private NonBlocking.ConcurrentDictionary _persistedLastSeens = new(); private bool _lastPersistedReachedReorgBoundary; private Task _pruningTask = Task.CompletedTask; @@ -589,9 +594,9 @@ static void ThrowMissingNode(Hash256 keccak) } public virtual byte[]? LoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => LoadRlp(address, path, hash, null, flags); - public byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => TryLoadRlp(address, path, hash, null, flags); + public virtual byte[]? TryLoadRlp(Hash256? address, in TreePath path, Hash256 hash, ReadFlags flags = ReadFlags.None) => TryLoadRlp(address, path, hash, null, flags); - public bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) + public virtual bool IsPersisted(Hash256? address, in TreePath path, in ValueHash256 keccak) { byte[]? rlp = _nodeStorage.Get(address, path, keccak, ReadFlags.None); @@ -609,9 +614,8 @@ public IReadOnlyTrieStore AsReadOnly(INodeStorage? store) => new ReadOnlyTrieStore(this, store); public bool IsNodeCached(Hash256? address, in TreePath path, Hash256? hash) => _dirtyNodes.IsNodeCached(new DirtyNodesCache.Key(address, path, hash)); - private bool IsNodeCached(DirtyNodesCache.Key key) => _dirtyNodes.IsNodeCached(key); - public TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256? hash) => + public virtual TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256? hash) => FindCachedOrUnknown(address, path, hash, false); internal TrieNode FindCachedOrUnknown(Hash256? address, in TreePath path, Hash256? hash, bool isReadOnly) @@ -649,7 +653,8 @@ public void Prune() Stopwatch sw = Stopwatch.StartNew(); if (_logger.IsDebug) _logger.Debug($"Locked {nameof(TrieStore)} for pruning."); - if (!_pruningTaskCancellationTokenSource.IsCancellationRequested && _pruningStrategy.ShouldPrune(MemoryUsedByDirtyCache)) + long memoryUsedByDirtyCache = MemoryUsedByDirtyCache; + if (!_pruningTaskCancellationTokenSource.IsCancellationRequested && _pruningStrategy.ShouldPrune(memoryUsedByDirtyCache)) { // Most of the time in memory pruning is on `PrunePersistedRecursively`. So its // usually faster to just SaveSnapshot causing most of the entry to be persisted. @@ -667,10 +672,10 @@ public void Prune() SaveSnapshot(); PruneCache(); - } - Metrics.PruningTime = sw.ElapsedMilliseconds; - if (_logger.IsInfo) _logger.Info($"Executed memory prune. Took {sw.Elapsed.TotalSeconds:0.##} seconds."); + Metrics.PruningTime = sw.ElapsedMilliseconds; + if (_logger.IsInfo) _logger.Info($"Executed memory prune. Took {sw.Elapsed.TotalSeconds:0.##} seconds. From {memoryUsedByDirtyCache / 1.MiB()}MB to {MemoryUsedByDirtyCache / 1.MiB()}MB"); + } } } @@ -717,7 +722,7 @@ private bool SaveSnapshot() bool shouldDeletePersistedNode = // Its disabled - _pastPathHash != null && + _pastPathHash is not null && // Full pruning need to visit all node, so can't delete anything. !_persistenceStrategy.IsFullPruning && // If more than one candidate set, its a reorg, we can't remove node as persisted node may not be canonical @@ -765,12 +770,12 @@ private bool SaveSnapshot() private void RemovePastKeys(Dictionary? persistedHashes) { - if (persistedHashes == null) return; + if (persistedHashes is null) return; - bool CanRemove(Hash256? address, TinyTreePath path, in TreePath fullPath, ValueHash256 keccak, Hash256? currentlyPersistingKeccak) + bool CanRemove(in ValueHash256 address, TinyTreePath path, in TreePath fullPath, in ValueHash256 keccak, Hash256? currentlyPersistingKeccak) { // Multiple current hash that we don't keep track for simplicity. Just ignore this case. - if (currentlyPersistingKeccak == null) return false; + if (currentlyPersistingKeccak is null) return false; // The persisted hash is the same as currently persisting hash. Do nothing. if (currentlyPersistingKeccak == keccak) return false; @@ -786,32 +791,48 @@ bool CanRemove(Hash256? address, TinyTreePath path, in TreePath fullPath, ValueH return true; } - using INodeStorage.WriteBatch writeBatch = _nodeStorage.StartWriteBatch(); + ActionBlock actionBlock = + new ActionBlock(static (batch) => batch.Dispose()); - void DoAct(KeyValuePair keyValuePair) + INodeStorage.WriteBatch writeBatch = _nodeStorage.StartWriteBatch(); + try { - HashAndTinyPath key = keyValuePair.Key; - if (_pastPathHash.TryGet(new(key.addr, in key.path), out ValueHash256 prevHash)) + int round = 0; + foreach (KeyValuePair keyValuePair in persistedHashes) { - TreePath fullPath = key.path.ToTreePath(); // Micro op to reduce double convert - if (CanRemove(key.addr, key.path, fullPath, prevHash, keyValuePair.Value)) + HashAndTinyPath key = keyValuePair.Key; + if (_pastPathHash.TryGet(key, out ValueHash256 prevHash)) { - Metrics.RemovedNodeCount++; - writeBatch.Remove(key.addr, fullPath, prevHash); + TreePath fullPath = key.path.ToTreePath(); // Micro op to reduce double convert + if (CanRemove(key.addr, key.path, fullPath, prevHash, keyValuePair.Value)) + { + Metrics.RemovedNodeCount++; + Hash256? address = key.addr == default ? null : key.addr.ToCommitment(); + writeBatch.Set(address, fullPath, prevHash, default, WriteFlags.DisableWAL); + round++; + } + } + + // Batches of 256 + if (round > 256) + { + actionBlock.Post(writeBatch); + writeBatch = _nodeStorage.StartWriteBatch(); + round = 0; } } } - - ActionBlock> actionBlock = - new ActionBlock>(DoAct); - - foreach (KeyValuePair keyValuePair in persistedHashes) + catch (Exception ex) { - actionBlock.Post(keyValuePair); + if (_logger.IsError) _logger.Error($"Failed to remove past keys. {ex}"); + } + finally + { + writeBatch.Dispose(); + actionBlock.Complete(); + actionBlock.Completion.Wait(); + _nodeStorage.Compact(); } - - actionBlock.Complete(); - actionBlock.Completion.Wait(); } /// @@ -843,7 +864,7 @@ private void PruneCache(bool skipRecalculateMemory = false) Stopwatch stopwatch = Stopwatch.StartNew(); // Run in parallel - bool shouldTrackPersistedNode = _pastPathHash != null && !_persistenceStrategy.IsFullPruning; + bool shouldTrackPersistedNode = _pastPathHash is not null && !_persistenceStrategy.IsFullPruning; ActionBlock<(DirtyNodesCache.Key key, TrieNode node)>? trackNodesAction = shouldTrackPersistedNode ? new ActionBlock<(DirtyNodesCache.Key key, TrieNode node)>( entry => TrackPrunedPersistedNodes(entry.key, entry.node)) @@ -869,7 +890,7 @@ private void PruneCache(bool skipRecalculateMemory = false) if (keccak is null) { TreePath path2 = key.Path; - keccak = node.GenerateKey(this.GetTrieStore(key.Address), ref path2, isRoot: true); + keccak = node.GenerateKey(this.GetTrieStore(key.AddressAsHash256), ref path2, isRoot: true); if (keccak != key.Keccak) { throw new InvalidOperationException($"Persisted {node} {key} != {keccak}"); @@ -916,7 +937,7 @@ private void TrackPrunedPersistedNodes(in DirtyNodesCache.Key key, TrieNode node TinyTreePath treePath = new(key.Path); // Persisted node with LastSeen is a node that has been re-committed, likely due to processing // recalculated to the same hash. - if (node.LastSeen != null) + if (node.LastSeen is not null) { // Update _persistedLastSeen to later value. _persistedLastSeens.AddOrUpdate( @@ -1016,7 +1037,7 @@ private void PersistBlockCommitSet( { void PersistNode(TrieNode tn, Hash256? address2, TreePath path) { - if (persistedHashes != null && path.Length <= TinyTreePath.MaxNibbleLength) + if (persistedHashes is not null && path.Length <= TinyTreePath.MaxNibbleLength) { HashAndTinyPath key = new(address2, new TinyTreePath(path)); ref Hash256? hash = ref CollectionsMarshal.GetValueRefOrAddDefault(persistedHashes, key, out bool exists); @@ -1243,10 +1264,10 @@ void ClearCommitSetQueue() PruneCache(); KeyValuePair[] nodesCopy = _dirtyNodes.AllNodes.ToArray(); - ConcurrentDictionary wasPersisted = new(); + NonBlocking.ConcurrentDictionary wasPersisted = new(); void PersistNode(TrieNode n, Hash256? address, TreePath path) { - if (n.Keccak == null) return; + if (n.Keccak is null) return; DirtyNodesCache.Key key = new DirtyNodesCache.Key(address, path, n.Keccak); if (wasPersisted.TryAdd(key, true)) { @@ -1254,12 +1275,13 @@ void PersistNode(TrieNode n, Hash256? address, TreePath path) n.IsPersisted = true; } } - Parallel.For(0, nodesCopy.Length, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount / 2 }, i => + Parallel.For(0, nodesCopy.Length, RuntimeInformation.ParallelOptionsPhysicalCores, i => { if (cancellationToken.IsCancellationRequested) return; DirtyNodesCache.Key key = nodesCopy[i].Key; TreePath path = key.Path; - nodesCopy[i].Value.CallRecursively(PersistNode, key.Address, ref path, GetTrieStore(key.Address), false, _logger, false); + Hash256? address = key.AddressAsHash256; + nodesCopy[i].Value.CallRecursively(PersistNode, address, ref path, GetTrieStore(address), false, _logger, false); }); PruneCache(); @@ -1319,43 +1341,185 @@ public bool HasRoot(Hash256 stateRoot) return true; } - private readonly struct HashAndTinyPath(Hash256? hash, in TinyTreePath path) : IEquatable + [StructLayout(LayoutKind.Auto)] + private readonly struct HashAndTinyPath : IEquatable { - public readonly Hash256? addr = hash; - public readonly TinyTreePath path = path; + public readonly ValueHash256 addr; + public readonly TinyTreePath path; + + public HashAndTinyPath(Hash256? hash, in TinyTreePath path) + { + addr = hash ?? default; + this.path = path; + } + public HashAndTinyPath(in ValueHash256 hash, in TinyTreePath path) + { + addr = hash; + this.path = path; + } - public bool Equals(HashAndTinyPath other) => addr == other.addr && path.Equals(other.path); + public bool Equals(HashAndTinyPath other) => addr == other.addr && path.Equals(in other.path); public override bool Equals(object? obj) => obj is HashAndTinyPath other && Equals(other); public override int GetHashCode() { - Hash256? address = addr; - var addressHash = 0; - if (address is not null) - { - addressHash = address.ValueHash256.GetHashCode(); - } + var addressHash = addr != default ? addr.GetHashCode() : 1; return path.GetHashCode() ^ addressHash; } } - private readonly struct HashAndTinyPathAndHash(Hash256? hash, in TinyTreePath path, in ValueHash256 valueHash) : IEquatable + [StructLayout(LayoutKind.Auto)] + private readonly struct HashAndTinyPathAndHash : IEquatable { - public readonly Hash256? hash = hash; - public readonly TinyTreePath path = path; - public readonly ValueHash256 valueHash = valueHash; + public readonly ValueHash256 hash; + public readonly TinyTreePath path; + public readonly ValueHash256 valueHash; + + public HashAndTinyPathAndHash(Hash256? hash, in TinyTreePath path, in ValueHash256 valueHash) + { + this.hash = hash ?? default; + this.path = path; + this.valueHash = valueHash; + } + public HashAndTinyPathAndHash(in ValueHash256 hash, in TinyTreePath path, in ValueHash256 valueHash) + { + this.hash = hash; + this.path = path; + this.valueHash = valueHash; + } - public bool Equals(HashAndTinyPathAndHash other) => hash == other.hash && path.Equals(other.path) && valueHash.Equals(in other.valueHash); + public bool Equals(HashAndTinyPathAndHash other) => hash == other.hash && path.Equals(in other.path) && valueHash.Equals(in other.valueHash); public override bool Equals(object? obj) => obj is HashAndTinyPath other && Equals(other); public override int GetHashCode() { - var hashHash = 0; - if (hash is not null) + var hashHash = hash != default ? hash.GetHashCode() : 1; + return valueHash.GetChainedHashCode((uint)path.GetHashCode()) ^ hashHash; + } + } + + internal static class HashHelpers + { + private const int HashPrime = 101; + + private static bool IsPrime(int candidate) + { + if ((candidate & 1) != 0) { - hashHash = hash.ValueHash256.GetHashCode(); + int limit = (int)Math.Sqrt(candidate); + for (int divisor = 3; divisor <= limit; divisor += 2) + { + if ((candidate % divisor) == 0) + return false; + } + return true; } + return candidate == 2; + } - return valueHash.GetChainedHashCode((uint)path.GetHashCode()) ^ hashHash; + public static int GetPrime(int min) + { + foreach (int prime in Primes) + { + if (prime >= min) + return prime; + } + + // Outside of our predefined table. Compute the hard way. + for (int i = (min | 1); i < int.MaxValue; i += 2) + { + if (IsPrime(i) && ((i - 1) % HashPrime != 0)) + return i; + } + return min; } + + // Table of prime numbers to use as hash table sizes. + // A typical resize algorithm would pick the smallest prime number in this array + // that is larger than twice the previous capacity. + // Suppose our Hashtable currently has capacity x and enough elements are added + // such that a resize needs to occur. Resizing first computes 2x then finds the + // first prime in the table greater than 2x, i.e. if primes are ordered + // p_1, p_2, ..., p_i, ..., it finds p_n such that p_n-1 < 2x < p_n. + // Doubling is important for preserving the asymptotic complexity of the + // hashtable operations such as add. Having a prime guarantees that double + // hashing does not lead to infinite loops. IE, your hash function will be + // h1(key) + i*h2(key), 0 <= i < size. h2 and the size must be relatively prime. + // We prefer the low computation costs of higher prime numbers over the increased + // memory allocation of a fixed prime number i.e. when right sizing a HashSet. + private static ReadOnlySpan Primes => + [ + 3, + 7, + 11, + 17, + 23, + 29, + 37, + 47, + 59, + 71, + 89, + 107, + 131, + 163, + 197, + 239, + 293, + 353, + 431, + 521, + 631, + 761, + 919, + 1103, + 1327, + 1597, + 1931, + 2333, + 2801, + 3371, + 4049, + 4861, + 5839, + 7013, + 8419, + 10103, + 12143, + 14591, + 17519, + 21023, + 25229, + 30293, + 36353, + 43627, + 52361, + 62851, + 75431, + 90523, + 108631, + 130363, + 156437, + 187751, + 225307, + 270371, + 324449, + 389357, + 467237, + 560689, + 672827, + 807403, + 968897, + 1162687, + 1395263, + 1674319, + 2009191, + 2411033, + 2893249, + 3471899, + 4166287, + 4999559, + 5999471, + 7199369 + ]; } } } diff --git a/src/Nethermind/Nethermind.Trie/RangeQueryVisitor.cs b/src/Nethermind/Nethermind.Trie/RangeQueryVisitor.cs index 4301869c26e..b6ec647fec3 100644 --- a/src/Nethermind/Nethermind.Trie/RangeQueryVisitor.cs +++ b/src/Nethermind/Nethermind.Trie/RangeQueryVisitor.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Threading; using Nethermind.Core; using Nethermind.Core.Buffers; @@ -39,10 +40,8 @@ public class RangeQueryVisitor : ITreeVisitor, IDisposable private readonly ILeafValueCollector _valueCollector; // For determining proofs - private (TreePath, TrieNode)?[] _leftmostNodes = new (TreePath, TrieNode)?[65]; - private (TreePath, TrieNode)?[] _rightmostNodes = new (TreePath, TrieNode)?[65]; - private (TreePath, TrieNode)? _leftLeafProof = null; - private TreePath _rightmostLeafPath; + private TrieNode?[] _leftmostNodes = new TrieNode?[65]; + private TrieNode?[] _rightmostNodes = new TrieNode?[65]; private readonly int _nodeLimit; private readonly long _byteLimit; @@ -78,12 +77,6 @@ public RangeQueryVisitor( private bool ShouldVisit(in TreePath path) { - if (_cancellationToken.IsCancellationRequested) - { - StoppedEarly = true; - return false; - } - if (_lastNodeFound) { StoppedEarly = true; @@ -126,41 +119,31 @@ public long GetBytesSize() public ArrayPoolList GetProofs() { - if (_leftLeafProof is null) return ArrayPoolList.Empty(); - HashSet proofs = new(); - // Note: although nethermind works just fine without left proof if start with zero starting hash, - // its out of spec. - (TreePath leftmostPath, TrieNode leftmostLeafProof) = _leftLeafProof.Value; - proofs.Add(leftmostLeafProof.FullRlp.ToArray()); - - for (int i = 64; i >= 0; i--) - { - if (!_leftmostNodes[i].HasValue) continue; - (TreePath path, TrieNode node) = _leftmostNodes[i].Value; - leftmostPath.TruncateMut(i); - if (leftmostPath != path) continue; + AddToProof(_leftmostNodes); + AddToProof(_rightmostNodes); - proofs.Add(node.FullRlp.ToArray()); - } + return proofs.ToPooledList(); - TreePath rightmostPath = _rightmostLeafPath; - if (rightmostPath.Length != 0) + void AddToProof(IReadOnlyList boundaryNodes) { - for (int i = 64; i >= 0; i--) + int i = 0; + while (true) { - if (!_rightmostNodes[i].HasValue) continue; - - (TreePath path, TrieNode node) = _rightmostNodes[i].Value; - rightmostPath.TruncateMut(i); - if (rightmostPath != path) continue; + TrieNode node = boundaryNodes[i]; + if (node is null) break; proofs.Add(node.FullRlp.ToArray()); + + if (node.IsBranch) + i++; + else if (node.IsExtension) + i += node.Key.Length; + else + break; } } - - return proofs.ToPooledList(); } public void VisitTree(in TreePathContext nodeContext, Hash256 rootHash, TrieVisitContext trieVisitContext) @@ -175,31 +158,32 @@ public void VisitMissingNode(in TreePathContext ctx, Hash256 nodeHash, TrieVisit public void VisitBranch(in TreePathContext ctx, TrieNode node, TrieVisitContext trieVisitContext) { - if (!_leftmostNodes[ctx.Path.Length].HasValue) _leftmostNodes[ctx.Path.Length] = (ctx.Path, node); - _rightmostNodes[ctx.Path.Length] = (ctx.Path, node); + _leftmostNodes[ctx.Path.Length] ??= node; + _rightmostNodes[ctx.Path.Length] = node; } public void VisitExtension(in TreePathContext ctx, TrieNode node, TrieVisitContext trieVisitContext) { - if (!_leftmostNodes[ctx.Path.Length].HasValue) _leftmostNodes[ctx.Path.Length] = (ctx.Path, node); - _rightmostNodes[ctx.Path.Length] = (ctx.Path, node); + _leftmostNodes[ctx.Path.Length] ??= node; + _rightmostNodes[ctx.Path.Length] = node; } public void VisitLeaf(in TreePathContext ctx, TrieNode node, TrieVisitContext trieVisitContext, ReadOnlySpan value) { + _leftmostNodes[ctx.Path.Length] ??= node; + _rightmostNodes[ctx.Path.Length] = node; + TreePath path = ctx.Path.Append(node.Key); - _rightmostNodes[ctx.Path.Length] = (ctx.Path, node); // Yes, this is needed. Yes, you can make a special variable like _rightLeafProof. if (!ShouldVisit(path)) { - if (!_lastNodeFound) _leftLeafProof = (path, node); return; } - _leftLeafProof ??= (path, node); - - if (path.Path.CompareTo(_limitHash) >= 0) + if (path.Path.CompareTo(_limitHash) >= 0 || _cancellationToken.IsCancellationRequested) { // This leaf is after or at limitHash. This will cause all further ShouldVisit to return false. + // Yes, we do need to include this as part of the response. + // Note: Cancellation must happen at the leaf or the proof may break _lastNodeFound = true; } @@ -215,8 +199,6 @@ public void VisitCode(in TreePathContext nodeContext, Hash256 codeHash, TrieVisi private void CollectNode(in TreePath path, CappedArray value) { - _rightmostLeafPath = path; - int encodedSize = _valueCollector.Collect(path.Path, value); _currentBytesCount += encodedSize; _currentLeafCount++; diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs b/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs index 55b522e38d1..b32c4f95a59 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs @@ -6,7 +6,10 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; using Nethermind.Core.Buffers; +using Nethermind.Core.Cpu; using Nethermind.Core.Crypto; using Nethermind.Serialization.Rlp; using Nethermind.Trie.Pruning; @@ -125,12 +128,12 @@ private static void ThrowNullKey(TrieNode node) throw new TrieException($"Hex prefix of a leaf node is null at node {node.Keccak}"); } - public static CappedArray RlpEncodeBranch(TrieNode item, ITrieNodeResolver tree, ref TreePath path, ICappedArrayPool? pool) + public static CappedArray RlpEncodeBranch(TrieNode item, ITrieNodeResolver tree, ref TreePath path, ICappedArrayPool? pool, bool canBeParallel) { Metrics.TreeNodeRlpEncodings++; int valueRlpLength = AllowBranchValues ? Rlp.LengthOf(item.Value.AsSpan()) : 1; - int contentLength = valueRlpLength + GetChildrenRlpLengthForBranch(tree, ref path, item, pool); + int contentLength = valueRlpLength + (UseParallel(canBeParallel) ? GetChildrenRlpLengthForBranchParallel(tree, ref path, item, pool) : GetChildrenRlpLengthForBranch(tree, ref path, item, pool)); int sequenceLength = Rlp.LengthOfSequence(contentLength); CappedArray result = pool.SafeRentBuffer(sequenceLength); Span resultSpan = result.AsSpan(); @@ -147,20 +150,61 @@ public static CappedArray RlpEncodeBranch(TrieNode item, ITrieNodeResolver } return result; + + static bool UseParallel(bool canBeParallel) => Environment.ProcessorCount > 1 && canBeParallel; } private static int GetChildrenRlpLengthForBranch(ITrieNodeResolver tree, ref TreePath path, TrieNode item, ICappedArrayPool? bufferPool) { item.EnsureInitialized(); // Tail call optimized. - if (item.HasRlp) - { - return GetChildrenRlpLengthForBranchRlp(tree, ref path, item, bufferPool); - } - else - { - return GetChildrenRlpLengthForBranchNonRlp(tree, ref path, item, bufferPool); - } + return item.HasRlp + ? GetChildrenRlpLengthForBranchRlp(tree, ref path, item, bufferPool) + : GetChildrenRlpLengthForBranchNonRlp(tree, ref path, item, bufferPool); + } + + private static int GetChildrenRlpLengthForBranchParallel(ITrieNodeResolver tree, ref TreePath path, TrieNode item, ICappedArrayPool? bufferPool) + { + item.EnsureInitialized(); + // Tail call optimized. + return item.HasRlp + ? GetChildrenRlpLengthForBranchRlpParallel(tree, path, item, bufferPool) + : GetChildrenRlpLengthForBranchNonRlpParallel(tree, path, item, bufferPool); + } + + private static int GetChildrenRlpLengthForBranchNonRlpParallel(ITrieNodeResolver tree, TreePath rootPath, TrieNode item, ICappedArrayPool bufferPool) + { + int totalLength = 0; + Parallel.For(0, BranchesCount, RuntimeInformation.ParallelOptionsLogicalCores, + () => 0, + (i, _, local) => + { + object? data = item._data[i]; + if (ReferenceEquals(data, _nullNode) || data is null) + { + local++; + } + else if (data is Hash256) + { + local += Rlp.LengthOfKeccakRlp; + } + else + { + TreePath path = rootPath; + path.AppendMut(i); + TrieNode childNode = Unsafe.As(data); + childNode.ResolveKey(tree, ref path, isRoot: false, bufferPool: bufferPool); + local += childNode.Keccak is null ? childNode.FullRlp.Length : Rlp.LengthOfKeccakRlp; + } + + return local; + }, + local => + { + Interlocked.Add(ref totalLength, local); + }); + + return totalLength; } private static int GetChildrenRlpLengthForBranchNonRlp(ITrieNodeResolver tree, ref TreePath path, TrieNode item, ICappedArrayPool bufferPool) @@ -168,7 +212,7 @@ private static int GetChildrenRlpLengthForBranchNonRlp(ITrieNodeResolver tree, r int totalLength = 0; for (int i = 0; i < BranchesCount; i++) { - object data = item._data[i]; + object? data = item._data[i]; if (ReferenceEquals(data, _nullNode) || data is null) { totalLength++; @@ -180,7 +224,7 @@ private static int GetChildrenRlpLengthForBranchNonRlp(ITrieNodeResolver tree, r else { path.AppendMut(i); - TrieNode childNode = (TrieNode)data; + TrieNode childNode = Unsafe.As(data); childNode.ResolveKey(tree, ref path, isRoot: false, bufferPool: bufferPool); path.TruncateOne(); totalLength += childNode.Keccak is null ? childNode.FullRlp.Length : Rlp.LengthOfKeccakRlp; @@ -189,6 +233,48 @@ private static int GetChildrenRlpLengthForBranchNonRlp(ITrieNodeResolver tree, r return totalLength; } + private static int GetChildrenRlpLengthForBranchRlpParallel(ITrieNodeResolver tree, TreePath rootPath, TrieNode item, ICappedArrayPool? bufferPool) + { + int totalLength = 0; + Parallel.For(0, BranchesCount, RuntimeInformation.ParallelOptionsLogicalCores, + () => 0, + (i, _, local) => + { + ValueRlpStream rlpStream = item.RlpStream; + item.SeekChild(ref rlpStream, i); + object? data = item._data[i]; + if (data is null) + { + local += rlpStream.PeekNextRlpLength(); + } + else if (ReferenceEquals(data, _nullNode)) + { + local++; + } + else if (data is Hash256) + { + local += Rlp.LengthOfKeccakRlp; + } + else + { + TreePath path = rootPath; + path.AppendMut(i); + Debug.Assert(data is TrieNode, "Data is not TrieNode"); + TrieNode childNode = Unsafe.As(data); + childNode.ResolveKey(tree, ref path, isRoot: false, bufferPool: bufferPool); + local += childNode.Keccak is null ? childNode.FullRlp.Length : Rlp.LengthOfKeccakRlp; + } + + return local; + }, + local => + { + Interlocked.Add(ref totalLength, local); + }); + + return totalLength; + } + private static int GetChildrenRlpLengthForBranchRlp(ITrieNodeResolver tree, ref TreePath path, TrieNode item, ICappedArrayPool? bufferPool) { int totalLength = 0; diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.Visitor.cs b/src/Nethermind/Nethermind.Trie/TrieNode.Visitor.cs index 67b43655dde..947ae4b2f9c 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.Visitor.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.Visitor.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Nethermind.Core; using Nethermind.Core.Buffers; +using Nethermind.Core.Cpu; using Nethermind.Core.Crypto; using Nethermind.Serialization.Rlp; using Nethermind.Trie.Pruning; @@ -192,7 +193,7 @@ void VisitMultiThread(TreePath parentPath, ITreeVisitor treeVisito var copy = nodeContext; // multithreaded route - Parallel.For(0, BranchesCount, i => + Parallel.For(0, BranchesCount, RuntimeInformation.ParallelOptionsPhysicalCores, i => { visitContext.Semaphore.Wait(); try @@ -216,7 +217,7 @@ static void VisitAllSingleThread(TrieNode currentNode, ref TreePath path, ITreeV path.AppendMut(0); for (int i = 0; i < 16; i++) { - if (output[i] == null) continue; + if (output[i] is null) continue; TrieNode child = output[i]; path.SetLast(i); child.ResolveKey(nodeResolver, ref path, false); diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.cs b/src/Nethermind/Nethermind.Trie/TrieNode.cs index 85fd6dcd04a..f9f7396b722 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.cs @@ -57,7 +57,7 @@ public sealed partial class TrieNode public Hash256? Keccak { get; internal set; } - public bool HasRlp => _rlp != null; + public bool HasRlp => _rlp is not null; public ref readonly CappedArray FullRlp { @@ -491,7 +491,7 @@ private bool DecodeRlp(ValueRlpStream rlpStream, ICappedArrayPool bufferPool, ou return true; } - public void ResolveKey(ITrieNodeResolver tree, ref TreePath path, bool isRoot, ICappedArrayPool? bufferPool = null) + public void ResolveKey(ITrieNodeResolver tree, ref TreePath path, bool isRoot, ICappedArrayPool? bufferPool = null, bool canBeParallel = true) { if (Keccak is not null) { @@ -500,17 +500,17 @@ public void ResolveKey(ITrieNodeResolver tree, ref TreePath path, bool isRoot, I return; } - Keccak = GenerateKey(tree, ref path, isRoot, bufferPool); + Keccak = GenerateKey(tree, ref path, isRoot, bufferPool, canBeParallel); } - public Hash256? GenerateKey(ITrieNodeResolver tree, ref TreePath path, bool isRoot, ICappedArrayPool? bufferPool = null) + public Hash256? GenerateKey(ITrieNodeResolver tree, ref TreePath path, bool isRoot, ICappedArrayPool? bufferPool = null, bool canBeParallel = true) { RlpFactory rlp = _rlp; if (rlp is null || IsDirty) { ref readonly CappedArray oldRlp = ref rlp is not null ? ref rlp.Data : ref CappedArray.Empty; CappedArray fullRlp = NodeType == NodeType.Branch ? - TrieNodeDecoder.RlpEncodeBranch(this, tree, ref path, bufferPool) : + TrieNodeDecoder.RlpEncodeBranch(this, tree, ref path, bufferPool, canBeParallel: isRoot && canBeParallel) : RlpEncode(tree, ref path, bufferPool); if (fullRlp.IsNotNullOrEmpty) @@ -536,7 +536,7 @@ internal CappedArray RlpEncode(ITrieNodeResolver tree, ref TreePath path, { return NodeType switch { - NodeType.Branch => TrieNodeDecoder.RlpEncodeBranch(this, tree, ref path, bufferPool), + NodeType.Branch => TrieNodeDecoder.RlpEncodeBranch(this, tree, ref path, bufferPool, canBeParallel: false), NodeType.Extension => TrieNodeDecoder.EncodeExtension(this, tree, ref path, bufferPool), NodeType.Leaf => TrieNodeDecoder.EncodeLeaf(this, bufferPool), _ => ThrowUnhandledNodeType(this) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs index c8cd8e1b4af..6cd66531054 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs @@ -361,8 +361,8 @@ public void should_not_broadcast_tx_with_MaxFeePerGas_lower_than_70_percent_of_C // tx should be immediately broadcasted only if MaxFeePerGas is equal at least 70% of current base fee peer.Received(shouldBroadcast ? 1 : 0).SendNewTransaction(Arg.Any()); - // tx should always be added to persistent collection, without any fee restrictions - _broadcaster.GetSnapshot().Length.Should().Be(1); + // tx should only be added to persistent collection, if it is above the fee restriction + _broadcaster.GetSnapshot().Length.Should().Be(shouldBroadcast ? 1 : 0); } [Test] diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index e2555cd1bd1..a6a96c029a2 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -273,6 +273,7 @@ public void get_next_pending_nonce() // LatestPendingNonce=1, when the current nonce of the account=1 and no pending transactions _stateProvider.IncrementNonce(TestItem.AddressA); + _txPool.ResetAddress(TestItem.AddressA); latestNonce = _txPool.GetLatestPendingNonce(TestItem.AddressA); Assert.That((UInt256)1, Is.EqualTo(latestNonce)); @@ -505,7 +506,7 @@ public void should_add_underpaid_txs_to_full_TxPool_only_if_local(bool isLocal) AcceptTxResult result = _txPool.SubmitTx(tx, txHandlingOptions); _txPool.GetPendingTransactionsCount().Should().Be(30); _txPool.GetOwnPendingTransactions().Length.Should().Be(isLocal ? 1 : 0); - result.ToString().Should().Contain(isLocal ? nameof(AcceptTxResult.FeeTooLowToCompete) : nameof(AcceptTxResult.FeeTooLow)); + result.ToString().Should().Contain(isLocal ? nameof(AcceptTxResult.Accepted) : nameof(AcceptTxResult.FeeTooLow)); } [TestCase(0)] @@ -564,6 +565,7 @@ public void should_not_count_txs_with_stale_nonces_when_calculating_cumulative_c if (i < numberOfStaleTxsInBucket) { _stateProvider.IncrementNonce(TestItem.AddressA); + _txPool.ResetAddress(TestItem.AddressA); } } @@ -1041,6 +1043,7 @@ public void should_retrieve_added_persistent_transaction_correctly_even_if_was_e retrievedTransaction.Should().BeEquivalentTo(transaction); EnsureSenderBalance(transactionWithHigherFee); + _txPool.ResetAddress(transactionWithHigherFee.SenderAddress); _txPool.SubmitTx(transactionWithHigherFee, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); _txPool.TryGetPendingTransaction(transactionWithHigherFee.Hash, out var retrievedTransactionWithHigherFee).Should().BeTrue(); retrievedTransactionWithHigherFee.Should().BeEquivalentTo(transactionWithHigherFee); @@ -1381,7 +1384,7 @@ public void should_increase_nonce_when_transaction_not_included_in_txPool_but_br .WithMaxFeePerGas(1) .WithMaxPriorityFeePerGas(1) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - _txPool.SubmitTx(cheapTx, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.FeeTooLowToCompete); + _txPool.SubmitTx(cheapTx, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Accepted); _txPool.GetPendingTransactions().Should().NotContain(cheapTx); _txPool.GetOwnPendingTransactions().Should().Contain(cheapTx); peer.Received().SendNewTransaction(cheapTx); @@ -1393,7 +1396,7 @@ public void should_increase_nonce_when_transaction_not_included_in_txPool_but_br .WithMaxFeePerGas(1) .WithMaxPriorityFeePerGas(1) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - _txPool.SubmitTx(fourthTx, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.FeeTooLowToCompete); + _txPool.SubmitTx(fourthTx, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Accepted); _txPool.GetPendingTransactions().Should().NotContain(fourthTx); _txPool.GetOwnPendingTransactions().Should().Contain(fourthTx); peer.Received().SendNewTransaction(fourthTx); @@ -1638,6 +1641,53 @@ public void Should_not_add_underpaid_tx_even_if_lower_nonces_are_expensive(int g result.Should().Be(expectedResult ? AcceptTxResult.Accepted : AcceptTxResult.FeeTooLowToCompete); } + [Test] + public void Should_correctly_add_tx_to_local_pool_when_underpaid([Values] TxType txType) + { + // Should only add non-blob transactions to local pool when underpaid + bool expectedResult = txType != TxType.Blob; + + // No need to check for deposit tx + if (txType == TxType.DepositTx) return; + + ISpecProvider specProvider = GetCancunSpecProvider(); + TxPoolConfig txPoolConfig = new TxPoolConfig { Size = 30, PersistentBlobStorageSize = 0 }; + _txPool = CreatePool(txPoolConfig, specProvider); + + Transaction[] transactions = GetTransactions(GetPeers(3), true, false); + + foreach (Address address in transactions.Select(t => t.SenderAddress).Distinct()) + { + EnsureSenderBalance(address, UInt256.MaxValue); + } + + // setup full tx pool + foreach (Transaction transaction in transactions) + { + transaction.GasPrice = 10.GWei(); + _txPool.SubmitTx(transaction, TxHandlingOptions.None); + } + + _txPool.GetPendingTransactionsCount().Should().Be(30); + + Transaction testTx = Build.A.Transaction + .WithNonce(0) + .WithType(txType) + .WithShardBlobTxTypeAndFieldsIfBlobTx() + .WithMaxFeePerGas(9.GWei()) + .WithMaxPriorityFeePerGas(9.GWei()) + .WithTo(TestItem.AddressB) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + EnsureSenderBalance(TestItem.PrivateKeyA.Address, UInt256.MaxValue); + + AcceptTxResult result = _txPool.SubmitTx(testTx, TxHandlingOptions.PersistentBroadcast); + result.Should().Be(expectedResult ? AcceptTxResult.Accepted : AcceptTxResult.FeeTooLowToCompete); + _txPool.GetOwnPendingTransactions().Length.Should().Be(expectedResult ? 1 : 0); + _txPool.GetPendingBlobTransactionsCount().Should().Be(0); + _txPool.GetPendingTransactions().Should().NotContain(testTx); + } + private IDictionary GetPeers(int limit = 100) { var peers = new Dictionary(); diff --git a/src/Nethermind/Nethermind.TxPool/Collections/DistinctValueSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/DistinctValueSortedPool.cs index 301004f370f..c37a14346d0 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/DistinctValueSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/DistinctValueSortedPool.cs @@ -57,10 +57,19 @@ protected override void InsertCore(TKey key, TValue value, TGroupKey groupKey) _distinctDictionary[value] = new KeyValuePair(key, value); } - protected override bool Remove(TKey key, TValue value) + protected override bool Remove(TKey key, out TValue? value) { - _distinctDictionary.Remove(value); - return base.Remove(key, value); + if (base.Remove(key, out value)) + { + if (value is not null) + { + _distinctDictionary.Remove(value); + } + + return true; + } + + return false; } protected virtual bool AllowSameKeyReplacement => false; diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index 89aea5b28ac..593266a0de6 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -88,10 +88,20 @@ public override bool TryGetValue(ValueHash256 hash, [NotNullWhen(true)] out Tran return false; } - protected override bool Remove(ValueHash256 hash, Transaction tx) + protected override bool Remove(ValueHash256 hash, out Transaction? tx) { - _blobTxCache.Delete(hash); - _blobTxStorage.Delete(hash, tx.Timestamp); - return base.Remove(hash, tx); + if (base.Remove(hash, out tx)) + { + if (tx is not null) + { + _blobTxStorage.Delete(hash, tx.Timestamp); + } + + _blobTxCache.Delete(hash); + + return true; + } + + return false; } } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.Events.cs b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.Events.cs index 79d55df908c..9e6f0303765 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.Events.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.Events.cs @@ -12,28 +12,16 @@ public partial class SortedPool public event EventHandler? Removed; #pragma warning restore 67 - public class SortedPoolEventArgs + public class SortedPoolEventArgs(TKey key, TValue value) { - public TKey Key { get; } - public TValue Value { get; } - public TGroupKey Group { get; } - - public SortedPoolEventArgs(TKey key, TValue value, TGroupKey group) - { - Key = key; - Value = value; - Group = group; - } + public TKey Key { get; } = key; + public TValue Value { get; } = value; } - public class SortedPoolRemovedEventArgs : SortedPoolEventArgs + public class SortedPoolRemovedEventArgs(TKey key, TValue value, bool evicted) + : SortedPoolEventArgs(key, value) { - public bool Evicted { get; } - - public SortedPoolRemovedEventArgs(TKey key, TValue value, TGroupKey group, bool evicted) : base(key, value, group) - { - Evicted = evicted; - } + public bool Evicted { get; } = evicted; } } } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs index 317df34eb09..c1771f87338 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Threading; - using Nethermind.Core.Collections; using Nethermind.Core.Threading; using Nethermind.Logging; @@ -225,42 +224,52 @@ private bool TryRemove(TKey key, out TValue? value, [NotNullWhen(true)] out ICol protected bool TryRemoveNonLocked(TKey key, bool evicted, [NotNullWhen(true)] out TValue? value, out ICollection? bucket) { - if (_cacheMap.TryGetValue(key, out value) && value is not null) + if (Remove(key, out value) && value is not null) + { + if (RemoveFromBucket(value, out EnhancedSortedSet? bucketSet)) + { + bucket = bucketSet; + Removed?.Invoke(this, new SortedPoolRemovedEventArgs(key, value, evicted)); + return true; + } + + // just for safety + _worstSortedValues.Remove(value); + UpdateWorstValue(); + } + + value = default; + bucket = null; + return false; + } + + private bool RemoveFromBucket([DisallowNull] TValue value, out EnhancedSortedSet? bucketSet) + { + TGroupKey groupMapping = MapToGroup(value); + if (_buckets.TryGetValue(groupMapping, out bucketSet)) { - if (Remove(key, value)) + TValue? last = bucketSet.Max; + if (bucketSet.Remove(value)) { - TGroupKey groupMapping = MapToGroup(value); - if (_buckets.TryGetValue(groupMapping, out EnhancedSortedSet? bucketSet)) + if (bucketSet.Count == 0) { - bucket = bucketSet; - TValue? last = bucketSet.Max; - if (bucketSet.Remove(value!)) + _buckets.Remove(groupMapping); + if (last is not null) { - if (bucket.Count == 0) - { - _buckets.Remove(groupMapping); - if (last is not null) - { - _worstSortedValues.Remove(last); - UpdateWorstValue(); - } - } - else - { - UpdateSortedValues(bucketSet, last); - } - _snapshot = null; - - return true; + _worstSortedValues.Remove(last); + UpdateWorstValue(); } } + else + { + UpdateSortedValues(bucketSet, last); + } - Removed?.Invoke(this, new SortedPoolRemovedEventArgs(key, value, groupMapping, evicted)); + _snapshot = null; + return true; } } - value = default; - bucket = null; return false; } @@ -371,16 +380,30 @@ public virtual bool TryInsert(TKey key, TValue value, out TValue? removed) private bool RemoveLast(out TValue? removed) { - TKey? key = _worstValue.GetValueOrDefault().Value; + TryAgain: + KeyValuePair worstValue = _worstValue.GetValueOrDefault(); + TKey? key = worstValue.Value; if (key is not null) { - return TryRemoveNonLocked(key, true, out removed, out _); - } - else - { - removed = default; - return false; + if (TryRemoveNonLocked(key, true, out removed, out _)) + { + return true; + } + + if (worstValue.Key is not null && _worstSortedValues.Remove(worstValue)) + { + RemoveFromBucket(worstValue.Key, out _); + } + else + { + UpdateWorstValue(); + } + + goto TryAgain; } + + removed = default; + return false; } /// @@ -413,7 +436,7 @@ protected virtual void InsertCore(TKey key, TValue value, TGroupKey groupKey) UpdateIsFull(); UpdateSortedValues(bucket, last); _snapshot = null; - Inserted?.Invoke(this, new SortedPoolEventArgs(key, value, groupKey)); + Inserted?.Invoke(this, new SortedPoolEventArgs(key, value)); } } @@ -439,15 +462,17 @@ private void UpdateSortedValues(EnhancedSortedSet bucket, TValue? previo /// /// Actual removal mechanism. /// - protected virtual bool Remove(TKey key, TValue value) + protected virtual bool Remove(TKey key, out TValue? value) { - if (_cacheMap.Remove(key)) + // Now remove from cache + if (_cacheMap.Remove(key, out value)) { UpdateIsFull(); _snapshot = null; return true; } + value = default; return false; } @@ -527,8 +552,8 @@ protected void EnsureCapacity(int? expectedCapacity = null) private string GetInfoAboutWorstValues() { TKey? key = _worstValue.GetValueOrDefault().Value; - var isWorstValueInPool = _cacheMap.TryGetValue(key, out TValue? value) && value != null; - return $"Number of items in worstSortedValues: {_worstSortedValues.Count}; IsWorstValueInPool: {isWorstValueInPool}; Worst value: {_worstValue}; GetValue: {_worstValue.GetValueOrDefault()}; Current max in worstSortedValues: {_worstSortedValues.Max};"; + var isWorstValueInPool = _cacheMap.TryGetValue(key, out TValue? value) && value is not null; + return $"Number of items in worstSortedValues: {_worstSortedValues.Count}; IsWorstValueInPool: {isWorstValueInPool};"; } public void UpdatePool(Func, IEnumerable<(TValue Tx, Action? Change)>> changingElements) diff --git a/src/Nethermind/Nethermind.TxPool/HashCache.cs b/src/Nethermind/Nethermind.TxPool/HashCache.cs index ac685b6e882..d49960d6bbc 100644 --- a/src/Nethermind/Nethermind.TxPool/HashCache.cs +++ b/src/Nethermind/Nethermind.TxPool/HashCache.cs @@ -23,14 +23,12 @@ internal class HashCache { private const int SafeCapacity = 1024 * 16; - private readonly LruKeyCache _longTermCache = new( + private readonly LruKeyCacheLowObject _longTermCache = new( MemoryAllowance.TxHashCacheSize, - Math.Min(SafeCapacity, MemoryAllowance.TxHashCacheSize), "long term hash cache"); - private readonly LruKeyCache _currentBlockCache = new( + private readonly LruKeyCacheLowObject _currentBlockCache = new( SafeCapacity, - Math.Min(SafeCapacity, MemoryAllowance.TxHashCacheSize), "current block hash cache"); public bool Get(Hash256 hash) diff --git a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs index 1505b21e9fb..de7eb80c9d0 100644 --- a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs +++ b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs @@ -97,31 +97,33 @@ public TxBroadcaster(IComparer comparer, // only for testing reasons internal Transaction[] GetSnapshot() => _persistentTxs.GetSnapshot(); - public void Broadcast(Transaction tx, bool isPersistent) + public bool Broadcast(Transaction tx, bool isPersistent) { if (isPersistent) { - StartBroadcast(tx); - } - else - { - BroadcastOnce(tx); + return StartBroadcast(tx); } + + BroadcastOnce(tx); + + return true; } - private void StartBroadcast(Transaction tx) + private bool StartBroadcast(Transaction tx) { // broadcast local tx only if MaxFeePerGas is not lower than configurable percent of current base fee // (70% by default). Otherwise only add to persistent txs and broadcast when tx will be ready for inclusion - if (tx.MaxFeePerGas >= _baseFeeThreshold || tx.IsFree()) + + if (tx is not null + && (tx.MaxFeePerGas >= _baseFeeThreshold || tx.IsFree()) + && _persistentTxs.TryInsert(tx.Hash, tx.SupportsBlobs ? new LightTransaction(tx) : tx, out Transaction? removed) + && removed?.Hash != tx.Hash) { NotifyPeersAboutLocalTx(tx); + return true; } - if (tx.Hash is not null) - { - _persistentTxs.TryInsert(tx.Hash, tx.SupportsBlobs ? new LightTransaction(tx) : tx); - } + return false; } private void BroadcastOnce(Transaction tx) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 8b062191f47..fe806ad91e6 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Channels; @@ -44,6 +45,7 @@ public class TxPool : ITxPool, IDisposable private readonly IChainHeadSpecProvider _specProvider; private readonly IAccountStateProvider _accounts; + private readonly AccountCache _accountCache; private readonly IEthereumEcdsa _ecdsa; private readonly IBlobTxStorage _blobTxStorage; private readonly IChainHeadInfoProvider _headInfo; @@ -65,6 +67,8 @@ public class TxPool : ITxPool, IDisposable private readonly ITimer? _timer; private Transaction[]? _transactionSnapshot; private Transaction[]? _blobTransactionSnapshot; + private long _lastBlockNumber = -1; + private Hash256? _lastBlockHash; /// /// This class stores all known pending transactions that can be used for block production @@ -97,7 +101,7 @@ public TxPool(IEthereumEcdsa ecdsa, _headInfo = chainHeadInfoProvider ?? throw new ArgumentNullException(nameof(chainHeadInfoProvider)); _txPoolConfig = txPoolConfig; _blobReorgsSupportEnabled = txPoolConfig.BlobsSupport.SupportsReorgs(); - _accounts = _headInfo.AccountStateProvider; + _accounts = _accountCache = new AccountCache(_headInfo.AccountStateProvider); _specProvider = _headInfo.SpecProvider; MemoryAllowance.MemPoolSize = txPoolConfig.Size; @@ -206,6 +210,23 @@ private void ProcessNewHeads() { try { + ArrayPoolList? accountChanges = args.Block.AccountChanges; + if (!CanUseCache(args.Block, accountChanges)) + { + // Not sequential block, reset cache + _accountCache.Reset(); + } + else + { + // Sequential block, just remove changed accounts from cache + _accountCache.RemoveAccounts(accountChanges); + } + args.Block.AccountChanges = null; + accountChanges?.Dispose(); + + _lastBlockNumber = args.Block.Number; + _lastBlockHash = args.Block.Hash; + ReAddReorganisedTransactions(args.PreviousBlock); RemoveProcessedTransactions(args.Block); UpdateBuckets(); @@ -219,6 +240,11 @@ private void ProcessNewHeads() } } } + + bool CanUseCache(Block block, [NotNullWhen(true)] ArrayPoolList? accountChanges) + { + return accountChanges is not null && block.ParentHash == _lastBlockHash && _lastBlockNumber + 1 == block.Number; + } }, TaskCreationOptions.LongRunning).ContinueWith(t => { if (t.IsFaulted) @@ -454,14 +480,17 @@ private AcceptTxResult AddCore(Transaction tx, ref TxFilteringState state, bool if (tx.Hash == removed?.Hash) { // it means it was added and immediately evicted - pool was full of better txs - if (isPersistentBroadcast) + if (!isPersistentBroadcast || tx.SupportsBlobs || !_broadcaster.Broadcast(tx, true)) { // we are adding only to persistent broadcast - not good enough for standard pool, // but can be good enough for TxBroadcaster pool - for local txs only - _broadcaster.Broadcast(tx, isPersistentBroadcast); + Metrics.PendingTransactionsPassedFiltersButCannotCompeteOnFees++; + return AcceptTxResult.FeeTooLowToCompete; + } + else + { + return AcceptTxResult.Accepted; } - Metrics.PendingTransactionsPassedFiltersButCannotCompeteOnFees++; - return AcceptTxResult.FeeTooLowToCompete; } relevantPool.UpdateGroup(tx.SenderAddress!, state.SenderAccount, _updateBucketAdded); @@ -734,6 +763,74 @@ private void TimerOnElapsed(object? sender, EventArgs e) _timer!.Enabled = true; } + internal void ResetAddress(Address address) + { + ArrayPoolList arrayPoolList = new(1); + arrayPoolList.Add(address); + _accountCache.RemoveAccounts(arrayPoolList); + } + + private sealed class AccountCache : IAccountStateProvider + { + private readonly IAccountStateProvider _provider; + private readonly LruCacheLowObject[] _caches; + + public AccountCache(IAccountStateProvider provider) + { + _provider = provider; + _caches = new LruCacheLowObject[16]; + for (int i = 0; i < _caches.Length; i++) + { + // Cache per nibble to reduce contention as TxPool is very parallel + _caches[i] = new LruCacheLowObject(1_024, ""); + } + } + + public bool TryGetAccount(Address address, out AccountStruct account) + { + var cache = _caches[GetCacheIndex(address)]; + if (!cache.TryGet(new AddressAsKey(address), out account)) + { + if (!_provider.TryGetAccount(address, out account)) + { + cache.Set(address, AccountStruct.TotallyEmpty); + return false; + } + cache.Set(address, account); + } + else + { + Db.Metrics.IncrementStateTreeCacheHits(); + } + + return true; + } + + public void RemoveAccounts(ArrayPoolList address) + { + Parallel.ForEach(address.GroupBy(a => GetCacheIndex(a.Value)), + n => + { + LruCacheLowObject cache = _caches[n.Key]; + foreach (AddressAsKey a in n) + { + cache.Delete(a); + } + } + ); + } + + private static int GetCacheIndex(Address address) => address.Bytes[^1] & 0xf; + + public void Reset() + { + for (int i = 0; i < _caches.Length; i++) + { + _caches[i].Clear(); + } + } + } + private static void WriteTxPoolReport(in ILogger logger) { if (!logger.IsInfo) diff --git a/src/Nethermind/Nethermind.sln b/src/Nethermind/Nethermind.sln index 87f05329dfb..9409f66783c 100644 --- a/src/Nethermind/Nethermind.sln +++ b/src/Nethermind/Nethermind.sln @@ -224,6 +224,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution nuget.config = nuget.config EndProjectSection EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nethermind.Optimism.Test", "Nethermind.Optimism.Test\Nethermind.Optimism.Test.csproj", "{2438958D-46EA-4A7E-B89F-29E069DA0CCA}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Signer", "Signer", "{89311B58-AF36-4956-883D-54531BC1D5A3}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Nethermind.ExternalSigner.Plugin", "Nethermind.ExternalSigner.Plugin\Nethermind.ExternalSigner.Plugin.csproj", "{6528010D-7DCE-4935-9785-5270FF515F3E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -614,6 +620,14 @@ Global {AD09FBCB-5496-499B-9129-B6D139A65B6F}.Debug|Any CPU.Build.0 = Debug|Any CPU {AD09FBCB-5496-499B-9129-B6D139A65B6F}.Release|Any CPU.ActiveCfg = Release|Any CPU {AD09FBCB-5496-499B-9129-B6D139A65B6F}.Release|Any CPU.Build.0 = Release|Any CPU + {2438958D-46EA-4A7E-B89F-29E069DA0CCA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2438958D-46EA-4A7E-B89F-29E069DA0CCA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2438958D-46EA-4A7E-B89F-29E069DA0CCA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2438958D-46EA-4A7E-B89F-29E069DA0CCA}.Release|Any CPU.Build.0 = Release|Any CPU + {6528010D-7DCE-4935-9785-5270FF515F3E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6528010D-7DCE-4935-9785-5270FF515F3E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6528010D-7DCE-4935-9785-5270FF515F3E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6528010D-7DCE-4935-9785-5270FF515F3E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -678,6 +692,8 @@ Global {2EDE2554-59C0-4E78-92F7-EB8077AA597D} = {78BED57D-720E-4E6C-ABA2-397B73B494F9} {48E50409-26FE-4FD8-AF6E-2A0E79F794CE} = {2EDE2554-59C0-4E78-92F7-EB8077AA597D} {E1E7BEFC-52C0-49ED-B0A7-CB8C3250D120} = {4019B82F-1104-4D2C-9F96-05FD7D3575E8} + {89311B58-AF36-4956-883D-54531BC1D5A3} = {78BED57D-720E-4E6C-ABA2-397B73B494F9} + {6528010D-7DCE-4935-9785-5270FF515F3E} = {89311B58-AF36-4956-883D-54531BC1D5A3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {092CA5E3-6180-4ED7-A3CB-9B57FAC2AA85} diff --git a/tools/HiveConsensusWorkflowGenerator/Program.cs b/tools/HiveConsensusWorkflowGenerator/Program.cs index 18ed584f5e4..1324452862a 100644 --- a/tools/HiveConsensusWorkflowGenerator/Program.cs +++ b/tools/HiveConsensusWorkflowGenerator/Program.cs @@ -71,7 +71,7 @@ private static string FindDirectory(string searchPattern) string? currentDir = Environment.CurrentDirectory; do { - if (currentDir == null) + if (currentDir is null) { return ""; } @@ -80,7 +80,7 @@ private static string FindDirectory(string searchPattern) .EnumerateDirectories(currentDir, searchPattern, SearchOption.TopDirectoryOnly) .SingleOrDefault(); - if (dir != null) + if (dir is not null) { return dir; } diff --git a/tools/Nethermind.Tools.Kute/Auth/JwtAuth.cs b/tools/Nethermind.Tools.Kute/Auth/JwtAuth.cs index feb05054faa..a40bb7ef8ad 100644 --- a/tools/Nethermind.Tools.Kute/Auth/JwtAuth.cs +++ b/tools/Nethermind.Tools.Kute/Auth/JwtAuth.cs @@ -34,7 +34,7 @@ private string GenerateAuthToken() { var signingKey = new SymmetricSecurityKey(_secret); var credentials = new SigningCredentials(signingKey, SecurityAlgorithms.HmacSha256); - var claims = new[] { new Claim("iat", _clock.UtcNow.ToUnixTimeSeconds().ToString()) }; + var claims = new[] { new Claim(JwtRegisteredClaimNames.Iat, _clock.UtcNow.ToUnixTimeSeconds().ToString(), ClaimValueTypes.Integer64) }; var token = new JwtSecurityToken(claims: claims, signingCredentials: credentials); var handler = new JwtSecurityTokenHandler(); diff --git a/tools/SendBlobs/BlobSender.cs b/tools/SendBlobs/BlobSender.cs index 04cd72d2e4b..4b4d0889ced 100644 --- a/tools/SendBlobs/BlobSender.cs +++ b/tools/SendBlobs/BlobSender.cs @@ -362,7 +362,7 @@ private async static Task WaitForBlobInclusion(INodeManager nodeManager, Hash256 while (true) { var blockResult = await nodeManager.Post>("eth_getBlockByNumber", lastBlockNumber.ToString() ?? "latest", false); - if (blockResult != null) + if (blockResult is not null) { lastBlockNumber = blockResult.Number + 1; diff --git a/tools/SendBlobs/Dockerfile b/tools/SendBlobs/Dockerfile index 42e47ef7bf5..ba5cc72bf1c 100644 --- a/tools/SendBlobs/Dockerfile +++ b/tools/SendBlobs/Dockerfile @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited # SPDX-License-Identifier: LGPL-3.0-only -FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-jammy AS build +FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0-noble AS build ARG BUILD_CONFIG=release ARG BUILD_TIMESTAMP diff --git a/tools/SendBlobs/FundsDistributor.cs b/tools/SendBlobs/FundsDistributor.cs index 24a1d9ab975..6356d91d345 100644 --- a/tools/SendBlobs/FundsDistributor.cs +++ b/tools/SendBlobs/FundsDistributor.cs @@ -119,10 +119,10 @@ public async Task> DitributeFunds(Signer distributeFrom, uin .Encode(tx, RlpBehaviors.SkipTypedWrapping | RlpBehaviors.InMempoolForm).Bytes); string? result = await _nodeManager.Post("eth_sendRawTransaction", "0x" + txRlp); - if (result != null) + if (result is not null) txHash.Add(result); - if (keyWriter != null) + if (keyWriter is not null) keyWriter.WriteLine(key.ToString()); nonce++; @@ -194,7 +194,7 @@ public async Task> ReclaimFunds(Address beneficiary, UInt256 .Encode(tx, RlpBehaviors.SkipTypedWrapping | RlpBehaviors.InMempoolForm).Bytes); string? result = await _nodeManager.Post("eth_sendRawTransaction", "0x" + txRlp); - if (result != null) + if (result is not null) txHashes.Add(result); } return txHashes; diff --git a/tools/TxParser/Program.cs b/tools/TxParser/Program.cs index 53e921fa92d..409c755ef97 100644 --- a/tools/TxParser/Program.cs +++ b/tools/TxParser/Program.cs @@ -23,7 +23,7 @@ { EthereumEcdsa ecdsa = new(BlockchainIds.Mainnet, SimpleConsoleLogManager.Instance); Address? sender = ecdsa.RecoverAddress(tx); - if (sender == null) + if (sender is null) { throw new InvalidDataException("Could not recover sender address"); } diff --git a/tools/docgen/DBSizeGenerator.cs b/tools/docgen/DBSizeGenerator.cs new file mode 100644 index 00000000000..8877f66e8de --- /dev/null +++ b/tools/docgen/DBSizeGenerator.cs @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.DocGen; + +internal static class DBSizeGenerator +{ + private const string _chainSizesDir = "chainSizes"; + + private static readonly List _dbList = + [ + "state", + "receipts", + "blocks", + "bloom", + "headers", + "code", + "blobTransactions" + ]; + + internal static void Generate() + { + IList chainOrder = + [ + "mainnet", + "sepolia", + "holesky", + "gnosis", + "chiado", + "energyweb", + "volta" + ]; + var startMark = ""; + var endMark = ""; + var fileName = "database.md"; + + var chainSizesPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, _chainSizesDir); + var chains = Directory + .GetFiles(chainSizesPath) + .Select(Path.GetFileNameWithoutExtension) + .OrderBy(c => + { + var i = chainOrder.IndexOf(c!); + + return i == -1 ? int.MaxValue : i; + }) + .ToList(); + + File.Delete($"~{fileName}"); + + using var readStream = new StreamReader(File.OpenRead(fileName)); + using var writeStream = new StreamWriter(File.OpenWrite($"~{fileName}")); + + writeStream.NewLine = "\n"; + + var line = string.Empty; + + do + { + line = readStream.ReadLine(); + + writeStream.WriteLine(line); + } + while (!line?.Equals(startMark, StringComparison.Ordinal) ?? false); + + writeStream.WriteLine(); + + WriteMarkdown(writeStream, chains!); + + var skip = true; + + for (line = readStream.ReadLine(); line is not null; line = readStream.ReadLine()) + { + if (skip) + { + if (line?.Equals(endMark, StringComparison.Ordinal) ?? false) + skip = false; + else + continue; + } + + writeStream.WriteLine(line); + } + + readStream.Close(); + writeStream.Close(); + + File.Move($"~{fileName}", fileName, true); + + Console.WriteLine($"Updated {fileName}"); + } + + private static void WriteMarkdown(StreamWriter file, List chains) + { + file.WriteLine(""); + + foreach (var chain in chains) + WriteChainSize(file, chain); + + file.WriteLine(""" + + + """); + } + + private static void WriteChainSize(StreamWriter file, string chain) + { + using var json = JsonDocument.Parse(File.ReadAllText($"{_chainSizesDir}/{chain}.json")); + + if (json.RootElement.ValueKind != JsonValueKind.Object) + return; + + var chainCapitalized = $"{char.ToUpper(chain[0])}{chain[1..]}"; + + file.WriteLine($""" + + + """); + + var items = json.RootElement.EnumerateObject(); + + foreach (var db in _dbList) + { + var size = items + .FirstOrDefault(e => e.Name.Contains(db, StringComparison.Ordinal)) + .Value.ToString(); + + file.WriteLine($"- `{db}`: {FormatSize(size)}"); + } + + var totalSize = items + .FirstOrDefault(e => e.Name.EndsWith("nethermind_db", StringComparison.Ordinal)) + .Value.ToString(); + + file.WriteLine($""" + - ... + - **Total: {FormatSize(totalSize)}** + + + """); + } + + private static string FormatSize(string value) => value + .Replace("G", " GB") + .Replace("M", " MB") + .Replace("K", " KB"); +} diff --git a/tools/docgen/JsonRpcGenerator.cs b/tools/docgen/JsonRpcGenerator.cs index 424dcff4ec5..5257ccd7b3b 100644 --- a/tools/docgen/JsonRpcGenerator.cs +++ b/tools/docgen/JsonRpcGenerator.cs @@ -6,7 +6,6 @@ using Nethermind.JsonRpc.Modules.Evm; using Nethermind.JsonRpc.Modules.Rpc; using Nethermind.JsonRpc.Modules.Subscribe; -using Nethermind.JsonRpc.Modules.Witness; using Newtonsoft.Json; namespace Nethermind.DocGen; @@ -23,8 +22,7 @@ internal static void Generate() typeof(IEvmRpcModule).FullName, typeof(IRpcModule).FullName, typeof(IRpcRpcModule).FullName, - typeof(ISubscribeRpcModule).FullName, - typeof(IWitnessRpcModule).FullName + typeof(ISubscribeRpcModule).FullName }; var types = new[] { "Nethermind.JsonRpc", "Nethermind.Consensus.Clique" } .SelectMany(a => Assembly.Load(a).GetTypes()) diff --git a/tools/docgen/Program.cs b/tools/docgen/Program.cs index 95cf44b61d0..58ea9f3e5bb 100644 --- a/tools/docgen/Program.cs +++ b/tools/docgen/Program.cs @@ -3,6 +3,21 @@ using Nethermind.DocGen; -ConfigGenerator.Generate(); -JsonRpcGenerator.Generate(); -MetricsGenerator.Generate(); +foreach (var arg in args) +{ + switch (arg) + { + case "--config": + ConfigGenerator.Generate(); + break; + case "--dbsize": + DBSizeGenerator.Generate(); + break; + case "--jsonrpc": + JsonRpcGenerator.Generate(); + break; + case "--metrics": + MetricsGenerator.Generate(); + break; + } +} diff --git a/tools/docgen/Properties/launchSettings.json b/tools/docgen/Properties/launchSettings.json new file mode 100644 index 00000000000..7ac1a7f915c --- /dev/null +++ b/tools/docgen/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "DocGen": { + "commandName": "Project", + "commandLineArgs": "--dbsize --config --jsonrpc --metrics" + } + } +}