From 6bf64f70abc4e5cac3b7f0c4e069ed5707ef295f Mon Sep 17 00:00:00 2001 From: maciejnems Date: Wed, 8 Feb 2023 13:17:54 +0100 Subject: [PATCH 1/2] add pruning catch up tests --- .github/workflows/e2e-tests-main-devnet.yml | 12 ++++++--- local-tests/run_nodes.py | 3 +-- local-tests/test_catch_up.py | 22 +++++++++++++---- local-tests/test_multiple_restarts.py | 27 +++++++++++++++------ 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml index d7dadc2218..2e5cf67377 100644 --- a/.github/workflows/e2e-tests-main-devnet.yml +++ b/.github/workflows/e2e-tests-main-devnet.yml @@ -62,7 +62,7 @@ jobs: - name: GIT | Checkout source code uses: actions/checkout@v2 - - name: Install Rust toolchain + - name: Install Rust toolchainrun_e2e_testrun_e2e_test uses: actions-rs/toolchain@v1 - name: Install Protoc @@ -796,6 +796,9 @@ jobs: name: Test catching up runs-on: ubuntu-20.04 needs: build-new-node + strategy: + matrix: + pruning: ['', '--state-pruning 90'] steps: - name: Checkout source code uses: actions/checkout@v2 @@ -815,12 +818,15 @@ jobs: env: # Relative to local-tests/ directory ALEPH_NODE_BINARY: aleph-test-node/aleph-node - run: ./.github/scripts/test_catch_up.sh + run: ./.github/scripts/test_catch_up.sh ${{ matrix.pruning }} test-multiple-restarts: name: Test multiple restarts runs-on: ubuntu-20.04 needs: build-new-node + strategy: + matrix: + pruning: ['', '--state-pruning 2048'] steps: - name: Checkout source code uses: actions/checkout@v2 @@ -840,7 +846,7 @@ jobs: env: # Relative to local-tests/ directory ALEPH_NODE_BINARY: aleph-release-node/aleph-node - run: ./.github/scripts/test_multiple_restarts.sh + run: ./.github/scripts/test_multiple_restarts.sh ${{ matrix.pruning }} check-runtime-change: name: Inspect whether runtime version has been changed (compared with main) diff --git a/local-tests/run_nodes.py b/local-tests/run_nodes.py index 25592e8f15..1cc9cb0103 100755 --- a/local-tests/run_nodes.py +++ b/local-tests/run_nodes.py @@ -35,8 +35,7 @@ unit_creation_delay=500, execution='Native', rpc_cors='all', - rpc_methods='Unsafe', - state_pruning='archive') + rpc_methods='Unsafe') addresses = [n.address() for n in chain] chain.set_flags(bootnodes=addresses[0], public_addr=addresses) diff --git a/local-tests/test_catch_up.py b/local-tests/test_catch_up.py index 1ef727fda9..0609ca497e 100755 --- a/local-tests/test_catch_up.py +++ b/local-tests/test_catch_up.py @@ -3,11 +3,18 @@ import sys from os.path import abspath, join from time import sleep, ctime +import argparse from chainrunner import Chain, Seq, generate_keys, check_finalized + def printt(s): print(ctime() + ' | ' + s) + +argParser = argparse.ArgumentParser() +argParser.add_argument("--state-pruning", help="state pruning argument") +state_pruning = argParser.parse_args().state_pruning + # Path to working directory, where chainspec, logs and nodes' dbs are written: workdir = abspath(os.getenv('WORKDIR', '/tmp/workdir')) # Path to the aleph-node binary (important use short-session feature): @@ -23,6 +30,8 @@ def printt(s): print(ctime() + ' | ' + s) nonvalidators=all_accounts[4:], sudo_account_id=keys[phrases[0]], chain_type='local') +printt('Purging previous chain') +chain.purge() chain.set_flags('no-mdns', port=Seq(30334), @@ -30,12 +39,15 @@ def printt(s): print(ctime() + ' | ' + s) ws_port=Seq(9944), rpc_port=Seq(9933), unit_creation_delay=200, - execution='Native', - state_pruning='archive') + execution='Native') addresses = [n.address() for n in chain] validator_addresses = [n.validator_address() for n in chain] chain.set_flags(bootnodes=addresses[0]) -chain.set_flags_validator(public_addr=addresses, public_validator_addresses=validator_addresses) +chain.set_flags_validator(public_addr=addresses, + public_validator_addresses=validator_addresses) +if state_pruning is not None: + chain.set_flags('experimental-pruning', state_pruning=state_pruning, + ) chain.set_flags_validator('validator') @@ -46,10 +58,10 @@ def printt(s): print(ctime() + ' | ' + s) chain.start('aleph', nodes=[4, 5]) -printt('Waiting for finalization') -chain.wait_for_finalization(0) printt('Waiting for authorities') chain.wait_for_authorities() +printt('Waiting for finalization') +chain.wait_for_finalization(90) printt('Killing one validator and one nonvalidator') chain.stop(nodes=[3, 4]) diff --git a/local-tests/test_multiple_restarts.py b/local-tests/test_multiple_restarts.py index 4b062cc8c3..701ba0e906 100755 --- a/local-tests/test_multiple_restarts.py +++ b/local-tests/test_multiple_restarts.py @@ -3,11 +3,18 @@ import sys from os.path import abspath, join from time import sleep, ctime +import argparse from chainrunner import Chain, Seq, generate_keys, check_finalized + def printt(s): print(ctime() + ' | ' + s) + +argParser = argparse.ArgumentParser() +argParser.add_argument("--state-pruning", help="state pruning argument") +state_pruning = argParser.parse_args().state_pruning + # Path to working directory, where chainspec, logs and nodes' dbs are written: workdir = abspath(os.getenv('WORKDIR', '/tmp/workdir')) # Path to the aleph-node binary (important DON'T use short-session feature): @@ -22,28 +29,34 @@ def printt(s): print(ctime() + ' | ' + s) sudo_account_id=keys[phrases[0]], chain_type='local') +printt('Purging previous chain') +chain.purge() + chain.set_flags('no-mdns', port=Seq(30334), validator_port=Seq(30343), ws_port=Seq(9944), rpc_port=Seq(9933), unit_creation_delay=200, - execution='Native', - state_pruning='archive') + execution='Native') addresses = [n.address() for n in chain] validator_addresses = [n.validator_address() for n in chain] chain.set_flags(bootnodes=addresses[0]) -chain.set_flags_validator(public_addr=addresses, public_validator_addresses=validator_addresses) +chain.set_flags_validator(public_addr=addresses, + public_validator_addresses=validator_addresses) +if state_pruning is not None: + chain.set_flags('experimental-pruning', state_pruning=state_pruning, + ) chain.set_flags_validator('validator') printt('Starting the chain') chain.start('aleph') -printt('Waiting for finalization') -chain.wait_for_finalization(0) printt('Waiting for authorities') chain.wait_for_authorities() +printt('Waiting for finalization') +chain.wait_for_finalization(256) delta = 5 @@ -63,7 +76,7 @@ def printt(s): print(ctime() + ' | ' + s) sys.exit(1) printt('Restarting nodes') - chain[3].start('aleph') + chain.start('aleph', nodes=[3]) printt('Waiting for finalization') chain.wait_for_finalization(finalized_before_start[3], nodes=[3]) @@ -73,5 +86,5 @@ def printt(s): print(ctime() + ' | ' + s) # Check if the murdered node started catching up with reasonable nr of blocks. if diff <= delta: - printt(f'Too small catch up for validators: {validator_diff}') + printt(f'Too small catch up for validators: {diff}') sys.exit(1) From 46100897807ea20f24f021f78f3f2c6cb2757a86 Mon Sep 17 00:00:00 2001 From: maciejnems Date: Thu, 9 Feb 2023 17:37:08 +0100 Subject: [PATCH 2/2] apply review changes --- .github/workflows/e2e-tests-main-devnet.yml | 2 +- local-tests/test_catch_up.py | 10 ++++++---- local-tests/test_multiple_restarts.py | 11 ++++++----- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml index 2e5cf67377..bea032fd05 100644 --- a/.github/workflows/e2e-tests-main-devnet.yml +++ b/.github/workflows/e2e-tests-main-devnet.yml @@ -62,7 +62,7 @@ jobs: - name: GIT | Checkout source code uses: actions/checkout@v2 - - name: Install Rust toolchainrun_e2e_testrun_e2e_test + - name: Install Rust toolchain uses: actions-rs/toolchain@v1 - name: Install Protoc diff --git a/local-tests/test_catch_up.py b/local-tests/test_catch_up.py index 0609ca497e..cb871338ba 100755 --- a/local-tests/test_catch_up.py +++ b/local-tests/test_catch_up.py @@ -30,8 +30,6 @@ def printt(s): print(ctime() + ' | ' + s) nonvalidators=all_accounts[4:], sudo_account_id=keys[phrases[0]], chain_type='local') -printt('Purging previous chain') -chain.purge() chain.set_flags('no-mdns', port=Seq(30334), @@ -58,10 +56,14 @@ def printt(s): print(ctime() + ' | ' + s) chain.start('aleph', nodes=[4, 5]) +printt('Waiting for finalization') +chain.wait_for_finalization(0) printt('Waiting for authorities') chain.wait_for_authorities() -printt('Waiting for finalization') -chain.wait_for_finalization(90) +if state_pruning is not None and state_pruning.isnumeric(): + bound = min(256, int(state_pruning)) + printt(f'Pruning turned on. Waiting for {bound} blocks to finalize') + chain.wait_for_finalization(bound) printt('Killing one validator and one nonvalidator') chain.stop(nodes=[3, 4]) diff --git a/local-tests/test_multiple_restarts.py b/local-tests/test_multiple_restarts.py index 701ba0e906..18995274cc 100755 --- a/local-tests/test_multiple_restarts.py +++ b/local-tests/test_multiple_restarts.py @@ -29,9 +29,6 @@ def printt(s): print(ctime() + ' | ' + s) sudo_account_id=keys[phrases[0]], chain_type='local') -printt('Purging previous chain') -chain.purge() - chain.set_flags('no-mdns', port=Seq(30334), validator_port=Seq(30343), @@ -53,10 +50,14 @@ def printt(s): print(ctime() + ' | ' + s) printt('Starting the chain') chain.start('aleph') +printt('Waiting for finalization') +chain.wait_for_finalization(0) printt('Waiting for authorities') chain.wait_for_authorities() -printt('Waiting for finalization') -chain.wait_for_finalization(256) +if state_pruning is not None and state_pruning.isnumeric(): + bound = min(256, int(state_pruning)) + printt(f'Pruning turned on. Waiting for {bound} blocks to finalize') + chain.wait_for_finalization(bound) delta = 5