From 619fbcd53c7e5c02781d9088c6acdde05f3553eb Mon Sep 17 00:00:00 2001 From: Dmitri Tsumak Date: Mon, 30 Jan 2023 15:22:26 +0200 Subject: [PATCH] Load local deposit data (#17) * Load deposit data locally * Update oracles address * Update logging messages * Fix index var name --- poetry.lock | 81 ++++++++------- pyproject.toml | 2 +- src/common/abi/IEthVault.json | 11 -- src/config/networks.py | 8 +- src/config/settings.py | 3 + src/main.py | 28 ++--- src/validators/database.py | 88 +--------------- src/validators/execution.py | 186 +++++++++++++--------------------- src/validators/ipfs.py | 29 ------ src/validators/tasks.py | 60 ++++++----- src/validators/typings.py | 15 +-- src/validators/utils.py | 56 ++++++++-- 12 files changed, 221 insertions(+), 346 deletions(-) delete mode 100644 src/validators/ipfs.py diff --git a/poetry.lock b/poetry.lock index 74f3273a..1e4bb8c3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -908,14 +908,14 @@ test = ["eth-utils (>=1.0.1,<3)", "hypothesis (>=3.44.24,<=6.31.6)", "pytest (>= [[package]] name = "identify" -version = "2.5.15" +version = "2.5.16" description = "File identification library for Python" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "identify-2.5.15-py2.py3-none-any.whl", hash = "sha256:1f4b36c5f50f3f950864b2a047308743f064eaa6f6645da5e5c780d1c7125487"}, - {file = "identify-2.5.15.tar.gz", hash = "sha256:c22aa206f47cc40486ecf585d27ad5f40adbfc494a3fa41dc3ed0499a23b123f"}, + {file = "identify-2.5.16-py2.py3-none-any.whl", hash = "sha256:832832a58ecc1b8f33d5e8cb4f7d3db2f5c7fbe922dfee5f958b48fed691501a"}, + {file = "identify-2.5.16.tar.gz", hash = "sha256:c47acedfe6495b1c603ed7e93469b26e839cab38db4155113f36f718f8b3dc47"}, ] [package.extras] @@ -1603,38 +1603,45 @@ files = [ [[package]] name = "pycryptodome" -version = "3.16.0" +version = "3.17" description = "Cryptographic library for Python" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ - {file = "pycryptodome-3.16.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:e061311b02cefb17ea93d4a5eb1ad36dca4792037078b43e15a653a0a4478ead"}, - {file = "pycryptodome-3.16.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:dab9359cc295160ba96738ba4912c675181c84bfdf413e5c0621cf00b7deeeaa"}, - {file = "pycryptodome-3.16.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:0198fe96c22f7bc31e7a7c27a26b2cec5af3cf6075d577295f4850856c77af32"}, - {file = "pycryptodome-3.16.0-cp27-cp27m-manylinux2014_aarch64.whl", hash = "sha256:58172080cbfaee724067a3c017add6a1a3cc167bbc8478dc5f2e5f45fa658763"}, - {file = "pycryptodome-3.16.0-cp27-cp27m-win32.whl", hash = "sha256:4d950ed2a887905b3fa709b86be5a163e26e1b174703ed59d34eb6832f213222"}, - {file = "pycryptodome-3.16.0-cp27-cp27m-win_amd64.whl", hash = "sha256:c69e19afc734b2a17b9d78b7bcb544aabd5a52ff628e14283b6e9404d27d0517"}, - {file = "pycryptodome-3.16.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:1fc16c80a5da8231fd1f953a7b8dfeb415f68120248e8d68383c5c2c4b18708c"}, - {file = "pycryptodome-3.16.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5df582f2112dd72331de7e567837e136a9629181a8ab69ef8949e4bc294a0b99"}, - {file = "pycryptodome-3.16.0-cp27-cp27mu-manylinux2014_aarch64.whl", hash = "sha256:2bf2a270906a02b7b255e1a0d7b3aea4f06b3983c51ddec1673c380e0dff5b30"}, - {file = "pycryptodome-3.16.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b12a88566a98617b1a34b4e5a805dff2da98d83fc74262aff3c3d724d0f525d6"}, - {file = "pycryptodome-3.16.0-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:69adf32522b75968e1cbf25b5d83e87c04cd9a55610ce1e4a19012e58e7e4023"}, - {file = "pycryptodome-3.16.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d67a2d2fe344953e4572a7d30668cceb516b04287b8638170d562065e53ee2e0"}, - {file = "pycryptodome-3.16.0-cp35-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e750a21d8a265b1f9bfb1a28822995ea33511ba7db5e2b55f41fb30781d0d073"}, - {file = "pycryptodome-3.16.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:47c71a0347847b747ba1349767b16cde049bc36f21654eb09cc82306ef5fdcf8"}, - {file = "pycryptodome-3.16.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:856ebf822d08d754af62c22e2b93626509a72773214f92db1551e2b68d9e2a1b"}, - {file = "pycryptodome-3.16.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6016269bb56caf0327f6d42e7bad1247e08b78407446dff562240c65f85d5a5e"}, - {file = "pycryptodome-3.16.0-cp35-abi3-win32.whl", hash = "sha256:1047ac2b9847ae84ea454e6e20db7dcb755a81c1b1631a879213d2b0ad835ff2"}, - {file = "pycryptodome-3.16.0-cp35-abi3-win_amd64.whl", hash = "sha256:13b3e610a2f8938c61a90b20625069ab7a77ccea20d65a9a0f926cc0cc1314b1"}, - {file = "pycryptodome-3.16.0-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:265bfcbbf20d58e6871ce695a7a08aac9b41a0553060d9c05363abd6f3391bdd"}, - {file = "pycryptodome-3.16.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:54d807314c66785c69cd25425933d4bd4c23547a593cdcf49d962fa3e0081336"}, - {file = "pycryptodome-3.16.0-pp27-pypy_73-win32.whl", hash = "sha256:63165fbdc247450017eb9ef04cfe15cb3a72ca48ffcc3a3b75b08c0340bf3647"}, - {file = "pycryptodome-3.16.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:95069fd9e2813668a2713a1efcc65cc26d2c7e741401ac46628f1ec957511f1b"}, - {file = "pycryptodome-3.16.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:d1daec4d31bb00918e4e178297ac6ca6f86ec4c851ba584770533ece554d29e2"}, - {file = "pycryptodome-3.16.0-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:48d99869d58f3979d72f6fa0c50f48d16f14973bc4a3adb0ce3b8325fdd7e223"}, - {file = "pycryptodome-3.16.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:c82e3bc1e70dde153b0956bffe20a15715a1fe3e00bc23e88d6973eda4505944"}, - {file = "pycryptodome-3.16.0.tar.gz", hash = "sha256:0e45d2d852a66ecfb904f090c3f87dc0dfb89a499570abad8590f10d9cffb350"}, + {file = "pycryptodome-3.17-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:2c5631204ebcc7ae33d11c43037b2dafe25e2ab9c1de6448eb6502ac69c19a56"}, + {file = "pycryptodome-3.17-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:04779cc588ad8f13c80a060b0b1c9d1c203d051d8a43879117fe6b8aaf1cd3fa"}, + {file = "pycryptodome-3.17-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:f812d58c5af06d939b2baccdda614a3ffd80531a26e5faca2c9f8b1770b2b7af"}, + {file = "pycryptodome-3.17-cp27-cp27m-manylinux2014_aarch64.whl", hash = "sha256:9453b4e21e752df8737fdffac619e93c9f0ec55ead9a45df782055eb95ef37d9"}, + {file = "pycryptodome-3.17-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:121d61663267f73692e8bde5ec0d23c9146465a0d75cad75c34f75c752527b01"}, + {file = "pycryptodome-3.17-cp27-cp27m-win32.whl", hash = "sha256:ba2d4fcb844c6ba5df4bbfee9352ad5352c5ae939ac450e06cdceff653280450"}, + {file = "pycryptodome-3.17-cp27-cp27m-win_amd64.whl", hash = "sha256:87e2ca3aa557781447428c4b6c8c937f10ff215202ab40ece5c13a82555c10d6"}, + {file = "pycryptodome-3.17-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:f44c0d28716d950135ff21505f2c764498eda9d8806b7c78764165848aa419bc"}, + {file = "pycryptodome-3.17-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5a790bc045003d89d42e3b9cb3cc938c8561a57a88aaa5691512e8540d1ae79c"}, + {file = "pycryptodome-3.17-cp27-cp27mu-manylinux2014_aarch64.whl", hash = "sha256:d086d46774e27b280e4cece8ab3d87299cf0d39063f00f1e9290d096adc5662a"}, + {file = "pycryptodome-3.17-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:5587803d5b66dfd99e7caa31ed91fba0fdee3661c5d93684028ad6653fce725f"}, + {file = "pycryptodome-3.17-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:e7debd9c439e7b84f53be3cf4ba8b75b3d0b6e6015212355d6daf44ac672e210"}, + {file = "pycryptodome-3.17-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ca1ceb6303be1282148f04ac21cebeebdb4152590842159877778f9cf1634f09"}, + {file = "pycryptodome-3.17-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:dc22cc00f804485a3c2a7e2010d9f14a705555f67020eb083e833cabd5bd82e4"}, + {file = "pycryptodome-3.17-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80ea8333b6a5f2d9e856ff2293dba2e3e661197f90bf0f4d5a82a0a6bc83a626"}, + {file = "pycryptodome-3.17-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c133f6721fba313722a018392a91e3c69d3706ae723484841752559e71d69dc6"}, + {file = "pycryptodome-3.17-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:333306eaea01fde50a73c4619e25631e56c4c61bd0fb0a2346479e67e3d3a820"}, + {file = "pycryptodome-3.17-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:1a30f51b990994491cec2d7d237924e5b6bd0d445da9337d77de384ad7f254f9"}, + {file = "pycryptodome-3.17-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:909e36a43fe4a8a3163e9c7fc103867825d14a2ecb852a63d3905250b308a4e5"}, + {file = "pycryptodome-3.17-cp35-abi3-win32.whl", hash = "sha256:a3228728a3808bc9f18c1797ec1179a0efb5068c817b2ffcf6bcd012494dffb2"}, + {file = "pycryptodome-3.17-cp35-abi3-win_amd64.whl", hash = "sha256:9ec565e89a6b400eca814f28d78a9ef3f15aea1df74d95b28b7720739b28f37f"}, + {file = "pycryptodome-3.17-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:e1819b67bcf6ca48341e9b03c2e45b1c891fa8eb1a8458482d14c2805c9616f2"}, + {file = "pycryptodome-3.17-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:f8e550caf52472ae9126953415e4fc554ab53049a5691c45b8816895c632e4d7"}, + {file = "pycryptodome-3.17-pp27-pypy_73-win32.whl", hash = "sha256:afbcdb0eda20a0e1d44e3a1ad6d4ec3c959210f4b48cabc0e387a282f4c7deb8"}, + {file = "pycryptodome-3.17-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a74f45aee8c5cc4d533e585e0e596e9f78521e1543a302870a27b0ae2106381e"}, + {file = "pycryptodome-3.17-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38bbd6717eac084408b4094174c0805bdbaba1f57fc250fd0309ae5ec9ed7e09"}, + {file = "pycryptodome-3.17-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f68d6c8ea2974a571cacb7014dbaada21063a0375318d88ac1f9300bc81e93c3"}, + {file = "pycryptodome-3.17-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:8198f2b04c39d817b206ebe0db25a6653bb5f463c2319d6f6d9a80d012ac1e37"}, + {file = "pycryptodome-3.17-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3a232474cd89d3f51e4295abe248a8b95d0332d153bf46444e415409070aae1e"}, + {file = "pycryptodome-3.17-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4992ec965606054e8326e83db1c8654f0549cdb26fce1898dc1a20bc7684ec1c"}, + {file = "pycryptodome-3.17-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53068e33c74f3b93a8158dacaa5d0f82d254a81b1002e0cd342be89fcb3433eb"}, + {file = "pycryptodome-3.17-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:74794a2e2896cd0cf56fdc9db61ef755fa812b4a4900fa46c49045663a92b8d0"}, + {file = "pycryptodome-3.17.tar.gz", hash = "sha256:bce2e2d8e82fcf972005652371a3e8731956a0c1fbb719cc897943b3695ad91b"}, ] [[package]] @@ -2062,14 +2069,14 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "66.1.1" +version = "67.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-66.1.1-py3-none-any.whl", hash = "sha256:6f590d76b713d5de4e49fe4fbca24474469f53c83632d5d0fd056f7ff7e8112b"}, - {file = "setuptools-66.1.1.tar.gz", hash = "sha256:ac4008d396bc9cd983ea483cb7139c0240a07bbc74ffb6232fceffedc6cf03a8"}, + {file = "setuptools-67.0.0-py3-none-any.whl", hash = "sha256:9d790961ba6219e9ff7d9557622d2fe136816a264dd01d5997cfc057d804853d"}, + {file = "setuptools-67.0.0.tar.gz", hash = "sha256:883131c5b6efa70b9101c7ef30b2b7b780a4283d5fc1616383cdf22c83cbefe6"}, ] [package.extras] @@ -2134,7 +2141,7 @@ pbr = ">=2.0.0,<2.1.0 || >2.1.0" [[package]] name = "sw-utils" -version = "0.2.12" +version = "0.2.13" description = "StakeWise Python utils" category = "main" optional = false @@ -2152,8 +2159,8 @@ web3 = "^6.0.0b9" [package.source] type = "git" url = "https://github.com/stakewise/sw-utils.git" -reference = "v0.2.12" -resolved_reference = "f2452c92b096447cf26708ac8ac563c01d029983" +reference = "v0.2.13" +resolved_reference = "91f19253fed83cc79089a93161a7aa6fd22047a5" [[package]] name = "toml" @@ -2576,4 +2583,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "f8ead2643b7c29194ff954c4f56d785868dabd1e976850b77cdbc92d9695b699" +content-hash = "b6427f9e39923a830422120f0288368fa750806cf9dd0c5e333a0e1df230831b" diff --git a/pyproject.toml b/pyproject.toml index c4573964..4707c2f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ sentry-sdk = "==1.12.1" web3 = "==6.0.0b9" py-ecc = "==6.0.0" multiproof = { git = "https://github.com/stakewise/multiproof.git", rev = "v0.1.2" } -sw-utils = { git = "https://github.com/stakewise/sw-utils.git", rev = "v0.2.12" } +sw-utils = { git = "https://github.com/stakewise/sw-utils.git", rev = "v0.2.13" } staking-deposit = { git = "https://github.com/ethereum/staking-deposit-cli.git", rev = "v2.3.0" } pycryptodomex = "==3.16.0" milagro-bls-binding = "==1.9.0" diff --git a/src/common/abi/IEthVault.json b/src/common/abi/IEthVault.json index 3c38a8b3..56d707ec 100644 --- a/src/common/abi/IEthVault.json +++ b/src/common/abi/IEthVault.json @@ -297,12 +297,6 @@ "internalType": "bytes32", "name": "validatorsRoot", "type": "bytes32" - }, - { - "indexed": false, - "internalType": "string", - "name": "validatorsIpfsHash", - "type": "string" } ], "name": "ValidatorsRootUpdated", @@ -978,11 +972,6 @@ "internalType": "bytes32", "name": "_validatorsRoot", "type": "bytes32" - }, - { - "internalType": "string", - "name": "validatorsIpfsHash", - "type": "string" } ], "name": "setValidatorsRoot", diff --git a/src/config/networks.py b/src/config/networks.py index 8376ee4f..2e84b296 100644 --- a/src/config/networks.py +++ b/src/config/networks.py @@ -21,7 +21,6 @@ class NetworkConfig: ORACLES_CONTRACT_ADDRESS: ChecksumAddress ORACLES_GENESIS_BLOCK: BlockNumber GENESIS_VALIDATORS_ROOT: Bytes32 - VAULT_GENESIS_BLOCK: BlockNumber SECONDS_PER_BLOCK: Decimal CONFIRMATION_BLOCKS: int GENESIS_FORK_VERSION: bytes @@ -42,7 +41,6 @@ class NetworkConfig: hexstr=HexStr('0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95') ) ), - VAULT_GENESIS_BLOCK=BlockNumber(0), SECONDS_PER_BLOCK=Decimal(12), CONFIRMATION_BLOCKS=64, GENESIS_FORK_VERSION=bytes.fromhex('00000000'), @@ -54,15 +52,14 @@ class NetworkConfig: ), VALIDATORS_REGISTRY_GENESIS_BLOCK=BlockNumber(4367321), ORACLES_CONTRACT_ADDRESS=Web3.to_checksum_address( - '0xb1A899f03a7F68C81f0d80fD2162214B8562E3e2' + '0x7f6F787feC4735B914EE4836A78487F2bFA9e70B' ), - ORACLES_GENESIS_BLOCK=BlockNumber(8368601), + ORACLES_GENESIS_BLOCK=BlockNumber(8398334), GENESIS_VALIDATORS_ROOT=Bytes32( Web3.to_bytes( hexstr=HexStr('0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb') ) ), - VAULT_GENESIS_BLOCK=BlockNumber(8368606), SECONDS_PER_BLOCK=Decimal(12), CONFIRMATION_BLOCKS=64, GENESIS_FORK_VERSION=bytes.fromhex('00001020'), @@ -81,7 +78,6 @@ class NetworkConfig: hexstr=HexStr('0xf5dcb5564e829aab27264b9becd5dfaa017085611224cb3036f573368dbb9d47') ) ), - VAULT_GENESIS_BLOCK=BlockNumber(0), SECONDS_PER_BLOCK=Decimal('6.8'), CONFIRMATION_BLOCKS=24, GENESIS_FORK_VERSION=bytes.fromhex('00000064'), diff --git a/src/config/settings.py b/src/config/settings.py index 1a1cc8a2..a670dafc 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -20,6 +20,9 @@ KEYSTORES_PASSWORD_PATH = config('KEYSTORES_PASSWORD_PATH') KEYSTORES_PATH = config('KEYSTORES_PATH') +# deposit data +DEPOSIT_DATA_PATH = config('DEPOSIT_DATA_PATH') + # operator OPERATOR_PRIVATE_KEY = config('OPERATOR_PRIVATE_KEY') diff --git a/src/main.py b/src/main.py index 8f1d7898..20cc7b86 100644 --- a/src/main.py +++ b/src/main.py @@ -9,12 +9,9 @@ from src.common.clients import execution_client from src.config.settings import LOG_LEVEL, NETWORK, NETWORK_CONFIG, SENTRY_DSN from src.validators.database import setup as validators_db_setup -from src.validators.execution import ( - NetworkValidatorsProcessor, - VaultValidatorsProcessor, -) +from src.validators.execution import NetworkValidatorsProcessor from src.validators.tasks import load_genesis_validators, register_validators -from src.validators.utils import load_private_keys +from src.validators.utils import load_deposit_data, load_keystores logging.basicConfig( format='%(asctime)s %(levelname)-8s %(message)s', @@ -39,31 +36,24 @@ async def main() -> None: # load genesis validators for some networks await load_genesis_validators() - # extract private keys from the keystores - private_keys = load_private_keys() + # load keystores + keystores = load_keystores() + + # load deposit data + deposit_data = await load_deposit_data() # start operator tasks interrupt_handler = InterruptHandler() - # periodically scan validators root updates - vault_validators_processor = VaultValidatorsProcessor() - vault_validators_scanner = EventScanner( - vault_validators_processor - ) - # periodically scan network validator updates network_validators_processor = NetworkValidatorsProcessor() - network_validators_scanner = EventScanner( - network_validators_processor - ) + network_validators_scanner = EventScanner(network_validators_processor) while not interrupt_handler.exit: to_block = await get_safe_block_number() await asyncio.gather( # check and register new validators - register_validators(private_keys), - # process validators root updates - vault_validators_scanner.process_new_events(to_block), + register_validators(keystores, deposit_data), # process new network validators network_validators_scanner.process_new_events(to_block), ) diff --git a/src/validators/database.py b/src/validators/database.py index c0f0c251..aa7b45b9 100644 --- a/src/validators/database.py +++ b/src/validators/database.py @@ -6,77 +6,13 @@ from src.common.clients import db_client from src.config.settings import NETWORK -from src.validators.typings import DepositData, NetworkValidator, ValidatorsRoot +from src.validators.typings import NetworkValidator NETWORK_VALIDATORS_TABLE = f'{NETWORK}_network_validators' -DEPOSIT_DATA_TABLE = f'{NETWORK}_deposit_data' -VALIDATORS_ROOT_TABLE = f'{NETWORK}_validators_root' logger = logging.getLogger(__name__) -def save_validators_root(root: ValidatorsRoot) -> None: - """Cleans up previous validators root and saves the new one.""" - with db_client.get_db_connection() as conn: - with conn.cursor() as cur: - cur.execute(SQL('DELETE FROM {}').format(Identifier(VALIDATORS_ROOT_TABLE))) - cur.execute( - SQL( - 'INSERT INTO {} (root, ipfs_hash, block_number) VALUES (%s, %s, %s)' - ).format(Identifier(VALIDATORS_ROOT_TABLE)), - (root.root, root.ipfs_hash, root.block_number), - ) - - -def get_last_validators_root() -> ValidatorsRoot | None: - """Fetches the last vault validators root.""" - with db_client.get_db_connection() as conn: - with conn.cursor(row_factory=class_row(ValidatorsRoot)) as cur: - cur.execute( - SQL('SELECT * FROM {} ORDER BY block_number DESC LIMIT 1').format( - Identifier(VALIDATORS_ROOT_TABLE) - ), - ) - return cur.fetchone() - - -def save_deposit_data(deposit_data: list[DepositData]) -> None: - """Cleans up previous deposit data and saves the new one.""" - with db_client.get_db_connection() as conn: - with conn.cursor() as cur: - cur.execute(SQL('DELETE FROM {}').format(Identifier(DEPOSIT_DATA_TABLE))) - if not deposit_data: - return - - cur.executemany( - SQL( - 'INSERT INTO {} (validator_index, public_key, signature) VALUES (%s, %s, %s)' - ).format( - Identifier(DEPOSIT_DATA_TABLE), - ), - [ - ( - data.validator_index, - data.public_key, - data.signature - ) - for data in deposit_data - ], - ) - - -def get_deposit_data() -> list[DepositData]: - """Fetches all the deposit data.""" - with db_client.get_db_connection() as conn: - with conn.cursor(row_factory=class_row(DepositData)) as cur: - cur.execute( - SQL('SELECT * FROM {} ORDER BY validator_index ASC').format( - Identifier(DEPOSIT_DATA_TABLE), - ), - ) - return cur.fetchall() - - def save_network_validators(validators: list[NetworkValidator]) -> None: """Saves network validators.""" with db_client.get_db_connection() as conn: @@ -151,25 +87,3 @@ def setup() -> None: """ ).format(Identifier(NETWORK_VALIDATORS_TABLE)), ) - cur.execute( - SQL( - """ - CREATE TABLE IF NOT EXISTS {} ( - validator_index INTEGER UNIQUE NOT NULL, - public_key VARCHAR(98) UNIQUE NOT NULL, - signature VARCHAR(194) UNIQUE NOT NULL - ) - """ - ).format(Identifier(DEPOSIT_DATA_TABLE)), - ) - cur.execute( - SQL( - """ - CREATE TABLE IF NOT EXISTS {} ( - root VARCHAR(66) NOT NULL, - ipfs_hash VARCHAR(66) NOT NULL, - block_number INTEGER NOT NULL - ) - """ - ).format(Identifier(VALIDATORS_ROOT_TABLE)), - ) diff --git a/src/validators/execution.py b/src/validators/execution.py index 132d0ac0..6e5815e5 100644 --- a/src/validators/execution.py +++ b/src/validators/execution.py @@ -30,29 +30,23 @@ VAULT_CONTRACT_ADDRESS, ) from src.validators.database import ( - get_deposit_data, get_last_network_validator, - get_last_validators_root, is_validator_registered, - save_deposit_data, save_network_validators, - save_validators_root, ) -from src.validators.ipfs import fetch_vault_deposit_data from src.validators.typings import ( - BLSPrivkey, DepositData, KeeperApprovalParams, + Keystores, MultipleValidatorRegistration, NetworkValidator, Oracles, OraclesApproval, SingleValidatorRegistration, - ValidatorsRoot, + Validator, ) VALIDATORS_REGISTRY_GENESIS_BLOCK: BlockNumber = NETWORK_CONFIG.VALIDATORS_REGISTRY_GENESIS_BLOCK -VAULT_GENESIS_BLOCK: BlockNumber = NETWORK_CONFIG.VAULT_GENESIS_BLOCK GENESIS_FORK_VERSION: bytes = NETWORK_CONFIG.GENESIS_FORK_VERSION logger = logging.getLogger(__name__) @@ -76,37 +70,6 @@ async def process_events(events: list[EventData]) -> None: save_network_validators(validators) -class VaultValidatorsProcessor(EventProcessor): - contract = vault_contract - contract_event = 'ValidatorsRootUpdated' - - @staticmethod - async def get_from_block() -> BlockNumber: - last_root = get_last_validators_root() - if not last_root: - return VAULT_GENESIS_BLOCK - - return BlockNumber(last_root.block_number + 1) - - @staticmethod - async def process_events(events: list[EventData]) -> None: - if not events: - return - - last_event = events[-1] - new_root = ValidatorsRoot( - root=Web3.to_hex(last_event['args']['validatorsRoot']), - ipfs_hash=last_event['args']['validatorsIpfsHash'], - block_number=BlockNumber(last_event['blockNumber']) - ) - if not new_root.ipfs_hash: - raise ValueError('Invalid validators root IPFS hash') - - deposit_data = await fetch_vault_deposit_data(new_root.ipfs_hash) - save_deposit_data(deposit_data) - save_validators_root(new_root) - - def process_network_validator_events(events: list[EventData]) -> list[NetworkValidator]: """ Processes `ValidatorsRegistry` registration events @@ -182,54 +145,45 @@ async def get_vault_validators_root() -> Bytes32: @backoff.on_exception(backoff.expo, Exception, max_time=300) -async def get_available_deposit_data( - private_keys: dict[HexStr, BLSPrivkey], - validators_count: int -) -> tuple[list[DepositData], StandardMerkleTree | None]: - """Fetches vault's available deposit data.""" - if validators_count <= 0: - return [], None +async def get_vault_validators_index() -> int: + """Fetches vault's current validators index.""" + return await vault_contract.functions.validatorIndex().call() - credentials = get_eth1_withdrawal_credentials(VAULT_CONTRACT_ADDRESS) - current_index = await vault_contract.functions.validatorIndex().call() - deposit_data = get_deposit_data() - available_deposit_data = [] - current_count = 0 - leaves: list[tuple[bytes, int]] = [] - for data in deposit_data: - leaves.append((_encode_tx_validator(credentials, data), data.validator_index)) - if data.validator_index < current_index or validators_count <= current_count: - continue +async def check_deposit_data_root(deposit_data_root: str) -> None: + """Checks whether deposit data root matches validators root in Vault.""" + if deposit_data_root != Web3.to_hex(await get_vault_validators_root()): + raise RuntimeError("Deposit data tree root and vault's validators root don't match") + - if data.public_key not in private_keys: +async def get_available_validators( + keystores: Keystores, deposit_data: DepositData, count: int +) -> list[Validator]: + """Fetches vault's available validators.""" + await check_deposit_data_root(deposit_data.tree.root) + + start_index = await get_vault_validators_index() + validators: list[Validator] = [] + for i in range(start_index, start_index + count): + validator = deposit_data.validators[i] + if validator.public_key not in keystores: logger.warning( 'Cannot find validator with public key %s in imported keystores.', - data.public_key + validator.public_key, ) - continue + break - if is_validator_registered(data.public_key): + if is_validator_registered(validator.public_key): logger.warning( 'Validator with public key %s is already registered.' ' You must upload new deposit data.', - data.public_key + validator.public_key, ) - continue - - available_deposit_data.append(data) - current_count += 1 + break - if not available_deposit_data: - logger.warning( - 'Failed to find available validator. You must upload new deposit data.' - ) - - tree = StandardMerkleTree.of(leaves, ['bytes', 'uint256']) - if tree.root != Web3.to_hex(await get_vault_validators_root()): - raise RuntimeError("Reconstructed tree root and vault's validators root don't match") + validators.append(validator) - return available_deposit_data, tree + return validators @backoff.on_exception(backoff.expo, Exception, max_time=300) @@ -261,98 +215,98 @@ async def get_oracles() -> Oracles: threshold=threshold, rsa_public_keys=rsa_public_keys, endpoints=endpoints, - addresses=addresses + addresses=addresses, ) async def register_single_validator( - deposit_data_tree: StandardMerkleTree, - deposit_data: DepositData, - approval: OraclesApproval + tree: StandardMerkleTree, validator: Validator, approval: OraclesApproval ) -> None: """Registers single validator.""" if NETWORK not in ETH_NETWORKS: raise NotImplementedError('networks other than Ethereum not supported') credentials = get_eth1_withdrawal_credentials(VAULT_CONTRACT_ADDRESS) - validator = _encode_tx_validator(credentials, deposit_data) - proof = deposit_data_tree.get_proof([validator, deposit_data.validator_index]) # type: ignore + tx_validator = _encode_tx_validator(credentials, validator) + proof = tree.get_proof([tx_validator, validator.deposit_data_index]) # type: ignore tx_data = SingleValidatorRegistration( keeperParams=KeeperApprovalParams( validatorsRegistryRoot=approval.validators_registry_root, - validators=validator, + validators=tx_validator, signatures=approval.signatures, exitSignaturesIpfsHash=approval.ipfs_hash, ), - proof=proof + proof=proof, ) + logger.info('Submitting registration transaction') tx = await vault_contract.functions.registerValidator( - (tx_data.keeperParams.validatorsRegistryRoot, - tx_data.keeperParams.validators, - tx_data.keeperParams.signatures, - tx_data.keeperParams.exitSignaturesIpfsHash - ), - tx_data.proof + ( + tx_data.keeperParams.validatorsRegistryRoot, + tx_data.keeperParams.validators, + tx_data.keeperParams.signatures, + tx_data.keeperParams.exitSignaturesIpfsHash, + ), + tx_data.proof, ).transact() # type: ignore + logger.info('Waiting for transaction %s confirmation', Web3.to_hex(tx)) await execution_client.eth.wait_for_transaction_receipt(tx, timeout=300) # type: ignore async def register_multiple_validator( - deposit_data_tree: StandardMerkleTree, - deposit_data: list[DepositData], - approval: OraclesApproval + tree: StandardMerkleTree, + validators: list[Validator], + approval: OraclesApproval, ) -> None: """Registers multiple validators.""" if NETWORK not in ETH_NETWORKS: raise NotImplementedError('networks other than Ethereum not supported') credentials = get_eth1_withdrawal_credentials(VAULT_CONTRACT_ADDRESS) - deposit_data = sorted(deposit_data, key=lambda d: d.validator_index) - validators: list[bytes] = [] + tx_validators: list[bytes] = [] leaves: list[tuple[bytes, int]] = [] - for deposit in deposit_data: - validator = _encode_tx_validator(credentials, deposit) - validators.append(validator) - leaves.append((validator, deposit.validator_index)) - - multi_proof = deposit_data_tree.get_multi_proof(leaves) - sorted_validators = [v[0] for v in multi_proof.leaves] - indexes = [sorted_validators.index(v) for v in validators] + for validator in validators: + tx_validator = _encode_tx_validator(credentials, validator) + tx_validators.append(tx_validator) + leaves.append((tx_validator, validator.deposit_data_index)) + + multi_proof = tree.get_multi_proof(leaves) + sorted_tx_validators: list[bytes] = [v[0] for v in multi_proof.leaves] + indexes = [sorted_tx_validators.index(v) for v in tx_validators] tx_data = MultipleValidatorRegistration( keeperParams=KeeperApprovalParams( validatorsRegistryRoot=approval.validators_registry_root, - validators=b''.join(validators), + validators=b''.join(tx_validators), signatures=approval.signatures, exitSignaturesIpfsHash=approval.ipfs_hash, ), indexes=indexes, proofFlags=multi_proof.proof_flags, - proof=multi_proof.proof + proof=multi_proof.proof, ) + logger.info('Submitting registration transaction') tx = await vault_contract.functions.registerValidators( - (tx_data.keeperParams.validatorsRegistryRoot, - tx_data.keeperParams.validators, - tx_data.keeperParams.signatures, - tx_data.keeperParams.exitSignaturesIpfsHash - ), + ( + tx_data.keeperParams.validatorsRegistryRoot, + tx_data.keeperParams.validators, + tx_data.keeperParams.signatures, + tx_data.keeperParams.exitSignaturesIpfsHash, + ), indexes, multi_proof.proof_flags, - multi_proof.proof + multi_proof.proof, ).transact() # type: ignore + logger.info('Waiting for transaction %s confirmation', Web3.to_hex(tx)) await execution_client.eth.wait_for_transaction_receipt(tx, timeout=300) # type: ignore -def _encode_tx_validator( - withdrawal_credentials: bytes, - deposit_data: DepositData -) -> bytes: - public_key = Web3.to_bytes(hexstr=deposit_data.public_key) - signature = Web3.to_bytes(hexstr=deposit_data.signature) +def _encode_tx_validator(withdrawal_credentials: bytes, validator: Validator) -> bytes: + public_key = Web3.to_bytes(hexstr=validator.public_key) + signature = Web3.to_bytes(hexstr=validator.signature) deposit_root = compute_deposit_data( public_key=public_key, withdrawal_credentials=withdrawal_credentials, amount_gwei=DEPOSIT_AMOUNT_GWEI, - signature=signature + signature=signature, ).hash_tree_root return public_key + signature + deposit_root diff --git a/src/validators/ipfs.py b/src/validators/ipfs.py deleted file mode 100644 index e1a7ed9d..00000000 --- a/src/validators/ipfs.py +++ /dev/null @@ -1,29 +0,0 @@ -import backoff -from web3 import Web3 - -from src.common.clients import ipfs_fetch_client -from src.validators.typings import DepositData - -BLS_PUBLIC_KEY_LENGTH = 48 -BLS_SIGNATURE_LENGTH = 96 - - -@backoff.on_exception(backoff.expo, Exception, max_time=300) -async def fetch_vault_deposit_data(ipfs_hash: str) -> list[DepositData]: - """Fetches deposit data from the IPFS.""" - ipfs_data = await ipfs_fetch_client.fetch_bytes(ipfs_hash) - deposit_data_length = BLS_PUBLIC_KEY_LENGTH + BLS_SIGNATURE_LENGTH - - result = [] - validator_index = 0 - for i in range(0, len(ipfs_data), deposit_data_length): - public_key = ipfs_data[i: i + BLS_PUBLIC_KEY_LENGTH] - signature = ipfs_data[i + BLS_PUBLIC_KEY_LENGTH: i + deposit_data_length] - result.append(DepositData( - validator_index=validator_index, - public_key=Web3.to_hex(public_key), - signature=Web3.to_hex(signature) - )) - validator_index += 1 - - return result diff --git a/src/validators/tasks.py b/src/validators/tasks.py index 6135663f..e3db14a4 100644 --- a/src/validators/tasks.py +++ b/src/validators/tasks.py @@ -1,7 +1,6 @@ import logging import backoff -from eth_typing import HexStr from web3 import Web3 from web3.types import Wei @@ -23,7 +22,7 @@ save_network_validators, ) from src.validators.execution import ( - get_available_deposit_data, + get_available_validators, get_latest_network_validator_public_keys, get_oracles, get_validators_registry_root, @@ -34,10 +33,11 @@ from src.validators.signing import get_exit_signature_shards from src.validators.typings import ( ApprovalRequest, - BLSPrivkey, DepositData, + Keystores, NetworkValidator, OraclesApproval, + Validator, ) from src.validators.utils import send_approval_requests @@ -45,7 +45,7 @@ @backoff.on_exception(backoff.expo, Exception, max_time=300) -async def register_validators(private_keys: dict[HexStr, BLSPrivkey]) -> None: +async def register_validators(keystores: Keystores, deposit_data: DepositData) -> None: """Registers vault validators.""" vault_balance = await get_withdrawable_assets() if NETWORK == GNOSIS: @@ -53,34 +53,37 @@ async def register_validators(private_keys: dict[HexStr, BLSPrivkey]) -> None: vault_balance = Wei(int(vault_balance * MGNO_RATE // WAD)) # calculate number of validators that can be registered - validators_count: int = min( - APPROVAL_MAX_VALIDATORS, vault_balance // DEPOSIT_AMOUNT - ) + validators_count: int = min(APPROVAL_MAX_VALIDATORS, vault_balance // DEPOSIT_AMOUNT) if not validators_count: # not enough balance to register validators return - deposit_data, deposit_data_tree = await get_available_deposit_data( - private_keys=private_keys, - validators_count=validators_count + logger.info('Started registration of %d validators', validators_count) + + validators: list[Validator] = await get_available_validators( + keystores, deposit_data, validators_count ) - if not (deposit_data and deposit_data_tree): + if not validators: + logger.warning('Failed to find available validator. You must upload new deposit data.') return - oracles_approval = await get_oracles_approval(private_keys, deposit_data) - if len(deposit_data) == 1: - return await register_single_validator(deposit_data_tree, deposit_data[0], oracles_approval) + oracles_approval = await get_oracles_approval(keystores, validators) + if len(validators) == 1: + validator = validators[0] + await register_single_validator(deposit_data.tree, validator, oracles_approval) + logger.info('Successfully registered validator with public key %s', validator.public_key) - if len(deposit_data) > 1: - return await register_multiple_validator(deposit_data_tree, deposit_data, oracles_approval) + if len(validators) > 1: + await register_multiple_validator(deposit_data.tree, validators, oracles_approval) + pub_keys = ', '.join([val.public_key for val in validators]) + logger.info('Successfully registered validators with public keys %s', pub_keys) @backoff.on_exception(backoff.expo, Exception, max_tries=10) async def get_oracles_approval( - private_keys: dict[HexStr, BLSPrivkey], - deposit_data: list[DepositData] + keystores: Keystores, validators: list[Validator] ) -> OraclesApproval: - """Updates vote for the new rewards.""" + """Fetches approval from oracles.""" # get latest oracles oracles = await get_oracles() @@ -102,25 +105,30 @@ async def get_oracles_approval( public_keys=[], deposit_signatures=[], public_key_shards=[], - exit_signature_shards=[] + exit_signature_shards=[], ) - for deposit in deposit_data: + for validator in validators: shards = get_exit_signature_shards( validator_index=validator_index, - private_key=private_keys[deposit.public_key], + private_key=keystores[validator.public_key], oracles=oracles, - fork=fork + fork=fork, ) if not shards: break - request.public_keys.append(deposit.public_key) - request.deposit_signatures.append(deposit.signature) + request.public_keys.append(validator.public_key) + request.deposit_signatures.append(validator.signature) request.public_key_shards.append(shards.public_keys) request.exit_signature_shards.append(shards.exit_signatures) # send approval request to oracles signatures, ipfs_hash = await send_approval_requests(oracles, request) + logger.info( + 'Fetched oracles approval for validators: count=%d, start index=%d', + len(validators), + validator_index, + ) return OraclesApproval( signatures=signatures, ipfs_hash=ipfs_hash, @@ -142,7 +150,7 @@ async def load_genesis_validators() -> None: for i in range(0, len(pub_keys), 48): genesis_validators.append( NetworkValidator( - public_key=Web3.to_hex(pub_keys[i: i + 48]), + public_key=Web3.to_hex(pub_keys[i : i + 48]), block_number=NETWORK_CONFIG.VALIDATORS_REGISTRY_GENESIS_BLOCK, ) ) diff --git a/src/validators/typings.py b/src/validators/typings.py index 5820d709..a0d23bef 100644 --- a/src/validators/typings.py +++ b/src/validators/typings.py @@ -3,9 +3,11 @@ from Cryptodome.PublicKey import RSA from eth_typing import BlockNumber, ChecksumAddress, HexStr +from multiproof import StandardMerkleTree from sw_utils.typings import Bytes32 BLSPrivkey = NewType('BLSPrivkey', bytes) +Keystores = NewType('Keystores', dict[HexStr, BLSPrivkey]) @dataclass @@ -15,17 +17,16 @@ class NetworkValidator: @dataclass -class ValidatorsRoot: - root: HexStr - ipfs_hash: str - block_number: BlockNumber +class Validator: + deposit_data_index: int + public_key: HexStr + signature: HexStr @dataclass class DepositData: - validator_index: int - public_key: HexStr - signature: HexStr + validators: list[Validator] + tree: StandardMerkleTree @dataclass diff --git a/src/validators/utils.py b/src/validators/utils.py index b5ce55ee..b15678dd 100644 --- a/src/validators/utils.py +++ b/src/validators/utils.py @@ -1,4 +1,5 @@ import dataclasses +import json import logging import random from multiprocessing import Pool @@ -9,12 +10,29 @@ import backoff import milagro_bls_binding as bls from eth_typing import ChecksumAddress, HexStr +from eth_utils import add_0x_prefix +from multiproof import StandardMerkleTree from staking_deposit.key_handling.keystore import ScryptKeystore +from sw_utils import get_eth1_withdrawal_credentials from tqdm import tqdm from web3 import Web3 -from src.config.settings import KEYSTORES_PASSWORD_PATH, KEYSTORES_PATH -from src.validators.typings import ApprovalRequest, BLSPrivkey, OracleApproval, Oracles +from src.config.settings import ( + DEPOSIT_DATA_PATH, + KEYSTORES_PASSWORD_PATH, + KEYSTORES_PATH, + VAULT_CONTRACT_ADDRESS, +) +from src.validators.execution import _encode_tx_validator, check_deposit_data_root +from src.validators.typings import ( + ApprovalRequest, + BLSPrivkey, + DepositData, + Keystores, + OracleApproval, + Oracles, + Validator, +) logger = logging.getLogger(__name__) @@ -65,7 +83,7 @@ async def send_approval_request( ) -def load_private_keys() -> dict[HexStr, BLSPrivkey]: +def load_keystores() -> Keystores: """Extracts private keys from the keystores.""" keystores_password = _load_keystores_password() @@ -96,10 +114,34 @@ def _stop_pool(*args, **kwargs): break existing_keys: list[tuple[HexStr, BLSPrivkey]] = [key for key in keys if key] - private_keys = dict(existing_keys) - - logger.info('Loaded %d keystores', len(private_keys)) - return private_keys + keystores = Keystores(dict(existing_keys)) + + logger.info('Loaded %d keystores', len(keystores)) + return keystores + + +async def load_deposit_data() -> DepositData: + """Loads and verifies deposit data.""" + with open(DEPOSIT_DATA_PATH, 'r', encoding='utf-8') as f: + deposit_data = json.load(f) + + credentials = get_eth1_withdrawal_credentials(VAULT_CONTRACT_ADDRESS) + leaves: list[tuple[bytes, int]] = [] + validators: list[Validator] = [] + for i, data in enumerate(deposit_data): + validator = Validator( + deposit_data_index=i, + public_key=add_0x_prefix(data['pubkey']), + signature=add_0x_prefix(data['signature']), + ) + leaves.append((_encode_tx_validator(credentials, validator), i)) + validators.append(validator) + + tree = StandardMerkleTree.of(leaves, ['bytes', 'uint256']) + await check_deposit_data_root(tree.root) + + logger.info('Loaded deposit data file %s', DEPOSIT_DATA_PATH) + return DepositData(validators=validators, tree=tree) def _process_keystore_file(