Web3py7

pyproject.toml

@@ -23,19 +23,20 @@ timeout-decorator = "^0.5.0"
 pytest = "^7.2.1"
 pytest-xdist = "^3.2.1"
 more-itertools = "^10.1.0"
-web3 = "^6.10.0"
-web3-multi-provider = "^0.6.0"
+web3 = "^7.8.0"
+web3-multi-provider = "^2.0.0"
 json-stream = "^2.3.2"
 lazy-object-proxy = "^1.9.0"
 oz-merkle-tree = {git = "https://github.com/lidofinance/oz-merkle-tree"}
 py-multiformats-cid = "^0.4.4"
+pydantic = "^2.4.0"
+polyfactory = "^2.19.0"
 
 [tool.poetry.group.dev.dependencies]
 base58 = "^2.1.1"
 ipfshttpclient = "^0.7.0"
-pydantic = "1.10.6"
+pydantic = "^2.4.0"
 pytest-cov = "^4.0.0"
-pydantic-factories = "^1.17.2"
 # {{{ stubs for mypy
 types-requests = "^2.28.11.15"
 types-setuptools = "^67.6.0.0"

src/main.py

@@ -3,20 +3,20 @@
 
 from packaging.version import Version
 from prometheus_client import start_http_server
-from web3.middleware import simple_cache_middleware
 
 from src import variables
 from src.metrics.healthcheck_server import start_pulse_server
 from src.metrics.logging import logging
 from src.metrics.prometheus.basic import ENV_VARIABLES_INFO, BUILD_INFO
 from src.modules.accounting.accounting import Accounting
-from src.modules.ejector.ejector import Ejector
 from src.modules.checks.checks_module import ChecksModule
 from src.modules.csm.csm import CSOracle
+from src.modules.ejector.ejector import Ejector
 from src.providers.ipfs import GW3, IPFSProvider, MultiIPFSProvider, Pinata, PublicIPFS
 from src.types import OracleModule
 from src.utils.build import get_build_info
 from src.utils.exception import IncompatibleException
+from src.web3py.contract_tweak import tweak_w3_contracts
 from src.web3py.extensions import (
     LidoContracts,
     TransactionUtils,
@@ -29,8 +29,6 @@
 from src.web3py.middleware import metrics_collector
 from src.web3py.types import Web3
 
-from src.web3py.contract_tweak import tweak_w3_contracts
-
 logger = logging.getLogger(__name__)
 
 
@@ -56,7 +54,8 @@ def main(module_name: OracleModule):
     logger.info({'msg': 'Initialize multi web3 provider.'})
     web3 = Web3(FallbackProviderModule(
         variables.EXECUTION_CLIENT_URI,
-        request_kwargs={'timeout': variables.HTTP_REQUEST_TIMEOUT_EXECUTION}
+        request_kwargs={'timeout': variables.HTTP_REQUEST_TIMEOUT_EXECUTION},
+        cache_allowed_requests=True,
     ))
 
     logger.info({'msg': 'Modify web3 with custom contract function call.'})
@@ -92,7 +91,6 @@ def main(module_name: OracleModule):
 
     logger.info({'msg': 'Add metrics middleware for ETH1 requests.'})
     web3.middleware_onion.add(metrics_collector)
-    web3.middleware_onion.add(simple_cache_middleware)
 
     logger.info({'msg': 'Sanity checks.'})
 

src/modules/ejector/ejector.py

@@ -123,6 +123,7 @@ def get_validators_to_eject(self, blockstamp: ReferenceBlockStamp) -> list[tuple
         chain_config = self.get_chain_config(blockstamp)
         validators_iterator = iter(ValidatorExitIterator(
             w3=self.w3,
+            consensus_version=self.get_consensus_version(blockstamp),
             blockstamp=blockstamp,
             seconds_per_slot=chain_config.seconds_per_slot
         ))

src/modules/submodules/consensus.py

@@ -9,9 +9,6 @@
 
 from src import variables
 from src.metrics.prometheus.basic import ORACLE_SLOT_NUMBER, ORACLE_BLOCK_NUMBER, GENESIS_TIME, ACCOUNT_BALANCE
-from src.providers.execution.contracts.base_oracle import BaseOracleContract
-from src.providers.execution.contracts.hash_consensus import HashConsensusContract
-from src.types import BlockStamp, ReferenceBlockStamp, SlotNumber, FrameNumber
 from src.metrics.prometheus.business import (
     ORACLE_MEMBER_LAST_REPORT_REF_SLOT,
     FRAME_CURRENT_REF_SLOT,
@@ -20,16 +17,19 @@
 )
 from src.modules.submodules.exceptions import IsNotMemberException, IncompatibleOracleVersion, ContractVersionMismatch
 from src.modules.submodules.types import ChainConfig, MemberInfo, ZERO_HASH, CurrentFrame, FrameConfig
+from src.providers.execution.contracts.base_oracle import BaseOracleContract
+from src.providers.execution.contracts.hash_consensus import HashConsensusContract
+from src.types import BlockStamp, ReferenceBlockStamp, SlotNumber, FrameNumber
 from src.utils.blockstamp import build_blockstamp
-from src.utils.web3converter import Web3Converter
-from src.utils.slot import get_reference_blockstamp
 from src.utils.cache import global_lru_cache as lru_cache
+from src.utils.slot import get_reference_blockstamp
+from src.utils.web3converter import Web3Converter
 from src.web3py.types import Web3
 
 logger = logging.getLogger(__name__)
 
 # Initial epoch is in the future. Revert signature: '0xcd0883ea'
-InitialEpochIsYetToArriveRevert = Web3.keccak(text="InitialEpochIsYetToArrive()")[:4].hex()
+InitialEpochIsYetToArriveRevert = '0x' + Web3.keccak(text="InitialEpochIsYetToArrive()")[:4].hex()
 
 
 class ConsensusModule(ABC):

src/providers/execution/contracts/lido.py

@@ -1,7 +1,7 @@
 import logging
 
 from eth_typing import ChecksumAddress, HexStr
-from web3.types import Wei, BlockIdentifier, CallOverrideParams
+from web3.types import Wei, BlockIdentifier, StateOverrideParams
 
 from src.modules.accounting.types import LidoReportRebase, BeaconStat
 from src.providers.execution.base_interface import ContractInterface
@@ -86,7 +86,7 @@ def _handle_oracle_report(
         ref_slot: HexStr,
         block_identifier: BlockIdentifier = 'latest',
     ) -> LidoReportRebase:
-        state_override: dict[ChecksumAddress, CallOverrideParams] = {
+        state_override: dict[ChecksumAddress, StateOverrideParams] = {
             accounting_oracle_address: {
                 # Fix: insufficient funds for gas * price + value
                 'balance': Wei(100 * 10**18),

src/services/exit_order_iterator.py

@@ -4,15 +4,15 @@
 
 from more_itertools import ilen
 
-from src.constants import TOTAL_BASIS_POINTS
+from src.constants import TOTAL_BASIS_POINTS, LIDO_DEPOSIT_AMOUNT
 from src.metrics.prometheus.duration_meter import duration_meter
+from src.providers.consensus.types import Validator
 from src.services.validator_state import LidoValidatorStateService
-from src.types import ReferenceBlockStamp, NodeOperatorGlobalIndex, StakingModuleId
+from src.types import ReferenceBlockStamp, NodeOperatorGlobalIndex, StakingModuleId, Gwei
 from src.utils.validator_state import is_on_exit, get_validator_age
 from src.web3py.extensions.lido_validators import LidoValidator, StakingModule, NodeOperator, NodeOperatorLimitMode
 from src.web3py.types import Web3
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -28,6 +28,7 @@ class NodeOperatorStats:
     module_stats: StakingModuleStats
 
     predictable_validators: int = 0
+    predictable_effective_balance: Gwei = Gwei(0)
     delayed_validators: int = 0
     total_age: int = 0
     force_exit_to: int | None = None
@@ -60,10 +61,19 @@ class ValidatorExitIterator:
 
     max_validators_to_exit: int = 0
     no_penetration_threshold: float = 0
-    eth_validators_count: int = 0
 
-    def __init__(self, w3: Web3, blockstamp: ReferenceBlockStamp, seconds_per_slot: int):
+    eth_validators_count: int = 0
+    eth_validators_effective_balance: Gwei = Gwei(0)
+
+    def __init__(
+        self,
+        w3: Web3,
+        consensus_version: int,
+        blockstamp: ReferenceBlockStamp,
+        seconds_per_slot: int,
+    ):
         self.w3 = w3
+        self.consensus_version = consensus_version
         self.blockstamp = blockstamp
         self.seconds_per_slot = seconds_per_slot
 
@@ -138,6 +148,9 @@ def _calculate_lido_stats(self):
             self.total_lido_validators += no_predictable_validators
             self.module_stats[gid[0]].predictable_validators += no_predictable_validators
             self.node_operators_stats[gid].predictable_validators = no_predictable_validators
+            self.node_operators_stats[gid].predictable_effective_balance = (
+                self._calculate_effective_balance_non_exiting_validators(validators) + transient_validators_count * LIDO_DEPOSIT_AMOUNT
+            )
 
             self.node_operators_stats[gid].delayed_validators = delayed_validators[gid]
             self.node_operators_stats[gid].total_age = self.calculate_validators_age(validators)
@@ -159,6 +172,18 @@ def _load_blockchain_state(self):
 
         self.eth_validators_count = ilen(v for v in self.w3.cc.get_validators(self.blockstamp) if not is_on_exit(v))
 
+        self.eth_validators_effective_balance = self._calculate_effective_balance_non_exiting_validators(self.w3.cc.get_validators(self.blockstamp))
+
+    @staticmethod
+    def _calculate_effective_balance_non_exiting_validators(validators: list[Validator]) -> Gwei:
+        return sum(
+            (
+                v.validator.effective_balance for v in validators
+                if not is_on_exit(v)
+            ),
+            Gwei(0),
+        )
+
     def get_filter_non_exitable_validators(self, gid: NodeOperatorGlobalIndex):
         """Validators that are presented but not yet activated on CL can be requested to exit in advance."""
         indexes = self.lvs.get_operators_with_last_exited_validator_indexes(self.blockstamp)
@@ -199,29 +224,32 @@ def calculate_validators_age(self, validators: list[LidoValidator]) -> int:
         return result
 
     def _eject_validator(self, gid: NodeOperatorGlobalIndex) -> LidoValidator:
-        validator = self.exitable_validators[gid].pop(0)
+        lido_validator = self.exitable_validators[gid].pop(0)
 
         # Total validators
         self.eth_validators_count -= 1
+        self.eth_validators_effective_balance -= lido_validator.validator.effective_balance  # type: ignore
         # Change lido total
         self.total_lido_validators -= 1
         # Change module total
         self.module_stats[gid[0]].predictable_validators -= 1
         # Change node operator stats
         self.node_operators_stats[gid].predictable_validators -= 1
-        self.node_operators_stats[gid].total_age -= get_validator_age(validator, self.blockstamp.ref_epoch)
+        self.node_operators_stats[gid].predictable_effective_balance -= lido_validator.validator.effective_balance  # type: ignore
+        self.node_operators_stats[gid].total_age -= get_validator_age(lido_validator, self.blockstamp.ref_epoch)
 
         logger.debug({
             'msg': 'Iterator state change. Eject validator.',
             'eth_validators_count': self.eth_validators_count,
+            'eth_validators_effective_balance': self.eth_validators_effective_balance,
             'total_lido_validators': self.total_lido_validators,
             'no_gid': gid[0],
             'module_stats': self.module_stats[gid[0]].predictable_validators,
             'no_stats_exitable_validators': self.node_operators_stats[gid].predictable_validators,
             'no_stats_total_age': self.node_operators_stats[gid].total_age,
         })
 
-        return validator
+        return lido_validator
 
     def _no_predicate(self, node_operator: NodeOperatorStats) -> tuple:
         return (
@@ -232,7 +260,9 @@ def _no_predicate(self, node_operator: NodeOperatorStats) -> tuple:
             - self._stake_weight_coefficient_predicate(
                 node_operator,
                 self.eth_validators_count,
+                self.eth_validators_effective_balance,
                 self.no_penetration_threshold,
+                self.consensus_version > 2 and self.w3.cc.is_electra_activated(self.blockstamp.ref_epoch),
             ),
             - node_operator.predictable_validators,
             self._lowest_validator_index_predicate(node_operator),
@@ -272,13 +302,19 @@ def _max_share_rate_coefficient_predicate(self, node_operator: NodeOperatorStats
     def _stake_weight_coefficient_predicate(
         node_operator: NodeOperatorStats,
         total_validators: int,
+        total_effective_balance: Gwei,
         no_penetration: float,
+        is_post_pectra: bool,
     ) -> int:
         """
         The higher coefficient the higher priority to eject validator
         """
-        if total_validators * no_penetration < node_operator.predictable_validators:
-            return node_operator.total_age
+        if is_post_pectra:
+            if total_effective_balance * no_penetration < node_operator.predictable_effective_balance:
+                return node_operator.total_age
+        else:
+            if total_validators * no_penetration < node_operator.predictable_validators:
+                return node_operator.total_age
 
         return 0
 

src/web3py/contract_tweak.py

@@ -2,27 +2,31 @@
 from typing import Any, Callable, Tuple
 
 from eth_abi.exceptions import DecodingError
+from eth_typing import (
+    ABI,
+    ABIFunction,
+)
 from eth_typing import ChecksumAddress
+from eth_utils.abi import (
+    get_abi_output_types,
+)
 from web3 import Web3
 from web3._utils.abi import (
-    get_abi_output_types,
     map_abi_data,
     named_tree,
     recursive_dict_to_namedtuple,
 )
-from web3._utils.contracts import find_matching_fn_abi, prepare_transaction
+from web3._utils.contracts import prepare_transaction
 from web3._utils.normalizers import BASE_RETURN_NORMALIZERS
 from web3.contract import Contract as _Contract
 from web3.contract.contract import ContractFunction as _ContractFunction
 from web3.contract.contract import ContractFunctions as _ContractFunctions
 from web3.contract.utils import ACCEPTABLE_EMPTY_STRINGS
 from web3.exceptions import BadFunctionCallOutput
 from web3.types import (
-    ABI,
-    ABIFunction,
     BlockIdentifier,
-    CallOverride,
-    FunctionIdentifier,
+    StateOverride,
+    ABIElementIdentifier,
     TxParams,
 )
 
@@ -31,12 +35,12 @@ def call_contract_function(  # pylint: disable=keyword-arg-before-vararg
     w3: "Web3",
     address: ChecksumAddress,
     normalizers: Tuple[Callable[..., Any], ...],
-    function_identifier: FunctionIdentifier,
+    function_identifier: ABIElementIdentifier,
     transaction: TxParams,
     block_id: BlockIdentifier | None = None,
     contract_abi: ABI | None = None,
     fn_abi: ABIFunction | None = None,
-    state_override: CallOverride | None = None,
+    state_override: StateOverride | None = None,
     ccip_read_enabled: bool | None = None,
     decode_tuples: bool | None = False,
     *args: Any,
@@ -49,9 +53,9 @@ def call_contract_function(  # pylint: disable=keyword-arg-before-vararg
     call_transaction = prepare_transaction(
         address,
         w3,
-        fn_identifier=function_identifier,
+        abi_element_identifier=function_identifier,
         contract_abi=contract_abi,
-        fn_abi=fn_abi,
+        abi_callable=fn_abi,
         transaction=transaction,
         fn_args=args,
         fn_kwargs=kwargs,
@@ -65,7 +69,7 @@ def call_contract_function(  # pylint: disable=keyword-arg-before-vararg
     )
 
     if fn_abi is None:
-        fn_abi = find_matching_fn_abi(
+        fn_abi = Contract._find_matching_fn_abi(
             contract_abi, w3.codec, function_identifier, args, kwargs
         )
 
@@ -114,7 +118,7 @@ def call(
         self,
         transaction: TxParams | None = None,
         block_identifier: BlockIdentifier = "latest",
-        state_override: CallOverride | None = None,
+        state_override: StateOverride | None = None,
         ccip_read_enabled: bool | None = None,
     ) -> Any:
         call_transaction = self._get_call_txparams(transaction)
@@ -123,7 +127,7 @@ def call(
             self.w3,
             self.address,
             self._return_data_normalizers,
-            self.function_identifier,
+            self.abi_element_identifier,
             call_transaction,
             block_identifier,
             self.contract_abi,

tests/conftest.py

@@ -6,7 +6,6 @@
 from _pytest.fixtures import FixtureRequest
 from eth_typing import ChecksumAddress
 from hexbytes import HexBytes
-from web3.middleware import construct_simple_cache_middleware
 from web3.types import Timestamp
 
 import src.variables
@@ -50,7 +49,7 @@ def update_responses_provider(responses_path) -> UpdateResponsesProvider:
 
 @pytest.fixture()
 def mock_provider(responses_path) -> ResponseFromFile:
-    provider = ResponseFromFile(responses_path)
+    provider = ResponseFromFile(responses_path, cache_allowed_requests=True)
     return provider
 
 
@@ -66,7 +65,6 @@ def provider(request, responses_path) -> UpdateResponsesProvider | ResponseFromF
 def web3(provider) -> Web3:
     web3 = Web3(provider)
     tweak_w3_contracts(web3)
-    web3.middleware_onion.add(construct_simple_cache_middleware())
 
     with provider.use_mock(Path('common/chainId.json')):
         _ = web3.eth.chain_id