From f6438434785b41d630211f93b7d5b68a5fb49f83 Mon Sep 17 00:00:00 2001 From: Aaro Altonen Date: Sun, 3 Sep 2023 15:06:20 +0300 Subject: [PATCH] Rework the event system of `sc-network` This commit introduces a new concept called `NotificationService` which allows Polkadot protocols to communicate with the underlying notification protocol implementation directly, without routing events through `NetworkWorker`. This implies that each protocol has its own service which it uses to communicate with remote peers and that each `NotificationService` is unique with respect to the underlying notification protocol, meaning `NotificationService` for the transaction protocol can only be used to send and receive transaction-related notifications. The `NotificationService` concept introduces two additional benefits: * allow protocols to start using custom handshakes * allow protocols to accept/reject inbound peers Previously the validation of inbound connections was solely the responsibility of `ProtocolController`. This caused issues with light peers and `SyncingEngine` as `ProtocolController` would accept more peers than `SyncingEngine` could accept which caused peers to have differing views of their own states. `SyncingEngine` would reject excess peers but these rejections were not properly communicated to those peers causing them to assume that they were accepted. With `NotificationService`, the local handshake is not sent to remote peer if peer is rejected which allows it to detect that it was rejected. This commit also deprecates the use of `NetworkEventStream` for all notification-related events and going forward only DHT events are provided through `NetworkEventStream`. If protocols wish to follow each other's events, they must introduce additional abtractions, as is done for GRANDPA and transactions protocols by following the syncing protocol through `SyncEventStream`. Fixes https://github.com/paritytech/polkadot-sdk/issues/512 Fixes https://github.com/paritytech/polkadot-sdk/issues/514 Fixes https://github.com/paritytech/polkadot-sdk/issues/515 Fixes https://github.com/paritytech/polkadot-sdk/issues/554 Fixes https://github.com/paritytech/polkadot-sdk/issues/556 Co-authored-by: Dmitry Markin --- Cargo.lock | 220 ++-- .../relay-chain-minimal-node/Cargo.toml | 1 + .../src/collator_overseer.rs | 15 +- .../relay-chain-minimal-node/src/lib.rs | 14 +- .../relay-chain-minimal-node/src/network.rs | 25 +- polkadot/node/network/bridge/src/lib.rs | 1 + polkadot/node/network/bridge/src/network.rs | 60 +- polkadot/node/network/bridge/src/rx/mod.rs | 943 ++++++++++-------- polkadot/node/network/bridge/src/rx/tests.rs | 334 +++++-- polkadot/node/network/bridge/src/tx/mod.rs | 60 +- polkadot/node/network/bridge/src/tx/tests.rs | 142 ++- .../network/bridge/src/validator_discovery.rs | 13 +- .../node/network/protocol/src/peer_set.rs | 77 +- polkadot/node/service/Cargo.toml | 1 + polkadot/node/service/src/lib.rs | 54 +- polkadot/node/service/src/overseer.rs | 14 +- .../bin/node-template/node/src/service.rs | 7 +- substrate/bin/node/cli/src/service.rs | 28 +- .../consensus/beefy/src/communication/mod.rs | 12 +- substrate/client/consensus/beefy/src/lib.rs | 6 +- substrate/client/consensus/beefy/src/tests.rs | 41 +- .../client/consensus/beefy/src/worker.rs | 4 + .../grandpa/src/communication/mod.rs | 4 +- .../grandpa/src/communication/tests.rs | 227 +++-- substrate/client/consensus/grandpa/src/lib.rs | 22 +- .../client/consensus/grandpa/src/observer.rs | 23 +- .../client/consensus/grandpa/src/tests.rs | 82 +- substrate/client/network-gossip/Cargo.toml | 2 + substrate/client/network-gossip/src/bridge.rs | 236 +++-- .../network-gossip/src/state_machine.rs | 4 + substrate/client/network/Cargo.toml | 2 + substrate/client/network/common/src/role.rs | 14 +- substrate/client/network/src/config.rs | 90 +- substrate/client/network/src/error.rs | 9 + substrate/client/network/src/lib.rs | 17 +- substrate/client/network/src/mock.rs | 9 + substrate/client/network/src/peer_store.rs | 41 +- substrate/client/network/src/protocol.rs | 199 ++-- .../client/network/src/protocol/message.rs | 1 + .../network/src/protocol/notifications.rs | 2 + .../src/protocol/notifications/behaviour.rs | 586 ++++++++--- .../src/protocol/notifications/handler.rs | 102 +- .../protocol/notifications/service/metrics.rs | 124 +++ .../src/protocol/notifications/service/mod.rs | 576 +++++++++++ .../protocol/notifications/service/tests.rs | 770 ++++++++++++++ .../src/protocol/notifications/tests.rs | 29 +- .../client/network/src/protocol_controller.rs | 3 + substrate/client/network/src/service.rs | 67 +- .../client/network/src/service/signature.rs | 2 + .../client/network/src/service/traits.rs | 198 +++- substrate/client/network/statement/src/lib.rs | 118 +-- substrate/client/network/sync/src/engine.rs | 727 +++++++++++--- substrate/client/network/sync/src/lib.rs | 49 +- .../client/network/sync/src/service/mock.rs | 2 + substrate/client/network/test/src/lib.rs | 97 +- substrate/client/network/test/src/service.rs | 337 +++---- substrate/client/network/test/src/sync.rs | 86 +- .../client/network/transactions/src/lib.rs | 151 ++- substrate/client/offchain/src/api.rs | 6 +- substrate/client/offchain/src/lib.rs | 8 +- substrate/client/service/src/builder.rs | 26 +- 61 files changed, 5133 insertions(+), 1987 deletions(-) create mode 100644 substrate/client/network/src/protocol/notifications/service/metrics.rs create mode 100644 substrate/client/network/src/protocol/notifications/service/mod.rs create mode 100644 substrate/client/network/src/protocol/notifications/service/tests.rs diff --git a/Cargo.lock b/Cargo.lock index f3bbc2faf296..a1d226de0cfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" dependencies = [ "memchr", ] @@ -617,7 +617,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.27", + "time 0.3.28", ] [[package]] @@ -633,7 +633,7 @@ dependencies = [ "num-traits", "rusticata-macros", "thiserror", - "time 0.3.27", + "time 0.3.28", ] [[package]] @@ -1149,7 +1149,7 @@ checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", - "pin-project-lite 0.2.12", + "pin-project-lite 0.2.13", ] [[package]] @@ -1190,7 +1190,7 @@ dependencies = [ "futures-sink", "futures-util", "memchr", - "pin-project-lite 0.2.12", + "pin-project-lite 0.2.13", ] [[package]] @@ -1285,9 +1285,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.2" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "base64ct" @@ -2139,12 +2139,12 @@ dependencies = [ [[package]] name = "bstr" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" +checksum = "8042c26c77e5bd6897a7358e0abb3ec412ed126d826988135653fc669263899d" dependencies = [ "memchr", - "regex-automata 0.3.6", + "regex-automata 0.3.7", "serde", ] @@ -3795,6 +3795,7 @@ dependencies = [ "cumulus-relay-chain-interface", "cumulus-relay-chain-rpc-interface", "futures", + "parking_lot 0.12.1", "polkadot-availability-recovery", "polkadot-collator-protocol", "polkadot-core-primitives", @@ -4085,9 +4086,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28403c86fc49e3401fdf45499ba37fad6493d9329449d6449d7f0e10f4654d28" +checksum = "bbe98ba1789d56fb3db3bee5e032774d4f421b685de7ba703643584ba24effbe" dependencies = [ "cc", "cxxbridge-flags", @@ -4097,9 +4098,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78da94fef01786dc3e0c76eafcd187abcaa9972c78e05ff4041e24fdf059c285" +checksum = "c4ce20f6b8433da4841b1dadfb9468709868022d829d5ca1f2ffbda928455ea3" dependencies = [ "cc", "codespan-reporting", @@ -4112,15 +4113,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2a6f5e1dfb4b34292ad4ea1facbfdaa1824705b231610087b00b17008641809" +checksum = "20888d9e1d2298e2ff473cee30efe7d5036e437857ab68bbfea84c74dba91da2" [[package]] name = "cxxbridge-macro" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c49547d73ba8dcfd4ad7325d64c6d5391ff4224d498fc39a6f3f49825a530d" +checksum = "2fa16a70dd58129e4dfffdff535fb1bce66673f7bbeec4a5a1765a504e1ccd84" dependencies = [ "proc-macro2", "quote", @@ -4164,9 +4165,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.5.1" +version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd72493923899c6f10c641bdbdeddc7183d6396641d99c1a0d1597f37f92e28" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", "hashbrown 0.14.0", @@ -4782,9 +4783,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "erased-serde" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "837c0466252947ada828b975e12daf82e18bb5444e4df87be6038d4469e2a3d2" +checksum = "6c138974f9d5e7fe373eb04df7cae98833802ae4b11c24ac7039a21d5af4b26c" dependencies = [ "serde", ] @@ -4801,9 +4802,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", @@ -5565,7 +5566,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eeb4ed9e12f43b7fa0baae3f9cdda28352770132ef2e09a23760c29cae8bd47" dependencies = [ - "rustix 0.38.8", + "rustix 0.38.10", "windows-sys 0.48.0", ] @@ -5641,7 +5642,7 @@ dependencies = [ "futures-io", "memchr", "parking", - "pin-project-lite 0.2.12", + "pin-project-lite 0.2.13", "waker-fn", ] @@ -5663,7 +5664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2411eed028cdf8c8034eaf21f9915f956b6c3abec4d4c7949ee67f0721127bd" dependencies = [ "futures-io", - "rustls 0.20.8", + "rustls 0.20.9", "webpki 0.22.0", ] @@ -5698,7 +5699,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project-lite 0.2.12", + "pin-project-lite 0.2.13", "pin-utils", "slab", ] @@ -6107,7 +6108,7 @@ checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", - "pin-project-lite 0.2.12", + "pin-project-lite 0.2.13", ] [[package]] @@ -6150,7 +6151,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project-lite 0.2.12", + "pin-project-lite 0.2.13", "socket2 0.4.9", "tokio", "tower-service", @@ -6168,7 +6169,7 @@ dependencies = [ "http", "hyper", "log", - "rustls 0.21.6", + "rustls 0.21.7", "rustls-native-certs", "tokio", "tokio-rustls", @@ -6513,7 +6514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.2", - "rustix 0.38.8", + "rustix 0.38.10", "windows-sys 0.48.0", ] @@ -7261,7 +7262,7 @@ dependencies = [ "parking_lot 0.12.1", "quinn-proto", "rand 0.8.5", - "rustls 0.20.8", + "rustls 0.20.9", "thiserror", "tokio", ] @@ -7342,7 +7343,7 @@ dependencies = [ "libp2p-identity", "rcgen 0.10.0", "ring 0.16.20", - "rustls 0.20.8", + "rustls 0.20.9", "thiserror", "webpki 0.22.0", "x509-parser 0.14.0", @@ -7526,9 +7527,9 @@ dependencies = [ [[package]] name = "linregress" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de0b5f52a9f84544d268f5fabb71b38962d6aa3c6600b8bcd27d44ccf9c9c45" +checksum = "4de04dcecc58d366391f9920245b85ffa684558a5ef6e7736e754347c3aea9c2" dependencies = [ "nalgebra", ] @@ -7756,9 +7757,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f478948fd84d9f8e86967bf432640e46adfb5a4bd4f14ef7e864ab38220534ae" [[package]] name = "memfd" @@ -8169,16 +8170,15 @@ dependencies = [ [[package]] name = "nix" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" dependencies = [ "bitflags 1.3.2", "cfg-if", "libc", "memoffset 0.7.1", "pin-utils", - "static_assertions", ] [[package]] @@ -8239,7 +8239,7 @@ dependencies = [ "jsonrpsee", "kitchensink-runtime", "log", - "nix 0.26.2", + "nix 0.26.4", "node-executor", "node-inspect", "node-primitives", @@ -11055,9 +11055,9 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "3.6.4" +version = "3.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8e946cc0cc711189c0b0249fb8b599cbeeab9784d83c415719368bb8d4ac64" +checksum = "0dec8a8073036902368c2cdc0387e85ff9a37054d7e7c98e592145e0c92cd4fb" dependencies = [ "arrayvec 0.7.4", "bitvec", @@ -11070,9 +11070,9 @@ dependencies = [ [[package]] name = "parity-scale-codec-derive" -version = "3.6.4" +version = "3.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a296c3079b5fefbc499e1de58dc26c09b1b9a5952d26694ee89f04a43ebbb3e" +checksum = "312270ee71e1cd70289dacf597cab7b207aa107d2f28191c2ae45b2ece18a260" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -11385,9 +11385,9 @@ checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" [[package]] name = "pin-project-lite" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -11423,9 +11423,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "platforms" -version = "3.0.2" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d7ddaed09e0eb771a79ab0fd64609ba0afb0a8366421957936ad14cbd13630" +checksum = "4503fa043bf02cee09a9582e9554b4c6403b2ef55e4612e96561d294419429f8" [[package]] name = "plotters" @@ -11461,7 +11461,7 @@ version = "1.0.0" dependencies = [ "assert_cmd", "color-eyre", - "nix 0.26.2", + "nix 0.26.4", "polkadot-cli", "polkadot-core-primitives", "polkadot-node-core-pvf", @@ -12429,7 +12429,7 @@ dependencies = [ "hex-literal 0.4.1", "jsonrpsee", "log", - "nix 0.26.2", + "nix 0.26.4", "pallet-transaction-payment-rpc", "parachains-common", "parity-scale-codec", @@ -12841,6 +12841,7 @@ dependencies = [ "pallet-transaction-payment-rpc-runtime-api", "parity-db", "parity-scale-codec", + "parking_lot 0.12.1", "polkadot-approval-distribution", "polkadot-availability-bitfield-distribution", "polkadot-availability-distribution", @@ -13187,7 +13188,7 @@ dependencies = [ "concurrent-queue", "libc", "log", - "pin-project-lite 0.2.12", + "pin-project-lite 0.2.13", "windows-sys 0.48.0", ] @@ -13239,9 +13240,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f32154ba0af3a075eefa1eda8bb414ee928f62303a54ea85b8d6638ff1a6ee9e" +checksum = "31114a898e107c51bb1609ffaf55a0e011cf6a4d7f1170d0015a165082c0338b" [[package]] name = "portpicker" @@ -13263,7 +13264,7 @@ dependencies = [ "findshlibs", "libc", "log", - "nix 0.26.2", + "nix 0.26.4", "once_cell", "parking_lot 0.12.1", "smallvec", @@ -13423,9 +13424,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro-warning" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70550716265d1ec349c41f70dd4f964b4fd88394efe4405f0c1da679c4799a07" +checksum = "3d1eaa7fa0aa1929ffdf7eeb6eac234dde6268914a14ad44d23521ab6a9b258e" dependencies = [ "proc-macro2", "quote", @@ -13641,7 +13642,7 @@ dependencies = [ "rand 0.8.5", "ring 0.16.20", "rustc-hash", - "rustls 0.20.8", + "rustls 0.20.9", "slab", "thiserror", "tinyvec", @@ -13790,7 +13791,7 @@ checksum = "6413f3de1edee53342e6138e75b56d32e7bc6e332b3bd62d497b1929d4cfbcdd" dependencies = [ "pem", "ring 0.16.20", - "time 0.3.27", + "time 0.3.28", "x509-parser 0.13.2", "yasna", ] @@ -13803,7 +13804,7 @@ checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ "pem", "ring 0.16.20", - "time 0.3.27", + "time 0.3.28", "yasna", ] @@ -13883,14 +13884,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.6", - "regex-syntax 0.7.4", + "regex-automata 0.3.7", + "regex-syntax 0.7.5", ] [[package]] @@ -13904,13 +13905,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.4", + "regex-syntax 0.7.5", ] [[package]] @@ -13921,9 +13922,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "remote-ext-tests-bags-list" @@ -13950,7 +13951,7 @@ version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", "bytes", "encoding_rs", "futures-core", @@ -13966,8 +13967,8 @@ dependencies = [ "mime", "once_cell", "percent-encoding", - "pin-project-lite 0.2.12", - "rustls 0.21.6", + "pin-project-lite 0.2.13", + "rustls 0.21.7", "rustls-pemfile", "serde", "serde_json", @@ -14361,9 +14362,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "ed6248e1caa625eb708e266e06159f135e8c26f2bb7ceb72dc4b2766d0340964" dependencies = [ "bitflags 2.4.0", "errno", @@ -14387,9 +14388,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.8" +version = "0.20.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" dependencies = [ "log", "ring 0.16.20", @@ -14399,9 +14400,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.6" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", "ring 0.16.20", @@ -14427,7 +14428,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.2", + "base64 0.21.3", ] [[package]] @@ -15245,6 +15246,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tokio-stream", "tokio-test", "tokio-util", "unsigned-varint", @@ -15300,10 +15302,12 @@ name = "sc-network-gossip" version = "0.10.0-dev" dependencies = [ "ahash 0.8.3", + "async-trait", "futures", "futures-timer", "libp2p", "log", + "parity-scale-codec", "quickcheck", "sc-network", "sc-network-common", @@ -16528,7 +16532,7 @@ dependencies = [ "arrayvec 0.7.4", "async-lock", "atomic-take", - "base64 0.21.2", + "base64 0.21.3", "bip39", "blake2-rfc", "bs58 0.5.0", @@ -16581,7 +16585,7 @@ checksum = "256b5bad1d6b49045e95fe87492ce73d5af81545d8b4d8318a872d2007024c33" dependencies = [ "async-channel", "async-lock", - "base64 0.21.2", + "base64 0.21.3", "blake2-rfc", "derive_more", "either", @@ -18077,7 +18081,7 @@ version = "0.1.0" dependencies = [ "assert_cmd", "futures", - "nix 0.26.2", + "nix 0.26.4", "node-cli", "node-primitives", "regex", @@ -18515,7 +18519,7 @@ dependencies = [ "cfg-if", "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix 0.38.8", + "rustix 0.38.10", "windows-sys 0.48.0", ] @@ -18775,9 +18779,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb39ee79a6d8de55f48f2293a830e040392f1c5f16e336bdd1788cd0aadce07" +checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" dependencies = [ "deranged", "itoa", @@ -18794,9 +18798,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733d258752e9303d392b94b75230d07b0b9c489350c69b851fc6c065fde3e8f9" +checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" dependencies = [ "time-core", ] @@ -18866,7 +18870,7 @@ dependencies = [ "mio", "num_cpus", "parking_lot 0.12.1", - "pin-project-lite 0.2.12", + "pin-project-lite 0.2.13", "signal-hook-registry", "socket2 0.5.3", "tokio-macros", @@ -18901,7 +18905,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.6", + "rustls 0.21.7", "tokio", ] @@ -18912,7 +18916,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", - "pin-project-lite 0.2.12", + "pin-project-lite 0.2.13", "tokio", "tokio-util", ] @@ -18952,7 +18956,7 @@ dependencies = [ "futures-core", "futures-io", "futures-sink", - "pin-project-lite 0.2.12", + "pin-project-lite 0.2.13", "tokio", "tracing", ] @@ -19024,7 +19028,7 @@ dependencies = [ "http", "http-body", "http-range-header", - "pin-project-lite 0.2.12", + "pin-project-lite 0.2.13", "tower-layer", "tower-service", ] @@ -19049,7 +19053,7 @@ checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "log", - "pin-project-lite 0.2.12", + "pin-project-lite 0.2.13", "tracing-attributes", "tracing-core", ] @@ -19480,9 +19484,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna 0.4.0", @@ -19725,9 +19729,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "wasm-encoder" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41763f20eafed1399fff1afb466496d3a959f58241436cfdc17e3f5ca954de16" +checksum = "1ba64e81215916eaeb48fee292f29401d69235d62d8b8fd92a7b2844ec5ae5f7" dependencies = [ "leb128", ] @@ -19899,7 +19903,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c86437fa68626fe896e5afc69234bb2b5894949083586535f200385adfd71213" dependencies = [ "anyhow", - "base64 0.21.2", + "base64 0.21.3", "bincode", "directories-next", "file-per-thread-logger", @@ -20052,9 +20056,9 @@ dependencies = [ [[package]] name = "wast" -version = "63.0.0" +version = "64.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2560471f60a48b77fccefaf40796fda61c97ce1e790b59dfcec9dc3995c9f63a" +checksum = "a259b226fd6910225aa7baeba82f9d9933b6d00f2ce1b49b80fa4214328237cc" dependencies = [ "leb128", "memchr", @@ -20064,9 +20068,9 @@ dependencies = [ [[package]] name = "wat" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bdc306c2c4c2f2bf2ba69e083731d0d2a77437fc6a350a19db139636e7e416c" +checksum = "53253d920ab413fca1c7dc2161d601c79b4fdf631d0ba51dd4343bf9b556c3f6" dependencies = [ "wast", ] @@ -20151,7 +20155,7 @@ dependencies = [ "sha2 0.10.7", "stun", "thiserror", - "time 0.3.27", + "time 0.3.28", "tokio", "turn", "url", @@ -20759,7 +20763,7 @@ dependencies = [ "ring 0.16.20", "rusticata-macros", "thiserror", - "time 0.3.27", + "time 0.3.28", ] [[package]] @@ -20777,7 +20781,7 @@ dependencies = [ "oid-registry 0.6.1", "rusticata-macros", "thiserror", - "time 0.3.27", + "time 0.3.28", ] [[package]] @@ -20947,7 +20951,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" dependencies = [ - "time 0.3.27", + "time 0.3.28", ] [[package]] diff --git a/cumulus/client/relay-chain-minimal-node/Cargo.toml b/cumulus/client/relay-chain-minimal-node/Cargo.toml index 39056d6b6511..7ecc7ef15537 100644 --- a/cumulus/client/relay-chain-minimal-node/Cargo.toml +++ b/cumulus/client/relay-chain-minimal-node/Cargo.toml @@ -40,6 +40,7 @@ schnellru = "0.2.1" tracing = "0.1.37" async-trait = "0.1.73" futures = "0.3.28" +parking_lot = "0.12.1" [features] network-protocol-staging = [ diff --git a/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs b/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs index 491758c1329a..c7d536d116ca 100644 --- a/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs +++ b/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs @@ -15,8 +15,9 @@ // along with Polkadot. If not, see . use futures::{select, StreamExt}; +use parking_lot::Mutex; use schnellru::{ByLength, LruMap}; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use polkadot_availability_recovery::AvailabilityRecoverySubsystem; use polkadot_collator_protocol::{CollatorProtocolSubsystem, ProtocolSide}; @@ -27,7 +28,7 @@ use polkadot_network_bridge::{ use polkadot_node_collation_generation::CollationGenerationSubsystem; use polkadot_node_core_runtime_api::RuntimeApiSubsystem; use polkadot_node_network_protocol::{ - peer_set::PeerSetProtocolNames, + peer_set::{PeerSet, PeerSetProtocolNames}, request_response::{ v1::{self, AvailableDataFetchingRequest}, vstaging, IncomingRequestReceiver, ReqProtocolNames, @@ -41,7 +42,7 @@ use polkadot_overseer::{ use polkadot_primitives::CollatorPair; use sc_authority_discovery::Service as AuthorityDiscoveryService; -use sc_network::NetworkStateInfo; +use sc_network::{NetworkStateInfo, NotificationService}; use sc_service::TaskManager; use sp_runtime::traits::Block as BlockT; @@ -77,6 +78,8 @@ pub(crate) struct CollatorOverseerGenArgs<'a> { pub req_protocol_names: ReqProtocolNames, /// Peerset protocols name mapping pub peer_set_protocol_names: PeerSetProtocolNames, + /// Notification services for validation/collation protocols. + pub notification_services: HashMap>, } fn build_overseer( @@ -94,6 +97,7 @@ fn build_overseer( collator_pair, req_protocol_names, peer_set_protocol_names, + notification_services, }: CollatorOverseerGenArgs<'_>, ) -> Result< (Overseer, Arc>, OverseerHandle), @@ -101,6 +105,8 @@ fn build_overseer( > { let spawner = SpawnGlue(spawner); let network_bridge_metrics: NetworkBridgeMetrics = Metrics::register(registry)?; + let notification_sinks = Arc::new(Mutex::new(HashMap::new())); + let builder = Overseer::builder() .availability_distribution(DummySubsystem) .availability_recovery(AvailabilityRecoverySubsystem::with_availability_store_skip( @@ -131,6 +137,8 @@ fn build_overseer( sync_oracle, network_bridge_metrics.clone(), peer_set_protocol_names.clone(), + notification_services, + notification_sinks.clone(), )) .network_bridge_tx(NetworkBridgeTxSubsystem::new( network_service, @@ -138,6 +146,7 @@ fn build_overseer( network_bridge_metrics, req_protocol_names, peer_set_protocol_names, + notification_sinks, )) .provisioner(DummySubsystem) .runtime_api(RuntimeApiSubsystem::new( diff --git a/cumulus/client/relay-chain-minimal-node/src/lib.rs b/cumulus/client/relay-chain-minimal-node/src/lib.rs index 366d428eda70..cae093678ad6 100644 --- a/cumulus/client/relay-chain-minimal-node/src/lib.rs +++ b/cumulus/client/relay-chain-minimal-node/src/lib.rs @@ -21,7 +21,7 @@ use cumulus_relay_chain_rpc_interface::{RelayChainRpcClient, RelayChainRpcInterf use network::build_collator_network; use polkadot_network_bridge::{peer_sets_info, IsAuthority}; use polkadot_node_network_protocol::{ - peer_set::PeerSetProtocolNames, + peer_set::{PeerSet, PeerSetProtocolNames}, request_response::{ v1, vstaging, IncomingRequest, IncomingRequestReceiver, Protocol, ReqProtocolNames, }, @@ -176,10 +176,13 @@ async fn new_minimal_relay_chain( let peer_set_protocol_names = PeerSetProtocolNames::new(genesis_hash, config.chain_spec.fork_id()); let is_authority = if role.is_authority() { IsAuthority::Yes } else { IsAuthority::No }; - - for config in peer_sets_info(is_authority, &peer_set_protocol_names) { - net_config.add_notification_protocol(config); - } + let notification_services = peer_sets_info(is_authority, &peer_set_protocol_names) + .into_iter() + .map(|(config, (peerset, service))| { + net_config.add_notification_protocol(config); + (peerset, service) + }) + .collect::>>(); let request_protocol_names = ReqProtocolNames::new(genesis_hash, config.chain_spec.fork_id()); let (collation_req_receiver_v1, collation_req_receiver_vstaging, available_data_req_receiver) = @@ -219,6 +222,7 @@ async fn new_minimal_relay_chain( collator_pair, req_protocol_names: request_protocol_names, peer_set_protocol_names, + notification_services, }; let overseer_handle = diff --git a/cumulus/client/relay-chain-minimal-node/src/network.rs b/cumulus/client/relay-chain-minimal-node/src/network.rs index f39d7a26dd88..0258cf9ddd58 100644 --- a/cumulus/client/relay-chain-minimal-node/src/network.rs +++ b/cumulus/client/relay-chain-minimal-node/src/network.rs @@ -25,10 +25,9 @@ use sc_network::{ NetworkService, }; -use sc_network::config::FullNetworkConfiguration; +use sc_network::{config::FullNetworkConfiguration, NotificationService}; use sc_network_common::{role::Roles, sync::message::BlockAnnouncesHandshake}; use sc_service::{error::Error, Configuration, NetworkStarter, SpawnTaskHandle}; -use sc_utils::mpsc::tracing_unbounded; use std::{iter, sync::Arc}; @@ -44,7 +43,7 @@ pub(crate) fn build_collator_network( Error, > { let protocol_id = config.protocol_id(); - let block_announce_config = get_block_announce_proto_config::( + let (block_announce_config, _notification_service) = get_block_announce_proto_config::( protocol_id.clone(), &None, Roles::from(&config.role), @@ -64,8 +63,6 @@ pub(crate) fn build_collator_network( let peer_store_handle = peer_store.handle(); spawn_handle.spawn("peer-store", Some("networking"), peer_store.run()); - // RX is not used for anything because syncing is not started for the minimal node - let (tx, _rx) = tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let network_params = sc_network::config::Params:: { role: config.role.clone(), executor: { @@ -81,7 +78,6 @@ pub(crate) fn build_collator_network( protocol_id, metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()), block_announce_config, - tx, }; let network_worker = sc_network::NetworkWorker::new(network_params)?; @@ -133,7 +129,7 @@ fn get_block_announce_proto_config( best_number: NumberFor, best_hash: B::Hash, genesis_hash: B::Hash, -) -> NonDefaultSetConfig { +) -> (NonDefaultSetConfig, Box) { let block_announces_protocol = { let genesis_hash = genesis_hash.as_ref(); if let Some(ref fork_id) = fork_id { @@ -143,12 +139,11 @@ fn get_block_announce_proto_config( } }; - NonDefaultSetConfig { - notifications_protocol: block_announces_protocol.into(), - fallback_names: iter::once(format!("/{}/block-announces/1", protocol_id.as_ref()).into()) - .collect(), - max_notification_size: 1024 * 1024, - handshake: Some(NotificationHandshake::new(BlockAnnouncesHandshake::::build( + NonDefaultSetConfig::new( + block_announces_protocol.into(), + iter::once(format!("/{}/block-announces/1", protocol_id.as_ref()).into()).collect(), + 1024 * 1024, + Some(NotificationHandshake::new(BlockAnnouncesHandshake::::build( roles, best_number, best_hash, @@ -156,11 +151,11 @@ fn get_block_announce_proto_config( ))), // NOTE: `set_config` will be ignored by `protocol.rs` as the block announcement // protocol is still hardcoded into the peerset. - set_config: SetConfig { + SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - } + ) } diff --git a/polkadot/node/network/bridge/src/lib.rs b/polkadot/node/network/bridge/src/lib.rs index 46d4a00faace..ddce99d5c2a8 100644 --- a/polkadot/node/network/bridge/src/lib.rs +++ b/polkadot/node/network/bridge/src/lib.rs @@ -83,6 +83,7 @@ pub(crate) enum WireMessage { ViewUpdate(View), } +#[derive(Debug)] pub(crate) struct PeerData { /// The Latest view sent by the peer. view: View, diff --git a/polkadot/node/network/bridge/src/network.rs b/polkadot/node/network/bridge/src/network.rs index 4f21212dcb64..1bdc602dfe04 100644 --- a/polkadot/node/network/bridge/src/network.rs +++ b/polkadot/node/network/bridge/src/network.rs @@ -14,21 +14,24 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::{collections::HashSet, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; use async_trait::async_trait; -use futures::{prelude::*, stream::BoxStream}; +use parking_lot::Mutex; use parity_scale_codec::Encode; use sc_network::{ - config::parse_addr, multiaddr::Multiaddr, types::ProtocolName, Event as NetworkEvent, - IfDisconnected, NetworkEventStream, NetworkNotification, NetworkPeers, NetworkRequest, - NetworkService, OutboundFailure, ReputationChange, RequestFailure, + config::parse_addr, multiaddr::Multiaddr, types::ProtocolName, IfDisconnected, MessageSink, + NetworkPeers, NetworkRequest, NetworkService, OutboundFailure, ReputationChange, + RequestFailure, }; use polkadot_node_network_protocol::{ - peer_set::{PeerSet, PeerSetProtocolNames, ProtocolVersion}, + peer_set::{PeerSet, ProtocolVersion}, request_response::{OutgoingRequest, Recipient, ReqProtocolNames, Requests}, PeerId, }; @@ -45,51 +48,50 @@ const LOG_TARGET: &'static str = "parachain::network-bridge-net"; /// messages that are compatible with the passed peer set, as that is currently not enforced by /// this function. These are messages of type `WireMessage` parameterized on the matching type. pub(crate) fn send_message( - net: &mut impl Network, mut peers: Vec, peer_set: PeerSet, version: ProtocolVersion, - protocol_names: &PeerSetProtocolNames, message: M, metrics: &super::Metrics, + network_notification_sinks: &Arc>>>, ) where M: Encode + Clone, { if peers.is_empty() { return } + let message = { let encoded = message.encode(); metrics.on_notification_sent(peer_set, version, encoded.len(), peers.len()); encoded }; + let notification_sinks = network_notification_sinks.lock(); + // optimization: avoid cloning the message for the last peer in the // list. The message payload can be quite large. If the underlying // network used `Bytes` this would not be necessary. + // + // peer may have gotten disconnect by the time `send_message()` is called + // at which point the the sink is not available. let last_peer = peers.pop(); - - // We always send messages on the "main" name even when a negotiated - // fallback is used. The libp2p implementation handles the fallback - // under the hood. - let protocol_name = protocol_names.get_main_name(peer_set); peers.into_iter().for_each(|peer| { - net.write_notification(peer, protocol_name.clone(), message.clone()); + if let Some(sink) = notification_sinks.get(&(peer_set, peer)) { + sink.send_sync_notification(message.clone()); + } }); + if let Some(peer) = last_peer { - net.write_notification(peer, protocol_name, message); + if let Some(sink) = notification_sinks.get(&(peer_set, peer)) { + sink.send_sync_notification(message.clone()); + } } } /// An abstraction over networking for the purposes of this subsystem. #[async_trait] pub trait Network: Clone + Send + 'static { - /// Get a stream of all events occurring on the network. This may include events unrelated - /// to the Polkadot protocol - the user of this function should filter only for events related - /// to the [`VALIDATION_PROTOCOL_NAME`](VALIDATION_PROTOCOL_NAME) - /// or [`COLLATION_PROTOCOL_NAME`](COLLATION_PROTOCOL_NAME) - fn event_stream(&mut self) -> BoxStream<'static, NetworkEvent>; - /// Ask the network to keep a substream open with these nodes and not disconnect from them /// until removed from the protocol's peer set. /// Note that `out_peers` setting has no effect on this. @@ -121,16 +123,12 @@ pub trait Network: Clone + Send + 'static { /// Disconnect a given peer from the protocol specified without harming reputation. fn disconnect_peer(&self, who: PeerId, protocol: ProtocolName); - /// Write a notification to a peer on the given protocol. - fn write_notification(&self, who: PeerId, protocol: ProtocolName, message: Vec); + /// Get peer role. + fn peer_role(&self, who: PeerId, handshake: Vec) -> Option; } #[async_trait] impl Network for Arc> { - fn event_stream(&mut self) -> BoxStream<'static, NetworkEvent> { - NetworkService::event_stream(self, "polkadot-network-bridge").boxed() - } - async fn set_reserved_peers( &mut self, protocol: ProtocolName, @@ -155,10 +153,6 @@ impl Network for Arc> { NetworkService::disconnect_peer(&**self, who, protocol); } - fn write_notification(&self, who: PeerId, protocol: ProtocolName, message: Vec) { - NetworkService::write_notification(&**self, who, protocol, message); - } - async fn start_request( &self, authority_discovery: &mut AD, @@ -230,6 +224,10 @@ impl Network for Arc> { if_disconnected, ); } + + fn peer_role(&self, who: PeerId, handshake: Vec) -> Option { + NetworkService::peer_role(self, who, handshake) + } } /// We assume one `peer_id` per `authority_id`. diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 51d248ca2d49..378fbd3a1a2b 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -20,10 +20,13 @@ use super::*; use always_assert::never; use bytes::Bytes; -use futures::stream::BoxStream; use parity_scale_codec::{Decode, DecodeAll}; +use parking_lot::Mutex; -use sc_network::Event as NetworkEvent; +use sc_network::{ + service::traits::{NotificationEvent, ValidationResult}, + MessageSink, NotificationService, +}; use sp_consensus::SyncOracle; use polkadot_node_network_protocol::{ @@ -85,6 +88,9 @@ pub struct NetworkBridgeRx { shared: Shared, metrics: Metrics, peerset_protocol_names: PeerSetProtocolNames, + validation_service: Box, + collation_service: Box, + notification_sinks: Arc>>>, } impl NetworkBridgeRx { @@ -99,8 +105,18 @@ impl NetworkBridgeRx { sync_oracle: Box, metrics: Metrics, peerset_protocol_names: PeerSetProtocolNames, + mut notification_services: HashMap>, + notification_sinks: Arc>>>, ) -> Self { let shared = Shared::default(); + + let validation_service = notification_services + .remove(&PeerSet::Validation) + .expect("validation protocol was enabled so `NotificationService` must exist; qed"); + let collation_service = notification_services + .remove(&PeerSet::Collation) + .expect("collation protocol was enabled so `NotificationService` must exist; qed"); + Self { network_service, authority_discovery_service, @@ -108,6 +124,9 @@ impl NetworkBridgeRx { shared, metrics, peerset_protocol_names, + validation_service, + collation_service, + notification_sinks, } } } @@ -118,423 +137,542 @@ where Net: Network + Sync, AD: validator_discovery::AuthorityDiscovery + Clone + Sync, { - fn start(mut self, ctx: Context) -> SpawnedSubsystem { - // The stream of networking events has to be created at initialization, otherwise the - // networking might open connections before the stream of events has been grabbed. - let network_stream = self.network_service.event_stream(); - + fn start(self, ctx: Context) -> SpawnedSubsystem { // Swallow error because failure is fatal to the node and we log with more precision // within `run_network`. - let future = run_network_in(self, ctx, network_stream) + let future = run_network_in(self, ctx) .map_err(|e| SubsystemError::with_origin("network-bridge", e)) .boxed(); SpawnedSubsystem { name: "network-bridge-rx-subsystem", future } } } -async fn handle_network_messages( - mut sender: impl overseer::NetworkBridgeRxSenderTrait, - mut network_service: impl Network, - network_stream: BoxStream<'static, NetworkEvent>, - mut authority_discovery_service: AD, - metrics: Metrics, - shared: Shared, - peerset_protocol_names: PeerSetProtocolNames, -) -> Result<(), Error> -where +/// Handle notification event received over the validation protocol. +async fn handle_validation_message( + event: NotificationEvent, + network_service: &mut impl Network, + sender: &mut impl overseer::NetworkBridgeRxSenderTrait, + authority_discovery_service: &mut AD, + metrics: &Metrics, + shared: &Shared, + peerset_protocol_names: &PeerSetProtocolNames, + notification_service: &mut Box, + notification_sinks: &mut Arc>>>, +) where AD: validator_discovery::AuthorityDiscovery + Send, { - let mut network_stream = network_stream.fuse(); - loop { - match network_stream.next().await { - None => return Err(Error::EventStreamConcluded), - Some(NetworkEvent::Dht(_)) => {}, - Some(NetworkEvent::NotificationStreamOpened { - remote: peer, - protocol, - role, - negotiated_fallback, - received_handshake: _, - }) => { - let role = ObservedRole::from(role); - let (peer_set, version) = { - let (peer_set, version) = - match peerset_protocol_names.try_get_protocol(&protocol) { - None => continue, - Some(p) => p, - }; - - if let Some(fallback) = negotiated_fallback { - match peerset_protocol_names.try_get_protocol(&fallback) { - None => { + match event { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + let _ = result_tx.send(ValidationResult::Accept); + }, + NotificationEvent::NotificationStreamOpened { + peer, + handshake, + negotiated_fallback, + .. + } => { + let role = match network_service.peer_role(peer, handshake) { + Some(role) => ObservedRole::from(role), + None => { + gum::debug!( + target: LOG_TARGET, + ?peer, + "Failed to determine peer role for validation protocol", + ); + return + }, + }; + + let (peer_set, version) = { + let (peer_set, version) = + (PeerSet::Validation, PeerSet::Validation.get_main_version()); + + if let Some(fallback) = negotiated_fallback { + match peerset_protocol_names.try_get_protocol(&fallback) { + None => { + gum::debug!( + target: LOG_TARGET, + fallback = &*fallback, + ?peer, + ?peer_set, + "Unknown fallback", + ); + + return + }, + Some((p2, v2)) => { + if p2 != peer_set { gum::debug!( target: LOG_TARGET, fallback = &*fallback, - ?peer, - ?peer_set, - "Unknown fallback", + fallback_peerset = ?p2, + peerset = ?peer_set, + "Fallback mismatched peer-set", ); - continue - }, - Some((p2, v2)) => { - if p2 != peer_set { - gum::debug!( - target: LOG_TARGET, - fallback = &*fallback, - fallback_peerset = ?p2, - protocol = &*protocol, - peerset = ?peer_set, - "Fallback mismatched peer-set", - ); - - continue - } - - (p2, v2) - }, - } - } else { - (peer_set, version) - } - }; - - gum::debug!( - target: LOG_TARGET, - action = "PeerConnected", - peer_set = ?peer_set, - version = %version, - peer = ?peer, - role = ?role - ); - - let local_view = { - let mut shared = shared.0.lock(); - let peer_map = match peer_set { - PeerSet::Validation => &mut shared.validation_peers, - PeerSet::Collation => &mut shared.collation_peers, - }; + return + } - match peer_map.entry(peer) { - hash_map::Entry::Occupied(_) => continue, - hash_map::Entry::Vacant(vacant) => { - vacant.insert(PeerData { view: View::default(), version }); + (p2, v2) }, } - - metrics.on_peer_connected(peer_set, version); - metrics.note_peer_count(peer_set, version, peer_map.len()); - - shared.local_view.clone().unwrap_or(View::default()) - }; - - let maybe_authority = - authority_discovery_service.get_authority_ids_by_peer_id(peer).await; - - match peer_set { - PeerSet::Validation => { - dispatch_validation_events_to_all( - vec![ - NetworkBridgeEvent::PeerConnected( - peer, - role, - version, - maybe_authority, - ), - NetworkBridgeEvent::PeerViewChange(peer, View::default()), - ], - &mut sender, - ) - .await; - - match ValidationVersion::try_from(version) - .expect("try_get_protocol has already checked version is known; qed") - { - ValidationVersion::V1 => send_message( - &mut network_service, - vec![peer], - PeerSet::Validation, - version, - &peerset_protocol_names, - WireMessage::::ViewUpdate( - local_view, - ), - &metrics, - ), - ValidationVersion::VStaging => send_message( - &mut network_service, - vec![peer], - PeerSet::Validation, - version, - &peerset_protocol_names, - WireMessage::::ViewUpdate( - local_view, - ), - &metrics, - ), - } - }, - PeerSet::Collation => { - dispatch_collation_events_to_all( - vec![ - NetworkBridgeEvent::PeerConnected( - peer, - role, - version, - maybe_authority, - ), - NetworkBridgeEvent::PeerViewChange(peer, View::default()), - ], - &mut sender, - ) - .await; - - match CollationVersion::try_from(version) - .expect("try_get_protocol has already checked version is known; qed") - { - CollationVersion::V1 => send_message( - &mut network_service, - vec![peer], - PeerSet::Collation, - version, - &peerset_protocol_names, - WireMessage::::ViewUpdate( - local_view, - ), - &metrics, - ), - CollationVersion::VStaging => send_message( - &mut network_service, - vec![peer], - PeerSet::Collation, - version, - &peerset_protocol_names, - WireMessage::::ViewUpdate( - local_view, - ), - &metrics, - ), - } + } else { + (peer_set, version) + } + }; + // store the notification sink to `notification_sinks` so both `NetworkBridgeRx` + // and `NetworkBridgeTx` can send messages to the peer. + match notification_service.message_sink(&peer) { + Some(sink) => { + notification_sinks.lock().insert((peer_set, peer), sink); + }, + None => { + gum::warn!( + target: LOG_TARGET, + peer_set = ?peer_set, + version = %version, + peer = ?peer, + role = ?role, + "Message sink not available for peer", + ); + return + }, + } + + gum::debug!( + target: LOG_TARGET, + action = "PeerConnected", + peer_set = ?peer_set, + version = %version, + peer = ?peer, + role = ?role + ); + + let local_view = { + let mut shared = shared.0.lock(); + let peer_map = &mut shared.validation_peers; + + match peer_map.entry(peer) { + hash_map::Entry::Occupied(_) => return, + hash_map::Entry::Vacant(vacant) => { + vacant.insert(PeerData { view: View::default(), version }); }, } - }, - Some(NetworkEvent::NotificationStreamClosed { remote: peer, protocol }) => { - let (peer_set, version) = match peerset_protocol_names.try_get_protocol(&protocol) { - None => continue, - Some(peer_set) => peer_set, - }; - gum::debug!( - target: LOG_TARGET, - action = "PeerDisconnected", - peer_set = ?peer_set, - peer = ?peer - ); - - let was_connected = { - let mut shared = shared.0.lock(); - let peer_map = match peer_set { - PeerSet::Validation => &mut shared.validation_peers, - PeerSet::Collation => &mut shared.collation_peers, - }; + metrics.on_peer_connected(peer_set, version); + metrics.note_peer_count(peer_set, version, peer_map.len()); - let w = peer_map.remove(&peer).is_some(); + shared.local_view.clone().unwrap_or(View::default()) + }; - metrics.on_peer_disconnected(peer_set, version); - metrics.note_peer_count(peer_set, version, peer_map.len()); + let maybe_authority = + authority_discovery_service.get_authority_ids_by_peer_id(peer).await; - w - }; + dispatch_validation_events_to_all( + vec![ + NetworkBridgeEvent::PeerConnected(peer, role, version, maybe_authority), + NetworkBridgeEvent::PeerViewChange(peer, View::default()), + ], + sender, + ) + .await; - if was_connected && version == peer_set.get_main_version() { - match peer_set { - PeerSet::Validation => - dispatch_validation_event_to_all( - NetworkBridgeEvent::PeerDisconnected(peer), - &mut sender, - ) - .await, - PeerSet::Collation => - dispatch_collation_event_to_all( - NetworkBridgeEvent::PeerDisconnected(peer), - &mut sender, - ) - .await, - } + match ValidationVersion::try_from(version) + .expect("try_get_protocol has already checked version is known; qed") + { + ValidationVersion::V1 => send_message( + vec![peer], + PeerSet::Validation, + version, + WireMessage::::ViewUpdate(local_view), + metrics, + notification_sinks, + ), + ValidationVersion::VStaging => send_message( + vec![peer], + PeerSet::Validation, + version, + WireMessage::::ViewUpdate(local_view), + metrics, + notification_sinks, + ), + } + }, + NotificationEvent::NotificationStreamClosed { peer } => { + let (peer_set, version) = (PeerSet::Validation, PeerSet::Validation.get_main_version()); + + gum::debug!( + target: LOG_TARGET, + action = "PeerDisconnected", + ?peer_set, + ?peer + ); + + let was_connected = { + let mut shared = shared.0.lock(); + let peer_map = &mut shared.validation_peers; + + let w = peer_map.remove(&peer).is_some(); + + metrics.on_peer_disconnected(peer_set, version); + metrics.note_peer_count(peer_set, version, peer_map.len()); + + w + }; + + if was_connected && version == peer_set.get_main_version() { + dispatch_validation_event_to_all( + NetworkBridgeEvent::PeerDisconnected(peer), + sender, + ) + .await; + } + }, + NotificationEvent::NotificationReceived { peer, notification } => { + let expected_versions = { + let mut versions = PerPeerSet::>::default(); + let shared = shared.0.lock(); + + if let Some(peer_data) = shared.validation_peers.get(&peer) { + versions[PeerSet::Validation] = Some(peer_data.version); } - }, - Some(NetworkEvent::NotificationsReceived { remote, messages }) => { - let expected_versions = { - let mut versions = PerPeerSet::>::default(); - let shared = shared.0.lock(); - if let Some(peer_data) = shared.validation_peers.get(&remote) { - versions[PeerSet::Validation] = Some(peer_data.version); - } - if let Some(peer_data) = shared.collation_peers.get(&remote) { - versions[PeerSet::Collation] = Some(peer_data.version); - } + versions + }; + + gum::trace!( + target: LOG_TARGET, + action = "PeerMessage", + peerset = ?PeerSet::Validation, + ?peer, + ); + + let (events, reports) = + if expected_versions[PeerSet::Validation] == Some(ValidationVersion::V1.into()) { + handle_peer_messages::( + peer, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + vec![notification.into()], + metrics, + ) + } else if expected_versions[PeerSet::Validation] == + Some(ValidationVersion::VStaging.into()) + { + handle_peer_messages::( + peer, + PeerSet::Validation, + &mut shared.0.lock().validation_peers, + vec![notification.into()], + metrics, + ) + } else { + gum::warn!( + target: LOG_TARGET, + version = ?expected_versions[PeerSet::Validation], + "Major logic bug. Peer somehow has unsupported validation protocol version." + ); - versions - }; + never!("Only versions 1 and 2 are supported; peer set connection checked above; qed"); - // non-decoded, but version-checked validation messages. - let v_messages: Result, _> = messages - .iter() - .filter_map(|(protocol, msg_bytes)| { - // version doesn't matter because we always receive on the 'correct' - // protocol name, not the negotiated fallback. - let (peer_set, _version) = - peerset_protocol_names.try_get_protocol(protocol)?; - if peer_set == PeerSet::Validation { - if expected_versions[PeerSet::Validation].is_none() { - return Some(Err(UNCONNECTED_PEERSET_COST)) - } + // If a peer somehow triggers this, we'll disconnect them + // eventually. + (Vec::new(), vec![UNCONNECTED_PEERSET_COST]) + }; - Some(Ok(msg_bytes.clone())) - } else { - None - } - }) - .collect(); + for report in reports { + network_service.report_peer(peer, report.into()); + } - let v_messages = match v_messages { - Err(rep) => { - gum::debug!(target: LOG_TARGET, action = "ReportPeer"); - network_service.report_peer(remote, rep.into()); + dispatch_validation_events_to_all(events, sender).await; + }, + } +} - continue - }, - Ok(v) => v, - }; +/// Handle notification event received over the collation protocol. +async fn handle_collation_message( + event: NotificationEvent, + network_service: &mut impl Network, + sender: &mut impl overseer::NetworkBridgeRxSenderTrait, + authority_discovery_service: &mut AD, + metrics: &Metrics, + shared: &Shared, + peerset_protocol_names: &PeerSetProtocolNames, + notification_service: &mut Box, + notification_sinks: &mut Arc>>>, +) where + AD: validator_discovery::AuthorityDiscovery + Send, +{ + match event { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + let _ = result_tx.send(ValidationResult::Accept); + }, + NotificationEvent::NotificationStreamOpened { + peer, + handshake, + negotiated_fallback, + .. + } => { + let role = match network_service.peer_role(peer, handshake) { + Some(role) => ObservedRole::from(role), + None => { + gum::debug!( + target: LOG_TARGET, + ?peer, + "Failed to determine peer role for validation protocol", + ); + return + }, + }; + + let (peer_set, version) = { + let (peer_set, version) = + (PeerSet::Collation, PeerSet::Collation.get_main_version()); + + if let Some(fallback) = negotiated_fallback { + match peerset_protocol_names.try_get_protocol(&fallback) { + None => { + gum::debug!( + target: LOG_TARGET, + fallback = &*fallback, + ?peer, + ?peer_set, + "Unknown fallback", + ); + + return + }, + Some((p2, v2)) => { + if p2 != peer_set { + gum::debug!( + target: LOG_TARGET, + fallback = &*fallback, + fallback_peerset = ?p2, + peerset = ?peer_set, + "Fallback mismatched peer-set", + ); - // non-decoded, but version-checked collation messages. - let c_messages: Result, _> = messages - .iter() - .filter_map(|(protocol, msg_bytes)| { - // version doesn't matter because we always receive on the 'correct' - // protocol name, not the negotiated fallback. - let (peer_set, _version) = - peerset_protocol_names.try_get_protocol(protocol)?; - - if peer_set == PeerSet::Collation { - if expected_versions[PeerSet::Collation].is_none() { - return Some(Err(UNCONNECTED_PEERSET_COST)) + return } - Some(Ok(msg_bytes.clone())) - } else { - None - } - }) - .collect(); - - let c_messages = match c_messages { - Err(rep) => { - gum::debug!(target: LOG_TARGET, action = "ReportPeer"); - network_service.report_peer(remote, rep.into()); - - continue + (p2, v2) + }, + } + } else { + (peer_set, version) + } + }; + + // store the notification sink to `notification_sinks` so both `NetworkBridgeRx` + // and `NetworkBridgeTx` can send messages to the peer. + match notification_service.message_sink(&peer) { + Some(sink) => { + notification_sinks.lock().insert((peer_set, peer), sink); + }, + None => { + gum::warn!( + target: LOG_TARGET, + peer_set = ?peer_set, + version = %version, + peer = ?peer, + role = ?role, + "Message sink not available for peer", + ); + return + }, + } + + gum::debug!( + target: LOG_TARGET, + action = "PeerConnected", + peer_set = ?peer_set, + version = %version, + peer = ?peer, + role = ?role + ); + + let local_view = { + let mut shared = shared.0.lock(); + let peer_map = &mut shared.collation_peers; + + match peer_map.entry(peer) { + hash_map::Entry::Occupied(_) => return, + hash_map::Entry::Vacant(vacant) => { + vacant.insert(PeerData { view: View::default(), version }); }, - Ok(v) => v, - }; - - if v_messages.is_empty() && c_messages.is_empty() { - continue } - gum::trace!( - target: LOG_TARGET, - action = "PeerMessages", - peer = ?remote, - num_validation_messages = %v_messages.len(), - num_collation_messages = %c_messages.len() - ); - - if !v_messages.is_empty() { - let (events, reports) = if expected_versions[PeerSet::Validation] == - Some(ValidationVersion::V1.into()) - { - handle_peer_messages::( - remote, - PeerSet::Validation, - &mut shared.0.lock().validation_peers, - v_messages, - &metrics, - ) - } else if expected_versions[PeerSet::Validation] == - Some(ValidationVersion::VStaging.into()) - { - handle_peer_messages::( - remote, - PeerSet::Validation, - &mut shared.0.lock().validation_peers, - v_messages, - &metrics, - ) - } else { - gum::warn!( - target: LOG_TARGET, - version = ?expected_versions[PeerSet::Validation], - "Major logic bug. Peer somehow has unsupported validation protocol version." - ); + metrics.on_peer_connected(peer_set, version); + metrics.note_peer_count(peer_set, version, peer_map.len()); - never!("Only versions 1 and 2 are supported; peer set connection checked above; qed"); + shared.local_view.clone().unwrap_or(View::default()) + }; - // If a peer somehow triggers this, we'll disconnect them - // eventually. - (Vec::new(), vec![UNCONNECTED_PEERSET_COST]) - }; + let maybe_authority = + authority_discovery_service.get_authority_ids_by_peer_id(peer).await; - for report in reports { - network_service.report_peer(remote, report.into()); - } + dispatch_collation_events_to_all( + vec![ + NetworkBridgeEvent::PeerConnected(peer, role, version, maybe_authority), + NetworkBridgeEvent::PeerViewChange(peer, View::default()), + ], + sender, + ) + .await; - dispatch_validation_events_to_all(events, &mut sender).await; + match CollationVersion::try_from(version) + .expect("try_get_protocol has already checked version is known; qed") + { + CollationVersion::V1 => send_message( + vec![peer], + PeerSet::Collation, + version, + WireMessage::::ViewUpdate(local_view), + metrics, + notification_sinks, + ), + CollationVersion::VStaging => send_message( + vec![peer], + PeerSet::Collation, + version, + WireMessage::::ViewUpdate(local_view), + metrics, + notification_sinks, + ), + } + }, + NotificationEvent::NotificationStreamClosed { peer } => { + let (peer_set, version) = (PeerSet::Collation, PeerSet::Collation.get_main_version()); + + gum::debug!( + target: LOG_TARGET, + action = "PeerDisconnected", + ?peer_set, + ?peer + ); + + let was_connected = { + let mut shared = shared.0.lock(); + let peer_map = &mut shared.collation_peers; + + let w = peer_map.remove(&peer).is_some(); + + metrics.on_peer_disconnected(peer_set, version); + metrics.note_peer_count(peer_set, version, peer_map.len()); + + w + }; + + if was_connected && version == peer_set.get_main_version() { + dispatch_collation_event_to_all(NetworkBridgeEvent::PeerDisconnected(peer), sender) + .await; + } + }, + NotificationEvent::NotificationReceived { peer, notification } => { + let expected_versions = { + let mut versions = PerPeerSet::>::default(); + let shared = shared.0.lock(); + + if let Some(peer_data) = shared.collation_peers.get(&peer) { + versions[PeerSet::Collation] = Some(peer_data.version); } - if !c_messages.is_empty() { - let (events, reports) = if expected_versions[PeerSet::Collation] == - Some(CollationVersion::V1.into()) - { - handle_peer_messages::( - remote, - PeerSet::Collation, - &mut shared.0.lock().collation_peers, - c_messages, - &metrics, - ) - } else if expected_versions[PeerSet::Collation] == - Some(CollationVersion::VStaging.into()) - { - handle_peer_messages::( - remote, - PeerSet::Collation, - &mut shared.0.lock().collation_peers, - c_messages, - &metrics, - ) - } else { - gum::warn!( - target: LOG_TARGET, - version = ?expected_versions[PeerSet::Collation], - "Major logic bug. Peer somehow has unsupported collation protocol version." - ); + versions + }; + + gum::trace!( + target: LOG_TARGET, + action = "PeerMessage", + peerset = ?PeerSet::Collation, + ?peer, + ); + + let (events, reports) = + if expected_versions[PeerSet::Collation] == Some(CollationVersion::V1.into()) { + handle_peer_messages::( + peer, + PeerSet::Collation, + &mut shared.0.lock().collation_peers, + vec![notification.into()], + metrics, + ) + } else if expected_versions[PeerSet::Collation] == + Some(CollationVersion::VStaging.into()) + { + handle_peer_messages::( + peer, + PeerSet::Collation, + &mut shared.0.lock().collation_peers, + vec![notification.into()], + metrics, + ) + } else { + gum::warn!( + target: LOG_TARGET, + version = ?expected_versions[PeerSet::Collation], + "Major logic bug. Peer somehow has unsupported collation protocol version." + ); - never!("Only versions 1 and 2 are supported; peer set connection checked above; qed"); + never!("Only versions 1 and 2 are supported; peer set connection checked above; qed"); - // If a peer somehow triggers this, we'll disconnect them - // eventually. - (Vec::new(), vec![UNCONNECTED_PEERSET_COST]) - }; + // If a peer somehow triggers this, we'll disconnect them + // eventually. + (Vec::new(), vec![UNCONNECTED_PEERSET_COST]) + }; - for report in reports { - network_service.report_peer(remote, report.into()); - } + for report in reports { + network_service.report_peer(peer, report.into()); + } - dispatch_collation_events_to_all(events, &mut sender).await; - } + dispatch_collation_events_to_all(events, sender).await; + }, + } +} + +async fn handle_network_messages( + mut sender: impl overseer::NetworkBridgeRxSenderTrait, + mut network_service: impl Network, + mut authority_discovery_service: AD, + metrics: Metrics, + shared: Shared, + peerset_protocol_names: PeerSetProtocolNames, + mut validation_service: Box, + mut collation_service: Box, + mut notification_sinks: Arc>>>, +) -> Result<(), Error> +where + AD: validator_discovery::AuthorityDiscovery + Send, +{ + loop { + futures::select! { + event = validation_service.next_event().fuse() => match event { + Some(event) => handle_validation_message( + event, + &mut network_service, + &mut sender, + &mut authority_discovery_service, + &metrics, + &shared, + &peerset_protocol_names, + &mut validation_service, + &mut notification_sinks, + ).await, + None => return Err(Error::EventStreamConcluded), }, + event = collation_service.next_event().fuse() => match event { + Some(event) => handle_collation_message( + event, + &mut network_service, + &mut sender, + &mut authority_discovery_service, + &metrics, + &shared, + &peerset_protocol_names, + &mut collation_service, + &mut notification_sinks, + ).await, + None => return Err(Error::EventStreamConcluded), + } } } } @@ -560,17 +698,15 @@ where } #[overseer::contextbounds(NetworkBridgeRx, prefix = self::overseer)] -async fn run_incoming_orchestra_signals( +async fn run_incoming_orchestra_signals( mut ctx: Context, - mut network_service: N, mut authority_discovery_service: AD, shared: Shared, sync_oracle: Box, metrics: Metrics, - peerset_protocol_names: PeerSetProtocolNames, + notification_sinks: Arc>>>, ) -> Result<(), Error> where - N: Network, AD: validator_discovery::AuthorityDiscovery + Clone, { // This is kept sorted, descending, by block number. @@ -662,13 +798,12 @@ where mode = Mode::Active; update_our_view( - &mut network_service, &mut ctx, &live_heads, &shared, finalized_number, &metrics, - &peerset_protocol_names, + ¬ification_sinks, ); } } @@ -702,7 +837,6 @@ where async fn run_network_in( bridge: NetworkBridgeRx, mut ctx: Context, - network_stream: BoxStream<'static, NetworkEvent>, ) -> Result<(), Error> where N: Network, @@ -715,16 +849,21 @@ where sync_oracle, shared, peerset_protocol_names, + validation_service, + collation_service, + notification_sinks, } = bridge; let (task, network_event_handler) = handle_network_messages( ctx.sender().clone(), network_service.clone(), - network_stream, authority_discovery_service.clone(), metrics.clone(), shared.clone(), peerset_protocol_names.clone(), + validation_service, + collation_service, + notification_sinks.clone(), ) .remote_handle(); @@ -733,12 +872,11 @@ where let orchestra_signal_handler = run_incoming_orchestra_signals( ctx, - network_service, authority_discovery_service, shared, sync_oracle, metrics, - peerset_protocol_names, + notification_sinks, ); futures::pin_mut!(orchestra_signal_handler); @@ -758,17 +896,14 @@ fn construct_view( } #[overseer::contextbounds(NetworkBridgeRx, prefix = self::overseer)] -fn update_our_view( - net: &mut Net, +fn update_our_view( ctx: &mut Context, live_heads: &[ActivatedLeaf], shared: &Shared, finalized_number: BlockNumber, metrics: &Metrics, - peerset_protocol_names: &PeerSetProtocolNames, -) where - Net: Network, -{ + notification_sinks: &Arc>>>, +) { let new_view = construct_view(live_heads.iter().map(|v| v.hash), finalized_number); let (validation_peers, collation_peers) = { @@ -817,35 +952,31 @@ fn update_our_view( filter_by_version(&collation_peers, ValidationVersion::VStaging.into()); send_validation_message_v1( - net, v1_validation_peers, - peerset_protocol_names, WireMessage::ViewUpdate(new_view.clone()), metrics, + notification_sinks, ); send_collation_message_v1( - net, v1_collation_peers, - peerset_protocol_names, WireMessage::ViewUpdate(new_view.clone()), metrics, + notification_sinks, ); send_validation_message_vstaging( - net, vstaging_validation_peers, - peerset_protocol_names, WireMessage::ViewUpdate(new_view.clone()), metrics, + notification_sinks, ); send_collation_message_vstaging( - net, vstaging_collation_peers, - peerset_protocol_names, WireMessage::ViewUpdate(new_view), metrics, + notification_sinks, ); let our_view = OurView::new( @@ -918,74 +1049,66 @@ fn handle_peer_messages>( } fn send_validation_message_v1( - net: &mut impl Network, peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { send_message( - net, peers, PeerSet::Validation, ValidationVersion::V1.into(), - peerset_protocol_names, message, metrics, + notification_sinks, ); } fn send_collation_message_v1( - net: &mut impl Network, peers: Vec, - peerset_protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { send_message( - net, peers, PeerSet::Collation, CollationVersion::V1.into(), - peerset_protocol_names, message, metrics, + notification_sinks, ); } fn send_validation_message_vstaging( - net: &mut impl Network, peers: Vec, - protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { send_message( - net, peers, PeerSet::Validation, ValidationVersion::VStaging.into(), - protocol_names, message, metrics, + notification_sinks, ); } fn send_collation_message_vstaging( - net: &mut impl Network, peers: Vec, - protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { send_message( - net, peers, PeerSet::Collation, CollationVersion::VStaging.into(), - protocol_names, message, metrics, + notification_sinks, ); } diff --git a/polkadot/node/network/bridge/src/rx/tests.rs b/polkadot/node/network/bridge/src/rx/tests.rs index 88a4b247fdc6..3161801aa2c2 100644 --- a/polkadot/node/network/bridge/src/rx/tests.rs +++ b/polkadot/node/network/bridge/src/rx/tests.rs @@ -15,7 +15,7 @@ // along with Polkadot. If not, see . use super::*; -use futures::{channel::oneshot, executor, stream::BoxStream}; +use futures::{channel::oneshot, executor}; use polkadot_node_network_protocol::{self as net_protocol, OurView}; use polkadot_node_subsystem::{messages::NetworkBridgeEvent, ActivatedLeaf}; @@ -28,7 +28,11 @@ use std::{ task::Poll, }; -use sc_network::{Event as NetworkEvent, IfDisconnected, ProtocolName, ReputationChange}; +use sc_network::{ + service::traits::{Direction, MessageSink, NotificationService}, + IfDisconnected, Multiaddr, ObservedRole as SubstrateObservedRole, ProtocolName, + ReputationChange, Roles, +}; use polkadot_node_network_protocol::{ peer_set::PeerSetProtocolNames, @@ -49,7 +53,6 @@ use polkadot_node_subsystem_test_helpers::{ use polkadot_node_subsystem_util::metered; use polkadot_primitives::{AuthorityDiscoveryId, CandidateHash, Hash}; -use sc_network::Multiaddr; use sp_keyring::Sr25519Keyring; use crate::{network::Network, validator_discovery::AuthorityDiscovery}; @@ -64,10 +67,9 @@ pub enum NetworkAction { WriteNotification(PeerId, PeerSet, Vec), } -// The subsystem's view of the network - only supports a single call to `event_stream`. +// The subsystem's view of the network. #[derive(Clone)] struct TestNetwork { - net_events: Arc>>>, action_tx: Arc>>, protocol_names: Arc, } @@ -79,37 +81,42 @@ struct TestAuthorityDiscovery; // of `NetworkAction`s. struct TestNetworkHandle { action_rx: metered::UnboundedMeteredReceiver, - net_tx: SingleItemSink, - protocol_names: PeerSetProtocolNames, + validation_tx: SingleItemSink, + collation_tx: SingleItemSink, } fn new_test_network( protocol_names: PeerSetProtocolNames, -) -> (TestNetwork, TestNetworkHandle, TestAuthorityDiscovery) { - let (net_tx, net_rx) = polkadot_node_subsystem_test_helpers::single_item_sink(); +) -> ( + TestNetwork, + TestNetworkHandle, + TestAuthorityDiscovery, + Box, + Box, +) { let (action_tx, action_rx) = metered::unbounded(); + let (validation_tx, validation_rx) = polkadot_node_subsystem_test_helpers::single_item_sink(); + let (collation_tx, collation_rx) = polkadot_node_subsystem_test_helpers::single_item_sink(); + let action_tx = Arc::new(Mutex::new(action_tx)); ( TestNetwork { - net_events: Arc::new(Mutex::new(Some(net_rx))), - action_tx: Arc::new(Mutex::new(action_tx)), + action_tx: action_tx.clone(), protocol_names: Arc::new(protocol_names.clone()), }, - TestNetworkHandle { action_rx, net_tx, protocol_names }, + TestNetworkHandle { action_rx, validation_tx, collation_tx }, TestAuthorityDiscovery, + Box::new(TestNotificationService::new( + PeerSet::Validation, + action_tx.clone(), + validation_rx, + )), + Box::new(TestNotificationService::new(PeerSet::Collation, action_tx, collation_rx)), ) } #[async_trait] impl Network for TestNetwork { - fn event_stream(&mut self) -> BoxStream<'static, NetworkEvent> { - self.net_events - .lock() - .take() - .expect("Subsystem made more than one call to `event_stream`") - .boxed() - } - async fn set_reserved_peers( &mut self, _protocol: ProtocolName, @@ -143,7 +150,8 @@ impl Network for TestNetwork { } fn disconnect_peer(&self, who: PeerId, protocol: ProtocolName) { - let (peer_set, _) = self.protocol_names.try_get_protocol(&protocol).unwrap(); + let (peer_set, version) = self.protocol_names.try_get_protocol(&protocol).unwrap(); + assert_eq!(version, peer_set.get_main_version()); self.action_tx .lock() @@ -151,13 +159,10 @@ impl Network for TestNetwork { .unwrap(); } - fn write_notification(&self, who: PeerId, protocol: ProtocolName, message: Vec) { - let (peer_set, _) = self.protocol_names.try_get_protocol(&protocol).unwrap(); - - self.action_tx - .lock() - .unbounded_send(NetworkAction::WriteNotification(who, peer_set, message)) - .unwrap(); + fn peer_role(&self, _peer_id: PeerId, handshake: Vec) -> Option { + Roles::decode_all(&mut &handshake[..]) + .ok() + .and_then(|role| Some(SubstrateObservedRole::from(role))) } } @@ -194,42 +199,93 @@ impl TestNetworkHandle { v } + // async fn connect_peer( + // &mut self, + // peer: PeerId, + // protocol_version: ValidationVersion, + // peer_set: PeerSet, + // role: ObservedRole, + // ) { + // // let protocol_version = ProtocolVersion::from(protocol_version); + // // self.send_network_event(NetworkEvent::NotificationStreamOpened { + // // remote: peer, + // // protocol: self.protocol_names.get_name(peer_set, protocol_version), + // // negotiated_fallback: None, + // // role: role.into(), + // // received_handshake: vec![], + // // }) + // // .await; + // } + async fn connect_peer( &mut self, peer: PeerId, - protocol_version: ValidationVersion, + _protocol_version: ValidationVersion, peer_set: PeerSet, role: ObservedRole, ) { - let protocol_version = ProtocolVersion::from(protocol_version); - self.send_network_event(NetworkEvent::NotificationStreamOpened { - remote: peer, - protocol: self.protocol_names.get_name(peer_set, protocol_version), - negotiated_fallback: None, - role: role.into(), - received_handshake: vec![], - }) - .await; + fn observed_role_to_handshake(role: &ObservedRole) -> Vec { + match role { + &ObservedRole::Light => Roles::LIGHT.encode(), + &ObservedRole::Authority => Roles::AUTHORITY.encode(), + &ObservedRole::Full => Roles::FULL.encode(), + } + } + + match peer_set { + PeerSet::Validation => { + self.validation_tx + .send(NotificationEvent::NotificationStreamOpened { + peer, + direction: Direction::Inbound, + handshake: observed_role_to_handshake(&role), + negotiated_fallback: None, + }) + .await + .expect("subsystem concluded early"); + }, + PeerSet::Collation => { + self.collation_tx + .send(NotificationEvent::NotificationStreamOpened { + peer, + direction: Direction::Inbound, + handshake: observed_role_to_handshake(&role), + negotiated_fallback: None, + }) + .await + .expect("subsystem concluded early"); + }, + } } async fn disconnect_peer(&mut self, peer: PeerId, peer_set: PeerSet) { - self.send_network_event(NetworkEvent::NotificationStreamClosed { - remote: peer, - protocol: self.protocol_names.get_main_name(peer_set), - }) - .await; + match peer_set { + PeerSet::Validation => self + .validation_tx + .send(NotificationEvent::NotificationStreamClosed { peer }) + .await + .expect("subsystem concluded early"), + PeerSet::Collation => self + .collation_tx + .send(NotificationEvent::NotificationStreamClosed { peer }) + .await + .expect("subsystem concluded early"), + } } async fn peer_message(&mut self, peer: PeerId, peer_set: PeerSet, message: Vec) { - self.send_network_event(NetworkEvent::NotificationsReceived { - remote: peer, - messages: vec![(self.protocol_names.get_main_name(peer_set), message.into())], - }) - .await; - } - - async fn send_network_event(&mut self, event: NetworkEvent) { - self.net_tx.send(event).await.expect("subsystem concluded early"); + match peer_set { + PeerSet::Validation => self + .validation_tx + .send(NotificationEvent::NotificationReceived { peer, notification: message }) + .await + .expect("subsystem concluded early"), + PeerSet::Collation => self + .collation_tx + .send(NotificationEvent::NotificationReceived { peer, notification: message }) + .await + .expect("subsystem concluded early"), + } } } @@ -240,6 +296,119 @@ fn assert_network_actions_contains(actions: &[NetworkAction], action: &NetworkAc } } +struct TestNotificationService { + peer_set: PeerSet, + action_tx: Arc>>, + rx: SingleItemStream, +} + +impl std::fmt::Debug for TestNotificationService { + fn fmt(&self, _: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + Ok(()) + } +} + +impl TestNotificationService { + pub fn new( + peer_set: PeerSet, + action_tx: Arc>>, + rx: SingleItemStream, + ) -> Self { + Self { peer_set, action_tx, rx } + } +} + +struct TestMessageSink { + peer: PeerId, + peer_set: PeerSet, + action_tx: Arc>>, +} + +impl TestMessageSink { + fn new( + peer: PeerId, + peer_set: PeerSet, + action_tx: Arc>>, + ) -> TestMessageSink { + Self { peer, peer_set, action_tx } + } +} + +#[async_trait::async_trait] +impl MessageSink for TestMessageSink { + fn send_sync_notification(&self, notification: Vec) { + self.action_tx + .lock() + .unbounded_send(NetworkAction::WriteNotification( + self.peer, + self.peer_set, + notification, + )) + .unwrap(); + } + + async fn send_async_notification( + &self, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } +} + +// TODO: how to implement this? +#[async_trait::async_trait] +impl NotificationService for TestNotificationService { + /// Instruct `Notifications` to open a new substream for `peer`. + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Instruct `Notifications` to close substream for `peer`. + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification(&self, _peer: &PeerId, _notification: Vec) { + unimplemented!(); + } + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + async fn send_async_notification( + &self, + _peer: &PeerId, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option { + self.rx.next().await + } + + // Clone [`NotificationService`] + fn clone(&mut self) -> Result, ()> { + unimplemented!(); + } + + /// Get protocol name. + fn protocol(&self) -> &ProtocolName { + unimplemented!(); + } + + /// Get notification sink of the peer. + fn message_sink(&self, peer: &PeerId) -> Option> { + // TODO: verify that peer exists? + Some(Box::new(TestMessageSink::new(*peer, self.peer_set, self.action_tx.clone()))) + } +} + #[derive(Clone)] struct TestSyncOracle { is_major_syncing: Arc, @@ -334,11 +503,14 @@ fn test_harness>( let fork_id = None; let peerset_protocol_names = PeerSetProtocolNames::new(genesis_hash, fork_id); + println!("protocol names: {peerset_protocol_names:#?}"); + let pool = sp_core::testing::TaskExecutor::new(); - let (mut network, network_handle, discovery) = new_test_network(peerset_protocol_names.clone()); + let (network, network_handle, discovery, validation_service, collation_service) = + new_test_network(peerset_protocol_names.clone()); let (context, virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context(pool); - let network_stream = network.event_stream(); + let notification_sinks = Arc::new(Mutex::new(HashMap::new())); let shared = Shared::default(); let bridge = NetworkBridgeRx { @@ -348,9 +520,12 @@ fn test_harness>( sync_oracle, shared: shared.clone(), peerset_protocol_names, + validation_service, + collation_service, + notification_sinks, }; - let network_bridge = run_network_in(bridge, context, network_stream) + let network_bridge = run_network_in(bridge, context) .map_err(|_| panic!("subsystem execution failed")) .map(|_| ()); @@ -972,8 +1147,6 @@ fn relays_collation_protocol_messages() { .await; } - // peer A gets reported for sending a collation message. - let collator_protocol_message = protocol_v1::CollatorProtocolMessage::Declare( Sr25519Keyring::Alice.public().into(), Default::default(), @@ -983,19 +1156,23 @@ fn relays_collation_protocol_messages() { let message_v1 = protocol_v1::CollationProtocol::CollatorProtocol(collator_protocol_message.clone()); - network_handle - .peer_message( - peer_a, - PeerSet::Collation, - WireMessage::ProtocolMessage(message_v1.clone()).encode(), - ) - .await; - - let actions = network_handle.next_network_actions(3).await; - assert_network_actions_contains( - &actions, - &NetworkAction::ReputationChange(peer_a, UNCONNECTED_PEERSET_COST.into()), - ); + // peer A gets reported for sending a collation message. + // NOTE: this is not possible since peer A cannot send + // a collation message if it has not opened a collation protocol + + // network_handle + // .peer_message( + // peer_a, + // PeerSet::Collation, + // WireMessage::ProtocolMessage(message_v1.clone()).encode(), + // ) + // .await; + + // let actions = network_handle.next_network_actions(3).await; + // assert_network_actions_contains( + // &actions, + // &NetworkAction::ReputationChange(peer_a, UNCONNECTED_PEERSET_COST.into()), + // ); // peer B has the message relayed. @@ -1252,7 +1429,7 @@ fn our_view_updates_decreasing_order_and_limited_to_max() { fn network_protocol_versioning_view_update() { let (oracle, handle) = make_sync_oracle(false); test_harness(Box::new(oracle), |test_harness| async move { - let TestHarness { mut network_handle, mut virtual_overseer, .. } = test_harness; + let TestHarness { mut network_handle, mut virtual_overseer, shared } = test_harness; let peer_ids: Vec<_> = (0..4).map(|_| PeerId::random()).collect(); let peers = [ @@ -1276,12 +1453,22 @@ fn network_protocol_versioning_view_update() { handle.await_mode_switch().await; + let mut total_validation_peers = 0; + let mut total_collation_peers = 0; + for &(peer_id, peer_set, version) in &peers { network_handle .connect_peer(peer_id, version, peer_set, ObservedRole::Full) .await; + + match peer_set { + PeerSet::Validation => total_validation_peers += 1, + PeerSet::Collation => total_collation_peers += 1, + } } + await_peer_connections(&shared, total_validation_peers, total_collation_peers).await; + let view = view![head]; let actions = network_handle.next_network_actions(4).await; @@ -1304,11 +1491,13 @@ fn network_protocol_versioning_view_update() { }); } +// TODO: discuss with parachains engineering #[test] +#[cfg(feature = "network-protocol-staging")] fn network_protocol_versioning_subsystem_msg() { let (oracle, _handle) = make_sync_oracle(false); test_harness(Box::new(oracle), |test_harness| async move { - let TestHarness { mut network_handle, mut virtual_overseer, .. } = test_harness; + let TestHarness { mut network_handle, mut virtual_overseer, shared } = test_harness; let peer = PeerId::random(); @@ -1320,6 +1509,7 @@ fn network_protocol_versioning_subsystem_msg() { ObservedRole::Full, ) .await; + await_peer_connections(&shared, 1, 0).await; // bridge will inform about all connected peers. { diff --git a/polkadot/node/network/bridge/src/tx/mod.rs b/polkadot/node/network/bridge/src/tx/mod.rs index 1b386ce1239b..2cbe7883d38a 100644 --- a/polkadot/node/network/bridge/src/tx/mod.rs +++ b/polkadot/node/network/bridge/src/tx/mod.rs @@ -33,7 +33,7 @@ use polkadot_node_subsystem::{ /// /// To be passed to [`FullNetworkConfiguration::add_notification_protocol`](). pub use polkadot_node_network_protocol::peer_set::{peer_sets_info, IsAuthority}; -use sc_network::ReputationChange; +use sc_network::{MessageSink, ReputationChange}; use crate::validator_discovery; @@ -58,6 +58,7 @@ pub struct NetworkBridgeTx { metrics: Metrics, req_protocol_names: ReqProtocolNames, peerset_protocol_names: PeerSetProtocolNames, + notification_sinks: Arc>>>, } impl NetworkBridgeTx { @@ -72,6 +73,7 @@ impl NetworkBridgeTx { metrics: Metrics, req_protocol_names: ReqProtocolNames, peerset_protocol_names: PeerSetProtocolNames, + notification_sinks: Arc>>>, ) -> Self { Self { network_service, @@ -79,6 +81,7 @@ impl NetworkBridgeTx { metrics, req_protocol_names, peerset_protocol_names, + notification_sinks, } } } @@ -105,6 +108,7 @@ async fn handle_subsystem_messages( metrics: Metrics, req_protocol_names: ReqProtocolNames, peerset_protocol_names: PeerSetProtocolNames, + notification_sinks: Arc>>>, ) -> Result<(), Error> where N: Network, @@ -128,6 +132,7 @@ where &metrics, &req_protocol_names, &peerset_protocol_names, + ¬ification_sinks, ) .await; }, @@ -138,13 +143,14 @@ where #[overseer::contextbounds(NetworkBridgeTx, prefix = self::overseer)] async fn handle_incoming_subsystem_communication( _ctx: &mut Context, - mut network_service: N, + network_service: N, validator_discovery: &mut validator_discovery::Service, mut authority_discovery_service: AD, msg: NetworkBridgeTxMessage, metrics: &Metrics, req_protocol_names: &ReqProtocolNames, peerset_protocol_names: &PeerSetProtocolNames, + notification_sinks: &Arc>>>, ) -> (N, AD) where N: Network, @@ -191,18 +197,16 @@ where match msg { Versioned::V1(msg) => send_validation_message_v1( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), Versioned::VStaging(msg) => send_validation_message_vstaging( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), } }, @@ -216,18 +220,16 @@ where for (peers, msg) in msgs { match msg { Versioned::V1(msg) => send_validation_message_v1( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), Versioned::VStaging(msg) => send_validation_message_vstaging( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), } } @@ -241,18 +243,16 @@ where match msg { Versioned::V1(msg) => send_collation_message_v1( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), Versioned::VStaging(msg) => send_collation_message_vstaging( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), } }, @@ -266,18 +266,16 @@ where for (peers, msg) in msgs { match msg { Versioned::V1(msg) => send_collation_message_v1( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), Versioned::VStaging(msg) => send_collation_message_vstaging( - &mut network_service, peers, - peerset_protocol_names, WireMessage::ProtocolMessage(msg), &metrics, + notification_sinks, ), } } @@ -359,6 +357,7 @@ where metrics, req_protocol_names, peerset_protocol_names, + notification_sinks, } = bridge; handle_subsystem_messages( @@ -368,6 +367,7 @@ where metrics, req_protocol_names, peerset_protocol_names, + notification_sinks, ) .await?; @@ -375,73 +375,65 @@ where } fn send_validation_message_v1( - net: &mut impl Network, peers: Vec, - protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { send_message( - net, peers, PeerSet::Validation, ValidationVersion::V1.into(), - protocol_names, message, metrics, + notification_sinks, ); } fn send_collation_message_v1( - net: &mut impl Network, peers: Vec, - protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { send_message( - net, peers, PeerSet::Collation, CollationVersion::V1.into(), - protocol_names, message, metrics, + notification_sinks, ); } fn send_validation_message_vstaging( - net: &mut impl Network, peers: Vec, - protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { send_message( - net, peers, PeerSet::Validation, ValidationVersion::VStaging.into(), - protocol_names, message, metrics, + notification_sinks, ); } fn send_collation_message_vstaging( - net: &mut impl Network, peers: Vec, - protocol_names: &PeerSetProtocolNames, message: WireMessage, metrics: &Metrics, + notification_sinks: &Arc>>>, ) { send_message( - net, peers, PeerSet::Collation, CollationVersion::VStaging.into(), - protocol_names, message, metrics, + notification_sinks, ); } diff --git a/polkadot/node/network/bridge/src/tx/tests.rs b/polkadot/node/network/bridge/src/tx/tests.rs index 21cd134c54f2..a19df51dc575 100644 --- a/polkadot/node/network/bridge/src/tx/tests.rs +++ b/polkadot/node/network/bridge/src/tx/tests.rs @@ -15,15 +15,18 @@ // along with Polkadot. If not, see . use super::*; -use futures::{executor, stream::BoxStream}; +use futures::executor; use polkadot_node_subsystem_util::TimeoutExt; use async_trait::async_trait; use parking_lot::Mutex; use std::collections::HashSet; -use sc_network::{Event as NetworkEvent, IfDisconnected, ProtocolName, ReputationChange}; +use sc_network::{ + IfDisconnected, ObservedRole as SubstrateObservedRole, ProtocolName, ReputationChange, Roles, +}; +use parity_scale_codec::DecodeAll; use polkadot_node_network_protocol::{ peer_set::PeerSetProtocolNames, request_response::{outgoing::Requests, ReqProtocolNames}, @@ -51,10 +54,9 @@ pub enum NetworkAction { WriteNotification(PeerId, PeerSet, Vec), } -// The subsystem's view of the network - only supports a single call to `event_stream`. +// The subsystem's view of the network. #[derive(Clone)] struct TestNetwork { - net_events: Arc>>>, action_tx: Arc>>, peerset_protocol_names: Arc, } @@ -66,37 +68,98 @@ struct TestAuthorityDiscovery; // of `NetworkAction`s. struct TestNetworkHandle { action_rx: metered::UnboundedMeteredReceiver, - net_tx: metered::MeteredSender, - peerset_protocol_names: PeerSetProtocolNames, + _peerset_protocol_names: PeerSetProtocolNames, + notification_sinks: Arc>>>, + action_tx: Arc>>, +} + +struct TestMessageSink { + peer: PeerId, + peer_set: PeerSet, + action_tx: Arc>>, +} + +impl TestMessageSink { + fn new( + peer: PeerId, + peer_set: PeerSet, + action_tx: Arc>>, + ) -> TestMessageSink { + Self { peer, peer_set, action_tx } + } +} + +#[async_trait::async_trait] +impl MessageSink for TestMessageSink { + fn send_sync_notification(&self, notification: Vec) { + self.action_tx + .lock() + .unbounded_send(NetworkAction::WriteNotification( + self.peer, + self.peer_set, + notification, + )) + .unwrap(); + } + + async fn send_async_notification( + &self, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } } fn new_test_network( peerset_protocol_names: PeerSetProtocolNames, -) -> (TestNetwork, TestNetworkHandle, TestAuthorityDiscovery) { - let (net_tx, net_rx) = metered::channel(10); +) -> ( + TestNetwork, + TestNetworkHandle, + TestAuthorityDiscovery, + Arc>>>, +) { let (action_tx, action_rx) = metered::unbounded(); + let notification_sinks = Arc::new(Mutex::new(HashMap::new())); + let action_tx = Arc::new(Mutex::new(action_tx)); ( TestNetwork { - net_events: Arc::new(Mutex::new(Some(net_rx))), - action_tx: Arc::new(Mutex::new(action_tx)), + action_tx: action_tx.clone(), peerset_protocol_names: Arc::new(peerset_protocol_names.clone()), }, - TestNetworkHandle { action_rx, net_tx, peerset_protocol_names }, + TestNetworkHandle { + action_rx, + _peerset_protocol_names: peerset_protocol_names, + action_tx, + notification_sinks: notification_sinks.clone(), + }, TestAuthorityDiscovery, + notification_sinks, ) } -#[async_trait] -impl Network for TestNetwork { - fn event_stream(&mut self) -> BoxStream<'static, NetworkEvent> { - self.net_events - .lock() - .take() - .expect("Subsystem made more than one call to `event_stream`") - .boxed() +// wait until all needed validation and collation peers have connected. +async fn await_peer_connections( + shared: &Shared, + num_validation_peers: usize, + num_collation_peers: usize, +) { + loop { + { + let shared = shared.0.lock(); + if shared.validation_peers.len() == num_validation_peers && + shared.collation_peers.len() == num_collation_peers + { + break + } + } + + futures_timer::Delay::new(std::time::Duration::from_millis(100)).await; } +} +#[async_trait] +impl Network for TestNetwork { async fn set_reserved_peers( &mut self, _protocol: ProtocolName, @@ -130,7 +193,8 @@ impl Network for TestNetwork { } fn disconnect_peer(&self, who: PeerId, protocol: ProtocolName) { - let (peer_set, _) = self.peerset_protocol_names.try_get_protocol(&protocol).unwrap(); + let (peer_set, version) = self.peerset_protocol_names.try_get_protocol(&protocol).unwrap(); + assert_eq!(version, peer_set.get_main_version()); self.action_tx .lock() @@ -138,13 +202,10 @@ impl Network for TestNetwork { .unwrap(); } - fn write_notification(&self, who: PeerId, protocol: ProtocolName, message: Vec) { - let (peer_set, _) = self.peerset_protocol_names.try_get_protocol(&protocol).unwrap(); - - self.action_tx - .lock() - .unbounded_send(NetworkAction::WriteNotification(who, peer_set, message)) - .unwrap(); + fn peer_role(&self, _peer_id: PeerId, handshake: Vec) -> Option { + Roles::decode_all(&mut &handshake[..]) + .ok() + .and_then(|role| Some(SubstrateObservedRole::from(role))) } } @@ -174,23 +235,14 @@ impl TestNetworkHandle { async fn connect_peer( &mut self, peer: PeerId, - protocol_version: ValidationVersion, + _protocol_version: ValidationVersion, peer_set: PeerSet, - role: ObservedRole, + _role: ObservedRole, ) { - let protocol_version = ProtocolVersion::from(protocol_version); - self.send_network_event(NetworkEvent::NotificationStreamOpened { - remote: peer, - protocol: self.peerset_protocol_names.get_name(peer_set, protocol_version), - negotiated_fallback: None, - role: role.into(), - received_handshake: vec![], - }) - .await; - } - - async fn send_network_event(&mut self, event: NetworkEvent) { - self.net_tx.send(event).await.expect("subsystem concluded early"); + self.notification_sinks.lock().insert( + (peer_set, peer), + Box::new(TestMessageSink::new(peer, peer_set, self.action_tx.clone())), + ); } } @@ -208,7 +260,8 @@ fn test_harness>(test: impl FnOnce(TestHarne let peerset_protocol_names = PeerSetProtocolNames::new(genesis_hash, fork_id); let pool = sp_core::testing::TaskExecutor::new(); - let (network, network_handle, discovery) = new_test_network(peerset_protocol_names.clone()); + let (network, network_handle, discovery, network_notification_sinks) = + new_test_network(peerset_protocol_names.clone()); let (context, virtual_overseer) = polkadot_node_subsystem_test_helpers::make_subsystem_context(pool); @@ -219,6 +272,7 @@ fn test_harness>(test: impl FnOnce(TestHarne Metrics(None), req_protocol_names, peerset_protocol_names, + network_notification_sinks, ); let network_bridge_out_fut = run_network_out(bridge_out, context) @@ -334,7 +388,9 @@ fn send_messages_to_peers() { }); } +// TODO: discuss with parachains engineering #[test] +#[cfg(feature = "network-protocol-staging")] fn network_protocol_versioning_send() { test_harness(|test_harness| async move { let TestHarness { mut network_handle, mut virtual_overseer } = test_harness; diff --git a/polkadot/node/network/bridge/src/validator_discovery.rs b/polkadot/node/network/bridge/src/validator_discovery.rs index 86e861fbc5b5..b11af8a8a089 100644 --- a/polkadot/node/network/bridge/src/validator_discovery.rs +++ b/polkadot/node/network/bridge/src/validator_discovery.rs @@ -169,13 +169,12 @@ mod tests { use crate::network::Network; use async_trait::async_trait; - use futures::stream::BoxStream; use polkadot_node_network_protocol::{ request_response::{outgoing::Requests, ReqProtocolNames}, PeerId, }; use polkadot_primitives::Hash; - use sc_network::{Event as NetworkEvent, IfDisconnected, ProtocolName, ReputationChange}; + use sc_network::{IfDisconnected, ProtocolName, ReputationChange}; use sp_keyring::Sr25519Keyring; use std::collections::{HashMap, HashSet}; @@ -224,10 +223,6 @@ mod tests { #[async_trait] impl Network for TestNetwork { - fn event_stream(&mut self) -> BoxStream<'static, NetworkEvent> { - panic!() - } - async fn set_reserved_peers( &mut self, _protocol: ProtocolName, @@ -263,7 +258,11 @@ mod tests { panic!() } - fn write_notification(&self, _: PeerId, _: ProtocolName, _: Vec) { + fn peer_role( + &self, + _peer_id: PeerId, + _handshake: Vec, + ) -> Option { panic!() } } diff --git a/polkadot/node/network/protocol/src/peer_set.rs b/polkadot/node/network/protocol/src/peer_set.rs index c2163783c2ce..9db5d4771d4b 100644 --- a/polkadot/node/network/protocol/src/peer_set.rs +++ b/polkadot/node/network/protocol/src/peer_set.rs @@ -21,6 +21,7 @@ use polkadot_primitives::Hash; use sc_network::{ config::{NonDefaultSetConfig, SetConfig}, types::ProtocolName, + NotificationService, }; use std::{ collections::{hash_map::Entry, HashMap}, @@ -68,7 +69,7 @@ impl PeerSet { self, is_authority: IsAuthority, peerset_protocol_names: &PeerSetProtocolNames, - ) -> NonDefaultSetConfig { + ) -> (NonDefaultSetConfig, (PeerSet, Box)) { // Networking layer relies on `get_main_name()` being the main name of the protocol // for peersets and connection management. let protocol = peerset_protocol_names.get_main_name(self); @@ -76,39 +77,47 @@ impl PeerSet { let max_notification_size = self.get_max_notification_size(is_authority); match self { - PeerSet::Validation => NonDefaultSetConfig { - notifications_protocol: protocol, - fallback_names, - max_notification_size, - handshake: None, - set_config: SetConfig { - // we allow full nodes to connect to validators for gossip - // to ensure any `MIN_GOSSIP_PEERS` always include reserved peers - // we limit the amount of non-reserved slots to be less - // than `MIN_GOSSIP_PEERS` in total - in_peers: super::MIN_GOSSIP_PEERS as u32 / 2 - 1, - out_peers: super::MIN_GOSSIP_PEERS as u32 / 2 - 1, - reserved_nodes: Vec::new(), - non_reserved_mode: sc_network::config::NonReservedPeerMode::Accept, - }, + PeerSet::Validation => { + let (config, notification_service) = NonDefaultSetConfig::new( + protocol, + fallback_names, + max_notification_size, + None, + SetConfig { + // we allow full nodes to connect to validators for gossip + // to ensure any `MIN_GOSSIP_PEERS` always include reserved peers + // we limit the amount of non-reserved slots to be less + // than `MIN_GOSSIP_PEERS` in total + in_peers: super::MIN_GOSSIP_PEERS as u32 / 2 - 1, + out_peers: super::MIN_GOSSIP_PEERS as u32 / 2 - 1, + reserved_nodes: Vec::new(), + non_reserved_mode: sc_network::config::NonReservedPeerMode::Accept, + }, + ); + + (config, (PeerSet::Validation, notification_service)) }, - PeerSet::Collation => NonDefaultSetConfig { - notifications_protocol: protocol, - fallback_names, - max_notification_size, - handshake: None, - set_config: SetConfig { - // Non-authority nodes don't need to accept incoming connections on this peer - // set: - in_peers: if is_authority == IsAuthority::Yes { 100 } else { 0 }, - out_peers: 0, - reserved_nodes: Vec::new(), - non_reserved_mode: if is_authority == IsAuthority::Yes { - sc_network::config::NonReservedPeerMode::Accept - } else { - sc_network::config::NonReservedPeerMode::Deny + PeerSet::Collation => { + let (config, notification_service) = NonDefaultSetConfig::new( + protocol, + fallback_names, + max_notification_size, + None, + SetConfig { + // Non-authority nodes don't need to accept incoming connections on this + // peer set: + in_peers: if is_authority == IsAuthority::Yes { 100 } else { 0 }, + out_peers: 0, + reserved_nodes: Vec::new(), + non_reserved_mode: if is_authority == IsAuthority::Yes { + sc_network::config::NonReservedPeerMode::Accept + } else { + sc_network::config::NonReservedPeerMode::Deny + }, }, - }, + ); + + (config, (PeerSet::Collation, notification_service)) }, } } @@ -202,7 +211,7 @@ impl IndexMut for PerPeerSet { pub fn peer_sets_info( is_authority: IsAuthority, peerset_protocol_names: &PeerSetProtocolNames, -) -> Vec { +) -> Vec<(NonDefaultSetConfig, (PeerSet, Box))> { PeerSet::iter() .map(|s| s.get_info(is_authority, &peerset_protocol_names)) .collect() @@ -281,7 +290,7 @@ impl From for ProtocolVersion { } /// On the wire protocol name to [`PeerSet`] mapping. -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct PeerSetProtocolNames { protocols: HashMap, names: HashMap<(PeerSet, ProtocolVersion), ProtocolName>, diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index 90881aa051a0..f9ebb03d497b 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -84,6 +84,7 @@ kvdb = "0.13.0" kvdb-rocksdb = { version = "0.19.0", optional = true } parity-db = { version = "0.4.8", optional = true } codec = { package = "parity-scale-codec", version = "3.6.1" } +parking_lot = "0.12.1" async-trait = "0.1.57" schnellru = "0.2.1" diff --git a/polkadot/node/service/src/lib.rs b/polkadot/node/service/src/lib.rs index 95c887947c98..7d10a212ae30 100644 --- a/polkadot/node/service/src/lib.rs +++ b/polkadot/node/service/src/lib.rs @@ -51,7 +51,8 @@ use { }, polkadot_node_core_dispute_coordinator::Config as DisputeCoordinatorConfig, polkadot_node_network_protocol::{ - peer_set::PeerSetProtocolNames, request_response::ReqProtocolNames, + peer_set::{PeerSet, PeerSetProtocolNames}, + request_response::ReqProtocolNames, }, sc_client_api::BlockBackend, sc_transaction_pool_api::OffchainTransactionPoolFactory, @@ -74,7 +75,7 @@ pub use { #[cfg(feature = "full-node")] use polkadot_node_subsystem::jaeger; -use std::{path::PathBuf, sync::Arc, time::Duration}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; use prometheus_endpoint::Registry; #[cfg(feature = "full-node")] @@ -810,9 +811,9 @@ pub fn new_full( // anything in terms of behaviour, but makes the logs more consistent with the other // Substrate nodes. let grandpa_protocol_name = grandpa::protocol_standard_name(&genesis_hash, &config.chain_spec); - net_config.add_notification_protocol(grandpa::grandpa_peers_set_config( - grandpa_protocol_name.clone(), - )); + let (grandpa_protocol_config, grandpa_notification_service) = + grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone()); + net_config.add_notification_protocol(grandpa_protocol_config); let beefy_gossip_proto_name = beefy::gossip_protocol_name(&genesis_hash, config.chain_spec.fork_id()); @@ -825,12 +826,17 @@ pub fn new_full( client.clone(), prometheus_registry.clone(), ); - if enable_beefy { - net_config.add_notification_protocol(beefy::communication::beefy_peers_set_config( - beefy_gossip_proto_name.clone(), - )); - net_config.add_request_response_protocol(beefy_req_resp_cfg); - } + let beefy_notification_service = match enable_beefy { + false => None, + true => { + let (beefy_notification_config, beefy_notification_service) = + beefy::communication::beefy_peers_set_config(beefy_gossip_proto_name.clone()); + + net_config.add_notification_protocol(beefy_notification_config); + net_config.add_request_response_protocol(beefy_req_resp_cfg); + Some(beefy_notification_service) + }, + }; // validation/collation protocols are enabled only if `Overseer` is enabled let peerset_protocol_names = @@ -841,13 +847,21 @@ pub fn new_full( // // Collators and parachain full nodes require the collator and validator networking to send // collations and to be able to recover PoVs. - if role.is_authority() || is_parachain_node.is_running_alongside_parachain_node() { - use polkadot_network_bridge::{peer_sets_info, IsAuthority}; - let is_authority = if role.is_authority() { IsAuthority::Yes } else { IsAuthority::No }; - for config in peer_sets_info(is_authority, &peerset_protocol_names) { - net_config.add_notification_protocol(config); - } - } + let notification_services = + if role.is_authority() || is_parachain_node.is_running_alongside_parachain_node() { + use polkadot_network_bridge::{peer_sets_info, IsAuthority}; + let is_authority = if role.is_authority() { IsAuthority::Yes } else { IsAuthority::No }; + + peer_sets_info(is_authority, &peerset_protocol_names) + .into_iter() + .map(|(config, (peerset, service))| { + net_config.add_notification_protocol(config); + (peerset, service) + }) + .collect::>>() + } else { + std::collections::HashMap::new() + }; let req_protocol_names = ReqProtocolNames::new(&genesis_hash, config.chain_spec.fork_id()); @@ -1074,6 +1088,7 @@ pub fn new_full( offchain_transaction_pool_factory: OffchainTransactionPoolFactory::new( transaction_pool.clone(), ), + notification_services, }, ) .map_err(|e| { @@ -1182,6 +1197,8 @@ pub fn new_full( sync: sync_service.clone(), gossip_protocol_name: beefy_gossip_proto_name, justifications_protocol_name, + notification_service: beefy_notification_service + .expect("BEEFY was enabled so `NotificationService` exists; qed"), _phantom: core::marker::PhantomData::, }; let payload_provider = beefy_primitives::mmr::MmrRootProvider::new(client.clone()); @@ -1279,6 +1296,7 @@ pub fn new_full( prometheus_registry: prometheus_registry.clone(), shared_voter_state, telemetry: telemetry.as_ref().map(|x| x.handle()), + notification_service: grandpa_notification_service, offchain_tx_pool_factory: OffchainTransactionPoolFactory::new(transaction_pool.clone()), }; diff --git a/polkadot/node/service/src/overseer.rs b/polkadot/node/service/src/overseer.rs index 33127b638e5a..99265541c0bb 100644 --- a/polkadot/node/service/src/overseer.rs +++ b/polkadot/node/service/src/overseer.rs @@ -26,7 +26,7 @@ use polkadot_node_core_candidate_validation::Config as CandidateValidationConfig use polkadot_node_core_chain_selection::Config as ChainSelectionConfig; use polkadot_node_core_dispute_coordinator::Config as DisputeCoordinatorConfig; use polkadot_node_network_protocol::{ - peer_set::PeerSetProtocolNames, + peer_set::{PeerSet, PeerSetProtocolNames}, request_response::{ v1 as request_v1, vstaging as request_vstaging, IncomingRequestReceiver, ReqProtocolNames, }, @@ -42,15 +42,16 @@ use polkadot_overseer::{ }; use schnellru::{ByLength, LruMap}; +use parking_lot::Mutex; use polkadot_primitives::runtime_api::ParachainHost; use sc_authority_discovery::Service as AuthorityDiscoveryService; use sc_client_api::AuxStore; use sc_keystore::LocalKeystore; -use sc_network::NetworkStateInfo; +use sc_network::{NetworkStateInfo, NotificationService}; use sp_api::ProvideRuntimeApi; use sp_blockchain::HeaderBackend; use sp_consensus_babe::BabeApi; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; pub use polkadot_approval_distribution::ApprovalDistribution as ApprovalDistributionSubsystem; pub use polkadot_availability_bitfield_distribution::BitfieldDistribution as BitfieldDistributionSubsystem; @@ -143,6 +144,8 @@ where pub peerset_protocol_names: PeerSetProtocolNames, /// The offchain transaction pool factory. pub offchain_transaction_pool_factory: OffchainTransactionPoolFactory, + /// Notification services for validation/collation protocols. + pub notification_services: HashMap>, } /// Obtain a prepared `OverseerBuilder`, that is initialized @@ -176,6 +179,7 @@ pub fn prepared_overseer_builder( req_protocol_names, peerset_protocol_names, offchain_transaction_pool_factory, + notification_services, }: OverseerGenArgs, ) -> Result< InitializedOverseerBuilder< @@ -221,6 +225,7 @@ where use polkadot_node_subsystem_util::metrics::Metrics; let metrics = ::register(registry)?; + let notification_sinks = Arc::new(Mutex::new(HashMap::new())); let spawner = SpawnGlue(spawner); @@ -238,6 +243,7 @@ where network_bridge_metrics.clone(), req_protocol_names, peerset_protocol_names.clone(), + notification_sinks.clone(), )) .network_bridge_rx(NetworkBridgeRxSubsystem::new( network_service.clone(), @@ -245,6 +251,8 @@ where Box::new(sync_service.clone()), network_bridge_metrics, peerset_protocol_names, + notification_services, + notification_sinks, )) .availability_distribution(AvailabilityDistributionSubsystem::new( keystore.clone(), diff --git a/substrate/bin/node-template/node/src/service.rs b/substrate/bin/node-template/node/src/service.rs index 7303f5cd6dd6..f80dd1bb73a6 100644 --- a/substrate/bin/node-template/node/src/service.rs +++ b/substrate/bin/node-template/node/src/service.rs @@ -163,9 +163,9 @@ pub fn new_full(config: Configuration) -> Result { &client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"), &config.chain_spec, ); - net_config.add_notification_protocol(sc_consensus_grandpa::grandpa_peers_set_config( - grandpa_protocol_name.clone(), - )); + let (grandpa_protocol_config, grandpa_notification_service) = + sc_consensus_grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone()); + net_config.add_notification_protocol(grandpa_protocol_config); let warp_sync = Arc::new(sc_consensus_grandpa::warp_proof::NetworkProvider::new( backend.clone(), @@ -315,6 +315,7 @@ pub fn new_full(config: Configuration) -> Result { link: grandpa_link, network, sync: Arc::new(sync_service), + notification_service: grandpa_notification_service, voting_rule: sc_consensus_grandpa::VotingRulesBuilder::default().build(), prometheus_registry, shared_voter_state: SharedVoterState::empty(), diff --git a/substrate/bin/node/cli/src/service.rs b/substrate/bin/node/cli/src/service.rs index ecca5c60db51..6ed890c8a7b3 100644 --- a/substrate/bin/node/cli/src/service.rs +++ b/substrate/bin/node/cli/src/service.rs @@ -359,19 +359,20 @@ pub fn new_full_base( &client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"), &config.chain_spec, ); - net_config.add_notification_protocol(grandpa::grandpa_peers_set_config( - grandpa_protocol_name.clone(), - )); - - let statement_handler_proto = sc_network_statement::StatementHandlerPrototype::new( - client - .block_hash(0u32.into()) - .ok() - .flatten() - .expect("Genesis block exists; qed"), - config.chain_spec.fork_id(), - ); - net_config.add_notification_protocol(statement_handler_proto.set_config()); + let (grandpa_protocol_config, grandpa_notification_service) = + grandpa::grandpa_peers_set_config(grandpa_protocol_name.clone()); + net_config.add_notification_protocol(grandpa_protocol_config); + + let (statement_handler_proto, statement_config) = + sc_network_statement::StatementHandlerPrototype::new( + client + .block_hash(0u32.into()) + .ok() + .flatten() + .expect("Genesis block exists; qed"), + config.chain_spec.fork_id(), + ); + net_config.add_notification_protocol(statement_config); let warp_sync = Arc::new(grandpa::warp_proof::NetworkProvider::new( backend.clone(), @@ -551,6 +552,7 @@ pub fn new_full_base( link: grandpa_link, network: network.clone(), sync: Arc::new(sync_service.clone()), + notification_service: grandpa_notification_service, telemetry: telemetry.as_ref().map(|x| x.handle()), voting_rule: grandpa::VotingRulesBuilder::default().build(), prometheus_registry: prometheus_registry.clone(), diff --git a/substrate/client/consensus/beefy/src/communication/mod.rs b/substrate/client/consensus/beefy/src/communication/mod.rs index 7f9535bfc23f..d92d3be7d687 100644 --- a/substrate/client/consensus/beefy/src/communication/mod.rs +++ b/substrate/client/consensus/beefy/src/communication/mod.rs @@ -67,10 +67,16 @@ pub(crate) mod beefy_protocol_name { /// For standard protocol name see [`beefy_protocol_name::gossip_protocol_name`]. pub fn beefy_peers_set_config( gossip_protocol_name: sc_network::ProtocolName, -) -> sc_network::config::NonDefaultSetConfig { - let mut cfg = sc_network::config::NonDefaultSetConfig::new(gossip_protocol_name, 1024 * 1024); +) -> (sc_network::config::NonDefaultSetConfig, Box) { + let (mut cfg, notification_service) = sc_network::config::NonDefaultSetConfig::new( + gossip_protocol_name, + Vec::new(), + 1024 * 1024, + None, + Default::default(), + ); cfg.allow_non_reserved(25, 25); - cfg + (cfg, notification_service) } // cost scalars for reporting peers. diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs index 72df3cab8550..2b62dec4a626 100644 --- a/substrate/client/consensus/beefy/src/lib.rs +++ b/substrate/client/consensus/beefy/src/lib.rs @@ -38,7 +38,7 @@ use parking_lot::Mutex; use prometheus::Registry; use sc_client_api::{Backend, BlockBackend, BlockchainEvents, FinalityNotifications, Finalizer}; use sc_consensus::BlockImport; -use sc_network::{NetworkRequest, ProtocolName}; +use sc_network::{NetworkRequest, NotificationService, ProtocolName}; use sc_network_gossip::{GossipEngine, Network as GossipNetwork, Syncing as GossipSyncing}; use sp_api::{HeaderT, NumberFor, ProvideRuntimeApi}; use sp_blockchain::{ @@ -178,6 +178,8 @@ pub struct BeefyNetworkParams { pub network: Arc, /// Syncing service implementing a sync oracle and an event stream for peers. pub sync: Arc, + /// Handle for receiving notification events. + pub notification_service: Box, /// Chain specific BEEFY gossip protocol name. See /// [`communication::beefy_protocol_name::gossip_protocol_name`]. pub gossip_protocol_name: ProtocolName, @@ -243,6 +245,7 @@ pub async fn start_beefy_gadget( let BeefyNetworkParams { network, sync, + notification_service, gossip_protocol_name, justifications_protocol_name, .. @@ -264,6 +267,7 @@ pub async fn start_beefy_gadget( let gossip_engine = GossipEngine::new( network.clone(), sync.clone(), + notification_service, gossip_protocol_name.clone(), gossip_validator.clone(), None, diff --git a/substrate/client/consensus/beefy/src/tests.rs b/substrate/client/consensus/beefy/src/tests.rs index 3bb65e9d57f4..5bc183387599 100644 --- a/substrate/client/consensus/beefy/src/tests.rs +++ b/substrate/client/consensus/beefy/src/tests.rs @@ -71,7 +71,7 @@ use substrate_test_runtime_client::{BlockBuilderExt, ClientExt}; use tokio::time::Duration; const GENESIS_HASH: H256 = H256::zero(); -fn beefy_gossip_proto_name() -> ProtocolName { +pub(crate) fn beefy_gossip_proto_name() -> ProtocolName { gossip_protocol_name(GENESIS_HASH, None) } @@ -370,6 +370,7 @@ async fn voter_init_setup( let mut gossip_engine = sc_network_gossip::GossipEngine::new( net.peer(0).network_service().clone(), net.peer(0).sync_service().clone(), + net.peer(0).take_notification_service(&beefy_gossip_proto_name()).unwrap(), "/beefy/whatever", gossip_validator, None, @@ -391,6 +392,14 @@ where { let tasks = FuturesUnordered::new(); + let mut notification_services = peers + .iter() + .map(|(peer_id, _, _)| { + let peer = &mut net.peers[*peer_id]; + (*peer_id, peer.take_notification_service(&beefy_gossip_proto_name()).unwrap()) + }) + .collect::>(); + for (peer_id, key, api) in peers.into_iter() { let peer = &net.peers[peer_id]; @@ -408,6 +417,7 @@ where let network_params = crate::BeefyNetworkParams { network: peer.network_service().clone(), sync: peer.sync_service().clone(), + notification_service: notification_services.remove(&peer_id).unwrap(), gossip_protocol_name: beefy_gossip_proto_name(), justifications_protocol_name: on_demand_justif_handler.protocol_name(), _phantom: PhantomData, @@ -1031,7 +1041,25 @@ async fn should_initialize_voter_at_custom_genesis() { net.peer(0).client().as_client().finalize_block(hashes[8], None).unwrap(); // load persistent state - nothing in DB, should init at genesis - let persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap(); + // + // NOTE: code from `voter_init_setup()` is moved here because the new network event system + // doesn't allow creating a new `GossipEngine` as the notification handle is consumed by the + // first `GossipEngine` + let known_peers = Arc::new(Mutex::new(KnownPeers::new())); + let (gossip_validator, _) = GossipValidator::new(known_peers); + let gossip_validator = Arc::new(gossip_validator); + let mut gossip_engine = sc_network_gossip::GossipEngine::new( + net.peer(0).network_service().clone(), + net.peer(0).sync_service().clone(), + net.peer(0).take_notification_service(&beefy_gossip_proto_name()).unwrap(), + "/beefy/whatever", + gossip_validator, + None, + ); + let (beefy_genesis, best_grandpa) = + wait_for_runtime_pallet(&api, &mut gossip_engine, &mut finality).await.unwrap(); + let persisted_state = + load_or_init_voter_state(&*backend, &api, beefy_genesis, best_grandpa, 1).unwrap(); // Test initialization at session boundary. // verify voter initialized with single session starting at block `custom_pallet_genesis` (7) @@ -1061,7 +1089,11 @@ async fn should_initialize_voter_at_custom_genesis() { net.peer(0).client().as_client().finalize_block(hashes[10], None).unwrap(); // load persistent state - state preset in DB, but with different pallet genesis - let new_persisted_state = voter_init_setup(&mut net, &mut finality, &api).await.unwrap(); + // the network state persists and uses the old `GossipEngine` initialized for `peer(0)` + let (beefy_genesis, best_grandpa) = + wait_for_runtime_pallet(&api, &mut gossip_engine, &mut finality).await.unwrap(); + let new_persisted_state = + load_or_init_voter_state(&*backend, &api, beefy_genesis, best_grandpa, 1).unwrap(); // verify voter initialized with single session starting at block `new_pallet_genesis` (10) let sessions = new_persisted_state.voting_oracle().sessions(); @@ -1309,7 +1341,7 @@ async fn gossipped_finality_proofs() { let api = Arc::new(TestApi::with_validator_set(&validator_set)); let beefy_peers = peers.iter().enumerate().map(|(id, key)| (id, key, api.clone())).collect(); - let charlie = &net.peers[2]; + let charlie = &mut net.peers[2]; let known_peers = Arc::new(Mutex::new(KnownPeers::::new())); // Charlie will run just the gossip engine and not the full voter. let (gossip_validator, _) = GossipValidator::new(known_peers); @@ -1322,6 +1354,7 @@ async fn gossipped_finality_proofs() { let mut charlie_gossip_engine = sc_network_gossip::GossipEngine::new( charlie.network_service().clone(), charlie.sync_service().clone(), + charlie.take_notification_service(&beefy_gossip_proto_name()).unwrap(), beefy_gossip_proto_name(), charlie_gossip_validator.clone(), None, diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs index a239e34030c2..a50fdab28ec4 100644 --- a/substrate/client/consensus/beefy/src/worker.rs +++ b/substrate/client/consensus/beefy/src/worker.rs @@ -1131,12 +1131,16 @@ pub(crate) mod tests { let api = Arc::new(TestApi::with_validator_set(&genesis_validator_set)); let network = peer.network_service().clone(); let sync = peer.sync_service().clone(); + let notification_service = peer + .take_notification_service(&crate::tests::beefy_gossip_proto_name()) + .unwrap(); let known_peers = Arc::new(Mutex::new(KnownPeers::new())); let (gossip_validator, gossip_report_stream) = GossipValidator::new(known_peers.clone()); let gossip_validator = Arc::new(gossip_validator); let gossip_engine = GossipEngine::new( network.clone(), sync.clone(), + notification_service, "/beefy/1", gossip_validator.clone(), None, diff --git a/substrate/client/consensus/grandpa/src/communication/mod.rs b/substrate/client/consensus/grandpa/src/communication/mod.rs index c0749858568f..22785639e1f6 100644 --- a/substrate/client/consensus/grandpa/src/communication/mod.rs +++ b/substrate/client/consensus/grandpa/src/communication/mod.rs @@ -46,7 +46,7 @@ use finality_grandpa::{ Message::{Precommit, Prevote, PrimaryPropose}, }; use parity_scale_codec::{Decode, DecodeAll, Encode}; -use sc_network::{NetworkBlock, NetworkSyncForkRequest, ReputationChange}; +use sc_network::{NetworkBlock, NetworkSyncForkRequest, NotificationService, ReputationChange}; use sc_network_gossip::{GossipEngine, Network as GossipNetwork}; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO}; use sp_keystore::KeystorePtr; @@ -247,6 +247,7 @@ impl, S: Syncing> NetworkBridge { pub(crate) fn new( service: N, sync: S, + notification_service: Box, config: crate::Config, set_state: crate::environment::SharedVoterSetState, prometheus_registry: Option<&Registry>, @@ -260,6 +261,7 @@ impl, S: Syncing> NetworkBridge { let gossip_engine = Arc::new(Mutex::new(GossipEngine::new( service.clone(), sync.clone(), + notification_service, protocol, validator.clone(), prometheus_registry, diff --git a/substrate/client/consensus/grandpa/src/communication/tests.rs b/substrate/client/consensus/grandpa/src/communication/tests.rs index 10c4772fc76d..3fbc954d89cf 100644 --- a/substrate/client/consensus/grandpa/src/communication/tests.rs +++ b/substrate/client/consensus/grandpa/src/communication/tests.rs @@ -24,17 +24,18 @@ use super::{ }; use crate::{communication::grandpa_protocol_name, environment::SharedVoterSetState}; use futures::prelude::*; -use parity_scale_codec::Encode; +use parity_scale_codec::{DecodeAll, Encode}; use sc_network::{ config::{MultiaddrWithPeerId, Role}, event::Event as NetworkEvent, + service::traits::{Direction, MessageSink, NotificationEvent, NotificationService}, types::ProtocolName, Multiaddr, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, NetworkSyncForkRequest, NotificationSenderError, NotificationSenderT as NotificationSender, PeerId, ReputationChange, }; use sc_network_common::{ - role::ObservedRole, + role::{ObservedRole, Roles}, sync::{SyncEvent as SyncStreamEvent, SyncEventStream}, }; use sc_network_gossip::Validator; @@ -125,6 +126,12 @@ impl NetworkPeers for TestNetwork { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, handshake: Vec) -> Option { + Roles::decode_all(&mut &handshake[..]) + .ok() + .and_then(|role| Some(ObservedRole::from(role))) + } } impl NetworkEventStream for TestNetwork { @@ -213,10 +220,65 @@ impl NetworkSyncForkRequest> for TestSync { fn set_sync_fork_request(&self, _peers: Vec, _hash: Hash, _number: NumberFor) {} } +#[derive(Debug)] +pub(crate) struct TestNotificationService { + rx: TracingUnboundedReceiver, +} + +#[async_trait::async_trait] +impl NotificationService for TestNotificationService { + /// Instruct `Notifications` to open a new substream for `peer`. + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Instruct `Notifications` to close substream for `peer`. + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification(&self, _peer: &PeerId, _notification: Vec) { + unimplemented!(); + } + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + async fn send_async_notification( + &self, + _peer: &PeerId, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option { + self.rx.next().await + } + + fn clone(&mut self) -> Result, ()> { + unimplemented!(); + } + + fn protocol(&self) -> &ProtocolName { + unimplemented!(); + } + + fn message_sink(&self, _peer: &PeerId) -> Option> { + unimplemented!(); + } +} + pub(crate) struct Tester { pub(crate) net_handle: super::NetworkBridge, gossip_validator: Arc>, pub(crate) events: TracingUnboundedReceiver, + pub(crate) notification_tx: TracingUnboundedSender, } impl Tester { @@ -281,6 +343,9 @@ fn voter_set_state() -> SharedVoterSetState { // needs to run in a tokio runtime. pub(crate) fn make_test_network() -> (impl Future, TestNetwork) { let (tx, rx) = tracing_unbounded("test", 100_000); + let (notification_tx, notification_rx) = tracing_unbounded("test-notification", 100_000); + + let notification_service = TestNotificationService { rx: notification_rx }; let net = TestNetwork { sender: tx }; let sync = TestSync {}; @@ -295,14 +360,22 @@ pub(crate) fn make_test_network() -> (impl Future, TestNetwork) } } - let bridge = - super::NetworkBridge::new(net.clone(), sync, config(), voter_set_state(), None, None); + let bridge = super::NetworkBridge::new( + net.clone(), + sync, + Box::new(notification_service), + config(), + voter_set_state(), + None, + None, + ); ( futures::future::ready(Tester { gossip_validator: bridge.validator.clone(), net_handle: bridge, events: rx, + notification_tx, }), net, ) @@ -387,63 +460,62 @@ fn good_commit_leads_to_relay() { let commit_to_send = encoded_commit.clone(); let network_bridge = tester.net_handle.clone(); - // asking for global communication will cause the test network - // to send us an event asking us for a stream. use it to - // send a message. + // `NetworkBridge` will be operational as soon as it's created and it's + // waiting for events from the network. Send it events that inform that + // a notification stream was opened and that a notification was received. + // + // Since each protocol has its own notification stream, events need not be filtered. let sender_id = id; - let send_message = tester.filter_network_events(move |event| match event { - Event::EventStream(sender) => { - // Add the sending peer and send the commit - let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { - remote: sender_id, - protocol: grandpa_protocol_name::NAME.into(), + + let send_message = async move { + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationStreamOpened { + peer: sender_id, + direction: Direction::Inbound, + negotiated_fallback: None, + handshake: Roles::FULL.encode(), + }, + ); + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationReceived { + peer: sender_id, + notification: commit_to_send.clone(), + }, + ); + + // Add a random peer which will be the recipient of this message + let receiver_id = PeerId::random(); + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationStreamOpened { + peer: receiver_id, + direction: Direction::Inbound, negotiated_fallback: None, - role: ObservedRole::Full, - received_handshake: vec![], + handshake: Roles::FULL.encode(), + }, + ); + + // Announce its local set being on the current set id through a neighbor + // packet, otherwise it won't be eligible to receive the commit + let _ = { + let update = gossip::VersionedNeighborPacket::V1(gossip::NeighborPacket { + round: Round(round), + set_id: SetId(set_id), + commit_finalized_height: 1, }); - let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived { - remote: sender_id, - messages: vec![( - grandpa_protocol_name::NAME.into(), - commit_to_send.clone().into(), - )], - }); + let msg = gossip::GossipMessage::::Neighbor(update); - // Add a random peer which will be the recipient of this message - let receiver_id = PeerId::random(); - let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { - remote: receiver_id, - protocol: grandpa_protocol_name::NAME.into(), - negotiated_fallback: None, - role: ObservedRole::Full, - received_handshake: vec![], - }); + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationReceived { + peer: receiver_id, + notification: msg.encode(), + }, + ); + }; - // Announce its local set has being on the current set id through a neighbor - // packet, otherwise it won't be eligible to receive the commit - let _ = { - let update = gossip::VersionedNeighborPacket::V1(gossip::NeighborPacket { - round: Round(round), - set_id: SetId(set_id), - commit_finalized_height: 1, - }); - - let msg = gossip::GossipMessage::::Neighbor(update); - - sender.unbounded_send(NetworkEvent::NotificationsReceived { - remote: receiver_id, - messages: vec![( - grandpa_protocol_name::NAME.into(), - msg.encode().into(), - )], - }) - }; - - true - }, - _ => false, - }); + tester + } + .boxed(); // when the commit comes in, we'll tell the callback it was good. let handle_commit = commits_in.into_future().map(|(item, _)| match item.unwrap() { @@ -539,31 +611,32 @@ fn bad_commit_leads_to_report() { let commit_to_send = encoded_commit.clone(); let network_bridge = tester.net_handle.clone(); - // asking for global communication will cause the test network - // to send us an event asking us for a stream. use it to - // send a message. + // `NetworkBridge` will be operational as soon as it's created and it's + // waiting for events from the network. Send it events that inform that + // a notification stream was opened and that a notification was received. + // + // Since each protocol has its own notification stream, events need not be filtered. let sender_id = id; - let send_message = tester.filter_network_events(move |event| match event { - Event::EventStream(sender) => { - let _ = sender.unbounded_send(NetworkEvent::NotificationStreamOpened { - remote: sender_id, - protocol: grandpa_protocol_name::NAME.into(), + + let send_message = async move { + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationStreamOpened { + peer: sender_id, + direction: Direction::Inbound, negotiated_fallback: None, - role: ObservedRole::Full, - received_handshake: vec![], - }); - let _ = sender.unbounded_send(NetworkEvent::NotificationsReceived { - remote: sender_id, - messages: vec![( - grandpa_protocol_name::NAME.into(), - commit_to_send.clone().into(), - )], - }); + handshake: Roles::FULL.encode(), + }, + ); + let _ = tester.notification_tx.unbounded_send( + NotificationEvent::NotificationReceived { + peer: sender_id, + notification: commit_to_send.clone(), + }, + ); - true - }, - _ => false, - }); + tester + } + .boxed(); // when the commit comes in, we'll tell the callback it was bad. let handle_commit = commits_in.into_future().map(|(item, _)| match item.unwrap() { diff --git a/substrate/client/consensus/grandpa/src/lib.rs b/substrate/client/consensus/grandpa/src/lib.rs index da621abd254c..34e5b375de8e 100644 --- a/substrate/client/consensus/grandpa/src/lib.rs +++ b/substrate/client/consensus/grandpa/src/lib.rs @@ -67,7 +67,7 @@ use sc_client_api::{ BlockchainEvents, CallExecutor, ExecutorProvider, Finalizer, LockImportRun, StorageProvider, }; use sc_consensus::BlockImport; -use sc_network::types::ProtocolName; +use sc_network::{types::ProtocolName, NotificationService}; use sc_telemetry::{telemetry, TelemetryHandle, CONSENSUS_DEBUG, CONSENSUS_INFO}; use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver}; @@ -690,6 +690,8 @@ pub struct GrandpaParams { pub network: N, /// Event stream for syncing-related events. pub sync: S, + /// Handle for interacting with `Notifications`. + pub notification_service: Box, /// A voting rule used to potentially restrict target votes. pub voting_rule: VR, /// The prometheus metrics registry. @@ -710,21 +712,21 @@ pub struct GrandpaParams { /// For standard protocol name see [`crate::protocol_standard_name`]. pub fn grandpa_peers_set_config( protocol_name: ProtocolName, -) -> sc_network::config::NonDefaultSetConfig { +) -> (sc_network::config::NonDefaultSetConfig, Box) { use communication::grandpa_protocol_name; - sc_network::config::NonDefaultSetConfig { - notifications_protocol: protocol_name, - fallback_names: grandpa_protocol_name::LEGACY_NAMES.iter().map(|&n| n.into()).collect(), + sc_network::config::NonDefaultSetConfig::new( + protocol_name, + grandpa_protocol_name::LEGACY_NAMES.iter().map(|&n| n.into()).collect(), // Notifications reach ~256kiB in size at the time of writing on Kusama and Polkadot. - max_notification_size: 1024 * 1024, - handshake: None, - set_config: sc_network::config::SetConfig { + 1024 * 1024, + None, + sc_network::config::SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: sc_network::config::NonReservedPeerMode::Deny, }, - } + ) } /// Run a GRANDPA voter as a task. Provide configuration and a link to a @@ -747,6 +749,7 @@ where link, network, sync, + notification_service, voting_rule, prometheus_registry, shared_voter_state, @@ -773,6 +776,7 @@ where let network = NetworkBridge::new( network, sync, + notification_service, config.clone(), persistent_data.set_state.clone(), prometheus_registry.as_ref(), diff --git a/substrate/client/consensus/grandpa/src/observer.rs b/substrate/client/consensus/grandpa/src/observer.rs index 8541baa822bb..608ff5e46a0e 100644 --- a/substrate/client/consensus/grandpa/src/observer.rs +++ b/substrate/client/consensus/grandpa/src/observer.rs @@ -28,6 +28,7 @@ use futures::prelude::*; use log::{debug, info, warn}; use sc_client_api::backend::Backend; +use sc_network::NotificationService; use sc_telemetry::TelemetryHandle; use sc_utils::mpsc::TracingUnboundedReceiver; use sp_blockchain::HeaderMetadata; @@ -168,6 +169,7 @@ pub fn run_grandpa_observer( link: LinkHalf, network: N, sync: S, + notification_service: Box, ) -> sp_blockchain::Result + Send> where BE: Backend + Unpin + 'static, @@ -189,6 +191,7 @@ where let network = NetworkBridge::new( network, sync, + notification_service, config.clone(), persistent_data.set_state.clone(), None, @@ -414,14 +417,14 @@ mod tests { use futures::executor; - /// Ensure `Future` implementation of `ObserverWork` is polling its `NetworkBridge`. Regression - /// test for bug introduced in d4fbb897c and fixed in b7af8b339. + /// Ensure `Future` implementation of `ObserverWork` is polling its `NetworkBridge`. + /// Regression test for bug introduced in d4fbb897c and fixed in b7af8b339. /// - /// When polled, `NetworkBridge` forwards reputation change requests from the `GossipValidator` - /// to the underlying `dyn Network`. This test triggers a reputation change by calling - /// `GossipValidator::validate` with an invalid gossip message. After polling the `ObserverWork` - /// which should poll the `NetworkBridge`, the reputation change should be forwarded to the test - /// network. + /// When polled, `NetworkBridge` forwards reputation change requests from the + /// `GossipValidator` to the underlying `dyn Network`. This test triggers a reputation change + /// by calling `GossipValidator::validate` with an invalid gossip message. After polling the + /// `ObserverWork` which should poll the `NetworkBridge`, the reputation change should be + /// forwarded to the test network. #[test] fn observer_work_polls_underlying_network_bridge() { // Create a test network. @@ -463,12 +466,6 @@ mod tests { // validator to the test network. assert!(observer.now_or_never().is_none()); - // Ignore initial event stream request by gossip engine. - match tester.events.next().now_or_never() { - Some(Some(Event::EventStream(_))) => {}, - _ => panic!("expected event stream request"), - }; - assert_matches!(tester.events.next().now_or_never(), Some(Some(Event::Report(_, _)))); }); } diff --git a/substrate/client/consensus/grandpa/src/tests.rs b/substrate/client/consensus/grandpa/src/tests.rs index 0175f7d1b473..6fa3f3637671 100644 --- a/substrate/client/consensus/grandpa/src/tests.rs +++ b/substrate/client/consensus/grandpa/src/tests.rs @@ -317,6 +317,9 @@ fn initialize_grandpa( (net.peers[peer_id].network_service().clone(), link) }; let sync = net.peers[peer_id].sync_service().clone(); + let notification_service = net.peers[peer_id] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(); let grandpa_params = GrandpaParams { config: Config { @@ -332,6 +335,7 @@ fn initialize_grandpa( link, network: net_service, sync, + notification_service, voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -472,6 +476,9 @@ async fn finalize_3_voters_1_full_observer() { let net_service = net.peers[peer_id].network_service().clone(); let sync = net.peers[peer_id].sync_service().clone(); let link = net.peers[peer_id].data.lock().take().expect("link initialized at startup; qed"); + let notification_service = net.peers[peer_id] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(); let grandpa_params = GrandpaParams { config: Config { @@ -487,6 +494,7 @@ async fn finalize_3_voters_1_full_observer() { link, network: net_service, sync, + notification_service, voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -557,14 +565,17 @@ async fn transition_3_voters_twice_1_full_observer() { for (peer_id, local_key) in all_peers.clone().into_iter().enumerate() { let keystore = create_keystore(local_key); - let (net_service, link, sync) = { - let net = net.lock(); + let (net_service, link, sync, notification_service) = { + let mut net = net.lock(); let link = net.peers[peer_id].data.lock().take().expect("link initialized at startup; qed"); ( net.peers[peer_id].network_service().clone(), link, net.peers[peer_id].sync_service().clone(), + net.peers[peer_id] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(), ) }; @@ -582,6 +593,7 @@ async fn transition_3_voters_twice_1_full_observer() { link, network: net_service, sync, + notification_service, voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1019,6 +1031,9 @@ async fn voter_persists_its_votes() { communication::NetworkBridge::new( net.peers[1].network_service().clone(), net.peers[1].sync_service().clone(), + net.peers[1] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(), config.clone(), set_state, None, @@ -1037,6 +1052,9 @@ async fn voter_persists_its_votes() { (net.peers[0].network_service().clone(), link) }; let sync = net.peers[0].sync_service().clone(); + let notification_service = net.peers[0] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(); let grandpa_params = GrandpaParams { config: Config { @@ -1052,6 +1070,7 @@ async fn voter_persists_its_votes() { link, network: net_service, sync, + notification_service, voting_rule: VotingRulesBuilder::default().build(), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1076,6 +1095,9 @@ async fn voter_persists_its_votes() { net.add_authority_peer(); let net_service = net.peers[2].network_service().clone(); let sync = net.peers[2].sync_service().clone(); + let notification_service = net.peers[2] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(); // but we'll reuse the client from the first peer (alice_voter1) // since we want to share the same database, so that we can // read the persisted state after aborting alice_voter1. @@ -1098,6 +1120,7 @@ async fn voter_persists_its_votes() { link, network: net_service, sync, + notification_service, voting_rule: VotingRulesBuilder::default().build(), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1249,6 +1272,9 @@ async fn finalize_3_voters_1_light_observer() { let mut net = GrandpaTestNet::new(TestApi::new(voters), 3, 1); let voters = initialize_grandpa(&mut net, authorities); + let notification_service = net.peers[3] + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(); let observer = observer::run_grandpa_observer( Config { gossip_duration: TEST_GOSSIP_DURATION, @@ -1263,6 +1289,7 @@ async fn finalize_3_voters_1_light_observer() { net.peers[3].data.lock().take().expect("link initialized at startup; qed"), net.peers[3].network_service().clone(), net.peers[3].sync_service().clone(), + notification_service, ) .unwrap(); net.peer(0).push_blocks(20, false); @@ -1311,6 +1338,10 @@ async fn voter_catches_up_to_latest_round_when_behind() { link, network: net.peer(peer_id).network_service().clone(), sync: net.peer(peer_id).sync_service().clone(), + notification_service: net + .peer(peer_id) + .take_notification_service(&grandpa_protocol_name::NAME.into()) + .unwrap(), voting_rule: (), prometheus_registry: None, shared_voter_state: SharedVoterState::empty(), @@ -1403,6 +1434,7 @@ fn test_environment_with_select_chain( keystore: Option, network_service: N, sync_service: S, + notification_service: Box, select_chain: SC, voting_rule: VR, ) -> TestEnvironment @@ -1427,6 +1459,7 @@ where let network = NetworkBridge::new( network_service.clone(), sync_service, + notification_service, config.clone(), set_state.clone(), None, @@ -1456,6 +1489,7 @@ fn test_environment( keystore: Option, network_service: N, sync_service: S, + notification_service: Box, voting_rule: VR, ) -> TestEnvironment, VR> where @@ -1468,6 +1502,7 @@ where keystore, network_service, sync_service, + notification_service, link.select_chain.clone(), voting_rule, ) @@ -1484,14 +1519,22 @@ async fn grandpa_environment_respects_voting_rules() { let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let mut notification_service = + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); // add 21 blocks let hashes = peer.push_blocks(21, false); // create an environment with no voting rule restrictions - let unrestricted_env = - test_environment(&link, None, network_service.clone(), sync_service.clone(), ()); + let unrestricted_env = test_environment( + &link, + None, + network_service.clone(), + sync_service.clone(), + notification_service.clone().unwrap(), + (), + ); // another with 3/4 unfinalized chain voting rule restriction let three_quarters_env = test_environment( @@ -1499,6 +1542,7 @@ async fn grandpa_environment_respects_voting_rules() { None, network_service.clone(), sync_service.clone(), + notification_service.clone().unwrap(), voting_rule::ThreeQuartersOfTheUnfinalizedChain, ); @@ -1509,6 +1553,7 @@ async fn grandpa_environment_respects_voting_rules() { None, network_service.clone(), sync_service, + notification_service, VotingRulesBuilder::default().build(), ); @@ -1602,6 +1647,8 @@ async fn grandpa_environment_passes_actual_best_block_to_voting_rules() { let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let notification_service = + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let client = peer.client().as_client().clone(); let select_chain = MockSelectChain::default(); @@ -1616,6 +1663,7 @@ async fn grandpa_environment_passes_actual_best_block_to_voting_rules() { None, network_service.clone(), sync_service, + notification_service, select_chain.clone(), voting_rule::BeforeBestBlockBy(5), ); @@ -1663,6 +1711,8 @@ async fn grandpa_environment_checks_if_best_block_is_descendent_of_finality_targ let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let notification_service = + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let client = peer.client().as_client().clone(); let select_chain = MockSelectChain::default(); @@ -1672,6 +1722,7 @@ async fn grandpa_environment_checks_if_best_block_is_descendent_of_finality_targ None, network_service.clone(), sync_service.clone(), + notification_service, select_chain.clone(), voting_rule.clone(), ); @@ -1774,11 +1825,19 @@ async fn grandpa_environment_never_overwrites_round_voter_state() { let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let notification_service = + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let keystore = create_keystore(peers[0]); - let environment = - test_environment(&link, Some(keystore), network_service.clone(), sync_service, ()); + let environment = test_environment( + &link, + Some(keystore), + network_service.clone(), + sync_service, + notification_service, + (), + ); let round_state = || finality_grandpa::round::State::genesis(Default::default()); let base = || Default::default(); @@ -2001,9 +2060,18 @@ async fn grandpa_environment_doesnt_send_equivocation_reports_for_itself() { let peer = net.peer(0); let network_service = peer.network_service().clone(); let sync_service = peer.sync_service().clone(); + let notification_service = + peer.take_notification_service(&grandpa_protocol_name::NAME.into()).unwrap(); let link = peer.data.lock().take().unwrap(); let keystore = create_keystore(alice); - test_environment(&link, Some(keystore), network_service.clone(), sync_service, ()) + test_environment( + &link, + Some(keystore), + network_service.clone(), + sync_service, + notification_service, + (), + ) }; let signed_prevote = { diff --git a/substrate/client/network-gossip/Cargo.toml b/substrate/client/network-gossip/Cargo.toml index 73d2d3fa051e..98c75fc2159d 100644 --- a/substrate/client/network-gossip/Cargo.toml +++ b/substrate/client/network-gossip/Cargo.toml @@ -28,5 +28,7 @@ sp-runtime = { path = "../../primitives/runtime" } [dev-dependencies] tokio = "1.22.0" +async-trait = "0.1.73" +codec = { package = "parity-scale-codec", version = "3.0.0", features = [ "derive" ] } quickcheck = { version = "1.0.3", default-features = false } substrate-test-runtime-client = { path = "../../test-utils/runtime/client" } diff --git a/substrate/client/network-gossip/src/bridge.rs b/substrate/client/network-gossip/src/bridge.rs index 6a3790ee2b2b..595cca9b773b 100644 --- a/substrate/client/network-gossip/src/bridge.rs +++ b/substrate/client/network-gossip/src/bridge.rs @@ -21,7 +21,11 @@ use crate::{ Network, Syncing, Validator, }; -use sc_network::{event::Event, types::ProtocolName, ReputationChange}; +use sc_network::{ + service::traits::{NotificationEvent, ValidationResult}, + types::ProtocolName, + NotificationService, ReputationChange, +}; use sc_network_common::sync::SyncEvent; use futures::{ @@ -48,10 +52,10 @@ pub struct GossipEngine { periodic_maintenance_interval: futures_timer::Delay, protocol: ProtocolName, - /// Incoming events from the network. - network_event_stream: Pin + Send>>, /// Incoming events from the syncing service. sync_event_stream: Pin + Send>>, + /// Handle for polling notification-related events. + notification_service: Box, /// Outgoing events to the consumer. message_sinks: HashMap>>, /// Buffered messages (see [`ForwardingState`]). @@ -81,6 +85,7 @@ impl GossipEngine { pub fn new( network: N, sync: S, + notification_service: Box, protocol: impl Into, validator: Arc>, metrics_registry: Option<&Registry>, @@ -91,17 +96,16 @@ impl GossipEngine { S: Syncing + Send + Clone + 'static, { let protocol = protocol.into(); - let network_event_stream = network.event_stream("network-gossip"); let sync_event_stream = sync.event_stream("network-gossip"); GossipEngine { state_machine: ConsensusGossip::new(validator, protocol.clone(), metrics_registry), network: Box::new(network), sync: Box::new(sync), + notification_service, periodic_maintenance_interval: futures_timer::Delay::new(PERIODIC_MAINTENANCE_INTERVAL), protocol, - network_event_stream, sync_event_stream, message_sinks: HashMap::new(), forwarding_state: ForwardingState::Idle, @@ -173,6 +177,11 @@ impl GossipEngine { pub fn announce(&self, block: B::Hash, associated_data: Option>) { self.sync.announce_block(block, associated_data); } + + /// Consume [`GossipEngine`] and return the notification service. + pub fn take_notification_service(self) -> Box { + self.notification_service + } } impl Future for GossipEngine { @@ -184,46 +193,40 @@ impl Future for GossipEngine { 'outer: loop { match &mut this.forwarding_state { ForwardingState::Idle => { - let net_event_stream = this.network_event_stream.poll_next_unpin(cx); + let next_notification_event = + this.notification_service.next_event().poll_unpin(cx); let sync_event_stream = this.sync_event_stream.poll_next_unpin(cx); - if net_event_stream.is_pending() && sync_event_stream.is_pending() { + if next_notification_event.is_pending() && sync_event_stream.is_pending() { break } - match net_event_stream { + match next_notification_event { Poll::Ready(Some(event)) => match event { - Event::NotificationStreamOpened { remote, protocol, role, .. } => - if protocol == this.protocol { - this.state_machine.new_peer(&mut *this.network, remote, role); - }, - Event::NotificationStreamClosed { remote, protocol } => { - if protocol == this.protocol { - this.state_machine - .peer_disconnected(&mut *this.network, remote); - } + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + let _ = result_tx.send(ValidationResult::Accept); }, - Event::NotificationsReceived { remote, messages } => { - let messages = messages - .into_iter() - .filter_map(|(engine, data)| { - if engine == this.protocol { - Some(data.to_vec()) - } else { - None - } - }) - .collect(); - + NotificationEvent::NotificationStreamOpened { + peer, handshake, .. + } => { + let Some(role) = this.network.peer_role(peer, handshake) else { + log::debug!(target: "gossip", "role for {peer} couldn't be determined"); + continue + }; + + this.state_machine.new_peer(&mut *this.network, peer, role); + }, + NotificationEvent::NotificationStreamClosed { peer } => { + this.state_machine.peer_disconnected(&mut *this.network, peer); + }, + NotificationEvent::NotificationReceived { peer, notification } => { let to_forward = this.state_machine.on_incoming( &mut *this.network, - remote, - messages, + peer, + vec![notification], ); - this.forwarding_state = ForwardingState::Busy(to_forward.into()); }, - Event::Dht(_) => {}, }, // The network event stream closed. Do the same for [`GossipValidator`]. Poll::Ready(None) => { @@ -328,15 +331,19 @@ impl futures::future::FusedFuture for GossipEngine { mod tests { use super::*; use crate::{multiaddr::Multiaddr, ValidationResult, ValidatorContext}; + use codec::{DecodeAll, Encode}; use futures::{ - channel::mpsc::{unbounded, UnboundedSender}, + channel::mpsc::{unbounded, UnboundedReceiver, UnboundedSender}, executor::{block_on, block_on_stream}, future::poll_fn, }; use quickcheck::{Arbitrary, Gen, QuickCheck}; use sc_network::{ - config::MultiaddrWithPeerId, NetworkBlock, NetworkEventStream, NetworkNotification, - NetworkPeers, NotificationSenderError, NotificationSenderT as NotificationSender, + config::MultiaddrWithPeerId, + service::traits::{Direction, MessageSink, NotificationEvent}, + Event, NetworkBlock, NetworkEventStream, NetworkNotification, NetworkPeers, + NotificationSenderError, NotificationSenderT as NotificationSender, NotificationService, + Roles, }; use sc_network_common::{role::ObservedRole, sync::SyncEventStream}; use sp_runtime::{ @@ -350,14 +357,10 @@ mod tests { use substrate_test_runtime_client::runtime::Block; #[derive(Clone, Default)] - struct TestNetwork { - inner: Arc>, - } + struct TestNetwork {} #[derive(Clone, Default)] - struct TestNetworkInner { - event_senders: Vec>, - } + struct TestNetworkInner {} impl NetworkPeers for TestNetwork { fn set_authorized_peers(&self, _peers: HashSet) { @@ -421,14 +424,17 @@ mod tests { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, handshake: Vec) -> Option { + Roles::decode_all(&mut &handshake[..]) + .ok() + .and_then(|role| Some(ObservedRole::from(role))) + } } impl NetworkEventStream for TestNetwork { fn event_stream(&self, _name: &'static str) -> Pin + Send>> { - let (tx, rx) = unbounded(); - self.inner.lock().unwrap().event_senders.push(tx); - - Box::pin(rx) + unimplemented!(); } } @@ -500,6 +506,54 @@ mod tests { } } + #[derive(Debug)] + pub(crate) struct TestNotificationService { + rx: UnboundedReceiver, + } + + #[async_trait::async_trait] + impl sc_network::service::traits::NotificationService for TestNotificationService { + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + unimplemented!(); + } + + fn send_sync_notification(&self, _peer: &PeerId, _notification: Vec) { + unimplemented!(); + } + + async fn send_async_notification( + &self, + _peer: &PeerId, + _notification: Vec, + ) -> Result<(), sc_network::error::Error> { + unimplemented!(); + } + + async fn set_handshake(&mut self, _handshake: Vec) -> Result<(), ()> { + unimplemented!(); + } + + async fn next_event(&mut self) -> Option { + self.rx.next().await + } + + fn clone(&mut self) -> Result, ()> { + unimplemented!(); + } + + fn protocol(&self) -> &ProtocolName { + unimplemented!(); + } + + fn message_sink(&self, _peer: &PeerId) -> Option> { + unimplemented!(); + } + } + struct AllowAll; impl Validator for AllowAll { fn validate( @@ -520,16 +574,19 @@ mod tests { fn returns_when_network_event_stream_closes() { let network = TestNetwork::default(); let sync = Arc::new(TestSync::default()); + let (tx, rx) = unbounded(); + let notification_service = Box::new(TestNotificationService { rx }); let mut gossip_engine = GossipEngine::::new( network.clone(), sync, + notification_service, "/my_protocol", Arc::new(AllowAll {}), None, ); - // Drop network event stream sender side. - drop(network.inner.lock().unwrap().event_senders.pop()); + // drop notification service sender side. + drop(tx); block_on(poll_fn(move |ctx| { if let Poll::Pending = gossip_engine.poll_unpin(ctx) { @@ -549,42 +606,37 @@ mod tests { let remote_peer = PeerId::random(); let network = TestNetwork::default(); let sync = Arc::new(TestSync::default()); + let (mut tx, rx) = unbounded(); + let notification_service = Box::new(TestNotificationService { rx }); let mut gossip_engine = GossipEngine::::new( network.clone(), sync.clone(), + notification_service, protocol.clone(), Arc::new(AllowAll {}), None, ); - let mut event_sender = network.inner.lock().unwrap().event_senders.pop().unwrap(); - // Register the remote peer. - event_sender - .start_send(Event::NotificationStreamOpened { - remote: remote_peer, - protocol: protocol.clone(), - negotiated_fallback: None, - role: ObservedRole::Authority, - received_handshake: vec![], - }) - .expect("Event stream is unbounded; qed."); + tx.send(NotificationEvent::NotificationStreamOpened { + peer: remote_peer, + direction: Direction::Inbound, + negotiated_fallback: None, + handshake: Roles::FULL.encode(), + }) + .await + .unwrap(); let messages = vec![vec![1], vec![2]]; - let events = messages - .iter() - .cloned() - .map(|m| Event::NotificationsReceived { - remote: remote_peer, - messages: vec![(protocol.clone(), m.into())], - }) - .collect::>(); // Send first event before subscribing. - event_sender - .start_send(events[0].clone()) - .expect("Event stream is unbounded; qed."); + tx.send(NotificationEvent::NotificationReceived { + peer: remote_peer, + notification: messages[0].clone().into(), + }) + .await + .unwrap(); let mut subscribers = vec![]; for _ in 0..2 { @@ -592,9 +644,12 @@ mod tests { } // Send second event after subscribing. - event_sender - .start_send(events[1].clone()) - .expect("Event stream is unbounded; qed."); + tx.send(NotificationEvent::NotificationReceived { + peer: remote_peer, + notification: messages[1].clone().into(), + }) + .await + .unwrap(); tokio::spawn(gossip_engine); @@ -671,6 +726,8 @@ mod tests { let remote_peer = PeerId::random(); let network = TestNetwork::default(); let sync = Arc::new(TestSync::default()); + let (mut tx, rx) = unbounded(); + let notification_service = Box::new(TestNotificationService { rx }); let num_channels_per_topic = channels.iter().fold( HashMap::new(), @@ -698,6 +755,7 @@ mod tests { let mut gossip_engine = GossipEngine::::new( network.clone(), sync.clone(), + notification_service, protocol.clone(), Arc::new(TestValidator {}), None, @@ -723,22 +781,18 @@ mod tests { } } - let mut event_sender = network.inner.lock().unwrap().event_senders.pop().unwrap(); - // Register the remote peer. - event_sender - .start_send(Event::NotificationStreamOpened { - remote: remote_peer, - protocol: protocol.clone(), - negotiated_fallback: None, - role: ObservedRole::Authority, - received_handshake: vec![], - }) - .expect("Event stream is unbounded; qed."); + tx.start_send(NotificationEvent::NotificationStreamOpened { + peer: remote_peer, + direction: Direction::Inbound, + negotiated_fallback: None, + handshake: Roles::FULL.encode(), + }) + .unwrap(); // Send messages into the network event stream. for (i_notification, messages) in notifications.iter().enumerate() { - let messages = messages + let messages: Vec> = messages .into_iter() .enumerate() .map(|(i_message, Message { topic })| { @@ -751,13 +805,17 @@ mod tests { message.push(i_notification.try_into().unwrap()); message.push(i_message.try_into().unwrap()); - (protocol.clone(), message.into()) + message.into() }) .collect(); - event_sender - .start_send(Event::NotificationsReceived { remote: remote_peer, messages }) - .expect("Event stream is unbounded; qed."); + for message in messages { + tx.start_send(NotificationEvent::NotificationReceived { + peer: remote_peer, + notification: message, + }) + .unwrap(); + } } let mut received_msgs_per_topic_all_chan = HashMap::::new(); diff --git a/substrate/client/network-gossip/src/state_machine.rs b/substrate/client/network-gossip/src/state_machine.rs index 4bfb5a7d37f4..724a4e3e07b8 100644 --- a/substrate/client/network-gossip/src/state_machine.rs +++ b/substrate/client/network-gossip/src/state_machine.rs @@ -651,6 +651,10 @@ mod tests { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, _handshake: Vec) -> Option { + None + } } impl NetworkEventStream for NoOpNetwork { diff --git a/substrate/client/network/Cargo.toml b/substrate/client/network/Cargo.toml index 72e38ce498d3..2ed366037a51 100644 --- a/substrate/client/network/Cargo.toml +++ b/substrate/client/network/Cargo.toml @@ -37,6 +37,8 @@ serde = { version = "1.0.188", features = ["derive"] } serde_json = "1.0.85" smallvec = "1.11.0" thiserror = "1.0" +tokio = { version = "1.22.0", features = ["macros", "sync"] } +tokio-stream = "0.1.7" unsigned-varint = { version = "0.7.1", features = ["futures", "asynchronous_codec"] } zeroize = "1.4.3" prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" } diff --git a/substrate/client/network/common/src/role.rs b/substrate/client/network/common/src/role.rs index cd43f6655b72..487ad7a5107f 100644 --- a/substrate/client/network/common/src/role.rs +++ b/substrate/client/network/common/src/role.rs @@ -24,7 +24,7 @@ use codec::{self, Encode, EncodeLike, Input, Output}; /// > **Note**: This enum is different from the `Role` enum. The `Role` enum indicates what a /// > node says about itself, while `ObservedRole` is a `Role` merged with the /// > information known locally about that node. -#[derive(Debug, Clone)] +#[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum ObservedRole { /// Full node. Full, @@ -41,6 +41,18 @@ impl ObservedRole { } } +impl From for ObservedRole { + fn from(roles: Roles) -> Self { + if roles.is_authority() { + ObservedRole::Authority + } else if roles.is_full() { + ObservedRole::Full + } else { + ObservedRole::Light + } + } +} + /// Role of the local node. #[derive(Debug, Clone)] pub enum Role { diff --git a/substrate/client/network/src/config.rs b/substrate/client/network/src/config.rs index d069c3f458ff..57250b1abd92 100644 --- a/substrate/client/network/src/config.rs +++ b/substrate/client/network/src/config.rs @@ -23,10 +23,11 @@ pub use crate::{ discovery::DEFAULT_KADEMLIA_REPLICATION_FACTOR, - protocol::NotificationsSink, + protocol::{notification_service, NotificationsSink, ProtocolHandlePair}, request_responses::{ IncomingRequest, OutgoingResponse, ProtocolConfig as RequestResponseConfig, }, + service::traits::NotificationService, types::ProtocolName, }; @@ -42,7 +43,6 @@ pub use sc_network_common::{ sync::{warp::WarpSyncProvider, SyncMode}, ExHashT, }; -use sc_utils::mpsc::TracingUnboundedSender; use sp_runtime::traits::Block as BlockT; use std::{ @@ -454,14 +454,14 @@ impl Default for SetConfig { /// /// > **Note**: As new fields might be added in the future, please consider using the `new` method /// > and modifiers instead of creating this struct manually. -#[derive(Clone, Debug)] +#[derive(Debug)] pub struct NonDefaultSetConfig { /// Name of the notifications protocols of this set. A substream on this set will be /// considered established once this protocol is open. /// /// > **Note**: This field isn't present for the default set, as this is handled internally /// > by the networking code. - pub notifications_protocol: ProtocolName, + protocol_name: ProtocolName, /// If the remote reports that it doesn't support the protocol indicated in the /// `notifications_protocol` field, then each of these fallback names will be tried one by @@ -469,37 +469,84 @@ pub struct NonDefaultSetConfig { /// /// If a fallback is used, it will be reported in /// `sc_network::protocol::event::Event::NotificationStreamOpened::negotiated_fallback` - pub fallback_names: Vec, + fallback_names: Vec, /// Handshake of the protocol /// /// NOTE: Currently custom handshakes are not fully supported. See issue #5685 for more /// details. This field is temporarily used to allow moving the hardcoded block announcement /// protocol out of `protocol.rs`. - pub handshake: Option, + handshake: Option, /// Maximum allowed size of single notifications. - pub max_notification_size: u64, + max_notification_size: u64, /// Base configuration. - pub set_config: SetConfig, + set_config: SetConfig, + + /// Notification handle. + /// + /// Notification handle is created during `NonDefaultSetConfig` creation and its other half, + /// `Box` is given to the protocol created the config and + /// `ProtocolHandle` is given to `Notifications` when it initializes itself. This handle allows + /// `Notifications ` to communicate with the protocol directly without relaying events through + /// `sc-network.` + protocol_handle_pair: ProtocolHandlePair, } impl NonDefaultSetConfig { /// Creates a new [`NonDefaultSetConfig`]. Zero slots and accepts only reserved nodes. - pub fn new(notifications_protocol: ProtocolName, max_notification_size: u64) -> Self { - Self { - notifications_protocol, - max_notification_size, - fallback_names: Vec::new(), - handshake: None, - set_config: SetConfig { - in_peers: 0, - out_peers: 0, - reserved_nodes: Vec::new(), - non_reserved_mode: NonReservedPeerMode::Deny, + /// Also returns an object which allows the protocol to communicate with `Notifications`. + pub fn new( + protocol_name: ProtocolName, + fallback_names: Vec, + max_notification_size: u64, + handshake: Option, + set_config: SetConfig, + ) -> (Self, Box) { + let (protocol_handle_pair, notification_service) = + notification_service(protocol_name.clone()); + ( + Self { + protocol_name, + max_notification_size, + fallback_names, + handshake, + set_config, + protocol_handle_pair, }, - } + notification_service, + ) + } + + /// Get reference to protocol name. + pub fn protocol_name(&self) -> &ProtocolName { + &self.protocol_name + } + + /// Get reference to fallback protocol names. + pub fn fallback_names(&self) -> impl Iterator { + self.fallback_names.iter() + } + + /// Get reference to handshake. + pub fn handshake(&self) -> &Option { + &self.handshake + } + + /// Get maximum notification size. + pub fn max_notification_size(&self) -> u64 { + self.max_notification_size + } + + /// Get reference to `SetConfig`. + pub fn set_config(&self) -> &SetConfig { + &self.set_config + } + + /// Take `ProtocolHandlePair` from `NonDefaultSetConfig` + pub fn take_protocol_handle(self) -> ProtocolHandlePair { + self.protocol_handle_pair } /// Modifies the configuration to allow non-reserved nodes. @@ -703,9 +750,6 @@ pub struct Params { /// Block announce protocol configuration pub block_announce_config: NonDefaultSetConfig, - - /// TX channel for direct communication with `SyncingEngine` and `Protocol`. - pub tx: TracingUnboundedSender>, } /// Full network configuration. diff --git a/substrate/client/network/src/error.rs b/substrate/client/network/src/error.rs index f0828fb821f3..01e8356fb553 100644 --- a/substrate/client/network/src/error.rs +++ b/substrate/client/network/src/error.rs @@ -68,6 +68,15 @@ pub enum Error { /// Name of the protocol registered multiple times. protocol: ProtocolName, }, + /// Peer does not exist. + #[error("Peer `{0}` does not exist.")] + PeerDoesntExist(PeerId), + /// Channel closed. + #[error("Channel closed")] + ChannelClosed, + /// Connection closed. + #[error("Connection closed")] + ConnectionClosed, } // Make `Debug` use the `Display` implementation. diff --git a/substrate/client/network/src/lib.rs b/substrate/client/network/src/lib.rs index ee3075968784..2f297f45ccc6 100644 --- a/substrate/client/network/src/lib.rs +++ b/substrate/client/network/src/lib.rs @@ -244,7 +244,6 @@ mod behaviour; mod protocol; -mod service; #[cfg(test)] mod mock; @@ -258,6 +257,7 @@ pub mod peer_info; pub mod peer_store; pub mod protocol_controller; pub mod request_responses; +pub mod service; pub mod transport; pub mod types; pub mod utils; @@ -267,7 +267,7 @@ pub use event::{DhtEvent, Event, SyncEvent}; pub use libp2p::{multiaddr, Multiaddr, PeerId}; pub use request_responses::{Config, IfDisconnected, RequestFailure}; pub use sc_network_common::{ - role::ObservedRole, + role::{ObservedRole, Roles}, sync::{ warp::{WarpSyncPhase, WarpSyncProgress}, ExtendedPeerInfo, StateDownloadProgress, SyncEventStream, SyncState, SyncStatusProvider, @@ -277,13 +277,14 @@ pub use sc_network_common::{ pub use service::{ signature::Signature, traits::{ - KademliaKey, NetworkBlock, NetworkDHTProvider, NetworkEventStream, NetworkNotification, - NetworkPeers, NetworkRequest, NetworkSigner, NetworkStateInfo, NetworkStatus, - NetworkStatusProvider, NetworkSyncForkRequest, NotificationSender as NotificationSenderT, - NotificationSenderError, NotificationSenderReady, + KademliaKey, MessageSink, NetworkBlock, NetworkDHTProvider, NetworkEventStream, + NetworkNotification, NetworkPeers, NetworkRequest, NetworkSigner, NetworkStateInfo, + NetworkStatus, NetworkStatusProvider, NetworkSyncForkRequest, + NotificationSender as NotificationSenderT, NotificationSenderError, + NotificationSenderReady, NotificationService, }, - DecodingError, Keypair, NetworkService, NetworkWorker, NotificationSender, NotificationsSink, - OutboundFailure, PublicKey, + DecodingError, Keypair, NetworkService, NetworkWorker, NotificationCommand, NotificationSender, + NotificationsSink, OutboundFailure, ProtocolHandle, PublicKey, }; pub use types::ProtocolName; diff --git a/substrate/client/network/src/mock.rs b/substrate/client/network/src/mock.rs index bc596b0fa579..534b81189707 100644 --- a/substrate/client/network/src/mock.rs +++ b/substrate/client/network/src/mock.rs @@ -20,6 +20,7 @@ use crate::{peer_store::PeerStoreProvider, protocol_controller::ProtocolHandle, ReputationChange}; use libp2p::PeerId; +use sc_network_common::role::ObservedRole; use std::collections::HashSet; /// No-op `PeerStore`. @@ -49,6 +50,14 @@ impl PeerStoreProvider for MockPeerStore { 0 } + fn peer_role(&self, _peer_id: &PeerId) -> Option { + None + } + + fn set_peer_role(&mut self, _peer_id: &PeerId, _role: ObservedRole) { + unimplemented!(); + } + fn outgoing_candidates(&self, _count: usize, _ignored: HashSet<&PeerId>) -> Vec { unimplemented!() } diff --git a/substrate/client/network/src/peer_store.rs b/substrate/client/network/src/peer_store.rs index 35d17e588cb9..4b28b8e75440 100644 --- a/substrate/client/network/src/peer_store.rs +++ b/substrate/client/network/src/peer_store.rs @@ -23,7 +23,7 @@ use libp2p::PeerId; use log::trace; use parking_lot::Mutex; use partial_sort::PartialSort; -use sc_network_common::types::ReputationChange; +use sc_network_common::{role::ObservedRole, types::ReputationChange}; use std::{ cmp::{Ord, Ordering, PartialOrd}, collections::{hash_map::Entry, HashMap, HashSet}, @@ -66,9 +66,15 @@ pub trait PeerStoreProvider: Debug + Send { /// Adjust peer reputation. fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange); + /// Set peer role. + fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole); + /// Get peer reputation. fn peer_reputation(&self, peer_id: &PeerId) -> i32; + /// Get peer role, if available. + fn peer_role(&self, peer_id: &PeerId) -> Option; + /// Get candidates with highest reputations for initiating outgoing connections. fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec; } @@ -96,10 +102,18 @@ impl PeerStoreProvider for PeerStoreHandle { self.inner.lock().report_peer(peer_id, change) } + fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole) { + self.inner.lock().set_peer_role(peer_id, role) + } + fn peer_reputation(&self, peer_id: &PeerId) -> i32 { self.inner.lock().peer_reputation(peer_id) } + fn peer_role(&self, peer_id: &PeerId) -> Option { + self.inner.lock().peer_role(peer_id) + } + fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { self.inner.lock().outgoing_candidates(count, ignored) } @@ -122,13 +136,19 @@ impl PeerStoreHandle { #[derive(Debug, Clone, Copy)] struct PeerInfo { + /// Reputation of the peer. reputation: i32, + + /// Instant when the peer was last updated. last_updated: Instant, + + /// Role of the peer, if known. + role: Option, } impl Default for PeerInfo { fn default() -> Self { - Self { reputation: 0, last_updated: Instant::now() } + Self { reputation: 0, last_updated: Instant::now(), role: None } } } @@ -242,10 +262,27 @@ impl PeerStoreInner { } } + fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole) { + log::trace!(target: LOG_TARGET, "Set {peer_id} role to {role:?}"); + + match self.peers.entry(*peer_id) { + Entry::Occupied(mut entry) => { + entry.get_mut().role = Some(role); + }, + Entry::Vacant(entry) => { + entry.insert(PeerInfo { role: Some(role), ..Default::default() }); + }, + } + } + fn peer_reputation(&self, peer_id: &PeerId) -> i32 { self.peers.get(peer_id).map_or(0, |info| info.reputation) } + fn peer_role(&self, peer_id: &PeerId) -> Option { + self.peers.get(peer_id).map_or(None, |info| info.role) + } + fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { let mut candidates = self .peers diff --git a/substrate/client/network/src/protocol.rs b/substrate/client/network/src/protocol.rs index 9b94f2883528..c78dfebdcb52 100644 --- a/substrate/client/network/src/protocol.rs +++ b/substrate/client/network/src/protocol.rs @@ -25,7 +25,7 @@ use crate::{ use bytes::Bytes; use codec::{DecodeAll, Encode}; -use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; +use futures::{stream::FuturesUnordered, StreamExt}; use libp2p::{ core::Endpoint, swarm::{ @@ -36,8 +36,9 @@ use libp2p::{ }; use log::{debug, error, warn}; -use sc_network_common::{role::Roles, sync::message::BlockAnnouncesHandshake}; -use sc_utils::mpsc::{TracingUnboundedReceiver, TracingUnboundedSender}; +use prometheus_endpoint::Registry; +use sc_network_common::role::Roles; +use sc_utils::mpsc::TracingUnboundedReceiver; use sp_runtime::traits::Block as BlockT; use std::{ @@ -48,10 +49,12 @@ use std::{ task::Poll, }; -use message::{generic::Message as GenericMessage, Message}; use notifications::{Notifications, NotificationsOut}; -pub use notifications::{NotificationsSink, NotifsHandlerError, Ready}; +pub use notifications::{ + notification_service, NotificationCommand, NotificationsSink, NotifsHandlerError, + ProtocolHandle, ProtocolHandlePair, Ready, +}; mod notifications; @@ -91,7 +94,6 @@ pub struct Protocol { /// Connected peers on sync protocol. peers: HashMap, sync_substream_validations: FuturesUnordered, - tx: TracingUnboundedSender>, _marker: std::marker::PhantomData, } @@ -99,45 +101,68 @@ impl Protocol { /// Create a new instance. pub fn new( roles: Roles, + registry: &Option, notification_protocols: Vec, block_announces_protocol: config::NonDefaultSetConfig, peer_store_handle: PeerStoreHandle, protocol_controller_handles: Vec, from_protocol_controllers: TracingUnboundedReceiver, - tx: TracingUnboundedSender>, ) -> error::Result { - let behaviour = { - Notifications::new( - protocol_controller_handles, - from_protocol_controllers, - // NOTE: Block announcement protocol is still very much hardcoded into `Protocol`. - // This protocol must be the first notification protocol given to - // `Notifications` - iter::once(notifications::ProtocolConfig { - name: block_announces_protocol.notifications_protocol.clone(), - fallback_names: block_announces_protocol.fallback_names.clone(), - handshake: block_announces_protocol.handshake.as_ref().unwrap().to_vec(), - max_notification_size: block_announces_protocol.max_notification_size, - }) - .chain(notification_protocols.iter().map(|s| notifications::ProtocolConfig { - name: s.notifications_protocol.clone(), - fallback_names: s.fallback_names.clone(), - handshake: s.handshake.as_ref().map_or(roles.encode(), |h| (*h).to_vec()), - max_notification_size: s.max_notification_size, - })), + let (behaviour, notification_protocols) = { + let installed_protocols = iter::once(block_announces_protocol.protocol_name().clone()) + .chain(notification_protocols.iter().map(|p| p.protocol_name().clone())) + .collect::>(); + + ( + Notifications::new( + protocol_controller_handles, + from_protocol_controllers, + registry, + // NOTE: Block announcement protocol is still very much hardcoded into + // `Protocol`. This protocol must be the first notification protocol given to + // `Notifications` + iter::once(( + notifications::ProtocolConfig { + name: block_announces_protocol.protocol_name().clone(), + fallback_names: block_announces_protocol + .fallback_names() + .cloned() + .collect(), + handshake: block_announces_protocol + .handshake() + .as_ref() + .unwrap() + .to_vec(), + max_notification_size: block_announces_protocol.max_notification_size(), + }, + block_announces_protocol.take_protocol_handle(), + )) + .chain(notification_protocols.into_iter().map(|s| { + ( + notifications::ProtocolConfig { + name: s.protocol_name().clone(), + fallback_names: s.fallback_names().cloned().collect(), + handshake: s + .handshake() + .as_ref() + .map_or(roles.encode(), |h| (*h).to_vec()), + max_notification_size: s.max_notification_size(), + }, + s.take_protocol_handle(), + ) + })), + ), + installed_protocols, ) }; let protocol = Self { peer_store_handle, behaviour, - notification_protocols: iter::once(block_announces_protocol.notifications_protocol) - .chain(notification_protocols.iter().map(|s| s.notifications_protocol.clone())) - .collect(), + notification_protocols, bad_handshake_substreams: Default::default(), peers: HashMap::new(), sync_substream_validations: FuturesUnordered::new(), - tx, // TODO: remove when `BlockAnnouncesHandshake` is moved away from `Protocol` _marker: Default::default(), }; @@ -165,7 +190,7 @@ impl Protocol { /// Returns the number of peers we're connected to on sync protocol. pub fn num_connected_peers(&self) -> usize { - self.peers.len() + self.behaviour.num_connected_peers(HARDCODED_PEERSETS_SYNC) } /// Set handshake for the notification protocol. @@ -310,103 +335,9 @@ impl NetworkBehaviour for Protocol { received_handshake, notifications_sink, negotiated_fallback, - inbound, + .. } => { - // Set number 0 is hardcoded the default set of peers we sync from. - if set_id == HARDCODED_PEERSETS_SYNC { - // `received_handshake` can be either a `Status` message if received from the - // legacy substream ,or a `BlockAnnouncesHandshake` if received from the block - // announces substream. - match as DecodeAll>::decode_all(&mut &received_handshake[..]) { - Ok(GenericMessage::Status(handshake)) => { - let roles = handshake.roles; - let handshake = BlockAnnouncesHandshake:: { - roles: handshake.roles, - best_number: handshake.best_number, - best_hash: handshake.best_hash, - genesis_hash: handshake.genesis_hash, - }; - - let (tx, rx) = oneshot::channel(); - let _ = self.tx.unbounded_send( - crate::SyncEvent::NotificationStreamOpened { - inbound, - remote: peer_id, - received_handshake: handshake, - sink: notifications_sink, - tx, - }, - ); - self.sync_substream_validations.push(Box::pin(async move { - match rx.await { - Ok(accepted) => - if accepted { - Ok((peer_id, roles)) - } else { - Err(peer_id) - }, - Err(_) => Err(peer_id), - } - })); - - CustomMessageOutcome::None - }, - Ok(msg) => { - debug!( - target: "sync", - "Expected Status message from {}, but got {:?}", - peer_id, - msg, - ); - self.peer_store_handle.report_peer(peer_id, rep::BAD_MESSAGE); - CustomMessageOutcome::None - }, - Err(err) => { - match as DecodeAll>::decode_all( - &mut &received_handshake[..], - ) { - Ok(handshake) => { - let roles = handshake.roles; - - let (tx, rx) = oneshot::channel(); - let _ = self.tx.unbounded_send( - crate::SyncEvent::NotificationStreamOpened { - inbound, - remote: peer_id, - received_handshake: handshake, - sink: notifications_sink, - tx, - }, - ); - self.sync_substream_validations.push(Box::pin(async move { - match rx.await { - Ok(accepted) => - if accepted { - Ok((peer_id, roles)) - } else { - Err(peer_id) - }, - Err(_) => Err(peer_id), - } - })); - CustomMessageOutcome::None - }, - Err(err2) => { - log::debug!( - target: "sync", - "Couldn't decode handshake sent by {}: {:?}: {} & {}", - peer_id, - received_handshake, - err, - err2, - ); - self.peer_store_handle.report_peer(peer_id, rep::BAD_MESSAGE); - CustomMessageOutcome::None - }, - } - }, - } - } else { + if set_id != HARDCODED_PEERSETS_SYNC { match ( Roles::decode_all(&mut &received_handshake[..]), self.peers.get(&peer_id), @@ -441,16 +372,14 @@ impl NetworkBehaviour for Protocol { CustomMessageOutcome::None }, } + } else { + CustomMessageOutcome::None } }, NotificationsOut::CustomProtocolReplaced { peer_id, notifications_sink, set_id } => if self.bad_handshake_substreams.contains(&(peer_id, set_id)) { CustomMessageOutcome::None } else if set_id == HARDCODED_PEERSETS_SYNC { - let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationSinkReplaced { - remote: peer_id, - sink: notifications_sink, - }); CustomMessageOutcome::None } else { CustomMessageOutcome::NotificationStreamReplaced { @@ -466,10 +395,6 @@ impl NetworkBehaviour for Protocol { // substream, and consequently shouldn't receive a closing event either. CustomMessageOutcome::None } else if set_id == HARDCODED_PEERSETS_SYNC { - let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationStreamClosed { - remote: peer_id, - }); - self.peers.remove(&peer_id); CustomMessageOutcome::None } else { CustomMessageOutcome::NotificationStreamClosed { @@ -482,10 +407,6 @@ impl NetworkBehaviour for Protocol { if self.bad_handshake_substreams.contains(&(peer_id, set_id)) { CustomMessageOutcome::None } else if set_id == HARDCODED_PEERSETS_SYNC { - let _ = self.tx.unbounded_send(crate::SyncEvent::NotificationsReceived { - remote: peer_id, - messages: vec![message.freeze()], - }); CustomMessageOutcome::None } else { let protocol_name = self.notification_protocols[usize::from(set_id)].clone(); diff --git a/substrate/client/network/src/protocol/message.rs b/substrate/client/network/src/protocol/message.rs index 66dca2975375..247580083f99 100644 --- a/substrate/client/network/src/protocol/message.rs +++ b/substrate/client/network/src/protocol/message.rs @@ -29,6 +29,7 @@ use sc_network_common::message::RequestId; use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; /// Type alias for using the message type using block type parameters. +#[allow(unused)] pub type Message = generic::Message< ::Header, ::Hash, diff --git a/substrate/client/network/src/protocol/notifications.rs b/substrate/client/network/src/protocol/notifications.rs index aa49cfcf9d44..b5b8b697456e 100644 --- a/substrate/client/network/src/protocol/notifications.rs +++ b/substrate/client/network/src/protocol/notifications.rs @@ -22,9 +22,11 @@ pub use self::{ behaviour::{Notifications, NotificationsOut, ProtocolConfig}, handler::{NotificationsSink, NotifsHandlerError, Ready}, + service::{notification_service, NotificationCommand, ProtocolHandle, ProtocolHandlePair}, }; mod behaviour; mod handler; +mod service; mod tests; mod upgrade; diff --git a/substrate/client/network/src/protocol/notifications/behaviour.rs b/substrate/client/network/src/protocol/notifications/behaviour.rs index 89513e004c6d..1ae8c35b4921 100644 --- a/substrate/client/network/src/protocol/notifications/behaviour.rs +++ b/substrate/client/network/src/protocol/notifications/behaviour.rs @@ -17,16 +17,19 @@ // along with this program. If not, see . use crate::{ - protocol::notifications::handler::{ - self, NotificationsSink, NotifsHandler, NotifsHandlerIn, NotifsHandlerOut, + config::ProtocolHandlePair, + protocol::notifications::{ + handler::{self, NotificationsSink, NotifsHandler, NotifsHandlerIn, NotifsHandlerOut}, + service::{metrics, NotificationCommand, ProtocolHandle}, }, protocol_controller::{self, IncomingIndex, Message, SetId}, + service::traits::{Direction, ValidationResult}, types::ProtocolName, }; use bytes::BytesMut; use fnv::FnvHashMap; -use futures::prelude::*; +use futures::{future::BoxFuture, prelude::*, stream::FuturesUnordered}; use libp2p::{ core::{ConnectedPoint, Endpoint, Multiaddr}, swarm::{ @@ -38,9 +41,13 @@ use libp2p::{ }; use log::{debug, error, info, trace, warn}; use parking_lot::RwLock; +use prometheus_endpoint::Registry; use rand::distributions::{Distribution as _, Uniform}; use sc_utils::mpsc::TracingUnboundedReceiver; use smallvec::SmallVec; +use tokio::sync::oneshot::error::RecvError; +use tokio_stream::StreamMap; + use std::{ cmp, collections::{hash_map::Entry, VecDeque}, @@ -51,6 +58,13 @@ use std::{ time::{Duration, Instant}, }; +/// Type representing a pending substream validation. +type PendingInboundValidation = + BoxFuture<'static, (Result, IncomingIndex)>; + +/// Logging target for the file. +const LOG_TARGET: &str = "sub-libp2p"; + /// Network behaviour that handles opening substreams for custom protocols with other peers. /// /// # How it works @@ -106,6 +120,12 @@ pub struct Notifications { /// Notification protocols. Entries never change after initialization. notif_protocols: Vec, + /// Protocol handles. + protocol_handles: Vec, + + // Command streams. + command_streams: StreamMap + Send + Unpin>>, + /// Protocol controllers are responsible for peer connections management. protocol_controller_handles: Vec, @@ -138,6 +158,18 @@ pub struct Notifications { /// Events to produce from `poll()`. events: VecDeque>, + + /// Pending inbound substream validations. + // + // NOTE: it's possible to read a stale response from `pending_inbound_validations` + // as the substream may get closed by the remote peer before the protocol has had + // a chance to validate it. [`Notifications`] must compare the `crate::peerset::IncomingIndex` + // returned by the completed future against the `crate::peerset::IncomingIndex` stored in + // `PeerState::Incoming` to check whether the completed future is stale or not. + pending_inbound_validations: FuturesUnordered, + + /// Metrics for notifications. + metrics: Option, } /// Configuration for a notifications protocol. @@ -303,6 +335,8 @@ struct IncomingPeer { alive: bool, /// Id that the we sent to the peerset. incoming_id: IncomingIndex, + /// Received handshake. + handshake: Vec, } /// Event that can be emitted by the `Notifications`. @@ -367,21 +401,40 @@ impl Notifications { pub fn new( protocol_controller_handles: Vec, from_protocol_controllers: TracingUnboundedReceiver, - notif_protocols: impl Iterator, + registry: &Option, + notif_protocols: impl Iterator, ) -> Self { - let notif_protocols = notif_protocols - .map(|cfg| handler::ProtocolConfig { - name: cfg.name, - fallback_names: cfg.fallback_names, - handshake: Arc::new(RwLock::new(cfg.handshake)), - max_notification_size: cfg.max_notification_size, + let (notif_protocols, protocol_handle_pairs): (Vec<_>, Vec<_>) = notif_protocols + .map(|(cfg, protocol_handle_pair)| { + ( + handler::ProtocolConfig { + name: cfg.name, + fallback_names: cfg.fallback_names, + handshake: Arc::new(RwLock::new(cfg.handshake)), + max_notification_size: cfg.max_notification_size, + }, + protocol_handle_pair, + ) }) - .collect::>(); - + .unzip(); assert!(!notif_protocols.is_empty()); + let metrics = registry.as_ref().and_then(|registry| metrics::register(®istry).ok()); + let (protocol_handles, command_streams): (Vec<_>, Vec<_>) = protocol_handle_pairs + .into_iter() + .enumerate() + .map(|(set_id, protocol_handle_pair)| { + let (mut protocol_handle, command_stream) = protocol_handle_pair.split(); + protocol_handle.set_metrics(metrics.clone()); + + (protocol_handle, (set_id, command_stream)) + }) + .unzip(); + Self { notif_protocols, + protocol_handles, + command_streams: StreamMap::from_iter(command_streams.into_iter()), protocol_controller_handles, from_protocol_controllers, peers: FnvHashMap::default(), @@ -390,6 +443,8 @@ impl Notifications { incoming: SmallVec::new(), next_incoming_index: IncomingIndex(0), events: VecDeque::new(), + pending_inbound_validations: FuturesUnordered::new(), + metrics, } } @@ -407,6 +462,11 @@ impl Notifications { } } + /// Get the number of connected peers for a given set. + pub fn num_connected_peers(&self, set_id: SetId) -> usize { + self.protocol_handles[usize::from(set_id)].num_peers() + } + /// Returns the list of all the peers we have an open channel to. pub fn open_peers(&self) -> impl Iterator { self.peers.iter().filter(|(_, state)| state.is_open()).map(|((id, _), _)| id) @@ -458,6 +518,8 @@ impl Notifications { let event = NotificationsOut::CustomProtocolClosed { peer_id: *peer_id, set_id }; self.events.push_back(ToSwarm::GenerateEvent(event)); + let _ = self.protocol_handles[usize::from(set_id)] + .report_substream_closed(*peer_id); } for (connec_id, connec_state) in @@ -759,6 +821,8 @@ impl Notifications { let event = NotificationsOut::CustomProtocolClosed { peer_id: entry.key().0, set_id }; self.events.push_back(ToSwarm::GenerateEvent(event)); + let _ = self.protocol_handles[usize::from(set_id)] + .report_substream_closed(entry.key().0); } for (connec_id, connec_state) in @@ -823,9 +887,37 @@ impl Notifications { } } + /// Substream has been accepted by the `ProtocolController` and must now be sent + /// to the protocol for validation. + fn peerset_report_preaccept(&mut self, index: IncomingIndex) { + let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) else { + error!(target: LOG_TARGET, "PSM => Preaccept({:?}): Invalid index", index); + return + }; + + trace!( + target: LOG_TARGET, + "PSM => Preaccept({:?}): Sent to protocol for validation", + index + ); + let incoming = &self.incoming[pos]; + + match self.protocol_handles[usize::from(incoming.set_id)] + .report_incoming_substream(incoming.peer_id, incoming.handshake.clone()) + { + Ok(rx) => self + .pending_inbound_validations + .push(Box::pin(async move { (rx.await, index) })), + Err(err) => { + error!(target: LOG_TARGET, "protocol has exited: {err:?}"); + debug_assert!(false); + }, + } + } + /// Function that is called when the peerset wants us to accept a connection /// request from a peer. - fn peerset_report_accept(&mut self, index: IncomingIndex) { + fn protocol_report_accept(&mut self, index: IncomingIndex) { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) @@ -911,40 +1003,52 @@ impl Notifications { } } - /// Function that is called when the peerset wants us to reject an incoming peer. + /// Function that is called when `ProtocolController` wants us to reject an incoming peer. fn peerset_report_reject(&mut self, index: IncomingIndex) { + let _ = self.report_reject(index); + } + + /// Function that is called when the protocol wants us to reject an incoming peer. + fn protocol_report_reject(&mut self, index: IncomingIndex) { + if let Some((set_id, peer_id)) = self.report_reject(index) { + self.protocol_controller_handles[usize::from(set_id)].dropped(peer_id) + } + } + + /// Function that is called when the peerset wants us to reject an incoming peer. + fn report_reject(&mut self, index: IncomingIndex) -> Option<(SetId, PeerId)> { let incoming = if let Some(pos) = self.incoming.iter().position(|i| i.incoming_id == index) { self.incoming.remove(pos) } else { error!(target: "sub-libp2p", "PSM => Reject({:?}): Invalid index", index); - return + return None }; if !incoming.alive { trace!(target: "sub-libp2p", "PSM => Reject({:?}, {}, {:?}): Obsolete incoming, \ ignoring", index, incoming.peer_id, incoming.set_id); - return + return None } let state = match self.peers.get_mut(&(incoming.peer_id, incoming.set_id)) { Some(s) => s, None => { debug_assert!(false); - return + return None }, }; match mem::replace(state, PeerState::Poisoned) { // Incoming => Disabled - PeerState::Incoming { mut connections, backoff_until, incoming_index } => { + PeerState::Incoming { mut connections, backoff_until, incoming_index, .. } => { if index < incoming_index { warn!( target: "sub-libp2p", "PSM => Reject({:?}, {}, {:?}): Ignoring obsolete incoming index, we are already awaiting {:?}.", index, incoming.peer_id, incoming.set_id, incoming_index ); - return + return None } else if index > incoming_index { error!( target: "sub-libp2p", @@ -952,7 +1056,7 @@ impl Notifications { index, incoming.peer_id, incoming.set_id, incoming_index ); debug_assert!(false); - return + return None } trace!(target: "sub-libp2p", "PSM => Reject({:?}, {}, {:?}): Rejecting connections.", @@ -976,10 +1080,15 @@ impl Notifications { } *state = PeerState::Disabled { connections, backoff_until }; + Some((incoming.set_id, incoming.peer_id)) + }, + peer => { + error!( + target: LOG_TARGET, + "State mismatch in libp2p: Expected alive incoming. Got {peer:?}.", + ); + None }, - peer => error!(target: "sub-libp2p", - "State mismatch in libp2p: Expected alive incoming. Got {:?}.", - peer), } } } @@ -1021,6 +1130,7 @@ impl NetworkBehaviour for Notifications { send_back_addr: remote_addr.clone(), }, self.notif_protocols.clone(), + self.metrics.clone(), )) } @@ -1035,6 +1145,7 @@ impl NetworkBehaviour for Notifications { peer, ConnectedPoint::Dialer { address: addr.clone(), role_override }, self.notif_protocols.clone(), + self.metrics.clone(), )) } @@ -1195,7 +1306,12 @@ impl NetworkBehaviour for Notifications { }, // Incoming => Incoming | Disabled | Backoff | Ø - PeerState::Incoming { mut connections, backoff_until, incoming_index } => { + PeerState::Incoming { + mut connections, + backoff_until, + incoming_index, + .. + } => { trace!( target: "sub-libp2p", "Libp2p => Disconnected({}, {:?}, {:?}): OpenDesiredByRemote.", @@ -1313,9 +1429,14 @@ impl NetworkBehaviour for Notifications { let event = NotificationsOut::CustomProtocolReplaced { peer_id, set_id, - notifications_sink: replacement_sink, + notifications_sink: replacement_sink.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); + let _ = self.protocol_handles[usize::from(set_id)] + .report_notification_sink_replaced( + peer_id, + replacement_sink, + ); } } else { trace!( @@ -1327,6 +1448,8 @@ impl NetworkBehaviour for Notifications { set_id, }; self.events.push_back(ToSwarm::GenerateEvent(event)); + let _ = self.protocol_handles[usize::from(set_id)] + .report_substream_closed(peer_id); } } } else { @@ -1475,7 +1598,7 @@ impl NetworkBehaviour for Notifications { event: THandlerOutEvent, ) { match event { - NotifsHandlerOut::OpenDesiredByRemote { protocol_index } => { + NotifsHandlerOut::OpenDesiredByRemote { protocol_index, handshake } => { let set_id = SetId::from(protocol_index); trace!(target: "sub-libp2p", @@ -1589,6 +1712,7 @@ impl NetworkBehaviour for Notifications { set_id, alive: true, incoming_id, + handshake, }); *entry.into_mut() = PeerState::Incoming { @@ -1726,9 +1850,11 @@ impl NetworkBehaviour for Notifications { let event = NotificationsOut::CustomProtocolReplaced { peer_id, set_id, - notifications_sink: replacement_sink, + notifications_sink: replacement_sink.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); + let _ = self.protocol_handles[usize::from(set_id)] + .report_notification_sink_replaced(peer_id, replacement_sink); } *entry.into_mut() = PeerState::Enabled { connections }; @@ -1750,6 +1876,8 @@ impl NetworkBehaviour for Notifications { trace!(target: "sub-libp2p", "External API <= Closed({}, {:?})", peer_id, set_id); let event = NotificationsOut::CustomProtocolClosed { peer_id, set_id }; self.events.push_back(ToSwarm::GenerateEvent(event)); + let _ = self.protocol_handles[usize::from(set_id)] + .report_substream_closed(peer_id); } }, @@ -1831,11 +1959,21 @@ impl NetworkBehaviour for Notifications { peer_id, set_id, inbound, - negotiated_fallback, - received_handshake, + received_handshake: received_handshake.clone(), + negotiated_fallback: negotiated_fallback.clone(), notifications_sink: notifications_sink.clone(), }; self.events.push_back(ToSwarm::GenerateEvent(event)); + let direction = + if inbound { Direction::Inbound } else { Direction::Outbound }; + let _ = self.protocol_handles[protocol_index] + .report_substream_opened( + peer_id, + direction, + received_handshake, + negotiated_fallback, + notifications_sink.clone(), + ); } *connec_state = ConnectionState::Open(notifications_sink); } else if let Some((_, connec_state)) = @@ -1980,9 +2118,14 @@ impl NetworkBehaviour for Notifications { peer_id, set_id, ); - let event = NotificationsOut::Notification { peer_id, set_id, message }; - + let event = NotificationsOut::Notification { + peer_id, + set_id, + message: message.clone(), + }; self.events.push_back(ToSwarm::GenerateEvent(event)); + let _ = self.protocol_handles[protocol_index] + .report_notification_received(peer_id, message.to_vec()); } else { trace!( target: "sub-libp2p", @@ -2010,10 +2153,10 @@ impl NetworkBehaviour for Notifications { loop { match futures::Stream::poll_next(Pin::new(&mut self.from_protocol_controllers), cx) { Poll::Ready(Some(Message::Accept(index))) => { - self.peerset_report_accept(index); + self.peerset_report_preaccept(index); }, Poll::Ready(Some(Message::Reject(index))) => { - self.peerset_report_reject(index); + let _ = self.peerset_report_reject(index); }, Poll::Ready(Some(Message::Connect { peer_id, set_id, .. })) => { self.peerset_report_connect(peer_id, set_id); @@ -2032,6 +2175,43 @@ impl NetworkBehaviour for Notifications { } } + // poll commands from protocols + loop { + match futures::Stream::poll_next(Pin::new(&mut self.command_streams), cx) { + Poll::Ready(Some((set_id, command))) => match command { + NotificationCommand::SetHandshake(handshake) => { + self.set_notif_protocol_handshake(set_id.into(), handshake); + }, + NotificationCommand::OpenSubstream(_peer) | + NotificationCommand::CloseSubstream(_peer) => { + todo!("substream control not implemented"); + }, + }, + Poll::Ready(None) => { + error!(target: LOG_TARGET, "Protocol command streams have been shut down"); + break + }, + Poll::Pending => break, + } + } + + while let Poll::Ready(Some((result, index))) = + self.pending_inbound_validations.poll_next_unpin(cx) + { + match result { + Ok(ValidationResult::Accept) => { + self.protocol_report_accept(index); + }, + Ok(ValidationResult::Reject) => { + self.protocol_report_reject(index); + }, + Err(_) => { + error!(target: LOG_TARGET, "Protocol has shut down"); + break + }, + } + } + while let Poll::Ready(Some((delay_id, peer_id, set_id))) = Pin::new(&mut self.delays).poll_next(cx) { @@ -2154,7 +2334,10 @@ mod tests { } } - fn development_notifs() -> (Notifications, ProtocolController) { + fn development_notifs( + ) -> (Notifications, ProtocolController, Box) { + let (protocol_handle_pair, notif_service) = + crate::protocol::notifications::service::notification_service("/proto/1".into()); let (to_notifications, from_controller) = tracing_unbounded("test_controller_to_notifications", 10_000); @@ -2174,20 +2357,25 @@ mod tests { Notifications::new( vec![handle], from_controller, - iter::once(ProtocolConfig { - name: "/foo".into(), - fallback_names: Vec::new(), - handshake: vec![1, 2, 3, 4], - max_notification_size: u64::MAX, - }), + &None, + iter::once(( + ProtocolConfig { + name: "/foo".into(), + fallback_names: Vec::new(), + handshake: vec![1, 2, 3, 4], + max_notification_size: u64::MAX, + }, + protocol_handle_pair, + )), ), controller, + notif_service, ) } #[test] fn update_handshake() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let inner = notif.notif_protocols.get_mut(0).unwrap().handshake.read().clone(); assert_eq!(inner, vec![1, 2, 3, 4]); @@ -2202,14 +2390,14 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn update_unknown_handshake() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); notif.set_notif_protocol_handshake(1337.into(), vec![5, 6, 7, 8]); } #[test] fn disconnect_backoff_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); notif.peers.insert( @@ -2226,7 +2414,7 @@ mod tests { #[test] fn disconnect_pending_request() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); notif.peers.insert( @@ -2243,7 +2431,7 @@ mod tests { #[test] fn disconnect_requested_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); notif.peers.insert((peer, 0.into()), PeerState::Requested); @@ -2254,7 +2442,7 @@ mod tests { #[test] fn disconnect_disabled_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); notif.peers.insert( (peer, 0.into()), @@ -2270,7 +2458,7 @@ mod tests { #[test] fn remote_opens_connection_and_substream() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let connected = ConnectedPoint::Listener { @@ -2300,7 +2488,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); if let Some(&PeerState::Incoming { ref connections, backoff_until: None, .. }) = @@ -2320,7 +2511,7 @@ mod tests { #[tokio::test] async fn disconnect_remote_substream_before_handled_by_controller() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let connected = ConnectedPoint::Listener { @@ -2340,7 +2531,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); notif.disconnect_peer(&peer, 0.into()); @@ -2356,7 +2550,7 @@ mod tests { #[test] fn peerset_report_connect_backoff() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -2394,7 +2588,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2421,7 +2615,7 @@ mod tests { #[test] fn peerset_connect_incoming() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -2445,19 +2639,22 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); // attempt to connect to the peer and verify that the peer state is `Enabled`; // we rely on implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); } #[test] fn peerset_disconnect_disable_pending_enable() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -2504,7 +2701,7 @@ mod tests { #[test] fn peerset_disconnect_enabled() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -2526,11 +2723,14 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); // we rely on the implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); // disconnect peer and verify that the state is `Disabled` @@ -2540,7 +2740,7 @@ mod tests { #[test] fn peerset_disconnect_requested() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); @@ -2555,7 +2755,7 @@ mod tests { #[test] fn peerset_disconnect_pending_request() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -2588,7 +2788,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2608,7 +2808,7 @@ mod tests { #[test] fn peerset_accept_peer_not_alive() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -2632,7 +2832,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -2648,14 +2851,14 @@ mod tests { IncomingPeer { alive: false, incoming_id: IncomingIndex(0), .. }, )); - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert_eq!(notif.incoming.len(), 0); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(PeerState::Disabled { .. }))); } #[test] fn secondary_connection_peer_state_incoming() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -2679,7 +2882,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); if let Some(PeerState::Incoming { connections, .. }) = notif.peers.get(&(peer, set_id)) { assert_eq!(connections.len(), 1); @@ -2710,7 +2916,7 @@ mod tests { #[test] fn close_connection_for_disabled_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -2735,7 +2941,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2744,7 +2950,7 @@ mod tests { #[test] fn close_connection_for_incoming_peer_one_connection() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -2767,7 +2973,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -2776,7 +2985,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2789,7 +2998,7 @@ mod tests { #[test] fn close_connection_for_incoming_peer_two_connections() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let conn1 = ConnectionId::new_unchecked(1); @@ -2816,7 +3025,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -2843,7 +3055,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -2858,7 +3070,7 @@ mod tests { #[test] fn connection_and_substream_open() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -2883,13 +3095,16 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); // We rely on the implementation detail that incoming indices are counted // from 0 to not mock the `Peerset`. - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); // open new substream @@ -2912,7 +3127,7 @@ mod tests { #[test] fn connection_closed_sink_replaced() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -2948,7 +3163,10 @@ mod tests { notif.on_connection_handler_event( peer, conn2, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); if let Some(PeerState::Enabled { connections, .. }) = notif.peers.get(&(peer, set_id)) { @@ -2985,7 +3203,7 @@ mod tests { peer_id: peer, connection_id: conn1, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3006,7 +3224,7 @@ mod tests { #[test] fn dial_failure_for_requested_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); @@ -3029,7 +3247,7 @@ mod tests { #[tokio::test] async fn write_notification() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -3078,7 +3296,7 @@ mod tests { #[test] fn peerset_report_connect_backoff_expired() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3111,7 +3329,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3126,7 +3344,7 @@ mod tests { #[test] fn peerset_report_disconnect_disabled() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -3152,7 +3370,7 @@ mod tests { #[test] fn peerset_report_disconnect_backoff() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3185,7 +3403,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3198,7 +3416,7 @@ mod tests { #[test] fn peer_is_backed_off_if_both_connections_get_closed_while_peer_is_disabled_with_back_off() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); @@ -3248,7 +3466,7 @@ mod tests { peer_id: peer, connection_id: conn1, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected.clone(), vec![]), + handler: NotifsHandler::new(peer, connected.clone(), vec![], None), remaining_established: 0usize, }, )); @@ -3262,7 +3480,7 @@ mod tests { peer_id: peer, connection_id: conn2, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3271,7 +3489,7 @@ mod tests { #[test] fn inject_connection_closed_incoming_with_backoff() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -3295,7 +3513,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); // manually add backoff for the entry @@ -3313,7 +3534,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3322,7 +3543,7 @@ mod tests { #[test] fn two_connections_inactive_connection_gets_closed_peer_state_is_still_incoming() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -3356,7 +3577,10 @@ mod tests { notif.on_connection_handler_event( peer, conn1, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!( notif.peers.get_mut(&(peer, 0.into())), @@ -3368,7 +3592,7 @@ mod tests { peer_id: peer, connection_id: conn2, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3377,7 +3601,7 @@ mod tests { #[test] fn two_connections_active_connection_gets_closed_peer_state_is_disabled() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -3414,7 +3638,10 @@ mod tests { notif.on_connection_handler_event( peer, conn1, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!( notif.peers.get_mut(&(peer, 0.into())), @@ -3426,7 +3653,7 @@ mod tests { peer_id: peer, connection_id: conn1, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3435,7 +3662,7 @@ mod tests { #[test] fn inject_connection_closed_for_active_connection() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn1 = ConnectionId::new_unchecked(0); let conn2 = ConnectionId::new_unchecked(1); @@ -3495,7 +3722,7 @@ mod tests { peer_id: peer, connection_id: conn1, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3503,7 +3730,7 @@ mod tests { #[test] fn inject_dial_failure_for_pending_request() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3536,7 +3763,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3566,7 +3793,7 @@ mod tests { #[test] fn peerstate_incoming_open_desired_by_remote() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); let conn1 = ConnectionId::new_unchecked(0); @@ -3600,7 +3827,10 @@ mod tests { notif.on_connection_handler_event( peer, conn1, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -3608,7 +3838,10 @@ mod tests { notif.on_connection_handler_event( peer, conn2, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); if let Some(PeerState::Incoming { ref connections, .. }) = notif.peers.get(&(peer, set_id)) @@ -3620,7 +3853,7 @@ mod tests { #[tokio::test] async fn remove_backoff_peer_after_timeout() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -3653,7 +3886,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3698,7 +3931,7 @@ mod tests { #[tokio::test] async fn reschedule_disabled_pending_enable_when_connection_not_closed() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -3727,13 +3960,16 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); // we rely on the implementation detail that incoming indices are counted from 0 // to not mock the `Peerset` - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); let event = conn_yielder.open_substream(peer, 0, connected, vec![1, 2, 3, 4]); @@ -3816,7 +4052,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn peerset_report_connect_with_enabled_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -3841,7 +4077,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -3866,7 +4105,7 @@ mod tests { #[test] #[cfg(debug_assertions)] fn peerset_report_connect_with_disabled_pending_enable_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3912,7 +4151,7 @@ mod tests { #[test] #[cfg(debug_assertions)] fn peerset_report_connect_with_requested_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); @@ -3928,7 +4167,7 @@ mod tests { #[test] #[cfg(debug_assertions)] fn peerset_report_connect_with_pending_requested() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -3961,7 +4200,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -3985,7 +4224,7 @@ mod tests { #[test] #[cfg(debug_assertions)] fn peerset_report_connect_with_incoming_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -4009,7 +4248,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4020,7 +4262,7 @@ mod tests { #[test] #[cfg(debug_assertions)] fn peerset_report_disconnect_with_incoming_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); let conn = ConnectionId::new_unchecked(0); @@ -4044,7 +4286,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4055,8 +4300,8 @@ mod tests { #[test] #[should_panic] #[cfg(debug_assertions)] - fn peerset_report_accept_incoming_peer() { - let (mut notif, _controller) = development_notifs(); + fn protocol_report_accept_incoming_peer() { + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -4080,7 +4325,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4090,14 +4338,14 @@ mod tests { )); notif.peers.remove(&(peer, set_id)); - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); } #[test] #[should_panic] #[cfg(debug_assertions)] - fn peerset_report_accept_not_incoming_peer() { - let (mut notif, _controller) = development_notifs(); + fn protocol_report_accept_not_incoming_peer() { + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -4122,7 +4370,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4139,14 +4390,14 @@ mod tests { assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); notif.incoming[0].alive = true; - notif.peerset_report_accept(IncomingIndex(0)); + notif.protocol_report_accept(IncomingIndex(0)); } #[test] #[should_panic] #[cfg(debug_assertions)] fn inject_connection_closed_non_existent_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let endpoint = ConnectedPoint::Listener { local_addr: Multiaddr::empty(), @@ -4158,7 +4409,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(0), endpoint: &endpoint.clone(), - handler: NotifsHandler::new(peer, endpoint, vec![]), + handler: NotifsHandler::new(peer, endpoint, vec![], None), remaining_established: 0usize, }, )); @@ -4166,7 +4417,7 @@ mod tests { #[test] fn disconnect_non_existent_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let set_id = SetId::from(0); @@ -4178,9 +4429,9 @@ mod tests { #[test] fn accept_non_existent_connection() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); - notif.peerset_report_accept(0.into()); + notif.protocol_report_accept(0.into()); assert!(notif.peers.is_empty()); assert!(notif.incoming.is_empty()); @@ -4188,9 +4439,9 @@ mod tests { #[test] fn reject_non_existent_connection() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); - notif.peerset_report_reject(0.into()); + notif.protocol_report_reject(0.into()); assert!(notif.peers.is_empty()); assert!(notif.incoming.is_empty()); @@ -4198,7 +4449,7 @@ mod tests { #[test] fn reject_non_active_connection() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -4222,12 +4473,15 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); notif.incoming[0].alive = false; - notif.peerset_report_reject(0.into()); + notif.protocol_report_reject(0.into()); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); } @@ -4236,7 +4490,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn reject_non_existent_peer_but_alive_connection() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -4260,7 +4514,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); assert!(std::matches!( @@ -4269,14 +4526,14 @@ mod tests { )); notif.peers.remove(&(peer, set_id)); - notif.peerset_report_reject(0.into()); + notif.protocol_report_reject(0.into()); } #[test] #[should_panic] #[cfg(debug_assertions)] fn inject_non_existent_connection_closed_for_incoming_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -4300,7 +4557,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4309,7 +4569,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(1337), endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4319,7 +4579,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_non_existent_connection_closed_for_disabled_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -4344,7 +4604,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(1337), endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4354,7 +4614,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_non_existent_connection_closed_for_disabled_pending_enable() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -4395,7 +4655,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(1337), endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4405,7 +4665,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_connection_closed_for_incoming_peer_state_mismatch() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -4429,7 +4689,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); notif.incoming[0].alive = false; @@ -4439,7 +4702,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4449,7 +4712,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_connection_closed_for_enabled_state_mismatch() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); let set_id = SetId::from(0); @@ -4473,7 +4736,10 @@ mod tests { notif.on_connection_handler_event( peer, conn, - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index: 0, + handshake: vec![1, 3, 3, 7], + }, ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); @@ -4486,7 +4752,7 @@ mod tests { peer_id: peer, connection_id: ConnectionId::new_unchecked(1337), endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4496,7 +4762,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn inject_connection_closed_for_backoff_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let set_id = SetId::from(0); let peer = PeerId::random(); let conn = ConnectionId::new_unchecked(0); @@ -4529,7 +4795,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected.clone(), vec![]), + handler: NotifsHandler::new(peer, connected.clone(), vec![], None), remaining_established: 0usize, }, )); @@ -4540,7 +4806,7 @@ mod tests { peer_id: peer, connection_id: conn, endpoint: &connected.clone(), - handler: NotifsHandler::new(peer, connected, vec![]), + handler: NotifsHandler::new(peer, connected, vec![], None), remaining_established: 0usize, }, )); @@ -4550,7 +4816,7 @@ mod tests { #[should_panic] #[cfg(debug_assertions)] fn open_result_ok_non_existent_peer() { - let (mut notif, _controller) = development_notifs(); + let (mut notif, _controller, _notif_service) = development_notifs(); let conn = ConnectionId::new_unchecked(0); let connected = ConnectedPoint::Listener { local_addr: Multiaddr::empty(), diff --git a/substrate/client/network/src/protocol/notifications/handler.rs b/substrate/client/network/src/protocol/notifications/handler.rs index cffdec7d71ee..c2adcbc96f14 100644 --- a/substrate/client/network/src/protocol/notifications/handler.rs +++ b/substrate/client/network/src/protocol/notifications/handler.rs @@ -58,9 +58,12 @@ //! [`NotifsHandlerIn::Open`] has gotten an answer. use crate::{ - protocol::notifications::upgrade::{ - NotificationsIn, NotificationsInSubstream, NotificationsOut, NotificationsOutSubstream, - UpgradeCollec, + protocol::notifications::{ + service::metrics, + upgrade::{ + NotificationsIn, NotificationsInSubstream, NotificationsOut, NotificationsOutSubstream, + UpgradeCollec, + }, }, types::ProtocolName, }; @@ -92,7 +95,7 @@ use std::{ /// Number of pending notifications in asynchronous contexts. /// See [`NotificationsSink::reserve_notification`] for context. -const ASYNC_NOTIFICATIONS_BUFFER_SIZE: usize = 8; +pub(crate) const ASYNC_NOTIFICATIONS_BUFFER_SIZE: usize = 8; /// Number of pending notifications in synchronous contexts. const SYNC_NOTIFICATIONS_BUFFER_SIZE: usize = 2048; @@ -106,6 +109,28 @@ const OPEN_TIMEOUT: Duration = Duration::from_secs(10); /// open substreams. const INITIAL_KEEPALIVE_TIME: Duration = Duration::from_secs(5); +/// Metrics-related information. +#[derive(Debug)] +pub struct MetricsInfo { + /// Protocol name. + protocol: ProtocolName, + + /// Metrics. + metrics: Option, +} + +impl MetricsInfo { + /// Create new [`MetricsInfo`]. + pub fn new(protocol: ProtocolName, metrics: Option) -> Self { + Self { protocol, metrics } + } + + /// Create new empty [`MetricsInfo`]. + pub fn new_empty() -> Self { + Self { protocol: ProtocolName::from(""), metrics: None } + } +} + /// The actual handler once the connection has been established. /// /// See the documentation at the module level for more information. @@ -126,11 +151,24 @@ pub struct NotifsHandler { events_queue: VecDeque< ConnectionHandlerEvent, >, + + /// Metrics-related information. + metrics_info: Vec>, } impl NotifsHandler { /// Creates new [`NotifsHandler`]. - pub fn new(peer_id: PeerId, endpoint: ConnectedPoint, protocols: Vec) -> Self { + pub fn new( + peer_id: PeerId, + endpoint: ConnectedPoint, + protocols: Vec, + metrics: Option, + ) -> Self { + let metrics_info = protocols + .iter() + .map(|config| Arc::new(MetricsInfo::new(config.name.clone(), metrics.clone()))) + .collect(); + Self { protocols: protocols .into_iter() @@ -148,6 +186,7 @@ impl NotifsHandler { endpoint, when_connection_open: Instant::now(), events_queue: VecDeque::with_capacity(16), + metrics_info, } } } @@ -303,6 +342,8 @@ pub enum NotifsHandlerOut { OpenDesiredByRemote { /// Index of the protocol in the list of protocols passed at initialization. protocol_index: usize, + /// Received handshake. + handshake: Vec, }, /// The remote would like the substreams to be closed. Send a [`NotifsHandlerIn::Close`] in @@ -333,6 +374,30 @@ pub struct NotificationsSink { inner: Arc, } +// NOTE: only used for testing but must be `pub` as other crates in `client/network` use this. +impl NotificationsSink { + /// Create new + pub fn new( + peer_id: PeerId, + ) -> (Self, mpsc::Receiver, mpsc::Receiver) + { + let (async_tx, async_rx) = mpsc::channel(ASYNC_NOTIFICATIONS_BUFFER_SIZE); + let (sync_tx, sync_rx) = mpsc::channel(SYNC_NOTIFICATIONS_BUFFER_SIZE); + ( + NotificationsSink { + inner: Arc::new(NotificationsSinkInner { + peer_id, + async_channel: FuturesMutex::new(async_tx), + sync_channel: Mutex::new(Some(sync_tx)), + metrics_info: Arc::new(MetricsInfo::new_empty()), + }), + }, + async_rx, + sync_rx, + ) + } +} + #[derive(Debug)] struct NotificationsSinkInner { /// Target of the sink. @@ -346,12 +411,14 @@ struct NotificationsSinkInner { /// back-pressure cannot be properly exerted. /// It will be removed in a future version. sync_channel: Mutex>>, + /// Metrics-related information. + metrics_info: Arc, } /// Message emitted through the [`NotificationsSink`] and processed by the background task /// dedicated to the peer. -#[derive(Debug)] -enum NotificationsSinkMessage { +#[derive(Debug, PartialEq, Eq)] +pub enum NotificationsSinkMessage { /// Message emitted by [`NotificationsSink::reserve_notification`] and /// [`NotificationsSink::write_notification_now`]. Notification { message: Vec }, @@ -379,8 +446,14 @@ impl NotificationsSink { let mut lock = self.inner.sync_channel.lock(); if let Some(tx) = lock.as_mut() { - let result = - tx.try_send(NotificationsSinkMessage::Notification { message: message.into() }); + let message = message.into(); + metrics::register_notification_sent( + &self.inner.metrics_info.metrics, + &self.inner.metrics_info.protocol, + message.len(), + ); + + let result = tx.try_send(NotificationsSinkMessage::Notification { message }); if result.is_err() { // Cloning the `mpsc::Sender` guarantees the allocation of an extra spot in the @@ -476,7 +549,10 @@ impl ConnectionHandler for NotifsHandler { match protocol_info.state { State::Closed { pending_opening } => { self.events_queue.push_back(ConnectionHandlerEvent::Custom( - NotifsHandlerOut::OpenDesiredByRemote { protocol_index }, + NotifsHandlerOut::OpenDesiredByRemote { + protocol_index, + handshake: in_substream_open.handshake, + }, )); protocol_info.state = State::OpenDesiredByRemote { @@ -530,6 +606,7 @@ impl ConnectionHandler for NotifsHandler { peer_id: self.peer_id, async_channel: FuturesMutex::new(async_tx), sync_channel: Mutex::new(Some(sync_tx)), + metrics_info: self.metrics_info[protocol_index].clone(), }), }; @@ -880,6 +957,7 @@ pub mod tests { peer_id: peer, async_channel: FuturesMutex::new(async_tx), sync_channel: Mutex::new(Some(sync_tx)), + metrics_info: Arc::new(MetricsInfo::new_empty()), }), }; let (in_substream, out_substream) = MockSubstream::new(); @@ -1040,6 +1118,7 @@ pub mod tests { }, peer_id: PeerId::random(), events_queue: VecDeque::new(), + metrics_info: vec![Arc::new(MetricsInfo::new_empty())], } } @@ -1544,6 +1623,7 @@ pub mod tests { peer_id: PeerId::random(), async_channel: FuturesMutex::new(async_tx), sync_channel: Mutex::new(Some(sync_tx)), + metrics_info: Arc::new(MetricsInfo::new_empty()), }), }; @@ -1597,7 +1677,7 @@ pub mod tests { assert!(std::matches!( handler.poll(cx), Poll::Ready(ConnectionHandlerEvent::Custom( - NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0, .. }, )) )); assert!(std::matches!( diff --git a/substrate/client/network/src/protocol/notifications/service/metrics.rs b/substrate/client/network/src/protocol/notifications/service/metrics.rs new file mode 100644 index 000000000000..65ccedac0762 --- /dev/null +++ b/substrate/client/network/src/protocol/notifications/service/metrics.rs @@ -0,0 +1,124 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use crate::types::ProtocolName; + +use prometheus_endpoint::{ + self as prometheus, CounterVec, HistogramOpts, HistogramVec, Opts, PrometheusError, Registry, + U64, +}; + +/// Notification metrics. +#[derive(Debug, Clone)] +pub struct Metrics { + // Total number of opened substreams. + pub notifications_streams_opened_total: CounterVec, + + /// Total number of closed substreams. + pub notifications_streams_closed_total: CounterVec, + + /// In/outbound notification sizes. + pub notifications_sizes: HistogramVec, +} + +impl Metrics { + fn register(registry: &Registry) -> Result { + Ok(Self { + notifications_sizes: prometheus::register( + HistogramVec::new( + HistogramOpts { + common_opts: Opts::new( + "substrate_sub_libp2p_notification_service_total_sizes", + "Sizes of the notifications send to and received from all nodes", + ), + buckets: prometheus::exponential_buckets(64.0, 4.0, 8) + .expect("parameters are always valid values; qed"), + }, + &["direction", "protocol"], + )?, + registry, + )?, + notifications_streams_closed_total: prometheus::register( + CounterVec::new( + Opts::new( + "substrate_sub_libp2p_notification_service_streams_closed_total", + "Total number of notification substreams that have been closed", + ), + &["protocol"], + )?, + registry, + )?, + notifications_streams_opened_total: prometheus::register( + CounterVec::new( + Opts::new( + "substrate_sub_libp2p_notification_service_streams_opened_total", + "Total number of notification substreams that have been opened", + ), + &["protocol"], + )?, + registry, + )?, + }) + } +} + +/// Register metrics. +pub fn register(registry: &Registry) -> Result { + Metrics::register(registry) +} + +/// Register opened substream to Prometheus. +pub fn register_substream_opened(metrics: &Option, protocol: &ProtocolName) { + if let Some(metrics) = metrics { + metrics.notifications_streams_opened_total.with_label_values(&[&protocol]).inc(); + } +} + +/// Register closed substream to Prometheus. +pub fn register_substream_closed(metrics: &Option, protocol: &ProtocolName) { + if let Some(metrics) = metrics { + metrics + .notifications_streams_closed_total + .with_label_values(&[&protocol[..]]) + .inc(); + } +} + +/// Register sent notification to Prometheus. +pub fn register_notification_sent(metrics: &Option, protocol: &ProtocolName, size: usize) { + if let Some(metrics) = metrics { + metrics + .notifications_sizes + .with_label_values(&["out", protocol]) + .observe(size as f64); + } +} + +/// Register received notification to Prometheus. +pub fn register_notification_received( + metrics: &Option, + protocol: &ProtocolName, + size: usize, +) { + if let Some(metrics) = metrics { + metrics + .notifications_sizes + .with_label_values(&["in", protocol]) + .observe(size as f64); + } +} diff --git a/substrate/client/network/src/protocol/notifications/service/mod.rs b/substrate/client/network/src/protocol/notifications/service/mod.rs new file mode 100644 index 000000000000..05aa8fc41615 --- /dev/null +++ b/substrate/client/network/src/protocol/notifications/service/mod.rs @@ -0,0 +1,576 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Notification service implementation. + +use crate::{ + error, + protocol::notifications::handler::NotificationsSink, + service::traits::{ + Direction, MessageSink, NotificationEvent, NotificationService, ValidationResult, + }, + types::ProtocolName, +}; + +use futures::{ + stream::{FuturesUnordered, Stream}, + StreamExt, +}; +use libp2p::PeerId; +use parking_lot::Mutex; +use tokio::sync::{mpsc, oneshot}; +use tokio_stream::wrappers::ReceiverStream; + +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; + +use std::{collections::HashMap, fmt::Debug, sync::Arc}; + +pub(crate) mod metrics; +#[cfg(test)] +mod tests; + +/// Logging target for the file. +const LOG_TARGET: &str = "notification-service"; + +/// Default command queue size. +const COMMAND_QUEUE_SIZE: usize = 64; + +/// Type representing subscribers of a notification protocol. +type Subscribers = Arc>>>; + +/// Type represending a distributable message sink. +/// +/// See documentation for [`PeerContext`] for more details. +type NotificationSink = Arc>; + +#[async_trait::async_trait] +impl MessageSink for NotificationSink { + /// Send synchronous `notification` to the peer associated with this [`MessageSink`]. + fn send_sync_notification(&self, notification: Vec) { + self.lock().send_sync_notification(notification); + } + + /// Send an asynchronous `notification` to to the peer associated with this [`MessageSink`], + /// allowing sender to exercise backpressure. + /// + /// Returns an error if the peer does not exist. + async fn send_async_notification(&self, notification: Vec) -> Result<(), error::Error> { + // notification sink must be cloned because the lock cannot be held across `.await` + // this makes the implementation less efficient but not prohibitively so as the same + // method is also used by `NetworkService` when sending notifications. + let sink = self.lock().clone(); + let sink = sink.reserve_notification().await.map_err(|_| error::Error::ConnectionClosed)?; + sink.send(notification).map_err(|_| error::Error::ChannelClosed) + } +} + +/// Inner notification event to deal with `NotificationsSinks` without exposing that +/// implementation detail to [`NotificationService`] consumers. +#[derive(Debug)] +enum InnerNotificationEvent { + /// Validate inbound substream. + ValidateInboundSubstream { + /// Peer ID. + peer: PeerId, + + /// Received handshake. + handshake: Vec, + + /// `oneshot::Sender` for sending validation result back to `Notifications` + result_tx: oneshot::Sender, + }, + + /// Notification substream open to `peer`. + NotificationStreamOpened { + /// Peer ID. + peer: PeerId, + + /// Direction of the substream. + direction: Direction, + + /// Received handshake. + handshake: Vec, + + /// Negotiated fallback. + negotiated_fallback: Option, + + /// Notification sink. + sink: NotificationsSink, + }, + + /// Substream was closed. + NotificationStreamClosed { + /// Peer ID. + peer: PeerId, + }, + + /// Notification was received from the substream. + NotificationReceived { + /// Peer ID. + peer: PeerId, + + /// Received notification. + notification: Vec, + }, + + /// Notification sink has been replaced. + NotificationSinkReplaced { + /// Peer ID. + peer: PeerId, + + /// Notification sink. + sink: NotificationsSink, + }, +} + +/// Notification commands. +/// +/// Sent by the installed protocols to `Notifications` to open/close/modify substreams. +#[derive(Debug)] +pub enum NotificationCommand { + /// Instruct `Notifications` to open a substream to peer. + OpenSubstream(PeerId), + + /// Instruct `Notifications` to close the substream to peer. + CloseSubstream(PeerId), + + /// Set handshake for the notifications protocol. + SetHandshake(Vec), +} + +/// Context assigned to each peer. +/// +/// Contains `NotificationsSink` used by [`NotificationService`] to send notifications +/// and an additional, distributable `NotificationsSink` which the protocol may acquire +/// if it wishes to send notifications through `NotificationsSink` directly. +/// +/// The distributable `NoticationsSink` is wrapped in an `Arc>` to allow +/// `NotificationsService` to swap the underlying sink in case it's replaced. +#[derive(Debug, Clone)] +struct PeerContext { + /// Sink for sending notificaitons. + sink: NotificationsSink, + + /// Distributable notification sink. + shared_sink: NotificationSink, +} + +/// Handle that is passed on to the notifications protocol. +#[derive(Debug)] +pub struct NotificationHandle { + /// Protocol name. + protocol: ProtocolName, + + /// TX channel for sending commands to `Notifications`. + tx: mpsc::Sender, + + /// RX channel for receiving events from `Notifications`. + rx: TracingUnboundedReceiver, + + /// All subscribers of `NotificationEvent`s. + subscribers: Subscribers, + + /// Connected peers. + peers: HashMap, +} + +impl NotificationHandle { + /// Create new [`NotificationHandle`]. + fn new( + protocol: ProtocolName, + tx: mpsc::Sender, + rx: TracingUnboundedReceiver, + subscribers: Arc>>>, + ) -> Self { + Self { protocol, tx, rx, subscribers, peers: HashMap::new() } + } +} + +#[async_trait::async_trait] +impl NotificationService for NotificationHandle { + /// Instruct `Notifications` to open a new substream for `peer`. + async fn open_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + todo!("support for opening substreams not implemented yet"); + } + + /// Instruct `Notifications` to close substream for `peer`. + async fn close_substream(&mut self, _peer: PeerId) -> Result<(), ()> { + todo!("support for closing substreams not implemented yet, call `NetworkService::disconnect_peer()` instead"); + } + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification(&self, peer: &PeerId, notification: Vec) { + log::trace!(target: LOG_TARGET, "{}: send sync notification to {peer:?}", self.protocol); + + if let Some(info) = self.peers.get(&peer) { + let _ = info.sink.send_sync_notification(notification); + } + } + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + async fn send_async_notification( + &self, + peer: &PeerId, + notification: Vec, + ) -> Result<(), error::Error> { + log::trace!(target: LOG_TARGET, "{}: send async notification to {peer:?}", self.protocol); + + self.peers + .get(&peer) + .ok_or_else(|| error::Error::PeerDoesntExist(*peer))? + .sink + .reserve_notification() + .await + .map_err(|_| error::Error::ConnectionClosed)? + .send(notification) + .map_err(|_| error::Error::ChannelClosed) + } + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_handshake(&mut self, handshake: Vec) -> Result<(), ()> { + log::trace!(target: LOG_TARGET, "{}: set handshake to {handshake:?}", self.protocol); + + self.tx.send(NotificationCommand::SetHandshake(handshake)).await.map_err(|_| ()) + } + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option { + loop { + match self.rx.next().await? { + InnerNotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => + return Some(NotificationEvent::ValidateInboundSubstream { + peer, + handshake, + result_tx, + }), + InnerNotificationEvent::NotificationStreamOpened { + peer, + handshake, + negotiated_fallback, + direction, + sink, + } => { + self.peers.insert( + peer, + PeerContext { sink: sink.clone(), shared_sink: Arc::new(Mutex::new(sink)) }, + ); + return Some(NotificationEvent::NotificationStreamOpened { + peer, + handshake, + direction, + negotiated_fallback, + }) + }, + InnerNotificationEvent::NotificationStreamClosed { peer } => { + self.peers.remove(&peer); + return Some(NotificationEvent::NotificationStreamClosed { peer }) + }, + InnerNotificationEvent::NotificationReceived { peer, notification } => + return Some(NotificationEvent::NotificationReceived { peer, notification }), + InnerNotificationEvent::NotificationSinkReplaced { peer, sink } => { + match self.peers.get_mut(&peer) { + None => log::error!( + "{}: notification sink replaced for {peer} but peer does not exist", + self.protocol + ), + Some(context) => { + context.sink = sink.clone(); + *context.shared_sink.lock() = sink.clone(); + }, + } + }, + } + } + } + + // Clone [`NotificationService`] + fn clone(&mut self) -> Result, ()> { + let mut subscribers = self.subscribers.lock(); + let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); + subscribers.push(event_tx); + + Ok(Box::new(NotificationHandle { + protocol: self.protocol.clone(), + tx: self.tx.clone(), + rx: event_rx, + peers: self.peers.clone(), + subscribers: self.subscribers.clone(), + })) + } + + /// Get protocol name. + fn protocol(&self) -> &ProtocolName { + &self.protocol + } + + /// Get message sink of the peer. + fn message_sink(&self, peer: &PeerId) -> Option> { + match self.peers.get(peer) { + Some(context) => Some(Box::new(context.shared_sink.clone())), + None => None, + } + } +} + +/// Channel pair which allows `Notifications` to interact with a protocol. +#[derive(Debug)] +pub struct ProtocolHandlePair { + /// Protocol name. + protocol: ProtocolName, + + /// Subscribers of the notification protocol events. + subscribers: Subscribers, + + // Receiver for notification commands received from the protocol implementation. + rx: mpsc::Receiver, +} + +impl ProtocolHandlePair { + /// Create new [`ProtocolHandlePair`]. + fn new( + protocol: ProtocolName, + subscribers: Subscribers, + rx: mpsc::Receiver, + ) -> Self { + Self { protocol, subscribers, rx } + } + + /// Consume `self` and split [`ProtocolHandlePair`] into a handle which allows it to send events + /// to the protocol and a stream of commands received from the protocol. + pub fn split( + self, + ) -> (ProtocolHandle, Box + Send + Unpin>) { + ( + ProtocolHandle::new(self.protocol, self.subscribers), + Box::new(ReceiverStream::new(self.rx)), + ) + } +} + +/// Handle that is passed on to `Notifications` and allows it to directly communicate +/// with the protocol. +#[derive(Debug)] +pub struct ProtocolHandle { + /// Protocol name. + protocol: ProtocolName, + + /// Subscribers of the notification protocol. + subscribers: Subscribers, + + /// Number of connected peers. + num_peers: usize, + + /// Prometheus metrics. + metrics: Option, +} + +impl ProtocolHandle { + /// Create new [`ProtocolHandle`]. + fn new(protocol: ProtocolName, subscribers: Subscribers) -> Self { + Self { protocol, subscribers, num_peers: 0usize, metrics: None } + } + + /// Set metrics. + pub fn set_metrics(&mut self, metrics: Option) { + self.metrics = metrics; + } + + /// Report to the protocol that a substream has been opened and it must be validated by the + /// protocol. + /// + /// Return `oneshot::Receiver` which allows `Notifications` to poll for the validation result + /// from protocol. + pub fn report_incoming_substream( + &self, + peer: PeerId, + handshake: Vec, + ) -> Result, ()> { + let subscribers = self.subscribers.lock(); + + log::trace!( + target: LOG_TARGET, + "{}: report incoming substream for {peer}, handshake {handshake:?}", + self.protocol + ); + + // if there is only one subscriber, `Notifications` can wait directly on the + // `oneshot::channel()`'s RX half without indirection + if subscribers.len() == 1 { + let (result_tx, rx) = oneshot::channel(); + return subscribers[0] + .unbounded_send(InnerNotificationEvent::ValidateInboundSubstream { + peer, + handshake, + result_tx, + }) + .map(|_| rx) + .map_err(|_| ()) + } + + // if there are multiple subscribers, create a task which waits for all of the + // validations to finish and returns the combined result to `Notifications` + let mut results: FuturesUnordered<_> = subscribers + .iter() + .filter_map(|subscriber| { + let (result_tx, rx) = oneshot::channel(); + + subscriber + .unbounded_send(InnerNotificationEvent::ValidateInboundSubstream { + peer, + handshake: handshake.clone(), + result_tx, + }) + .is_ok() + .then_some(rx) + }) + .collect(); + + let (tx, rx) = oneshot::channel(); + tokio::spawn(async move { + while let Some(event) = results.next().await { + match event { + Err(_) | Ok(ValidationResult::Reject) => + return tx.send(ValidationResult::Reject), + Ok(ValidationResult::Accept) => {}, + } + } + + return tx.send(ValidationResult::Accept) + }); + + Ok(rx) + } + + /// Report to the protocol that a substream has been opened and that it can now use the handle + /// to send notifications to the remote peer. + pub fn report_substream_opened( + &mut self, + peer: PeerId, + direction: Direction, + handshake: Vec, + negotiated_fallback: Option, + sink: NotificationsSink, + ) -> Result<(), ()> { + metrics::register_substream_opened(&self.metrics, &self.protocol); + + let mut subscribers = self.subscribers.lock(); + log::trace!(target: LOG_TARGET, "{}: substream opened for {peer:?}", self.protocol); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationStreamOpened { + peer, + direction, + handshake: handshake.clone(), + negotiated_fallback: negotiated_fallback.clone(), + sink: sink.clone(), + }) + .is_ok() + }); + self.num_peers += 1; + + Ok(()) + } + + /// Substream was closed. + pub fn report_substream_closed(&mut self, peer: PeerId) -> Result<(), ()> { + metrics::register_substream_closed(&self.metrics, &self.protocol); + + let mut subscribers = self.subscribers.lock(); + log::trace!(target: LOG_TARGET, "{}: substream closed for {peer:?}", self.protocol); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationStreamClosed { peer }) + .is_ok() + }); + self.num_peers -= 1; + + Ok(()) + } + + /// Notification was received from the substream. + pub fn report_notification_received( + &mut self, + peer: PeerId, + notification: Vec, + ) -> Result<(), ()> { + metrics::register_notification_received(&self.metrics, &self.protocol, notification.len()); + + let mut subscribers = self.subscribers.lock(); + log::trace!(target: LOG_TARGET, "{}: notification received from {peer:?}", self.protocol); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationReceived { + peer, + notification: notification.clone(), + }) + .is_ok() + }); + + Ok(()) + } + + /// Notification sink was replaced. + pub fn report_notification_sink_replaced( + &mut self, + peer: PeerId, + sink: NotificationsSink, + ) -> Result<(), ()> { + let mut subscribers = self.subscribers.lock(); + + log::trace!( + target: LOG_TARGET, + "{}: notification sink replaced for {peer:?}", + self.protocol + ); + + subscribers.retain(|subscriber| { + subscriber + .unbounded_send(InnerNotificationEvent::NotificationSinkReplaced { + peer, + sink: sink.clone(), + }) + .is_ok() + }); + + Ok(()) + } + + /// Get the number of connected peers. + pub fn num_peers(&self) -> usize { + self.num_peers + } +} + +/// Create new (protocol, notification) handle pair. +/// +/// Handle pair allows `Notifications` and the protocol to communicate with each other directly. +pub fn notification_service( + protocol: ProtocolName, +) -> (ProtocolHandlePair, Box) { + let (cmd_tx, cmd_rx) = mpsc::channel(COMMAND_QUEUE_SIZE); + let (event_tx, event_rx) = tracing_unbounded("mpsc-notification-to-protocol", 100_000); + let subscribers = Arc::new(Mutex::new(vec![event_tx])); + + ( + ProtocolHandlePair::new(protocol.clone(), subscribers.clone(), cmd_rx), + Box::new(NotificationHandle::new(protocol.clone(), cmd_tx, event_rx, subscribers)), + ) +} diff --git a/substrate/client/network/src/protocol/notifications/service/tests.rs b/substrate/client/network/src/protocol/notifications/service/tests.rs new file mode 100644 index 000000000000..72e97b79b50f --- /dev/null +++ b/substrate/client/network/src/protocol/notifications/service/tests.rs @@ -0,0 +1,770 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use super::*; +use crate::protocol::notifications::handler::{ + NotificationsSinkMessage, ASYNC_NOTIFICATIONS_BUFFER_SIZE, +}; + +use std::future::Future; + +#[tokio::test] +async fn validate_and_accept_substream() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (handle, _stream) = proto.split(); + + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + + assert_eq!(rx.await.unwrap(), ValidationResult::Accept); +} + +#[tokio::test] +async fn substream_opened() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, _, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + + let peer_id = PeerId::random(); + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } +} + +#[tokio::test] +async fn send_sync_notification() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }) + ); +} + +#[tokio::test] +async fn send_async_notification() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }) + ); +} + +#[tokio::test] +async fn send_sync_notification_to_non_existent_peer() { + let (proto, notif) = notification_service("/proto/1".into()); + let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); + let (_handle, _stream) = proto.split(); + let peer = PeerId::random(); + + // as per the original implementation, the call doesn't fail + notif.send_sync_notification(&peer, vec![1, 3, 3, 7]) +} + +#[tokio::test] +async fn send_async_notification_to_non_existent_peer() { + let (proto, notif) = notification_service("/proto/1".into()); + let (_sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); + let (_handle, _stream) = proto.split(); + let peer = PeerId::random(); + + if let Err(error::Error::PeerDoesntExist(peer_id)) = + notif.send_async_notification(&peer, vec![1, 3, 3, 7]).await + { + assert_eq!(peer, peer_id); + } else { + panic!("invalid error received from `send_async_notification()`"); + } +} + +#[tokio::test] +async fn receive_notification() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, _, _sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + // notification is received + handle.report_notification_received(peer_id, vec![1, 3, 3, 8]).unwrap(); + + if let Some(NotificationEvent::NotificationReceived { peer, notification }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(notification, vec![1, 3, 3, 8]); + } else { + panic!("invalid event received"); + } +} + +#[tokio::test] +async fn backpressure_works() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, mut async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + // fill the message buffer with messages + for i in 0..=ASYNC_NOTIFICATIONS_BUFFER_SIZE { + assert!(futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, i as u8])) + .is_ready()); + } + + // try to send one more message and verify that the call blocks + assert!(futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, 9])).is_pending()); + + // release one slot from the buffer for new message + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 0] }) + ); + + // verify that a message can be sent + assert!(futures::poll!(notif.send_async_notification(&peer_id, vec![1, 3, 3, 9])).is_ready()); +} + +#[tokio::test] +async fn peer_disconnects_then_sync_notification_is_sent() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, _, sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + // report that a substream has been closed but don't poll `notif` to receive this + // information + handle.report_substream_closed(peer_id).unwrap(); + drop(sync_rx); + + // as per documentation, error is not reported but the notification is silently dropped + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 7]); +} + +#[tokio::test] +async fn peer_disconnects_then_async_notification_is_sent() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, async_rx, _) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + // report that a substream has been closed but don't poll `notif` to receive this + // information + handle.report_substream_closed(peer_id).unwrap(); + drop(async_rx); + + // as per documentation, error is not reported but the notification is silently dropped + if let Err(error::Error::ConnectionClosed) = + notif.send_async_notification(&peer_id, vec![1, 3, 3, 7]).await + { + } else { + panic!("invalid state after calling `send_async_notificatio()` on closed connection") + } +} + +#[tokio::test] +async fn cloned_service_opening_substream_works() { + let (proto, mut notif1) = notification_service("/proto/1".into()); + let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); + let (handle, _stream) = proto.split(); + let mut notif2 = notif1.clone().unwrap(); + let peer_id = PeerId::random(); + + // validate inbound substream + let mut result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + // verify that `notif1` gets the event + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif1.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + + // verify that because only one listener has thus far send their result, the result is + // pending + assert!(result_rx.try_recv().is_err()); + + // verify that `notif2` also gets the event + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif2.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); +} + +#[tokio::test] +async fn cloned_service_one_service_rejects_substream() { + let (proto, mut notif1) = notification_service("/proto/1".into()); + let (_sink, _async_rx, _) = NotificationsSink::new(PeerId::random()); + let (handle, _stream) = proto.split(); + let mut notif2 = notif1.clone().unwrap(); + let mut notif3 = notif2.clone().unwrap(); + let peer_id = PeerId::random(); + + // validate inbound substream + let mut result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + for notif in vec![&mut notif1, &mut notif2] { + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + } + + // `notif3` has not yet sent their validation result + assert!(result_rx.try_recv().is_err()); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif3.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Reject).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Reject); +} + +#[tokio::test] +async fn cloned_service_opening_substream_sending_and_receiving_notifications_work() { + let (proto, mut notif1) = notification_service("/proto/1".into()); + let (sink, _, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let mut notif2 = notif1.clone().unwrap(); + let mut notif3 = notif1.clone().unwrap(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + // accept the inbound substream for all services + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that then notification stream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + } + // receive a notification from peer and verify all services receive it + handle.report_notification_received(peer_id, vec![1, 3, 3, 8]).unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + if let Some(NotificationEvent::NotificationReceived { peer, notification }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(notification, vec![1, 3, 3, 8]); + } else { + panic!("invalid event received"); + } + } + + for (i, notif) in vec![&mut notif1, &mut notif2, &mut notif3].iter().enumerate() { + // send notification from each service and verify peer receives it + notif.send_sync_notification(&peer_id, vec![1, 3, 3, i as u8]); + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, i as u8] }) + ); + } + + // close the substream for peer and verify all services receive the event + handle.report_substream_closed(peer_id).unwrap(); + + for notif in vec![&mut notif1, &mut notif2, &mut notif3] { + if let Some(NotificationEvent::NotificationStreamClosed { peer }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + } else { + panic!("invalid event received"); + } + } +} + +#[tokio::test] +async fn sending_notifications_using_notifications_sink_works() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, mut async_rx, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + // get a copy of the notification sink and send a synchronous notification using. + let sink = notif.message_sink(&peer_id).unwrap(); + sink.send_sync_notification(vec![1, 3, 3, 6]); + + // send an asynchronous notification using the acquired notifications sink. + let _ = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); + + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 6] }), + ); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 7] }), + ); + + // send notifications using the stored notification sink as well. + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }), + ); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }), + ); +} + +#[test] +fn try_to_get_notifications_sink_for_non_existent_peer() { + let (_proto, notif) = notification_service("/proto/1".into()); + assert!(notif.message_sink(&PeerId::random()).is_none()); +} + +#[tokio::test] +async fn notification_sink_replaced() { + let (proto, mut notif) = notification_service("/proto/1".into()); + let (sink, mut async_rx, mut sync_rx) = NotificationsSink::new(PeerId::random()); + let (mut handle, _stream) = proto.split(); + let peer_id = PeerId::random(); + + // validate inbound substream + let result_rx = handle.report_incoming_substream(peer_id, vec![1, 3, 3, 7]).unwrap(); + + if let Some(NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx }) = + notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(handshake, vec![1, 3, 3, 7]); + let _ = result_tx.send(ValidationResult::Accept).unwrap(); + } else { + panic!("invalid event received"); + } + assert_eq!(result_rx.await.unwrap(), ValidationResult::Accept); + + // report that a substream has been opened + handle + .report_substream_opened(peer_id, Direction::Inbound, vec![1, 3, 3, 7], None, sink) + .unwrap(); + + if let Some(NotificationEvent::NotificationStreamOpened { + peer, + negotiated_fallback, + handshake, + direction, + }) = notif.next_event().await + { + assert_eq!(peer_id, peer); + assert_eq!(negotiated_fallback, None); + assert_eq!(handshake, vec![1, 3, 3, 7]); + assert_eq!(direction, Direction::Inbound); + } else { + panic!("invalid event received"); + } + + // get a copy of the notification sink and send a synchronous notification using. + let sink = notif.message_sink(&peer_id).unwrap(); + sink.send_sync_notification(vec![1, 3, 3, 6]); + + // send an asynchronous notification using the acquired notifications sink. + let _ = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); + + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 6] }), + ); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 7] }), + ); + + // send notifications using the stored notification sink as well. + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + + assert_eq!( + sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }), + ); + assert_eq!( + async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }), + ); + + // the initial connection was closed and `Notifications` switched to secondary connection + // and emitted `CustomProtocolReplaced` which informs the local `NotificationService` that + // the notification sink was replaced. + let (new_sink, mut new_async_rx, mut new_sync_rx) = NotificationsSink::new(PeerId::random()); + handle.report_notification_sink_replaced(peer_id, new_sink).unwrap(); + + // drop the old sinks and poll `notif` once to register the sink replacement + drop(sync_rx); + drop(async_rx); + + futures::future::poll_fn(|cx| { + let _ = std::pin::Pin::new(&mut notif.next_event()).poll(cx); + std::task::Poll::Ready(()) + }) + .await; + + // verify that using the `NotificationService` API automatically results in using the correct + // sink + notif.send_sync_notification(&peer_id, vec![1, 3, 3, 8]); + notif.send_async_notification(&peer_id, vec![1, 3, 3, 9]).await.unwrap(); + + assert_eq!( + new_sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 8] }), + ); + assert_eq!( + new_async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 9] }), + ); + + // now send two notifications using the acquired message sink and verify that + // it's also updated + sink.send_sync_notification(vec![1, 3, 3, 6]); + + // send an asynchronous notification using the acquired notifications sink. + let _ = sink.send_async_notification(vec![1, 3, 3, 7]).await.unwrap(); + + assert_eq!( + new_sync_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 6] }), + ); + assert_eq!( + new_async_rx.next().await, + Some(NotificationsSinkMessage::Notification { message: vec![1, 3, 3, 7] }), + ); +} diff --git a/substrate/client/network/src/protocol/notifications/tests.rs b/substrate/client/network/src/protocol/notifications/tests.rs index d57c24144f57..30bbe0275054 100644 --- a/substrate/client/network/src/protocol/notifications/tests.rs +++ b/substrate/client/network/src/protocol/notifications/tests.rs @@ -22,6 +22,7 @@ use crate::{ peer_store::PeerStore, protocol::notifications::{Notifications, NotificationsOut, ProtocolConfig}, protocol_controller::{ProtoSetConfig, ProtocolController, SetId}, + service::traits::{NotificationEvent, ValidationResult}, }; use futures::{future::BoxFuture, prelude::*}; @@ -70,6 +71,8 @@ fn build_nodes() -> (Swarm, Swarm) { .timeout(Duration::from_secs(20)) .boxed(); + let (protocol_handle_pair, mut notif_service) = + crate::protocol::notifications::service::notification_service("/foo".into()); let peer_store = PeerStore::new(if index == 0 { keypairs.iter().skip(1).map(|keypair| keypair.public().to_peer_id()).collect() } else { @@ -95,12 +98,16 @@ fn build_nodes() -> (Swarm, Swarm) { inner: Notifications::new( vec![controller_handle], from_controller, - iter::once(ProtocolConfig { - name: "/foo".into(), - fallback_names: Vec::new(), - handshake: Vec::new(), - max_notification_size: 1024 * 1024, - }), + &None, + iter::once(( + ProtocolConfig { + name: "/foo".into(), + fallback_names: Vec::new(), + handshake: Vec::new(), + max_notification_size: 1024 * 1024, + }, + protocol_handle_pair, + )), ), peer_store_future: peer_store.run().boxed(), protocol_controller_future: controller.run().boxed(), @@ -118,6 +125,16 @@ fn build_nodes() -> (Swarm, Swarm) { }; let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.spawn(async move { + loop { + if let NotificationEvent::ValidateInboundSubstream { result_tx, .. } = + notif_service.next_event().await.unwrap() + { + result_tx.send(ValidationResult::Accept).unwrap(); + } + } + }); + let mut swarm = SwarmBuilder::with_executor( transport, behaviour, diff --git a/substrate/client/network/src/protocol_controller.rs b/substrate/client/network/src/protocol_controller.rs index c9baa0a77d4b..1c26d8ce1d89 100644 --- a/substrate/client/network/src/protocol_controller.rs +++ b/substrate/client/network/src/protocol_controller.rs @@ -840,6 +840,7 @@ mod tests { use super::*; use crate::{peer_store::PeerStoreProvider, ReputationChange}; use libp2p::PeerId; + use sc_network_common::role::ObservedRole; use sc_utils::mpsc::{tracing_unbounded, TryRecvError}; use std::collections::HashSet; @@ -851,8 +852,10 @@ mod tests { fn is_banned(&self, peer_id: &PeerId) -> bool; fn register_protocol(&self, protocol_handle: ProtocolHandle); fn report_disconnect(&mut self, peer_id: PeerId); + fn set_peer_role(&mut self, peer_id: &PeerId, role: ObservedRole); fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange); fn peer_reputation(&self, peer_id: &PeerId) -> i32; + fn peer_role(&self, peer_id: &PeerId) -> Option; fn outgoing_candidates<'a>(&self, count: usize, ignored: HashSet<&'a PeerId>) -> Vec; } } diff --git a/substrate/client/network/src/service.rs b/substrate/client/network/src/service.rs index aca0072a31de..16ef1d1cd7df 100644 --- a/substrate/client/network/src/service.rs +++ b/substrate/client/network/src/service.rs @@ -54,6 +54,7 @@ use crate::{ ReputationChange, }; +use codec::DecodeAll; use either::Either; use futures::{channel::oneshot, prelude::*}; #[allow(deprecated)] @@ -74,7 +75,10 @@ use log::{debug, error, info, trace, warn}; use metrics::{Histogram, HistogramVec, MetricSources, Metrics}; use parking_lot::Mutex; -use sc_network_common::ExHashT; +use sc_network_common::{ + role::{ObservedRole, Roles}, + ExHashT, +}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_runtime::traits::Block as BlockT; @@ -94,7 +98,7 @@ use std::{ pub use behaviour::{InboundFailure, OutboundFailure, ResponseFailure}; pub use libp2p::identity::{DecodingError, Keypair, PublicKey}; -pub use protocol::NotificationsSink; +pub use protocol::{NotificationCommand, NotificationsSink, ProtocolHandle}; mod metrics; mod out_events; @@ -132,6 +136,8 @@ pub struct NetworkService { protocol_handles: Vec, /// Shortcut to sync protocol handle (`protocol_handles[0]`). sync_protocol_handle: protocol_controller::ProtocolHandle, + /// Handle to `PeerStore`. + peer_store_handle: PeerStoreHandle, /// Marker to pin the `H` generic. Serves no purpose except to not break backwards /// compatibility. _marker: PhantomData, @@ -199,7 +205,7 @@ where )?; for notification_protocol in ¬ification_protocols { ensure_addresses_consistent_with_transport( - notification_protocol.set_config.reserved_nodes.iter().map(|x| &x.multiaddr), + notification_protocol.set_config().reserved_nodes.iter().map(|x| &x.multiaddr), &network_config.transport, )?; } @@ -241,7 +247,7 @@ where .map(|cfg| usize::try_from(cfg.max_response_size).unwrap_or(usize::MAX)); let notifs_max = notification_protocols .iter() - .map(|cfg| usize::try_from(cfg.max_notification_size).unwrap_or(usize::MAX)); + .map(|cfg| usize::try_from(cfg.max_notification_size()).unwrap_or(usize::MAX)); // A "default" max is added to cover all the other protocols: ping, identify, // kademlia, block announces, and transactions. @@ -273,7 +279,7 @@ where // We must prepend a hardcoded default peer set to notification protocols. let all_peer_sets_iter = iter::once(&network_config.default_peers_set) - .chain(notification_protocols.iter().map(|protocol| &protocol.set_config)); + .chain(notification_protocols.iter().map(|protocol| protocol.set_config())); let (protocol_handles, protocol_controllers): (Vec<_>, Vec<_>) = all_peer_sets_iter .enumerate() @@ -312,21 +318,9 @@ where iter::once(¶ms.block_announce_config) .chain(notification_protocols.iter()) .enumerate() - .map(|(index, protocol)| { - (protocol.notifications_protocol.clone(), SetId::from(index)) - }) + .map(|(index, protocol)| (protocol.protocol_name().clone(), SetId::from(index))) .collect(); - let protocol = Protocol::new( - From::from(¶ms.role), - notification_protocols.clone(), - params.block_announce_config, - params.peer_store.clone(), - protocol_handles.clone(), - from_protocol_controllers, - params.tx, - )?; - let known_addresses = { // Collect all reserved nodes and bootnodes addresses. let mut addresses: Vec<_> = network_config @@ -336,7 +330,7 @@ where .map(|reserved| (reserved.peer_id, reserved.multiaddr.clone())) .chain(notification_protocols.iter().flat_map(|protocol| { protocol - .set_config + .set_config() .reserved_nodes .iter() .map(|reserved| (reserved.peer_id, reserved.multiaddr.clone())) @@ -389,6 +383,16 @@ where let num_connected = Arc::new(AtomicUsize::new(0)); let external_addresses = Arc::new(Mutex::new(HashSet::new())); + let protocol = Protocol::new( + From::from(¶ms.role), + ¶ms.metrics_registry, + notification_protocols, + params.block_announce_config, + params.peer_store.clone(), + protocol_handles.clone(), + from_protocol_controllers, + )?; + // Build the swarm. let (mut swarm, bandwidth): (Swarm>, _) = { let user_agent = @@ -525,6 +529,7 @@ where notification_protocol_ids, protocol_handles, sync_protocol_handle, + peer_store_handle: params.peer_store.clone(), _marker: PhantomData, _block: Default::default(), }); @@ -943,9 +948,10 @@ where peers: HashSet, ) -> Result<(), String> { let Some(set_id) = self.notification_protocol_ids.get(&protocol) else { - return Err( - format!("Cannot add peers to reserved set of unknown protocol: {}", protocol) - ) + return Err(format!( + "Cannot add peers to reserved set of unknown protocol: {}", + protocol + )) }; let peers = self.split_multiaddr_and_peer_id(peers)?; @@ -974,9 +980,10 @@ where peers: Vec, ) -> Result<(), String> { let Some(set_id) = self.notification_protocol_ids.get(&protocol) else { - return Err( - format!("Cannot remove peers from reserved set of unknown protocol: {}", protocol) - ) + return Err(format!( + "Cannot remove peers from reserved set of unknown protocol: {}", + protocol + )) }; for peer_id in peers.into_iter() { @@ -989,6 +996,16 @@ where fn sync_num_connected(&self) -> usize { self.num_connected.load(Ordering::Relaxed) } + + fn peer_role(&self, peer_id: PeerId, handshake: Vec) -> Option { + match Roles::decode_all(&mut &handshake[..]) { + Ok(role) => Some(role.into()), + Err(_) => { + log::debug!(target: "sub-libp2p", "handshake doesn't contain peer role: {handshake:?}"); + self.peer_store_handle.peer_role(&peer_id) + }, + } + } } impl NetworkEventStream for NetworkService diff --git a/substrate/client/network/src/service/signature.rs b/substrate/client/network/src/service/signature.rs index 024f60e4c466..5b2ba6be8cf8 100644 --- a/substrate/client/network/src/service/signature.rs +++ b/substrate/client/network/src/service/signature.rs @@ -18,6 +18,8 @@ // // If you read this, you are very thorough, congratulations. +//! Signature-related code + use libp2p::{ identity::{Keypair, PublicKey}, PeerId, diff --git a/substrate/client/network/src/service/traits.rs b/substrate/client/network/src/service/traits.rs index bed325ede4a8..a62283155166 100644 --- a/substrate/client/network/src/service/traits.rs +++ b/substrate/client/network/src/service/traits.rs @@ -18,8 +18,11 @@ // // If you read this, you are very thorough, congratulations. +//! Traits defined by `sc-network`. + use crate::{ config::MultiaddrWithPeerId, + error, event::Event, request_responses::{IfDisconnected, RequestFailure}, service::signature::Signature, @@ -30,7 +33,9 @@ use crate::{ use futures::{channel::oneshot, Stream}; use libp2p::{Multiaddr, PeerId}; -use std::{collections::HashSet, future::Future, pin::Pin, sync::Arc}; +use sc_network_common::role::ObservedRole; + +use std::{collections::HashSet, fmt::Debug, future::Future, pin::Pin, sync::Arc}; pub use libp2p::{identity::SigningError, kad::record::Key as KademliaKey}; @@ -221,6 +226,14 @@ pub trait NetworkPeers { /// Returns the number of peers in the sync peer set we're connected to. fn sync_num_connected(&self) -> usize; + + /// Attempt to get peer role. + /// + /// Right now the peer role is decoded from the received handshake for all protocols + /// (`/block-announces/1` has other information as well). If the handshake cannot be + /// decoded into a role, the role queried from `PeerStore` and if the role is not stored + /// there either, `None` is returned and the peer should be discarded. + fn peer_role(&self, peer_id: PeerId, handshake: Vec) -> Option; } // Manual implementation to avoid extra boxing here @@ -296,6 +309,10 @@ where fn sync_num_connected(&self) -> usize { T::sync_num_connected(self) } + + fn peer_role(&self, peer_id: PeerId, handshake: Vec) -> Option { + T::peer_role(self, peer_id, handshake) + } } /// Provides access to network-level event stream. @@ -611,3 +628,182 @@ where T::new_best_block_imported(self, hash, number) } } + +/// Substream acceptance result. +#[derive(Debug, PartialEq, Eq)] +pub enum ValidationResult { + /// Accept inbound substream. + Accept, + + /// Reject inbound substream. + Reject, +} + +/// Substream direction. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum Direction { + /// Substream opened by the remote node. + Inbound, + + /// Substream opened by the local node. + Outbound, +} + +impl Direction { + /// Is the direction inbound. + pub fn is_inbound(&self) -> bool { + std::matches!(self, Direction::Inbound) + } +} + +/// Events received by the protocol from `Notifications`. +#[derive(Debug)] +pub enum NotificationEvent { + /// Validate inbound substream. + ValidateInboundSubstream { + /// Peer ID. + peer: PeerId, + + /// Received handshake. + handshake: Vec, + + /// `oneshot::Sender` for sending validation result back to `Notifications` + result_tx: tokio::sync::oneshot::Sender, + }, + + /// Remote identified by `PeerId` opened a substream and sent `Handshake`. + /// Validate `Handshake` and report status (accept/reject) to `Notifications`. + NotificationStreamOpened { + /// Peer ID. + peer: PeerId, + + /// Is the substream inbound or outbound. + direction: Direction, + + /// Received handshake. + handshake: Vec, + + /// Negotiated fallback. + negotiated_fallback: Option, + }, + + /// Substream was closed. + NotificationStreamClosed { + /// Peer Id. + peer: PeerId, + }, + + /// Notification was received from the substream. + NotificationReceived { + /// Peer ID. + peer: PeerId, + + /// Received notification. + notification: Vec, + }, +} + +/// Notification service +/// +/// Defines behaviors that both the protocol implementations and `Notifications` can expect from +/// each other. +/// +/// `Notifications` can send two different kinds of information to protocol: +/// * substream-related information +/// * notification-related information +/// +/// When an unvalidated, inbound substream is received by `Notifications`, it sends the inbound +/// stream information (peer ID, handshake) to protocol for validation. Protocol must then verify +/// that the handshake is valid (and in the future that it has a slot it can allocate for the peer) +/// and then report back the `ValidationResult` which is either `Accept` or `Reject`. +/// +/// After the validation result has been received by `Notifications`, it prepares the +/// substream for communication by initializing the necessary sinks and emits +/// `NotificationStreamOpened` which informs the protocol that the remote peer is ready to receive +/// notifications. +/// +/// Two different flavors of sending options are provided: +/// * synchronous sending ([`NotificationService::send_sync_notification()`]) +/// * asynchronous sending ([`NotificationService::send_async_notification()`]) +/// +/// The former is used by the protocols not ready to exercise backpressure and the latter by the +/// protocols that can do it. +/// +/// Both local and remote peer can close the substream at any time. Local peer can do so by calling +/// [`NotificationService::close_substream()`] which instrucs `Notifications` to close the +/// substream. Remote closing the substream is indicated to the local peer by receiving +/// [`NotificationEvent::NotificationStreamClosed`] event. +/// +/// In case the protocol must update its handshake while it's operating (such as updating the best +/// block information), it can do so by calling [`NotificationService::set_handshake()`] +/// which instructs `Notifications` to update the handshake it stored during protocol +/// initialization. +/// +/// All peer events are multiplexed on the same incoming event stream from `Notifications` and thus +/// each event carries a `PeerId` so the protocol knows whose information to update when receiving +/// an event. +#[async_trait::async_trait] +pub trait NotificationService: Debug + Send { + /// Instruct `Notifications` to open a new substream for `peer`. + /// + /// `dial_if_disconnected` informs `Notifications` whether to dial + // the peer if there is currently no active connection to it. + // + // NOTE: not offered by the current implementation + async fn open_substream(&mut self, peer: PeerId) -> Result<(), ()>; + + /// Instruct `Notifications` to close substream for `peer`. + // + // NOTE: not offered by the current implementation + async fn close_substream(&mut self, peer: PeerId) -> Result<(), ()>; + + /// Send synchronous `notification` to `peer`. + fn send_sync_notification(&self, peer: &PeerId, notification: Vec); + + /// Send asynchronous `notification` to `peer`, allowing sender to exercise backpressure. + /// + /// Returns an error if the peer doesn't exist. + async fn send_async_notification( + &self, + peer: &PeerId, + notification: Vec, + ) -> Result<(), error::Error>; + + /// Set handshake for the notification protocol replacing the old handshake. + async fn set_handshake(&mut self, handshake: Vec) -> Result<(), ()>; + + /// Get next event from the `Notifications` event stream. + async fn next_event(&mut self) -> Option; + + /// Make a copy of the object so it can be shared between protocol components + /// who wish to have access to the same underlying notification protocol. + fn clone(&mut self) -> Result, ()>; + + /// Get protocol name of the `NotificationService`. + fn protocol(&self) -> &ProtocolName; + + /// Get message sink of the peer. + fn message_sink(&self, peer: &PeerId) -> Option>; +} + +/// Message sink for peers. +/// +/// If protocol cannot use [`NotificationService`] to send notifications to peers and requires, +/// e.g., notifications to be sent in another task, the protocol may acquire a [`MessageSink`] +/// object for each peer by calling [`NotificationService::message_sink()`]. Calling this +/// function returns an object which allows the protocol to send notifications to the remote peer. +/// +/// Use of this API is discouraged as it's not as performant as sending notifications through +/// [`NotificationService`] due to synchronization required to keep the underlying notification +/// sink up to date with possible sink replacement events. +#[async_trait::async_trait] +pub trait MessageSink: Send { + /// Send synchronous `notification` to the peer associated with this [`MessageSink`]. + fn send_sync_notification(&self, notification: Vec); + + /// Send an asynchronous `notification` to to the peer associated with this [`MessageSink`], + /// allowing sender to exercise backpressure. + /// + /// Returns an error if the peer does not exist. + async fn send_async_notification(&self, notification: Vec) -> Result<(), error::Error>; +} diff --git a/substrate/client/network/statement/src/lib.rs b/substrate/client/network/statement/src/lib.rs index c5d83b59b260..ae73034a975f 100644 --- a/substrate/client/network/statement/src/lib.rs +++ b/substrate/client/network/statement/src/lib.rs @@ -21,12 +21,13 @@ //! Usage: //! //! - Use [`StatementHandlerPrototype::new`] to create a prototype. -//! - Pass the return value of [`StatementHandlerPrototype::set_config`] to the network -//! configuration as an extra peers set. +//! - Pass the `NonDefaultSetConfig` returned from [`StatementHandlerPrototype::new`] to the network +//! configuration as an extra peers set. //! - Use [`StatementHandlerPrototype::build`] then [`StatementHandler::run`] to obtain a //! `Future` that processes statements. use crate::config::*; + use codec::{Decode, Encode}; use futures::{channel::oneshot, prelude::*, stream::FuturesUnordered, FutureExt}; use libp2p::{multiaddr, PeerId}; @@ -34,7 +35,7 @@ use prometheus_endpoint::{register, Counter, PrometheusError, Registry, U64}; use sc_network::{ config::{NonDefaultSetConfig, NonReservedPeerMode, SetConfig}, error, - event::Event, + service::traits::{NotificationEvent, NotificationService, ValidationResult}, types::ProtocolName, utils::{interval, LruHashSet}, NetworkEventStream, NetworkNotification, NetworkPeers, @@ -103,35 +104,35 @@ impl Metrics { /// Prototype for a [`StatementHandler`]. pub struct StatementHandlerPrototype { protocol_name: ProtocolName, + notification_service: Box, } impl StatementHandlerPrototype { /// Create a new instance. - pub fn new>(genesis_hash: Hash, fork_id: Option<&str>) -> Self { + pub fn new>( + genesis_hash: Hash, + fork_id: Option<&str>, + ) -> (Self, NonDefaultSetConfig) { let genesis_hash = genesis_hash.as_ref(); let protocol_name = if let Some(fork_id) = fork_id { format!("/{}/{}/statement/1", array_bytes::bytes2hex("", genesis_hash), fork_id) } else { format!("/{}/statement/1", array_bytes::bytes2hex("", genesis_hash)) }; - - Self { protocol_name: protocol_name.into() } - } - - /// Returns the configuration of the set to put in the network configuration. - pub fn set_config(&self) -> NonDefaultSetConfig { - NonDefaultSetConfig { - notifications_protocol: self.protocol_name.clone(), - fallback_names: Vec::new(), - max_notification_size: MAX_STATEMENT_SIZE, - handshake: None, - set_config: SetConfig { + let (config, notification_service) = NonDefaultSetConfig::new( + protocol_name.clone().into(), + Vec::new(), + MAX_STATEMENT_SIZE, + None, + SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - } + ); + + (Self { protocol_name: protocol_name.into(), notification_service }, config) } /// Turns the prototype into the actual handler. @@ -149,7 +150,6 @@ impl StatementHandlerPrototype { metrics_registry: Option<&Registry>, executor: impl Fn(Pin + Send>>) + Send, ) -> error::Result> { - let net_event_stream = network.event_stream("statement-handler-net"); let sync_event_stream = sync.event_stream("statement-handler-sync"); let (queue_sender, mut queue_receiver) = async_channel::bounded(100_000); @@ -178,6 +178,7 @@ impl StatementHandlerPrototype { let handler = StatementHandler { protocol_name: self.protocol_name, + notification_service: self.notification_service, propagate_timeout: (Box::pin(interval(PROPAGATE_TIMEOUT)) as Pin + Send>>) .fuse(), @@ -185,7 +186,6 @@ impl StatementHandlerPrototype { pending_statements_peers: HashMap::new(), network, sync, - net_event_stream: net_event_stream.fuse(), sync_event_stream: sync_event_stream.fuse(), peers: HashMap::new(), statement_store, @@ -221,10 +221,10 @@ pub struct StatementHandler< network: N, /// Syncing service. sync: S, - /// Stream of networking events. - net_event_stream: stream::Fuse + Send>>>, /// Receiver for syncing-related events. sync_event_stream: stream::Fuse + Send>>>, + /// Notification service. + notification_service: Box, // All connected peers peers: HashMap, statement_store: Arc, @@ -263,14 +263,6 @@ where log::warn!(target: LOG_TARGET, "Inconsistent state, no peers for pending statement!"); } }, - network_event = self.net_event_stream.next() => { - if let Some(network_event) = network_event { - self.handle_network_event(network_event).await; - } else { - // Networking has seemingly closed. Closing as well. - return; - } - }, sync_event = self.sync_event_stream.next() => { if let Some(sync_event) = sync_event { self.handle_sync_event(sync_event); @@ -279,6 +271,14 @@ where return; } } + event = self.notification_service.next_event().fuse() => { + if let Some(event) = event { + self.handle_notification_event(event) + } else { + // `Notifications` has seemingly closed. Closing as well. + return + } + } } } } @@ -308,14 +308,19 @@ where } } - async fn handle_network_event(&mut self, event: Event) { + fn handle_notification_event(&mut self, event: NotificationEvent) { match event { - Event::Dht(_) => {}, - Event::NotificationStreamOpened { remote, protocol, role, .. } - if protocol == self.protocol_name => - { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + let _ = result_tx.send(ValidationResult::Accept); + }, + NotificationEvent::NotificationStreamOpened { peer, handshake, .. } => { + let Some(role) = self.network.peer_role(peer, handshake) else { + log::debug!(target: LOG_TARGET, "role for {peer} couldn't be determined"); + return + }; + let _was_in = self.peers.insert( - remote, + peer, Peer { known_statements: LruHashSet::new( NonZeroUsize::new(MAX_KNOWN_STATEMENTS).expect("Constant is nonzero"), @@ -325,39 +330,26 @@ where ); debug_assert!(_was_in.is_none()); }, - Event::NotificationStreamClosed { remote, protocol } - if protocol == self.protocol_name => - { - let _peer = self.peers.remove(&remote); + NotificationEvent::NotificationStreamClosed { peer } => { + let _peer = self.peers.remove(&peer); debug_assert!(_peer.is_some()); }, + NotificationEvent::NotificationReceived { peer, notification } => { + // Accept statements only when node is not major syncing + if self.sync.is_major_syncing() { + log::trace!( + target: LOG_TARGET, + "{peer}: Ignoring statements while major syncing or offline" + ); + return + } - Event::NotificationsReceived { remote, messages } => { - for (protocol, message) in messages { - if protocol != self.protocol_name { - continue - } - // Accept statements only when node is not major syncing - if self.sync.is_major_syncing() { - log::trace!( - target: LOG_TARGET, - "{remote}: Ignoring statements while major syncing or offline" - ); - continue - } - if let Ok(statements) = ::decode(&mut message.as_ref()) { - self.on_statements(remote, statements); - } else { - log::debug!( - target: LOG_TARGET, - "Failed to decode statement list from {remote}" - ); - } + if let Ok(statements) = ::decode(&mut notification.as_ref()) { + self.on_statements(peer, statements); + } else { + log::debug!(target: LOG_TARGET, "Failed to decode statement list from {peer}"); } }, - - // Not our concern. - Event::NotificationStreamOpened { .. } | Event::NotificationStreamClosed { .. } => {}, } } diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 65bd56a28958..71abbcdd7f21 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -24,7 +24,7 @@ use crate::{ ChainSync, ClientError, SyncingService, }; -use codec::{Decode, Encode}; +use codec::{Decode, DecodeAll, Encode}; use futures::{FutureExt, StreamExt}; use futures_timer::Delay; use libp2p::PeerId; @@ -37,8 +37,11 @@ use sc_client_api::{BlockBackend, HeaderBackend, ProofProvider}; use sc_consensus::import_queue::ImportQueueService; use sc_network::{ config::{FullNetworkConfiguration, NonDefaultSetConfig, ProtocolId}, + peer_store::{PeerStoreHandle, PeerStoreProvider}, + service::traits::{Direction, NotificationEvent, ValidationResult}, + types::ProtocolName, utils::LruHashSet, - NotificationsSink, ProtocolName, ReputationChange, + NotificationService, ReputationChange, }; use sc_network_common::{ role::Roles, @@ -51,7 +54,7 @@ use sc_network_common::{ use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_blockchain::HeaderMetadata; use sp_consensus::block_validation::BlockAnnounceValidator; -use sp_runtime::traits::{Block as BlockT, Header, NumberFor, Zero}; +use sp_runtime::traits::{Block as BlockT, Header, Zero}; use std::{ collections::{HashMap, HashSet}, @@ -70,6 +73,9 @@ const TICK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(1100) /// Maximum number of known block hashes to keep for a peer. const MAX_KNOWN_BLOCKS: usize = 1024; // ~32kb per peer + LruHashSet overhead +/// Logging target for the file. +const LOG_TARGET: &str = "sync"; + /// If the block announces stream to peer has been inactive for 30 seconds meaning local node /// has not sent or received block announcements to/from the peer, report the node for inactivity, /// disconnect it and attempt to establish connection to some other peer. @@ -172,8 +178,6 @@ pub struct Peer { pub info: ExtendedPeerInfo, /// Holds a set of blocks known to this peer. pub known_blocks: LruHashSet, - /// Notification sink. - sink: NotificationsSink, /// Is the peer inbound. inbound: bool, } @@ -198,9 +202,6 @@ pub struct SyncingEngine { /// Channel for receiving service commands service_rx: TracingUnboundedReceiver>, - /// Channel for receiving inbound connections from `Protocol`. - rx: sc_utils::mpsc::TracingUnboundedReceiver>, - /// Assigned roles. roles: Roles, @@ -251,12 +252,18 @@ pub struct SyncingEngine { /// Prometheus metrics. metrics: Option, + /// Handle that is used to communicate with `sc_network::Notifications`. + notification_service: Box, + /// When the syncing was started. /// /// Stored as an `Option` so once the initial wait has passed, `SyncingEngine` /// can reset the peer timers and continue with the normal eviction process. syncing_started: Option, + /// Handle to `PeerStore`. + peer_store_handle: PeerStoreHandle, + /// Instant when the last notification was sent or received. last_notification_io: Instant, } @@ -286,7 +293,7 @@ where block_request_protocol_name: ProtocolName, state_request_protocol_name: ProtocolName, warp_sync_protocol_name: Option, - rx: sc_utils::mpsc::TracingUnboundedReceiver>, + peer_store_handle: PeerStoreHandle, ) -> Result<(Self, SyncingService, NonDefaultSetConfig), ClientError> { let mode = net_config.network_config.sync_mode; let max_parallel_downloads = net_config.network_config.max_parallel_downloads; @@ -308,7 +315,7 @@ where } for config in net_config.notification_protocols() { let peer_ids = config - .set_config + .set_config() .reserved_nodes .iter() .map(|info| info.peer_id) @@ -346,7 +353,7 @@ where total.saturating_sub(net_config.network_config.default_peers_set_num_full) as usize }; - let (chain_sync, block_announce_config) = ChainSync::new( + let (chain_sync, block_announce_config, notification_service) = ChainSync::new( mode, client.clone(), protocol_id, @@ -364,7 +371,7 @@ where warp_sync_protocol_name, )?; - let block_announce_protocol_name = block_announce_config.notifications_protocol.clone(); + let block_announce_protocol_name = block_announce_config.protocol_name().clone(); let (tx, service_rx) = tracing_unbounded("mpsc_chain_sync", 100_000); let num_connected = Arc::new(AtomicUsize::new(0)); let is_major_syncing = Arc::new(AtomicBool::new(false)); @@ -392,7 +399,6 @@ where num_connected: num_connected.clone(), is_major_syncing: is_major_syncing.clone(), service_rx, - rx, genesis_hash, important_peers, default_peers_set_no_slot_connected_peers: HashSet::new(), @@ -403,8 +409,10 @@ where num_in_peers: 0usize, max_in_peers, event_streams: Vec::new(), + notification_service, tick_timeout: Delay::new(TICK_TIMEOUT), syncing_started: None, + peer_store_handle, last_notification_io: Instant::now(), metrics: if let Some(r) = metrics_registry { match Metrics::register(r, is_major_syncing.clone()) { @@ -569,23 +577,11 @@ where }; self.last_notification_io = Instant::now(); - peer.sink.send_sync_notification(message.encode()); + let _ = self.notification_service.send_sync_notification(who, message.encode()); } } } - /// Inform sync about new best imported block. - pub fn new_best_block_imported(&mut self, hash: B::Hash, number: NumberFor) { - log::debug!(target: "sync", "New best block imported {:?}/#{}", hash, number); - - self.chain_sync.update_chain_info(&hash, number); - self.network_service.set_notification_handshake( - self.block_announce_protocol_name.clone(), - BlockAnnouncesHandshake::::build(self.roles, number, hash, self.genesis_hash) - .encode(), - ) - } - pub async fn run(mut self) { self.syncing_started = Some(Instant::now()); @@ -669,8 +665,25 @@ where } }, ToServiceCommand::AnnounceBlock(hash, data) => self.announce_block(hash, data), - ToServiceCommand::NewBestBlockImported(hash, number) => - self.new_best_block_imported(hash, number), + ToServiceCommand::NewBestBlockImported(hash, number) => { + log::debug!(target: "sync", "New best block imported {:?}/#{}", hash, number); + + self.chain_sync.update_chain_info(&hash, number); + // TODO: remove once `SyncingEngine` is refactored + while let Poll::Pending = self + .notification_service + .set_handshake( + BlockAnnouncesHandshake::::build( + self.roles, + number, + hash, + self.genesis_hash, + ) + .encode(), + ) + .poll_unpin(cx) + {} + }, ToServiceCommand::Status(tx) => { let mut status = self.chain_sync.status(); status.num_connected_peers = self.peers.len() as u32; @@ -707,64 +720,74 @@ where } } - while let Poll::Ready(Some(event)) = self.rx.poll_next_unpin(cx) { + loop { + let Poll::Ready(Some(event)) = self.notification_service.next_event().poll_unpin(cx) + else { + break + }; + match event { - sc_network::SyncEvent::NotificationStreamOpened { - remote, - received_handshake, - sink, - inbound, - tx, - } => match self.on_sync_peer_connected(remote, &received_handshake, sink, inbound) { - Ok(()) => { - let _ = tx.send(true); - }, - Err(()) => { - log::debug!( - target: "sync", - "Failed to register peer {remote:?}: {received_handshake:?}", - ); - let _ = tx.send(false); - }, - }, - sc_network::SyncEvent::NotificationStreamClosed { remote } => { - if self.on_sync_peer_disconnected(remote).is_err() { - log::trace!( - target: "sync", - "Disconnected peer which had earlier been refused by on_sync_peer_connected {}", - remote - ); - } + NotificationEvent::ValidateInboundSubstream { peer, handshake, result_tx } => { + let validation_result = self + .validate_connection(&peer, handshake, Direction::Inbound) + .map_or(ValidationResult::Reject, |_| ValidationResult::Accept); + + let _ = result_tx.send(validation_result); }, - sc_network::SyncEvent::NotificationsReceived { remote, messages } => { - for message in messages { - if self.peers.contains_key(&remote) { - if let Ok(announce) = BlockAnnounce::decode(&mut message.as_ref()) { - self.last_notification_io = Instant::now(); - self.push_block_announce_validation(remote, announce); - - // Make sure that the newly added block announce validation future - // was polled once to be registered in the task. - if let Poll::Ready(res) = - self.chain_sync.poll_block_announce_validation(cx) - { - self.process_block_announce_validation_result(res) - } - } else { - log::warn!(target: "sub-libp2p", "Failed to decode block announce"); + NotificationEvent::NotificationStreamOpened { + peer, handshake, direction, .. + } => { + log::debug!( + target: LOG_TARGET, + "Substream opened for {peer}, handshake {handshake:?}" + ); + + match self.validate_connection(&peer, handshake, direction) { + Ok(handshake) => { + if self.on_sync_peer_connected(peer, &handshake, direction).is_err() { + log::debug!(target: LOG_TARGET, "Failed to register peer {peer}"); + self.network_service.disconnect_peer( + peer, + self.block_announce_protocol_name.clone(), + ); } - } else { - log::trace!( - target: "sync", - "Received sync for peer earlier refused by sync layer: {}", - remote - ); - } + }, + Err(wrong_genesis) => { + log::debug!(target: LOG_TARGET, "`SyncingEngine` rejected {peer}"); + + if wrong_genesis { + self.peer_store_handle.report_peer(peer, rep::GENESIS_MISMATCH); + } + + self.network_service + .disconnect_peer(peer, self.block_announce_protocol_name.clone()); + }, } }, - sc_network::SyncEvent::NotificationSinkReplaced { remote, sink } => { - if let Some(peer) = self.peers.get_mut(&remote) { - peer.sink = sink; + NotificationEvent::NotificationStreamClosed { peer } => { + self.on_sync_peer_disconnected(peer); + }, + NotificationEvent::NotificationReceived { peer, notification } => { + if !self.peers.contains_key(&peer) { + log::error!( + target: LOG_TARGET, + "received notification from {peer} who had been earlier refused by `SyncingEngine`", + ); + continue + } + + let Ok(announce) = BlockAnnounce::decode(&mut notification.as_ref()) else { + log::warn!(target: LOG_TARGET, "failed to decode block announce"); + continue + }; + + // push the block announcement for validation and make sure it's polled + // once to register it in the task. + self.last_notification_io = Instant::now(); + self.push_block_announce_validation(peer, announce); + + if let Poll::Ready(res) = self.chain_sync.poll_block_announce_validation(cx) { + self.process_block_announce_validation_result(res) } }, } @@ -783,120 +806,166 @@ where /// Called by peer when it is disconnecting. /// /// Returns a result if the handshake of this peer was indeed accepted. - pub fn on_sync_peer_disconnected(&mut self, peer: PeerId) -> Result<(), ()> { - if let Some(info) = self.peers.remove(&peer) { - if self.important_peers.contains(&peer) { - log::warn!(target: "sync", "Reserved peer {} disconnected", peer); - } else { - log::debug!(target: "sync", "{} disconnected", peer); - } - - if !self.default_peers_set_no_slot_connected_peers.remove(&peer) && - info.inbound && info.info.roles.is_full() - { - match self.num_in_peers.checked_sub(1) { - Some(value) => { - self.num_in_peers = value; - }, - None => { - log::error!( - target: "sync", - "trying to disconnect an inbound node which is not counted as inbound" - ); - debug_assert!(false); - }, - } - } + pub fn on_sync_peer_disconnected(&mut self, peer: PeerId) { + let Some(info) = self.peers.remove(&peer) else { + log::debug!(target: LOG_TARGET, "{peer} does not exist in `SyncingEngine`"); + return + }; - self.chain_sync.peer_disconnected(&peer); - self.event_streams - .retain(|stream| stream.unbounded_send(SyncEvent::PeerDisconnected(peer)).is_ok()); - Ok(()) + if self.important_peers.contains(&peer) { + log::warn!(target: "sync", "Reserved peer {} disconnected", peer); } else { - Err(()) + log::debug!(target: "sync", "{} disconnected", peer); } - } - /// Called on the first connection between two peers on the default set, after their exchange - /// of handshake. - /// - /// Returns `Ok` if the handshake is accepted and the peer added to the list of peers we sync - /// from. - pub fn on_sync_peer_connected( - &mut self, - who: PeerId, - status: &BlockAnnouncesHandshake, - sink: NotificationsSink, - inbound: bool, - ) -> Result<(), ()> { - log::trace!(target: "sync", "New peer {} {:?}", who, status); - - if self.peers.contains_key(&who) { - log::error!(target: "sync", "Called on_sync_peer_connected with already connected peer {}", who); - debug_assert!(false); - return Err(()) + if !self.default_peers_set_no_slot_connected_peers.remove(&peer) && + info.inbound && info.info.roles.is_full() + { + match self.num_in_peers.checked_sub(1) { + Some(value) => { + self.num_in_peers = value; + }, + None => { + log::error!( + target: "sync", + "trying to disconnect an inbound node which is not counted as inbound" + ); + debug_assert!(false); + }, + } } - if status.genesis_hash != self.genesis_hash { - self.network_service.report_peer(who, rep::GENESIS_MISMATCH); + self.chain_sync.peer_disconnected(&peer); + self.event_streams + .retain(|stream| stream.unbounded_send(SyncEvent::PeerDisconnected(peer)).is_ok()); + } - if self.important_peers.contains(&who) { + /// Validate received handshake. + fn validate_handshake( + &mut self, + peer_id: &PeerId, + handshake: Vec, + ) -> Result, bool> { + log::trace!(target: LOG_TARGET, "Validate handshake for {peer_id}"); + + let handshake = as DecodeAll>::decode_all(&mut &handshake[..]) + .map_err(|error| { + log::debug!(target: LOG_TARGET, "Failed to decode handshake for {peer_id}: {error:?}"); + false + })?; + + if handshake.genesis_hash != self.genesis_hash { + if self.important_peers.contains(&peer_id) { log::error!( - target: "sync", - "Reserved peer id `{}` is on a different chain (our genesis: {} theirs: {})", - who, + target: LOG_TARGET, + "Reserved peer id `{peer_id}` is on a different chain (our genesis: {} theirs: {})", self.genesis_hash, - status.genesis_hash, + handshake.genesis_hash, ); - } else if self.boot_node_ids.contains(&who) { + } else if self.boot_node_ids.contains(&peer_id) { log::error!( - target: "sync", - "Bootnode with peer id `{}` is on a different chain (our genesis: {} theirs: {})", - who, + target: LOG_TARGET, + "Bootnode with peer id `{peer_id}` is on a different chain (our genesis: {} theirs: {})", self.genesis_hash, - status.genesis_hash, + handshake.genesis_hash, ); } else { log::debug!( - target: "sync", + target: LOG_TARGET, "Peer is on different chain (our genesis: {} theirs: {})", - self.genesis_hash, status.genesis_hash + self.genesis_hash, + handshake.genesis_hash ); } - return Err(()) + return Err(true) } - let no_slot_peer = self.default_peers_set_no_slot_peers.contains(&who); - let this_peer_reserved_slot: usize = if no_slot_peer { 1 } else { 0 }; + Ok(handshake) + } - // make sure to accept no more than `--in-peers` many full nodes - if !no_slot_peer && - status.roles.is_full() && - inbound && self.num_in_peers == self.max_in_peers - { - log::debug!(target: "sync", "All inbound slots have been consumed, rejecting {who}"); - return Err(()) + /// Validate connection. + // NOTE Returning `Err(bool)` is a really ugly hack to work around the issue + // that `ProtocolController` thinks the peer is connected when in fact it can + // still be under validation. If the peer has different genesis than the + // local node the validation fails but the peer cannot be reported in + // `validate_connection()` as that is also called by + // `ValiateInboundSubstream` which means that the peer is still being + // validated and banning the peer when handling that event would + // result in peer getting dropped twice. + // + // The proper way to fix this is to integrate `ProtocolController` more + // tightly with `NotificationService` or add an additional API call for + // banning pre-accepted peers (which is not desirable) + fn validate_connection( + &mut self, + peer_id: &PeerId, + handshake: Vec, + direction: Direction, + ) -> Result, bool> { + log::trace!(target: LOG_TARGET, "New peer {peer_id} {handshake:?}"); + + let handshake = self.validate_handshake(peer_id, handshake)?; + + if self.peers.contains_key(&peer_id) { + log::error!( + target: LOG_TARGET, + "Called `validate_connection()` with already connected peer {peer_id}", + ); + debug_assert!(false); + return Err(false) } - if status.roles.is_full() && + let no_slot_peer = self.default_peers_set_no_slot_peers.contains(&peer_id); + let this_peer_reserved_slot: usize = if no_slot_peer { 1 } else { 0 }; + + if handshake.roles.is_full() && self.chain_sync.num_peers() >= self.default_peers_set_num_full + self.default_peers_set_no_slot_connected_peers.len() + this_peer_reserved_slot { - log::debug!(target: "sync", "Too many full nodes, rejecting {}", who); - return Err(()) + log::debug!(target: LOG_TARGET, "Too many full nodes, rejecting {peer_id}"); + return Err(false) + } + + // make sure to accept no more than `--in-peers` many full nodes + if !no_slot_peer && + handshake.roles.is_full() && + direction.is_inbound() && + self.num_in_peers == self.max_in_peers + { + log::debug!(target: LOG_TARGET, "All inbound slots have been consumed, rejecting {peer_id}"); + return Err(false) } - if status.roles.is_light() && + // make sure that all slots are not occupied by light peers + // + // `ChainSync` only accepts full peers whereas `SyncingEngine` accepts both full and light + // peers. Verify that there is a slot in `SyncingEngine` for the inbound light peer + if handshake.roles.is_light() && (self.peers.len() - self.chain_sync.num_peers()) >= self.default_peers_set_num_light { - // Make sure that not all slots are occupied by light clients. - log::debug!(target: "sync", "Too many light nodes, rejecting {}", who); - return Err(()) + log::debug!(target: LOG_TARGET, "Too many light nodes, rejecting {peer_id}"); + return Err(false) } + Ok(handshake) + } + + /// Called on the first connection between two peers on the default set, after their exchange + /// of handshake. + /// + /// Returns `Ok` if the handshake is accepted and the peer added to the list of peers we sync + /// from. + pub fn on_sync_peer_connected( + &mut self, + who: PeerId, + status: &BlockAnnouncesHandshake, + direction: Direction, + ) -> Result<(), ()> { + log::trace!(target: "sync", "New peer {} {:?}", who, status); + let peer = Peer { info: ExtendedPeerInfo { roles: status.roles, @@ -906,8 +975,7 @@ where known_blocks: LruHashSet::new( NonZeroUsize::new(MAX_KNOWN_BLOCKS).expect("Constant is nonzero"), ), - sink, - inbound, + inbound: direction.is_inbound(), }; let req = if peer.info.roles.is_full() { @@ -922,13 +990,14 @@ where None }; - log::debug!(target: "sync", "Connected {}", who); + log::debug!(target: LOG_TARGET, "Peer connected {who}"); self.peers.insert(who, peer); + self.peer_store_handle.set_peer_role(&who, status.roles.into()); - if no_slot_peer { + if self.default_peers_set_no_slot_peers.contains(&who) { self.default_peers_set_no_slot_connected_peers.insert(who); - } else if inbound && status.roles.is_full() { + } else if direction.is_inbound() && status.roles.is_full() { self.num_in_peers += 1; } @@ -942,3 +1011,331 @@ where Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::service::network::NetworkServiceProvider; + use sc_block_builder::BlockBuilderProvider; + use sc_network::{ + config::NetworkConfiguration, peer_store::PeerStore, service::traits::Direction, + NetworkBlock, NotificationCommand, NotificationsSink, + }; + use sp_consensus::block_validation::DefaultBlockAnnounceValidator; + use substrate_test_runtime_client::{ + runtime::Block, DefaultTestClientBuilderExt, TestClient, TestClientBuilder, + TestClientBuilderExt, + }; + + fn make_syncing_engine() -> ( + Arc, + SyncingEngine, + SyncingService, + NonDefaultSetConfig, + ) { + let mut net_config = NetworkConfiguration::new_local(); + + // 8 outbound full nodes, 32 inbound full nodes and 100 inbound light nodes + net_config.default_peers_set_num_full = 40; + net_config.default_peers_set.in_peers = 132; + net_config.default_peers_set.out_peers = 8; + + let full_config = FullNetworkConfiguration::new(&net_config); + let client = Arc::new(TestClientBuilder::new().build()); + let block_announce_validator = Box::new(DefaultBlockAnnounceValidator); + let import_queue = Box::new(sc_consensus::import_queue::mock::MockImportQueueHandle::new()); + let (_chain_sync_network_provider, chain_sync_network_handle) = + NetworkServiceProvider::new(); + let peer_store = PeerStore::new(vec![]); + let handle = peer_store.handle(); + + let (engine, service, config) = SyncingEngine::new( + Roles::FULL, + client.clone(), + None, + &full_config, + ProtocolId::from("test-protocol-name"), + &None, + block_announce_validator, + None, + chain_sync_network_handle, + import_queue, + ProtocolName::from("/block-request/1"), + ProtocolName::from("/state-request/1"), + None, + handle, + ) + .unwrap(); + + (client, engine, service, config) + } + + #[tokio::test] + async fn reject_invalid_handshake() { + let (_client, engine, _service, config) = make_syncing_engine(); + let (handle, _) = config.take_protocol_handle().split(); + + tokio::spawn(engine.run()); + + let rx = handle.report_incoming_substream(PeerId::random(), vec![1, 2, 3, 4]).unwrap(); + + assert_eq!(rx.await, Ok(ValidationResult::Reject)); + } + + #[tokio::test] + async fn accept_valid_peer() { + let (client, engine, _service, config) = make_syncing_engine(); + let (handle, _) = config.take_protocol_handle().split(); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + let handshake = BlockAnnouncesHandshake::::build( + engine.roles, + *block.header().number(), + block.hash(), + engine.genesis_hash, + ) + .encode(); + + tokio::spawn(engine.run()); + + let rx = handle.report_incoming_substream(PeerId::random(), handshake).unwrap(); + + assert_eq!(rx.await, Ok(ValidationResult::Accept)); + } + + #[tokio::test] + async fn valid_peer_included_into_peers() { + let (client, engine, service, config) = make_syncing_engine(); + let (mut handle, _) = config.take_protocol_handle().split(); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + let handshake = BlockAnnouncesHandshake::::build( + engine.roles, + *block.header().number(), + block.hash(), + engine.genesis_hash, + ) + .encode(); + + tokio::spawn(engine.run()); + + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, handshake.clone()).unwrap(); + assert_eq!(rx.await, Ok(ValidationResult::Accept)); + let (sink, _, _) = NotificationsSink::new(peer_id); + + handle + .report_substream_opened(peer_id, Direction::Inbound, handshake, None, sink) + .unwrap(); + + if let Err(_) = tokio::time::timeout(Duration::from_secs(10), async move { + loop { + if service.num_sync_peers().await.unwrap() == 1 { + break + } + } + }) + .await + { + panic!("failed to add sync peer"); + } + } + + #[tokio::test] + async fn syncing_engine_full_of_inbound_peers() { + let (client, engine, service, config) = make_syncing_engine(); + let (mut handle, _) = config.take_protocol_handle().split(); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + let handshake = BlockAnnouncesHandshake::::build( + engine.roles, + *block.header().number(), + block.hash(), + engine.genesis_hash, + ) + .encode(); + + tokio::spawn(engine.run()); + + // add maximum number of allowed full inbound peers to `SyncingEngine` + let num_in_peers = 32; + for i in 0..num_in_peers { + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, handshake.clone()).unwrap(); + assert_eq!(rx.await, Ok(ValidationResult::Accept)); + let (sink, _, _) = NotificationsSink::new(peer_id); + + handle + .report_substream_opened(peer_id, Direction::Inbound, handshake.clone(), None, sink) + .unwrap(); + let sync_service = service.clone(); + + if let Err(_) = tokio::time::timeout(Duration::from_secs(10), async move { + loop { + if sync_service.num_sync_peers().await.unwrap() == i + 1 { + break + } + } + }) + .await + { + panic!("failed to add sync peer"); + } + } + + // try to add one more peer and verify the peer is rejected during handshake validation + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, handshake.clone()).unwrap(); + assert_eq!(rx.await, Ok(ValidationResult::Reject)); + } + + #[tokio::test] + async fn syncing_engine_full_of_inbound_peers_outbound_accepted() { + let (client, engine, service, config) = make_syncing_engine(); + let (mut handle, _) = config.take_protocol_handle().split(); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + let handshake = BlockAnnouncesHandshake::::build( + engine.roles, + *block.header().number(), + block.hash(), + engine.genesis_hash, + ) + .encode(); + + tokio::spawn(engine.run()); + + // add maximum number of allowed inbound peers to `SyncingEngine` + let num_in_peers = 32; + for i in 0..num_in_peers { + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, handshake.clone()).unwrap(); + assert_eq!(rx.await, Ok(ValidationResult::Accept)); + let (sink, _, _) = NotificationsSink::new(peer_id); + + handle + .report_substream_opened(peer_id, Direction::Inbound, handshake.clone(), None, sink) + .unwrap(); + let sync_service = service.clone(); + + if let Err(_) = tokio::time::timeout(Duration::from_secs(10), async move { + loop { + if sync_service.num_sync_peers().await.unwrap() == i + 1 { + break + } + } + }) + .await + { + panic!("failed to add sync peer"); + } + } + + // try to add one more peer, this time outbound and verify it's accepted + let peer_id = PeerId::random(); + let (sink, _, _) = NotificationsSink::new(peer_id); + handle + .report_substream_opened(peer_id, Direction::Outbound, handshake.clone(), None, sink) + .unwrap(); + let sync_service = service.clone(); + + if let Err(_) = tokio::time::timeout(Duration::from_secs(10), async move { + loop { + if sync_service.num_sync_peers().await.unwrap() == num_in_peers + 1 { + break + } + } + }) + .await + { + panic!("failed to add sync peer"); + } + } + + #[tokio::test] + async fn syncing_engine_full_of_inbound_full_peers_but_not_light_peers() { + let (client, engine, service, config) = make_syncing_engine(); + let (mut handle, _) = config.take_protocol_handle().split(); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + let genesis = engine.genesis_hash.clone(); + let handshake = BlockAnnouncesHandshake::::build( + engine.roles, + *block.header().number(), + block.hash(), + engine.genesis_hash, + ) + .encode(); + + tokio::spawn(engine.run()); + + // add maximum number of allowed inbound peers to `SyncingEngine` + let num_in_peers = 32; + for i in 0..num_in_peers { + let peer_id = PeerId::random(); + let rx = handle.report_incoming_substream(peer_id, handshake.clone()).unwrap(); + assert_eq!(rx.await, Ok(ValidationResult::Accept)); + let (sink, _, _) = NotificationsSink::new(peer_id); + + handle + .report_substream_opened(peer_id, Direction::Inbound, handshake.clone(), None, sink) + .unwrap(); + let sync_service = service.clone(); + + if let Err(_) = tokio::time::timeout(Duration::from_secs(10), async move { + loop { + if sync_service.num_sync_peers().await.unwrap() == i + 1 { + break + } + } + }) + .await + { + panic!("failed to add sync peer"); + } + } + + // try to add one more peer, this time outbound and verify it's accepted + let handshake = BlockAnnouncesHandshake::::build( + Roles::LIGHT, + *block.header().number(), + block.hash(), + genesis, + ) + .encode(); + let peer_id = PeerId::random(); + let (sink, _, _) = NotificationsSink::new(peer_id); + handle + .report_substream_opened(peer_id, Direction::Inbound, handshake.clone(), None, sink) + .unwrap(); + let sync_service = service.clone(); + + if let Err(_) = tokio::time::timeout(Duration::from_secs(10), async move { + loop { + if sync_service.status().await.unwrap().num_connected_peers == num_in_peers + 1 { + break + } + } + }) + .await + { + panic!("failed to add sync peer"); + } + } + + #[tokio::test] + async fn best_block_import_updates_handshake() { + let (client, engine, service, config) = make_syncing_engine(); + let (_handle, mut commands) = config.take_protocol_handle().split(); + + tokio::spawn(engine.run()); + + let block = client.new_block(Default::default()).unwrap().build().unwrap().block; + service.new_best_block_imported(block.hash(), *block.header().number()); + + assert!(std::matches!( + commands.next().await.unwrap(), + NotificationCommand::SetHandshake(_) + )); + } +} diff --git a/substrate/client/network/sync/src/lib.rs b/substrate/client/network/sync/src/lib.rs index 175c1c43f46f..bb7fea9b0ae1 100644 --- a/substrate/client/network/sync/src/lib.rs +++ b/substrate/client/network/sync/src/lib.rs @@ -55,6 +55,7 @@ use sc_network::{ }, request_responses::{IfDisconnected, RequestFailure}, types::ProtocolName, + NotificationService, }; use sc_network_common::{ role::Roles, @@ -1405,8 +1406,8 @@ where block_request_protocol_name: ProtocolName, state_request_protocol_name: ProtocolName, warp_sync_protocol_name: Option, - ) -> Result<(Self, NonDefaultSetConfig), ClientError> { - let block_announce_config = Self::get_block_announce_proto_config( + ) -> Result<(Self, NonDefaultSetConfig, Box), ClientError> { + let (block_announce_config, notification_service) = Self::get_block_announce_proto_config( protocol_id, fork_id, roles, @@ -1445,10 +1446,7 @@ where state_request_protocol_name, warp_sync_params, warp_sync_protocol_name, - block_announce_protocol_name: block_announce_config - .notifications_protocol - .clone() - .into(), + block_announce_protocol_name: block_announce_config.protocol_name().clone().into(), pending_responses: HashMap::new(), import_queue, metrics: if let Some(r) = &metrics_registry { @@ -1465,7 +1463,7 @@ where }; sync.reset_sync_start_point()?; - Ok((sync, block_announce_config)) + Ok((sync, block_announce_config, notification_service)) } /// Returns the median seen block number. @@ -1933,7 +1931,7 @@ where best_number: NumberFor, best_hash: B::Hash, genesis_hash: B::Hash, - ) -> NonDefaultSetConfig { + ) -> (NonDefaultSetConfig, Box) { let block_announces_protocol = { let genesis_hash = genesis_hash.as_ref(); if let Some(ref fork_id) = fork_id { @@ -1947,14 +1945,11 @@ where } }; - NonDefaultSetConfig { - notifications_protocol: block_announces_protocol.into(), - fallback_names: iter::once( - format!("/{}/block-announces/1", protocol_id.as_ref()).into(), - ) - .collect(), - max_notification_size: MAX_BLOCK_ANNOUNCE_SIZE, - handshake: Some(NotificationHandshake::new(BlockAnnouncesHandshake::::build( + NonDefaultSetConfig::new( + block_announces_protocol.into(), + iter::once(format!("/{}/block-announces/1", protocol_id.as_ref()).into()).collect(), + MAX_BLOCK_ANNOUNCE_SIZE, + Some(NotificationHandshake::new(BlockAnnouncesHandshake::::build( roles, best_number, best_hash, @@ -1962,13 +1957,13 @@ where ))), // NOTE: `set_config` will be ignored by `protocol.rs` as the block announcement // protocol is still hardcoded into the peerset. - set_config: SetConfig { + SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - } + ) } fn decode_block_response(response: &[u8]) -> Result { @@ -3189,7 +3184,7 @@ mod test { let import_queue = Box::new(sc_consensus::import_queue::mock::MockImportQueueHandle::new()); let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -3256,7 +3251,7 @@ mod test { let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -3438,7 +3433,7 @@ mod test { let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -3565,7 +3560,7 @@ mod test { NetworkServiceProvider::new(); let info = client.info(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -3723,7 +3718,7 @@ mod test { let info = client.info(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -3866,7 +3861,7 @@ mod test { let info = client.info(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -4011,7 +4006,7 @@ mod test { let mut client = Arc::new(TestClientBuilder::new().build()); let blocks = (0..3).map(|_| build_block(&mut client, None, false)).collect::>(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), @@ -4057,7 +4052,7 @@ mod test { let empty_client = Arc::new(TestClientBuilder::new().build()); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, empty_client.clone(), ProtocolId::from("test-protocol-name"), @@ -4111,7 +4106,7 @@ mod test { let import_queue = Box::new(sc_consensus::import_queue::mock::MockImportQueueHandle::new()); let (_chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); - let (mut sync, _) = ChainSync::new( + let (mut sync, _, _) = ChainSync::new( SyncMode::Full, client.clone(), ProtocolId::from("test-protocol-name"), diff --git a/substrate/client/network/sync/src/service/mock.rs b/substrate/client/network/sync/src/service/mock.rs index 885eb1f8da59..47986a71d012 100644 --- a/substrate/client/network/sync/src/service/mock.rs +++ b/substrate/client/network/sync/src/service/mock.rs @@ -27,6 +27,7 @@ use sc_network::{ NetworkNotification, NetworkPeers, NetworkRequest, NetworkSyncForkRequest, NotificationSenderError, NotificationSenderT, ReputationChange, }; +use sc_network_common::role::ObservedRole; use sp_runtime::traits::{Block as BlockT, NumberFor}; use std::collections::HashSet; @@ -105,6 +106,7 @@ mockall::mock! { peers: Vec ) -> Result<(), String>; fn sync_num_connected(&self) -> usize; + fn peer_role(&self, peer_id: PeerId, handshake: Vec) -> Option; } #[async_trait::async_trait] diff --git a/substrate/client/network/test/src/lib.rs b/substrate/client/network/test/src/lib.rs index 2a20da5a556b..7aaac5d78c3a 100644 --- a/substrate/client/network/test/src/lib.rs +++ b/substrate/client/network/test/src/lib.rs @@ -58,7 +58,7 @@ use sc_network::{ request_responses::ProtocolConfig as RequestResponseConfig, types::ProtocolName, Multiaddr, NetworkBlock, NetworkService, NetworkStateInfo, NetworkSyncForkRequest, - NetworkWorker, + NetworkWorker, NotificationService, }; use sc_network_common::{ role::Roles, @@ -241,6 +241,7 @@ pub struct Peer { imported_blocks_stream: Pin> + Send>>, finality_notification_stream: Pin> + Send>>, listen_addr: Multiaddr, + notification_services: HashMap>, } impl Peer @@ -265,8 +266,8 @@ where } /// Returns the number of peers we're connected to. - pub fn num_peers(&self) -> usize { - self.network.num_connected_peers() + pub async fn num_peers(&self) -> usize { + self.sync_service.status().await.unwrap().num_connected_peers as usize } /// Returns the number of downloaded blocks. @@ -505,10 +506,19 @@ where self.network.service() } + /// Get `SyncingService`. pub fn sync_service(&self) -> &Arc> { &self.sync_service } + /// Take notification handle for enabled protocol. + pub fn take_notification_service( + &mut self, + protocol: &ProtocolName, + ) -> Option> { + self.notification_services.remove(protocol) + } + /// Get a reference to the network worker. pub fn network(&self) -> &NetworkWorker::Hash> { &self.network @@ -781,6 +791,23 @@ pub trait TestNetFactory: Default + Sized + Send { network_config.transport = TransportConfig::MemoryOnly; network_config.listen_addresses = vec![listen_addr.clone()]; network_config.allow_non_globals_in_dht = true; + + let (notif_configs, notif_handles): (Vec<_>, Vec<_>) = config + .notifications_protocols + .into_iter() + .map(|p| { + let (config, handle) = NonDefaultSetConfig::new( + p.clone(), + Vec::new(), + 1024 * 1024, + None, + Default::default(), + ); + + (config, (p, handle)) + }) + .unzip(); + if let Some(connect_to) = config.connect_to_peers { let addrs = connect_to .iter() @@ -846,13 +873,18 @@ pub trait TestNetFactory: Default + Sized + Send { protocol_config }; + let peer_store = PeerStore::new( + network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(), + ); + let peer_store_handle = peer_store.handle(); + self.spawn_task(peer_store.run().boxed()); + let block_announce_validator = config .block_announce_validator .unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator)); let (chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); - let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (engine, sync_service, block_announce_config) = sc_network_sync::engine::SyncingEngine::new( Roles::from(if config.is_authority { &Role::Authority } else { &Role::Full }), @@ -868,7 +900,7 @@ pub trait TestNetFactory: Default + Sized + Send { block_request_protocol_config.name.clone(), state_request_protocol_config.name.clone(), Some(warp_protocol_config.name.clone()), - rx, + peer_store_handle.clone(), ) .unwrap(); let sync_service_import_queue = Box::new(sync_service.clone()); @@ -886,22 +918,10 @@ pub trait TestNetFactory: Default + Sized + Send { full_net_config.add_request_response_protocol(config); } - for protocol in config.notifications_protocols { - full_net_config.add_notification_protocol(NonDefaultSetConfig { - notifications_protocol: protocol, - fallback_names: Vec::new(), - max_notification_size: 1024 * 1024, - handshake: None, - set_config: Default::default(), - }); + for config in notif_configs { + full_net_config.add_notification_protocol(config); } - let peer_store = PeerStore::new( - network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(), - ); - let peer_store_handle = peer_store.handle(); - self.spawn_task(peer_store.run().boxed()); - let genesis_hash = client.hash(Zero::zero()).ok().flatten().expect("Genesis block exists; qed"); let network = NetworkWorker::new(sc_network::config::Params { @@ -916,7 +936,6 @@ pub trait TestNetFactory: Default + Sized + Send { fork_id, metrics_registry: None, block_announce_config, - tx, }) .unwrap(); @@ -952,6 +971,7 @@ pub trait TestNetFactory: Default + Sized + Send { backend: Some(backend), imported_blocks_stream, finality_notification_stream, + notification_services: HashMap::from_iter(notif_handles.into_iter()), block_import, verifier, network, @@ -966,20 +986,6 @@ pub trait TestNetFactory: Default + Sized + Send { tokio::spawn(f); } - /// Polls the testnet until all peers are connected to each other. - /// - /// Must be executed in a task context. - fn poll_until_connected(&mut self, cx: &mut FutureContext) -> Poll<()> { - self.poll(cx); - - let num_peers = self.peers().len(); - if self.peers().iter().all(|p| p.num_peers() == num_peers - 1) { - return Poll::Ready(()) - } - - Poll::Pending - } - async fn is_in_sync(&mut self) -> bool { let mut highest = None; let peers = self.peers_mut(); @@ -1057,10 +1063,27 @@ pub trait TestNetFactory: Default + Sized + Send { } /// Run the network until all peers are connected to each other. - /// - /// Calls `poll_until_connected` repeatedly with the runtime passed as parameter. async fn run_until_connected(&mut self) { - futures::future::poll_fn::<(), _>(|cx| self.poll_until_connected(cx)).await; + let num_peers = self.peers().len(); + let sync_services = + self.peers().iter().map(|info| info.sync_service.clone()).collect::>(); + + 'outer: loop { + for sync_service in &sync_services { + if sync_service.status().await.unwrap().num_connected_peers as usize != + num_peers - 1 + { + futures::future::poll_fn::<(), _>(|cx| { + self.poll(cx); + Poll::Ready(()) + }) + .await; + continue 'outer + } + } + + break + } } /// Polls the testnet. Processes all the pending actions. diff --git a/substrate/client/network/test/src/service.rs b/substrate/client/network/test/src/service.rs index 68e780545bb1..6fda1b86a51c 100644 --- a/substrate/client/network/test/src/service.rs +++ b/substrate/client/network/test/src/service.rs @@ -24,8 +24,9 @@ use sc_network::{ config::{self, FullNetworkConfiguration, MultiaddrWithPeerId, ProtocolId, TransportConfig}, event::Event, peer_store::PeerStore, - NetworkEventStream, NetworkNotification, NetworkPeers, NetworkService, NetworkStateInfo, - NetworkWorker, + service::traits::{NotificationEvent, ValidationResult}, + NetworkEventStream, NetworkPeers, NetworkService, NetworkStateInfo, NetworkWorker, + NotificationService, }; use sc_network_common::role::Roles; use sc_network_light::light_client_requests::handler::LightClientRequestHandler; @@ -116,7 +117,7 @@ impl TestNetworkBuilder { self } - pub fn build(mut self) -> TestNetwork { + pub fn build(mut self) -> (TestNetwork, Option>) { let client = self.client.as_mut().map_or( Arc::new(TestClientBuilder::with_default_backend().build_with_longest_chain().0), |v| v.clone(), @@ -177,9 +178,14 @@ impl TestNetworkBuilder { protocol_config }; + let peer_store = PeerStore::new( + network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(), + ); + let peer_store_handle = peer_store.handle(); + tokio::spawn(peer_store.run().boxed()); + let (chain_sync_network_provider, chain_sync_network_handle) = self.chain_sync_network.unwrap_or(NetworkServiceProvider::new()); - let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (engine, chain_sync_service, block_announce_config) = SyncingEngine::new( Roles::from(&config::Role::Full), client.clone(), @@ -194,24 +200,27 @@ impl TestNetworkBuilder { block_request_protocol_config.name.clone(), state_request_protocol_config.name.clone(), None, - rx, + peer_store_handle.clone(), ) .unwrap(); let mut link = self.link.unwrap_or(Box::new(chain_sync_service.clone())); - if !self.notification_protocols.is_empty() { + let handle = if !self.notification_protocols.is_empty() { for config in self.notification_protocols { full_net_config.add_notification_protocol(config); } + None } else { - full_net_config.add_notification_protocol(config::NonDefaultSetConfig { - notifications_protocol: PROTOCOL_NAME.into(), - fallback_names: Vec::new(), - max_notification_size: 1024 * 1024, - handshake: None, - set_config: self.set_config.unwrap_or_default(), - }); - } + let (config, handle) = config::NonDefaultSetConfig::new( + PROTOCOL_NAME.into(), + Vec::new(), + 1024 * 1024, + None, + self.set_config.unwrap_or_default(), + ); + full_net_config.add_notification_protocol(config); + Some(handle) + }; for config in [ block_request_protocol_config, @@ -221,12 +230,6 @@ impl TestNetworkBuilder { full_net_config.add_request_response_protocol(config); } - let peer_store = PeerStore::new( - network_config.boot_nodes.iter().map(|bootnode| bootnode.peer_id).collect(), - ); - let peer_store_handle = peer_store.handle(); - tokio::spawn(peer_store.run().boxed()); - let genesis_hash = client.hash(Zero::zero()).ok().flatten().expect("Genesis block exists; qed"); let worker = NetworkWorker::< @@ -244,7 +247,6 @@ impl TestNetworkBuilder { protocol_id, fork_id, metrics_registry: None, - tx, }) .unwrap(); @@ -264,7 +266,7 @@ impl TestNetworkBuilder { }); tokio::spawn(engine.run()); - TestNetwork::new(worker) + (TestNetwork::new(worker), handle) } } @@ -272,18 +274,18 @@ impl TestNetworkBuilder { /// The nodes are connected together and have the `PROTOCOL_NAME` protocol registered. fn build_nodes_one_proto() -> ( Arc, - impl Stream, + Option>, Arc, - impl Stream, + Option>, ) { let listen_addr = config::build_multiaddr![Memory(rand::random::())]; - let (node1, events_stream1) = TestNetworkBuilder::new() + let (network1, handle1) = TestNetworkBuilder::new() .with_listen_addresses(vec![listen_addr.clone()]) - .build() - .start_network(); + .build(); + let (node1, _) = network1.start_network(); - let (node2, events_stream2) = TestNetworkBuilder::new() + let (network2, handle2) = TestNetworkBuilder::new() .with_set_config(config::SetConfig { reserved_nodes: vec![MultiaddrWithPeerId { multiaddr: listen_addr, @@ -291,10 +293,11 @@ fn build_nodes_one_proto() -> ( }], ..Default::default() }) - .build() - .start_network(); + .build(); + + let (node2, _) = network2.start_network(); - (node1, events_stream1, node2, events_stream2) + (node1, handle1, node2, handle2) } #[tokio::test] @@ -302,22 +305,15 @@ async fn notifications_state_consistent() { // Runs two nodes and ensures that events are propagated out of the API in a consistent // correct order, which means no notification received on a closed substream. - let (node1, mut events_stream1, node2, mut events_stream2) = build_nodes_one_proto(); + let (node1, handle1, node2, handle2) = build_nodes_one_proto(); + let (mut handle1, mut handle2) = (handle1.unwrap(), handle2.unwrap()); // Write some initial notifications that shouldn't get through. for _ in 0..(rand::random::() % 5) { - node1.write_notification( - node2.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle1.send_sync_notification(&node2.local_peer_id(), b"hello world".to_vec()); } for _ in 0..(rand::random::() % 5) { - node2.write_notification( - node1.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle2.send_sync_notification(&node1.local_peer_id(), b"hello world".to_vec()); } // True if we have an active substream from node1 to node2. @@ -339,18 +335,10 @@ async fn notifications_state_consistent() { // Start by sending a notification from node1 to node2 and vice-versa. Part of the // test consists in ensuring that notifications get ignored if the stream isn't open. if rand::random::() % 5 >= 3 { - node1.write_notification( - node2.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle1.send_sync_notification(&node2.local_peer_id(), b"hello world".to_vec()); } if rand::random::() % 5 >= 3 { - node2.write_notification( - node1.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle2.send_sync_notification(&node1.local_peer_id(), b"hello world".to_vec()); } // Also randomly disconnect the two nodes from time to time. @@ -363,8 +351,8 @@ async fn notifications_state_consistent() { // Grab next event from either `events_stream1` or `events_stream2`. let next_event = { - let next1 = events_stream1.next(); - let next2 = events_stream2.next(); + let next1 = handle1.next_event(); + let next2 = handle2.next_event(); // We also await on a small timer, otherwise it is possible for the test to wait // forever while nothing at all happens on the network. let continue_test = futures_timer::Delay::new(Duration::from_millis(20)); @@ -379,58 +367,55 @@ async fn notifications_state_consistent() { }; match next_event { - future::Either::Left(Event::NotificationStreamOpened { remote, protocol, .. }) => - if protocol == PROTOCOL_NAME.into() { - something_happened = true; - assert!(!node1_to_node2_open); - node1_to_node2_open = true; - assert_eq!(remote, node2.local_peer_id()); - }, - future::Either::Right(Event::NotificationStreamOpened { remote, protocol, .. }) => - if protocol == PROTOCOL_NAME.into() { - something_happened = true; - assert!(!node2_to_node1_open); - node2_to_node1_open = true; - assert_eq!(remote, node1.local_peer_id()); - }, - future::Either::Left(Event::NotificationStreamClosed { remote, protocol, .. }) => - if protocol == PROTOCOL_NAME.into() { - assert!(node1_to_node2_open); - node1_to_node2_open = false; - assert_eq!(remote, node2.local_peer_id()); - }, - future::Either::Right(Event::NotificationStreamClosed { remote, protocol, .. }) => - if protocol == PROTOCOL_NAME.into() { - assert!(node2_to_node1_open); - node2_to_node1_open = false; - assert_eq!(remote, node1.local_peer_id()); - }, - future::Either::Left(Event::NotificationsReceived { remote, .. }) => { + future::Either::Left(NotificationEvent::ValidateInboundSubstream { + result_tx, .. + }) => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + future::Either::Right(NotificationEvent::ValidateInboundSubstream { + result_tx, + .. + }) => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + future::Either::Left(NotificationEvent::NotificationStreamOpened { peer, .. }) => { + something_happened = true; + assert!(!node1_to_node2_open); + node1_to_node2_open = true; + assert_eq!(peer, node2.local_peer_id()); + }, + future::Either::Right(NotificationEvent::NotificationStreamOpened { peer, .. }) => { + something_happened = true; + assert!(!node2_to_node1_open); + node2_to_node1_open = true; + assert_eq!(peer, node1.local_peer_id()); + }, + future::Either::Left(NotificationEvent::NotificationStreamClosed { peer, .. }) => { assert!(node1_to_node2_open); - assert_eq!(remote, node2.local_peer_id()); + node1_to_node2_open = false; + assert_eq!(peer, node2.local_peer_id()); + }, + future::Either::Right(NotificationEvent::NotificationStreamClosed { peer, .. }) => { + assert!(node2_to_node1_open); + node2_to_node1_open = false; + assert_eq!(peer, node1.local_peer_id()); + }, + future::Either::Left(NotificationEvent::NotificationReceived { peer, .. }) => { + assert!(node1_to_node2_open); + assert_eq!(peer, node2.local_peer_id()); if rand::random::() % 5 >= 4 { - node1.write_notification( - node2.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle1 + .send_sync_notification(&node2.local_peer_id(), b"hello world".to_vec()); } }, - future::Either::Right(Event::NotificationsReceived { remote, .. }) => { + future::Either::Right(NotificationEvent::NotificationReceived { peer, .. }) => { assert!(node2_to_node1_open); - assert_eq!(remote, node1.local_peer_id()); + assert_eq!(peer, node1.local_peer_id()); if rand::random::() % 5 >= 4 { - node2.write_notification( - node1.local_peer_id(), - PROTOCOL_NAME.into(), - b"hello world".to_vec(), - ); + let _ = handle2 + .send_sync_notification(&node1.local_peer_id(), b"hello world".to_vec()); } }, - - // Add new events here. - future::Either::Left(Event::Dht(_)) => {}, - future::Either::Right(Event::Dht(_)) => {}, }; } } @@ -440,20 +425,29 @@ async fn lots_of_incoming_peers_works() { sp_tracing::try_init_simple(); let listen_addr = config::build_multiaddr![Memory(rand::random::())]; - let (main_node, _) = TestNetworkBuilder::new() + let (main_node, handle1) = TestNetworkBuilder::new() .with_listen_addresses(vec![listen_addr.clone()]) .with_set_config(config::SetConfig { in_peers: u32::MAX, ..Default::default() }) - .build() - .start_network(); + .build(); + let mut handle1 = handle1.unwrap(); + let (main_node, _) = main_node.start_network(); let main_node_peer_id = main_node.local_peer_id(); + tokio::spawn(async move { + while let Some(event) = handle1.next_event().await { + if let NotificationEvent::ValidateInboundSubstream { result_tx, .. } = event { + result_tx.send(ValidationResult::Accept).unwrap(); + } + } + }); + // We spawn background tasks and push them in this `Vec`. They will all be waited upon before // this test ends. let mut background_tasks_to_wait = Vec::new(); for _ in 0..32 { - let (_dialing_node, event_stream) = TestNetworkBuilder::new() + let (dialing_node, handle) = TestNetworkBuilder::new() .with_set_config(config::SetConfig { reserved_nodes: vec![MultiaddrWithPeerId { multiaddr: listen_addr.clone(), @@ -461,8 +455,9 @@ async fn lots_of_incoming_peers_works() { }], ..Default::default() }) - .build() - .start_network(); + .build(); + let mut handle = handle.unwrap(); + let (_, _) = dialing_node.start_network(); background_tasks_to_wait.push(tokio::spawn(async move { // Create a dummy timer that will "never" fire, and that will be overwritten when we @@ -470,34 +465,23 @@ async fn lots_of_incoming_peers_works() { // make the code below way more complicated. let mut timer = futures_timer::Delay::new(Duration::from_secs(3600 * 24 * 7)).fuse(); - let mut event_stream = event_stream.fuse(); - let mut sync_protocol_name = None; loop { futures::select! { _ = timer => { // Test succeeds when timer fires. return; } - ev = event_stream.next() => { - match ev.unwrap() { - Event::NotificationStreamOpened { protocol, remote, .. } => { - if let None = sync_protocol_name { - sync_protocol_name = Some(protocol.clone()); - } - - assert_eq!(remote, main_node_peer_id); - // Test succeeds after 5 seconds. This timer is here in order to - // detect a potential problem after opening. - timer = futures_timer::Delay::new(Duration::from_secs(5)).fuse(); - } - Event::NotificationStreamClosed { protocol, .. } => { - if Some(protocol) != sync_protocol_name { - // Test failed. - panic!(); - } - } - _ => {} + ev = handle.next_event().fuse() => match ev.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); } + NotificationEvent::NotificationStreamOpened { peer, .. } => { + assert_eq!(peer, main_node_peer_id); + // Test succeeds after 5 seconds. This timer is here in order to + // detect a potential problem after opening. + timer = futures_timer::Delay::new(Duration::from_secs(5)).fuse(); + } + _ => {} } } } @@ -514,33 +498,27 @@ async fn notifications_back_pressure() { const TOTAL_NOTIFS: usize = 10_000; - let (node1, mut events_stream1, node2, mut events_stream2) = build_nodes_one_proto(); + let (_node1, handle1, node2, handle2) = build_nodes_one_proto(); + let (mut handle1, mut handle2) = (handle1.unwrap(), handle2.unwrap()); let node2_id = node2.local_peer_id(); let receiver = tokio::spawn(async move { let mut received_notifications = 0; - let mut sync_protocol_name = None; while received_notifications < TOTAL_NOTIFS { - match events_stream2.next().await.unwrap() { - Event::NotificationStreamOpened { protocol, .. } => { - if let None = sync_protocol_name { - sync_protocol_name = Some(protocol); - } + match handle2.next_event().await.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); }, - Event::NotificationStreamClosed { protocol, .. } => { - if Some(&protocol) != sync_protocol_name.as_ref() { - panic!() - } + NotificationEvent::NotificationReceived { notification, .. } => { + assert_eq!( + notification, + format!("hello #{}", received_notifications).into_bytes() + ); + received_notifications += 1; }, - Event::NotificationsReceived { messages, .. } => - for message in messages { - assert_eq!(message.0, PROTOCOL_NAME.into()); - assert_eq!(message.1, format!("hello #{}", received_notifications)); - received_notifications += 1; - }, _ => {}, - }; + } if rand::random::() < 2 { tokio::time::sleep(Duration::from_millis(rand::random::() % 750)).await; @@ -550,20 +528,20 @@ async fn notifications_back_pressure() { // Wait for the `NotificationStreamOpened`. loop { - match events_stream1.next().await.unwrap() { - Event::NotificationStreamOpened { .. } => break, + match handle1.next_event().await.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + NotificationEvent::NotificationStreamOpened { .. } => break, _ => {}, }; } // Sending! for num in 0..TOTAL_NOTIFS { - let notif = node1.notification_sender(node2_id, PROTOCOL_NAME.into()).unwrap(); - notif - .ready() + handle1 + .send_async_notification(&node2_id, format!("hello #{}", num).into_bytes()) .await - .unwrap() - .send(format!("hello #{}", num).into_bytes()) .unwrap(); } @@ -572,28 +550,31 @@ async fn notifications_back_pressure() { #[tokio::test] async fn fallback_name_working() { + sp_tracing::try_init_simple(); // Node 1 supports the protocols "new" and "old". Node 2 only supports "old". Checks whether // they can connect. const NEW_PROTOCOL_NAME: &str = "/new-shiny-protocol-that-isnt-PROTOCOL_NAME"; let listen_addr = config::build_multiaddr![Memory(rand::random::())]; - let (node1, mut events_stream1) = TestNetworkBuilder::new() - .with_notification_protocol(config::NonDefaultSetConfig { - notifications_protocol: NEW_PROTOCOL_NAME.into(), - fallback_names: vec![PROTOCOL_NAME.into()], - max_notification_size: 1024 * 1024, - handshake: None, - set_config: Default::default(), - }) + let (config, mut handle1) = config::NonDefaultSetConfig::new( + NEW_PROTOCOL_NAME.into(), + vec![PROTOCOL_NAME.into()], + 1024 * 1024, + None, + Default::default(), + ); + let (network1, _) = TestNetworkBuilder::new() + .with_notification_protocol(config) .with_config(config::NetworkConfiguration { listen_addresses: vec![listen_addr.clone()], transport: TransportConfig::MemoryOnly, ..config::NetworkConfiguration::new_local() }) - .build() - .start_network(); + .build(); - let (_, mut events_stream2) = TestNetworkBuilder::new() + let (node1, _) = network1.start_network(); + + let (network2, handle2) = TestNetworkBuilder::new() .with_set_config(config::SetConfig { reserved_nodes: vec![MultiaddrWithPeerId { multiaddr: listen_addr, @@ -601,34 +582,38 @@ async fn fallback_name_working() { }], ..Default::default() }) - .build() - .start_network(); + .build(); + let mut handle2 = handle2.unwrap(); + let _ = network2.start_network(); let receiver = tokio::spawn(async move { // Wait for the `NotificationStreamOpened`. loop { - match events_stream2.next().await.unwrap() { - Event::NotificationStreamOpened { protocol, negotiated_fallback, .. } => { - assert_eq!(protocol, PROTOCOL_NAME.into()); + match handle2.next_event().await.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + NotificationEvent::NotificationStreamOpened { negotiated_fallback, .. } => { assert_eq!(negotiated_fallback, None); break }, _ => {}, - }; + } } }); // Wait for the `NotificationStreamOpened`. loop { - match events_stream1.next().await.unwrap() { - Event::NotificationStreamOpened { protocol, negotiated_fallback, .. } - if protocol == NEW_PROTOCOL_NAME.into() => - { + match handle1.next_event().await.unwrap() { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + result_tx.send(ValidationResult::Accept).unwrap(); + }, + NotificationEvent::NotificationStreamOpened { negotiated_fallback, .. } => { assert_eq!(negotiated_fallback, Some(PROTOCOL_NAME.into())); break }, _ => {}, - }; + } } receiver.await.unwrap(); @@ -651,6 +636,7 @@ async fn ensure_listen_addresses_consistent_with_transport_memory() { ) }) .build() + .0 .start_network(); } @@ -670,6 +656,7 @@ async fn ensure_listen_addresses_consistent_with_transport_not_memory() { ) }) .build() + .0 .start_network(); } @@ -695,6 +682,7 @@ async fn ensure_boot_node_addresses_consistent_with_transport_memory() { ) }) .build() + .0 .start_network(); } @@ -719,6 +707,7 @@ async fn ensure_boot_node_addresses_consistent_with_transport_not_memory() { ) }) .build() + .0 .start_network(); } @@ -747,6 +736,7 @@ async fn ensure_reserved_node_addresses_consistent_with_transport_memory() { ) }) .build() + .0 .start_network(); } @@ -774,6 +764,7 @@ async fn ensure_reserved_node_addresses_consistent_with_transport_not_memory() { ) }) .build() + .0 .start_network(); } @@ -796,6 +787,7 @@ async fn ensure_public_addresses_consistent_with_transport_memory() { ) }) .build() + .0 .start_network(); } @@ -817,5 +809,6 @@ async fn ensure_public_addresses_consistent_with_transport_not_memory() { ) }) .build() + .0 .start_network(); } diff --git a/substrate/client/network/test/src/sync.rs b/substrate/client/network/test/src/sync.rs index 389177b4aaf1..f2be662ada16 100644 --- a/substrate/client/network/test/src/sync.rs +++ b/substrate/client/network/test/src/sync.rs @@ -44,16 +44,16 @@ async fn sync_peers_works() { sp_tracing::try_init_simple(); let mut net = TestNet::new(3); - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - for peer in 0..3 { - if net.peer(peer).num_peers() != 2 { - return Poll::Pending - } - } - Poll::Ready(()) - }) - .await; + while net.peer(0).num_peers().await != 2 && + net.peer(1).num_peers().await != 2 && + net.peer(2).num_peers().await != 2 + { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); + Poll::Ready(()) + }) + .await; + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -412,15 +412,13 @@ async fn can_sync_small_non_best_forks() { assert!(net.peer(1).client().header(small_hash).unwrap().is_none()); // poll until the two nodes connect, otherwise announcing the block will not work - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { - Poll::Pending - } else { + while net.peer(0).num_peers().await == 0 || net.peer(1).num_peers().await == 0 { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); Poll::Ready(()) - } - }) - .await; + }) + .await; + } // synchronization: 0 synced to longer chain and 1 didn't sync to small chain. @@ -465,6 +463,7 @@ async fn can_sync_forks_ahead_of_the_best_chain() { net.peer(1).push_blocks(1, false); net.run_until_connected().await; + // Peer 0 is on 2-block fork which is announced with is_best=false let fork_hash = net .peer(0) @@ -516,15 +515,13 @@ async fn can_sync_explicit_forks() { assert!(net.peer(1).client().header(small_hash).unwrap().is_none()); // poll until the two nodes connect, otherwise announcing the block will not work - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { - Poll::Pending - } else { + while net.peer(0).num_peers().await == 0 || net.peer(1).num_peers().await == 0 { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); Poll::Ready(()) - } - }) - .await; + }) + .await; + } // synchronization: 0 synced to longer chain and 1 didn't sync to small chain. @@ -613,15 +610,14 @@ async fn full_sync_requires_block_body() { net.peer(0).push_headers(1); // Wait for nodes to connect - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(0).num_peers() == 0 || net.peer(1).num_peers() == 0 { - Poll::Pending - } else { + while net.peer(0).num_peers().await == 0 || net.peer(1).num_peers().await == 0 { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); Poll::Ready(()) - } - }) - .await; + }) + .await; + } + net.run_until_idle().await; assert_eq!(net.peer(1).client.info().best_number, 0); } @@ -917,18 +913,16 @@ async fn block_announce_data_is_propagated() { }); // Wait until peer 1 is connected to both nodes. - futures::future::poll_fn::<(), _>(|cx| { - net.poll(cx); - if net.peer(1).num_peers() == 2 && - net.peer(0).num_peers() == 1 && - net.peer(2).num_peers() == 1 - { + while net.peer(1).num_peers().await != 2 || + net.peer(0).num_peers().await != 1 || + net.peer(2).num_peers().await != 1 + { + futures::future::poll_fn::<(), _>(|cx| { + net.poll(cx); Poll::Ready(()) - } else { - Poll::Pending - } - }) - .await; + }) + .await; + } let block_hash = net .peer(0) @@ -1010,7 +1004,7 @@ async fn multiple_requests_are_accepted_as_long_as_they_are_not_fulfilled() { tokio::time::sleep(tokio::time::Duration::from_secs(10)).await; net.peer(0).push_blocks(1, false); net.run_until_sync().await; - assert_eq!(1, net.peer(0).num_peers()); + assert_eq!(1, net.peer(0).num_peers().await); } let hashof10 = hashes[9]; diff --git a/substrate/client/network/transactions/src/lib.rs b/substrate/client/network/transactions/src/lib.rs index b46733d42723..835efe0e6db7 100644 --- a/substrate/client/network/transactions/src/lib.rs +++ b/substrate/client/network/transactions/src/lib.rs @@ -21,8 +21,8 @@ //! Usage: //! //! - Use [`TransactionsHandlerPrototype::new`] to create a prototype. -//! - Pass the return value of [`TransactionsHandlerPrototype::set_config`] to the network -//! configuration as an extra peers set. +//! - Pass the `NonDefaultSetConfig` returned from [`TransactionsHandlerPrototype::new`] to the +//! network configuration as an extra peers set. //! - Use [`TransactionsHandlerPrototype::build`] then [`TransactionsHandler::run`] to obtain a //! `Future` that processes transactions. @@ -37,7 +37,7 @@ use prometheus_endpoint::{register, Counter, PrometheusError, Registry, U64}; use sc_network::{ config::{NonDefaultSetConfig, NonReservedPeerMode, ProtocolId, SetConfig}, error, - event::Event, + service::traits::{NotificationEvent, NotificationService, ValidationResult}, types::ProtocolName, utils::{interval, LruHashSet}, NetworkEventStream, NetworkNotification, NetworkPeers, @@ -118,8 +118,11 @@ impl Future for PendingTransaction { /// Prototype for a [`TransactionsHandler`]. pub struct TransactionsHandlerPrototype { + /// Name of the transaction protocol. protocol_name: ProtocolName, - fallback_protocol_names: Vec, + + /// Handle that is used to communicate with `sc_network::Notifications`. + notification_service: Box, } impl TransactionsHandlerPrototype { @@ -128,35 +131,28 @@ impl TransactionsHandlerPrototype { protocol_id: ProtocolId, genesis_hash: Hash, fork_id: Option<&str>, - ) -> Self { + ) -> (Self, NonDefaultSetConfig) { let genesis_hash = genesis_hash.as_ref(); - let protocol_name = if let Some(fork_id) = fork_id { + let protocol_name: ProtocolName = if let Some(fork_id) = fork_id { format!("/{}/{}/transactions/1", array_bytes::bytes2hex("", genesis_hash), fork_id) } else { format!("/{}/transactions/1", array_bytes::bytes2hex("", genesis_hash)) - }; - let legacy_protocol_name = format!("/{}/transactions/1", protocol_id.as_ref()); - - Self { - protocol_name: protocol_name.into(), - fallback_protocol_names: iter::once(legacy_protocol_name.into()).collect(), } - } - - /// Returns the configuration of the set to put in the network configuration. - pub fn set_config(&self) -> NonDefaultSetConfig { - NonDefaultSetConfig { - notifications_protocol: self.protocol_name.clone(), - fallback_names: self.fallback_protocol_names.clone(), - max_notification_size: MAX_TRANSACTIONS_SIZE, - handshake: None, - set_config: SetConfig { + .into(); + let (config, notification_service) = NonDefaultSetConfig::new( + protocol_name.clone(), + vec![format!("/{}/transactions/1", protocol_id.as_ref()).into()], + MAX_TRANSACTIONS_SIZE, + None, + SetConfig { in_peers: 0, out_peers: 0, reserved_nodes: Vec::new(), non_reserved_mode: NonReservedPeerMode::Deny, }, - } + ); + + (Self { protocol_name, notification_service }, config) } /// Turns the prototype into the actual handler. Returns a controller that allows controlling @@ -176,12 +172,12 @@ impl TransactionsHandlerPrototype { transaction_pool: Arc>, metrics_registry: Option<&Registry>, ) -> error::Result<(TransactionsHandler, TransactionsHandlerController)> { - let net_event_stream = network.event_stream("transactions-handler-net"); let sync_event_stream = sync.event_stream("transactions-handler-sync"); let (to_handler, from_controller) = tracing_unbounded("mpsc_transactions_handler", 100_000); let handler = TransactionsHandler { protocol_name: self.protocol_name, + notification_service: self.notification_service, propagate_timeout: (Box::pin(interval(PROPAGATE_TIMEOUT)) as Pin + Send>>) .fuse(), @@ -189,7 +185,6 @@ impl TransactionsHandlerPrototype { pending_transactions_peers: HashMap::new(), network, sync, - net_event_stream: net_event_stream.fuse(), sync_event_stream: sync_event_stream.fuse(), peers: HashMap::new(), transaction_pool, @@ -256,8 +251,6 @@ pub struct TransactionsHandler< network: N, /// Syncing service. sync: S, - /// Stream of networking events. - net_event_stream: stream::Fuse + Send>>>, /// Receiver for syncing-related events. sync_event_stream: stream::Fuse + Send>>>, // All connected peers @@ -266,6 +259,8 @@ pub struct TransactionsHandler< from_controller: TracingUnboundedReceiver>, /// Prometheus metrics. metrics: Option, + /// Handle that is used to communicate with `sc_network::Notifications`. + notification_service: Box, } /// Peer information @@ -298,14 +293,6 @@ where warn!(target: "sub-libp2p", "Inconsistent state, no peers for pending transaction!"); } }, - network_event = self.net_event_stream.next() => { - if let Some(network_event) = network_event { - self.handle_network_event(network_event).await; - } else { - // Networking has seemingly closed. Closing as well. - return; - } - }, sync_event = self.sync_event_stream.next() => { if let Some(sync_event) = sync_event { self.handle_sync_event(sync_event); @@ -320,10 +307,56 @@ where ToHandler::PropagateTransactions => self.propagate_transactions(), } }, + event = self.notification_service.next_event().fuse() => { + if let Some(event) = event { + self.handle_notification_event(event) + } else { + // `Notifications` has seemingly closed. Closing as well. + return + } + } } } } + fn handle_notification_event(&mut self, event: NotificationEvent) { + match event { + NotificationEvent::ValidateInboundSubstream { result_tx, .. } => { + let _ = result_tx.send(ValidationResult::Accept); + }, + NotificationEvent::NotificationStreamOpened { peer, handshake, .. } => { + let Some(role) = self.network.peer_role(peer, handshake) else { + log::debug!(target: "sub-libp2p", "role for {peer} couldn't be determined"); + return + }; + + let _was_in = self.peers.insert( + peer, + Peer { + known_transactions: LruHashSet::new( + NonZeroUsize::new(MAX_KNOWN_TRANSACTIONS).expect("Constant is nonzero"), + ), + role, + }, + ); + debug_assert!(_was_in.is_none()); + }, + NotificationEvent::NotificationStreamClosed { peer } => { + let _peer = self.peers.remove(&peer); + debug_assert!(_peer.is_some()); + }, + NotificationEvent::NotificationReceived { peer, notification } => { + if let Ok(m) = + as Decode>::decode(&mut notification.as_ref()) + { + self.on_transactions(peer, m); + } else { + warn!(target: "sub-libp2p", "Failed to decode transactions list"); + } + }, + } + } + fn handle_sync_event(&mut self, event: SyncEvent) { match event { SyncEvent::PeerConnected(remote) => { @@ -349,51 +382,6 @@ where } } - async fn handle_network_event(&mut self, event: Event) { - match event { - Event::Dht(_) => {}, - Event::NotificationStreamOpened { remote, protocol, role, .. } - if protocol == self.protocol_name => - { - let _was_in = self.peers.insert( - remote, - Peer { - known_transactions: LruHashSet::new( - NonZeroUsize::new(MAX_KNOWN_TRANSACTIONS).expect("Constant is nonzero"), - ), - role, - }, - ); - debug_assert!(_was_in.is_none()); - }, - Event::NotificationStreamClosed { remote, protocol } - if protocol == self.protocol_name => - { - let _peer = self.peers.remove(&remote); - debug_assert!(_peer.is_some()); - }, - - Event::NotificationsReceived { remote, messages } => { - for (protocol, message) in messages { - if protocol != self.protocol_name { - continue - } - - if let Ok(m) = - as Decode>::decode(&mut message.as_ref()) - { - self.on_transactions(remote, m); - } else { - warn!(target: "sub-libp2p", "Failed to decode transactions list"); - } - } - }, - - // Not our concern. - Event::NotificationStreamOpened { .. } | Event::NotificationStreamClosed { .. } => {}, - } - } - /// Called when peer sends us new transactions fn on_transactions(&mut self, who: PeerId, transactions: Transactions) { // Accept transactions only when node is not major syncing @@ -485,8 +473,7 @@ where propagated_to.entry(hash).or_default().push(who.to_base58()); } trace!(target: "sync", "Sending {} transactions to {}", to_send.len(), who); - self.network - .write_notification(*who, self.protocol_name.clone(), to_send.encode()); + let _ = self.notification_service.send_sync_notification(who, to_send.encode()); } } diff --git a/substrate/client/offchain/src/api.rs b/substrate/client/offchain/src/api.rs index c7df5784d329..2901bab2f267 100644 --- a/substrate/client/offchain/src/api.rs +++ b/substrate/client/offchain/src/api.rs @@ -223,7 +223,7 @@ mod tests { use sc_client_db::offchain::LocalStorage; use sc_network::{ config::MultiaddrWithPeerId, types::ProtocolName, NetworkPeers, NetworkStateInfo, - ReputationChange, + ObservedRole, ReputationChange, }; use sp_core::offchain::{storage::OffchainDb, DbExternalities, Externalities, StorageKind}; use std::time::SystemTime; @@ -294,6 +294,10 @@ mod tests { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, _handshake: Vec) -> Option { + None + } } impl NetworkStateInfo for TestNetwork { diff --git a/substrate/client/offchain/src/lib.rs b/substrate/client/offchain/src/lib.rs index a11ac7d86ecb..704999aa334c 100644 --- a/substrate/client/offchain/src/lib.rs +++ b/substrate/client/offchain/src/lib.rs @@ -330,7 +330,9 @@ mod tests { use libp2p::{Multiaddr, PeerId}; use sc_block_builder::BlockBuilderProvider as _; use sc_client_api::Backend as _; - use sc_network::{config::MultiaddrWithPeerId, types::ProtocolName, ReputationChange}; + use sc_network::{ + config::MultiaddrWithPeerId, types::ProtocolName, ObservedRole, ReputationChange, + }; use sc_transaction_pool::BasicPool; use sc_transaction_pool_api::{InPoolTransaction, TransactionPool}; use sp_consensus::BlockOrigin; @@ -422,6 +424,10 @@ mod tests { fn sync_num_connected(&self) -> usize { unimplemented!(); } + + fn peer_role(&self, _peer_id: PeerId, _handshake: Vec) -> Option { + None + } } #[test] diff --git a/substrate/client/service/src/builder.rs b/substrate/client/service/src/builder.rs index fe18d1d002d5..9e80483954bd 100644 --- a/substrate/client/service/src/builder.rs +++ b/substrate/client/service/src/builder.rs @@ -834,17 +834,17 @@ where } // create transactions protocol and add it to the list of supported protocols of - // `network_params` - let transactions_handler_proto = sc_network_transactions::TransactionsHandlerPrototype::new( - protocol_id.clone(), - client - .block_hash(0u32.into()) - .ok() - .flatten() - .expect("Genesis block exists; qed"), - config.chain_spec.fork_id(), - ); - net_config.add_notification_protocol(transactions_handler_proto.set_config()); + let (transactions_handler_proto, transactions_config) = + sc_network_transactions::TransactionsHandlerPrototype::new( + protocol_id.clone(), + client + .block_hash(0u32.into()) + .ok() + .flatten() + .expect("Genesis block exists; qed"), + config.chain_spec.fork_id(), + ); + net_config.add_notification_protocol(transactions_config); // Create `PeerStore` and initialize it with bootnode peer ids. let peer_store = PeerStore::new( @@ -858,7 +858,6 @@ where let peer_store_handle = peer_store.handle(); spawn_handle.spawn("peer-store", Some("networking"), peer_store.run()); - let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); let (engine, sync_service, block_announce_config) = SyncingEngine::new( Roles::from(&config.role), @@ -874,7 +873,7 @@ where block_request_protocol_name, state_request_protocol_name, warp_request_protocol_name, - rx, + peer_store_handle.clone(), )?; let sync_service_import_queue = sync_service.clone(); let sync_service = Arc::new(sync_service); @@ -895,7 +894,6 @@ where fork_id: config.chain_spec.fork_id().map(ToOwned::to_owned), metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()), block_announce_config, - tx, }; let has_bootnodes = !network_params.network_config.network_config.boot_nodes.is_empty();