diff --git a/build_readmes.sh b/build_readmes.sh new file mode 100755 index 0000000..16c2399 --- /dev/null +++ b/build_readmes.sh @@ -0,0 +1,7 @@ +all_dirs=("remote-externalities" "offline-election" "sub-storage" "sub-du") + +for d in "${all_dirs[@]}"; do + if [ -d $d ]; then + cd $d && cargo readme > README.md && cd .. + fi +done diff --git a/laboratory/remote-ext-elections-phragmen/js/kusama-preimage-0xca8474fb509d5a4254342130a3462da5c6140c5aacc8ea4b49d3af3ab04126b5.bin b/laboratory/remote-ext-elections-phragmen/js/kusama-preimage-0xca8474fb509d5a4254342130a3462da5c6140c5aacc8ea4b49d3af3ab04126b5.bin new file mode 100644 index 0000000..e912a72 --- /dev/null +++ b/laboratory/remote-ext-elections-phragmen/js/kusama-preimage-0xca8474fb509d5a4254342130a3462da5c6140c5aacc8ea4b49d3af3ab04126b5.bin @@ -0,0 +1 @@ +0x1800a804026d6f646c70792f74727372790000000000000000000000000000000000000000005a433c8e151c81ae948544f4cb07622b18a8430480de6fc6e1a38bc46f9c27070068ecccf604026d6f646c70792f7472737279000000000000000000000000000000000000000001861165f38113498b4e5fb2f0cc12c0a36f1d9ffe1a1e270d2345a937518fc2070068ecccf604026d6f646c70792f747273727900000000000000000000000000000000000000000239b7eaacc3ea409786f0336a8976075623acef319fc5777c76033dd585b840070068ecccf604026d6f646c70792f74727372790000000000000000000000000000000000000000224346ed63fa298c8064eddece0712330342a50f2f152a5fafb454f009a568550baa76ccf6110104026d6f646c70792f7472737279000000000000000000000000000000000000000026e93371e3e2b7c38be52ac91e8ea621a5ad7518d5580fa7b36d250b9ed3571a070068ecccf604026d6f646c70792f747273727900000000000000000000000000000000000000002c24642cef14e77315bf467c00917c749a19c3e5a6df705548a67aa7ad0ad1380b001804f0f70a04026d6f646c70792f7472737279000000000000000000000000000000000000000030599dba50b5f3ba0b36f856a761eb3c0aee61e830d4beb448ef94b6ad92be390b0098db4ab10304026d6f646c70792f7472737279000000000000000000000000000000000000000052ffd72c532815b3ec9d13291ad1989418129c3e89499a918e884d5bf3fbb0350b007891a1df0104026d6f646c70792f7472737279000000000000000000000000000000000000000086b7409a11700afb027924cb40fa43889d98709ea35319d48fea85dd35004e640b0014fbd1bc0204026d6f646c70792f74727372790000000000000000000000000000000000000000b2dbc35ccf086294a0d24e1091d08ca3b5c2a487071c4fb54070e666cc99e02d0bfc97db4ab10304026d6f646c70792f74727372790000000000000000000000000000000000000000bef3f1aa71b32bba775b3886b900a2e3fb4f4163d58c1bce0aaecfe0b55c1b5f0b00ec6a6f110a04026d6f646c70792f74727372790000000000000000000000000000000000000000d8db10fc7f845a393f39ca5611818f89daee51ad7273bd7f4a56b956f95a6414070068ecccf604026d6f646c70792f74727372790000000000000000000000000000000000000000868cd54faea1a0e45836635b2bf658733436ec69c5567d651be592392cbb69dc070068ecccf604026d6f646c70792f747273727900000000000000000000000000000000000000004e1550920067048086a9f30f799a1749508e222ad1a9d999f586e6f3e782c932070084e078f404026d6f646c70792f747273727900000000000000000000000000000000000000008242caf22b1da0a2842f09f335cce7a275990fa2c31bcf9050d2474bd664822d070084e078f404026d6f646c70792f74727372790000000000000000000000000000000000000000ca5bc1915da74aba3aadd7ce7b809045d5eb5b73559259755fdcd85a40a5dc6e0b00a09c3c9a0504026d6f646c70792f747273727900000000000000000000000000000000000000001994df5bf0f44342b1719bfbb1561286bd81b6d84f577f55ef45fe7ad6f50e4a070068ecccf604026d6f646c70792f747273727900000000000000000000000000000000000000003a45f85ffebdb18f19c75c8bed61ea11dbfa288429caebeecb23211a49eda520070068ecccf604026d6f646c70792f747273727900000000000000000000000000000000000000004adf51a47b72795366d52285e329229c836ea7bbfe139dbe8fa0700c4f86fc560b001804f0f70a04026d6f646c70792f74727372790000000000000000000000000000000000000000a493de655d6a57c136e22828f46b8532e2d31cc6f2e7a5c7ba2a20e689f7540c070010a5d4e804026d6f646c70792f747273727900000000000000000000000000000000000000002cf0838b05fb182718de859525fa1e6d53d557e5fcf631ee9ff44c619810d43b0b548d6d12160304026d6f646c70792f7472737279000000000000000000000000000000000000000052c74d5f51f34d09c9383ff7afcf64298a851b2797027dca0187ff4537871970070010a5d4e804026d6f646c70792f74727372790000000000000000000000000000000000000000124b024b427d378b0db0f7941b39a9eab8561c1f9bf301fdd46f9eee13f25176070010a5d4e804026d6f646c70792f7472737279000000000000000000000000000000000000000040a40af94843458d2a32dffda8c113143c4c263b689fb22feb2817d44b5574470b002453c70c0a04026d6f646c70792f747273727900000000000000000000000000000000000000009c988a9c500ac472e4b1ecbeab3b10514dcb8bcd5008b71a20eaa1bf281da80f0b0030ef7dba0204026d6f646c70792f74727372790000000000000000000000000000000000000000e20dc5d1d5e6a07db452dde05bff17b6c445903d045f9a776b051f7849b8601f070010a5d4e804026d6f646c70792f74727372790000000000000000000000000000000000000000cea3dabe52b2a665b1e19bf8c6913a5d54e06d6413ca3ddbec8f9a22415ec4770b54855f41940704026d6f646c70792f74727372790000000000000000000000000000000000000000367b412b107710468b5900e0145fe95adf8a9e2b5cee21b7327877621022fd3b0b00409452a30304026d6f646c70792f7472737279000000000000000000000000000000000000000082be679b08a867b9b6e92189ffaf6fe921e50e38c5204ae46450705fa0f398780b005039278c0404026d6f646c70792f74727372790000000000000000000000000000000000000000f48bf5db2175c4d131e009da423e7b56de7ce54ec953b9884af4a764e81b2f220b00204aa9d10104026d6f646c70792f747273727900000000000000000000000000000000000000008e39f979b623dc5d8dea82f9bb5a7e9864b5b53d5502c98bc88c0e73c731c00e0b00b01723010a04026d6f646c70792f74727372790000000000000000000000000000000000000000e47d82b4a3661a945005648b365901c567ddb8e893f167e3fba3f724870b5075070010a5d4e804026d6f646c70792f74727372790000000000000000000000000000000000000000294c5ac1ca255ec7328b2bfdae5b98a4a3b50dc9956407b7a1910d8fd7e581e8070084e078f404026d6f646c70792f74727372790000000000000000000000000000000000000000d71c3176c00b028a841880392ba8cd903836b938a971ad12857ca842ba29973b0b007083d05d0604026d6f646c70792f74727372790000000000000000000000000000000000000000c0b248917ec51942009e6cb18f634b944827d04edee517c29deab27d755cd1000b00204aa9d10104026d6f646c70792f74727372790000000000000000000000000000000000000000f6b21d624832094b03aa672e016462a020e217cc67b1434785b99114a2b4fa5a0b00409452a30304026d6f646c70792f74727372790000000000000000000000000000000000000000327497753b33243d0ce0f0e0d4a4c6e102b628d0ca2f49d606cff851203ee36a075405379c4d04026d6f646c70792f74727372790000000000000000000000000000000000000000b09623e54e9673f7d84c080b8b36a6ed747bd4b4f2cfe80396d31f8e2f34b471075405379c4d04026d6f646c70792f747273727900000000000000000000000000000000000000000aff6865635ae11013a83835c019d44ec3f865145943f487ae82a8e7bed3a66b0b4e9d2edbaa0104026d6f646c70792f7472737279000000000000000000000000000000000000000066823a843da63136d51a51334c6a771685c15258684e29d2821c3c395646187b07528d8906c204026d6f646c70792f74727372790000000000000000000000000000000000000000a86d962e9922cd2213f9ea746767e3513957f86a335d940c8529d8357c10641307a80a6e389b04026d6f646c70792f74727372790000000000000000000000000000000000000000fe56be5933800b45a21ee8e9817eae9f49099fdf4a20076718497092ed43c62b07aa821bce26 \ No newline at end of file diff --git a/laboratory/remote-ext-elections-phragmen/js/polkadot-preimage-0x683b144f5dc9fe9875261fc75ffb49c7d047669a887ab639d7c322783cf6593d.bin b/laboratory/remote-ext-elections-phragmen/js/polkadot-preimage-0x683b144f5dc9fe9875261fc75ffb49c7d047669a887ab639d7c322783cf6593d.bin new file mode 100644 index 0000000..63a1fe4 --- /dev/null +++ b/laboratory/remote-ext-elections-phragmen/js/polkadot-preimage-0x683b144f5dc9fe9875261fc75ffb49c7d047669a887ab639d7c322783cf6593d.bin @@ -0,0 +1 @@ +0x1a005005026d6f646c70792f747273727900000000000000000000000000000000000000001650c532ed1a8641e8922aa24ade0ff411d03edd9ed1c6b7fe42f1a801cee37c0b000196a3230105026d6f646c70792f747273727900000000000000000000000000000000000000001baa453966c043ca367ccfa19f450244447b9d32f4b7af2d9749e55a57ac09cc0b0022f836760105026d6f646c70792f747273727900000000000000000000000000000000000000001ebd2c29909eb603331b960308a070b839ee78e80fe12ef05e4639a176ab743e0b000196a3230105026d6f646c70792f7472737279000000000000000000000000000000000000000026171b3ad75723bf624a27485e51e4c2fe38f4b2d24ee52b86a979fb772c513b0b000196a3230105026d6f646c70792f74727372790000000000000000000000000000000000000000424998bac8f20fda695c2941c6214ed4ae8225801a1e8f01c43a9bded567d8bc0b00318521de0305026d6f646c70792f747273727900000000000000000000000000000000000000004e1550920067048086a9f30f799a1749508e222ad1a9d999f586e6f3e782c9320b008d5aff170105026d6f646c70792f747273727900000000000000000000000000000000000000007cccebb5c0b8fee894e3381be6ecc25f8d99e278cd390461aa1f308a2b18902f0b005182838d0205026d6f646c70792f74727372790000000000000000000000000000000000000000caafae0aaa6333fcf4dc193146945fe8e4da74aa6c16d481eef0ca35b8279d730b006b9cc8470105026d6f646c70792f7472737279000000000000000000000000000000000000000043fa61b298e82f9f207ddea327900cee26b554756c4a533f36cd875e3e7bcf060b80f30513180105026d6f646c70792f7472737279000000000000000000000000000000000000000056613c69103858e51c79e3525fcfbc8315d93383b9280f0acd551bdc75a91463070010a5d4e805026d6f646c70792f74727372790000000000000000000000000000000000000000b8e06dc2e6bb6bd269319ace4cf8f663338f8f285a0564d6a139063c985d23100b8076e910a40105026d6f646c70792f74727372790000000000000000000000000000000000000000a627338579cf7cda0b8ae74e483448f719c1f17f4e0a60ca32d08ea0e4cf4d6e070010a5d4e805026d6f646c70792f747273727900000000000000000000000000000000000000001280a479ee3beca7af1636aca17582f30829782e2c9b1b9c72aaf8060563ab370b007e6460750105026d6f646c70792f747273727900000000000000000000000000000000000000003235f7d984058bb410c163fc1d7a90e5475c0917aad77deb241093a50b4f683f0b00b0eef7260105026d6f646c70792f747273727900000000000000000000000000000000000000005661eeafb5d22cc01c4dfe7ddaf4af6210da32aa407b91be8af3f75bc65898710b8077b8a8460105026d6f646c70792f747273727900000000000000000000000000000000000000006464f15335eee136dd7c216b994ea6ac3394eb723590e9e065fd9061e05d00350b008d5aff170105026d6f646c70792f74727372790000000000000000000000000000000000000000ca5bc1915da74aba3aadd7ce7b809045d5eb5b73559259755fdcd85a40a5dc6e0b80ebf34c520105026d6f646c70792f74727372790000000000000000000000000000000000000000a6996ba5fce10b5419b5a9fd55998b5d7fde4922e61f93b021cb2cf6dfd5707e0b80ebf34c520105026d6f646c70792f74727372790000000000000000000000000000000000000000c08d5de7a5d97bea2c7ddf516d0635bddc43f326ae2f80e2595b49d4a08c46190b0094854ddd0105026d6f646c70792f74727372790000000000000000000000000000000000000000e2680e2c991a18cf7ba4710a51ba147b6856879a3047a03f8c647c69b953d630070010a5d4e8 \ No newline at end of file diff --git a/offline-election/README.md b/offline-election/README.md index 86de83b..1cd53af 100644 --- a/offline-election/README.md +++ b/offline-election/README.md @@ -1,4 +1,6 @@ -# Offline elections +# offline-election + +## Offline elections Run election algorithms of substrate (all under `sp-npos-elections`) offline. @@ -13,7 +15,7 @@ Run election algorithms of substrate (all under `sp-npos-elections`) offline. > [youtube.com/watch?v=H9OvpAOebTs](https://www.youtube.com/watch?v=H9OvpAOebTs) -### Builders +#### Builders Several tools have already built on top of this repo, such: @@ -28,7 +30,7 @@ way. As of this writing, the validator election of Polkadot/Kusama is as such: seq-phragmen -> random iterations of balancing -> reduce. This translates to: -``` +```rust cargo run -- staking -i 10 -r ``` @@ -36,11 +38,11 @@ And **if executed at the correct time** (i.e. while the election window is open) *accurately predict the next validator set*, but the nominator stake distribution will be different, because the random number of iterations is not known. -## Usage +### Usage Simply run `--help`. -``` +```rust Offline elections app. Provides utilities and debug tools around the election pallets of a substrate chain offline. @@ -84,43 +86,43 @@ SUBCOMMANDS: staking Run the staking election validator-check The general checkup of a validators ``` -## Install +### Install TODO: -## Example usage +### Example usage - Run the council election with 25 members. -``` +```rust RUST_LOG=offline-phragmen=trace cargo run -- council --count 25 ``` - Run the staking election with no equalization at a particular block number -``` +```rust cargo run --at 8b7d6e14221b4fefc4b007660c80af6d4a9ac740c50b6e918f61d521553cd17e staking ``` - Run the election with only 50 slots, and print all the nominator distributions -``` +```rust cargo run -- -vv staking --count 50 ``` - Run the above again now with `reduce()` and see how most nominator edges are... reduced. -``` +```rust cargo run -- -vv staking --count 50 --reduce ``` - Run the above again now against a remote node. -``` +```rust cargo run -- --uri wss://kusama-rpc.polkadot.io/ -vv staking --count 50 --reduce ``` -## Connecting to a node +### Connecting to a node > Both Polkadot and Kusama are growing fast and scraping the data is becoming harder and harder. I > really recommend you to try this script against a local node, or be prepared to wait for a while. @@ -132,7 +134,7 @@ Connect to a different node using the `--uri` argument e.g. `--uri wss://kusama- - **`ws://`** prefix: plain (unencrypted) websockets connection. - **`wss://`** prefix: TLS (encrypted) websockets connection. -## Logging +### Logging Scripts output additional information as logs. You need to enable them by setting `RUST_LOG` environment variable. diff --git a/offline-election/src/main.rs b/offline-election/src/main.rs index 512b2a4..d3fe78c 100644 --- a/offline-election/src/main.rs +++ b/offline-election/src/main.rs @@ -14,7 +14,146 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! TODO +//! # Offline elections +//! +//! Run election algorithms of substrate (all under `sp-npos-elections`) offline. +//! +//! > Substrate seminar about offchain phragmen and how the staking pallet works in substrate. +//! > [youtube.com/watch?v=MjOvVhc1oXw](https://www.youtube.com/watch?v=MjOvVhc1oXw). +//! +//! > Substrate seminar session about this repo prior to the overhaul (`offline-phragmen`): +//! > [youtube.com/watch?v=6omrrY11HEg](youtube.com/watch?v=6omrrY11HEg) +//! +//! > Sub0 Talk about offchain phragmen: +//! > [crowdcast.io/e/sub0-online/7](https://www.crowdcast.io/e/sub0-online/7) / +//! > [youtube.com/watch?v=H9OvpAOebTs](https://www.youtube.com/watch?v=H9OvpAOebTs) +//! +//! +//! ### Builders +//! +//! Several tools have already built on top of this repo, such: +//! +//! - https://polkadot.pro/phragmen.php +//! - https://polkadot.staking4all.org/ +//! +//! Note that the npos results generate by this repo or any of the above tools will not be exactly equal +//! to that of polkadot and kusama. This is highly dependent on the arguments passed to the `staking` +//! sub-command. The NPoS solution of both polkadot and kusama is being computed in a non-deterministic +//! way. +//! +//! As of this writing, the validator election of Polkadot/Kusama is as such: seq-phragmen -> random +//! iterations of balancing -> reduce. This translates to: +//! +//! ``` +//! cargo run -- staking -i 10 -r +//! ``` +//! +//! And **if executed at the correct time** (i.e. while the election window is open), this should +//! *accurately predict the next validator set*, but the nominator stake distribution will be different, +//! because the random number of iterations is not known. +//! +//! ## Usage +//! +//! Simply run `--help`. +//! +//! ``` +//! Offline elections app. +//! +//! Provides utilities and debug tools around the election pallets of a substrate chain offline. +//! +//! Can be used to predict next elections, diagnose previous ones, and perform checks on validators and nominators. +//! +//! USAGE: +//! offline-election [FLAGS] [OPTIONS] +//! +//! FLAGS: +//! -h, --help +//! Prints help information +//! +//! -V, --version +//! Prints version information +//! +//! -v +//! Print more output +//! +//! +//! OPTIONS: +//! --at +//! The block number at which the scrap should happen. Use only the hex value, no need for a `0x` prefix +//! +//! -n, --network +//! Network address format. Can be kusama|polkadot|substrate. +//! +//! This will also change the token display name. [default: polkadot] +//! --uri +//! The node to connect to [default: ws://localhost:9944] +//! +//! +//! SUBCOMMANDS: +//! command-center Display the command center of the staking panel +//! council Run the council election +//! current Display the current validators +//! dangling-nominators Show the nominators who are dangling: +//! help Prints this message or the help of the given subcommand(s) +//! next Display the next queued validators +//! nominator-check The general checkup of a nominator +//! staking Run the staking election +//! validator-check The general checkup of a validators +//! ``` +//! ## Install +//! +//! TODO: +//! +//! ## Example usage +//! +//! - Run the council election with 25 members. +//! +//! ``` +//! RUST_LOG=offline-phragmen=trace cargo run -- council --count 25 +//! ``` +//! +//! - Run the staking election with no equalization at a particular block number +//! +//! ``` +//! cargo run --at 8b7d6e14221b4fefc4b007660c80af6d4a9ac740c50b6e918f61d521553cd17e staking +//! ``` +//! +//! - Run the election with only 50 slots, and print all the nominator distributions +//! +//! ``` +//! cargo run -- -vv staking --count 50 +//! ``` +//! +//! - Run the above again now with `reduce()` and see how most nominator edges are... reduced. +//! +//! ``` +//! cargo run -- -vv staking --count 50 --reduce +//! ``` +//! +//! - Run the above again now against a remote node. +//! +//! ``` +//! cargo run -- --uri wss://kusama-rpc.polkadot.io/ -vv staking --count 50 --reduce +//! ``` +//! +//! ## Connecting to a node +//! +//! > Both Polkadot and Kusama are growing fast and scraping the data is becoming harder and harder. I +//! > really recommend you to try this script against a local node, or be prepared to wait for a while. +//! +//! By default it will attempt to connect to a locally running node running at `ws://127.0.0.1:9944`. +//! +//! Connect to a different node using the `--uri` argument e.g. `--uri wss://kusama-rpc.polkadot.io/`. +//! +//! - **`ws://`** prefix: plain (unencrypted) websockets connection. +//! - **`wss://`** prefix: TLS (encrypted) websockets connection. +//! +//! ## Logging +//! +//! Scripts output additional information as logs. You need to enable them by setting `RUST_LOG` +//! environment variable. +//! +//! Also, you can always use `-v`, `-vv`, ... to get more output out of each script. #![warn(missing_docs)] #![warn(unused_extern_crates)] diff --git a/remote-externalities/README.md b/remote-externalities/README.md index 78eeb8b..e31f836 100644 --- a/remote-externalities/README.md +++ b/remote-externalities/README.md @@ -1,4 +1,6 @@ -# Remote Externalities +# remote-externalities + +## Remote Externalities An equivalent of `TestExternalities` that can load its state from a remote substrate based chain. @@ -8,7 +10,7 @@ An equivalent of `TestExternalities` that can load its state from a remote subst the types that you want to query and **they must be the same as the one used in chain**. -### Example +#### Example With a test runtime @@ -25,7 +27,7 @@ impl_outer_origin! { impl frame_system::Trait for TestRuntime { .. - // we only care about these two for now. The rest can be mock. The block number type of + // we only care about these two for now. The rest can be mock. The block number type of // kusama is u32. type BlockNumber = u32; type Header = Header; diff --git a/remote-externalities/src/lib.rs b/remote-externalities/src/lib.rs index 4c8ffd8..30df58c 100644 --- a/remote-externalities/src/lib.rs +++ b/remote-externalities/src/lib.rs @@ -1,11 +1,74 @@ -//! An equivalent of `TestExternalities` that can load its state from a remote substrate based -//! chain. +//! # Remote Externalities //! -//! - For now, the `build()` method os not async and will block. This is so that the test code would -//! be freed from dealing with an executor or async tests. -//! - You typically have two options, either use a mock runtime file. In this case, you only care -//! about the types that you want to query and they must be the same as the one used in chain. Or, -//! simply use the runtime struct of the chain that you want to scrape. +//! An equivalent of `TestExternalities` that can load its state from a remote substrate based chain. +//! +//! - For now, the `build()` method is not async and will block. This is so that the test code would be +//! freed from dealing with an executor or async tests. +//! - You typically have two options, either use a mock runtime or a real one. In the case of a mock, you only care about +//! the types that you want to query and **they must be the same as the one used in chain**. +//! +//! +//! ### Example +//! +//! With a test runtime +//! +//! ```ignore +//! use remote_externalities::Builder; +//! +//! #[derive(Clone, Eq, PartialEq, Debug, Default)] +//! pub struct TestRuntime; +//! +//! use frame_system as system; +//! impl_outer_origin! { +//! pub enum Origin for TestRuntime {} +//! } +//! +//! impl frame_system::Trait for TestRuntime { +//! .. +//! // we only care about these two for now. The rest can be mock. The block number type of +//! // kusama is u32. +//! type BlockNumber = u32; +//! type Header = Header; +//! .. +//! } +//! +//! #[test] +//! fn test_runtime_works() { +//! let hash: Hash = +//! hex!["f9a4ce984129569f63edc01b1c13374779f9384f1befd39931ffdcc83acf63a7"].into(); +//! let parent: Hash = +//! hex!["540922e96a8fcaf945ed23c6f09c3e189bd88504ec945cc2171deaebeaf2f37e"].into(); +//! Builder::new() +//! .at(hash) +//! .module("System") +//! .build() +//! .execute_with(|| { +//! assert_eq!( +//! // note: the hash corresponds to 3098546. We can check only the parent. +//! // https://polkascan.io/kusama/block/3098546 +//! >::block_hash(3098545u32), +//! parent, +//! ) +//! }); +//! } +//! ``` +//! +//! Or with the real kusama runtime. +//! +//! ```ignore +//! use remote_externalities::Builder; +//! use kusama_runtime::Runtime; +//! +//! #[test] +//! fn test_runtime_works() { +//! let hash: Hash = +//! hex!["f9a4ce984129569f63edc01b1c13374779f9384f1befd39931ffdcc83acf63a7"].into(); +//! Builder::new() +//! .at(hash) +//! .module("Staking") +//! .build() +//! .execute_with(|| assert_eq!(>::validator_count(), 400)); +//! } use log::*; use sp_core::hashing::twox_128; diff --git a/sub-du/README.md b/sub-du/README.md index ddadadd..52d2d3a 100644 --- a/sub-du/README.md +++ b/sub-du/README.md @@ -1,228 +1 @@ # sub-du - -A du-like tool to scrap a chain's storage at a given block. This works with metadata, so it shoudl -work for any substrate-based chain. - -Example output from Polkadot CC1 in early July 2020: - -``` -2 M ├─┬ Staking -841K │ │── ErasStakersClipped => Value(861,517 bytes, 3615 keys) -841K │ │── ErasStakers => Value(861,517 bytes, 3615 keys) -252K │ │── Nominators => Value(258,168 bytes, 756 keys) -127K │ │── ErasRewardPoints => Value(130,310 bytes, 37 keys) -110K │ │── Ledger => Value(113,041 bytes, 1146 keys) -35 K │ │── Bonded => Value(36,672 bytes, 1146 keys) -13 K │ │── ErasValidatorPrefs => Value(13,752 bytes, 3615 keys) -1 K │ │── Validators => Value(1,274 bytes, 350 keys) -1 K │ │── Payee => Value(1,146 bytes, 1146 keys) -576B │ │── ErasTotalStake => Value(576 bytes, 36 keys) -576B │ │── ErasValidatorReward => Value(576 bytes, 36 keys) -293B │ │── SlashingSpans => Value(293 bytes, 17 keys) -233B │ │── BondedEras => Value(233 bytes) -148B │ │── ErasStartSessionIndex => Value(148 bytes, 37 keys) -16 B │ │── CanceledSlashPayout => Value(16 bytes) -13 B │ │── ActiveEra => Value(13 bytes) -4 B │ │── EarliestUnappliedSlash => Value(4 bytes) -4 B │ │── SlashRewardFraction => Value(4 bytes) -4 B │ │── CurrentEra => Value(4 bytes) -4 B │ │── MinimumValidatorCount => Value(4 bytes) -4 B │ │── ValidatorCount => Value(4 bytes) -4 B │ │── HistoryDepth => Value(4 bytes) -1 B │ │── StorageVersion => Value(1 bytes) -1 B │ │── IsCurrentSessionFinal => Value(1 bytes) -1 B │ │── EraElectionStatus => Value(1 bytes) -1 B │ │── Invulnerables => Value(1 bytes) -0 B │ │── QueuedScore => Value(0 bytes) -0 B │ │── QueuedElected => Value(0 bytes) -0 B │ │── SnapshotNominators => Value(0 bytes) -0 B │ │── SnapshotValidators => Value(0 bytes) -0 B │ │── SpanSlash => Value(0 bytes, 0 keys) -0 B │ │── NominatorSlashInEra => Value(0 bytes, 0 keys) -0 B │ │── ValidatorSlashInEra => Value(0 bytes, 0 keys) -0 B │ │── UnappliedSlashes => Value(0 bytes, 0 keys) -0 B │ │── ForceEra => Value(0 bytes) -221K ├─┬ System -146K │ │── Account => Value(149,799 bytes, 2171 keys) -75 K │ │── BlockHash => Value(76,832 bytes, 2401 keys) -37 B │ │── Events => Value(37 bytes) -16 B │ │── BlockWeight => Value(16 bytes) -10 B │ │── LastRuntimeUpgrade => Value(10 bytes) -4 B │ │── EventCount => Value(4 bytes) -0 B │ │── ExecutionPhase => Value(0 bytes) -0 B │ │── EventTopics => Value(0 bytes, 0 keys) -0 B │ │── Digest => Value(0 bytes) -0 B │ │── ExtrinsicsRoot => Value(0 bytes) -0 B │ │── ParentHash => Value(0 bytes) -0 B │ │── Number => Value(0 bytes) -0 B │ │── ExtrinsicData => Value(0 bytes, 0 keys) -0 B │ │── AllExtrinsicsLen => Value(0 bytes) -0 B │ │── ExtrinsicCount => Value(0 bytes) -161K ├─┬ Session -59 K │ │── KeyOwner => Value(60,800 bytes, 1900 keys) -59 K │ │── NextKeys => Value(60,800 bytes, 380 keys) -36 K │ │── QueuedKeys => Value(37,826 bytes) -6 K │ │── Validators => Value(6,306 bytes) -4 B │ │── CurrentIndex => Value(4 bytes) -1 B │ │── QueuedChanged => Value(1 bytes) -0 B │ │── DisabledValidators => Value(0 bytes) -78 K ├─┬ Indices -78 K │ │── Accounts => Value(80,164 bytes, 1636 keys) -58 K ├─┬ Claims -38 K │ │── Claims => Value(39,024 bytes, 2439 keys) -13 K │ │── Preclaims => Value(13,840 bytes, 692 keys) -4 K │ │── Vesting => Value(4,752 bytes, 132 keys) -2 K │ │── Signing => Value(2,461 bytes, 2461 keys) -16 B │ │── Total => Value(16 bytes) -34 K ├─┬ Balances -34 K │ │── Locks => Value(35,764 bytes, 1214 keys) -16 B │ │── TotalIssuance => Value(16 bytes) -1 B │ │── StorageVersion => Value(1 bytes) -0 B │ │── Account => Value(0 bytes, 0 keys) -24 K ├─┬ Babe -16 K │ │── UnderConstruction => Value(16,838 bytes, 3 keys) -7 K │ │── Authorities => Value(7,882 bytes) -32 B │ │── NextRandomness => Value(32 bytes) -32 B │ │── Randomness => Value(32 bytes) -8 B │ │── CurrentSlot => Value(8 bytes) -8 B │ │── GenesisSlot => Value(8 bytes) -8 B │ │── EpochIndex => Value(8 bytes) -4 B │ │── SegmentIndex => Value(4 bytes) -0 B │ │── Lateness => Value(0 bytes) -0 B │ │── Initialized => Value(0 bytes) -0 B │ │── NextEpochConfig => Value(0 bytes) -16 K ├─┬ Identity -6 K │ │── IdentityOf => Value(6,910 bytes, 66 keys) -5 K │ │── SuperOf => Value(5,216 bytes, 137 keys) -4 K │ │── SubsOf => Value(4,843 bytes, 27 keys) -115B │ │── Registrars => Value(115 bytes) -8 K ├─┬ Vesting -8 K │ │── Vesting => Value(8,496 bytes, 236 keys) -7 K ├─┬ ImOnline -6 K │ │── Keys => Value(6,306 bytes) -784B │ │── AuthoredBlocks => Value(784 bytes, 196 keys) -198B │ │── ReceivedHeartbeats => Value(198 bytes, 2 keys) -4 B │ │── HeartbeatAfter => Value(4 bytes) -6 K ├─┬ Offences -4 K │ │── Reports => Value(4,494 bytes, 32 keys) -1 K │ │── ReportsByKindIndex => Value(1,155 bytes, 1 keys) -1 K │ │── ConcurrentReportsIndex => Value(1,043 bytes, 19 keys) -1 B │ │── DeferredOffences => Value(1 bytes) -6 K ├─┬ Parachains -6 K │ │── Authorities => Value(6,306 bytes) -1 B │ │── DidUpdate => Value(1 bytes) -1 B │ │── PastCodePruning => Value(1 bytes) -0 B │ │── DownwardMessageQueue => Value(0 bytes, 0 keys) -0 B │ │── NeedsDispatch => Value(0 bytes) -0 B │ │── RelayDispatchQueueSize => Value(0 bytes, 0 keys) -0 B │ │── RelayDispatchQueue => Value(0 bytes, 0 keys) -0 B │ │── Heads => Value(0 bytes, 0 keys) -0 B │ │── FutureCode => Value(0 bytes, 0 keys) -0 B │ │── FutureCodeUpgrades => Value(0 bytes, 0 keys) -0 B │ │── PastCode => Value(0 bytes, 0 keys) -0 B │ │── PastCodeMeta => Value(0 bytes, 0 keys) -0 B │ │── Code => Value(0 bytes, 0 keys) -2 K ├─┬ RandomnessCollectiveFlip -2 K │ │── RandomMaterial => Value(2,594 bytes) -2 K ├─┬ Proxy -2 K │ │── Proxies => Value(2,133 bytes, 42 keys) -817B ├─┬ FinalityTracker -406B │ │── OrderedHints => Value(406 bytes) -406B │ │── RecentHints => Value(406 bytes) -4 B │ │── Median => Value(4 bytes) -1 B │ │── Initialized => Value(1 bytes) -0 B │ │── Update => Value(0 bytes) -280B ├─┬ Grandpa -272B │ │── SetIdSession => Value(272 bytes, 68 keys) -8 B │ │── CurrentSetId => Value(8 bytes) -0 B │ │── Stalled => Value(0 bytes) -0 B │ │── NextForced => Value(0 bytes) -0 B │ │── PendingChange => Value(0 bytes) -0 B │ │── State => Value(0 bytes) -97 B ├─┬ TechnicalCommittee -97 B │ │── Members => Value(97 bytes) -0 B │ │── Prime => Value(0 bytes) -0 B │ │── ProposalCount => Value(0 bytes) -0 B │ │── Voting => Value(0 bytes, 0 keys) -0 B │ │── ProposalOf => Value(0 bytes, 0 keys) -0 B │ │── Proposals => Value(0 bytes) -89 B ├─┬ Multisig -89 B │ │── Multisigs => Value(89 bytes, 1 keys) -0 B │ │── Calls => Value(0 bytes, 0 keys) -32 B ├─┬ Sudo -32 B │ │── Key => Value(32 bytes) -17 B ├─┬ TransactionPayment -16 B │ │── NextFeeMultiplier => Value(16 bytes) -1 B │ │── StorageVersion => Value(1 bytes) -13 B ├─┬ Democracy -4 B │ │── LowestUnbaked => Value(4 bytes) -4 B │ │── ReferendumCount => Value(4 bytes) -4 B │ │── PublicPropCount => Value(4 bytes) -1 B │ │── StorageVersion => Value(1 bytes) -0 B │ │── Cancellations => Value(0 bytes, 0 keys) -0 B │ │── Blacklist => Value(0 bytes, 0 keys) -0 B │ │── NextExternal => Value(0 bytes) -0 B │ │── LastTabledWasExternal => Value(0 bytes) -0 B │ │── Locks => Value(0 bytes, 0 keys) -0 B │ │── VotingOf => Value(0 bytes, 0 keys) -0 B │ │── ReferendumInfoOf => Value(0 bytes, 0 keys) -0 B │ │── Preimages => Value(0 bytes, 0 keys) -0 B │ │── DepositOf => Value(0 bytes, 0 keys) -0 B │ │── PublicProps => Value(0 bytes) -8 B ├─┬ Timestamp -8 B │ │── Now => Value(8 bytes) -0 B │ │── DidUpdate => Value(0 bytes) -6 B ├─┬ Registrar -3 B │ │── SelectedThreads => Value(3 bytes) -1 B │ │── RetryQueue => Value(1 bytes) -1 B │ │── Active => Value(1 bytes) -1 B │ │── Parachains => Value(1 bytes) -0 B │ │── Debtors => Value(0 bytes, 0 keys) -0 B │ │── Paras => Value(0 bytes, 0 keys) -0 B │ │── PendingSwap => Value(0 bytes, 0 keys) -0 B │ │── NextFreeId => Value(0 bytes) -0 B │ │── ThreadCount => Value(0 bytes) -6 B ├─┬ ElectionsPhragmen -4 B │ │── ElectionRounds => Value(4 bytes) -1 B │ │── RunnersUp => Value(1 bytes) -1 B │ │── Members => Value(1 bytes) -0 B │ │── Candidates => Value(0 bytes) -0 B │ │── Voting => Value(0 bytes, 0 keys) -2 B ├─┬ Scheduler -2 B │ │── Agenda => Value(2 bytes, 1 keys) -0 B │ │── Lookup => Value(0 bytes, 0 keys) -1 B ├─┬ Slots -1 B │ │── ManagedIds => Value(1 bytes) -0 B │ │── Offboarding => Value(0 bytes, 0 keys) -0 B │ │── Onboarding => Value(0 bytes, 0 keys) -0 B │ │── OnboardQueue => Value(0 bytes, 0 keys) -0 B │ │── ReservedAmounts => Value(0 bytes, 0 keys) -0 B │ │── Winning => Value(0 bytes, 0 keys) -0 B │ │── AuctionInfo => Value(0 bytes) -0 B │ │── Deposits => Value(0 bytes, 0 keys) -0 B │ │── AuctionCounter => Value(0 bytes) -1 B ├─┬ Treasury -1 B │ │── Approvals => Value(1 bytes) -0 B │ │── Reasons => Value(0 bytes, 0 keys) -0 B │ │── Tips => Value(0 bytes, 0 keys) -0 B │ │── Proposals => Value(0 bytes, 0 keys) -0 B │ │── ProposalCount => Value(0 bytes) -1 B ├─┬ TechnicalMembership -1 B │ │── Members => Value(1 bytes) -0 B │ │── Prime => Value(0 bytes) -1 B ├─┬ Council -1 B │ │── Members => Value(1 bytes) -0 B │ │── Prime => Value(0 bytes) -0 B │ │── ProposalCount => Value(0 bytes) -0 B │ │── Voting => Value(0 bytes, 0 keys) -0 B │ │── ProposalOf => Value(0 bytes, 0 keys) -0 B │ │── Proposals => Value(0 bytes) -1 B ├─┬ Authorship -1 B │ │── Uncles => Value(1 bytes) -0 B │ │── DidSetUncles => Value(0 bytes) -0 B │ │── Author => Value(0 bytes) -0 B ├─┬ Attestations -0 B │ │── DidUpdate => Value(0 bytes) -0 B │ │── ParaBlockAttestations => Value(0 bytes, 0 keys) -0 B │ │── RecentParaBlocks => Value(0 bytes, 0 keys) -``` diff --git a/sub-storage/Cargo.toml b/sub-storage/Cargo.toml index 01969bd..5729a21 100644 --- a/sub-storage/Cargo.toml +++ b/sub-storage/Cargo.toml @@ -3,6 +3,7 @@ name = "sub-storage" version = "0.1.0" authors = ["Parity Technologies "] edition = "2018" +build = "build.rs" [dependencies] codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = ["derive"] } @@ -32,6 +33,7 @@ pallet-proxy = { version = "2.0.0" } [features] remote-test-kusama = [] remote-test-polkadot = [] +build-docs = [] default = [] helpers = [ "frame-system", diff --git a/sub-storage/README.md b/sub-storage/README.md new file mode 100644 index 0000000..2a41d1d --- /dev/null +++ b/sub-storage/README.md @@ -0,0 +1,21 @@ +# sub-storage + +## Sub-Storage. + +A thing wrapper around substrate's RPC calls that work with storage. This module is an +equivalent ot the polkadojs-api `api.query` and `api.const`, but written in Rust. + +This crate is heavily dependent on the `jsonspsee` crate and uses it internally to connect to +nodes. + + +The base functions of this crate make no assumption about the runtime. Some runtime-dependent +functions are provided under the `helpers` module. + +### Unsafe RPC calls. + +The most useful features provided by this crate are often marked as unsafe by the substrate +nodes. Namely, [`get_pairs`] and [`enumerate_map`] can only be used against nodes that such +external RPCs. + +THIS IS A TEST. diff --git a/sub-storage/build.rs b/sub-storage/build.rs new file mode 100644 index 0000000..331e9cc --- /dev/null +++ b/sub-storage/build.rs @@ -0,0 +1,16 @@ +#[cfg(feature = "build-docs")] +use std::process::Command; + +#[cfg(not(feature = "build-docs"))] +fn main() {} + +#[cfg(feature = "build-docs")] +fn main() { + println!("cargo:rerun-if-changed=src/lib.rs"); + let _ = Command::new("cargo") + .arg("readme") + .arg(">") + .arg("README.md") + .output() + .unwrap(); +} diff --git a/sub-storage/src/lib.rs b/sub-storage/src/lib.rs index f877767..18364d1 100644 --- a/sub-storage/src/lib.rs +++ b/sub-storage/src/lib.rs @@ -15,6 +15,8 @@ //! The most useful features provided by this crate are often marked as unsafe by the substrate //! nodes. Namely, [`get_pairs`] and [`enumerate_map`] can only be used against nodes that such //! external RPCs. +//! +//! THIS IS A TEST. use codec::Decode; use frame_support::StorageHasher; @@ -81,7 +83,6 @@ pub fn map_prefix_key(module: &[u8], storage: &[u8]) -> StorageKey { } /// create key prefix for a module as vec bytes. Basically twox128 hash of the given values. -/// TODO: can't we use stuff from frame_support::storage directly here? This is for now a duplicate. pub fn module_prefix_raw(module: &[u8], storage: &[u8]) -> Vec { let module_key = twox_128(module); let storage_key = twox_128(storage); @@ -104,6 +105,11 @@ pub async fn read(key: StorageKey, client: &Client, at: Hash) -> Opti ::decode(&mut encoded.as_slice()).ok() } +/// Get all storage pairs located under a certain prefix. +/// +/// ## Warning +/// +/// This is an unsafe RPC call. It requires connecting to a node that allows it. pub async fn get_pairs( prefix: StorageKey, client: &Client, @@ -118,6 +124,8 @@ pub async fn get_pairs( } /// Enumerate all keys and values in a storage map. +/// +/// It is basically a wrapper around `get_pairs` that also decodes types. pub async fn enumerate_map( module: &[u8], storage: &[u8], @@ -160,6 +168,7 @@ pub fn unwrap_decoded( client: &Client, module: &str, @@ -215,35 +224,36 @@ pub async fn get_head(client: &Client) -> Hash { pub async fn get_header( client: &Client, at: Hash, -) -> sp_runtime::generic::Header { +) -> Option> { let at = to_json_value(at).expect("Block hash serialization infallible"); - let data: Option> = client + client .request("chain_getHeader", Params::Array(vec![at])) .await - .expect("get chain header request failed"); - data.unwrap() + .expect("get chain header request failed") } /// Get the metadata of a chain. +/// +/// Cannot fail. Runtime must always have some bytes as metadata. pub async fn get_metadata(client: &Client, at: Hash) -> sp_core::Bytes { let at = to_json_value(at).expect("Block hash serialization infallible"); let data: Option = client .request("state_getMetadata", Params::Array(vec![at])) .await .expect("Failed to decode block"); - - data.unwrap() + data.expect("Metadata must exist") } /// Get the runtime version at the given block. +/// +/// Cannot fail. Runtime must always have some version. pub async fn get_runtime_version(client: &Client, at: Hash) -> sp_version::RuntimeVersion { let at = to_json_value(at).expect("Block hash serialization infallible"); let data: Option = client .request("state_getRuntimeVersion", Params::Array(vec![at])) .await - .expect("Failed to decode block"); - - data.unwrap() + .expect("Failed to fetch version"); + data.expect("Version must exist") } /// Get the size of a storage map. diff --git a/sub-tokens/README.md b/sub-tokens/README.md new file mode 100644 index 0000000..fc19bb7 --- /dev/null +++ b/sub-tokens/README.md @@ -0,0 +1,53 @@ +# sub-tokens + +Small crate to represent tokens in Polkadot, Kusama and Westend. Also provides macro and +functionalities for any substrate based chain. + +## Usage: + +### Default Tokens: `DOT`, `KSM`, and `WND`. +Current crate provides implementations for DOT, KSM, and WND. note that DOTs are 10 decimal +points and the other two are 12. + +```rust +use sub_tokens::DOT; + +// 100 new dot, 1 old dot. +let dots = DOT::from(1_000_000_000_000u128); + +// provides display and format implementations. +assert_eq!(format!("{}", dots), "100,000 DOT"); +assert_eq!(format!("{:?}", dots), "100,000 DOT (1,000,000,000,000)"); +``` + +### Custom tokens + +New tokens can be built from the provided macro. + +```rust +use sub_tokens::impl_token; + +// u32 token with 3 decimal points named KIZ. +impl_token!(KIZ, 1000u32, u32); + +let kiz = KIZ::from(100); +assert_eq!(format!("{}", kiz), "0,100 KIZ"); +assert_eq!(format!("{:?}", kiz), "0,100 KIZ (100)"); +``` + +### Dynamic Tokens + +A dynamic token is also provided that can be used in applications that need to dynamically +decide to which chain to connect. This token type works only with u128. + +```rust +// the alias that you will use in your crate. +type MyToken = sub_tokens::dynamic::DynamicToken; + +// set the name +sub_tokens::dynamic::set_name("CST"); +sub_tokens::dynamic::set_decimal_points(1000); + +assert_eq!(format!("{}", MyToken::from(100)), "0,100 CST"); +assert_eq!(format!("{:?}", MyToken::from(100)), "0,100 CST (100)"); +```