From bfeb28ff49a7c6026b38936d44b426d37d7c8d06 Mon Sep 17 00:00:00 2001 From: Rjan Date: Mon, 28 Mar 2022 14:50:39 +0200 Subject: [PATCH 01/31] Draft for PoSt workers page Initial draft for PoSt workers. A couple of things is needed before it can be merged: --- .../seal-workers/post-workers.md | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 content/en/storage-providers/seal-workers/post-workers.md diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md new file mode 100644 index 00000000..cb0c24d4 --- /dev/null +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -0,0 +1,107 @@ +--- +title: "PoSt workers" +description: "PoSt workers are specilized instances of a `lotus-worker`. It enables you to offload windowPoSt and winningPoSt to seperate workers." +lead: "PoSt workers are specilized instances of a `lotus-worker`. It enables you to offload windowPoSt and winningPoSt to seperate workers." +draft: false +menu: + storage-providers: + parent: "storage-providers-seal-workers" +aliases: + - /docs/storage-providers/seal-workers/ +weight: 120 +toc: true +--- + +While the Lotus Miner runs _windowPoSt_ and _winningPoSt_ by default, you can use specialized lotus workers to create a cluster of windowPoSt and winningPoSt workers to handle multiple Proof-of-Spacetime partitions simultaneously. + +One lotus worker instance can only be one of the following: +- WindowPoSt worker +- WinningPoSt worker +- Worker for sealing tasks + +## Launching PoSt workers + +Before launching your PoSt workers you need to ensure that your meets the minial requirements for the task. + +### Minminal spec requirements + +| **Task** | **GPU** | **VRAM** | **RAM** | +|:-----------:|:------------------:|:--------:|:-------:| +| winningPoSt | Highly recommended | | | +| windowPoSt | Highly recommended | 8\.5GiB | 128GiB | + +### Remote storage access + + + +### Environment variables + +Ensure that workers have access to the following environment variables when they run. These are similar to those used by the Miner daemon ([explained in the setup guide]({{< relref "setup" >}})): + +``` +# MINER_API_INFO as obtained before +export TMPDIR=/fast/disk/folder3 # used when sealing +export MINER_API_INFO::/ip4//tcp//http` +export MARKETS_API_INFO::/ip4//tcp//http` +export BELLMAN_CPU_UTILIZATION=0.875 # optimal value depends on exact hardware +export FIL_PROOFS_MAXIMIZE_CACHING=1 +export FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1 # when GPU is available +export FIL_PROOFS_USE_GPU_TREE_BUILDER=1 # when GPU is available +export FIL_PROOFS_PARAMETER_CACHE=/fast/disk/folder # > 100GiB! +export FIL_PROOFS_PARENT_CACHE=/fast/disk/folder2 # > 50GiB! +export FIL_PROOFS_USE_MULTICORE_SDR=1 +``` + +{{< alert icon="tip" >}} +When initially fetching parameter files, remember to set the [`IPFS_GATEWAY` variable when running from China]({{< relref "nodes-in-china" >}}) +{{< /alert >}} + +### Run the PoSt worker + +```shell +lotus-worker run +``` + +The above command will start the worker. You´ll need to specify which PoSt operation you want the worker to perform with one of the following flags set to true. + +``` + --winningpost enable winning post (default: false) + --windowpost enable window post (default: false) +``` + +When a winningPoSt or windowPoSt worker connects to the _lotus-miner_, the lotus miner will delegate all winningPoSt or windowPoSt tasks to that worker. If both tasks are delegated to seperate PoSt workers, no PoSt taks will be executed locally on the miner instance. If a worker is stopped, lotus-miner switches back to local PoSt automatically. + +You can verify that PoSt workers are connected to the lotus-miner with `lotus-miner proving workers`: + +```shell +$ lotus-miner proving workers +Worker 0, host computer + CPU: [ ] 0 core(s) in use + RAM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB + VMEM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB + +Worker 1, host othercomputer + CPU: [ ] 0 core(s) in use + RAM: [|||||||||||||| ] 23% 14 GiB/119.2 GiB + VMEM: [|||||||||||||| ] 23% 14 GiB/119.2 GiB + GPU: GeForce RTX 2080, not used +``` + +You can also verify that PoSt workers are connected with `lotus-miner info` in the workers-section. + +```shell +$ lotus-miner info +Workers: + Seal: 15 + WdPoSt: 4 + WinPoSt: 3 +``` + +### Optional settings + +You can configure some advanced settings for PoSt workers. + +``` + --post-parallel-reads value maximum number of parallel challenge reads (0 = no limit) (default: 0) + --post-read-timeout value time limit for reading PoSt challenges (0 = no limit) (default: 0s) +``` \ No newline at end of file From d578cba0f8ffd966e79427d0f64bf98f267b2ded Mon Sep 17 00:00:00 2001 From: Rjan Date: Tue, 29 Mar 2022 09:21:26 +0200 Subject: [PATCH 02/31] Add optional settings Adds which optional settings can be set when running PoSt-workers --- .../seal-workers/post-workers.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index cb0c24d4..2c1b7bcb 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -14,7 +14,7 @@ toc: true While the Lotus Miner runs _windowPoSt_ and _winningPoSt_ by default, you can use specialized lotus workers to create a cluster of windowPoSt and winningPoSt workers to handle multiple Proof-of-Spacetime partitions simultaneously. -One lotus worker instance can only be one of the following: +A lotus worker instance can only be one of the following: - WindowPoSt worker - WinningPoSt worker - Worker for sealing tasks @@ -69,7 +69,7 @@ The above command will start the worker. You´ll need to specify which PoSt oper --windowpost enable window post (default: false) ``` -When a winningPoSt or windowPoSt worker connects to the _lotus-miner_, the lotus miner will delegate all winningPoSt or windowPoSt tasks to that worker. If both tasks are delegated to seperate PoSt workers, no PoSt taks will be executed locally on the miner instance. If a worker is stopped, lotus-miner switches back to local PoSt automatically. +When a winningPoSt or windowPoSt worker connects to the _lotus-miner_, the lotus miner will delegate all winningPoSt or windowPoSt tasks to that worker. If both tasks are delegated to seperate PoSt workers, no PoSt tasks will be executed locally on the miner instance. If a worker is stopped, lotus-miner switches back to local PoSt automatically. You can verify that PoSt workers are connected to the lotus-miner with `lotus-miner proving workers`: @@ -87,7 +87,7 @@ Worker 1, host othercomputer GPU: GeForce RTX 2080, not used ``` -You can also verify that PoSt workers are connected with `lotus-miner info` in the workers-section. +You can also verify that PoSt workers are connected in the workers-section of `lotus-miner info`. ```shell $ lotus-miner info @@ -99,9 +99,16 @@ Workers: ### Optional settings -You can configure some advanced settings for PoSt workers. +Although the default settings are reasonable you can configure some advanced settings when running the PoSt workers, that can be tested for local optimizations of your hardware. ``` --post-parallel-reads value maximum number of parallel challenge reads (0 = no limit) (default: 0) +``` + +Lets you set an upper boundary of how many challenges it reads from your storage simultaneously. At defualt this is set to no limit, meaning that it will try to read all challenges at the same time from your storage. + +``` --post-read-timeout value time limit for reading PoSt challenges (0 = no limit) (default: 0s) -``` \ No newline at end of file +``` + +Lets you set a cut off time for reading challenges from storage, after which it will abort the job. At default this is set to no limit. \ No newline at end of file From 945b4c7ed56e2e1cb1cd0761053d6b5199116ed1 Mon Sep 17 00:00:00 2001 From: Rjan Date: Tue, 29 Mar 2022 10:20:38 +0200 Subject: [PATCH 03/31] Multiple partitions section Added a section to clarify how the PoSt workers will work in the case where a SP has multiple partitions in a single proving deadline. --- .../seal-workers/post-workers.md | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 2c1b7bcb..75744ecd 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -21,18 +21,25 @@ A lotus worker instance can only be one of the following: ## Launching PoSt workers -Before launching your PoSt workers you need to ensure that your meets the minial requirements for the task. +Before launching your PoSt worker you need to ensure that your worker meets the minimal requirements for the job it is going to perfom. You should also consider ### Minminal spec requirements +These are the minimal requirements for running each of the PoSt tasks: + | **Task** | **GPU** | **VRAM** | **RAM** | |:-----------:|:------------------:|:--------:|:-------:| | winningPoSt | Highly recommended | | | | windowPoSt | Highly recommended | 8\.5GiB | 128GiB | +{{< alert icon="callout" >}} +Although both tasks can be run with a powerful CPU, it is highly recommended to run on GPU as it speeds up the process significantly, and therefore reducing the chance to miss windowPoSt or winningPoSt +{{< /alert >}} + ### Remote storage access +PoSt workers do not need ### Environment variables @@ -111,4 +118,19 @@ Lets you set an upper boundary of how many challenges it reads from your storage --post-read-timeout value time limit for reading PoSt challenges (0 = no limit) (default: 0s) ``` -Lets you set a cut off time for reading challenges from storage, after which it will abort the job. At default this is set to no limit. \ No newline at end of file +Lets you set a cut off time for reading challenges from storage, after which it will abort the job. At default this is set to no limit. + +## Multiple partitions + +If you have multiple partitions in a single proving deadline, each partition will run on seperate workers in parallel, up to the number of partitions. + +Consider this proving deadline with four full partitions: + +``` +lotus-miner proving deadlines +Miner: f023467 +deadline partitions sectors (faults) proven partitions +0 4 9396 (0) 0 +``` + +If that storage provider has four windowPoSt workers connected, each of the partitions will be computed on each of the workers in parallel. If one windowPoSt worker gets disconnected, leaving you with only three windowPoSt workers, the first three partitions will be computed in parallel on each worker. While the last partition will be picked up by the first windowPoSt worker to finish its computation. \ No newline at end of file From 23055335bddae8f17036ed29f86c14f67a29f45e Mon Sep 17 00:00:00 2001 From: Rjan Date: Tue, 29 Mar 2022 10:43:17 +0200 Subject: [PATCH 04/31] File descriptor limt setting Add FD-limit section --- .../en/storage-providers/seal-workers/post-workers.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 75744ecd..b576031c 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -38,8 +38,7 @@ Although both tasks can be run with a powerful CPU, it is highly recommended to ### Remote storage access - -PoSt workers do not need +PoSt workers do not necessarily need local access to the sealed sectors. Workers without local access will ask workers with local access to read challenges from their storage. ### Environment variables @@ -47,7 +46,6 @@ Ensure that workers have access to the following environment variables when they ``` # MINER_API_INFO as obtained before -export TMPDIR=/fast/disk/folder3 # used when sealing export MINER_API_INFO::/ip4//tcp//http` export MARKETS_API_INFO::/ip4//tcp//http` export BELLMAN_CPU_UTILIZATION=0.875 # optimal value depends on exact hardware @@ -56,9 +54,10 @@ export FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1 # when GPU is available export FIL_PROOFS_USE_GPU_TREE_BUILDER=1 # when GPU is available export FIL_PROOFS_PARAMETER_CACHE=/fast/disk/folder # > 100GiB! export FIL_PROOFS_PARENT_CACHE=/fast/disk/folder2 # > 50GiB! -export FIL_PROOFS_USE_MULTICORE_SDR=1 ``` +The PoSt workers will fail to start if the file descriptor limit is not set high enough. So raise the the file descriptor limit with `ulimit -n 1048576` or by following the [Permanently Setting Your ULIMIT System Value](https://github.com/filecoin-project/lotus/discussions/6198) guide. + {{< alert icon="tip" >}} When initially fetching parameter files, remember to set the [`IPFS_GATEWAY` variable when running from China]({{< relref "nodes-in-china" >}}) {{< /alert >}} @@ -128,9 +127,9 @@ Consider this proving deadline with four full partitions: ``` lotus-miner proving deadlines -Miner: f023467 +Miner: f011235 deadline partitions sectors (faults) proven partitions 0 4 9396 (0) 0 ``` -If that storage provider has four windowPoSt workers connected, each of the partitions will be computed on each of the workers in parallel. If one windowPoSt worker gets disconnected, leaving you with only three windowPoSt workers, the first three partitions will be computed in parallel on each worker. While the last partition will be picked up by the first windowPoSt worker to finish its computation. \ No newline at end of file +If the storage provider has four windowPoSt workers connected, each of the partitions will be computed on each of the workers in parallel. If one windowPoSt worker gets disconnected, leaving you with only three windowPoSt workers, the first three partitions will be computed in parallel. While the last partition will be picked up by the first windowPoSt worker to finish its computation. \ No newline at end of file From b40bbf6404a2de185b256a1874835c3d281994ff Mon Sep 17 00:00:00 2001 From: Rjan Date: Tue, 29 Mar 2022 14:26:36 +0200 Subject: [PATCH 05/31] Remote Storage Access Add Remote Storage Access section --- .../seal-workers/post-workers.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index b576031c..50b425ea 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -38,7 +38,15 @@ Although both tasks can be run with a powerful CPU, it is highly recommended to ### Remote storage access -PoSt workers do not necessarily need local access to the sealed sectors. Workers without local access will ask workers with local access to read challenges from their storage. +The windowPoSt process requires reading random leafs of all the sealed sectors in a proving deadline, so when setting up windowPoSt workers one needs to consider how the workers can access those files. By default + +At least one PoSt worker needs local read access to the sealed sectors. If you have multiple PoSt workers, workers without local sector access will ask workers with sector access to read challenges from their storage. + +The lotus-miner instance disables PoSt-work when a PoSt worker is connected, meaning that a single windowPoSt worker can not rely on reading challenges from the lotus-miner instance. And therefore needs read access to the sealed sectors. + +{{< alert icon="warning" >}} +Storage providers should design their sealed sector access depending on their setup size and also redundancy required. If only one PoSt worker has access to the sealed sector that can create a single point of failure. If you have multiple partitions in a deadline, having multiple PoSt workers reading challenges from the same source may cause unwanted I/O load. +{{< /alert >}} ### Environment variables @@ -56,7 +64,7 @@ export FIL_PROOFS_PARAMETER_CACHE=/fast/disk/folder # > 100GiB! export FIL_PROOFS_PARENT_CACHE=/fast/disk/folder2 # > 50GiB! ``` -The PoSt workers will fail to start if the file descriptor limit is not set high enough. So raise the the file descriptor limit with `ulimit -n 1048576` or by following the [Permanently Setting Your ULIMIT System Value](https://github.com/filecoin-project/lotus/discussions/6198) guide. +The PoSt workers will fail to start if the file descriptor limit is not set high enough. Raise the the file descriptor limit temporarily before running with `ulimit -n 1048576` or permanently by following the [Permanently Setting Your ULIMIT System Value](https://github.com/filecoin-project/lotus/discussions/6198) guide. {{< alert icon="tip" >}} When initially fetching parameter files, remember to set the [`IPFS_GATEWAY` variable when running from China]({{< relref "nodes-in-china" >}}) @@ -75,7 +83,9 @@ The above command will start the worker. You´ll need to specify which PoSt oper --windowpost enable window post (default: false) ``` -When a winningPoSt or windowPoSt worker connects to the _lotus-miner_, the lotus miner will delegate all winningPoSt or windowPoSt tasks to that worker. If both tasks are delegated to seperate PoSt workers, no PoSt tasks will be executed locally on the miner instance. If a worker is stopped, lotus-miner switches back to local PoSt automatically. +A PoSt worker instance can only be either a winningPoSt worker, or a windowPoSt worker. + +When a winningPoSt or windowPoSt worker connects to the _lotus-miner_, the lotus miner will delegate all winningPoSt or windowPoSt tasks to that worker. If both tasks are delegated to seperate PoSt workers, no PoSt tasks will be executed locally on the miner instance. If a worker is stopped, the lotus-miner instance switches back to local PoSt automatically. You can verify that PoSt workers are connected to the lotus-miner with `lotus-miner proving workers`: @@ -132,4 +142,4 @@ deadline partitions sectors (faults) proven partitions 0 4 9396 (0) 0 ``` -If the storage provider has four windowPoSt workers connected, each of the partitions will be computed on each of the workers in parallel. If one windowPoSt worker gets disconnected, leaving you with only three windowPoSt workers, the first three partitions will be computed in parallel. While the last partition will be picked up by the first windowPoSt worker to finish its computation. \ No newline at end of file +If the storage provider has four windowPoSt workers connected, each of the partitions will be computed on each of the workers in parallel. If one windowPoSt worker gets disconnected, leaving you with only three windowPoSt workers, the first three partitions will be computed in parallel, while the last partition will be picked up by the first windowPoSt worker to finish its computation. \ No newline at end of file From 87ffd0da3cf4caa08933e84ea1d3b60e3333e80c Mon Sep 17 00:00:00 2001 From: Rjan Date: Tue, 29 Mar 2022 15:30:37 +0200 Subject: [PATCH 06/31] Testing the setup section Adds a testing the setup section, removes aliases, and some small fixes --- .../seal-workers/post-workers.md | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 50b425ea..431d7cc9 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -6,8 +6,6 @@ draft: false menu: storage-providers: parent: "storage-providers-seal-workers" -aliases: - - /docs/storage-providers/seal-workers/ weight: 120 toc: true --- @@ -38,14 +36,14 @@ Although both tasks can be run with a powerful CPU, it is highly recommended to ### Remote storage access -The windowPoSt process requires reading random leafs of all the sealed sectors in a proving deadline, so when setting up windowPoSt workers one needs to consider how the workers can access those files. By default +The windowPoSt process requires reading random leafs of all the sealed sectors in a proving deadline. So when setting up windowPoSt workers one needs to consider how the workers can access those files. By default At least one PoSt worker needs local read access to the sealed sectors. If you have multiple PoSt workers, workers without local sector access will ask workers with sector access to read challenges from their storage. The lotus-miner instance disables PoSt-work when a PoSt worker is connected, meaning that a single windowPoSt worker can not rely on reading challenges from the lotus-miner instance. And therefore needs read access to the sealed sectors. {{< alert icon="warning" >}} -Storage providers should design their sealed sector access depending on their setup size and also redundancy required. If only one PoSt worker has access to the sealed sector that can create a single point of failure. If you have multiple partitions in a deadline, having multiple PoSt workers reading challenges from the same source may cause unwanted I/O load. +Storage providers should design their workers sector access depending on their setup size and also redundancy required. If only one worker has read access to the sealed sectors, it can create a single point of failure. If you have multiple partitions in a deadline, having multiple PoSt workers reading challenges from the same source may also cause unwanted I/O load. {{< /alert >}} ### Environment variables @@ -55,8 +53,7 @@ Ensure that workers have access to the following environment variables when they ``` # MINER_API_INFO as obtained before export MINER_API_INFO::/ip4//tcp//http` -export MARKETS_API_INFO::/ip4//tcp//http` -export BELLMAN_CPU_UTILIZATION=0.875 # optimal value depends on exact hardware +export BELLMAN_CUSTOM_GPU="MODEL-NAME:CORES" # If you´re using a custom GPU export FIL_PROOFS_MAXIMIZE_CACHING=1 export FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1 # when GPU is available export FIL_PROOFS_USE_GPU_TREE_BUILDER=1 # when GPU is available @@ -113,7 +110,7 @@ Workers: WinPoSt: 3 ``` -### Optional settings +### Advanced settings Although the default settings are reasonable you can configure some advanced settings when running the PoSt workers, that can be tested for local optimizations of your hardware. @@ -131,7 +128,7 @@ Lets you set a cut off time for reading challenges from storage, after which it ## Multiple partitions -If you have multiple partitions in a single proving deadline, each partition will run on seperate workers in parallel, up to the number of partitions. +If you have multiple partitions in a single proving deadline and multiple windowPoSt workers, each partition will run on seperate workers in parallel, up to the number of partitions. Consider this proving deadline with four full partitions: @@ -142,4 +139,14 @@ deadline partitions sectors (faults) proven partitions 0 4 9396 (0) 0 ``` -If the storage provider has four windowPoSt workers connected, each of the partitions will be computed on each of the workers in parallel. If one windowPoSt worker gets disconnected, leaving you with only three windowPoSt workers, the first three partitions will be computed in parallel, while the last partition will be picked up by the first windowPoSt worker to finish its computation. \ No newline at end of file +If the storage provider has four windowPoSt workers connected, each of the partitions will be computed on each of the workers in parallel. If one windowPoSt worker gets disconnected, leaving you with only three windowPoSt workers, the first three partitions will be computed in parallel, while the last partition will be picked up by the first windowPoSt worker to finish its computation. + +### Testing the setup + +When doing changes to your PoSt setup it is useful to verify that the changes works as intended without testing it on a real proving period, and risk failing windowPoSt. + +After you have set up your windowPoSt workers you can manually trigger a windowPoSt with `lotus-miner proving compute window-post`. + +{{< alert icon="tip" >}} +This should be scheduled outside of any proving deadlines. Check `lotus-miner proving info` to see when your next proving period starts. +{{< /alert >}} \ No newline at end of file From af5f759f2b4990ee2c3cf40d23112e9ec5df6156 Mon Sep 17 00:00:00 2001 From: Rjan Date: Tue, 29 Mar 2022 15:38:07 +0200 Subject: [PATCH 07/31] Full path relref Adds full path in the relref for `explained in the setup guide` --- content/en/storage-providers/seal-workers/post-workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 431d7cc9..5090c474 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -48,7 +48,7 @@ Storage providers should design their workers sector access depending on their s ### Environment variables -Ensure that workers have access to the following environment variables when they run. These are similar to those used by the Miner daemon ([explained in the setup guide]({{< relref "setup" >}})): +Ensure that workers have access to the following environment variables when they run. These are similar to those used by the Miner daemon ([explained in the setup guide]({{< relref "configure#setup" >}})): ``` # MINER_API_INFO as obtained before From 93f133656b6742775f6622a012fb9413d812ae96 Mon Sep 17 00:00:00 2001 From: Rjan Date: Wed, 30 Mar 2022 07:56:45 +0200 Subject: [PATCH 08/31] Copy Proof-params Adds a copy proof-params from the lotus-miner instance for faster init --- content/en/storage-providers/seal-workers/post-workers.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 5090c474..f53363fe 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -36,9 +36,7 @@ Although both tasks can be run with a powerful CPU, it is highly recommended to ### Remote storage access -The windowPoSt process requires reading random leafs of all the sealed sectors in a proving deadline. So when setting up windowPoSt workers one needs to consider how the workers can access those files. By default - -At least one PoSt worker needs local read access to the sealed sectors. If you have multiple PoSt workers, workers without local sector access will ask workers with sector access to read challenges from their storage. +The windowPoSt process requires reading random leafs of all the sealed sectors in a proving deadline. When setting up windowPoSt workers one needs to consider how the workers can access those files. The PoSt workers can read challenges from any other workers, including the lotus-miner process, but it will prefer reading it from local paths. The lotus-miner instance disables PoSt-work when a PoSt worker is connected, meaning that a single windowPoSt worker can not rely on reading challenges from the lotus-miner instance. And therefore needs read access to the sealed sectors. @@ -61,6 +59,8 @@ export FIL_PROOFS_PARAMETER_CACHE=/fast/disk/folder # > 100GiB! export FIL_PROOFS_PARENT_CACHE=/fast/disk/folder2 # > 50GiB! ``` +For the PoSt worker to start, it will need to read and verify the Filecoin proof parameters. These are the same parameters as is currently on the lotus-miner instance. We recomend copying them over from your lotus-miner machine, else they will be downloaded on first run. + The PoSt workers will fail to start if the file descriptor limit is not set high enough. Raise the the file descriptor limit temporarily before running with `ulimit -n 1048576` or permanently by following the [Permanently Setting Your ULIMIT System Value](https://github.com/filecoin-project/lotus/discussions/6198) guide. {{< alert icon="tip" >}} @@ -145,7 +145,7 @@ If the storage provider has four windowPoSt workers connected, each of the parti When doing changes to your PoSt setup it is useful to verify that the changes works as intended without testing it on a real proving period, and risk failing windowPoSt. -After you have set up your windowPoSt workers you can manually trigger a windowPoSt with `lotus-miner proving compute window-post`. +After you have set up your windowPoSt workers you can manually trigger a windowPoSt with `lotus-miner proving compute window-post [deadline index]`. It will not send any messages to the chain. {{< alert icon="tip" >}} This should be scheduled outside of any proving deadlines. Check `lotus-miner proving info` to see when your next proving period starts. From d4a006be9f8c30e510623729b937955344da838b Mon Sep 17 00:00:00 2001 From: Rjan Date: Wed, 30 Mar 2022 13:35:16 +0200 Subject: [PATCH 09/31] Updated parallel-read Updates parallel-reads values, and adds a caution --- content/en/storage-providers/seal-workers/post-workers.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index f53363fe..afd8146a 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -114,11 +114,15 @@ Workers: Although the default settings are reasonable you can configure some advanced settings when running the PoSt workers, that can be tested for local optimizations of your hardware. +{{< alert icon="tip" >}} +Use with caution as changing these values to extremes might cause you to miss windowPoSt. +{{< /alert >}} + ``` - --post-parallel-reads value maximum number of parallel challenge reads (0 = no limit) (default: 0) + --post-parallel-reads value maximum number of parallel challenge reads (0 = no limit) (default: 128) ``` -Lets you set an upper boundary of how many challenges it reads from your storage simultaneously. At defualt this is set to no limit, meaning that it will try to read all challenges at the same time from your storage. +Lets you set an upper boundary of how many challenges it reads from your storage simultaneously. At defualt this is set to 128. ``` --post-read-timeout value time limit for reading PoSt challenges (0 = no limit) (default: 0s) From d65639717ff497dcf78c3b2ae735614e5fc4f9fa Mon Sep 17 00:00:00 2001 From: Rjan Date: Wed, 30 Mar 2022 14:30:05 +0200 Subject: [PATCH 10/31] Considerations Adding som considerations before running. --- content/en/storage-providers/seal-workers/post-workers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index afd8146a..aee5ea39 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -19,7 +19,7 @@ A lotus worker instance can only be one of the following: ## Launching PoSt workers -Before launching your PoSt worker you need to ensure that your worker meets the minimal requirements for the job it is going to perfom. You should also consider +Before launching your PoSt worker you need to ensure that your worker meets the minimal requirements for the job it is going to perfom. You should also consider how they can access sectors and setting correct enviroment variables be you run the workers. ### Minminal spec requirements @@ -147,7 +147,7 @@ If the storage provider has four windowPoSt workers connected, each of the parti ### Testing the setup -When doing changes to your PoSt setup it is useful to verify that the changes works as intended without testing it on a real proving period, and risk failing windowPoSt. +When doing changes to your PoSt setup it is useful to verify that the changes works as intended without testing it on a real proving period and risk failing windowPoSt. After you have set up your windowPoSt workers you can manually trigger a windowPoSt with `lotus-miner proving compute window-post [deadline index]`. It will not send any messages to the chain. From bbba97e1a3075bc6fb4fab23d0e8717121e73ec6 Mon Sep 17 00:00:00 2001 From: Rjan Date: Wed, 30 Mar 2022 17:09:29 +0200 Subject: [PATCH 11/31] Cleaning more env-variables Cleaning more env-variables --- content/en/storage-providers/seal-workers/post-workers.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index aee5ea39..1c22d309 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -49,12 +49,8 @@ Storage providers should design their workers sector access depending on their s Ensure that workers have access to the following environment variables when they run. These are similar to those used by the Miner daemon ([explained in the setup guide]({{< relref "configure#setup" >}})): ``` -# MINER_API_INFO as obtained before export MINER_API_INFO::/ip4//tcp//http` export BELLMAN_CUSTOM_GPU="MODEL-NAME:CORES" # If you´re using a custom GPU -export FIL_PROOFS_MAXIMIZE_CACHING=1 -export FIL_PROOFS_USE_GPU_COLUMN_BUILDER=1 # when GPU is available -export FIL_PROOFS_USE_GPU_TREE_BUILDER=1 # when GPU is available export FIL_PROOFS_PARAMETER_CACHE=/fast/disk/folder # > 100GiB! export FIL_PROOFS_PARENT_CACHE=/fast/disk/folder2 # > 50GiB! ``` From 6ceeb23c02f502b3140efa807afdcd88b86f769b Mon Sep 17 00:00:00 2001 From: Rjan Date: Wed, 30 Mar 2022 19:02:35 +0200 Subject: [PATCH 12/31] Remove a false statement Remove a false statement about sector storage access. --- content/en/storage-providers/seal-workers/post-workers.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 1c22d309..6bfbbe5b 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -38,8 +38,6 @@ Although both tasks can be run with a powerful CPU, it is highly recommended to The windowPoSt process requires reading random leafs of all the sealed sectors in a proving deadline. When setting up windowPoSt workers one needs to consider how the workers can access those files. The PoSt workers can read challenges from any other workers, including the lotus-miner process, but it will prefer reading it from local paths. -The lotus-miner instance disables PoSt-work when a PoSt worker is connected, meaning that a single windowPoSt worker can not rely on reading challenges from the lotus-miner instance. And therefore needs read access to the sealed sectors. - {{< alert icon="warning" >}} Storage providers should design their workers sector access depending on their setup size and also redundancy required. If only one worker has read access to the sealed sectors, it can create a single point of failure. If you have multiple partitions in a deadline, having multiple PoSt workers reading challenges from the same source may also cause unwanted I/O load. {{< /alert >}} From e936fa301c0329df1871b630f850bad8c4a47169 Mon Sep 17 00:00:00 2001 From: Rjan Date: Wed, 30 Mar 2022 19:08:23 +0200 Subject: [PATCH 13/31] Small fix Small change in sentence --- content/en/storage-providers/seal-workers/post-workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 6bfbbe5b..95495ae9 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -36,7 +36,7 @@ Although both tasks can be run with a powerful CPU, it is highly recommended to ### Remote storage access -The windowPoSt process requires reading random leafs of all the sealed sectors in a proving deadline. When setting up windowPoSt workers one needs to consider how the workers can access those files. The PoSt workers can read challenges from any other workers, including the lotus-miner process, but it will prefer reading it from local paths. +The windowPoSt process requires reading random leafs of all the sealed sectors in a proving deadline. When setting up windowPoSt workers one needs to consider how the workers can access those files. The PoSt workers can ask any other worker to read challenges for them, including the lotus-miner process, but it will prefer reading it from local paths. {{< alert icon="warning" >}} Storage providers should design their workers sector access depending on their setup size and also redundancy required. If only one worker has read access to the sealed sectors, it can create a single point of failure. If you have multiple partitions in a deadline, having multiple PoSt workers reading challenges from the same source may also cause unwanted I/O load. From 07cdc672a90b8bb4b84aeda737ab3d9f80297dfe Mon Sep 17 00:00:00 2001 From: Rjan Date: Thu, 31 Mar 2022 07:10:23 +0200 Subject: [PATCH 14/31] Update storage section --- content/en/storage-providers/seal-workers/post-workers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 95495ae9..9197a72f 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -19,7 +19,7 @@ A lotus worker instance can only be one of the following: ## Launching PoSt workers -Before launching your PoSt worker you need to ensure that your worker meets the minimal requirements for the job it is going to perfom. You should also consider how they can access sectors and setting correct enviroment variables be you run the workers. +Before launching your PoSt worker you need to ensure that your worker meets the minimal requirements for the job it is going to perfom. You should also consider how they can access sectors and setting the correct enviroment variables before you run the workers. ### Minminal spec requirements @@ -39,7 +39,7 @@ Although both tasks can be run with a powerful CPU, it is highly recommended to The windowPoSt process requires reading random leafs of all the sealed sectors in a proving deadline. When setting up windowPoSt workers one needs to consider how the workers can access those files. The PoSt workers can ask any other worker to read challenges for them, including the lotus-miner process, but it will prefer reading it from local paths. {{< alert icon="warning" >}} -Storage providers should design their workers sector access depending on their setup size and also redundancy required. If only one worker has read access to the sealed sectors, it can create a single point of failure. If you have multiple partitions in a deadline, having multiple PoSt workers reading challenges from the same source may also cause unwanted I/O load. +Storage providers should design their worker sectors access according to their setup size and also redundancy required. If only one worker has read access to the sealed sectors, it can create a single point of failure. If you have multiple partitions in a deadline, having multiple PoSt workers requesting a single worker to read challenges for them may cause unwanted I/O load. {{< /alert >}} ### Environment variables From 4c4373ae325c9e4b53d73126a49e265c8e8e38ce Mon Sep 17 00:00:00 2001 From: Rjan Date: Thu, 31 Mar 2022 16:03:20 +0200 Subject: [PATCH 15/31] Nvidia-driver mentions Add OpenCL / CUDA & Nvidia-driver mentions --- content/en/storage-providers/seal-workers/post-workers.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 9197a72f..6e59a66f 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -19,7 +19,7 @@ A lotus worker instance can only be one of the following: ## Launching PoSt workers -Before launching your PoSt worker you need to ensure that your worker meets the minimal requirements for the job it is going to perfom. You should also consider how they can access sectors and setting the correct enviroment variables before you run the workers. +Before launching your PoSt worker you need to ensure that your worker meets the minimal requirements for the job it is going to perfom. You should also consider how the PoSt workers can access the sealed sectors<> and setting the correct enviroment variables before you run the workers. ### Minminal spec requirements @@ -44,6 +44,8 @@ Storage providers should design their worker sectors access according to their s ### Environment variables +Remember to have the appropriate Nvidia-drivers and nvidia-opencl-icd installed if running OpenCL on your worker. If using CUDA, install the CUDA-toolkit and build Lotus with FFI_USE_CUDA=1 + Ensure that workers have access to the following environment variables when they run. These are similar to those used by the Miner daemon ([explained in the setup guide]({{< relref "configure#setup" >}})): ``` From 55e159f558dadaa051bac5b5f175629bec06b076 Mon Sep 17 00:00:00 2001 From: Rjan Date: Mon, 4 Apr 2022 18:33:37 +0200 Subject: [PATCH 16/31] Adding output of `proving workers` Adding output of `proving workers` --- .../seal-workers/post-workers.md | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 6e59a66f..ed0409e5 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -49,7 +49,7 @@ Remember to have the appropriate Nvidia-drivers and nvidia-opencl-icd installed Ensure that workers have access to the following environment variables when they run. These are similar to those used by the Miner daemon ([explained in the setup guide]({{< relref "configure#setup" >}})): ``` -export MINER_API_INFO::/ip4//tcp//http` +export MINER_API_INFO::/ip4//tcp//http export BELLMAN_CUSTOM_GPU="MODEL-NAME:CORES" # If you´re using a custom GPU export FIL_PROOFS_PARAMETER_CACHE=/fast/disk/folder # > 100GiB! export FIL_PROOFS_PARENT_CACHE=/fast/disk/folder2 # > 50GiB! @@ -60,7 +60,7 @@ For the PoSt worker to start, it will need to read and verify the Filecoin proof The PoSt workers will fail to start if the file descriptor limit is not set high enough. Raise the the file descriptor limit temporarily before running with `ulimit -n 1048576` or permanently by following the [Permanently Setting Your ULIMIT System Value](https://github.com/filecoin-project/lotus/discussions/6198) guide. {{< alert icon="tip" >}} -When initially fetching parameter files, remember to set the [`IPFS_GATEWAY` variable when running from China]({{< relref "nodes-in-china" >}}) +When fetching parameter files, remember to set the [`IPFS_GATEWAY` variable when running from China]({{< relref "nodes-in-china" >}}) {{< /alert >}} ### Run the PoSt worker @@ -84,16 +84,19 @@ You can verify that PoSt workers are connected to the lotus-miner with `lotus-mi ```shell $ lotus-miner proving workers -Worker 0, host computer - CPU: [ ] 0 core(s) in use - RAM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB - VMEM: [|||||||||||||||||| ] 28% 18.1 GiB/62.7 GiB - -Worker 1, host othercomputer - CPU: [ ] 0 core(s) in use - RAM: [|||||||||||||| ] 23% 14 GiB/119.2 GiB - VMEM: [|||||||||||||| ] 23% 14 GiB/119.2 GiB - GPU: GeForce RTX 2080, not used +Worker 0, host windowPoSt + CPU: [ ] 0/48 core(s) in use + RAM: [| ] 1% 4.005 GiB/119.2 GiB + VMEM: [| ] 1% 4.005 GiB/119.2 GiB + GPU: [ ] 0% 0.00/1 gpu(s) in use + GPU: GeForce RTX 2080 Ti, not used + +Worker 1, host winningPoSt + CPU: [ ] 0/16 core(s) in use + RAM: [| ] 1% 4.005 GiB/119.2 GiB + VMEM: [| ] 1% 4.005 GiB/119.2 GiB + GPU: [ ] 0% 0.00/1 gpu(s) in use + GPU: GeForce RTX 2080 Ti, not used ``` You can also verify that PoSt workers are connected in the workers-section of `lotus-miner info`. From 003fb7408b32607cb9ed0e8a526f3e2a0cc6fc57 Mon Sep 17 00:00:00 2001 From: Phi-rjan Date: Mon, 4 Apr 2022 20:16:54 +0200 Subject: [PATCH 17/31] Fix spelling/typos Fix spelling/typos Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com> --- content/en/storage-providers/seal-workers/post-workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index ed0409e5..a03a774b 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -55,7 +55,7 @@ export FIL_PROOFS_PARAMETER_CACHE=/fast/disk/folder # > 100GiB! export FIL_PROOFS_PARENT_CACHE=/fast/disk/folder2 # > 50GiB! ``` -For the PoSt worker to start, it will need to read and verify the Filecoin proof parameters. These are the same parameters as is currently on the lotus-miner instance. We recomend copying them over from your lotus-miner machine, else they will be downloaded on first run. +When the PoSt worker starts, it needs to read and verify the Filecoin proof parameters. We recommend copying them over from your lotus-miner machine. Otherwise, they will be downloaded locally on first run. The PoSt workers will fail to start if the file descriptor limit is not set high enough. Raise the the file descriptor limit temporarily before running with `ulimit -n 1048576` or permanently by following the [Permanently Setting Your ULIMIT System Value](https://github.com/filecoin-project/lotus/discussions/6198) guide. From ad88f83178fe7ac37a96a81bf442140bcb0f9cc6 Mon Sep 17 00:00:00 2001 From: Phi-rjan Date: Mon, 4 Apr 2022 20:17:56 +0200 Subject: [PATCH 18/31] Some cosmetic changes Some cosmetic changes Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com> --- content/en/storage-providers/seal-workers/post-workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index a03a774b..1b059f16 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -44,7 +44,7 @@ Storage providers should design their worker sectors access according to their s ### Environment variables -Remember to have the appropriate Nvidia-drivers and nvidia-opencl-icd installed if running OpenCL on your worker. If using CUDA, install the CUDA-toolkit and build Lotus with FFI_USE_CUDA=1 +Remember to have the appropriate Nvidia-drivers and nvidia-opencl-icd installed if running OpenCL on your worker. If using CUDA, install the CUDA-toolkit and build lotus binaries with `FFI_USE_CUDA=1` Ensure that workers have access to the following environment variables when they run. These are similar to those used by the Miner daemon ([explained in the setup guide]({{< relref "configure#setup" >}})): From 105ed5697bf92f72731b19199c12e104f7c75341 Mon Sep 17 00:00:00 2001 From: Phi-rjan Date: Mon, 4 Apr 2022 20:18:44 +0200 Subject: [PATCH 19/31] Fix spelling Fix spelling Co-authored-by: TippyFlits --- content/en/storage-providers/seal-workers/post-workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 1b059f16..2b6f6e84 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -146,7 +146,7 @@ If the storage provider has four windowPoSt workers connected, each of the parti ### Testing the setup -When doing changes to your PoSt setup it is useful to verify that the changes works as intended without testing it on a real proving period and risk failing windowPoSt. +When making changes to your PoSt setup it is useful to verify that the changes works as intended without testing it on a real proving period and risk failing windowPoSt. After you have set up your windowPoSt workers you can manually trigger a windowPoSt with `lotus-miner proving compute window-post [deadline index]`. It will not send any messages to the chain. From e7289a416b151ff8525b98a760885f7f95c581d2 Mon Sep 17 00:00:00 2001 From: Phi-rjan Date: Mon, 4 Apr 2022 20:20:14 +0200 Subject: [PATCH 20/31] Fix spelling Fix spelling Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com> --- content/en/storage-providers/seal-workers/post-workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 2b6f6e84..2793ec05 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -46,7 +46,7 @@ Storage providers should design their worker sectors access according to their s Remember to have the appropriate Nvidia-drivers and nvidia-opencl-icd installed if running OpenCL on your worker. If using CUDA, install the CUDA-toolkit and build lotus binaries with `FFI_USE_CUDA=1` -Ensure that workers have access to the following environment variables when they run. These are similar to those used by the Miner daemon ([explained in the setup guide]({{< relref "configure#setup" >}})): +The following environment variables are required to be set before starting the worker. These variables are similar to those set on the lotus-miner node as [explained in the setup guide]({{< relref "configure#setup" >}}): ``` export MINER_API_INFO::/ip4//tcp//http From 8a06944144f212b0359367443eda39873b0f7e9e Mon Sep 17 00:00:00 2001 From: Phi-rjan Date: Mon, 4 Apr 2022 20:22:37 +0200 Subject: [PATCH 21/31] Fix spelling Fix spelling Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com> --- content/en/storage-providers/seal-workers/post-workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 2793ec05..8fca51dd 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -57,7 +57,7 @@ export FIL_PROOFS_PARENT_CACHE=/fast/disk/folder2 # > 50GiB! When the PoSt worker starts, it needs to read and verify the Filecoin proof parameters. We recommend copying them over from your lotus-miner machine. Otherwise, they will be downloaded locally on first run. -The PoSt workers will fail to start if the file descriptor limit is not set high enough. Raise the the file descriptor limit temporarily before running with `ulimit -n 1048576` or permanently by following the [Permanently Setting Your ULIMIT System Value](https://github.com/filecoin-project/lotus/discussions/6198) guide. +The PoSt workers will fail to start if the file descriptor limit is not set high enough. This limit can be raised temporarily before starting the worker by running the command `ulimit -n 1048576`. Although, we recommend setting it permanently by following the [Permanently Setting Your ULIMIT System Value](https://github.com/filecoin-project/lotus/discussions/6198) guide. {{< alert icon="tip" >}} When fetching parameter files, remember to set the [`IPFS_GATEWAY` variable when running from China]({{< relref "nodes-in-china" >}}) From a55d3e0e8cceb1f7efc0fc34dfb83fd74903926c Mon Sep 17 00:00:00 2001 From: Rjan Date: Mon, 4 Apr 2022 20:31:14 +0200 Subject: [PATCH 22/31] Link to local fd-limit guide Link to local fd-limit guide --- content/en/storage-providers/seal-workers/post-workers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 8fca51dd..f76a6536 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -31,7 +31,7 @@ These are the minimal requirements for running each of the PoSt tasks: | windowPoSt | Highly recommended | 8\.5GiB | 128GiB | {{< alert icon="callout" >}} -Although both tasks can be run with a powerful CPU, it is highly recommended to run on GPU as it speeds up the process significantly, and therefore reducing the chance to miss windowPoSt or winningPoSt +Although both tasks can be run with a powerful CPU, it is highly recommended to run on GPU as it speeds up the process significantly, therefore reducing the chance to miss windowPoSt or winningPoSt {{< /alert >}} ### Remote storage access @@ -57,7 +57,7 @@ export FIL_PROOFS_PARENT_CACHE=/fast/disk/folder2 # > 50GiB! When the PoSt worker starts, it needs to read and verify the Filecoin proof parameters. We recommend copying them over from your lotus-miner machine. Otherwise, they will be downloaded locally on first run. -The PoSt workers will fail to start if the file descriptor limit is not set high enough. This limit can be raised temporarily before starting the worker by running the command `ulimit -n 1048576`. Although, we recommend setting it permanently by following the [Permanently Setting Your ULIMIT System Value](https://github.com/filecoin-project/lotus/discussions/6198) guide. +The PoSt workers will fail to start if the file descriptor limit is not set high enough. This limit can be raised temporarily before starting the worker by running the command `ulimit -n 1048576`. Although, we recommend setting it permanently by following the [Permanently Setting Your ULIMIT System Value]({{< relref "kb#soft-fd-limit" >}}) guide. {{< alert icon="tip" >}} When fetching parameter files, remember to set the [`IPFS_GATEWAY` variable when running from China]({{< relref "nodes-in-china" >}}) From 35e74ba1a2c99ad4d35f7b9d60ed3459071a55a9 Mon Sep 17 00:00:00 2001 From: Rjan Date: Mon, 4 Apr 2022 23:08:00 +0200 Subject: [PATCH 23/31] Remove parant-cache Remove parent cache, as it is not needed for the PoSt worker to function --- content/en/storage-providers/seal-workers/post-workers.md | 1 - 1 file changed, 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index f76a6536..189da6e0 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -52,7 +52,6 @@ The following environment variables are required to be set before starting the w export MINER_API_INFO::/ip4//tcp//http export BELLMAN_CUSTOM_GPU="MODEL-NAME:CORES" # If you´re using a custom GPU export FIL_PROOFS_PARAMETER_CACHE=/fast/disk/folder # > 100GiB! -export FIL_PROOFS_PARENT_CACHE=/fast/disk/folder2 # > 50GiB! ``` When the PoSt worker starts, it needs to read and verify the Filecoin proof parameters. We recommend copying them over from your lotus-miner machine. Otherwise, they will be downloaded locally on first run. From a77d65d3a14051b92d2fc9548d4834e9edbc0b11 Mon Sep 17 00:00:00 2001 From: Rjan Date: Tue, 5 Apr 2022 11:24:03 +0200 Subject: [PATCH 24/31] PoSt task will disable other tasks Enabling a PoSt task will disable other tasks --- content/en/storage-providers/seal-workers/post-workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 189da6e0..45770b59 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -75,7 +75,7 @@ The above command will start the worker. You´ll need to specify which PoSt oper --windowpost enable window post (default: false) ``` -A PoSt worker instance can only be either a winningPoSt worker, or a windowPoSt worker. +A PoSt worker instance can only be either a winningPoSt worker, or a windowPoSt worker. Enabling a PoSt tasks will automatically disable all other tasks. When a winningPoSt or windowPoSt worker connects to the _lotus-miner_, the lotus miner will delegate all winningPoSt or windowPoSt tasks to that worker. If both tasks are delegated to seperate PoSt workers, no PoSt tasks will be executed locally on the miner instance. If a worker is stopped, the lotus-miner instance switches back to local PoSt automatically. From 9a8ace29fdc5cad7f477fa3541891f93c785ac38 Mon Sep 17 00:00:00 2001 From: Rjan Date: Thu, 7 Apr 2022 09:52:48 +0200 Subject: [PATCH 25/31] winningPoSt spec Add very conservative winningPoSt spec. These are most likely lower --- content/en/storage-providers/seal-workers/post-workers.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 45770b59..e799e510 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -27,9 +27,11 @@ These are the minimal requirements for running each of the PoSt tasks: | **Task** | **GPU** | **VRAM** | **RAM** | |:-----------:|:------------------:|:--------:|:-------:| -| winningPoSt | Highly recommended | | | +| winningPoSt | Highly recommended | 6GiB* | 64GiB* | | windowPoSt | Highly recommended | 8\.5GiB | 128GiB | +*These are conservative numbers and we are currently exploring the lower boundaries for winningPoSt. + {{< alert icon="callout" >}} Although both tasks can be run with a powerful CPU, it is highly recommended to run on GPU as it speeds up the process significantly, therefore reducing the chance to miss windowPoSt or winningPoSt {{< /alert >}} From 2342ed1a29a6d3d602ed4b5201812c7208a7e15f Mon Sep 17 00:00:00 2001 From: Johnny <9611008+johnnymatthews@users.noreply.github.com> Date: Thu, 7 Apr 2022 13:53:55 -0400 Subject: [PATCH 26/31] Fixes configure link. --- content/en/storage-providers/seal-workers/post-workers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index e799e510..b96b47c7 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -48,7 +48,7 @@ Storage providers should design their worker sectors access according to their s Remember to have the appropriate Nvidia-drivers and nvidia-opencl-icd installed if running OpenCL on your worker. If using CUDA, install the CUDA-toolkit and build lotus binaries with `FFI_USE_CUDA=1` -The following environment variables are required to be set before starting the worker. These variables are similar to those set on the lotus-miner node as [explained in the setup guide]({{< relref "configure#setup" >}}): +The following environment variables are required to be set before starting the worker. These variables are similar to those set on the lotus-miner node as [explained in the setup guide]({{< relref "/storage-providers/configure/setup/" >}}): ``` export MINER_API_INFO::/ip4//tcp//http @@ -153,4 +153,4 @@ After you have set up your windowPoSt workers you can manually trigger a windowP {{< alert icon="tip" >}} This should be scheduled outside of any proving deadlines. Check `lotus-miner proving info` to see when your next proving period starts. -{{< /alert >}} \ No newline at end of file +{{< /alert >}} From e0daf11673f8230b72edbef8f9171687c1c7b691 Mon Sep 17 00:00:00 2001 From: Rjan Date: Mon, 11 Apr 2022 12:31:15 +0200 Subject: [PATCH 27/31] Remove advanced settings Removes advanced PoSt settings, as advanced configs have gotten a separate section now. --- .../seal-workers/post-workers.md | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index b96b47c7..dc9688a7 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -110,26 +110,6 @@ Workers: WinPoSt: 3 ``` -### Advanced settings - -Although the default settings are reasonable you can configure some advanced settings when running the PoSt workers, that can be tested for local optimizations of your hardware. - -{{< alert icon="tip" >}} -Use with caution as changing these values to extremes might cause you to miss windowPoSt. -{{< /alert >}} - -``` - --post-parallel-reads value maximum number of parallel challenge reads (0 = no limit) (default: 128) -``` - -Lets you set an upper boundary of how many challenges it reads from your storage simultaneously. At defualt this is set to 128. - -``` - --post-read-timeout value time limit for reading PoSt challenges (0 = no limit) (default: 0s) -``` - -Lets you set a cut off time for reading challenges from storage, after which it will abort the job. At default this is set to no limit. - ## Multiple partitions If you have multiple partitions in a single proving deadline and multiple windowPoSt workers, each partition will run on seperate workers in parallel, up to the number of partitions. From 2739ed602b78427fd11bc82c4ae44cfe1b725c12 Mon Sep 17 00:00:00 2001 From: Johnny <9611008+johnnymatthews@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:34:02 -0400 Subject: [PATCH 28/31] Fixes relref link. --- content/en/storage-providers/seal-workers/post-workers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index dc9688a7..49d4882d 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -48,7 +48,7 @@ Storage providers should design their worker sectors access according to their s Remember to have the appropriate Nvidia-drivers and nvidia-opencl-icd installed if running OpenCL on your worker. If using CUDA, install the CUDA-toolkit and build lotus binaries with `FFI_USE_CUDA=1` -The following environment variables are required to be set before starting the worker. These variables are similar to those set on the lotus-miner node as [explained in the setup guide]({{< relref "/storage-providers/configure/setup/" >}}): +The following environment variables are required to be set before starting the worker. These variables are similar to those set on the lotus-miner node as [explained in the setup guide]({{< relref "../configure/setup" >}}): ``` export MINER_API_INFO::/ip4//tcp//http From 0258be9f755acae178b8db48c550b9c746dc1521 Mon Sep 17 00:00:00 2001 From: Rjan Date: Tue, 12 Apr 2022 20:56:24 +0200 Subject: [PATCH 29/31] Typo and remove a line Fix a typo, and remove an excessive line from the `Environment variables` section --- content/en/storage-providers/seal-workers/post-workers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 49d4882d..09f41390 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -19,7 +19,7 @@ A lotus worker instance can only be one of the following: ## Launching PoSt workers -Before launching your PoSt worker you need to ensure that your worker meets the minimal requirements for the job it is going to perfom. You should also consider how the PoSt workers can access the sealed sectors<> and setting the correct enviroment variables before you run the workers. +Before launching your PoSt worker you need to ensure that your worker meets the minimal requirements for the job it is going to perfom. You should also consider how the PoSt workers can access the sealed sectors and setting the correct enviroment variables before you run the workers. ### Minminal spec requirements @@ -48,7 +48,7 @@ Storage providers should design their worker sectors access according to their s Remember to have the appropriate Nvidia-drivers and nvidia-opencl-icd installed if running OpenCL on your worker. If using CUDA, install the CUDA-toolkit and build lotus binaries with `FFI_USE_CUDA=1` -The following environment variables are required to be set before starting the worker. These variables are similar to those set on the lotus-miner node as [explained in the setup guide]({{< relref "../configure/setup" >}}): +The following environment variables are required to be set before starting the worker. ``` export MINER_API_INFO::/ip4//tcp//http From 38bb67d0637193c25c3ce9dd8cf33de7732ec82b Mon Sep 17 00:00:00 2001 From: Jiaying Wang <42981373+jennijuju@users.noreply.github.com> Date: Tue, 12 Apr 2022 23:18:50 +0200 Subject: [PATCH 30/31] Update content/en/storage-providers/seal-workers/post-workers.md --- content/en/storage-providers/seal-workers/post-workers.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index 09f41390..fce90dc6 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -9,7 +9,9 @@ menu: weight: 120 toc: true --- - +{{< alert icon="warning" >}} +This feature is supported since lotus v1.15.2. +{{< /alert >}} While the Lotus Miner runs _windowPoSt_ and _winningPoSt_ by default, you can use specialized lotus workers to create a cluster of windowPoSt and winningPoSt workers to handle multiple Proof-of-Spacetime partitions simultaneously. A lotus worker instance can only be one of the following: From 7e2760701c9a30f1119d13861657bd23e40ab3c5 Mon Sep 17 00:00:00 2001 From: Jiaying Wang <42981373+jennijuju@users.noreply.github.com> Date: Tue, 12 Apr 2022 23:21:46 +0200 Subject: [PATCH 31/31] Update post-workers.md --- content/en/storage-providers/seal-workers/post-workers.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/content/en/storage-providers/seal-workers/post-workers.md b/content/en/storage-providers/seal-workers/post-workers.md index fce90dc6..7ad8d376 100644 --- a/content/en/storage-providers/seal-workers/post-workers.md +++ b/content/en/storage-providers/seal-workers/post-workers.md @@ -9,9 +9,11 @@ menu: weight: 120 toc: true --- -{{< alert icon="warning" >}} + +{{< alert icon="callout" >}} This feature is supported since lotus v1.15.2. {{< /alert >}} + While the Lotus Miner runs _windowPoSt_ and _winningPoSt_ by default, you can use specialized lotus workers to create a cluster of windowPoSt and winningPoSt workers to handle multiple Proof-of-Spacetime partitions simultaneously. A lotus worker instance can only be one of the following: