diff --git a/.fleek.json b/.fleek.json index 0dae6993b..5dadc440c 100644 --- a/.fleek.json +++ b/.fleek.json @@ -1,6 +1,6 @@ { "build": { - "image": "hugomrdias/puppeteer", + "image": "node:20.2", "command": "npm install && npm run build", "publicDir": "public" } diff --git a/.gitignore b/.gitignore index 57dea231d..b82708cff 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,8 @@ yarn.lock package-lock.json node_modules resources -static/_gen data/toc.json api/dist api/worker +.hugo_build.lock +assets/jsconfig.json diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 000000000..6ed5da955 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +v20.2.0 diff --git a/api/index.js b/api/index.js index 4fb6e7538..a7868699e 100644 --- a/api/index.js +++ b/api/index.js @@ -31,21 +31,41 @@ async function handleRequest(event) { async function cov(event) { const url = new URL(event.request.url) // https://github.com/filecoin-project/lotus - const repo = url.searchParams.get('repo').split('/').slice(3).join('/') - const data = await get(event, { - url: `https://codecov.io/api/gh/${repo}`, + const [owner, repo] = url.searchParams.get('repo').split('/').slice(3) + const headers = { + 'User-Agent': 'ianconsolata', + Accept: 'application/json', + Authorization: `Bearer ${CODECOV_TOKEN}`, + } + const repo_resp = await get(event, { + url: `https://api.codecov.io/api/v2/github/${owner}/repos/${repo}/`, transform: (data) => { const out = { - cov: dlv(data, 'commit.totals.c', 0), - ci: dlv(data, 'commit.ci_passed', false), - repo: dlv(data, 'repo.name', 'N/A'), - org: dlv(data, 'owner.username', 'N/A'), - lang: dlv(data, 'repo.language', 'N/A'), + branch: dlv(data, 'branch', 'master'), + lang: dlv(data, 'language', 'N/A'), } return out }, + headers, }) - return data + const repo_data = await repo_resp.json() + + const cov_data = await get(event, { + url: `https://api.codecov.io/api/v2/github/${owner}/repos/${repo}/branches/${repo_data.branch}/`, + transform: (data) => { + const out = { + cov: dlv(data, 'head_commit.totals.coverage', 0), + ci: dlv(data, 'head_commit.ci_passed', false), + repo: repo, + org: owner, + lang: repo_data.lang, + branch: repo_data.branch, + } + return out + }, + headers, + }) + return cov_data } async function github(event) { @@ -56,7 +76,7 @@ async function github(event) { const path = file.slice(7).join('/') const ref = file[6] const headers = { - 'User-Agent': 'hugomrdias', + 'User-Agent': 'ianconsolata', Authorization: `token ${GITHUB_TOKEN}`, } @@ -82,7 +102,7 @@ async function github(event) { async function releases(event) { const headers = { - 'User-Agent': 'hugomrdias', + 'User-Agent': 'ianconsolata', Authorization: `token ${GITHUB_TOKEN}`, } const rsp = await get(event, { diff --git a/api/package.json b/api/package.json index 9d1ae7c58..89fa46913 100644 --- a/api/package.json +++ b/api/package.json @@ -12,6 +12,7 @@ "dlv": "^1.1.3", "merge-options": "^3.0.3", "nanoid": "^3.1.16", - "p-map": "^4.0.0" + "p-map": "^4.0.0", + "wrangler": "^2.17.0" } } diff --git a/api/wrangler.toml b/api/wrangler.toml index 96b98f576..ffbd9ff42 100644 --- a/api/wrangler.toml +++ b/api/wrangler.toml @@ -1,7 +1,5 @@ name = "specs-api" -type = "webpack" -account_id = "fffa4b4363a7e5250af8357087263b3a" +account_id = "e85b6fdda17f8c464c343dc84d1373e8" workers_dev = true -route = "" -zone_id = "" -webpack_config = "webpack.config.js" +main = "index.js" +compatibility_date = "2023-04-27" diff --git a/config.toml b/config.toml index 18de27b79..297d7771c 100644 --- a/config.toml +++ b/config.toml @@ -5,6 +5,10 @@ canonifyurls = false theme = "book" # relativeURLs = true +# allow building even if getting remote JSON fails +# needed because rust-fil-proofs has no coverage +ignoreErrors = ["error-remote-getjson"] + # Book configuration disablePathToLower = true enableGitInfo = true @@ -41,7 +45,7 @@ enableGitInfo = true # weight = 10 [params] - API = 'https://specs-api.protocol-labs.workers.dev' + API = 'https://specs-api.filecoin-foundation.workers.dev' # (Optional, default true) Controls table of contents visibility on right side of pages. # Start and end levels can be controlled with markup.tableOfContents setting. # You can also specify this parameter per page in front matter. diff --git a/content/algorithms/crypto/poseidon.md b/content/algorithms/crypto/poseidon.md index 8d0b6564e..e3a130639 100644 --- a/content/algorithms/crypto/poseidon.md +++ b/content/algorithms/crypto/poseidon.md @@ -356,7 +356,15 @@ while providing security against known attacks (statistical, interpolation, and
-{{}}$\constb R_F, R_P = \texttt{calc\_round\_numbers}(p, M, t, \alpha)${{}}\ +{{}}$\constb R_F, R_P = \texttt{calc\_round\_numbers}(p, M, t, c_{\alpha}})${{}}\ +where the S-box case, {{}}$c_{\alpha}${{}}, is given by +{{}} +$c_{\alpha} = \begin{cases} + 0 & \if \alpha = 3 \cr + 1 & \if \alpha = 5 \cr + 2 & \if \alpha = \neg 1 +\end{cases} +{{}}\ The number of full and partial rounds, both are positive integers $R_F, R_P \typecolon \mathbb{Z}_{>0}$ and $R_F$ is even. $R_F$ and $R_P$ are calculated using either the Python script [`calc_round_numbers.py`](https://extgit.iaik.tugraz.at/krypto/hadeshash/-/blob/9d80ec0473ad7cde5a12f3aac46439ad0da68c0a/code/scripts/calc_round_numbers.py) or the [`neptune`](https://github.com/filecoin-project/neptune) Rust library, denoted {{}}$\texttt{calc\_round\_numbers}${{}}. Both methods calculate the round numbers via brute-force; by iterating over all reasonable values for $R_F$ and $R_P$ and choosing the pair that satisfies the security inequalities (provided below) while minimizing the number of S-boxes. diff --git a/content/algorithms/expected_consensus/_index.md b/content/algorithms/expected_consensus/_index.md index 7b5d5be85..78997d59f 100644 --- a/content/algorithms/expected_consensus/_index.md +++ b/content/algorithms/expected_consensus/_index.md @@ -11,7 +11,7 @@ dashboardTests: 0 ## Algorithm -Expected Consensus (EC) is a probabilistic Byzantine fault-tolerant consensus protocol. At a high level, it operates by running a leader election every epoch in which, on expectation, a set number of participants may be eligible to submit a block. EC guarantees that these winners will be anonymous until they reveal themselves by submitting a proof that they have been elected, the `ElectionProof`. Each winning miner can submit one such proof per round and will be rewarded proportionally to its power. From this point on, each wining miner also creates a proof of storage (aka Winning PoSt). Each proof can be derived from a [properly formatted beacon entry](randomness), as described below. +Expected Consensus (EC) is a probabilistic Byzantine fault-tolerant consensus protocol. At a high level, it operates by running a leader election every epoch in which, on expectation, a set number of participants may be eligible to submit a block. EC guarantees that these winners will be anonymous until they reveal themselves by submitting a proof that they have been elected, the `ElectionProof`. Each winning miner can submit one such proof per round and will be rewarded proportionally to its power. From this point on, each winning miner also creates a proof of storage (aka Winning PoSt). Each proof can be derived from a [properly formatted beacon entry](randomness), as described below. All valid blocks submitted in a given round form a `Tipset`. Every block in a Tipset adds weight to its chain. The 'best' chain is the one with the highest weight, which is to say that the fork choice rule is to choose the heaviest known chain. For more details on how to select the heaviest chain, see [Chain Selection](expected_consensus#chain-selection). While on expectation at least one block will be generated at every round, in cases where no one finds a block in a given round, a miner can simply run leader election again for the next epoch with the appropriate random seed, thereby ensuring liveness in the protocol. @@ -173,9 +173,9 @@ WinCount guarantees that a lucky single block will earn the same reward as the r **Alternative Options for the Distribution/Sortition** -Bernouli, Binomial and Poisson distributions have been considered for the _WinCount_ of a miner with power `p` out of a total network power of `N`. There are the following options: +Bernoulli, Binomial and Poisson distributions have been considered for the _WinCount_ of a miner with power `p` out of a total network power of `N`. There are the following options: -- Option 1: WinCount(p,N) ~ Bernouli(pE/N) +- Option 1: WinCount(p,N) ~ Bernoulli(pE/N) - Option 2: WinCount(p,N) ~ Binomial(E, p/N) - Option 3: WinCount(p,N) ~ Binomial(p, E/N) - Option 4: WinCount(p,N) ~ Binomial(p/M, ME/N) @@ -189,7 +189,7 @@ In Option 5 (the one used in Filecoin Leader Election), the ticket targets for e **Rationale for the Poisson Sortition choice** -- Option 1 - Bernouli(pE/N): this option is easy to implement, but comes with a drawback: if the miner's power exceeds 1/E, the miner's WinCount is always 1, but never higher than 1. +- Option 1 - Bernoulli(pE/N): this option is easy to implement, but comes with a drawback: if the miner's power exceeds 1/E, the miner's WinCount is always 1, but never higher than 1. - Option 2 - Binomial(E, p/N): the expectation of WinCount stays the same irrespectively of whether the miner splits their power into more than one Sybil nodes, but the variance increases if they choose to Sybil. Risk-seeking miners will prefer to Sybil, while risk-averse miners will prefer to pool, none of which is a behaviour the protocol should encourage. This option is not computationally-expensive as it would involve calculation of factorials and fixed-point multiplications (or small integer exponents) only. - Option 3 - Binomial(p, E/N): this option is computationally inefficient. It involves very large integer exponents. - Option 4 - Binomial(p/M, ME/N): the complexity of this option depends on the value of M. A small M results in high computational cost, similarly to Option 3. A large M, on the other hand, leads to a situation similar to that of Option 2, where a risk-seeking miner is incentivized to Sybil. Clearly none of these are desirable properties. @@ -342,7 +342,7 @@ The exact value for these parameters remain to be determined, but for testing pu - `log2b(X) = floor(log2(x)) = (binary length of X) - 1` and `log2b(0) = 0`. Note that that special case should never be used (given it would mean an empty power table). {{< hint warning >}} -**Note that if your implementation does not allow for rounding to the fourth decimal**, miners should apply the [tie-breaker below](selecting-between-tipsets-with-equal-weight). Weight changes will be on the order of single digit numbers on expectation, so this should not have an outsized impact on chain consensus across implementations. +**Note that if your implementation does not allow for rounding to the fourth decimal**, miners should apply the [tie-breaker below](#section-algorithms.expected_consensus.selecting-between-tipsets-with-equal-weight). Weight changes will be on the order of single digit numbers on expectation, so this should not have an outsized impact on chain consensus across implementations. {{< /hint >}} `ParentWeight` is the aggregate chain weight of a given block's parent set. It is calculated as @@ -397,7 +397,7 @@ This is detectable when a given miner submits two blocks that satisfy any of the A single consensus fault results into: -- miner termination and removal of power from the power table, +- miner suspension - loss of all pledge collateral (which includes the initial pledge and blocks rewards yet to be vested) ### Detection and Reporting diff --git a/content/algorithms/pos/post.md b/content/algorithms/pos/post.md index fee395007..6102f4c94 100644 --- a/content/algorithms/pos/post.md +++ b/content/algorithms/pos/post.md @@ -30,7 +30,7 @@ Before continuing into more details of the _WinnningPoSt_ and _WindowPoSt_ algor ## WinningPoSt -At the beginning of each epoch, a small number of storage miners (up to five, currently) are elected to mine new blocks, by Filecoin's [Expected Consensus](algorithms#expected_consensus) algorithm. Recall that the Filecoin blockchain operates on the basis of _tipsets_, which are groups of up to five blocks. This means that multiple blocks (up to five) can be mined at the same height. +At the beginning of each epoch, a small number of storage miners are elected to mine new blocks, by Filecoin's [Expected Consensus](algorithms#expected_consensus) algorithm. Recall that the Filecoin blockchain operates on the basis of _tipsets_, therefore multiple blocks can be mined at the same height. Each of the miners that are elected to mine a block have to submit a proof that they keep a sealed copy of the data which they have included in their proposed block, before the end of the current epoch. Successful submission of this proof is the _WinningPoSt_, which in turn grants the miner the Filecoin Block Reward, as well as the opportunity to charge other nodes fees in order to include their messages in the block. If a miner misses the epoch-end deadline, then the miner misses the opportunity to mine a block and get a Block Reward. No penalty is incurred in this case. diff --git a/content/algorithms/sdr/_index.md b/content/algorithms/sdr/_index.md index ec257f229..d654dc631 100644 --- a/content/algorithms/sdr/_index.md +++ b/content/algorithms/sdr/_index.md @@ -1156,8 +1156,8 @@ $\line{11}{}{\return \Sha{254}(\preimage) \as \Fq}$ **Implementation:** -- [`storage_proofs::porep::stacked::circuit::proof::StackedCircuit::synthesize()`](https://github.com/filecoin-project/rust-fil-proofs/blob/master/storage-proofs/porep/src/stacked/circuit/proof.rs#L77) -- [`storage_proofs::porep::stacked::circuit::params::Proof::synthesize()`](https://github.com/filecoin-project/rust-fil-proofs/blob/master/storage-proofs/porep/src/stacked/circuit/params.rs#L77) +- [`storage_proofs::porep::stacked::circuit::proof::StackedCircuit::synthesize()`](https://github.com/filecoin-project/rust-fil-proofs/blob/108517271f8166cabc23e1966d8e8f397ec75ae0/storage-proofs-porep/src/stacked/circuit/proof.rs#L64) +- [`storage_proofs::porep::stacked::circuit::params::Proof::synthesize()`](https://github.com/filecoin-project/rust-fil-proofs/blob/108517271f8166cabc23e1966d8e8f397ec75ae0/storage-proofs-porep/src/stacked/circuit/proof.rs#L92) **Additional Notation:** diff --git a/content/appendix/audit_reports.md b/content/appendix/audit_reports.md index 32da7c394..c03051c9e 100644 --- a/content/appendix/audit_reports.md +++ b/content/appendix/audit_reports.md @@ -10,6 +10,17 @@ dashboardAudit: n/a Security is a critical component in ensuring Filecoin can fulfill its mission to be the storage network for humanity. In addition to robust secure development processes, trainings, theory audits, and investing in external security research, the Filecoin project has engaged reputable third party auditing specialists to ensure that the theory behind the protocol and its implementation delivers the intended value, enabling Filecoin to be a safe and secure network. This section covers a selection of audit reports that have been published on Filecoin's theory and implementation. +## Filecoin Virtual Machine + +### `2023-03-09` Filecoin EVM (FEVM) + +- Report: [Filecoin EVM Audit]() +- Audit conducted by **Oak Security** + +The audit covers the implementation of: + +- FEVM's [builtin actors](https://github.com/filecoin-project/builtin-actors/tree/1b11df4b399550753a4105f45f58bc07015af2a3/actors/evm) out of which only [actors/evm](https://github.com/filecoin-project/builtin-actors/tree/1b11df4b399550753a4105f45f58bc07015af2a3/actors/evm) and [actors/eam](https://github.com/filecoin-project/builtin-actors/tree/1b11df4b399550753a4105f45f58bc07015af2a3/actors/eam) were included in scope along with code base of [ref-fvm](https://github.com/filecoin-project/ref-fvm). The report included auditing EVM runtime action and implementation, correctness of EVM opcodes, including Ethereum Address Manager(EAM). The report also included issues and enhancements methods for gas model and F4 addresses. The audit team also reviewed the message execution flow and kernel setup, WASM integration and FVM logs. All the valid issues raised by the audit were resolved and acknowledged including a few informational issues. More details on these issues are available in the report. + ## Lotus ### `2020-10-20` Lotus Mainnet Ready Security Audit @@ -22,6 +33,17 @@ The scope of this audit covered: - The Lotus Daemon: Core component responsible for handling the Blockchain node logic by handling peer- to-peer networking, chain syncing, block validation, data retrieval and transfer, etc. - The Lotus Storage Miner: Mining component used to manage a single storage miner by contributing to the network through Sector commitments and Proofs-of-Spacetime data proving it is storing the sectors it has committed to. This component communicates with the Lotus daemon via JSON-RPC API calls. +## Venus + +### `2021-06-29` Venus Security Audit + +- Report: [Venus Security Assessment](https://leastauthority.com/static/publications/LeastAuthority_Filecoin_Foundation_Venus_Final_Audit_Report.pdf) +- Audit conducted by: **Least Authority** + +The scope of this audit covered: + +- The Venus Daemon: Core component responsible for handling the Filecoin node logic by handling peer-to-peer networking, chain syncing, block validation, etc. + ## Actors ### `2020-10-19` Actors Mainnet Ready Security Audit diff --git a/content/glossary/_index.md b/content/glossary/_index.md index 5b622c9a7..ab7e7ef2f 100644 --- a/content/glossary/_index.md +++ b/content/glossary/_index.md @@ -377,7 +377,7 @@ Every mined block has a computed `weight`, also called its `WinCount`. Together, [_Window Proof-of-Spacetime_ (WindowPoSt)](post#windowpost) is the mechanism by which the commitments made by [storage miners](glossary#storage-miner-actor) are audited. It sees each 24-hour period broken down into a series of windows. Correspondingly, each storage miner's set of pledged [sectors](glossary#sector) is partitioned into subsets, one subset for each window. Within a given window, each storage miner must submit a [Proof-of-Spacetime](glossary#proof-of-spacetime-post) for each sector in their respective subset. This requires ready access to each of the challenged sectors, and will result in a [zk-SNARK-compressed](glossary#zksnark) proof published to the Filecoin [blockchain](glossary#blockchain) as a [message](glossary#message) in a [block](glossary#block). In this way, every sector of [pledged storage](glossary#pledged-storage) is audited at least once in any 24-hour period, and a permanent, verifiable, and public record attesting to each storage miner's continued commitment is kept. -The Filecoin network expects constant availability of stored data. Failing to submit WindowPoSt for a sector will result in a [fault](glossary#fault), and the storage miner supplying the sector will be [slashed](glossary#slash). +The Filecoin network expects constant availability of stored data. Failing to submit WindowPoSt for a sector will result in a [fault](glossary#fault), and the storage miner supplying the sector will be [slashed](glossary#slashing). ## Winning Proof-of-Spacetime (WinningPoSt) diff --git a/content/implementations/_index.md b/content/implementations/_index.md index 1d71e04d5..a6b3b620a 100644 --- a/content/implementations/_index.md +++ b/content/implementations/_index.md @@ -14,7 +14,7 @@ Filecoin is targeting [multiple implementations](https://filecoin.io/blog/announ - [Lotus](https://lotu.sh): the Go-based implementation, supported by [Protocol Labs](https://protocol.ai), - [Forest](https://github.com/ChainSafe/forest): the Rust-based implementation, supported by [ChainSafe](https://chainsafe.io), -- [Fuhon](https://github.com/filecoin-project/cpp-filecoin): the C++-based implementation, supported by [Soramitsu](https://soramitsu.co.jp), +- [Fuhon (deprecated)](https://github.com/filecoin-project/cpp-filecoin): the C++-based implementation, supported by [Soramitsu](https://soramitsu.co.jp), - [Venus](https://github.com/filecoin-project/go-filecoin): a second Go-based implementation of Filecoin, previously called `go-filecoin`, which is maintained by the [IPFS-Force Community](https://github.com/ipfs-force-community). {{}} diff --git a/content/implementations/fuhon.md b/content/implementations/fuhon.md index 1192f1b38..05ccccec5 100644 --- a/content/implementations/fuhon.md +++ b/content/implementations/fuhon.md @@ -1,5 +1,5 @@ --- -title: Fuhon +title: Fuhon (deprecated) weight: 4 dashboardWeight: 1 dashboardState: reliable @@ -15,3 +15,5 @@ Fuhon is the C++ implementation of Filecoin. The implementation uses Rust librar You can find the Fuhon codebase [here](https://github.com/filecoin-project/cpp-filecoin). The Fuhon implementation of Filecoin is supported by [Soramitsu](https://soramitsu.co.jp/). + +Since May 2022, this implementation has been deprecated and out of support. The existing code repositories will remain public. diff --git a/content/intro/systems/impl_systems.md b/content/intro/systems/impl_systems.md index feec70378..e8a7d6707 100644 --- a/content/intro/systems/impl_systems.md +++ b/content/intro/systems/impl_systems.md @@ -41,7 +41,7 @@ For this purpose, we use the `FilecoinNode` data structure, which is passed into Further, Systems MUST abide by the following limitations: - **Random crashes.** A Filecoin Node may crash at any moment. Systems must be secure and consistent through crashes. - This is primarily achived by limiting the use of persistent state, persisting such state through Ipld data structures, + This is primarily achieved by limiting the use of persistent state, persisting such state through Ipld data structures, and through the use of initialization routines that check state, and perhaps correct errors. - **Isolation.** Systems must communicate over well-defined, isolated interfaces. They must not build their critical functionality over a shared memory space. (Note: for performance, shared memory abstractions can be used to power diff --git a/content/intro/systems/why_systems.md b/content/intro/systems/why_systems.md index d6f3cab6f..a74bead61 100644 --- a/content/intro/systems/why_systems.md +++ b/content/intro/systems/why_systems.md @@ -29,12 +29,12 @@ This decoupling is useful for: - **Security Isolation:** some systems require higher operational security than others. System decoupling allows implementations to meet their security and functionality needs. A good example of this is separating Blockchain processing from Data Transfer. -- **Scalability:** systems, and various use cases, may drive different performance requirements for different opertators. +- **Scalability:** systems, and various use cases, may drive different performance requirements for different operators. System decoupling makes it easier for operators to scale their deployments along system boundaries. ## Filecoin Nodes don't need all the systems -Filecoin Nodes vary significantly, and do not need all the systems. +Filecoin Nodes vary significantly and do not need all the systems. Most systems are only needed for a subset of use cases. For example, the Blockchain System is required for synchronizing the chain, participating in secure consensus, @@ -68,7 +68,7 @@ Sometimes this is harder, and it requires detangling, adding, or removing abstra example, the `StoragePowerActor` and the `StorageMarketActor` were a single `Actor` previously. This caused a large coupling of functionality across `StorageDeal` making, the `StorageMarket`, markets in general, with Storage Mining, Sector Sealing, PoSt Generation, and more. Detangling these two sets of related functionality -requried breaking apart the one actor into two. +required breaking apart the one actor into two. ## Decomposing within a System diff --git a/content/libraries/drand/_index.md b/content/libraries/drand/_index.md index 37b3ca8ac..2f3b6f33d 100644 --- a/content/libraries/drand/_index.md +++ b/content/libraries/drand/_index.md @@ -18,7 +18,7 @@ Any node that has gathered t of the signatures can reconstruct the full BLS sign drand assumes that at least t of the n nodes are honest (and online -- for liveness). If this threshold is broken, the adversary can permanently halt randomness production but cannot otherwise bias the randomness. -You can learn more about how drand works, by visiting its [repository](https://github.com/drand/drand), or reading its [spec](https://github.com/drand/drand/blob/master/docs/SPECS.md). +You can learn more about how drand works, by visiting its [repository](https://github.com/drand/drand), or reading its [specification](https://drand.love/docs/specification/). In the following sections we look in turn at how the Filecoin protocol makes use of drand randomness, and at some of the characteristics of the specific drand network Filecoin uses. @@ -29,6 +29,7 @@ By polling the appropriate endpoint (see below for specifics on the drand networ ```json { "round": 367, + "randomness": "3439d92d58e47d342131d446a3abe264396dd264717897af30525c98408c834f", "signature": "b62dd642e939191af1f9e15bef0f0b0e9562a5f570a12a231864afe468377e2a6424a92ccfc34ef1471cbd58c37c6b020cf75ce9446d2aa1252a090250b2b1441f8a2a0d22208dcc09332eaa0143c4a508be13de63978dbed273e3b9813130d5", "previous_signature": "afc545efb57f591dbdf833c339b3369f569566a93e49578db46b6586299422483b7a2d595814046e2847494b401650a0050981e716e531b6f4b620909c2bf1476fd82cf788a110becbc77e55746a7cccd47fb171e8ae2eea2a22fcc6a512486d" } @@ -36,11 +37,12 @@ By polling the appropriate endpoint (see below for specifics on the drand networ Specifically, we have: +- `Randomness` -- SHA256 hash of the signature - `Signature` -- the threshold BLS signature on the previous signature value `Previous` and the current round number `round`. - `PreviousSignature` -- the threshold BLS signature from the previous drand round. - `Round` -- the index of Randomness in the sequence of all random values produced by this drand network. -Specifically, the message signed is the concatenation of the round number treated as a uint64 and the previous signature. At the moment, drand uses BLS signatures on the BLS12-381 curve with the latest v7 RFC of hash-to-curve and the signature is made over G1 (for more see the [drand spec](https://github.com/drand/drand/blob/master/docs/SPECS.md#cryptographic-specification)). +Specifically, the message signed is the concatenation of the round number treated as a uint64 and the previous signature. At the moment, drand uses BLS signatures on the BLS12-381 curve with the latest v7 RFC of hash-to-curve and the signature is made over G1 (for more see the [drand specification](https://drand.love/docs/specification/#cryptographic-specification). ## Polling the drand network @@ -69,7 +71,7 @@ endpoint. Thereafter, the Filecoin client can call drand's endpoints: - `/public/latest` to get the latest randomness value produced by the beacon -- `/public/` to get the randoomness value produced by the beacon at a given round +- `/public/` to get the randomness value produced by the beacon at a given round ## Using drand in Filecoin @@ -90,7 +92,7 @@ Upon receiving a new drand randomness value from a beacon, a Filecoin node shoul - that the `Signature` field is verified by the beacon's `PublicKey` as the beacon's signature of `SHA256(PreviousSignature || Round)`. - that the `Randomness` field is `SHA256(Signature)`. -See [drand](https://github.com/drand/drand/blob/master/beacon/beacon.go#L63) for an example. +See [drand](https://github.com/drand/drand/blob/0df91a710b4366d41e88ad487814a16cf88494f9/crypto/schemes.go#L68) for an example. ### Fetching the appropriate drand value while mining diff --git a/content/systems/filecoin_blockchain/storage_power_consensus/storage_power_actor.md b/content/systems/filecoin_blockchain/storage_power_consensus/storage_power_actor.md index a050d25a3..0ec9ea51e 100644 --- a/content/systems/filecoin_blockchain/storage_power_consensus/storage_power_actor.md +++ b/content/systems/filecoin_blockchain/storage_power_consensus/storage_power_actor.md @@ -16,7 +16,7 @@ dashboardTests: 0 {{}} -{{}} +{{}} ## The Power Table @@ -36,7 +36,7 @@ The Miner lifecycle in the power table should be roughly as follows: - Power of a partition is decremented immediately after a missed WindowPoSt (`DetectedFault`). - A particular sector's power is decremented when it enters into a faulty state either through Declared Faults or Skipped Faults. - A particular sector's power is added back after recovery is declared and proven by PoSt. - - A particular sector's power is removed when the sector is expired or terminated through miner invovation. + - A particular sector's power is removed when the sector is expired or terminated through miner invocation. To summarize, only sectors in the Active state will command power. A Sector becomes Active when it is added upon `ProveCommit`. Power is immediately decremented when it enters into the faulty state. Power will be restored when its declared recovery is proven. A sector's power is removed when it is expired or terminated through miner invocation. diff --git a/content/systems/filecoin_files/data_transfer/_index.md b/content/systems/filecoin_files/data_transfer/_index.md index eeb62857d..fc977c75a 100644 --- a/content/systems/filecoin_files/data_transfer/_index.md +++ b/content/systems/filecoin_files/data_transfer/_index.md @@ -44,7 +44,7 @@ There are two basic phases to any data transfer: Note that the Negotiation and Transfer stages can occur in separate round trips, or potentially the same round trip, where the requesting party implicitly agrees by sending the request, and the responding party can agree and immediately send or receive data. Whether the process is taking place in a single or multiple round-trips depends in part on whether the request is a push request (storage deal) or a pull request (retrieval deal), and on whether the data transfer negotiation process is able to piggy back on the underlying transport mechanism. -In the case of GraphSync as transport mechanism, data transfer requests can piggy back as an extension to the GraphSync protocol using [GraphSync's built-in extensibility](https://github.com/ipld/specs/blob/master/block-layer/graphsync/graphsync.md#extensions). So, only a single round trip is required for Pull Requests. However, because Graphsync is a request/response protocol with no direct support for `push` type requests, in the Push case, negotiation happens in a seperate request over data transfer's own libp2p protocol `/fil/datatransfer/1.0.0`. Other future transport mechinisms might handle both Push and Pull, either, or neither as a single round trip. +In the case of GraphSync as transport mechanism, data transfer requests can piggy back as an extension to the GraphSync protocol using [GraphSync's built-in extensibility](https://github.com/ipld/specs/blob/master/block-layer/graphsync/graphsync.md#extensions). So, only a single round trip is required for Pull Requests. However, because Graphsync is a request/response protocol with no direct support for `push` type requests, in the Push case, negotiation happens in a seperate request over data transfer's own libp2p protocol `/fil/datatransfer/1.0.0`. Other future transport mechanisms might handle both Push and Pull, either, or neither as a single round trip. Upon receiving a data transfer request, the data transfer module does the decoding the voucher and delivers it to the request validators. In storage deals, the request validator checks if the deal included is one that the recipient has agreed to before. For retrieval deals the request includes the proposal for the retrieval deal itself. As long as request validator accepts the deal proposal, everything is done at once as a single round-trip. It is worth noting that in the case of retrieval the provider can accept the deal and the data transfer request, but then pause the retrieval itself in order to carry out the unsealing process. The storage provider has to unseal all of the requested data before initiating the actual data transfer. Furthermore, the storage provider has the option of pausing the retrieval flow before starting the unsealing process in order to ask for an unsealing payment request. Storage providers have the option to request for this payment in order to cover unsealing computation costs and avoid falling victims of misbehaving clients. @@ -82,7 +82,7 @@ once the provider indicates their intent to accept and publish the client's deal A data transfer CAN be negotiated over the network via the Data Transfer Protocol, a libp2p protocol type. -Using the Data Transfer Protocol as an independent libp2p communciation mechanism is not a hard requirement -- as long as both parties have an implementation of the Data Transfer Subsystem that can talk to the other, any +Using the Data Transfer Protocol as an independent libp2p communication mechanism is not a hard requirement -- as long as both parties have an implementation of the Data Transfer Subsystem that can talk to the other, any transport mechanism (including offline mechanisms) is acceptable. ## Data Structures diff --git a/content/systems/filecoin_markets/storage_market/_index.md b/content/systems/filecoin_markets/storage_market/_index.md index 6dfc31344..44e70ac82 100644 --- a/content/systems/filecoin_markets/storage_market/_index.md +++ b/content/systems/filecoin_markets/storage_market/_index.md @@ -10,7 +10,7 @@ dashboardTests: 0 # Storage Market in Filecoin -Storage Market subsystem is the data entry point into the network. Storage miners only earn power from data stored in a storage deal and all deals live on the Filecoin network. Specific deal negotiation process happens off chain, clients and miners enter a storage deal after an agreement has been reached and post storage deals on the Filecoin network to earn block rewards and get paid for storing the data in the storage deal. A deal is only valid when it is posted on chain with signatures from both parties and at the time of posting, there are sufficient balances for both parties locked up to honor the deal in terms of deal price and deal collateral. +Storage Market subsystem is the data entry point into the network. Storage miners can earn power from data stored in a storage deal and all deals live on the Filecoin network. Specific deal negotiation process happens off chain, clients and miners enter a storage deal after an agreement has been reached and post storage deals on the Filecoin network to earn block rewards and get paid for storing the data in the storage deal. A deal is only valid when it is posted on chain with signatures from both parties and at the time of posting, there are sufficient balances for both parties locked up to honor the deal in terms of deal price and deal collateral. ## Terminology diff --git a/content/systems/filecoin_mining/sector/_index.md b/content/systems/filecoin_mining/sector/_index.md index 9fb8bd364..65dff3b1d 100644 --- a/content/systems/filecoin_mining/sector/_index.md +++ b/content/systems/filecoin_mining/sector/_index.md @@ -30,6 +30,6 @@ This diagram shows the composition of an unsealed sector and a sealed sector. **Sector Storage & Window PoSt** -The Lotus implementation of the Window PoSt scheduler can be found [here](https://github.com/filecoin-project/lotus/blob/master/storage/wdpost_sched.go) and the actual execution of Window PoSt on a sector can be found [here](https://github.com/filecoin-project/lotus/blob/master/storage/wdpost_run.go). +The Lotus implementation of the Window PoSt scheduler can be found [here](https://github.com/filecoin-project/lotus/blob/master/storage/wdpost/wdpost_sched.go) and the actual execution of Window PoSt on a sector can be found [here](https://github.com/filecoin-project/lotus/blob/master/storage/wdpost/wdpost_run.go). The Lotus block store implementation for sectors can be found [here](https://github.com/filecoin-project/lotus/blob/master/storage/sectorblocks/blocks.go). diff --git a/content/systems/filecoin_mining/sector/adding_storage.md b/content/systems/filecoin_mining/sector/adding_storage.md index 3fc1b322e..25c70be68 100644 --- a/content/systems/filecoin_mining/sector/adding_storage.md +++ b/content/systems/filecoin_mining/sector/adding_storage.md @@ -32,7 +32,7 @@ To facilitate this, there are _two types_ of Sectors that may be sealed and Prov - **Regular Sector**: A Sector that contains Client data - **Committed Capacity (CC) Sector**: A Sector with no data (all zeroes) -Miners are free to coose which types of Sectors to store. CC sectors, in particular, allow Miners to immediately make use of existing disk space: earning storage power and a higher chance at producing a block. Miners can decide if they should upgrade their CC sectors to take client deals or continue proving CC sectors. Currently, CC sectors store randomness by default in client implementation, but this does not preclude miners from storing any type of useful data that increase their private utility in CC sectors (as long as it is legal). The protocol expects that new use-cases and diversity will emerge out of such behaviour. +Miners are free to choose which types of Sectors to store. CC sectors, in particular, allow Miners to immediately make use of existing disk space: earning storage power and a higher chance at producing a block. Miners can decide if they should upgrade their CC sectors to take client deals or continue proving CC sectors. Currently, CC sectors store randomness by default in client implementation, but this does not preclude miners from storing any type of useful data that increase their private utility in CC sectors (as long as it is legal). The protocol expects that new use-cases and diversity will emerge out of such behaviour. To incentivize Miners to hoard storage space and dedicate it to Filecoin, CC Sectors have a unique capability: **they can be "upgraded" to Regular Sectors** (also called "replacing a CC Sector"). diff --git a/content/systems/filecoin_mining/sector/lifecycle.md b/content/systems/filecoin_mining/sector/lifecycle.md index 2f056bfcb..f1e346019 100644 --- a/content/systems/filecoin_mining/sector/lifecycle.md +++ b/content/systems/filecoin_mining/sector/lifecycle.md @@ -15,7 +15,7 @@ In particular, as sectors of shorter lifetime are added, the networkʼs capacity It is reasonable to assume that miners enter the network by adding Committed Capacity sectors, that is, sectors that do not contain user data. Once miners agree storage deals with clients, they upgrade their sectors to Regular Sectors. Alternatively, if they find Verified Clients and agree a storage deal with them, they upgrade their sector accordingly. Depending on whether or not a sector includes a (verified) deal, the miner acquires the corresponding storage power in the network. -All sectors are expected to remain live until the end of their sector lifetime and early dropping of sectors will result in slashing. This is done to provide clients a certain level of guarantee on the reliability of their hosted data. Sector termination can comes with a corresponding _termination fee_. +All sectors are expected to remain live until the end of their sector lifetime and early dropping of sectors will result in slashing. This is done to provide clients a certain level of guarantee on the reliability of their hosted data. Sector termination comes with a corresponding _termination fee_. As with every system it is expected that sectors will present faults. Although this might degrade the quality offered by the network, the reaction of the miner to the fault drives system decisions on whether or not the miner should be penalized. A miner can recover the faulty sector, let the system terminate the sector automatically after 42 days of faults, or proactively terminate the sector immediately in the case of unrecoverable data loss. In case of a faulty sector, a small penalty fee approximately equal to the block reward that the sector would win per day is applied. The fee is calculated per day of the sector being unavailable to the network, i.e. until the sector is recovered or terminated. diff --git a/content/systems/filecoin_mining/sector/sealing.md b/content/systems/filecoin_mining/sector/sealing.md index 108499f1b..2cf2944e1 100644 --- a/content/systems/filecoin_mining/sector/sealing.md +++ b/content/systems/filecoin_mining/sector/sealing.md @@ -18,7 +18,7 @@ Before a Sector can be used, the Miner must _seal_ the Sector: encode the data i Sealing a sector through Proof-of-Replication (PoRep) is a computation-intensive process that results in a unique encoding of the sector. Once data is sealed, storage miners: generate a proof; run a SNARK on the proof to compress it; and finally, submit the result of the compression to the blockchain as a certification of the storage commitment. Depending on the PoRep algorithm and protocol security parameters, cost profiles and performance characteristics vary and tradeoffs have to be made among sealing cost, security, onchain footprint, retrieval latency and so on. However, sectors can be sealed with commercial hardware and sealing cost is expected to decrease over time. The Filecoin Protocol will launch with Stacked Depth Robust (SDR) PoRep with a planned upgrade to Narrow Stacked Expander (NSE) PoRep with improvement in both cost and retrieval latency. -The Lotus-specific set of functions applied to the sealing of a sector can be found [here](https://github.com/filecoin-project/lotus/blob/master/storage/sealing.go). +The Lotus-specific set of functions applied to the sealing of a sector can be found [here](https://github.com/filecoin-project/lotus/blob/master/storage/pipeline/sealing.go). ## Randomness diff --git a/content/systems/filecoin_nodes/network/_index.md b/content/systems/filecoin_nodes/network/_index.md index 9b3dd6315..e3a244aad 100644 --- a/content/systems/filecoin_nodes/network/_index.md +++ b/content/systems/filecoin_nodes/network/_index.md @@ -16,9 +16,9 @@ Here is the list of libp2p protocols used by Filecoin. - **Graphsync:** Graphsync is a protocol to synchronize graphs across peers. It is used to reference, address, request and transfer blockchain and user data between Filecoin nodes. The [draft specification of GraphSync](https://github.com/ipld/specs/blob/master/block-layer/graphsync/graphsync.md) provides more details on the concepts, the interfaces and the network messages used by GraphSync. There are no Filecoin-specific modifications to the protocol id. -- **Gossipsub:** Block headers and messages are propagating through the Filecoin network using a gossip-based pubsub protocol acronymed _GossipSub_. As is traditionally the case with pubsub protocols, nodes subscribe to topics and receive messages published on those topics. When nodes receive messages from a topic they are subscribed to, they run a validation process and i) pass the message to the application, ii) forward the message further to nodes they know off being subscribed to the same topic. Furthermore, v1.1 version of GossipSub, which is the one used in Filecoin is enhanced with security mechanisms that make the protocol resilient against security attacks. The [GossipSub Specification](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) provides all the protocol details pertaining to its design and implementation, as well as specific settings for the protocols parameters. There have been no filecoin specific modifications to the protocol id. However the topic identifiers MUST be of the form `fil/blocks/` and `fil/msgs/` +- **Gossipsub:** Block headers and messages are propagating through the Filecoin network using a gossip-based pubsub protocol acronymed _GossipSub_. As is traditionally the case with pubsub protocols, nodes subscribe to topics and receive messages published on those topics. When nodes receive messages from a topic they are subscribed to, they run a validation process and i) pass the message to the application, ii) forward the message further to nodes they know of being subscribed to the same topic. Furthermore, v1.1 version of GossipSub, which is the one used in Filecoin is enhanced with security mechanisms that make the protocol resilient against security attacks. The [GossipSub Specification](https://github.com/libp2p/specs/tree/master/pubsub/gossipsub) provides all the protocol details pertaining to its design and implementation, as well as specific settings for the protocols parameters. There have been no Filecoin-specific modifications to the protocol id. However the topic identifiers MUST be of the form `fil/blocks/` and `fil/msgs/` -- **Kademlia DHT:** The Kademlia DHT is a distributed hash table with a logarithmic bound on the maximum number of lookups for a particular node. In the Filecoin network, the Kademlia DHT is used primarily for peer discovery and peer routing. In particular, when a node wants to store data in the Filecoin network, they get a list of miners and their node information. This node information includes (among other things) the PeerID of the miner. In order to connect to the miner and exchange data, the node that wants to store data in the network has to find the Multiaddress of the miner, which they do by queering the DHT. The [libp2p Kad DHT Specification](https://github.com/libp2p/go-libp2p-kad-dht) provides implementation details of the DHT structure. For the Filecoin network, the protocol id must be of the form `fil//kad/1.0.0`. +- **Kademlia DHT:** The Kademlia DHT is a distributed hash table with a logarithmic bound on the maximum number of lookups for a particular node. In the Filecoin network, the Kademlia DHT is used primarily for peer discovery and peer routing. In particular, when a node wants to store data in the Filecoin network, they get a list of miners and their node information. This node information includes (among other things) the PeerID of the miner. In order to connect to the miner and exchange data, the node that wants to store data in the network has to find the Multiaddress of the miner, which they do by querying the DHT. The [libp2p Kad DHT Specification](https://github.com/libp2p/go-libp2p-kad-dht) provides implementation details of the DHT structure. For the Filecoin network, the protocol id must be of the form `fil//kad/1.0.0`. - **Bootstrap List:** This is a list of nodes that a new node attempts to connect to upon joining the network. The list of bootstrap nodes and their addresses are defined by the users (i.e., applications). diff --git a/content/systems/filecoin_token/minting_model/_index.md b/content/systems/filecoin_token/minting_model/_index.md index 60155f518..34f7fea22 100644 --- a/content/systems/filecoin_token/minting_model/_index.md +++ b/content/systems/filecoin_token/minting_model/_index.md @@ -18,7 +18,7 @@ To encourage consistent storage onboarding and investment in long-term storage, Specifically, a hybrid exponential minting mechanism is introduced with a proportion of the reward coming from simple exponential decay, “Simple Minting” and the other proportion from network baseline, “Baseline Minting”. The total reward per epoch will be the sum of the two rewards. Mining Filecoin should be even more profitable with this mechanism. Simple minting allocation disproportionately rewards early miners and provides counter pressure to shocks. Baseline minting allocation mints more tokens when more value for the network has been created. More tokens are minted to facilitate greater trade when the network can unlock a greater potential. This should lead to increased creation of value for the network and lower risk of minting filecoin too quickly. -The protocol allocates 30% of Storage Mining Allocation in Simple Minting and the remaining 70% in Baseline Minting. 30% of Simple Minting can provide counter forces in the event of shocks. Baseline capacity can start from a smaller percentage of worldʼs storage today, grow at a rapid rate, and catch up to a higher but still reasonable percentage of worldʼs storage in the future. As such, the network baseline will start from 1EiB (which is less than 0.01% of the worldʼs storage today) and grow at an annual rate of 200% (higher than the usual world storage annual growth rate at 40%). The community can come together to slow down the rate of growth when the network is providing 1-10% of the worldʼs storage. +The protocol allocates 30% of Storage Mining Allocation in Simple Minting and the remaining 70% in Baseline Minting. 30% of Simple Minting can provide counter forces in the event of shocks. Baseline capacity can start from a smaller percentage of worldʼs storage today, grow at a rapid rate, and catch up to a higher but still reasonable percentage of worldʼs storage in the future. The network baseline will start from **2.5EiB**, or **2.88888888EB**, (which is less than 0.01% of the worldʼs storage today) and grow at an annual rate of 100% (higher than the usual world storage annual growth rate at 40%). The community can come together to slow down the rate of growth when the network is providing 1-10% of the worldʼs storage. There are many features that will make passing the baseline more efficient and economical and unleash a greater share of baseline minting. The community can come together to collectively achieve these goals: diff --git a/content/systems/filecoin_vm/sysactors/_index.md b/content/systems/filecoin_vm/sysactors/_index.md index 335cae9dd..c4e4162dc 100644 --- a/content/systems/filecoin_vm/sysactors/_index.md +++ b/content/systems/filecoin_vm/sysactors/_index.md @@ -14,11 +14,13 @@ dashboardTests: 0 There are eleven (11) builtin System Actors in total, but not all of them interact with the VM. Each actor is identified by a _Code ID_ (or CID). -There are two system actors required for VM processing: +There are four (4) system actors required for VM processing: - the [InitActor](sysactors#initactor), which initializes new actors and records the network name, and - the [CronActor](sysactors#cronactor), a scheduler actor that runs critical functions at every epoch. - There are another two actors that interact with the VM: + +There are another two actors that interact with the VM: + - the [AccountActor](sysactors#accountactor) responsible for user accounts (non-singleton), and - the [RewardActor](sysactors#rewardactor) for block reward and token vesting (singleton). diff --git a/package.json b/package.json index 228da67cf..6999df601 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "serve": "npm start", "watch-hugo": "npm run build-diagrams && hugo server --bind=0.0.0.0 --disableFastRender --renderToDisk", "watch-assets": "tools/watch.js --watch", - "build": "npm run build-diagrams && hugo && npm run build-toc && hugo --gc --minify", + "build": "hugo && npm run build-toc && hugo --gc", "build-diagrams": "tools/diagrams.js", "build-toc": "tools/toc.js", "clean": "premove public resources static/_gen", @@ -27,16 +27,16 @@ "tablesort": "^5.2.1" }, "devDependencies": { - "@mermaid-js/mermaid-cli": "^8.8.1", + "@mermaid-js/mermaid-cli": "^10.2.2", "chokidar": "^3.4.3", "concurrently": "^5.3.0", "conventional-github-releaser": "^3.1.5", "execa": "^4.0.3", "globby": "^11.0.1", - "graphviz-cli": "^1.0.0", - "hugo-extended": "^0.76.5", + "graphviz-cli": "^2.0.0", + "hugo-extended": "^0.113.0", "husky": ">=4", - "jsdom": "^16.4.0", + "jsdom": "^22.1.0", "lint-staged": ">=10", "np": "^6.5.0", "premove": "^3.0.1", diff --git a/static/_gen/diagrams/algorithms/expected_consensus/diagrams/double_fork.svg b/static/_gen/diagrams/algorithms/expected_consensus/diagrams/double_fork.svg new file mode 100644 index 000000000..8aa99723c --- /dev/null +++ b/static/_gen/diagrams/algorithms/expected_consensus/diagrams/double_fork.svg @@ -0,0 +1,72 @@ + + + + + + +G + + + +B1 + +B1 +MinerAny + + + +B2 + +B2 +MinerAny + + + +B2->B1 + + + + + +B3 + +B3 +MinerAny + + + +B3->B1 + + + + + +B4 + +B4 +MinerE + + + +B4->B2 + + + + + +B5 + +B5 +MinerE + + + +B5->B3 + + + + + diff --git a/static/_gen/diagrams/algorithms/expected_consensus/diagrams/parent_grinding.svg b/static/_gen/diagrams/algorithms/expected_consensus/diagrams/parent_grinding.svg new file mode 100644 index 000000000..a4d8772a3 --- /dev/null +++ b/static/_gen/diagrams/algorithms/expected_consensus/diagrams/parent_grinding.svg @@ -0,0 +1,89 @@ + + + + + + +G + + + +B1 + +B1 +MinerAny + + + +B2 + +B2 +MinerAny + + + +B2->B1 + + + + + +B3 + +B3 +MinerE + + + +B3->B1 + + + + + +B4 + +B4 +MinerE + + + +B4Null + + + + +B4->B4Null + + + + + +B5 + +B5 +MinerAny + + + +B5->B2 + + + + + +B5->B3 + + + + + +B4Null->B2 + + + + + diff --git a/static/_gen/diagrams/algorithms/expected_consensus/diagrams/time_offset.svg b/static/_gen/diagrams/algorithms/expected_consensus/diagrams/time_offset.svg new file mode 100644 index 000000000..c1206f094 --- /dev/null +++ b/static/_gen/diagrams/algorithms/expected_consensus/diagrams/time_offset.svg @@ -0,0 +1,60 @@ + + + + + + +G + + + +B1 + +B1 +MinerAny + + + +B2 + +B2 +MinerAny + + + +B2->B1 + + + + + +B3 + +B3 +MinerE + + + +B3->B2 + + + + + +B4 + +B4 +MinerE + + + +B4->B2 + + + + + + diff --git a/static/_gen/diagrams/intro/diagrams/buildsys/buildsys.svg b/static/_gen/diagrams/intro/diagrams/buildsys/buildsys.svg new file mode 100644 index 000000000..f33e900e8 --- /dev/null +++ b/static/_gen/diagrams/intro/diagrams/buildsys/buildsys.svg @@ -0,0 +1,257 @@ + + + + + + +G + + + +.md + +.md + + + +make pdf +make pdf + + + +.md->make pdf + + + + + +hugo build + +hugo build + + + +.md->hugo build + + + + + +.ipld + +.ipld + + + +bin/codeGen +bin/codeGen + + + +.ipld->bin/codeGen + + + + + +.go + +.go + + + +go test +go test + + + +.go->go test + + + + + +.go->hugo build + + + + + +.orient + +.orient + + + +orient +orient + + + +.orient->orient + + + + + +.org + +.org + + + +<emacs> + +<emacs> + + + +.org-><emacs> + + + + + +.dot + +.dot + + + +dot +dot + + + +.dot->dot + + + + + +[website] + +[website] + + + +spec.pdf + +spec.pdf + + + +emacs +emacs + + + +.svg + +.svg + + + +dot->.svg + + + + + +orient->.dot + + + + + +orient-><emacs> + + + + + +.html + +.html + + + +orient->.html + + + + + +bin/codeGen->.go + + + + + +hugo +hugo + + + +[ok] + +[ok] + + + +go test->[ok] + + + + + +make website +make website + + + +make website->[website] + + + + + +make pdf->spec.pdf + + + + + +<emacs>->.md + + + + + +hugo build->.html + + + + + +.html->make website + + + + + +.svg->make website + + + + + +.svg->make pdf + + + + + diff --git a/static/_gen/diagrams/intro/diagrams/orient/filecoin.svg b/static/_gen/diagrams/intro/diagrams/orient/filecoin.svg new file mode 100644 index 000000000..5db7eeac4 --- /dev/null +++ b/static/_gen/diagrams/intro/diagrams/orient/filecoin.svg @@ -0,0 +1,906 @@ + + + + + + + + + +69 + +ACTORS-MESSAGES-FRACTION + + + +68 + +ACTORS-MESSAGES-PER-BLOCK + + + +68->69 + + + + + +62 + +MESSAGES + + + +62->69 + + + + + +62->68 + + + + + +66 + +TX-MESSAGES-PER-BLOCK + + + +62->66 + + + + + +64 + +RECEIPTS + + + +62->64 + + + + + +63 + +MESSAGES-SIZE + + + +62->63 + + + + + +67 + +TX-MESSAGES-FRACTION + + + +67->66 + + + + + +66->62 + + + + + +65 + +RECEIPTS-SIZE + + + +64->65 + + + + + +61 + +PROOF-MESSAGES-FRACTION + + + +61->62 + + + + + +60 + +AVG-PROOFS-MESSAGES-PER-BLOCK + + + +60->62 + + + + + +59 + +AVG-POSTS-MESSAGES-PER-BLOCK + + + +59->60 + + + + + +58 + +AVG-SEALS-MESSAGES-PER-BLOCK + + + +58->60 + + + + + +56 + +ALL-POST-MESSAGES-PER-YEAR + + + +56->59 + + + + + +57 + +MINERS + + + +57->56 + + + + + +55 + +ALL-SEAL-SIZE-PER-YEAR + + + +54 + +ALL-SEAL-MESSAGES-PER-YEAR + + + +54->58 + + + + + +54->55 + + + + + +53 + +SECTORS-COUNT + + + +53->56 + + + + + +53->54 + + + + + +52 + +STORAGE-NETWORK-CAPACITY + + + +52->53 + + + + + +48 + +PROVING-PERIOD-SECONDS + + + +51 + +POSTS-PER-SECTOR-PER-YEAR + + + +48->51 + + + + + +51->56 + + + + + +50 + +SEALS-PER-SECTOR-PER-YEAR + + + +50->54 + + + + + +49 + +RESEAL + + + +49->50 + + + + + +47 + +PROVING-PERIOD-HOURS + + + +47->48 + + + + + +46 + +MAX-TICKETS + + + +45 + +ONE-BLOCK-IN-TEN-YEARS + + + +45->46 + + + + + +44 + +BLOCKS-IN-TEN-YEARS + + + +44->45 + + + + + +41 + +BLOCKS-IN-A-YEAR + + + +41->59 + + + + + +41->58 + + + + + +41->44 + + + + + +43 + +EXPECTED-WINNING-MINERS + + + +43->41 + + + + + +42 + +YEAR-IN-SECONDS + + + +42->51 + + + + + +42->41 + + + + + +40 + +TICKETS + + + +39 + +AVG-TICKETS + + + +39->40 + + + + + +36 + +POST-CHALLENGE-TIME + + + +38 + +POST-CHALLENGE-BLOCKS + + + +36->38 + + + + + +37 + +BLOCK-TIME + + + +37->41 + + + + + +37->38 + + + + + +35 + +POST-CHALLENGE-HOURS + + + +35->36 + + + + + +33 + +ENCODING-TIME + + + +34 + +KDF-HASH-SIZE + + + +34->33 + + + + + +32 + +KDF-CONTENT + + + +32->33 + + + + + +31 + +NODES + + + +30 + +NODE-SIZE + + + +30->33 + + + + + +30->31 + + + + + +29 + +SECTOR-SIZE-GIB + + + +28 + +SECTOR-SIZE + + + +29->28 + + + + + +28->53 + + + + + +28->31 + + + + + +27 + +DEGREE-EXPANDER + + + +26 + +DEGREE + + + +27->26 + + + + + +26->32 + + + + + +25 + +DEGREE-BASE + + + +25->26 + + + + + +23 + +MESSAGE-RECEIPT + + + +23->65 + + + + + +24 + +GAS-USED + + + +24->23 + + + + + +22 + +RETURN + + + +22->23 + + + + + +21 + +EXIT-CODE + + + +21->23 + + + + + +20 + +MESSAGE-RECEIPTS-CID + + + +12 + +MESSAGE-SIZE + + + +12->63 + + + + + +19 + +ACTOR-METHOD + + + +19->12 + + + + + +18 + +GAS-LIMIT + + + +18->12 + + + + + +17 + +GAS-PRICE + + + +17->12 + + + + + +16 + +VALUE + + + +16->12 + + + + + +15 + +U64 + + + +15->24 + + + + + +15->19 + + + + + +15->18 + + + + + +15->17 + + + + + +15->16 + + + + + +14 + +MESSAGE-NONCE + + + +14->12 + + + + + +13 + +VARINT + + + +13->22 + + + + + +13->21 + + + + + +13->14 + + + + + +11 + +FROM-ADDRESS + + + +11->12 + + + + + +10 + +TO-ADDRESS + + + +10->12 + + + + + +9 + +MESSAGES-ROOT-CID + + + +8 + +ADDRESS-SIZE + + + +8->11 + + + + + +8->10 + + + + + +7 + +CID-SIZE + + + +7->20 + + + + + +7->9 + + + + + +7->8 + + + + + +6 + +EIX + + + +6->52 + + + + + +5 + +PIB + + + +5->6 + + + + + +4 + +TIB + + + +4->5 + + + + + +3 + +GIB + + + +3->28 + + + + + +3->4 + + + + + +2 + +MIB + + + +2->3 + + + + + +1 + +KIB + + + +1->2 + + + + + diff --git a/static/_gen/diagrams/intro/diagrams/overview1/overview.svg b/static/_gen/diagrams/intro/diagrams/overview1/overview.svg new file mode 100644 index 000000000..f5e2703da --- /dev/null +++ b/static/_gen/diagrams/intro/diagrams/overview1/overview.svg @@ -0,0 +1,730 @@ + + + + + + +Filecoin + + +cluster_blockchain_subgroup + + + +cluster_blockchain + +Blockchain + + +cluster_vm + +Filecoin VM + + +cluster_messagepool + +Message Pool + + +cluster_storagepowerconsensus + +Storage Power Consensus + + +cluster_markets + + + +cluster_storagemarket + +Storage Market + + +cluster_retrievalmarket + +Retrieval Market + + +cluster_filesdata + + + +cluster_pieces + +Data Transfers + + +cluster_files + +Local File Storage + + +cluster_storageminerprocess + + + +cluster_storagemining + +Storage Mining + + +cluster_sectorindexer + +Sector Indexer + + +cluster_filproofs_subsystem + +Storage Proving + + +cluster_systems + + + +cluster_repository + +Repository + + +cluster_filproofs_lib + +Filecoin Proofs + + +cluster_payments + +Payments + + +cluster_net + + + +cluster_clock + +Clock + + +cluster_network + +Network Interface + + +cluster_ipfs + +IPFS + + +cluster_libp2p + +libp2p + + + +Chain + +Chain + + + +Block Producer + +Block Producer + + + + +VM Interpreter + +VM Interpreter + + + +Chain->VM Interpreter + + + + + +IPLD DagStore + +IPLD DagStore + + + +Chain->IPLD DagStore + + + + + +Message Storage + +Message Storage + + + +Block Producer->Message Storage + + + + + +Block Syncer + +Block Syncer + + + +Gossipsub + + + + + +Gossipsub + + + +Block Syncer->Gossipsub + + + + + +Messages + +Messages + + + + +Init Actor + + +Init +Actor + + + +Account Actor + + +Account +Actor + + + + +Cron Actor + + +Cron +Actor + + + +Message Syncer + +Message Syncer + + + + +Filecoin Network Node + +Filecoin Network Node + + + + +Message Syncer->Gossipsub + + + + + +Expected Consensus + +Expected Consensus + + + +Storage Power Actor + + +Storage +Power +Actor + + + + +Storage Provider + +Storage +Provider + + + +Storage Provider->Chain + + + + + +Data Transfers + +Data Transfers + + + +Storage Provider->Data Transfers + + + + + +Storage Client + +Storage +Client + + + +Storage Deal Protocol + + + + + +Storage +Deal +Protocol + + + +Storage Market Actor + + +Storage +Market +Actor + + + + + + +Retrieval Provider + +Retrieval +Provider + + + +Retrieval Provider->Chain + + + + + +Retrieval Provider->Data Transfers + + + + + +Retrieval Client + +Retrieval +Client + + + +Retrieval Deal Protocol + + + + + +Retrieval +Deal +Protocol + + + +PieceStore + +PieceStore + + + +Data Transfers->PieceStore + + + + + +Data Transfers Scheduler + +Data Transfers Scheduler + + + +AuthSwap + + + + + +data +transfer +protocol +(unnamed) + + + +Graphsync + + + + + +Graphsync + + + +AuthSwap->Graphsync + + + + + +FileStore + +FileStore + + + +Disk2 + + +Data Storage System +(Disks, etc) + + + +FileStore->Disk2 + + + + + +PieceStore->FileStore + + + + + +Mining Scheduler + +Mining Scheduler + + + +Sector Builder + +Sector Builder + + + +Mining Scheduler->Sector Builder + + + + + +Pledge Prover + +Pledge Prover + + + +Mining Scheduler->Pledge Prover + + + + + +Storage Miner Actor + + +Storage +Miner +Actor + + + +Sector Store + +Sector Store + + + +Sector Store->PieceStore + + + + + +Seal Prover + +Seal Prover + + + +PoSt Generator + +PoSt Generator + + + +PoSt Generator->Chain + + + + + +Pledge Prover->Storage Power Actor + + + + + +Stacked-DRG-Porep + +Stacked-DRG-Porep + + + +Pledge Prover->Stacked-DRG-Porep + + + + + +Config + +Config + + + +Local Storage + + +Local Storage +(Disk) + + + +Config->Local Storage + + + + + +KeyStore + +KeyStore + + + +KeyStore->Local Storage + + + + + +IPLD DagStore->Local Storage + + + + + +Rational PoSt + +Rational PoSt + + + + +Piece Inclusion Proofs + +Piece +Inclusion +Proofs + + + +VRFs + +VRFs + + + +VDFs + +VDFs + + + + + +Wallet + +Wallet + + + +Wallet->KeyStore + + + + + +Payment Channel Actor + + +Payment +Channel +Actor + + + +Wall Clock + +Wall Clock + + + +NTP + +NTP + + + +Wall Clock->NTP + + + + + +Switch + +Switch + + + +NTP->Switch + + + + + + +p2p.Node + +p2p.Node + + + +Graphsync->p2p.Node + + + + + +Bitswap + + + + + +Bitswap + + + +Bitswap->p2p.Node + + + + + +p2p.Node->Switch + + + + + +Switch->KeyStore + + + + + +Conn + +Conn + + + +Switch->Conn + + + + + +Identify + + + + + +Identify + + + +Identify->p2p.Node + + + + + +Gossipsub->p2p.Node + + + + + +Bootstrap List + + + + + +Bootstrap +List + + + +Bootstrap List->p2p.Node + + + + + diff --git a/static/_gen/diagrams/intro/diagrams/overview2/overview2.svg b/static/_gen/diagrams/intro/diagrams/overview2/overview2.svg new file mode 100644 index 000000000..64cc727a8 --- /dev/null +++ b/static/_gen/diagrams/intro/diagrams/overview2/overview2.svg @@ -0,0 +1 @@ +
libraries
FilecoinNode
FilesAndData
multiformats
libp2p
ipld
ipfs
libp2pCore
libp2pProtocols
Repository
Pos0
Bitswap
Graphsync
UnixFS
DagImports
ipld.CID
ipld.Selector
ipld.GraphStore
Gossipsub
BootstrapList
p2p.Node
p2p.PeerID
p2p.PeerInfo
multihash
multiaddr
Pos1
P2PNode
Clock
FileStore
Config
IpldStore
KeyStore
Pos2
File
Piece
DataTransfer
 \ No newline at end of file diff --git a/static/_gen/diagrams/intro/diagrams/sequence/full-deals-on-chain.svg b/static/_gen/diagrams/intro/diagrams/sequence/full-deals-on-chain.svg new file mode 100644 index 000000000..8a2d634a3 --- /dev/null +++ b/static/_gen/diagrams/intro/diagrams/sequence/full-deals-on-chain.svg @@ -0,0 +1 @@ +RetrievalClientRetrievalProviderStorageMarketParticipant1StorageMarketParticipant2StorageMarketActorPaymentChannelActorPaymentSubsystemBlockchainSubsystemBlockSyncerBlockProducerStoragePowerConsensusSubsystemStoragePowerActorStorageMiningSubsystemStorageMinerActorSectorIndexerSubsystemStorageProvingSubsystemFilecoinProofsSubsystemClockSubsystemlibp2pStorageMarketParticipantSectorSealerWallClockBlockchainMessagePoolMiningSchedulerStorageMinerSubsystemRetrievalMarketSubsystemStorageMarketSubsystemBlockchainGroupMiningGroupopt[RetrievalMarket]opt[RegisterStorageMiner]opt[StorageDealMake]opt[AddingDealToSector]opt[ClientQuery]alt[1 - success][0 - failure]opt[CommitSector]opt[SealingSector]opt[Round Cutoff]loop[BlockReception]epoch is incremented by 1alt[New Tipset][Null block last round]Using tickets derived in failed election proof in last epochalt[New Tipset][Null block last round]alt[1 - success][0 - failure]opt[SubmitPoSt]opt[1- success]loop[forEach StorageMiningSubsystem.StorageMiner]loop[BlockProduction]opt[Expired deals]Schedule and resume PoStsSchedule and resume SEALsMaintain FaultSetMaintain DoneSetopt[MiningScheduler]PoSt challenge if a node has not mined a block in a ProvingPeriodalt[1 - success][0 - failure]opt[SubmitPoSt]opt[ClockSubsystem]SectorStateSets := (FaultSet, RecoverSet, ExpireSet)opt[Declaration before receiving a challenge]FaultSet is all sectors if no post submittedsectors Faulted longer than threshold proving periods are destroyedopt[if miner ProvingPeriod ends]loop[forEach StorageMinerActor in StoragePowerActor.Miners]loop[EveryBlock]opt[Storage Fault]opt[Valid Fault]opt[Consensus Fault]NewRetrievalQuery(RetreivalQuery)RetrievalQueryResponseNewRetrievalDealProposal(RetrievalDealProposal)AcceptRetrievalDealProposal(RetrievalDealPropsal)NewPaymentChannelCreateMiner(ownerPubKey PubKey, workerPubKey PubKey, pledgeAmt TokenAmount)CreateStorageMiner(OwnerAddr, WorkerPubKey)StorageMinerActorRegisterParticipant(TokenAmount)AddBalance(TokenAmount)WithdrawBalance(TokenAmount)CheckLockedBalance(Address)TokenAmountNewStorageDealProposal(StorageDealProposal)VerifyBalance(StorageMarketParticipant1)SignStorageDealProposal(StorageDealProposal)NewStorageDeal(StorageDeal)PublishStorageDeal(StorageDeal)PublishStorageDeal(StorageDeal)HandleStorageDeal(Deal)AddDealToSector(Deal, SectorID)IndexSectorByDealExpiration(SectorID, Deal)(SectorID, Deal)NotifyStorageDealStaged(Deal,SectorID)QueryStorageDealProposalStatus(StorageDealProposalQuery)StorageDealProposalQueryResponseQueryStorageDealStatus(StorageDealQuery)StorageDealQueryResponseGetSealSeed(Chain, Epoch)SeedSealSector(Seed, SectorID, ReplicaCfg)Seal(Seed, SectorID, ReplicaCfg)SealOutputsSealOutputsCommitSector(Seed, SectorID, SealCommitment, SealProof, [&Deal], [Deal])VerifySeal(SectorID, OnSectorInfo){1,0}AddDeal(SectorID, [&Deal], DealStatusOnChain)AddDeal(SectorID, [Deal], DealStatusPending)IncrementPower(StorageMiner.WorkerPubKey)CommitSectorErrorSubscribe(OnNewBlock)Event(OnNewBlock, block)ValidateBlockSyntax(block)HandleBlock(block)ValidateBlock(block)ValidateBlock(block)ValidateBlock(block)StateTree ← TryGenerateStateTree(block)AssembleTipsets()[Tipset] ← AssembleTipsets()Tipset ← ChooseTipset([Tipset])ApplyStateTree(StateTree)OnNewTipset(Chain, Epoch)OnNewRound()GetElectionArtifacts(Chain, Epoch)TK ← TicketAtEpoch(Chain, Epoch-k)T1 ← TicketAtEpoch(Chain, Epoch-1)TK, T1EP ← DrawElectionProof(TK.randomness(), StorageMiner.WorkerKey)T0 ← GenerateNextTicket(T1.randomness(), StorageMiner.WorkerKey)T1 ← GenerateNextTicket(T0.randomness(), StorageMiner.WorkerKey)TryLeaderElection(EP){1, 0}GetPoStChallenge(Chain, Epoch)challengeGeneratePoSt(challenge, [SectorID])PoStProofSubmitPoSt(PoStProof, DoneSet)VerifyPoSt(PoStProof){1,0}UpdateDoneSet()HandleStorageDealPayment()CloseExpiredStorageDeal()PoStErrorSlashStorageCollateral()GenerateBlock(EP, T0, Tipset, StorageMiner.Address)GetMostProfitableMessages(StorageMiner.Address)[Message]block ← AssembleBlock([Message], Tipset, EP, T0, StorageMiner.Address)PropagateBlock(block)OnNewTipset(Chain, Epoch)[SectorID] ← LookupSectorByDealExpiry(Epoch)PurgeSectorsWithNoLiveDeals([SectorID])HandleExpiredDeals([SectorID])CloseExpiredStorageDeal(StorageDeal)GetPoStChallenge(Chain, Epoch)challengeGeneratePoSt(challenge, [SectorID])PoStProofSubmitPoSt(PoStProof, DoneSet)VerifyPoSt(PoStProof){1,0}UpdateDoneSet()HandleStorageDealPayment()CloseExpiredStorageDeal()PoStErrorSlashStorageCollateral()UpdateSectorStatus([FaultSet], SectorStateSets)RecomputeMinerPower()SlashStorageCollateral()ProvingPeriodUpdate()computeProvingPeriodEndSectorState()UpdateSectorStatus(newSectorState)RecomputeMinerPower()HandleFailedDeals([newSectorState.DestroyedSet])DeclareConsensusFault(ConsensusFaultProof)ValidateFault(ConsensusFaultProof)TerminateMiner(Address)SlashPledgeCollateral(Address)UpdateBalance(Reward)RetrievalClientRetrievalProviderStorageMarketParticipant1StorageMarketParticipant2StorageMarketActorPaymentChannelActorPaymentSubsystemBlockchainSubsystemBlockSyncerBlockProducerStoragePowerConsensusSubsystemStoragePowerActorStorageMiningSubsystemStorageMinerActorSectorIndexerSubsystemStorageProvingSubsystemFilecoinProofsSubsystemClockSubsystemlibp2pStorageMarketParticipantSectorSealerWallClockBlockchainMessagePoolMiningSchedulerStorageMinerSubsystem \ No newline at end of file diff --git a/static/_gen/diagrams/intro/new-state-diagram.svg b/static/_gen/diagrams/intro/new-state-diagram.svg new file mode 100644 index 000000000..1c810ae48 --- /dev/null +++ b/static/_gen/diagrams/intro/new-state-diagram.svg @@ -0,0 +1 @@ +
PreCommitSectors
CommitSectors
CronPreCommitExpiry (PCD)
SubmittedWindowPoSt
DeclareFault
SubmitWindowPoSt (SP)
ProvingDeadline (SP)
TerminateSectors
(TF)
SubmittedWindowPoSt (FF)
ProvingDeadline (FF)
DeclareFaultRecovered
EarlyExpiration (TF)
TerminateSectors (TF)
SubmittedWindowPoSt (FF)
DeclareFault
ProvingDeadline (SP)
TerminateSectors (TF)
SubmittedWindowPoSt
DeclareFault
SubmitWindowPoSt (SP)
ProvingDeadline (SP)
CronExpiration
TerminateSectors (TF)
CompactSectors
Null
Precommitted
Committed
Deleted
Active
Faulty
Terminated
Recovering
 \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_blockchain/chainsync/chainsync_fsm.svg b/static/_gen/diagrams/systems/filecoin_blockchain/chainsync/chainsync_fsm.svg new file mode 100644 index 000000000..740918629 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_blockchain/chainsync/chainsync_fsm.svg @@ -0,0 +1,150 @@ + + + + + + +sector + + + +INIT + +INIT + + + +init2boot +init2boot + + + +INIT->init2boot + + + + +follow2init +follow2init + + + +INIT->follow2init + + + + + +BOOTSTRAP + +BOOTSTRAP + + + +boot2check +boot2check + + + +BOOTSTRAP->boot2check + + + + +boot2catch +boot2catch + + + +BOOTSTRAP->boot2catch + + + + +SYNC_CHECKPOINT + +SYNC_CHECKPOINT + + + +check2catch +check2catch + + + +SYNC_CHECKPOINT->check2catch + + + + +CHAIN_CATCHUP + +CHAIN_CATCHUP + + + +catch2follow +catch2follow + + + +CHAIN_CATCHUP->catch2follow + + + + +follow2catch +follow2catch + + + +CHAIN_CATCHUP->follow2catch + + + + + +CHAIN_FOLLOW + +CHAIN_FOLLOW + + + +init2boot->BOOTSTRAP + + + + + +boot2check->SYNC_CHECKPOINT + + + + + +boot2catch->CHAIN_CATCHUP + + + + + +check2catch->CHAIN_CATCHUP + + + + + +catch2follow->CHAIN_FOLLOW + + + + + +follow2catch->CHAIN_FOLLOW + + + + diff --git a/static/_gen/diagrams/systems/filecoin_files/data_transfer/alternate-pull-flow.svg b/static/_gen/diagrams/systems/filecoin_files/data_transfer/alternate-pull-flow.svg new file mode 100644 index 000000000..6214020d4 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_files/data_transfer/alternate-pull-flow.svg @@ -0,0 +1 @@ +RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Responder)Scheduler (Responder)Data Transfer Module (Responder)ResponderOne System. Likely A ClientOne System. Likely A MinerInitiate PullSchedule TransferMake Graphsync Request w/ Data Transfer Request Piggy BackedSend Graphsync Request (w/ Data Transfer Request)Verify Request (validate & schedule)Validate Pull RequestPull Request validatedSchedule TransferSend response w/ DTR Accepted Piggy BackedSend response w/ DTR Accepted Piggy BackedResponse Progress (to end) (include DT Accepted)Request CompleteRequest Completed (if listening)RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Responder)Scheduler (Responder)Data Transfer Module (Responder)Responder \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_files/data_transfer/pull-flow.svg b/static/_gen/diagrams/systems/filecoin_files/data_transfer/pull-flow.svg new file mode 100644 index 000000000..2934c535d --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_files/data_transfer/pull-flow.svg @@ -0,0 +1 @@ +RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Responder)Scheduler (Responder)Data Transfer Module (Responder)ResponderOne System. Likely A ClientOne System. Likely A MinerInitiate PullSend Data Transfer RequestValidate Pull RequestPull Request validatedSchedule TransferSend Data Transfer Request AcceptedSchedule TransferMake Graphsync RequestSend Graphsync RequestVerify Transfer ScheduledRequest is scheduledSend ResponseResponse Progress (to end)Request CompleteRequest Completed (if listening)RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Responder)Scheduler (Responder)Data Transfer Module (Responder)Responder \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_files/data_transfer/push-flow.svg b/static/_gen/diagrams/systems/filecoin_files/data_transfer/push-flow.svg new file mode 100644 index 000000000..5eed429d2 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_files/data_transfer/push-flow.svg @@ -0,0 +1 @@ +RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Responder)Scheduler (Responder)Data Transfer Module (Responder)ResponderOne system. Likely A ClientOne system. Likely A MinerInitiate PushSchedule TransferSend Data Transfer RequestValidate Push RequestPush Request validatedSchedule TransferMake Graphsync RequestSend Graphsync RequestVerify Transfer ScheduledRequest is scheduledSend ResponseResponse Progress (to end)Request CompleteRequest Completed (if listening)RequestorData Transfer Module (Requestor)Scheduler (Requestor)Graphsync (Requestor)Graphsync (Responder)Scheduler (Responder)Data Transfer Module (Responder)Responder \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-flow.svg b/static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-flow.svg new file mode 100644 index 000000000..11f710f2c --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-flow.svg @@ -0,0 +1 @@ +StorageClientStorageProviderStorageMiningSubsystemStorageMarketActorDataTransferStorageMinerActorCronActorAddBalance()AddBalance()OnStorageDealProposal(DealRequest)DealResponseOpenPushTransferRequest(provider, voucher, payload, selector)ValidatePush(client, voucher, payload, selector)acceptOnDataTransferEvent(DataTransferComplete, state)verifyStorageDealProposal(proposal)verifyPieceCID(proposal, payloadCID)signStorageDealProposal(proposal)DealResponse (updated)publishStorageDealMessage(message)PublishStorageDeal(message)generateStorageDealID()lockFund()dealIDDealResponse (updated)HandleStorageDeal(deal)CommitSector()DealResponse (updated)NotifyOfPoStChallenge()SubmitPoSt()activateSectorNotifyOfOnChainDealStatus(DealActive)DealResponse (updated)NotifyOfPoStChallenge()/CheckSurprisePoStSubmissionHappened()NotifyOfOnChainDealStatus(DealExpired/DealFailing)DealResponse (updated)StorageClientStorageProviderStorageMiningSubsystemStorageMarketActorDataTransferStorageMinerActorCronActor \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-payment.svg b/static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-payment.svg new file mode 100644 index 000000000..fd660086d --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_markets/onchain_storage_market/diagrams/deal-payment.svg @@ -0,0 +1 @@ +StorageMiningSubsystemStorageMinerActorStorageMarketActorPublishStorageDeals_validateNewStorageDeal_assertValidClienSignature_assertValidDealTimingAtPublish_assertValidDealMinimum_assertSufficientBalanceAvailableForDeal_lockFundsForStorageDeal_generateStorageDealIDDealIDsPreCommitSectorVerifyPublishedDealIDs(dealIDs)_assertPublishedDealState_assertDealStartAfterCurrEpoch_assertDealExpireAfterMaxProveCommitWindowSuccessProveCommitSectorActivateDeals(dealIDs)_assertPublishedDealState_assertDealNotYetExpired_activateDeal(publishedDeal)Deals_initializeUtilizationInfo_onSuccessfulPoStProcessDealPayment_assertActiveDealState_getStorageFeeSinceLastPayment_transferBalance_submitPowerReport (if any deal expires)ProcessDealExpiration_assertActiveDealState_getStorageFeeSinceLastPayment_transferBalance_unlockCollateral_submitFaultReport(if TerminatedFault)ProcessDealSlash_slashTerminatedFaults_assertActiveDealState_terminateDealWithdrawBalancefundStorageMiningSubsystemStorageMinerActorStorageMarketActor \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v0.svg b/static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v0.svg new file mode 100644 index 000000000..58c322d5b --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v0.svg @@ -0,0 +1 @@ +RetrievalClientNodeRetrievalProviderBlockStore (Client)BlockStore (Provider)Storage Miner Subsystemopt[Discovery]opt[Create Payment Channel]opt[Unsealing]loop[Sending Data until Payment Required]loop[Retrieve In Pieces]Query Chain For Providers w/ PieceIDProviders With PieceIDSend Query for PieceSend information about Piece (price, size, etc)Send RetrievalDealProposalSend RetrievalDealResponse, acceptingCreate Payment ChannelMessage For Channel Gets On ChainUnseal SectorUnsealed Piece DataWrite Piece Blocks to BlockStoreBlocks writtenRead Blocks from BlockStoreBlocksSend BlocksVerify BlocksWrite BlocksBlocks writtenRequest PaymentCreate Voucher On Payment ChannelMessage For Voucher Gets On ChainSend Reference To Payment VoucherRedeem Voucher On Payment ChannelMessage For Voucher Redeemed On ChainRetrievalClientNodeRetrievalProviderBlockStore (Client)BlockStore (Provider)Storage Miner Subsystem \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v1.svg b/static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v1.svg new file mode 100644 index 000000000..14955ac62 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_markets/retrieval_market/retrieval_flow_v1.svg @@ -0,0 +1 @@ +RetrievalClientData Transfer SubsystemRetrievalProviderNodeBlockStore (Client)BlockStore (Provider)Storage Miner Subsystemopt[Discovery]opt[Create Payment Channel]opt[Unsealing]loop[Sending Data until Payment Required]loop[Retrieve In Pieces]Query Chain For Providers w/ PieceIDProviders With PieceIDSend Query for PieceSend information about Piece (price, size, etc)Open Pull Data Transfer w/ RetrievalDealProposal voucherValidate RetrievalDealProposalData Transfer AcceptedUpdate Transfer In ProgressPauseCreate Payment ChannelMessage For Channel Gets On ChainUnpausePauseUnseal SectorUnsealed Piece DataWrite Piece Blocks to BlockStoreBlocks writtenUnpauseRead Blocks from BlockStoreBlocksHandle Send+Verify Of BlocksWrite BlocksBlocks writtenPauseSend Request Payment Intermediate VoucherValidate Request Payment VoucherCreate Voucher On Payment ChannelMessage For Voucher Gets On ChainSend Reference To Payment VoucherValidate Reference To Payment VoucherRedeem Voucher On Payment ChannelMessage For Voucher Redeemed On ChainUnpauseUpdate Transfer CompleteRetrievalClientData Transfer SubsystemRetrievalProviderNodeBlockStore (Client)BlockStore (Provider)Storage Miner Subsystem \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_markets/storage_market/storage_market_flow.svg b/static/_gen/diagrams/systems/filecoin_markets/storage_market/storage_market_flow.svg new file mode 100644 index 000000000..73532418e --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_markets/storage_market/storage_market_flow.svg @@ -0,0 +1 @@ +StorageClientData TransferStorageProviderNodeStorageMarketActorStorage Mining SubsystemDiscoveryDiscovering Minersloop[Each Miner Actor]Querying For Asksloop[each StorageProvider]NegotiationProposing a dealVerifying a dealTransferring DataEnsuring CollateralPublishingPutting Deal On ChainVerify Deal PublishedHandoffStorageMiner Handoff And SealingStorage Miner adds piece to sector precommits, seals sector, activates deal See Storage Mining Subsystem for more detailList Miner Actors from chain stateList of Miner ActorsQuery actor properties from chain stateMiner actor properties from chain stateSend Query For Current AskCurrent Storage AskAdd funds for a deal as neccesaryFunds addedCalculate piece commitmentPropose a dealVerify deal parameters (price, size, etc)Check StorageClient balanceClient balanceVerify balance greater that total storage priceIndicate Intent To Accept DealOpen push request to clientSend push request to providerAccept and request dataSend request for dataSend DataFinished Sending DataNotify data transfer completeVerify data matches piece commitmentAdd collateral for deal as neededCollateral addedSign Deal ProposalPublish Deal On ChainDeal Accepted (w/ Publish Message CID)Deal PublishedVerify Publish Message Appears On ChainDeal PublishedAdd PieceStorageClientData TransferStorageProviderNodeStorageMarketActorStorage Mining Subsystem \ No newline at end of file diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine.svg new file mode 100644 index 000000000..94309cd54 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine.svg @@ -0,0 +1,62 @@ + + + + + + +miner_post + + +cluster_0 + +Lose power on all sectors + + + +OK + + +OK + + + +Challenged + +Challenged + + + +OK->Challenged + + + + + +Challenged->OK + + + + + +DetectedFault + +DetectedFault + + + +Challenged->DetectedFault + + + + + + +DetectedFault->OK + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine_legend.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine_legend.svg new file mode 100644 index 000000000..ac79bd504 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/miner_post_state_machine_legend.svg @@ -0,0 +1,51 @@ + + + + + + + + +cluster_01 + +Legend + + + +key +key + + + +key2 +key2 + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/mining_cycle.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/mining_cycle.svg new file mode 100644 index 000000000..fd4b00e4d --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/mining_cycle.svg @@ -0,0 +1,123 @@ + + + + + + +mining_cycle + + +cluster_local + +Storage Miner +(off chain) + + +cluster_local_mining + +Mining + + +cluster_chain + +Storage Miner Actor +(on chain) + + + +ActiveL + +ActiveL + + + +ChallengedL + +ChallengedL + + + + +ActiveL->ChallengedL + + +Chain issues challenge +(implicit) + + + +ActiveC + +ActiveC + + + +ActiveL->ActiveC + + +Add Sectors +(sm.CommitSector) + + + +ProducePost +Produce & submit PoSt +(sm.SubmitPost) + + + +ChallengedL->ProducePost + + + + +OwnerAccountC + +OwnerAccountC + + + + +OwnerAccountC->ActiveC + + +Create Miner +(spa.CreateMiner) + + + +ActiveC->ActiveL + + +Miner Created + + + +OwnerAccL +OwnerAccL + + + +OwnerAccL->ActiveC + + +Create Miner +(spa.CreateMiner) + + + +ProducePost->ActiveL + + + + + +ProducePost->ActiveC + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_chain_fsm.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_chain_fsm.svg new file mode 100644 index 000000000..15eccb32c --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_chain_fsm.svg @@ -0,0 +1,225 @@ + + + + + + +sector + + +cluster_chain + +Sector States in StorageMinerActor +(on chain) + + +cluster_postproving + +Proving Set + + +cluster_nopowerC1 + +Does not count for power + + +cluster_nopowerC2 + +Does not count for power + + +cluster_activate + + + +cluster_powerC + +Counts for power + + +cluster_nopowerC2 + +Does not count for power + + +cluster_expire_by_post + + + + +CommittedC + +Committed + + + +FirstPoSt + +1st PoSt +(sm.SubmitPost) +ActivateSectors + + + +CommittedC->FirstPoSt + + + + +ActiveC + +Active + + + + + +ExpireByPost + +Expiration Reached +(sma.CronTick +-> sm.ExpireSectors) + + + +CommittedC->ExpireByPost + + + + +FaultsDeclared + +Fault Declared +(sm.DeclareFaults) + + + +CommittedC->FaultsDeclared + + + + +FaultSpotted + +Fault Spotted +(ca.CronTick +-> sm.PostFailed) + + + +CommittedC->FaultSpotted + + + + +RecoveringC + +Recovering + + + +RecoveringC->FirstPoSt + + + + +RecoveringC->ExpireByPost + + + + +RecoveringC->FaultsDeclared + + + + +RecoveringC->FaultSpotted + + + + +FirstPoSt->ActiveC + + + + + +ActiveC->ExpireByPost + + + + +ActiveC->FaultsDeclared + + + + + +ActiveC->FaultSpotted + + + + + +FaultedC + +Faulted + + + +FaultedC->RecoveringC + + +sm.RecoverFaults + + + +ExpiredC + +Cleared + + + +FaultedC->ExpiredC + + +Expiry epoch reached +(sma.CronTick +-> sm.ExpireSectors) + + + +FaultedC->ExpiredC + + +Faulted 3+ times +(sma.CronTick +-> sm.SectorCleanup) + + + + + +ExpireByPost->ExpiredC + + + + + +FaultsDeclared->FaultedC + + + + + + +FaultSpotted->FaultedC + + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_fsm.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_fsm.svg new file mode 100644 index 000000000..6fcb27eb8 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_fsm.svg @@ -0,0 +1,335 @@ + + + + + + +sector + + +cluster_local + +Sector States in Storage Miner +(off chain) + + +cluster_power + + + +cluster_chain + +Sector States in StorageMinerActor +(on chain) + + +cluster_postproving + +must be proved in post + + +cluster_nopowerC1 + +Does not count for power + + +cluster_powerC + +Counts for power + + +cluster_nopowerC2 + +Does not count for power + + + +Empty + +Empty + + + +Packing + +Packing + + + + +Empty->Packing + + +Add Deal + + + +Packing->Packing + + + + + +Unsealed + +Unsealed + + + + +Packing->Unsealed + + +Sector is full + + + +Sealed + +Sealed + + + + +Unsealed->Sealed + + +Seal + + + +NotProving + +Not Proving + + + + +Proving + +Proving + + + +Sealed->Proving + + +Commit +(sm.CommitSector) + + + +CommittedC + +Unproven + + + +Sealed->CommittedC + + + + + +Deleted + +Deleted + + + + +NotProving->Deleted + + +Give up on recovery +or +Expiry + + + +NotProving->Proving + + +Recover +(sm.SubmitPost) + + + +FaultsDeclared + +Fault Declared +(sm.DeclareFaults) + + + +NotProving->FaultsDeclared + + + + +FaultSpotted + +Fault Spotted +(sma.CronTick +-> sm.PostFailed) + + + +NotProving->FaultSpotted + + + + + + +Proving->NotProving + + +Lost data +or +Stopped Proving + + + +Proving->Proving + + + + + +FirstPost +1st PoSt +(sm.SubmitPost +->sm.ActivateSector) + + + +Proving->FirstPost + + + + + + +CommittedC->Proving + + +Commit Sector +(wait for commit) + + + + + +CommittedC->FirstPost + + + + +CommittedC->FaultsDeclared + + + + +CommittedC->FaultSpotted + + + + +ActiveC + +Active + + + +ExpiredC + +Deleted + + + +ActiveC->ExpiredC + + +Expiry epoch reached +(sm.SubmitPost +-> sm.ExpireSectors) + + + + +ActiveC->FaultsDeclared + + + + +ActiveC->FaultSpotted + + + + + +FirstPost->ActiveC + + + + + + + +FaultedC + +Faulted + + + +FaultedC->CommittedC + + +Recover +(sm.SubmitPost) + + + +FaultedC->ExpiredC + + +Expiry epoch reached +(sma.CronTick +-> sm.ExpireSectors) + + + +FaultedC->ExpiredC + + +Faulted 3+ times +(sma.CronTick +-> sm.SectorCleanup) + + + + + + + + +FaultsDeclared->FaultedC + + + + + +FaultSpotted->FaultedC + + + + + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_offchain_fsm.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_offchain_fsm.svg new file mode 100644 index 000000000..9d3a3fd05 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_offchain_fsm.svg @@ -0,0 +1,157 @@ + + + + + + +sector + + +cluster_local + +Sector States in Storage Miner +(off chain) + + +cluster_power + + + + +Empty + +Empty + + + +Packing + +Packing + + + + +Empty->Packing + + +Add Deal + + + +Packing->Packing + + +Add Deal + + + +Unsealed + +Unsealed + + + + +Packing->Unsealed + + +Sector is full + + + +Sealed + +Sealed + + + + +Unsealed->Sealed + + +Seal + + + +NotProving + +Not Proving + + + + +Proving + +Proving + + + +Sealed->Proving + + +Commit Sector +(sm.CommitSector) + + + +NotProving->NotProving + + +Fault Declared +(sm.DeclareFaults) + + + +NotProving->NotProving + + +Fault Spotted +(sma.CronTick +-> sm.PostFailed) + + + +Deleted + +Deleted + + + + +NotProving->Deleted + + +Give up on recovery +or +Expiry + + + +NotProving->Proving + + +Recover +(sm.SubmitPost) + + + +Proving->NotProving + + +Lost data +or +Stopped Proving + + + +Proving->Proving + + +Produce Post +(sm.SubmitPost) + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_fsm.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_fsm.svg new file mode 100644 index 000000000..9607d61f0 --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_fsm.svg @@ -0,0 +1,140 @@ + + + + + + +sectoRecovering + + +cluster_0 + +Proving Set + + +cluster_1 + +Not Counted for Power + + + +Cleared + + +Cleared + + + +Precommitted + +Precommitted + + + +Cleared->Precommitted + + + + + +Precommitted->Cleared + + + + + + + +Committed + +Committed + + + +Precommitted->Committed + + + + + +Active + +Active +(Counted +for Power) + + + +Committed->Active + + + + + +Failing + +Failing + + + +Committed->Failing + + + + + + + +Active->Cleared + + + + + +Active->Failing + + + + + + + +Recovering + +Recovering + + + +Recovering->Active + + + + + +Recovering->Failing + + + + + + + +Failing->Cleared + + + + + + + + + +Failing->Recovering + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_legend.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_legend.svg new file mode 100644 index 000000000..2495d094f --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_legend.svg @@ -0,0 +1,81 @@ + + + + + + + + +cluster_01 + +Legend + + + +key +key + + + +key2 +key2 + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine.svg new file mode 100644 index 000000000..6b13ef76f --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine.svg @@ -0,0 +1,82 @@ + + + + + + +sector_state + + +cluster_0 + +Proving Set (Counted for Power) + + + +Cleared + + +Cleared + + + +PreCommit + +PreCommit + + + +Cleared->PreCommit + + + + + +PreCommit->Cleared + + + + + +Active + +Active + + + +PreCommit->Active + + + + + + +Active->Cleared + + + + + + + +TempFault + +TempFault + + + +Active->TempFault + + + + + +TempFault->Active + + + + + diff --git a/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine_legend.svg b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine_legend.svg new file mode 100644 index 000000000..a4388337a --- /dev/null +++ b/static/_gen/diagrams/systems/filecoin_mining/storage_mining/diagrams/sector_state_machine_legend.svg @@ -0,0 +1,69 @@ + + + + + + + + +cluster_01 + +Legend + + + +key +key + + + +key2 +key2 + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + +key:e->key2:w + + + + + diff --git a/tools/diagrams.js b/tools/diagrams.js index a2d5abead..71e7db06c 100755 --- a/tools/diagrams.js +++ b/tools/diagrams.js @@ -1,19 +1,15 @@ #!/usr/bin/env node - const globby = require('globby') -const execa = require('execa') const path = require('path') const fs = require('fs') +const graphviz = require('graphviz-cli') -const runMmd = (p) => { +const runMmd = async (p) => { + const mmdc = await import('@mermaid-js/mermaid-cli') const outDir = path.dirname(p).replace('content/', 'static/_gen/diagrams/') const outFile = path.basename(p).replace('.mmd', '.svg') - fs.mkdirSync(outDir, { recursive: true }) - const config = process.env.CI ? ['-p', 'tools/pptr.config'] : [] - return execa('mmdc', [...config, '-i', p, '-o', path.join(outDir, outFile)], { - preferLocal: true, - }) + return await mmdc.run(p, path.join(outDir, outFile)) } const runMmdAll = async () => { @@ -21,13 +17,15 @@ const runMmdAll = async () => { await Promise.all(paths.map(runMmd)) } -const runDot = (p) => { +const runDot = async (p) => { const outDir = path.dirname(p).replace('content/', 'static/_gen/diagrams/') const outFile = path.basename(p).replace('.dot', '.svg') fs.mkdirSync(outDir, { recursive: true }) - return execa('graphviz', ['-Tsvg', `-o${path.join(outDir, outFile)}`, p], { - preferLocal: true, - }) + + return await graphviz.renderGraphFromSource( + { name: p }, + { format: 'svg', name: path.join(outDir, outFile) } + ) } const runDotAll = async () => { @@ -43,7 +41,7 @@ const run = async () => { console.timeEnd('Processed *.{mmd,dot}') } -module.exports.configureWatcher = (watcher) => { +exports.configureWatcher = (watcher) => { watcher.on('all', async (_, p) => { const ext = path.extname(p) switch (ext) { diff --git a/tools/pptr.config b/tools/pptr.config deleted file mode 100644 index bc84d2dcc..000000000 --- a/tools/pptr.config +++ /dev/null @@ -1,4 +0,0 @@ -{ - "args": ["--no-sandbox"], - "executablePath": "google-chrome-stable" -}