Merge remote-tracking branch 'origin/feat/implement-index-validation-…

…api' into feat/tests-for-the-chainindexer

.github/actions/make-deps/action.yml

@@ -0,0 +1,40 @@
+name: Make Deps
+description: Runs `make deps` with caching
+
+inputs:
+  working-directory:
+    description: Specifies the working directory where the command is run.
+    required: false
+  github-token:
+    description: Specifies the token to use when calling GitHub APIs while building FFI.
+    required: false
+
+runs:
+  using: composite
+  steps:
+    - id: cache-ffi
+      uses: actions/cache/restore@v4
+      with:
+        key: ${{ runner.os }}-${{ runner.arch }}-ffi-${{ hashFiles('./.git/modules/extern/filecoin-ffi/HEAD') }}
+        path: |
+          ./extern/filecoin-ffi/filcrypto.h
+          ./extern/filecoin-ffi/libfilcrypto.a
+          ./extern/filecoin-ffi/filcrypto.pc
+          ./build/.filecoin-install
+          ./build/.update-modules
+    - if: steps.cache-ffi.outputs.cache-hit != 'true'
+      shell: bash
+      working-directory: ${{ inputs.working-directory || github.workspace }}
+      env:
+        GITHUB_TOKEN: ${{ inputs.github-token || github.token }}
+      run: FFI_PORTABLE=1 make deps
+    - if: steps.cache-ffi.outputs.cache-hit != 'true'
+      uses: actions/cache/save@v4
+      with:
+        key: ${{ runner.os }}-${{ runner.arch }}-ffi-${{ hashFiles('./.git/modules/extern/filecoin-ffi/HEAD') }}
+        path: |
+          ./extern/filecoin-ffi/filcrypto.h
+          ./extern/filecoin-ffi/libfilcrypto.a
+          ./extern/filecoin-ffi/filcrypto.pc
+          ./build/.filecoin-install
+          ./build/.update-modules

.github/workflows/builtin-actor-tests.yml

@@ -19,5 +19,5 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: 1.21
+          go-version: 1.22
       - run: go test -tags=release ./build

.github/workflows/check.yml

@@ -20,19 +20,6 @@ permissions:
   contents: read
 
 jobs:
-  check-docsgen:
-    name: Check (docs-check)
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: 'recursive'
-      - uses: ./.github/actions/install-system-dependencies
-      - uses: ./.github/actions/install-go
-      - run: go install golang.org/x/tools/cmd/goimports
-      - run: make deps
-      - run: make docsgen
-      - run: git diff --exit-code
   check-gen:
     name: Check (gen-check)
     runs-on: ubuntu-latest
@@ -42,8 +29,7 @@ jobs:
           submodules: 'recursive'
       - uses: ./.github/actions/install-system-dependencies
       - uses: ./.github/actions/install-go
-      - run: make deps lotus
-      - run: go install golang.org/x/tools/cmd/goimports
+      - uses: ./.github/actions/make-deps
       - run: make gen
       - run: git diff --exit-code
       - run: make docsgen-cli
@@ -57,8 +43,8 @@ jobs:
           submodules: 'recursive'
       - uses: ./.github/actions/install-system-dependencies
       - uses: ./.github/actions/install-go
+      - uses: ./.github/actions/make-deps
       - run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.0
-      - run: make deps
       - run: golangci-lint run -v --timeout 10m --concurrency 4
   check-fmt:
     name: Check (gofmt)

.github/workflows/test.yml

@@ -186,7 +186,7 @@ jobs:
           echo -e "path<<EOF\n$CACHE_PATH\nEOF" | tee -a $GITHUB_OUTPUT
       - id: make_deps
         env:
-          CACHE_KEY: ${{ runner.os }}-${{ runner.arch }}-make-deps-${{ hashFiles('./.git/modules/extern/filecoin-ffi/HEAD') }}
+          CACHE_KEY: ${{ runner.os }}-${{ runner.arch }}-make-deps-${{ hashFiles('./.git/modules/extern/filecoin-ffi/HEAD') }}-p
           CACHE_PATH: |
             ./extern/filecoin-ffi/filcrypto.h
             ./extern/filecoin-ffi/libfilcrypto.a
@@ -206,14 +206,14 @@ jobs:
           key: ${{ steps.make_deps.outputs.key }}
           path: ${{ steps.make_deps.outputs.path }}
           lookup-only: true
-      - if: steps.restore_fetch_params.outputs.cache-hit != 'true'
+      - if: steps.restore_fetch_params.outputs.cache-hit != 'true' || steps.restore_make_deps.outputs.cache-hit != 'true'
         uses: ./.github/actions/install-system-dependencies
-      - if: steps.restore_fetch_params.outputs.cache-hit != 'true'
+      - if: steps.restore_fetch_params.outputs.cache-hit != 'true' || steps.restore_make_deps.outputs.cache-hit != 'true'
         uses: ./.github/actions/install-go
       - if: steps.restore_fetch_params.outputs.cache-hit != 'true' || steps.restore_make_deps.outputs.cache-hit != 'true'
         env:
           GITHUB_TOKEN: ${{ github.token }}
-        run: make deps
+        run: FFI_PORTABLE=1 make deps
       - if: steps.restore_fetch_params.outputs.cache-hit != 'true'
         run: make lotus
       - if: steps.restore_fetch_params.outputs.cache-hit != 'true'

CHANGELOG.md

@@ -2,17 +2,52 @@
 
 # UNRELEASED
  (`ChainIndexer`) to index Filecoin chain state such as tipsets, messages, events and ETH transactions for accurate and faster RPC responses. The `ChainIndexer` replaces the existing `MsgIndex`, `EthTxIndex` and `EventIndex` implementations in Lotus, which [suffer from a multitude of known problems](https://github.com/filecoin-project/lotus/issues/12293).  If you are an RPC provider/node operator, please refer to the [ChainIndexer documentation for RPC providers](TODO: URL) for information on how to enable, configure and use the new Indexer.
+Add `EthGetBlockReceipts` RPC method to retrieve transaction receipts for a specified block. This method allows users to obtain Ethereum format receipts of all transactions included in a given tipset as specified by its Ethereum block equivalent. ([filecoin-project/lotus#12478](https://github.com/filecoin-project/lotus/pull/12478))
 
 
 ## ☢️ Upgrade Warnings ☢️
 
+- Minimum go-version been updated to v1.22.7 ([filecoin-project/lotus#12459](https://github.com/filecoin-project/lotus/pull/12459))
+
 ## New features
 
 * Add `EthSendRawTransactionUntrusted` RPC method to be used for the gateway when accepting `EthSendRawTransaction` and `eth_sendRawTransaction`. Applies a tighter limit on the number of messages in the queue from a single sender and applies additional restrictions on nonce increments. ([filecoin-project/lotus#12431](https://github.com/filecoin-project/lotus/pull/12431))
+* [Checkpoint TipSets finalized by F3](https://github.com/filecoin-project/lotus/pull/12460): Once a decision is made by F3, the TipSet is check-pointed in `ChainStore`. As part of this change, any missing TipSets are asynchronously synced as required by the `ChainStore` checkpointing mechanism.
+* Add an environment variable, `F3_INITIAL_POWERTABLE_CID` to allow specifying the initial power table used by F3 ([filecoin-project/lotus#12502](https://github.com/filecoin-project/lotus/pull/12502)). This may be used to help a lotus node re-sync the F3 chain when syncing from a snapshot after the F3 upgrade epoch. The precise CID to use here won't be known until the F3 is officially "live".
+* Added `StateMinerInitialPledgeForSector` RPC method and deprecated existing `StateMinerInitialPledgeCollateral` method. Since ProveCommitSectors3 and ProveReplicaUpdates3, sector onboarding no longer includes an explicit notion of "deals", and precommit messages no longer contain deal information. This makes the existing `StateMinerInitialPledgeCollateral` unable to properly calculate pledge requirements with only the precommit. `StateMinerInitialPledgeForSector` is a new simplified calculator that simply takes duration, sector size, and verified size and estimates pledge based on current network conditions. Please note that the `StateMinerInitialPledgeCollateral` method will be removed entirely in the next non-patch release. ([filecoin-project/lotus#12384](https://github.com/filecoin-project/lotus/pull/12384)
 
 ## Improvements
 
 - Reduce size of embedded genesis CAR files by removing WASM actor blocks and compressing with zstd. This reduces the `lotus` binary size by approximately 10 MiB. ([filecoin-project/lotus#12439](https://github.com/filecoin-project/lotus/pull/12439))
+- Add ChainSafe operated Calibration archival node to the bootstrap list ([filecoin-project/lotus#12517](https://github.com/filecoin-project/lotus/pull/12517))
+- Legacy/historical Drand lookups via `StateGetBeaconEntry` now work again for all historical epochs. `StateGetBeaconEntry` now uses the on-chain beacon entries and follows the same rules for historical Drand round matching as `StateGetRandomnessFromBeacon` and the `get_beacon_randomness` FVM syscall. Be aware that there will be some some variance in matching Filecoin epochs to Drand rounds where null Filecoin rounds are involved prior to network version 14. ([filecoin-project/lotus#12428](https://github.com/filecoin-project/lotus/pull/12428)).
+
+## Bug Fixes
+
+## Deps
+- chore: bump go-libp2p to v0.35.5 ([filecoin-project/lotus#12511](https://github.com/filecoin-project/lotus/pull/12511))
+
+# Node v1.29.1 / 2024-09-16
+
+This is a Lotus Node patch release that addresses a critical sync issue affecting users of the v1.29.0 release. The primary fix in this release is:
+
+- Downgrade of a dependency that was causing invalid BLS signatures, leading to sync failures for many Lotus nodes. See [#12467](https://github.com/filecoin-project/lotus/issues/12467) for more information about the bug.
+
+We strongly recommend that all users currently running Lotus v1.29.0 upgrade to this version to resolve the syncing problems.
+
+## Bug Fixes
+- Update BLS dependency to fix [filecoin-project/lotus#12467](https://github.com/filecoin-project/lotus/pull/12467).
+
+# 1.28.3 / 2024-09-16
+
+This is a Lotus Node patch release that addresses a critical sync issue affecting users of the v1.28.2 release. The primary fix in this patch release is downgrading of a dependency that was causing invalid BLS signatures, leading to sync failures for many Lotus nodes. See #12467 for more information about the issue/bug.
+
+## Bug Fixes
+- Update BLS dependency to fix [filecoin-project/lotus#12467](https://github.com/filecoin-project/lotus/pull/12467).
+
+# UNRELEASED v1.29.2
+
+See https://github.com/filecoin-project/lotus/blob/release/v1.29.2/CHANGELOG.md
 
 # Node v1.29.0 / 2024-09-02
 
@@ -84,6 +119,7 @@ From https://github.com/filecoin-project/lotus/compare/v1.28.2...release/v1.29.0
 - github.com/filecoin-project/jackc/pgx (v5.4.1 -> v5.6.0)
 - feat(f3): update from v0.0.7 to v0.1.0 (#12382) ([filecoin-project/lotus#12382](https://github.com/filecoin-project/lotus/pull/12382))
 - feat: f3: update go-f3 to 0.2.0 (#12390) ([filecoin-project/lotus#12390](https://github.com/filecoin-project/lotus/pull/12390))
+- feat: update cheggaaa's pb to v3 ([filecoin-project/lotus#12518](https://github.com/filecoin-project/lotus/pull/12518))
 
 ### Chores
 

Dockerfile

@@ -1,5 +1,5 @@
 #####################################
-FROM golang:1.21.7-bullseye AS lotus-builder
+FROM golang:1.22.7-bullseye AS lotus-builder
 MAINTAINER Lotus Development Team
 
 RUN apt-get update && apt-get install -y ca-certificates build-essential clang ocl-icd-opencl-dev ocl-icd-libopencl1 jq libhwloc-dev

GO_VERSION_MIN

@@ -1 +1 @@
-1.21.7
+1.22.7

LOTUS_RELEASE_FLOW.md

@@ -22,6 +22,7 @@
   - [Why do we call it "Lotus Node"?](#why-do-we-call-it-lotus-node)
   - [Why isn't Lotus Miner released more frequently?](#why-isnt-lotus-miner-released-more-frequently)
   - [Why is the `releases` branch deprecated and what are alternatives?](#why-is-the-releases-branch-deprecated-and-what-are-alternatives)
+  - [Why does Lotus still use a `master` branch instead of `main`?](#why-does-lotus-still-use-a-master-branch-instead-of-main)
 - [Related Items](#related-items)
 
 ## Purpose
@@ -153,6 +154,9 @@ That said, one can still programmatically get the latest release based on the [B
 * Lotus Node: `git tag -l 'v*' | grep -v "-" | sort -V -r | head -n 1` 
 * Lotus Miner: `git tag -l 'miner/v*' | grep -v "-" | sort -V -r | head -n 1` 
 
+### Why does Lotus still use a `master` branch instead of `main`?
+There was a [push in 202109](https://github.com/filecoin-project/lotus/issues/7356) on changing the default branch to `main` from `master` for good reason. 3 years later though, the migration was never completed and `master` has ossified 😔.  The effort's failure was acknowledged and commented on in the issue above in 202409.  
+
 ## Related Items
 
 1. [Release Issue template](https://github.com/filecoin-project/lotus/blob/master/documentation/misc/RELEASE_ISSUE_TEMPLATE.md)

Makefile

@@ -28,6 +28,7 @@ endif
 
 GOFLAGS+=-ldflags="$(ldflags)"
 
+FIX_IMPORTS = $(GOCC) run ./scripts/fiximports
 
 ## FFI
 
@@ -295,7 +296,7 @@ dist-clean:
 type-gen: api-gen
 	$(GOCC) run ./gen/main.go
 	$(GOCC) generate -x ./...
-	goimports -w api/
+	$(FIX_IMPORTS)
 
 actors-code-gen:
 	$(GOCC) run ./gen/inline-gen . gen/inlinegen-data.json
@@ -311,11 +312,9 @@ bundle-gen:
 	$(GOCC) fmt ./build/...
 .PHONY: bundle-gen
 
-
 api-gen:
 	$(GOCC) run ./gen/api
-	goimports -w api
-	goimports -w api
+	$(FIX_IMPORTS)
 .PHONY: api-gen
 
 cfgdoc-gen:
@@ -330,38 +329,13 @@ appimage: lotus
 	cp ./lotus AppDir/usr/bin/
 	appimage-builder
 
-docsgen: docsgen-md docsgen-openrpc fiximports
-
-docsgen-md-bin: api-gen actors-gen
-	$(GOCC) build $(GOFLAGS) -o docgen-md ./api/docgen/cmd
-docsgen-openrpc-bin: api-gen actors-gen
-	$(GOCC) build $(GOFLAGS) -o docgen-openrpc ./api/docgen-openrpc/cmd
-
-docsgen-md: docsgen-md-full docsgen-md-storage docsgen-md-worker
-
-docsgen-md-full: docsgen-md-bin
-	./docgen-md "api/api_full.go" "FullNode" "api" "./api" > documentation/en/api-v1-unstable-methods.md
-	./docgen-md "api/v0api/full.go" "FullNode" "v0api" "./api/v0api" > documentation/en/api-v0-methods.md
-docsgen-md-storage: docsgen-md-bin
-	./docgen-md "api/api_storage.go" "StorageMiner" "api" "./api" > documentation/en/api-v0-methods-miner.md
-docsgen-md-worker: docsgen-md-bin
-	./docgen-md "api/api_worker.go" "Worker" "api" "./api" > documentation/en/api-v0-methods-worker.md
-
-docsgen-openrpc: docsgen-openrpc-full docsgen-openrpc-storage docsgen-openrpc-worker docsgen-openrpc-gateway
-
-docsgen-openrpc-full: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_full.go" "FullNode" "api" "./api" > build/openrpc/full.json
-docsgen-openrpc-storage: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_storage.go" "StorageMiner" "api" "./api" > build/openrpc/miner.json
-docsgen-openrpc-worker: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_worker.go" "Worker" "api" "./api" > build/openrpc/worker.json
-docsgen-openrpc-gateway: docsgen-openrpc-bin
-	./docgen-openrpc "api/api_gateway.go" "Gateway" "api" "./api" > build/openrpc/gateway.json
-
-.PHONY: docsgen docsgen-md-bin docsgen-openrpc-bin
+docsgen: fiximports
+	$(GOCC) run ./gen/docs
+.PHONY: docsgen
 
 fiximports:
-	$(GOCC) run ./scripts/fiximports
+	$(FIX_IMPORTS)
+.PHONY: fiximports
 
 gen: actors-code-gen type-gen cfgdoc-gen docsgen api-gen
 	$(GOCC) run ./scripts/fiximports
@@ -376,7 +350,7 @@ snap: lotus lotus-miner lotus-worker
 
 # separate from gen because it needs binaries
 docsgen-cli: lotus lotus-miner lotus-worker
-	python3 ./scripts/generate-lotus-cli.py
+	$(GOCC) run ./scripts/docsgen-cli
 	./lotus config default > documentation/en/default-lotus-config.toml
 	./lotus-miner config default > documentation/en/default-lotus-miner-config.toml
 .PHONY: docsgen-cli

README.md

@@ -19,7 +19,8 @@ Lotus is an implementation of the Filecoin Distributed Storage Network. For more
 
 ## Building & Documentation
 
-> Note: The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the [latest release](https://github.com/filecoin-project/lotus/blob/master/LOTUS_RELEASE_FLOW.md#why-is-the-releases-branch-deprecated-and-what-are-alternatives).
+> [!NOTE]
+> The default `master` branch is the dev branch, please use with caution. For the latest stable version, checkout the [latest release](https://github.com/filecoin-project/lotus/blob/master/LOTUS_RELEASE_FLOW.md#why-is-the-releases-branch-deprecated-and-what-are-alternatives).
 
 For complete instructions on how to build, install and setup lotus, please visit [https://lotus.filecoin.io](https://lotus.filecoin.io/lotus/install/prerequisites/#supported-platforms). Basic build instructions can be found further down in this readme.
 
@@ -57,10 +58,10 @@ For other distributions you can find the required dependencies [here.](https://l
 
 #### Go
 
-To build Lotus, you need a working installation of [Go 1.21.7 or higher](https://golang.org/dl/):
+To build Lotus, you need a working installation of [Go 1.22.7 or higher](https://golang.org/dl/):
 
 ```bash
-wget -c https://golang.org/dl/go1.21.7.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
+wget -c https://golang.org/dl/go1.22.7.linux-amd64.tar.gz -O - | sudo tar -xz -C /usr/local
 ```
 
 **TIP:**