Merge pull request #61 from filecoin-project/feat/ffi-0.30.1

Fileconi-FFI 0.30.1
filecoin-project · Jun 30, 2020 · 4c1968f · 4c1968f
2 parents 98ef8e4 + a53fff7
commit 4c1968f
Show file tree

Hide file tree

Showing 6 changed files with 22 additions and 25 deletions.
diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi
diff --git a/ffiwrapper/sealer_test.go b/ffiwrapper/sealer_test.go
@@ -458,6 +458,10 @@ func openFDs(t *testing.T) int {
 		if strings.HasPrefix(l, "/dev/nvidia") {
 			skip++
 		}
+
+		if strings.HasPrefix(l, "/var/tmp/filecoin-proof-parameters/") {
+			skip++
+		}
 	}
 
 	return len(dent) - skip

diff --git a/go.mod b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/filecoin-project/go-bitfield v0.0.2-0.20200518150651-562fdb554b6e
 	github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5
 	github.com/filecoin-project/go-paramfetch v0.0.1
-	github.com/filecoin-project/specs-actors v0.6.0
+	github.com/filecoin-project/specs-actors v0.6.1
 	github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.4

diff --git a/go.sum b/go.sum
@@ -43,8 +43,8 @@ github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go
 github.com/filecoin-project/go-paramfetch v0.0.1 h1:gV7bs5YaqlgpGFMiLxInGK2L1FyCXUE0rimz4L7ghoE=
 github.com/filecoin-project/go-paramfetch v0.0.1/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc=
 github.com/filecoin-project/specs-actors v0.3.0/go.mod h1:nQYnFbQ7Y0bHZyq6HDEuVlCPR+U3z5Q3wMOQ+2aiV+Y=
-github.com/filecoin-project/specs-actors v0.6.0 h1:IepUsmDGY60QliENVTkBTAkwqGWw9kNbbHOcU/9oiC0=
-github.com/filecoin-project/specs-actors v0.6.0/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
+github.com/filecoin-project/specs-actors v0.6.1 h1:rhHlEzqcuuQU6oKc4csuq+/kQBDZ4EXtSomoN2XApCA=
+github.com/filecoin-project/specs-actors v0.6.1/go.mod h1:dRdy3cURykh2R8O/DKqy8olScl70rmIS7GrB4hB1IDY=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea h1:iixjULRQFPn7Q9KlIqfwLJnlAXO10bbkI+xy5GKGdLY=
 github.com/filecoin-project/specs-storage v0.1.1-0.20200622113353-88a9704877ea/go.mod h1:Pr5ntAaxsh+sLG/LYiL4tKzvA83Vk5vLODYhfNwOg7k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=

diff --git a/resources.go b/resources.go
@@ -20,8 +20,6 @@ func (r Resources) MultiThread() bool {
 	return r.Threads == -1
 }
 
-const MaxCachingOverhead = 32 << 30
-
 var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{
 	sealtasks.TTAddPiece: {
 		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ // This is probably a bit conservative
@@ -68,27 +66,27 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 	sealtasks.TTPreCommit1: {
 		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
 			MaxMemory: 128 << 30,
-			MinMemory: 96 << 30,
+			MinMemory: 112 << 30,
 
 			Threads: 1,
 
-			BaseMinMemory: 60 << 30,
+			BaseMinMemory: 10 << 20,
 		},
 		abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{
 			MaxMemory: 64 << 30,
-			MinMemory: 48 << 30,
+			MinMemory: 56 << 30,
 
 			Threads: 1,
 
-			BaseMinMemory: 30 << 30,
+			BaseMinMemory: 10 << 20,
 		},
 		abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{
-			MaxMemory: 3 << 29, // 1.5G
-			MinMemory: 1 << 30,
+			MaxMemory: 1 << 30,
+			MinMemory: 768 << 20,
 
 			Threads: 1,
 
-			BaseMinMemory: 1 << 30,
+			BaseMinMemory: 1 << 20,
 		},
 		abi.RegisteredSealProof_StackedDrg2KiBV1: Resources{
 			MaxMemory: 2 << 10,
@@ -195,7 +193,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources
 	},
 	sealtasks.TTCommit2: {
 		abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{
-			MaxMemory: 260 << 30, // TODO: Confirm
+			MaxMemory: 190 << 30, // TODO: Confirm
 			MinMemory: 60 << 30,
 
 			Threads: -1,

diff --git a/sched.go b/sched.go
@@ -250,7 +250,7 @@ func (sh *scheduler) maybeSchedRequest(req *workerRequest) (bool, error) {
 		}
 		tried++
 
-		if !canHandleRequest(needRes, sh.spt, wid, worker.info.Resources, worker.preparing) {
+		if !canHandleRequest(needRes, wid, worker.info.Resources, worker.preparing) {
 			continue
 		}
 
@@ -316,7 +316,7 @@ func (sh *scheduler) assignWorker(wid WorkerID, w *workerHandle, req *workerRequ
 			return
 		}
 
-		err = w.active.withResources(sh.spt, wid, w.info.Resources, needRes, &sh.workersLk, func() error {
+		err = w.active.withResources(wid, w.info.Resources, needRes, &sh.workersLk, func() error {
 			w.preparing.free(w.info.Resources, needRes)
 			sh.workersLk.Unlock()
 			defer sh.workersLk.Lock() // we MUST return locked from this function
@@ -350,8 +350,8 @@ func (sh *scheduler) assignWorker(wid WorkerID, w *workerHandle, req *workerRequ
 	return nil
 }
 
-func (a *activeResources) withResources(spt abi.RegisteredSealProof, id WorkerID, wr storiface.WorkerResources, r Resources, locker sync.Locker, cb func() error) error {
-	for !canHandleRequest(r, spt, id, wr, a) {
+func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerResources, r Resources, locker sync.Locker, cb func() error) error {
+	for !canHandleRequest(r, id, wr, a) {
 		if a.cond == nil {
 			a.cond = sync.NewCond(locker)
 		}
@@ -396,7 +396,7 @@ func (a *activeResources) free(wr storiface.WorkerResources, r Resources) {
 	a.memUsedMax -= r.MaxMemory
 }
 
-func canHandleRequest(needRes Resources, spt abi.RegisteredSealProof, wid WorkerID, res storiface.WorkerResources, active *activeResources) bool {
+func canHandleRequest(needRes Resources, wid WorkerID, res storiface.WorkerResources, active *activeResources) bool {
 
 	// TODO: dedupe needRes.BaseMinMemory per task type (don't add if that task is already running)
 	minNeedMem := res.MemReserved + active.memUsedMin + needRes.MinMemory + needRes.BaseMinMemory
@@ -406,12 +406,7 @@ func canHandleRequest(needRes Resources, spt abi.RegisteredSealProof, wid Worker
 	}
 
 	maxNeedMem := res.MemReserved + active.memUsedMax + needRes.MaxMemory + needRes.BaseMinMemory
-	if spt == abi.RegisteredSealProof_StackedDrg32GiBV1 {
-		maxNeedMem += MaxCachingOverhead
-	}
-	if spt == abi.RegisteredSealProof_StackedDrg64GiBV1 {
-		maxNeedMem += MaxCachingOverhead * 2 // ewwrhmwh
-	}
+
 	if maxNeedMem > res.MemSwap+res.MemPhysical {
 		log.Debugf("sched: not scheduling on worker %d; not enough virtual memory - need: %dM, have %dM", wid, maxNeedMem/mib, (res.MemSwap+res.MemPhysical)/mib)
 		return false
+13 −3		.circleci/config.yml
+56 −0		CHANGELOG.md
+7 −1		bls.go
+13 −0		bls_test.go
+12 −0		build.sh
+1 −2		go.mod
+2 −95		go.sum
+3 −0		run_tests.sh
+1,399 −1,642		rust/Cargo.lock
+10 −6		rust/Cargo.toml
+6 −4		rust/src/bls/api.rs
+3 −4		rust/src/proofs/api.rs
+10 −0		rust/src/util/api.rs
-Original file line number
+Diff line change
@@ Expand Up / @@ -458,6 +458,10 @@ func openFDs(t *testing.T) int { @@
     		if strings.HasPrefix(l, "/dev/nvidia") {
     			skip++
     		}
+    		if strings.HasPrefix(l, "/var/tmp/filecoin-proof-parameters/") {
+    			skip++
+    		}
     	}
     	return len(dent) - skip
@@ Expand Down @@