From 7ac81075444e49b627db9cf277327ac5b3c800ac Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 9 May 2019 15:13:47 +0800 Subject: [PATCH 1/9] reset --- torrentfs/monitor.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/torrentfs/monitor.go b/torrentfs/monitor.go index 8883ba4712..c876e5826a 100644 --- a/torrentfs/monitor.go +++ b/torrentfs/monitor.go @@ -351,11 +351,11 @@ func (m *Monitor) startWork() error { } func (m *Monitor) validateStorage() error { - if m.fs.LastListenBlockNumber > 2048 { - m.lastNumber = m.fs.LastListenBlockNumber - 2048 - } else { - m.lastNumber = uint64(0) //m.fs.LastListenBlockNumber - } + //if m.fs.LastListenBlockNumber > 2048 { + m.lastNumber = m.fs.LastListenBlockNumber - 2048 + //} else { + // m.lastNumber = uint64(0) //m.fs.LastListenBlockNumber + //} end := uint64(0) if m.lastNumber > 4096 { From 972ba29ac32cce6f91eba4db7013b678f4ba89f8 Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 9 May 2019 15:18:09 +0800 Subject: [PATCH 2/9] tfs validate --- torrentfs/monitor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torrentfs/monitor.go b/torrentfs/monitor.go index c876e5826a..311c5895f7 100644 --- a/torrentfs/monitor.go +++ b/torrentfs/monitor.go @@ -352,7 +352,7 @@ func (m *Monitor) startWork() error { func (m *Monitor) validateStorage() error { //if m.fs.LastListenBlockNumber > 2048 { - m.lastNumber = m.fs.LastListenBlockNumber - 2048 + m.lastNumber = m.fs.LastListenBlockNumber //} else { // m.lastNumber = uint64(0) //m.fs.LastListenBlockNumber //} From c15a67574b4d6cef30f534821b03f9afc9a4ccb8 Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 9 May 2019 15:54:19 +0800 Subject: [PATCH 3/9] tfs logic fix --- torrentfs/monitor.go | 23 ++++++++++++++++++----- torrentfs/storage.go | 3 ++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/torrentfs/monitor.go b/torrentfs/monitor.go index 311c5895f7..db5919f6a4 100644 --- a/torrentfs/monitor.go +++ b/torrentfs/monitor.go @@ -371,21 +371,34 @@ func (m *Monitor) validateStorage() error { return rpcErr } + if rpcBlock == nil { + log.Warn("No block found", "number", i) + m.lastNumber = uint64(i) + //m.fs.LastListenBlockNumber = uint64(i) + continue + } + stBlock := m.fs.GetBlockByNumber(uint64(i)) if stBlock == nil { - log.Warn("Vaidate Torrent FS Storage state invalid", "number", m.lastNumber, "error", "LastListenBlockNumber not persistent") - //return nil - continue + log.Warn("Vaidate Torrent FS Storage state failed, rescan", "number", m.lastNumber, "error", "LastListenBlockNumber not persistent", "dirty", m.fs.LastListenBlockNumber) + m.lastNumber = uint64(0) + //m.fs.LastListenBlockNumber = uint64(0) + return nil } if rpcBlock.Hash.Hex() == stBlock.Hash.Hex() { - log.Warn("Validate TFS failed", "number", m.lastNumber, "rpc", rpcBlock.Hash.Hex(), "store", stBlock.Hash.Hex()) - //return nil + log.Warn("Validate TFS continue", "number", m.lastNumber, "rpc", rpcBlock.Hash.Hex(), "store", stBlock.Hash.Hex()) continue } // block in storage invalid log.Info("Update invalid block in storage", "old hash", stBlock.Hash, "new hash", rpcBlock.Hash) + m.lastNumber = uint64(i) + if parseErr := m.parseBlockTorrentInfo(rpcBlock, true); parseErr != nil { + log.Error("Parse new block", "number", i, "block", rpcBlock, "error", parseErr) + m.lastNumber = uint64(0) + return nil + } m.fs.WriteBlock(rpcBlock) } log.Info("Validate Torrent FS Storage ended", "last IPC listen number", m.lastNumber, "end", end) diff --git a/torrentfs/storage.go b/torrentfs/storage.go index d6a7d3502d..1e50dd2a87 100644 --- a/torrentfs/storage.go +++ b/torrentfs/storage.go @@ -287,7 +287,8 @@ func (fs *FileStorage) WriteBlock(b *Block) error { return e }) - if err == nil && b.Number > fs.LastListenBlockNumber { + //if err == nil && b.Number > fs.LastListenBlockNumber { + if err == nil { fs.bnLock.Lock() fs.LastListenBlockNumber = b.Number fs.writeBlockNumber() From d437dfbb1cde94dd4bc81031e7f1683d202ec182 Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 9 May 2019 15:57:10 +0800 Subject: [PATCH 4/9] replace old with new --- torrentfs/monitor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torrentfs/monitor.go b/torrentfs/monitor.go index db5919f6a4..ad2930bc9c 100644 --- a/torrentfs/monitor.go +++ b/torrentfs/monitor.go @@ -387,19 +387,19 @@ func (m *Monitor) validateStorage() error { } if rpcBlock.Hash.Hex() == stBlock.Hash.Hex() { - log.Warn("Validate TFS continue", "number", m.lastNumber, "rpc", rpcBlock.Hash.Hex(), "store", stBlock.Hash.Hex()) + //log.Warn("Validate TFS continue", "number", m.lastNumber, "rpc", rpcBlock.Hash.Hex(), "store", stBlock.Hash.Hex()) continue } // block in storage invalid log.Info("Update invalid block in storage", "old hash", stBlock.Hash, "new hash", rpcBlock.Hash) - m.lastNumber = uint64(i) if parseErr := m.parseBlockTorrentInfo(rpcBlock, true); parseErr != nil { log.Error("Parse new block", "number", i, "block", rpcBlock, "error", parseErr) m.lastNumber = uint64(0) return nil } m.fs.WriteBlock(rpcBlock) + m.lastNumber = uint64(i) } log.Info("Validate Torrent FS Storage ended", "last IPC listen number", m.lastNumber, "end", end) From dc49f4843705e58e88442ee02726f812cf27b2ad Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 9 May 2019 15:58:46 +0800 Subject: [PATCH 5/9] tfs log --- torrentfs/monitor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torrentfs/monitor.go b/torrentfs/monitor.go index ad2930bc9c..b506ea192b 100644 --- a/torrentfs/monitor.go +++ b/torrentfs/monitor.go @@ -392,7 +392,7 @@ func (m *Monitor) validateStorage() error { } // block in storage invalid - log.Info("Update invalid block in storage", "old hash", stBlock.Hash, "new hash", rpcBlock.Hash) + log.Info("Update invalid block in storage", "old hash", stBlock.Hash, "new hash", rpcBlock.Hash, "latest", m.fs.LastListenBlockNumber) if parseErr := m.parseBlockTorrentInfo(rpcBlock, true); parseErr != nil { log.Error("Parse new block", "number", i, "block", rpcBlock, "error", parseErr) m.lastNumber = uint64(0) @@ -401,7 +401,7 @@ func (m *Monitor) validateStorage() error { m.fs.WriteBlock(rpcBlock) m.lastNumber = uint64(i) } - log.Info("Validate Torrent FS Storage ended", "last IPC listen number", m.lastNumber, "end", end) + log.Info("Validate Torrent FS Storage ended", "last IPC listen number", m.lastNumber, "end", end, "latest", m.fs.LastListenBlockNumber) return nil } From 7e60830549037db79de6b72f23ae1de540a39fbb Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 9 May 2019 16:09:53 +0800 Subject: [PATCH 6/9] safe for dirty tfs --- torrentfs/monitor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/torrentfs/monitor.go b/torrentfs/monitor.go index b506ea192b..ee1b0a2fa5 100644 --- a/torrentfs/monitor.go +++ b/torrentfs/monitor.go @@ -371,9 +371,9 @@ func (m *Monitor) validateStorage() error { return rpcErr } - if rpcBlock == nil { + if rpcBlock == nil || rpcBlock.Hash == common.EmptyHash{ log.Warn("No block found", "number", i) - m.lastNumber = uint64(i) + m.lastNumber = uint64(0) //m.fs.LastListenBlockNumber = uint64(i) continue } From 4bd5e1aeffc1ea27f8a4cab4a81f62159d65481e Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 9 May 2019 16:18:43 +0800 Subject: [PATCH 7/9] empty hash --- common/types.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/types.go b/common/types.go index be6e1f2d96..1ba835a516 100644 --- a/common/types.go +++ b/common/types.go @@ -43,8 +43,10 @@ const ( var ( hashT = reflect.TypeOf(Hash{}) addressT = reflect.TypeOf(Address{}) - EmptyHash = HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - EmptyAddress = HexToAddress("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + //EmptyHash = HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + EmptyHash = HexToHash("0000000000000000000000000000000000000000000000000000000000000000") +// EmptyAddress = HexToAddress("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + EmptyAddress = HexToAddress("0000000000000000000000000000000000000000000000000000000000000000") ) // Hash represents the 32 byte Keccak256 hash of arbitrary data. From d3dfa39239c5eda0d69bdd1984a67550917c4f1b Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 9 May 2019 17:30:51 +0800 Subject: [PATCH 8/9] opt tfs --- torrentfs/monitor.go | 75 ++++++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 23 deletions(-) diff --git a/torrentfs/monitor.go b/torrentfs/monitor.go index ee1b0a2fa5..fc956f28a1 100644 --- a/torrentfs/monitor.go +++ b/torrentfs/monitor.go @@ -358,11 +358,11 @@ func (m *Monitor) validateStorage() error { //} end := uint64(0) - if m.lastNumber > 4096 { - end = m.lastNumber - 4096 + if m.lastNumber > 2048 { + end = m.lastNumber - 2048 } - log.Info("Validate Torrent FS Storage", "last IPC listen number", m.lastNumber, "end", end) + log.Info("Validate Torrent FS Storage", "last IPC listen number", m.lastNumber, "end", end, "lastest", m.fs.LastListenBlockNumber) for i := m.lastNumber; i > end; i-- { rpcBlock, rpcErr := m.rpcBlockByNumber(uint64(i)) @@ -371,18 +371,17 @@ func (m *Monitor) validateStorage() error { return rpcErr } - if rpcBlock == nil || rpcBlock.Hash == common.EmptyHash{ + if rpcBlock == nil || rpcBlock.Hash == common.EmptyHash { log.Warn("No block found", "number", i) m.lastNumber = uint64(0) - //m.fs.LastListenBlockNumber = uint64(i) - continue + //continue + return nil } stBlock := m.fs.GetBlockByNumber(uint64(i)) if stBlock == nil { log.Warn("Vaidate Torrent FS Storage state failed, rescan", "number", m.lastNumber, "error", "LastListenBlockNumber not persistent", "dirty", m.fs.LastListenBlockNumber) m.lastNumber = uint64(0) - //m.fs.LastListenBlockNumber = uint64(0) return nil } @@ -393,13 +392,8 @@ func (m *Monitor) validateStorage() error { // block in storage invalid log.Info("Update invalid block in storage", "old hash", stBlock.Hash, "new hash", rpcBlock.Hash, "latest", m.fs.LastListenBlockNumber) - if parseErr := m.parseBlockTorrentInfo(rpcBlock, true); parseErr != nil { - log.Error("Parse new block", "number", i, "block", rpcBlock, "error", parseErr) - m.lastNumber = uint64(0) - return nil - } - m.fs.WriteBlock(rpcBlock) - m.lastNumber = uint64(i) + m.lastNumber = uint64(0) + return nil } log.Info("Validate Torrent FS Storage ended", "last IPC listen number", m.lastNumber, "end", end, "latest", m.fs.LastListenBlockNumber) @@ -507,17 +501,23 @@ func (m *Monitor) syncLastBlock() { break } + rpcBlock, rpcErr := m.rpcBlockByNumber(i) + if rpcErr != nil { + log.Error("Sync old block", "number", i, "error", rpcErr) + return + } + block := m.fs.GetBlockByNumber(i) if block == nil { - rpcBlock, rpcErr := m.rpcBlockByNumber(i) - if rpcErr != nil { - log.Error("Sync old block", "number", i, "error", rpcErr) - return - } + //rpcBlock, rpcErr := m.rpcBlockByNumber(i) + //if rpcErr != nil { + // log.Error("Sync old block", "number", i, "error", rpcErr) + // return + //} block = rpcBlock - if parseErr := m.parseBlockTorrentInfo(block, true); parseErr != nil { + /*if parseErr := m.parseBlockTorrentInfo(block, true); parseErr != nil { log.Error("Parse new block", "number", i, "block", block, "error", parseErr) return } @@ -525,11 +525,27 @@ func (m *Monitor) syncLastBlock() { if storeErr := m.fs.WriteBlock(block); storeErr != nil { log.Error("Store latest block", "number", i, "error", storeErr) return + }*/ + + if err := m.parseAndStore(block, true); err != nil { + log.Error("Fail to parse and storge latest block", "number", i, "error", err) + return } - } else if parseErr := m.parseBlockTorrentInfo(block, false); parseErr != nil { - log.Error("Parse old block", "number", i, "block", block, "error", parseErr) - return + } else { + if block.Hash.Hex() == rpcBlock.Hash.Hex() { + + if parseErr := m.parseBlockTorrentInfo(block, false); parseErr != nil { //dirty to do + log.Error("Parse old block", "number", i, "block", block, "error", parseErr) + return + } + } else { + //dirty tfs + if err := m.parseAndStore(rpcBlock, true); err != nil { + log.Error("Dirty tfs fail to parse and storge latest block", "number", i, "error", err) + return + } + } } //if (i-minNumber)%fetchBlockLogStep == 0 || i == maxNumber { @@ -540,3 +556,16 @@ func (m *Monitor) syncLastBlock() { m.lastNumber = maxNumber log.Info("Torrent scan finished", "from", minNumber, "to", maxNumber, "current", uint64(currentNumber), "progress", float64(maxNumber)/float64(currentNumber), "last", m.lastNumber) } + +func (m *Monitor) parseAndStore(block *Block, flow bool) error { + if parseErr := m.parseBlockTorrentInfo(block, flow); parseErr != nil { + log.Error("Parse new block", "number", block.Number, "block", block, "error", parseErr) + return parseErr + } + + if storeErr := m.fs.WriteBlock(block); storeErr != nil { + log.Error("Store latest block", "number", block.Number, "error", storeErr) + return storeErr + } + return nil +} From e63a139001c0a096addeb2811e2e71b2f9aa1fe3 Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 9 May 2019 18:01:13 +0800 Subject: [PATCH 9/9] loging --- torrentfs/monitor.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/torrentfs/monitor.go b/torrentfs/monitor.go index fc956f28a1..5b740cbd4f 100644 --- a/torrentfs/monitor.go +++ b/torrentfs/monitor.go @@ -351,11 +351,7 @@ func (m *Monitor) startWork() error { } func (m *Monitor) validateStorage() error { - //if m.fs.LastListenBlockNumber > 2048 { m.lastNumber = m.fs.LastListenBlockNumber - //} else { - // m.lastNumber = uint64(0) //m.fs.LastListenBlockNumber - //} end := uint64(0) if m.lastNumber > 2048 { @@ -374,7 +370,6 @@ func (m *Monitor) validateStorage() error { if rpcBlock == nil || rpcBlock.Hash == common.EmptyHash { log.Warn("No block found", "number", i) m.lastNumber = uint64(0) - //continue return nil }