From e78aa9a32eeb23ce41acbc974e8713057df986e5 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 30 Apr 2024 19:50:23 +0100 Subject: [PATCH 1/3] set existing torrent webseeds after download --- erigon-lib/downloader/downloader.go | 6 +++++- erigon-lib/downloader/webseed.go | 20 +++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index cd5c8f80d2b..34b873acf0a 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -318,7 +318,7 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, logger log.Logger, verbosi downloading: map[string]struct{}{}, webseedsDiscover: discover, } - d.webseeds.SetTorrent(d.torrentFS, lock.Downloads, cfg.DownloadTorrentFilesFromWebseed) + d.webseeds.SetTorrent(d, lock.Downloads, cfg.DownloadTorrentFilesFromWebseed) requestHandler.downloader = d @@ -1853,6 +1853,10 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { peersOfThisFile := t.PeerConns() weebseedPeersOfThisFile := t.WebseedPeerConns() + if len(weebseedPeersOfThisFile) == 0 { + t.WebseedPeerConns() + } + tLen := t.Length() var bytesCompleted int64 diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go index 905095a2788..66986829683 100644 --- a/erigon-lib/downloader/webseed.go +++ b/erigon-lib/downloader/webseed.go @@ -35,8 +35,8 @@ type WebSeeds struct { torrentUrls snaptype.TorrentUrls // HTTP urls of .torrent files downloadTorrentFile bool torrentsWhitelist snapcfg.Preverified - - seeds []*url.URL + downloader *Downloader + seeds []*url.URL logger log.Logger verbosity log.Lvl @@ -97,10 +97,11 @@ func (d *WebSeeds) getWebDownloadInfo(ctx context.Context, t *torrent.Torrent) ( return infos, seedHashMismatches, nil } -func (d *WebSeeds) SetTorrent(t *AtomicTorrentFS, whiteList snapcfg.Preverified, downloadTorrentFile bool) { +func (d *WebSeeds) SetTorrent(downloader *Downloader, whiteList snapcfg.Preverified, downloadTorrentFile bool) { d.downloadTorrentFile = downloadTorrentFile d.torrentsWhitelist = whiteList - d.torrentFiles = t + d.torrentFiles = downloader.torrentFS + d.downloader = downloader } func (d *WebSeeds) checkHasTorrents(manifestResponse snaptype.WebSeedsFromProvider, report *WebSeedCheckReport) { @@ -406,8 +407,17 @@ func (d *WebSeeds) makeWebSeedUrls(listsOfFiles []snaptype.WebSeedsFromProvider, } d.lock.Lock() - defer d.lock.Unlock() d.byFileName = webSeedUrls + d.lock.Unlock() + + d.downloader.lock.Lock() + defer d.downloader.lock.Unlock() + + for _, t := range d.downloader.torrentClient.Torrents() { + if urls, ok := d.ByFileName(t.Name()); ok { + t.AddWebSeeds(urls) + } + } } func (d *WebSeeds) TorrentUrls() snaptype.TorrentUrls { From f12286fa974fe37230b7551b236e04bb1f4acb7f Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 30 Apr 2024 19:59:06 +0100 Subject: [PATCH 2/3] remove debug statement --- erigon-lib/downloader/downloader.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 34b873acf0a..66683dc3b7b 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -1853,10 +1853,6 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { peersOfThisFile := t.PeerConns() weebseedPeersOfThisFile := t.WebseedPeerConns() - if len(weebseedPeersOfThisFile) == 0 { - t.WebseedPeerConns() - } - tLen := t.Length() var bytesCompleted int64 From 73c78117554e6697fb608fbae0152e0c1985814f Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 1 May 2024 10:14:07 +0100 Subject: [PATCH 3/3] move webseed torrent init to discovery loop --- erigon-lib/downloader/downloader.go | 11 ++++++++++- erigon-lib/downloader/webseed.go | 15 ++------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 66683dc3b7b..10653fbeed4 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -318,7 +318,7 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, logger log.Logger, verbosi downloading: map[string]struct{}{}, webseedsDiscover: discover, } - d.webseeds.SetTorrent(d, lock.Downloads, cfg.DownloadTorrentFilesFromWebseed) + d.webseeds.SetTorrent(d.torrentFS, lock.Downloads, cfg.DownloadTorrentFilesFromWebseed) requestHandler.downloader = d @@ -821,6 +821,15 @@ func (d *Downloader) mainLoop(silent bool) error { if err := d.addTorrentFilesFromDisk(true); err != nil && !errors.Is(err, context.Canceled) { d.logger.Warn("[snapshots] addTorrentFilesFromDisk", "err", err) } + + d.lock.Lock() + defer d.lock.Unlock() + + for _, t := range d.torrentClient.Torrents() { + if urls, ok := d.webseeds.ByFileName(t.Name()); ok { + t.AddWebSeeds(urls) + } + } }() } diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go index 66986829683..579fd09e7e7 100644 --- a/erigon-lib/downloader/webseed.go +++ b/erigon-lib/downloader/webseed.go @@ -35,7 +35,6 @@ type WebSeeds struct { torrentUrls snaptype.TorrentUrls // HTTP urls of .torrent files downloadTorrentFile bool torrentsWhitelist snapcfg.Preverified - downloader *Downloader seeds []*url.URL logger log.Logger @@ -97,11 +96,10 @@ func (d *WebSeeds) getWebDownloadInfo(ctx context.Context, t *torrent.Torrent) ( return infos, seedHashMismatches, nil } -func (d *WebSeeds) SetTorrent(downloader *Downloader, whiteList snapcfg.Preverified, downloadTorrentFile bool) { +func (d *WebSeeds) SetTorrent(torrentFS *AtomicTorrentFS, whiteList snapcfg.Preverified, downloadTorrentFile bool) { d.downloadTorrentFile = downloadTorrentFile d.torrentsWhitelist = whiteList - d.torrentFiles = downloader.torrentFS - d.downloader = downloader + d.torrentFiles = torrentFS } func (d *WebSeeds) checkHasTorrents(manifestResponse snaptype.WebSeedsFromProvider, report *WebSeedCheckReport) { @@ -409,15 +407,6 @@ func (d *WebSeeds) makeWebSeedUrls(listsOfFiles []snaptype.WebSeedsFromProvider, d.lock.Lock() d.byFileName = webSeedUrls d.lock.Unlock() - - d.downloader.lock.Lock() - defer d.downloader.lock.Unlock() - - for _, t := range d.downloader.torrentClient.Torrents() { - if urls, ok := d.ByFileName(t.Name()); ok { - t.AddWebSeeds(urls) - } - } } func (d *WebSeeds) TorrentUrls() snaptype.TorrentUrls {