downloader: prohibit_new_downloads.lock check missed download .torren…

…t from WebSeed (#9295) moved `.lock` file handling inside `TorrentFiles` class - which is available everywhere and guarded by mutex. fixing: #9294
erigontech · Jan 24, 2024 · 56db532 · 56db532
1 parent cc01ac3
commit 56db532
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 34 deletions.
diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go
@@ -21,8 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
-	"os"
-	"path/filepath"
 	"runtime"
 	"strings"
 	"sync"
@@ -41,7 +39,6 @@ import (
 	"github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/common/datadir"
 	"github.com/ledgerwatch/erigon-lib/common/dbg"
-	"github.com/ledgerwatch/erigon-lib/common/dir"
 	"github.com/ledgerwatch/erigon-lib/diagnostics"
 	"github.com/ledgerwatch/erigon-lib/downloader/downloadercfg"
 	"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
@@ -147,27 +144,6 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, dirs datadir.Dirs, logger
 	return d, nil
 }
 
-const ProhibitNewDownloadsFileName = "prohibit_new_downloads.lock"
-
-// Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast)
-// After "download once" - Erigon will produce and seed new files
-// Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts)
-func (d *Downloader) prohibitNewDownloads() error {
-	fPath := filepath.Join(d.SnapDir(), ProhibitNewDownloadsFileName)
-	f, err := os.Create(fPath)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	if err := f.Sync(); err != nil {
-		return err
-	}
-	return nil
-}
-func (d *Downloader) newDownloadsAreProhibited() bool {
-	return dir.FileExist(filepath.Join(d.SnapDir(), ProhibitNewDownloadsFileName))
-}
-
 func (d *Downloader) MainLoopInBackground(silent bool) {
 	d.wg.Add(1)
 	go func() {
@@ -605,7 +581,7 @@ func (d *Downloader) AddMagnetLink(ctx context.Context, infoHash metainfo.Hash,
 	if d.alreadyHaveThisName(name) {
 		return nil
 	}
-	if d.newDownloadsAreProhibited() {
+	if d.torrentFiles.newDownloadsAreProhibited() {
 		return nil
 	}
 

diff --git a/erigon-lib/downloader/downloader_grpc_server.go b/erigon-lib/downloader/downloader_grpc_server.go
@@ -46,7 +46,7 @@ type GrpcServer struct {
 }
 
 func (s *GrpcServer) ProhibitNewDownloads(context.Context, *proto_downloader.ProhibitNewDownloadsRequest) (*emptypb.Empty, error) {
-	if err := s.d.prohibitNewDownloads(); err != nil {
+	if err := s.d.torrentFiles.prohibitNewDownloads(); err != nil {
 		return nil, err
 	}
 	return nil, nil

diff --git a/erigon-lib/downloader/torrent_files.go b/erigon-lib/downloader/torrent_files.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/anacrolix/torrent"
@@ -28,8 +29,10 @@ func (tf *TorrentFiles) Exists(name string) bool {
 }
 
 func (tf *TorrentFiles) exists(name string) bool {
-	fPath := filepath.Join(tf.dir, name)
-	return dir2.FileExist(fPath + ".torrent")
+	if !strings.HasSuffix(name, ".torrent") {
+		name += ".torrent"
+	}
+	return dir2.FileExist(filepath.Join(tf.dir, name))
 }
 func (tf *TorrentFiles) Delete(name string) error {
 	tf.lock.Lock()
@@ -38,8 +41,10 @@ func (tf *TorrentFiles) Delete(name string) error {
 }
 
 func (tf *TorrentFiles) delete(name string) error {
-	fPath := filepath.Join(tf.dir, name)
-	return os.Remove(fPath + ".torrent")
+	if !strings.HasSuffix(name, ".torrent") {
+		name += ".torrent"
+	}
+	return os.Remove(filepath.Join(tf.dir, name))
 }
 
 func (tf *TorrentFiles) Create(torrentFilePath string, res []byte) error {
@@ -91,11 +96,10 @@ func (tf *TorrentFiles) createTorrentFromMetaInfo(fPath string, mi *metainfo.Met
 	return nil
 }
 
-func (tf *TorrentFiles) LoadByName(fName string) (*torrent.TorrentSpec, error) {
+func (tf *TorrentFiles) LoadByName(name string) (*torrent.TorrentSpec, error) {
 	tf.lock.Lock()
 	defer tf.lock.Unlock()
-	fPath := filepath.Join(tf.dir, fName+".torrent")
-	return tf.load(fPath)
+	return tf.load(filepath.Join(tf.dir, name))
 }
 
 func (tf *TorrentFiles) LoadByPath(fPath string) (*torrent.TorrentSpec, error) {
@@ -105,10 +109,41 @@ func (tf *TorrentFiles) LoadByPath(fPath string) (*torrent.TorrentSpec, error) {
 }
 
 func (tf *TorrentFiles) load(fPath string) (*torrent.TorrentSpec, error) {
+	if !strings.HasSuffix(fPath, ".torrent") {
+		fPath += ".torrent"
+	}
 	mi, err := metainfo.LoadFromFile(fPath)
 	if err != nil {
 		return nil, fmt.Errorf("LoadFromFile: %w, file=%s", err, fPath)
 	}
 	mi.AnnounceList = Trackers
 	return torrent.TorrentSpecFromMetaInfoErr(mi)
 }
+
+const ProhibitNewDownloadsFileName = "prohibit_new_downloads.lock"
+
+// Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast)
+// After "download once" - Erigon will produce and seed new files
+// Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts)
+func (tf *TorrentFiles) prohibitNewDownloads() error {
+	tf.lock.Lock()
+	defer tf.lock.Unlock()
+	return CreateProhibitNewDownloadsFile(tf.dir)
+}
+func (tf *TorrentFiles) newDownloadsAreProhibited() bool {
+	tf.lock.Lock()
+	defer tf.lock.Unlock()
+	return dir2.FileExist(filepath.Join(tf.dir, ProhibitNewDownloadsFileName))
+}
+func CreateProhibitNewDownloadsFile(dir string) error {
+	fPath := filepath.Join(dir, ProhibitNewDownloadsFileName)
+	f, err := os.Create(fPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if err := f.Sync(); err != nil {
+		return err
+	}
+	return nil
+}
diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go
@@ -237,6 +237,9 @@ func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDi
 	if len(d.TorrentUrls()) == 0 {
 		return
 	}
+	if d.torrentFiles.newDownloadsAreProhibited() {
+		return
+	}
 	var addedNew int
 	e, ctx := errgroup.WithContext(ctx)
 	e.SetLimit(1024)

diff --git a/eth/backend.go b/eth/backend.go
@@ -266,11 +266,15 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger
 			config.Sync.UseSnapshots = useSnapshots
 			config.Snapshot.Enabled = ethconfig.UseSnapshotsByChainName(config.Genesis.Config.ChainName) && useSnapshots
 		}
-
 		return nil
 	}); err != nil {
 		return nil, err
 	}
+	if !config.Sync.UseSnapshots {
+		if err := downloader.CreateProhibitNewDownloadsFile(dirs.Snap); err != nil {
+			return nil, err
+		}
+	}
 
 	ctx, ctxCancel := context.WithCancel(context.Background())