From 69650bfb9bf657f4a65ec10b1447a65a5d81e24f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 24 Jan 2024 08:59:43 +0700 Subject: [PATCH 1/5] save --- erigon-lib/downloader/downloader.go | 26 +------------------ .../downloader/downloader_grpc_server.go | 2 +- erigon-lib/downloader/torrent_files.go | 25 ++++++++++++++++++ erigon-lib/downloader/webseed.go | 3 +++ 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 8a42ed2c21f..ecb10ae267d 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -21,8 +21,6 @@ import ( "errors" "fmt" "net/url" - "os" - "path/filepath" "runtime" "strings" "sync" @@ -41,7 +39,6 @@ import ( "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dbg" - "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/erigon-lib/diagnostics" "github.com/ledgerwatch/erigon-lib/downloader/downloadercfg" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" @@ -147,27 +144,6 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, dirs datadir.Dirs, logger return d, nil } -const ProhibitNewDownloadsFileName = "prohibit_new_downloads.lock" - -// Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) -// After "download once" - Erigon will produce and seed new files -// Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts) -func (d *Downloader) prohibitNewDownloads() error { - fPath := filepath.Join(d.SnapDir(), ProhibitNewDownloadsFileName) - f, err := os.Create(fPath) - if err != nil { - return err - } - defer f.Close() - if err := f.Sync(); err != nil { - return err - } - return nil -} -func (d *Downloader) newDownloadsAreProhibited() bool { - return dir.FileExist(filepath.Join(d.SnapDir(), ProhibitNewDownloadsFileName)) -} - func (d *Downloader) MainLoopInBackground(silent bool) { d.wg.Add(1) go func() { @@ -605,7 +581,7 @@ func (d *Downloader) AddMagnetLink(ctx context.Context, infoHash metainfo.Hash, if d.alreadyHaveThisName(name) { return nil } - if d.newDownloadsAreProhibited() { + if d.torrentFiles.newDownloadsAreProhibited() { return nil } diff --git a/erigon-lib/downloader/downloader_grpc_server.go b/erigon-lib/downloader/downloader_grpc_server.go index 33410793475..8f448788e85 100644 --- a/erigon-lib/downloader/downloader_grpc_server.go +++ b/erigon-lib/downloader/downloader_grpc_server.go @@ -46,7 +46,7 @@ type GrpcServer struct { } func (s *GrpcServer) ProhibitNewDownloads(context.Context, *proto_downloader.ProhibitNewDownloadsRequest) (*emptypb.Empty, error) { - if err := s.d.prohibitNewDownloads(); err != nil { + if err := s.d.torrentFiles.prohibitNewDownloads(); err != nil { return nil, err } return nil, nil diff --git a/erigon-lib/downloader/torrent_files.go b/erigon-lib/downloader/torrent_files.go index 51d1c8ddd1a..ed58e5b2999 100644 --- a/erigon-lib/downloader/torrent_files.go +++ b/erigon-lib/downloader/torrent_files.go @@ -112,3 +112,28 @@ func (tf *TorrentFiles) load(fPath string) (*torrent.TorrentSpec, error) { mi.AnnounceList = Trackers return torrent.TorrentSpecFromMetaInfoErr(mi) } + +const ProhibitNewDownloadsFileName = "prohibit_new_downloads.lock" + +// Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast) +// After "download once" - Erigon will produce and seed new files +// Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts) +func (tf *TorrentFiles) prohibitNewDownloads() error { + tf.lock.Lock() + defer tf.lock.Unlock() + fPath := filepath.Join(tf.dir, ProhibitNewDownloadsFileName) + f, err := os.Create(fPath) + if err != nil { + return err + } + defer f.Close() + if err := f.Sync(); err != nil { + return err + } + return nil +} +func (tf *TorrentFiles) newDownloadsAreProhibited() bool { + tf.lock.Lock() + defer tf.lock.Unlock() + return dir2.FileExist(filepath.Join(tf.dir, ProhibitNewDownloadsFileName)) +} diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go index f6433103356..6dc4519dd74 100644 --- a/erigon-lib/downloader/webseed.go +++ b/erigon-lib/downloader/webseed.go @@ -237,6 +237,9 @@ func (d *WebSeeds) downloadTorrentFilesFromProviders(ctx context.Context, rootDi if len(d.TorrentUrls()) == 0 { return } + if d.torrentFiles.newDownloadsAreProhibited() { + return + } var addedNew int e, ctx := errgroup.WithContext(ctx) e.SetLimit(1024) From ebef6cb1835d66173ad7809cc3225a81051fa36a Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 24 Jan 2024 09:03:05 +0700 Subject: [PATCH 2/5] save --- erigon-lib/downloader/torrent_files.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/erigon-lib/downloader/torrent_files.go b/erigon-lib/downloader/torrent_files.go index ed58e5b2999..c1ebbd420db 100644 --- a/erigon-lib/downloader/torrent_files.go +++ b/erigon-lib/downloader/torrent_files.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "github.com/anacrolix/torrent" @@ -28,8 +29,10 @@ func (tf *TorrentFiles) Exists(name string) bool { } func (tf *TorrentFiles) exists(name string) bool { - fPath := filepath.Join(tf.dir, name) - return dir2.FileExist(fPath + ".torrent") + if !strings.HasSuffix(name, ".torrent") { + name += ".torrent" + } + return dir2.FileExist(filepath.Join(tf.dir, name)) } func (tf *TorrentFiles) Delete(name string) error { tf.lock.Lock() From 0f067bd930e5ec9842a7e0349ca101c25a9dfe7c Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 24 Jan 2024 09:04:29 +0700 Subject: [PATCH 3/5] save --- erigon-lib/downloader/torrent_files.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/erigon-lib/downloader/torrent_files.go b/erigon-lib/downloader/torrent_files.go index c1ebbd420db..9156630bf31 100644 --- a/erigon-lib/downloader/torrent_files.go +++ b/erigon-lib/downloader/torrent_files.go @@ -41,8 +41,10 @@ func (tf *TorrentFiles) Delete(name string) error { } func (tf *TorrentFiles) delete(name string) error { - fPath := filepath.Join(tf.dir, name) - return os.Remove(fPath + ".torrent") + if !strings.HasSuffix(name, ".torrent") { + name += ".torrent" + } + return os.Remove(filepath.Join(tf.dir, name)) } func (tf *TorrentFiles) Create(torrentFilePath string, res []byte) error { @@ -94,11 +96,10 @@ func (tf *TorrentFiles) createTorrentFromMetaInfo(fPath string, mi *metainfo.Met return nil } -func (tf *TorrentFiles) LoadByName(fName string) (*torrent.TorrentSpec, error) { +func (tf *TorrentFiles) LoadByName(name string) (*torrent.TorrentSpec, error) { tf.lock.Lock() defer tf.lock.Unlock() - fPath := filepath.Join(tf.dir, fName+".torrent") - return tf.load(fPath) + return tf.load(filepath.Join(tf.dir, name)) } func (tf *TorrentFiles) LoadByPath(fPath string) (*torrent.TorrentSpec, error) { @@ -108,6 +109,9 @@ func (tf *TorrentFiles) LoadByPath(fPath string) (*torrent.TorrentSpec, error) { } func (tf *TorrentFiles) load(fPath string) (*torrent.TorrentSpec, error) { + if !strings.HasSuffix(fPath, ".torrent") { + fPath += ".torrent" + } mi, err := metainfo.LoadFromFile(fPath) if err != nil { return nil, fmt.Errorf("LoadFromFile: %w, file=%s", err, fPath) From 020f731639d52be9a181723bd1c5f77315a34cde Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 24 Jan 2024 09:31:48 +0700 Subject: [PATCH 4/5] save --- erigon-lib/downloader/torrent_files.go | 15 +++++++++------ eth/backend.go | 5 +++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/erigon-lib/downloader/torrent_files.go b/erigon-lib/downloader/torrent_files.go index 9156630bf31..d8eb8c815c8 100644 --- a/erigon-lib/downloader/torrent_files.go +++ b/erigon-lib/downloader/torrent_files.go @@ -128,7 +128,15 @@ const ProhibitNewDownloadsFileName = "prohibit_new_downloads.lock" func (tf *TorrentFiles) prohibitNewDownloads() error { tf.lock.Lock() defer tf.lock.Unlock() - fPath := filepath.Join(tf.dir, ProhibitNewDownloadsFileName) + return CreateProhibitNewDownloadsFile(tf.dir) +} +func (tf *TorrentFiles) newDownloadsAreProhibited() bool { + tf.lock.Lock() + defer tf.lock.Unlock() + return dir2.FileExist(filepath.Join(tf.dir, ProhibitNewDownloadsFileName)) +} +func CreateProhibitNewDownloadsFile(dir string) error { + fPath := filepath.Join(dir, ProhibitNewDownloadsFileName) f, err := os.Create(fPath) if err != nil { return err @@ -139,8 +147,3 @@ func (tf *TorrentFiles) prohibitNewDownloads() error { } return nil } -func (tf *TorrentFiles) newDownloadsAreProhibited() bool { - tf.lock.Lock() - defer tf.lock.Unlock() - return dir2.FileExist(filepath.Join(tf.dir, ProhibitNewDownloadsFileName)) -} diff --git a/eth/backend.go b/eth/backend.go index 4f670b2ef65..d6590fa62ea 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -266,6 +266,11 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger config.Sync.UseSnapshots = useSnapshots config.Snapshot.Enabled = ethconfig.UseSnapshotsByChainName(config.Genesis.Config.ChainName) && useSnapshots } + if !config.Sync.UseSnapshots { + if err := downloader.CreateProhibitNewDownloadsFile(dirs.Snap); err != nil { + return err + } + } return nil }); err != nil { From de54717d6bf0b834beb4c691202b2b88f94fec39 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 24 Jan 2024 09:32:51 +0700 Subject: [PATCH 5/5] save --- eth/backend.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index d6590fa62ea..289ad335496 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -266,16 +266,15 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger config.Sync.UseSnapshots = useSnapshots config.Snapshot.Enabled = ethconfig.UseSnapshotsByChainName(config.Genesis.Config.ChainName) && useSnapshots } - if !config.Sync.UseSnapshots { - if err := downloader.CreateProhibitNewDownloadsFile(dirs.Snap); err != nil { - return err - } - } - return nil }); err != nil { return nil, err } + if !config.Sync.UseSnapshots { + if err := downloader.CreateProhibitNewDownloadsFile(dirs.Snap); err != nil { + return nil, err + } + } ctx, ctxCancel := context.WithCancel(context.Background())