From f0a108b15f5ca0239b9a848d717a1c26fb112c32 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 9 Apr 2020 14:50:29 -0700 Subject: [PATCH 1/8] open files on windows with shared delete mode --- flatfs.go | 9 +++-- go.mod | 1 + go.sum | 2 ++ util_unix.go | 16 +++++++++ util_windows.go | 93 +++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 util_unix.go create mode 100644 util_windows.go diff --git a/flatfs.go b/flatfs.go index 06d3ef0..af7ce79 100644 --- a/flatfs.go +++ b/flatfs.go @@ -7,7 +7,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "math" "math/rand" "os" @@ -608,7 +607,7 @@ func (fs *Datastore) Get(key datastore.Key) (value []byte, err error) { } _, path := fs.encode(key) - data, err := ioutil.ReadFile(path) + data, err := readFile(path) if err != nil { if os.IsNotExist(err) { return nil, datastore.ErrNotFound @@ -1013,7 +1012,7 @@ func (fs *Datastore) writeDiskUsageFile(du int64, doSync bool) { // readDiskUsageFile is only safe to call in Open() func (fs *Datastore) readDiskUsageFile() int64 { fpath := filepath.Join(fs.path, DiskUsageFile) - duB, err := ioutil.ReadFile(fpath) + duB, err := readFile(fpath) if err != nil { return 0 } @@ -1050,7 +1049,7 @@ func (fs *Datastore) Accuracy() string { } func (fs *Datastore) tempFile() (*os.File, error) { - file, err := ioutil.TempFile(fs.tempPath, "temp-") + file, err := tempFile(fs.tempPath, "temp-") return file, err } @@ -1093,7 +1092,7 @@ func (fs *Datastore) walk(path string, qrb *query.ResultBuilder) error { var result query.Result result.Key = key.String() if !qrb.Query.KeysOnly { - value, err := ioutil.ReadFile(filepath.Join(path, fn)) + value, err := readFile(filepath.Join(path, fn)) if err != nil { result.Error = err } else { diff --git a/go.mod b/go.mod index f7927f4..86e5ab4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/ipfs/go-ds-flatfs require ( + github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect github.com/ipfs/go-datastore v0.4.4 github.com/ipfs/go-log v1.0.3 github.com/jbenet/goprocess v0.1.4 diff --git a/go.sum b/go.sum index 6f02fd7..315861c 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= +github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= diff --git a/util_unix.go b/util_unix.go new file mode 100644 index 0000000..f05ac36 --- /dev/null +++ b/util_unix.go @@ -0,0 +1,16 @@ +// +build !windows + +package flatfs + +import ( + "io/ioutil" + "os" +) + +func tempFile(dir, pattern string) (f *os.File, err error) { + return ioutil.TempFile(dir, pattern) +} + +func readFile(filename string) ([]byte, error) { + return ioutil.ReadFile(filename) +} diff --git a/util_windows.go b/util_windows.go new file mode 100644 index 0000000..988fa58 --- /dev/null +++ b/util_windows.go @@ -0,0 +1,93 @@ +// +build windows + +package flatfs + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + goissue34681 "github.com/alexbrainman/goissue34681" +) + +var tmpRand uint32 +var randmu sync.Mutex + +func reseed() uint32 { + return uint32(time.Now().UnixNano() + int64(os.Getpid())) +} + +func nextRandom() string { + randmu.Lock() + r := tmpRand + if r == 0 { + r = reseed() + } + r = r*1664525 + 1013904223 // constants from Numerical Recipes + tmpRand = r + randmu.Unlock() + return strconv.Itoa(int(1e9 + r%1e9))[1:] +} + +func prefixAndSuffix(pattern string) (prefix, suffix string) { + if pos := strings.LastIndex(pattern, "*"); pos != -1 { + prefix, suffix = pattern[:pos], pattern[pos+1:] + } else { + prefix = pattern + } + return +} + +func tempFile(dir, pattern string) (f *os.File, err error) { + if dir == "" { + dir = os.TempDir() + } + + prefix, suffix := prefixAndSuffix(pattern) + + nconflict := 0 + for i := 0; i < 10000; i++ { + name := filepath.Join(dir, prefix+nextRandom()+suffix) + f, err = goissue34681.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) + if os.IsExist(err) { + if nconflict++; nconflict > 10 { + randmu.Lock() + tmpRand = reseed() + randmu.Unlock() + } + continue + } + break + } + return +} + +func readFile(filename string) ([]byte, error) { + f, err := goissue34681.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + // It's a good but not certain bet that FileInfo will tell us exactly how much to + // read, so let's try it but be prepared for the answer to be wrong. + var n int64 = bytes.MinRead + + if fi, err := f.Stat(); err == nil { + // As initial capacity for readAll, use Size + a little extra in case Size + // is zero, and to avoid another allocation after Read has filled the + // buffer. The readAll call will read into its allocated internal buffer + // cheaply. If the size was wrong, we'll either waste some space off the end + // or reallocate as needed, but in the overwhelmingly common case we'll get + // it just right. + if size := fi.Size() + bytes.MinRead; size > n { + n = size + } + } + + return ioutil.ReadAll(f) +} From 49c9b4dd560f4f3fa3939f34b66999dcafa062b3 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 9 Apr 2020 16:06:25 -0700 Subject: [PATCH 2/8] windows CI testing --- .circleci/config.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6ef40d4..4ec4a27 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,16 @@ version: 2.1 orbs: ci-go: ipfs/ci-go@0.2.1 + win: circleci/windows@2.4.0 + +jobs: + "windows-test": + executor: win/default + steps: + - checkout + - run: + name: "Go test" + command: go test . workflows: version: 2 @@ -9,3 +19,4 @@ workflows: - ci-go/build - ci-go/lint - ci-go/test + - windows-test From da6967dd8b27b995956e2f3b9c40c5a5dc18ed16 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 9 Apr 2020 16:30:57 -0700 Subject: [PATCH 3/8] remove feature added after the windows go 1.12 executor --- flatfs.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flatfs.go b/flatfs.go index af7ce79..8ff89df 100644 --- a/flatfs.go +++ b/flatfs.go @@ -240,12 +240,12 @@ func Open(path string, syncFiles bool) (*Datastore, error) { tempPath := filepath.Join(path, ".temp") err = os.RemoveAll(tempPath) if err != nil && !os.IsNotExist(err) { - return nil, fmt.Errorf("failed to remove temporary directory: %w", err) + return nil, fmt.Errorf("failed to remove temporary directory: %v", err) } err = os.Mkdir(tempPath, 0755) if err != nil { - return nil, fmt.Errorf("failed to create temporary directory: %w", err) + return nil, fmt.Errorf("failed to create temporary directory: %v", err) } shardId, err := ReadShardFunc(path) @@ -379,7 +379,7 @@ var putMaxRetries = 6 // will win. func (fs *Datastore) Put(key datastore.Key, value []byte) error { if !keyIsValid(key) { - return fmt.Errorf("when putting '%q': %w", key, ErrInvalidKey) + return fmt.Errorf("when putting '%q': %v", key, ErrInvalidKey) } fs.shutdownLock.RLock() @@ -1156,7 +1156,7 @@ func (fs *Datastore) Batch() (datastore.Batch, error) { func (bt *flatfsBatch) Put(key datastore.Key, val []byte) error { if !keyIsValid(key) { - return fmt.Errorf("when putting '%q': %w", key, ErrInvalidKey) + return fmt.Errorf("when putting '%q': %v", key, ErrInvalidKey) } bt.puts[key] = val return nil From 862f53401a240c42e379a0d1694ff710dbf79c6f Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 9 Apr 2020 19:29:34 -0700 Subject: [PATCH 4/8] additional opening in shared_delete mode --- flatfs.go | 4 ++-- util_unix.go | 6 +++++- util_windows.go | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/flatfs.go b/flatfs.go index 8ff89df..abd266b 100644 --- a/flatfs.go +++ b/flatfs.go @@ -728,7 +728,7 @@ func (fs *Datastore) Query(q query.Query) (query.Results, error) { } func (fs *Datastore) walkTopLevel(path string, result *query.ResultBuilder) error { - dir, err := os.Open(path) + dir, err := open(path) if err != nil { return err } @@ -1054,7 +1054,7 @@ func (fs *Datastore) tempFile() (*os.File, error) { } func (fs *Datastore) walk(path string, qrb *query.ResultBuilder) error { - dir, err := os.Open(path) + dir, err := open(path) if err != nil { if os.IsNotExist(err) { // not an error if the file disappeared diff --git a/util_unix.go b/util_unix.go index f05ac36..1390705 100644 --- a/util_unix.go +++ b/util_unix.go @@ -7,10 +7,14 @@ import ( "os" ) -func tempFile(dir, pattern string) (f *os.File, err error) { +func tempFile(dir, pattern string) (*os.File, error) { return ioutil.TempFile(dir, pattern) } func readFile(filename string) ([]byte, error) { return ioutil.ReadFile(filename) } + +func open(name string) (*os.File, error) { + return os.Open(name) +} diff --git a/util_windows.go b/util_windows.go index 988fa58..3a00688 100644 --- a/util_windows.go +++ b/util_windows.go @@ -91,3 +91,7 @@ func readFile(filename string) ([]byte, error) { return ioutil.ReadAll(f) } + +func open(name string) (*os.File, error) { + return goissue34681.Open(name) +} From 6787bc7821d6f6f0f32e7a899e0e2f89ed451caf Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 9 Apr 2020 20:07:10 -0700 Subject: [PATCH 5/8] fix: skip non-directories --- flatfs.go | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/flatfs.go b/flatfs.go index abd266b..75adf59 100644 --- a/flatfs.go +++ b/flatfs.go @@ -728,16 +728,20 @@ func (fs *Datastore) Query(q query.Query) (query.Results, error) { } func (fs *Datastore) walkTopLevel(path string, result *query.ResultBuilder) error { - dir, err := open(path) + dir, err := os.Open(path) if err != nil { return err } defer dir.Close() - names, err := dir.Readdirnames(-1) + entries, err := dir.Readdir(-1) if err != nil { return err } - for _, dir := range names { + for _, entry := range entries { + if !entry.IsDir() { + continue + } + dir := entry.Name() if len(dir) == 0 || dir[0] == '.' { continue } @@ -1053,8 +1057,9 @@ func (fs *Datastore) tempFile() (*os.File, error) { return file, err } +// only call this on directories. func (fs *Datastore) walk(path string, qrb *query.ResultBuilder) error { - dir, err := open(path) + dir, err := os.Open(path) if err != nil { if os.IsNotExist(err) { // not an error if the file disappeared @@ -1064,15 +1069,6 @@ func (fs *Datastore) walk(path string, qrb *query.ResultBuilder) error { } defer dir.Close() - // ignore non-directories - fileInfo, err := dir.Stat() - if err != nil { - return err - } - if !fileInfo.IsDir() { - return nil - } - names, err := dir.Readdirnames(-1) if err != nil { return err From ff9cd6d683b06237d10c61ca38389b36c8b46f57 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 9 Apr 2020 21:21:02 -0700 Subject: [PATCH 6/8] skip MoveRestart conversion test on windows --- convert_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/convert_test.go b/convert_test.go index 9e819ad..a8e8a5e 100644 --- a/convert_test.go +++ b/convert_test.go @@ -7,11 +7,12 @@ import ( "math/rand" "os" "path/filepath" + "runtime" "testing" "time" "github.com/ipfs/go-datastore" - "github.com/ipfs/go-ds-flatfs" + flatfs "github.com/ipfs/go-ds-flatfs" ) func TestMove(t *testing.T) { @@ -57,6 +58,9 @@ func TestMove(t *testing.T) { } func TestMoveRestart(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } tempdir, cleanup := tempdir(t) defer cleanup() From e4f1bbc48898f819342d0c843f873d03771d48d8 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 9 Apr 2020 21:24:33 -0700 Subject: [PATCH 7/8] remove extraneous open fn --- util_unix.go | 4 ---- util_windows.go | 4 ---- 2 files changed, 8 deletions(-) diff --git a/util_unix.go b/util_unix.go index 1390705..aea0777 100644 --- a/util_unix.go +++ b/util_unix.go @@ -14,7 +14,3 @@ func tempFile(dir, pattern string) (*os.File, error) { func readFile(filename string) ([]byte, error) { return ioutil.ReadFile(filename) } - -func open(name string) (*os.File, error) { - return os.Open(name) -} diff --git a/util_windows.go b/util_windows.go index 3a00688..988fa58 100644 --- a/util_windows.go +++ b/util_windows.go @@ -91,7 +91,3 @@ func readFile(filename string) ([]byte, error) { return ioutil.ReadAll(f) } - -func open(name string) (*os.File, error) { - return goissue34681.Open(name) -} From e20a965e7a895266683b2abd2a05396cc9f27d02 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 9 Apr 2020 21:34:02 -0700 Subject: [PATCH 8/8] attribution --- util_windows.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/util_windows.go b/util_windows.go index 988fa58..5f55b86 100644 --- a/util_windows.go +++ b/util_windows.go @@ -1,5 +1,13 @@ // +build windows +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Note: This file is a variant of a subset of the golang standard library +// src/io/ioutil/tempfile.go +// with calls to os.Open replaced with the goissue34681.Open variant. + package flatfs import (