open files on windows with FILE_SHARE_DELETE (#741)

Address the occasional rename related failures on Windows by using the FILE_SHARE_DELETE flag when opening files on Windows. As noted in #613, this unfortunately requires copying substantial code from the os package, as there's no way to use that flag with the existing os.Open api.
brimdata · May 8, 2020 · 0f599a5 · 0f599a5
1 parent 56221c9
commit 0f599a5
Show file tree

Hide file tree

Showing 23 changed files with 343 additions and 38 deletions.
diff --git a/archive/indexer.go b/archive/indexer.go
@@ -3,8 +3,8 @@ package archive
 import (
 	"errors"
 	"fmt"
-	"os"
 
+	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zdx"
 	"github.com/brimsec/zq/zio/zngio"
@@ -67,7 +67,7 @@ func run(zardir string, rules []Rule, progress chan<- string) error {
 			progress <- fmt.Sprintf("%s: creating index %s", logPath, indexer.Path())
 		}
 	}
-	file, err := os.Open(logPath)
+	file, err := fs.Open(logPath)
 	if err != nil {
 		return err
 	}

diff --git a/cmd/pcap/cut/command.go b/cmd/pcap/cut/command.go
@@ -12,6 +12,7 @@ import (
 
 	"github.com/brimsec/zq/cmd/pcap/root"
 	"github.com/brimsec/zq/pcap/pcapio"
+	"github.com/brimsec/zq/pkg/fs"
 	"github.com/mccanne/charm"
 )
 
@@ -115,7 +116,7 @@ func (c *Command) Run(args []string) error {
 	in := os.Stdin
 	if c.inputFile != "-" {
 		var err error
-		in, err = os.Open(c.inputFile)
+		in, err = fs.Open(c.inputFile)
 		if err != nil {
 			return err
 		}
@@ -124,7 +125,7 @@ func (c *Command) Run(args []string) error {
 
 	out := io.Writer(os.Stdout)
 	if c.outputFile != "-" {
-		f, err := os.OpenFile(c.outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+		f, err := fs.OpenFile(c.outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
 		if err != nil {
 			return err
 		}

diff --git a/cmd/pcap/index/command.go b/cmd/pcap/index/command.go
@@ -6,10 +6,10 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
-	"os"
 
 	"github.com/brimsec/zq/cmd/pcap/root"
 	"github.com/brimsec/zq/pcap"
+	"github.com/brimsec/zq/pkg/fs"
 	"github.com/mccanne/charm"
 )
 
@@ -61,7 +61,7 @@ func (c *Command) Run(args []string) error {
 	if len(args) != 0 || c.inputFile == "" {
 		return errors.New("pcap index: must be provide single pcap file as -r argument")
 	}
-	f, err := os.Open(c.inputFile)
+	f, err := fs.Open(c.inputFile)
 	if err != nil {
 		return err
 	}

diff --git a/cmd/pcap/slice/command.go b/cmd/pcap/slice/command.go
@@ -12,6 +12,7 @@ import (
 	"github.com/brimsec/zq/cmd/pcap/root"
 	"github.com/brimsec/zq/pcap"
 	"github.com/brimsec/zq/pcap/pcapio"
+	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/pkg/nano"
 	"github.com/mccanne/charm"
 )
@@ -117,7 +118,7 @@ func (c *Command) Run(args []string) error {
 	}
 	in := os.Stdin
 	if c.inputFile != "-" {
-		in, err = os.Open(c.inputFile)
+		in, err = fs.Open(c.inputFile)
 		if err != nil {
 			return err
 		}
@@ -141,7 +142,7 @@ func (c *Command) Run(args []string) error {
 	}
 	out := io.Writer(os.Stdout)
 	if c.outputFile != "-" {
-		f, err := os.OpenFile(c.outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+		f, err := fs.OpenFile(c.outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
 		if err != nil {
 			return err
 		}

diff --git a/cmd/pcap/ts/command.go b/cmd/pcap/ts/command.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/brimsec/zq/cmd/pcap/root"
 	"github.com/brimsec/zq/pcap/pcapio"
+	"github.com/brimsec/zq/pkg/fs"
 	"github.com/mccanne/charm"
 )
 
@@ -47,7 +48,7 @@ func (c *Command) Run(args []string) error {
 	in := os.Stdin
 	if c.inputFile != "-" {
 		var err error
-		in, err = os.Open(c.inputFile)
+		in, err = fs.Open(c.inputFile)
 		if err != nil {
 			return err
 		}
@@ -60,7 +61,7 @@ func (c *Command) Run(args []string) error {
 	out := os.Stdout
 	if c.outputFile != "-" {
 		var err error
-		out, err = os.OpenFile(c.outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+		out, err = fs.OpenFile(c.outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
 		if err != nil {
 			return err
 		}

diff --git a/cmd/zar/chop/command.go b/cmd/zar/chop/command.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/brimsec/zq/cmd/zar/root"
 	"github.com/brimsec/zq/pkg/bufwriter"
+	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/pkg/nano"
 	"github.com/brimsec/zq/zbuf"
 	"github.com/brimsec/zq/zio"
@@ -106,7 +107,7 @@ func (c *Command) Run(args []string) error {
 			path := filepath.Join(dir, ts.StringFloat()+".zng")
 			//XXX for now just truncate any existing file.
 			// a future PR will do a split/merge.
-			out, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644)
+			out, err := fs.OpenFile(path, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644)
 			if err != nil {
 				return err
 			}

diff --git a/cmd/zqd/logger/file.go b/cmd/zqd/logger/file.go
@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/brimsec/zq/pkg/fs"
 	"go.uber.org/zap/zapcore"
 	"gopkg.in/natefinch/lumberjack.v2"
 	"gopkg.in/yaml.v3"
@@ -51,9 +52,9 @@ func OpenFile(path string, mode FileMode) (zapcore.WriteSyncer, error) {
 	case FileModeRotate:
 		return logrotate(path, mode)
 	case FileModeTruncate:
-		return os.OpenFile(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
+		return fs.OpenFile(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
 	default: // FileModeAppend
-		return os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
+		return fs.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
 	}
 }
 

diff --git a/emitter/file.go b/emitter/file.go
@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/brimsec/zq/pkg/bufwriter"
+	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/zio"
 	"github.com/brimsec/zq/zio/detector"
 )
@@ -26,7 +27,7 @@ func NewFile(path string, flags *zio.WriterFlags) (*zio.Writer, error) {
 	} else {
 		var err error
 		flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
-		file, err := os.OpenFile(path, flags, 0600)
+		file, err := fs.OpenFile(path, flags, 0600)
 		if err != nil {
 			return nil, err
 		}

diff --git a/emitter/types.go b/emitter/types.go
@@ -7,6 +7,7 @@ import (
 	"strconv"
 
 	"github.com/brimsec/zq/pkg/bufwriter"
+	"github.com/brimsec/zq/pkg/fs"
 	"github.com/brimsec/zq/zng"
 )
 
@@ -24,7 +25,7 @@ func NewTypeLogger(path string, verbose bool) (*TypeLogger, error) {
 	} else {
 		var err error
 		flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
-		file, err := os.OpenFile(path, flags, 0600)
+		file, err := fs.OpenFile(path, flags, 0600)
 		if err != nil {
 			return nil, err
 		}

diff --git a/pcap/pcapio/read.go b/pcap/pcapio/read.go
@@ -69,7 +69,7 @@ const (
 // If the file format is not supported an error is returned.
 //
 //  // Create new reader:
-//  f, _ := os.Open("/tmp/file.pcap")
+//  f, _ := fs.Open("/tmp/file.pcap")
 //  defer f.Close()
 //  r, err := NewReader(f)
 //  data, info, err := r.Read()

diff --git a/pkg/fs/open.go b/pkg/fs/open.go
@@ -0,0 +1,17 @@
+// +build !windows
+
+package fs
+
+import "os"
+
+func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) {
+	return os.OpenFile(name, flag, perm)
+}
+
+func Open(name string) (*os.File, error) {
+	return OpenFile(name, os.O_RDONLY, 0)
+}
+
+func Create(name string) (*os.File, error) {
+	return OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
+}