diff --git a/archive/indexer.go b/archive/indexer.go index 37e187ca9d..e33cc9b49a 100644 --- a/archive/indexer.go +++ b/archive/indexer.go @@ -3,8 +3,8 @@ package archive import ( "errors" "fmt" - "os" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/zbuf" "github.com/brimsec/zq/zdx" "github.com/brimsec/zq/zio/zngio" @@ -67,7 +67,7 @@ func run(zardir string, rules []Rule, progress chan<- string) error { progress <- fmt.Sprintf("%s: creating index %s", logPath, indexer.Path()) } } - file, err := os.Open(logPath) + file, err := fs.Open(logPath) if err != nil { return err } diff --git a/cmd/pcap/cut/command.go b/cmd/pcap/cut/command.go index f5214d910e..44bb39ab34 100644 --- a/cmd/pcap/cut/command.go +++ b/cmd/pcap/cut/command.go @@ -12,6 +12,7 @@ import ( "github.com/brimsec/zq/cmd/pcap/root" "github.com/brimsec/zq/pcap/pcapio" + "github.com/brimsec/zq/pkg/fs" "github.com/mccanne/charm" ) @@ -115,7 +116,7 @@ func (c *Command) Run(args []string) error { in := os.Stdin if c.inputFile != "-" { var err error - in, err = os.Open(c.inputFile) + in, err = fs.Open(c.inputFile) if err != nil { return err } @@ -124,7 +125,7 @@ func (c *Command) Run(args []string) error { out := io.Writer(os.Stdout) if c.outputFile != "-" { - f, err := os.OpenFile(c.outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + f, err := fs.OpenFile(c.outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err } diff --git a/cmd/pcap/index/command.go b/cmd/pcap/index/command.go index 582adc6e65..292738b18b 100644 --- a/cmd/pcap/index/command.go +++ b/cmd/pcap/index/command.go @@ -6,10 +6,10 @@ import ( "flag" "fmt" "io/ioutil" - "os" "github.com/brimsec/zq/cmd/pcap/root" "github.com/brimsec/zq/pcap" + "github.com/brimsec/zq/pkg/fs" "github.com/mccanne/charm" ) @@ -61,7 +61,7 @@ func (c *Command) Run(args []string) error { if len(args) != 0 || c.inputFile == "" { return errors.New("pcap index: must be provide single pcap file as -r argument") } - f, err := os.Open(c.inputFile) + f, err := fs.Open(c.inputFile) if err != nil { return err } diff --git a/cmd/pcap/slice/command.go b/cmd/pcap/slice/command.go index 13eab7b420..181a6fc46e 100644 --- a/cmd/pcap/slice/command.go +++ b/cmd/pcap/slice/command.go @@ -12,6 +12,7 @@ import ( "github.com/brimsec/zq/cmd/pcap/root" "github.com/brimsec/zq/pcap" "github.com/brimsec/zq/pcap/pcapio" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/pkg/nano" "github.com/mccanne/charm" ) @@ -117,7 +118,7 @@ func (c *Command) Run(args []string) error { } in := os.Stdin if c.inputFile != "-" { - in, err = os.Open(c.inputFile) + in, err = fs.Open(c.inputFile) if err != nil { return err } @@ -141,7 +142,7 @@ func (c *Command) Run(args []string) error { } out := io.Writer(os.Stdout) if c.outputFile != "-" { - f, err := os.OpenFile(c.outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + f, err := fs.OpenFile(c.outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err } diff --git a/cmd/pcap/ts/command.go b/cmd/pcap/ts/command.go index bc5c069fa3..f2b23bef06 100644 --- a/cmd/pcap/ts/command.go +++ b/cmd/pcap/ts/command.go @@ -9,6 +9,7 @@ import ( "github.com/brimsec/zq/cmd/pcap/root" "github.com/brimsec/zq/pcap/pcapio" + "github.com/brimsec/zq/pkg/fs" "github.com/mccanne/charm" ) @@ -47,7 +48,7 @@ func (c *Command) Run(args []string) error { in := os.Stdin if c.inputFile != "-" { var err error - in, err = os.Open(c.inputFile) + in, err = fs.Open(c.inputFile) if err != nil { return err } @@ -60,7 +61,7 @@ func (c *Command) Run(args []string) error { out := os.Stdout if c.outputFile != "-" { var err error - out, err = os.OpenFile(c.outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + out, err = fs.OpenFile(c.outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) if err != nil { return err } diff --git a/cmd/zar/chop/command.go b/cmd/zar/chop/command.go index d483b1f86a..1b2e2823d4 100644 --- a/cmd/zar/chop/command.go +++ b/cmd/zar/chop/command.go @@ -9,6 +9,7 @@ import ( "github.com/brimsec/zq/cmd/zar/root" "github.com/brimsec/zq/pkg/bufwriter" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/pkg/nano" "github.com/brimsec/zq/zbuf" "github.com/brimsec/zq/zio" @@ -106,7 +107,7 @@ func (c *Command) Run(args []string) error { path := filepath.Join(dir, ts.StringFloat()+".zng") //XXX for now just truncate any existing file. // a future PR will do a split/merge. - out, err := os.OpenFile(path, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644) + out, err := fs.OpenFile(path, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644) if err != nil { return err } diff --git a/cmd/zqd/logger/file.go b/cmd/zqd/logger/file.go index 38c6c26d14..57968a955a 100644 --- a/cmd/zqd/logger/file.go +++ b/cmd/zqd/logger/file.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" + "github.com/brimsec/zq/pkg/fs" "go.uber.org/zap/zapcore" "gopkg.in/natefinch/lumberjack.v2" "gopkg.in/yaml.v3" @@ -51,9 +52,9 @@ func OpenFile(path string, mode FileMode) (zapcore.WriteSyncer, error) { case FileModeRotate: return logrotate(path, mode) case FileModeTruncate: - return os.OpenFile(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644) + return fs.OpenFile(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644) default: // FileModeAppend - return os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) + return fs.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) } } diff --git a/emitter/file.go b/emitter/file.go index a19fd12f59..7a6f840307 100644 --- a/emitter/file.go +++ b/emitter/file.go @@ -5,6 +5,7 @@ import ( "os" "github.com/brimsec/zq/pkg/bufwriter" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/zio" "github.com/brimsec/zq/zio/detector" ) @@ -26,7 +27,7 @@ func NewFile(path string, flags *zio.WriterFlags) (*zio.Writer, error) { } else { var err error flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC - file, err := os.OpenFile(path, flags, 0600) + file, err := fs.OpenFile(path, flags, 0600) if err != nil { return nil, err } diff --git a/emitter/types.go b/emitter/types.go index ff0ee3362e..038c28d99c 100644 --- a/emitter/types.go +++ b/emitter/types.go @@ -7,6 +7,7 @@ import ( "strconv" "github.com/brimsec/zq/pkg/bufwriter" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/zng" ) @@ -24,7 +25,7 @@ func NewTypeLogger(path string, verbose bool) (*TypeLogger, error) { } else { var err error flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC - file, err := os.OpenFile(path, flags, 0600) + file, err := fs.OpenFile(path, flags, 0600) if err != nil { return nil, err } diff --git a/pcap/pcapio/read.go b/pcap/pcapio/read.go index f508e9a497..89e33e12d0 100644 --- a/pcap/pcapio/read.go +++ b/pcap/pcapio/read.go @@ -69,7 +69,7 @@ const ( // If the file format is not supported an error is returned. // // // Create new reader: -// f, _ := os.Open("/tmp/file.pcap") +// f, _ := fs.Open("/tmp/file.pcap") // defer f.Close() // r, err := NewReader(f) // data, info, err := r.Read() diff --git a/pkg/fs/open.go b/pkg/fs/open.go new file mode 100644 index 0000000000..eba54b3526 --- /dev/null +++ b/pkg/fs/open.go @@ -0,0 +1,17 @@ +// +build !windows + +package fs + +import "os" + +func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { + return os.OpenFile(name, flag, perm) +} + +func Open(name string) (*os.File, error) { + return OpenFile(name, os.O_RDONLY, 0) +} + +func Create(name string) (*os.File, error) { + return OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) +} diff --git a/pkg/fs/open_windows.go b/pkg/fs/open_windows.go new file mode 100644 index 0000000000..a9cb4b45ae --- /dev/null +++ b/pkg/fs/open_windows.go @@ -0,0 +1,272 @@ +// This file contains code from these Go 1.13.8 source files: +// - os/file.go +// - os/file_windows.go +// - syscall/syscall_windows.go +// - os/path_windows.go +// +// That code is covered by the following copyright notice, +// and the license file mentioned is available at: +// https://github.com/golang/go/blob/go1.13.8/LICENSE +// +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fs + +import ( + "os" + "syscall" + "unsafe" +) + +// OpenFile mimics the os.OpenFile call, but always uses the FILE_SHARE_DELETE flag. +// Differences from os.OpenFile: +// - It's not possible to support the testlog package connection for tests since +// testlog is an internal package. +// - This will not try to open directories if there's a failure to open a file with +// the given name. +// - WriteAt will not return an error if the file is opened in append mode, as the +// required variable to do so is not exported from package os. +func OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { + if name == "" { + return nil, &os.PathError{"open", name, syscall.ENOENT} + } + h, e := syscallOpen(fixLongPath(name), flag|syscall.O_CLOEXEC, syscallMode(perm)) + if e != nil { + return nil, e + } + return os.NewFile(uintptr(h), name), nil +} + +func Open(name string) (*os.File, error) { + return OpenFile(name, os.O_RDONLY, 0) +} + +func Create(name string) (*os.File, error) { + return OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) +} + +// syscallMode returns the syscall-specific mode bits from Go's portable mode bits. +func syscallMode(i os.FileMode) (o uint32) { + o |= uint32(i.Perm()) + if i&os.ModeSetuid != 0 { + o |= syscall.S_ISUID + } + if i&os.ModeSetgid != 0 { + o |= syscall.S_ISGID + } + if i&os.ModeSticky != 0 { + o |= syscall.S_ISVTX + } + // No mapping for Go's ModeTemporary (plan9 only). + return +} + +// basename removes trailing slashes and the leading +// directory name and drive letter from path name. +func basename(name string) string { + // Remove drive letter + if len(name) == 2 && name[1] == ':' { + name = "." + } else if len(name) > 2 && name[1] == ':' { + name = name[2:] + } + i := len(name) - 1 + // Remove trailing slashes + for ; i > 0 && (name[i] == '/' || name[i] == '\\'); i-- { + name = name[:i] + } + // Remove leading directory name + for i--; i >= 0; i-- { + if name[i] == '/' || name[i] == '\\' { + name = name[i+1:] + break + } + } + return name +} + +func isAbs(path string) (b bool) { + v := volumeName(path) + if v == "" { + return false + } + path = path[len(v):] + if path == "" { + return false + } + return os.IsPathSeparator(path[0]) +} + +func volumeName(path string) (v string) { + if len(path) < 2 { + return "" + } + // with drive letter + c := path[0] + if path[1] == ':' && + ('0' <= c && c <= '9' || 'a' <= c && c <= 'z' || + 'A' <= c && c <= 'Z') { + return path[:2] + } + // is it UNC + if l := len(path); l >= 5 && os.IsPathSeparator(path[0]) && os.IsPathSeparator(path[1]) && + !os.IsPathSeparator(path[2]) && path[2] != '.' { + // first, leading `\\` and next shouldn't be `\`. its server name. + for n := 3; n < l-1; n++ { + // second, next '\' shouldn't be repeated. + if os.IsPathSeparator(path[n]) { + n++ + // third, following something characters. its share name. + if !os.IsPathSeparator(path[n]) { + if path[n] == '.' { + break + } + for ; n < l; n++ { + if os.IsPathSeparator(path[n]) { + break + } + } + return path[:n] + } + break + } + } + } + return "" +} + +// fixLongPath returns the extended-length (\\?\-prefixed) form of +// path when needed, in order to avoid the default 260 character file +// path limit imposed by Windows. If path is not easily converted to +// the extended-length form (for example, if path is a relative path +// or contains .. elements), or is short enough, fixLongPath returns +// path unmodified. +// +// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath +func fixLongPath(path string) string { + // Do nothing (and don't allocate) if the path is "short". + // Empirically (at least on the Windows Server 2013 builder), + // the kernel is arbitrarily okay with < 248 bytes. That + // matches what the docs above say: + // "When using an API to create a directory, the specified + // path cannot be so long that you cannot append an 8.3 file + // name (that is, the directory name cannot exceed MAX_PATH + // minus 12)." Since MAX_PATH is 260, 260 - 12 = 248. + // + // The MSDN docs appear to say that a normal path that is 248 bytes long + // will work; empirically the path must be less then 248 bytes long. + if len(path) < 248 { + // Don't fix. (This is how Go 1.7 and earlier worked, + // not automatically generating the \\?\ form) + return path + } + + // The extended form begins with \\?\, as in + // \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt. + // The extended form disables evaluation of . and .. path + // elements and disables the interpretation of / as equivalent + // to \. The conversion here rewrites / to \ and elides + // . elements as well as trailing or duplicate separators. For + // simplicity it avoids the conversion entirely for relative + // paths or paths containing .. elements. For now, + // \\server\share paths are not converted to + // \\?\UNC\server\share paths because the rules for doing so + // are less well-specified. + if len(path) >= 2 && path[:2] == `\\` { + // Don't canonicalize UNC paths. + return path + } + if !isAbs(path) { + // Relative path + return path + } + + const prefix = `\\?` + + pathbuf := make([]byte, len(prefix)+len(path)+len(`\`)) + copy(pathbuf, prefix) + n := len(path) + r, w := 0, len(prefix) + for r < n { + switch { + case os.IsPathSeparator(path[r]): + // empty block + r++ + case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])): + // /./ + r++ + case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])): + // /../ is currently unhandled + return path + default: + pathbuf[w] = '\\' + w++ + for ; r < n && !os.IsPathSeparator(path[r]); r++ { + pathbuf[w] = path[r] + w++ + } + } + } + // A drive's root directory needs a trailing \ + if w == len(`\\?\c:`) { + pathbuf[w] = '\\' + w++ + } + return string(pathbuf[:w]) +} +func makeInheritSa() *syscall.SecurityAttributes { + var sa syscall.SecurityAttributes + sa.Length = uint32(unsafe.Sizeof(sa)) + sa.InheritHandle = 1 + return &sa +} + +// syscallOpen is a copy of syscall.Open, modified to always use the +// FILE_SHARE_DELETE flag. +func syscallOpen(path string, mode int, perm uint32) (fd syscall.Handle, err error) { + if len(path) == 0 { + return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND + } + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return syscall.InvalidHandle, err + } + var access uint32 + switch mode & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) { + case syscall.O_RDONLY: + access = syscall.GENERIC_READ + case syscall.O_WRONLY: + access = syscall.GENERIC_WRITE + case syscall.O_RDWR: + access = syscall.GENERIC_READ | syscall.GENERIC_WRITE + } + if mode&syscall.O_CREAT != 0 { + access |= syscall.GENERIC_WRITE + } + if mode&syscall.O_APPEND != 0 { + access &^= syscall.GENERIC_WRITE + access |= syscall.FILE_APPEND_DATA + } + sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE) + var sa *syscall.SecurityAttributes + if mode&syscall.O_CLOEXEC == 0 { + sa = makeInheritSa() + } + var createmode uint32 + switch { + case mode&(syscall.O_CREAT|syscall.O_EXCL) == (syscall.O_CREAT | syscall.O_EXCL): + createmode = syscall.CREATE_NEW + case mode&(syscall.O_CREAT|syscall.O_TRUNC) == (syscall.O_CREAT | syscall.O_TRUNC): + createmode = syscall.CREATE_ALWAYS + case mode&syscall.O_CREAT == syscall.O_CREAT: + createmode = syscall.OPEN_ALWAYS + case mode&syscall.O_TRUNC == syscall.O_TRUNC: + createmode = syscall.TRUNCATE_EXISTING + default: + createmode = syscall.OPEN_EXISTING + } + h, e := syscall.CreateFile(pathp, access, sharemode, sa, createmode, syscall.FILE_ATTRIBUTE_NORMAL, 0) + return h, e +} diff --git a/proc/sort.go b/proc/sort.go index f9f69e7c9b..2357967a14 100644 --- a/proc/sort.go +++ b/proc/sort.go @@ -13,6 +13,7 @@ import ( "github.com/brimsec/zq/ast" "github.com/brimsec/zq/expr" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/pkg/nano" "github.com/brimsec/zq/zbuf" "github.com/brimsec/zq/zio" @@ -331,7 +332,7 @@ type runFile struct { // newRunFile writes sorted to filename and returns a runFile that reads the // file using zctx. func newRunFile(filename string, sorted []*zng.Record, zctx *resolver.Context) (*runFile, error) { - f, err := os.Create(filename) + f, err := fs.Create(filename) if err != nil { return nil, err } diff --git a/zdx/reader.go b/zdx/reader.go index 9cc4db8f8f..772fc4dfc0 100644 --- a/zdx/reader.go +++ b/zdx/reader.go @@ -3,6 +3,7 @@ package zdx import ( "os" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/zio/zngio" "github.com/brimsec/zq/zng/resolver" ) @@ -27,7 +28,7 @@ func NewReader(path string) (*Reader, error) { } func newReader(path string, level int) (*Reader, error) { - f, err := os.Open(filename(path, level)) + f, err := fs.Open(filename(path, level)) if err != nil { return nil, err } diff --git a/zdx/writer.go b/zdx/writer.go index a2bf7a1167..59df9cf05a 100644 --- a/zdx/writer.go +++ b/zdx/writer.go @@ -5,6 +5,7 @@ import ( "os" "github.com/brimsec/zq/pkg/bufwriter" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/zio" "github.com/brimsec/zq/zio/zngio" "github.com/brimsec/zq/zng" @@ -60,7 +61,7 @@ func newWriter(path string, framesize, level int) (*Writer, error) { panic("something wrong") } name := filename(path, level) - f, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + f, err := fs.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { return nil, err } diff --git a/zio/detector/file.go b/zio/detector/file.go index f1d0fd43ef..58b9e5c999 100644 --- a/zio/detector/file.go +++ b/zio/detector/file.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/zbuf" "github.com/brimsec/zq/zio/ndjsonio" "github.com/brimsec/zq/zio/parquetio" @@ -59,7 +60,7 @@ func OpenFile(zctx *resolver.Context, path string, cfg OpenConfig) (*zbuf.File, if info.IsDir() { return nil, errors.New("is a directory") } - f, err = os.Open(path) + f, err = fs.Open(path) if err != nil { return nil, err } diff --git a/zio/zngio/index_test.go b/zio/zngio/index_test.go index a5bb290f55..2de26d3e3d 100644 --- a/zio/zngio/index_test.go +++ b/zio/zngio/index_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/pkg/nano" "github.com/brimsec/zq/zbuf" "github.com/brimsec/zq/zio" @@ -107,7 +108,7 @@ func TestZngIndex(t *testing.T) { // First time we read the file we don't have an index, but a // search with a time span should still only return results // from the span. - fp, err = os.Open(fname) + fp, err = fs.Open(fname) require.NoError(t, err) ireader, err := index.NewReader(fp, resolver.NewContext(), span) require.NoError(t, err) @@ -118,7 +119,7 @@ func TestZngIndex(t *testing.T) { // Second time through, should get the same results, this time // verify that we didn't read the whole file. - fp, err = os.Open(fname) + fp, err = fs.Open(fname) require.NoError(t, err) ireader, err = index.NewReader(fp, resolver.NewContext(), span) require.NoError(t, err) diff --git a/zio/zngio/types.go b/zio/zngio/types.go index c767788d2f..e1a5e811fa 100644 --- a/zio/zngio/types.go +++ b/zio/zngio/types.go @@ -19,6 +19,7 @@ import ( "path/filepath" "sync" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/zng/resolver" ) @@ -40,7 +41,7 @@ func NewTypeFile(path string) (*TypeFile, error) { zctx := resolver.NewContext() info, err := os.Stat(path) if os.IsNotExist(err) { - f, err := os.Create(path) + f, err := fs.Create(path) if err != nil { return nil, err } @@ -51,7 +52,7 @@ func NewTypeFile(path string) (*TypeFile, error) { if info.IsDir() { return nil, fmt.Errorf("type file cannot be a directory: %s", path) } - file, err := os.Open(path) + file, err := fs.Open(path) if err != nil { return nil, err } diff --git a/zqd/handlers_test.go b/zqd/handlers_test.go index 7f1816c55f..5a02d7255e 100644 --- a/zqd/handlers_test.go +++ b/zqd/handlers_test.go @@ -17,6 +17,7 @@ import ( "sync" "testing" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/pkg/nano" "github.com/brimsec/zq/pkg/test" "github.com/brimsec/zq/zbuf" @@ -799,7 +800,7 @@ func (p *testZeekProcess) Wait() error { func writeLogsFn(logs []string) procFn { return func(t *testZeekProcess) error { for _, log := range logs { - r, err := os.Open(log) + r, err := fs.Open(log) if err != nil { return err } diff --git a/zqd/ingest/log.go b/zqd/ingest/log.go index 5f8c4b1bde..606ea2f50f 100644 --- a/zqd/ingest/log.go +++ b/zqd/ingest/log.go @@ -9,6 +9,7 @@ import ( "time" "github.com/brimsec/zq/driver" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/pkg/nano" "github.com/brimsec/zq/zbuf" "github.com/brimsec/zq/zio" @@ -87,7 +88,7 @@ func openIncomingLog(path string) (*readCounter, int64, error) { if info.IsDir() { return nil, 0, zqe.E(zqe.Invalid, "path is a directory") } - f, err := os.Open(path) + f, err := fs.Open(path) if err != nil { return nil, 0, err } diff --git a/zqd/ingest/pcap.go b/zqd/ingest/pcap.go index 90483eaefe..8a78f2d188 100644 --- a/zqd/ingest/pcap.go +++ b/zqd/ingest/pcap.go @@ -13,6 +13,7 @@ import ( "github.com/brimsec/zq/driver" "github.com/brimsec/zq/pcap" + "github.com/brimsec/zq/pkg/fs" "github.com/brimsec/zq/pkg/nano" "github.com/brimsec/zq/zbuf" "github.com/brimsec/zq/zio" @@ -144,7 +145,7 @@ outer: } func (p *Process) indexPcap() error { - pcapfile, err := os.Open(p.pcapPath) + pcapfile, err := fs.Open(p.pcapPath) if err != nil { return err } @@ -155,7 +156,7 @@ func (p *Process) indexPcap() error { } idxpath := p.space.DataPath(space.PcapIndexFile) tmppath := idxpath + ".tmp" - f, err := os.Create(tmppath) + f, err := fs.Create(tmppath) if err != nil { return err } @@ -173,7 +174,7 @@ func (p *Process) indexPcap() error { } func (p *Process) runZeek(ctx context.Context) error { - pcapfile, err := os.Open(p.pcapPath) + pcapfile, err := fs.Open(p.pcapPath) if err != nil { return err } diff --git a/zqd/space/space.go b/zqd/space/space.go index 623cb65385..6780020bc4 100644 --- a/zqd/space/space.go +++ b/zqd/space/space.go @@ -285,7 +285,7 @@ func (s *Space) PcapSearch(ctx context.Context, req api.PcapSearch) (*SearchRead default: return nil, fmt.Errorf("unsupported proto type: %s", req.Proto) } - f, err := os.Open(s.PcapPath()) + f, err := fs.Open(s.PcapPath()) if err != nil { return nil, err } @@ -345,7 +345,7 @@ func (s *Space) DataPath(elem ...string) string { func (s *Space) OpenZng(span nano.Span) (zbuf.ReadCloser, error) { zctx := resolver.NewContext() - f, err := os.Open(s.DataPath(AllZngFile)) + f, err := fs.Open(s.DataPath(AllZngFile)) if err != nil { if !os.IsNotExist(err) { return nil, err @@ -353,7 +353,7 @@ func (s *Space) OpenZng(span nano.Span) (zbuf.ReadCloser, error) { // Couldn't read all.zng, check for an old space with all.bzng bzngFile := strings.TrimSuffix(AllZngFile, filepath.Ext(AllZngFile)) + ".bzng" - f, err = os.Open(s.DataPath(bzngFile)) + f, err = fs.Open(s.DataPath(bzngFile)) if err != nil { if !os.IsNotExist(err) { return nil, err @@ -367,7 +367,7 @@ func (s *Space) OpenZng(span nano.Span) (zbuf.ReadCloser, error) { } func (s *Space) CreateFile(file string) (*os.File, error) { - return os.Create(s.DataPath(file)) + return fs.Create(s.DataPath(file)) } func (s *Space) HasFile(file string) bool { @@ -477,7 +477,7 @@ func loadConfig(spacePath string) (config, error) { func (c config) save(spacePath string) error { path := filepath.Join(spacePath, configFile) tmppath := path + ".tmp" - f, err := os.Create(tmppath) + f, err := fs.Create(tmppath) if err != nil { return err } @@ -508,7 +508,7 @@ func loadInfoFile(path string) (info, error) { func (i info) save(path string) error { path = filepath.Join(path, infoFile) tmppath := path + ".tmp" - f, err := os.Create(tmppath) + f, err := fs.Create(tmppath) if err != nil { return err } diff --git a/zql/zql_test.go b/zql/zql_test.go index 7249bca0e7..d6e407113f 100644 --- a/zql/zql_test.go +++ b/zql/zql_test.go @@ -3,16 +3,16 @@ package zql import ( "bufio" "fmt" - "os" "testing" "github.com/brimsec/zq/ast" + "github.com/brimsec/zq/pkg/fs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestValid(t *testing.T) { - file, err := os.Open("valid.zql") + file, err := fs.Open("valid.zql") require.NoError(t, err) scanner := bufio.NewScanner(file) for scanner.Scan() { @@ -23,7 +23,7 @@ func TestValid(t *testing.T) { } func TestInvalid(t *testing.T) { - file, err := os.Open("invalid.zql") + file, err := fs.Open("invalid.zql") require.NoError(t, err) scanner := bufio.NewScanner(file) for scanner.Scan() {