diff --git a/.golangci.yml b/.golangci.yml index 27061841158c..f9a821f6b31e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -222,8 +222,10 @@ issues: exclude-dirs: - test/testdata_etc # test files - internal/cache # extracted from Go code - - internal/renameio # extracted from Go code - internal/robustio # extracted from Go code + - internal/mmap # extracted from Go code + - internal/quoted # extracted from Go code + - internal/testenv # extracted from Go code run: timeout: 5m diff --git a/go.mod b/go.mod index 39ebc65db4c7..b1b717f30dd9 100644 --- a/go.mod +++ b/go.mod @@ -87,6 +87,7 @@ require ( github.com/polyfloyd/go-errorlint v1.6.0 github.com/quasilyte/go-ruleguard/dsl v0.3.22 github.com/raeperd/recvcheck v0.1.2 + github.com/rogpeppe/go-internal v1.13.1 github.com/ryancurrah/gomodguard v1.3.5 github.com/ryanrolds/sqlclosecheck v0.5.1 github.com/sanposhiho/wastedassign/v2 v2.0.7 @@ -127,6 +128,7 @@ require ( go-simpler.org/sloglint v0.7.2 go.uber.org/automaxprocs v1.6.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 + golang.org/x/sys v0.26.0 golang.org/x/tools v0.26.0 gopkg.in/yaml.v3 v3.0.1 honnef.co/go/tools v0.5.1 @@ -193,7 +195,6 @@ require ( golang.org/x/exp/typeparams v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.18.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index e089544dc025..e473209e2968 100644 --- a/go.sum +++ b/go.sum @@ -461,8 +461,8 @@ github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567/go.mod h1:DWNGW8 github.com/raeperd/recvcheck v0.1.2 h1:SjdquRsRXJc26eSonWIo8b7IMtKD3OAT2Lb5G3ZX1+4= github.com/raeperd/recvcheck v0.1.2/go.mod h1:n04eYkwIR0JbgD73wT8wL4JjPC3wm0nFtzBnWNocnYU= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryancurrah/gomodguard v1.3.5 h1:cShyguSwUEeC0jS7ylOiG/idnd1TpJ1LfHGpV3oJmPU= github.com/ryancurrah/gomodguard v1.3.5/go.mod h1:MXlEPQRxgfPQa62O8wzK3Ozbkv9Rkqr+wKjSxTdsNJE= diff --git a/internal/cache/LICENSE b/internal/cache/LICENSE new file mode 100644 index 000000000000..6a66aea5eafe --- /dev/null +++ b/internal/cache/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 299fd5279021..a59813236a3e 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -15,14 +15,16 @@ import ( "errors" "fmt" "io" + "io/fs" "os" "path/filepath" "strconv" "strings" "time" - "github.com/golangci/golangci-lint/internal/renameio" + "github.com/golangci/golangci-lint/internal/mmap" "github.com/golangci/golangci-lint/internal/robustio" + "github.com/rogpeppe/go-internal/lockedfile" ) // An ActionID is a cache action key, the hash of a complete description of a @@ -33,8 +35,50 @@ type ActionID [HashSize]byte // An OutputID is a cache output key, the hash of an output of a computation. type OutputID [HashSize]byte +// Cache is the interface as used by the cmd/go. +type Cache interface { + // Get returns the cache entry for the provided ActionID. + // On miss, the error type should be of type *entryNotFoundError. + // + // After a success call to Get, OutputFile(Entry.OutputID) must + // exist on disk for until Close is called (at the end of the process). + Get(ActionID) (Entry, error) + + // Put adds an item to the cache. + // + // The seeker is only used to seek to the beginning. After a call to Put, + // the seek position is not guaranteed to be in any particular state. + // + // As a special case, if the ReadSeeker is of type noVerifyReadSeeker, + // the verification from GODEBUG=goverifycache=1 is skipped. + // + // After a success call to Get, OutputFile(Entry.OutputID) must + // exist on disk for until Close is called (at the end of the process). + Put(ActionID, io.ReadSeeker) (_ OutputID, size int64, _ error) + + // Close is called at the end of the go process. Implementations can do + // cache cleanup work at this phase, or wait for and report any errors from + // background cleanup work started earlier. Any cache trimming should in one + // process should not violate cause the invariants of this interface to be + // violated in another process. Namely, a cache trim from one process should + // not delete an ObjectID from disk that was recently Get or Put from + // another process. As a rule of thumb, don't trim things used in the last + // day. + Close() error + + // OutputFile returns the path on disk where OutputID is stored. + // + // It's only called after a successful get or put call so it doesn't need + // to return an error; it's assumed that if the previous get or put succeeded, + // it's already on disk. + OutputFile(OutputID) string + + // FuzzDir returns where fuzz files are stored. + FuzzDir() string +} + // A Cache is a package cache, backed by a file system directory tree. -type Cache struct { +type DiskCache struct { dir string now func() time.Time } @@ -50,13 +94,13 @@ type Cache struct { // to share a cache directory (for example, if the directory were stored // in a network file system). File locking is notoriously unreliable in // network file systems and may not suffice to protect the cache. -func Open(dir string) (*Cache, error) { +func Open(dir string) (*DiskCache, error) { info, err := os.Stat(dir) if err != nil { return nil, err } if !info.IsDir() { - return nil, &os.PathError{Op: "open", Path: dir, Err: errors.New("not a directory")} + return nil, &fs.PathError{Op: "open", Path: dir, Err: fmt.Errorf("not a directory")} } for i := 0; i < 256; i++ { name := filepath.Join(dir, fmt.Sprintf("%02x", i)) @@ -64,7 +108,7 @@ func Open(dir string) (*Cache, error) { return nil, err } } - c := &Cache{ + c := &DiskCache{ dir: dir, now: time.Now, } @@ -72,14 +116,25 @@ func Open(dir string) (*Cache, error) { } // fileName returns the name of the file corresponding to the given id. -func (c *Cache) fileName(id [HashSize]byte, key string) string { +func (c *DiskCache) fileName(id [HashSize]byte, key string) string { return filepath.Join(c.dir, fmt.Sprintf("%02x", id[0]), fmt.Sprintf("%x", id)+"-"+key) } -var errMissing = errors.New("cache entry not found") +// An entryNotFoundError indicates that a cache entry was not found, with an +// optional underlying reason. +type entryNotFoundError struct { + Err error +} -func IsErrMissing(err error) bool { - return errors.Is(err, errMissing) +func (e *entryNotFoundError) Error() string { + if e.Err == nil { + return "cache entry not found" + } + return fmt.Sprintf("cache entry not found: %v", e.Err) +} + +func (e *entryNotFoundError) Unwrap() error { + return e.Err } const ( @@ -99,35 +154,41 @@ const ( // GODEBUG=gocacheverify=1. var verify = false +var errVerifyMode = errors.New("gocacheverify=1") + // DebugTest is set when GODEBUG=gocachetest=1 is in the environment. var DebugTest = false -func init() { initEnv() } - -func initEnv() { - verify = false - debugHash = false - debug := strings.Split(os.Getenv("GODEBUG"), ",") - for _, f := range debug { - if f == "gocacheverify=1" { - verify = true - } - if f == "gocachehash=1" { - debugHash = true - } - if f == "gocachetest=1" { - DebugTest = true - } - } -} +// func init() { initEnv() } + +// var ( +// gocacheverify = godebug.New("gocacheverify") +// gocachehash = godebug.New("gocachehash") +// gocachetest = godebug.New("gocachetest") +// ) + +// func initEnv() { +// if gocacheverify.Value() == "1" { +// gocacheverify.IncNonDefault() +// verify = true +// } +// if gocachehash.Value() == "1" { +// gocachehash.IncNonDefault() +// debugHash = true +// } +// if gocachetest.Value() == "1" { +// gocachetest.IncNonDefault() +// DebugTest = true +// } +// } // Get looks up the action ID in the cache, // returning the corresponding output ID and file size, if any. // Note that finding an output ID does not guarantee that the // saved file for that output ID is still available. -func (c *Cache) Get(id ActionID) (Entry, error) { +func (c *DiskCache) Get(id ActionID) (Entry, error) { if verify { - return Entry{}, errMissing + return Entry{}, &entryNotFoundError{Err: errVerifyMode} } return c.get(id) } @@ -135,99 +196,134 @@ func (c *Cache) Get(id ActionID) (Entry, error) { type Entry struct { OutputID OutputID Size int64 - Time time.Time + Time time.Time // when added to cache } // get is Get but does not respect verify mode, so that Put can use it. -func (c *Cache) get(id ActionID) (Entry, error) { - missing := func() (Entry, error) { - return Entry{}, errMissing - } - failed := func(err error) (Entry, error) { - return Entry{}, err +func (c *DiskCache) get(id ActionID) (Entry, error) { + missing := func(reason error) (Entry, error) { + return Entry{}, &entryNotFoundError{Err: reason} } - fileName := c.fileName(id, "a") - f, err := os.Open(fileName) + f, err := os.Open(c.fileName(id, "a")) if err != nil { - if os.IsNotExist(err) { - return missing() - } - return failed(err) + return missing(err) } defer f.Close() entry := make([]byte, entrySize+1) // +1 to detect whether f is too long - if n, readErr := io.ReadFull(f, entry); n != entrySize || readErr != io.ErrUnexpectedEOF { - return failed(fmt.Errorf("read %d/%d bytes from %s with error %w", n, entrySize, fileName, readErr)) + if n, err := io.ReadFull(f, entry); n > entrySize { + return missing(errors.New("too long")) + } else if err != io.ErrUnexpectedEOF { + if err == io.EOF { + return missing(errors.New("file is empty")) + } + return missing(err) + } else if n < entrySize { + return missing(errors.New("entry file incomplete")) } if entry[0] != 'v' || entry[1] != '1' || entry[2] != ' ' || entry[3+hexSize] != ' ' || entry[3+hexSize+1+hexSize] != ' ' || entry[3+hexSize+1+hexSize+1+20] != ' ' || entry[entrySize-1] != '\n' { - return failed(fmt.Errorf("bad data in %s", fileName)) + return missing(errors.New("invalid header")) } eid, entry := entry[3:3+hexSize], entry[3+hexSize:] eout, entry := entry[1:1+hexSize], entry[1+hexSize:] esize, entry := entry[1:1+20], entry[1+20:] - etime := entry[1 : 1+20] + etime, entry := entry[1:1+20], entry[1+20:] var buf [HashSize]byte - if _, err = hex.Decode(buf[:], eid); err != nil || buf != id { - return failed(fmt.Errorf("failed to hex decode eid data in %s: %w", fileName, err)) + if _, err := hex.Decode(buf[:], eid); err != nil { + return missing(fmt.Errorf("decoding ID: %v", err)) + } else if buf != id { + return missing(errors.New("mismatched ID")) } - if _, err = hex.Decode(buf[:], eout); err != nil { - return failed(fmt.Errorf("failed to hex decode eout data in %s: %w", fileName, err)) + if _, err := hex.Decode(buf[:], eout); err != nil { + return missing(fmt.Errorf("decoding output ID: %v", err)) } i := 0 for i < len(esize) && esize[i] == ' ' { i++ } size, err := strconv.ParseInt(string(esize[i:]), 10, 64) - if err != nil || size < 0 { - return failed(fmt.Errorf("failed to parse esize int from %s with error %w", fileName, err)) + if err != nil { + return missing(fmt.Errorf("parsing size: %v", err)) + } else if size < 0 { + return missing(errors.New("negative size")) } i = 0 for i < len(etime) && etime[i] == ' ' { i++ } tm, err := strconv.ParseInt(string(etime[i:]), 10, 64) - if err != nil || tm < 0 { - return failed(fmt.Errorf("failed to parse etime int from %s with error %w", fileName, err)) + if err != nil { + return missing(fmt.Errorf("parsing timestamp: %v", err)) + } else if tm < 0 { + return missing(errors.New("negative timestamp")) } - if err = c.used(fileName); err != nil { - return failed(fmt.Errorf("failed to mark %s as used: %w", fileName, err)) + err = c.used(c.fileName(id, "a")) + if err != nil { + return Entry{}, fmt.Errorf("failed to mark %s as used: %w", c.fileName(id, "a"), err) } return Entry{buf, size, time.Unix(0, tm)}, nil } +// GetFile looks up the action ID in the cache and returns +// the name of the corresponding data file. +func GetFile(c Cache, id ActionID) (file string, entry Entry, err error) { + entry, err = c.Get(id) + if err != nil { + return "", Entry{}, err + } + file = c.OutputFile(entry.OutputID) + info, err := os.Stat(file) + if err != nil { + return "", Entry{}, &entryNotFoundError{Err: err} + } + if info.Size() != entry.Size { + return "", Entry{}, &entryNotFoundError{Err: errors.New("file incomplete")} + } + return file, entry, nil +} + // GetBytes looks up the action ID in the cache and returns // the corresponding output bytes. // GetBytes should only be used for data that can be expected to fit in memory. -func (c *Cache) GetBytes(id ActionID) ([]byte, Entry, error) { +func GetBytes(c Cache, id ActionID) ([]byte, Entry, error) { entry, err := c.Get(id) if err != nil { return nil, entry, err } - outputFile, err := c.OutputFile(entry.OutputID) + data, err := robustio.ReadFile(c.OutputFile(entry.OutputID)) if err != nil { return nil, entry, err } + if sha256.Sum256(data) != entry.OutputID { + return nil, entry, &entryNotFoundError{Err: errors.New("bad checksum")} + } + return data, entry, nil +} - data, err := robustio.ReadFile(outputFile) +// GetMmap looks up the action ID in the cache and returns +// the corresponding output bytes. +// GetMmap should only be used for data that can be expected to fit in memory. +func GetMmap(c Cache, id ActionID) ([]byte, Entry, error) { + entry, err := c.Get(id) if err != nil { return nil, entry, err } - - if sha256.Sum256(data) != entry.OutputID { - return nil, entry, errMissing + md, err := mmap.Mmap(c.OutputFile(entry.OutputID)) + if err != nil { + return nil, Entry{}, err } - return data, entry, nil + if int64(len(md.Data)) != entry.Size { + return nil, Entry{}, &entryNotFoundError{Err: errors.New("file incomplete")} + } + return md.Data, entry, nil } // OutputFile returns the name of the cache file storing output with the given OutputID. -func (c *Cache) OutputFile(out OutputID) (string, error) { +func (c *DiskCache) OutputFile(out OutputID) string { file := c.fileName(out, "d") - if err := c.used(file); err != nil { - return "", err - } - return file, nil + c.used(file) + return file } // Time constants for cache expiration. @@ -255,39 +351,49 @@ const ( // and to reduce the amount of disk activity caused by using // cache entries, used only updates the mtime if the current // mtime is more than an hour old. This heuristic eliminates -// nearly all the mtime updates that would otherwise happen, +// nearly all of the mtime updates that would otherwise happen, // while still keeping the mtimes useful for cache trimming. -func (c *Cache) used(file string) error { +func (c *DiskCache) used(file string) error { info, err := os.Stat(file) + if err == nil && c.now().Sub(info.ModTime()) < mtimeInterval { + return nil + } + if err != nil { if os.IsNotExist(err) { - return errMissing + return &entryNotFoundError{Err: err} } - return fmt.Errorf("failed to stat file %s: %w", file, err) + return &entryNotFoundError{Err: fmt.Errorf("failed to stat file %s: %w", file, err)} } - if c.now().Sub(info.ModTime()) < mtimeInterval { - return nil - } - - if err := os.Chtimes(file, c.now(), c.now()); err != nil { + err = os.Chtimes(file, c.now(), c.now()) + if err != nil { return fmt.Errorf("failed to change time of file %s: %w", file, err) } return nil } +func (c *DiskCache) Close() error { return c.Trim() } + // Trim removes old cache entries that are likely not to be reused. -func (c *Cache) Trim() { +func (c *DiskCache) Trim() error { now := c.now() // We maintain in dir/trim.txt the time of the last completed cache trim. // If the cache has been trimmed recently enough, do nothing. // This is the common case. - data, _ := renameio.ReadFile(filepath.Join(c.dir, "trim.txt")) - t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) - if err == nil && now.Sub(time.Unix(t, 0)) < trimInterval { - return + // If the trim file is corrupt, detected if the file can't be parsed, or the + // trim time is too far in the future, attempt the trim anyway. It's possible that + // the cache was full when the corruption happened. Attempting a trim on + // an empty cache is cheap, so there wouldn't be a big performance hit in that case. + if data, err := lockedfile.Read(filepath.Join(c.dir, "trim.txt")); err == nil { + if t, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); err == nil { + lastTrim := time.Unix(t, 0) + if d := now.Sub(lastTrim); d < trimInterval && d > -mtimeInterval { + return nil + } + } } // Trim each of the 256 subdirectories. @@ -301,15 +407,21 @@ func (c *Cache) Trim() { // Ignore errors from here: if we don't write the complete timestamp, the // cache will appear older than it is, and we'll trim it again next time. - _ = renameio.WriteFile(filepath.Join(c.dir, "trim.txt"), []byte(fmt.Sprintf("%d", now.Unix())), 0666) + var b bytes.Buffer + fmt.Fprintf(&b, "%d", now.Unix()) + if err := lockedfile.Write(filepath.Join(c.dir, "trim.txt"), &b, 0666); err != nil { + return err + } + + return nil } // trimSubdir trims a single cache subdirectory. -func (c *Cache) trimSubdir(subdir string, cutoff time.Time) { +func (c *DiskCache) trimSubdir(subdir string, cutoff time.Time) { // Read all directory entries from subdir before removing // any files, in case removing files invalidates the file offset // in the directory scan. Also, ignore error from f.Readdirnames, - // because we don't care about reporting the error, and we still + // because we don't care about reporting the error and we still // want to process any entries found before the error. f, err := os.Open(subdir) if err != nil { @@ -333,7 +445,7 @@ func (c *Cache) trimSubdir(subdir string, cutoff time.Time) { // putIndexEntry adds an entry to the cache recording that executing the action // with the given id produces an output with the given output id (hash) and size. -func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify bool) error { +func (c *DiskCache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify bool) error { // Note: We expect that for one reason or another it may happen // that repeating an action produces a different output hash // (for example, if the output contains a time stamp or temp dir name). @@ -346,7 +458,6 @@ func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify // are entirely reproducible. As just noted, this may be unrealistic // in some cases but the check is also useful for shaking out real bugs. entry := fmt.Sprintf("v1 %x %x %20d %20d\n", id, out, size, time.Now().UnixNano()) - if verify && allowVerify { old, err := c.get(id) if err == nil && (old.OutputID != out || old.Size != size) { @@ -383,28 +494,40 @@ func (c *Cache) putIndexEntry(id ActionID, out OutputID, size int64, allowVerify os.Remove(file) return err } - if err = os.Chtimes(file, c.now(), c.now()); err != nil { // mainly for tests + err = os.Chtimes(file, c.now(), c.now()) // mainly for tests + if err != nil { return fmt.Errorf("failed to change time of file %s: %w", file, err) } return nil } +// noVerifyReadSeeker is an io.ReadSeeker wrapper sentinel type +// that says that Cache.Put should skip the verify check +// (from GODEBUG=goverifycache=1). +type noVerifyReadSeeker struct { + io.ReadSeeker +} + // Put stores the given output in the cache as the output for the action ID. // It may read file twice. The content of file must not change between the two passes. -func (c *Cache) Put(id ActionID, file io.ReadSeeker) (OutputID, int64, error) { - return c.put(id, file, true) +func (c *DiskCache) Put(id ActionID, file io.ReadSeeker) (OutputID, int64, error) { + wrapper, isNoVerify := file.(noVerifyReadSeeker) + if isNoVerify { + file = wrapper.ReadSeeker + } + return c.put(id, file, !isNoVerify) } // PutNoVerify is like Put but disables the verify check // when GODEBUG=goverifycache=1 is set. // It is meant for data that is OK to cache but that we expect to vary slightly from run to run, // like test output containing times and the like. -func (c *Cache) PutNoVerify(id ActionID, file io.ReadSeeker) (OutputID, int64, error) { - return c.put(id, file, false) +func PutNoVerify(c Cache, id ActionID, file io.ReadSeeker) (OutputID, int64, error) { + return c.Put(id, noVerifyReadSeeker{file}) } -func (c *Cache) put(id ActionID, file io.ReadSeeker, allowVerify bool) (OutputID, int64, error) { +func (c *DiskCache) put(id ActionID, file io.ReadSeeker, allowVerify bool) (OutputID, int64, error) { // Compute output ID. h := sha256.New() if _, err := file.Seek(0, 0); err != nil { @@ -427,21 +550,22 @@ func (c *Cache) put(id ActionID, file io.ReadSeeker, allowVerify bool) (OutputID } // PutBytes stores the given bytes in the cache as the output for the action ID. -func (c *Cache) PutBytes(id ActionID, data []byte) error { +func PutBytes(c Cache, id ActionID, data []byte) error { _, _, err := c.Put(id, bytes.NewReader(data)) return err } // copyFile copies file into the cache, expecting it to have the given // output ID and size, if that file is not present already. -func (c *Cache) copyFile(file io.ReadSeeker, out OutputID, size int64) error { +func (c *DiskCache) copyFile(file io.ReadSeeker, out OutputID, size int64) error { name := c.fileName(out, "d") info, err := os.Stat(name) if err == nil && info.Size() == size { // Check hash. - if f, openErr := os.Open(name); openErr == nil { + if f, err := os.Open(name); err == nil { h := sha256.New() - if _, copyErr := io.Copy(h, f); copyErr != nil { + _, copyErr := io.Copy(h, f) + if copyErr != nil { return fmt.Errorf("failed to copy to sha256: %w", copyErr) } @@ -477,49 +601,63 @@ func (c *Cache) copyFile(file io.ReadSeeker, out OutputID, size int64) error { // before returning, to avoid leaving bad bytes in the file. // Copy file to f, but also into h to double-check hash. - if _, err = file.Seek(0, 0); err != nil { - _ = f.Truncate(0) + if _, err := file.Seek(0, 0); err != nil { + f.Truncate(0) return err } h := sha256.New() w := io.MultiWriter(f, h) - if _, err = io.CopyN(w, file, size-1); err != nil { - _ = f.Truncate(0) + if _, err := io.CopyN(w, file, size-1); err != nil { + f.Truncate(0) return err } // Check last byte before writing it; writing it will make the size match // what other processes expect to find and might cause them to start // using the file. buf := make([]byte, 1) - if _, err = file.Read(buf); err != nil { - _ = f.Truncate(0) + if _, err := file.Read(buf); err != nil { + f.Truncate(0) return err } - if n, wErr := h.Write(buf); n != len(buf) { + n, wErr := h.Write(buf) + if n != len(buf) { return fmt.Errorf("wrote to hash %d/%d bytes with error %w", n, len(buf), wErr) } sum := h.Sum(nil) if !bytes.Equal(sum, out[:]) { - _ = f.Truncate(0) - return errors.New("file content changed underfoot") + f.Truncate(0) + return fmt.Errorf("file content changed underfoot") } // Commit cache file entry. - if _, err = f.Write(buf); err != nil { - _ = f.Truncate(0) + if _, err := f.Write(buf); err != nil { + f.Truncate(0) return err } - if err = f.Close(); err != nil { + if err := f.Close(); err != nil { // Data might not have been written, // but file may look like it is the right size. // To be extra careful, remove cached file. os.Remove(name) return err } - if err = os.Chtimes(name, c.now(), c.now()); err != nil { // mainly for tests + err = os.Chtimes(name, c.now(), c.now()) // mainly for tests + if err != nil { return fmt.Errorf("failed to change time of file %s: %w", name, err) } return nil } + +// FuzzDir returns a subdirectory within the cache for storing fuzzing data. +// The subdirectory may not exist. +// +// This directory is managed by the internal/fuzz package. Files in this +// directory aren't removed by the 'go clean -cache' command or by Trim. +// They may be removed with 'go clean -fuzzcache'. +// +// TODO(#48526): make Trim remove unused files from this directory. +func (c *DiskCache) FuzzDir() string { + return filepath.Join(c.dir, "fuzz") +} diff --git a/internal/cache/cache_gcil.go b/internal/cache/cache_gcil.go new file mode 100644 index 000000000000..b4f07738e623 --- /dev/null +++ b/internal/cache/cache_gcil.go @@ -0,0 +1,12 @@ +package cache + +import ( + "errors" +) + +// IsErrMissing allows to access to the internal error. +// TODO(ldez) the handling of this error inside runner_action.go should be refactored. +func IsErrMissing(err error) bool { + var errENF *entryNotFoundError + return errors.As(err, &errENF) +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 1c24cdfc6276..b201cc675051 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -12,6 +12,8 @@ import ( "path/filepath" "testing" "time" + + "github.com/golangci/golangci-lint/internal/testenv" ) func init() { @@ -19,8 +21,6 @@ func init() { } func TestBasic(t *testing.T) { - t.Parallel() - dir, err := os.MkdirTemp("", "cachetest-") if err != nil { t.Fatal(err) @@ -66,8 +66,6 @@ func TestBasic(t *testing.T) { } func TestGrowth(t *testing.T) { - t.Parallel() - dir, err := os.MkdirTemp("", "cachetest-") if err != nil { t.Fatal(err) @@ -81,7 +79,7 @@ func TestGrowth(t *testing.T) { n := 10000 if testing.Short() { - n = 1000 + n = 10 } for i := 0; i < n; i++ { @@ -109,43 +107,43 @@ func TestGrowth(t *testing.T) { } } -func TestVerifyPanic(t *testing.T) { - os.Setenv("GODEBUG", "gocacheverify=1") - initEnv() - defer func() { - os.Unsetenv("GODEBUG") - verify = false - }() - - if !verify { - t.Fatal("initEnv did not set verify") - } - - dir, err := os.MkdirTemp("", "cachetest-") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - - c, err := Open(dir) - if err != nil { - t.Fatalf("Open: %v", err) - } - - id := ActionID(dummyID(1)) - if err := c.PutBytes(id, []byte("abc")); err != nil { - t.Fatal(err) - } - - defer func() { - if err := recover(); err != nil { - t.Log(err) - return - } - }() - c.PutBytes(id, []byte("def")) - t.Fatal("mismatched Put did not panic in verify mode") -} +// func TestVerifyPanic(t *testing.T) { +// os.Setenv("GODEBUG", "gocacheverify=1") +// initEnv() +// defer func() { +// os.Unsetenv("GODEBUG") +// verify = false +// }() +// +// if !verify { +// t.Fatal("initEnv did not set verify") +// } +// +// dir, err := os.MkdirTemp("", "cachetest-") +// if err != nil { +// t.Fatal(err) +// } +// defer os.RemoveAll(dir) +// +// c, err := Open(dir) +// if err != nil { +// t.Fatalf("Open: %v", err) +// } +// +// id := ActionID(dummyID(1)) +// if err := PutBytes(c, id, []byte("abc")); err != nil { +// t.Fatal(err) +// } +// +// defer func() { +// if err := recover(); err != nil { +// t.Log(err) +// return +// } +// }() +// PutBytes(c, id, []byte("def")) +// t.Fatal("mismatched Put did not panic in verify mode") +// } func dummyID(x int) [HashSize]byte { var out [HashSize]byte @@ -154,8 +152,6 @@ func dummyID(x int) [HashSize]byte { } func TestCacheTrim(t *testing.T) { - t.Parallel() - dir, err := os.MkdirTemp("", "cachetest-") if err != nil { t.Fatal(err) @@ -183,9 +179,9 @@ func TestCacheTrim(t *testing.T) { } id := ActionID(dummyID(1)) - c.PutBytes(id, []byte("abc")) + PutBytes(c, id, []byte("abc")) entry, _ := c.Get(id) - c.PutBytes(ActionID(dummyID(2)), []byte("def")) + PutBytes(c, ActionID(dummyID(2)), []byte("def")) mtime := now checkTime(fmt.Sprintf("%x-a", id), mtime) checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime) @@ -207,7 +203,12 @@ func TestCacheTrim(t *testing.T) { checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime2) // Trim should leave everything alone: it's all too new. - c.Trim() + if err := c.Trim(); err != nil { + if testenv.SyscallIsNotSupported(err) { + t.Skipf("skipping: Trim is unsupported (%v)", err) + } + t.Fatal(err) + } if _, err := c.Get(id); err != nil { t.Fatal(err) } @@ -220,7 +221,9 @@ func TestCacheTrim(t *testing.T) { // Trim less than a day later should not do any work at all. now = start + 80000 - c.Trim() + if err := c.Trim(); err != nil { + t.Fatal(err) + } if _, err := c.Get(id); err != nil { t.Fatal(err) } @@ -233,14 +236,16 @@ func TestCacheTrim(t *testing.T) { t.Fatalf("second trim did work: %q -> %q", data, data2) } - // Fast-forward and do another trim just before the 5-day cutoff. + // Fast forward and do another trim just before the 5 day cutoff. // Note that because of usedQuantum the cutoff is actually 5 days + 1 hour. // We used c.Get(id) just now, so 5 days later it should still be kept. // On the other hand almost a full day has gone by since we wrote dummyID(2) // and we haven't looked at it since, so 5 days later it should be gone. now += 5 * 86400 checkTime(fmt.Sprintf("%x-a", dummyID(2)), start) - c.Trim() + if err := c.Trim(); err != nil { + t.Fatal(err) + } if _, err := c.Get(id); err != nil { t.Fatal(err) } @@ -254,7 +259,9 @@ func TestCacheTrim(t *testing.T) { // Check that another 5 days later it is still not gone, // but check by using checkTime, which doesn't bring mtime forward. now += 5 * 86400 - c.Trim() + if err := c.Trim(); err != nil { + t.Fatal(err) + } checkTime(fmt.Sprintf("%x-a", id), mtime3) checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime3) @@ -262,13 +269,17 @@ func TestCacheTrim(t *testing.T) { // Even though the entry for id is now old enough to be trimmed, // it gets a reprieve until the time comes for a new Trim scan. now += 86400 / 2 - c.Trim() + if err := c.Trim(); err != nil { + t.Fatal(err) + } checkTime(fmt.Sprintf("%x-a", id), mtime3) checkTime(fmt.Sprintf("%x-d", entry.OutputID), mtime3) // Another half a day later, Trim should actually run, and it should remove id. now += 86400/2 + 1 - c.Trim() + if err := c.Trim(); err != nil { + t.Fatal(err) + } if _, err := c.Get(dummyID(1)); err == nil { t.Fatal("Trim did not remove dummyID(1)") } diff --git a/internal/cache/default.go b/internal/cache/default.go index 399cc84cf0fd..7232f1ef3e60 100644 --- a/internal/cache/default.go +++ b/internal/cache/default.go @@ -6,23 +6,22 @@ package cache import ( "fmt" - "log" + base "log" "os" "path/filepath" "sync" ) -const envGolangciLintCache = "GOLANGCI_LINT_CACHE" - // Default returns the default cache to use. -func Default() (*Cache, error) { +// It never returns nil. +func Default() Cache { defaultOnce.Do(initDefaultCache) - return defaultCache, defaultDirErr + return defaultCache } var ( defaultOnce sync.Once - defaultCache *Cache + defaultCache Cache ) // cacheREADME is a message stored in a README in the cache directory. @@ -34,32 +33,46 @@ const cacheREADME = `This directory holds cached build artifacts from golangci-l // initDefaultCache does the work of finding the default cache // the first time Default is called. func initDefaultCache() { - dir := DefaultDir() + dir, _ := DefaultDir() + if dir == "off" { + if defaultDirErr != nil { + base.Fatalf("build cache is required, but could not be located: %v", defaultDirErr) + } + base.Fatalf("build cache is disabled by %s=off, but required", envGolangciLintCache) + } if err := os.MkdirAll(dir, 0744); err != nil { - log.Fatalf("failed to initialize build cache at %s: %s\n", dir, err) + base.Fatalf("failed to initialize build cache at %s: %s\n", dir, err) } if _, err := os.Stat(filepath.Join(dir, "README")); err != nil { // Best effort. if wErr := os.WriteFile(filepath.Join(dir, "README"), []byte(cacheREADME), 0666); wErr != nil { - log.Fatalf("Failed to write README file to cache dir %s: %s", dir, err) + base.Fatalf("Failed to write README file to cache dir %s: %s", dir, err) } } - c, err := Open(dir) + diskCache, err := Open(dir) if err != nil { - log.Fatalf("failed to initialize build cache at %s: %s\n", dir, err) + base.Fatalf("failed to initialize build cache at %s: %s\n", dir, err) + } + + if v := os.Getenv(envGolangciLintCacheProg); v != "" { + defaultCache = startCacheProg(v, diskCache) + } else { + defaultCache = diskCache } - defaultCache = c } var ( - defaultDirOnce sync.Once - defaultDir string - defaultDirErr error + defaultDirOnce sync.Once + defaultDir string + defaultDirChanged bool // effective value differs from $GOLANGCI_LINT_CACHE + defaultDirErr error ) // DefaultDir returns the effective GOLANGCI_LINT_CACHE setting. -func DefaultDir() string { +// It returns "off" if the cache is disabled, +// and reports whether the effective value differs from GOLANGCI_LINT_CACHE. +func DefaultDir() (string, bool) { // Save the result of the first call to DefaultDir for later use in // initDefaultCache. cmd/go/main.go explicitly sets GOCACHE so that // subprocesses will inherit it, but that means initDefaultCache can't @@ -67,10 +80,12 @@ func DefaultDir() string { defaultDirOnce.Do(func() { defaultDir = os.Getenv(envGolangciLintCache) - if filepath.IsAbs(defaultDir) { - return - } if defaultDir != "" { + defaultDirChanged = true + if filepath.IsAbs(defaultDir) || defaultDir == "off" { + return + } + defaultDir = "off" defaultDirErr = fmt.Errorf("%s is not an absolute path", envGolangciLintCache) return } @@ -78,11 +93,13 @@ func DefaultDir() string { // Compute default location. dir, err := os.UserCacheDir() if err != nil { + defaultDir = "off" + defaultDirChanged = true defaultDirErr = fmt.Errorf("%s is not defined and %w", envGolangciLintCache, err) return } defaultDir = filepath.Join(dir, "golangci-lint") }) - return defaultDir + return defaultDir, defaultDirChanged } diff --git a/internal/cache/default_gcil.go b/internal/cache/default_gcil.go new file mode 100644 index 000000000000..a801f67f4791 --- /dev/null +++ b/internal/cache/default_gcil.go @@ -0,0 +1,6 @@ +package cache + +const ( + envGolangciLintCache = "GOLANGCI_LINT_CACHE" + envGolangciLintCacheProg = "GOLANGCI_LINT_CACHEPROG" +) diff --git a/internal/cache/hash.go b/internal/cache/hash.go index 4ce79e325b24..d5169dd4c491 100644 --- a/internal/cache/hash.go +++ b/internal/cache/hash.go @@ -11,6 +11,7 @@ import ( "hash" "io" "os" + "strings" "sync" ) @@ -36,22 +37,26 @@ type Hash struct { // which are still addressed by unsalted SHA256. var hashSalt []byte -func SetSalt(b []byte) { - hashSalt = b +// stripExperiment strips any GOEXPERIMENT configuration from the Go +// version string. +func stripExperiment(version string) string { + if i := strings.Index(version, " X:"); i >= 0 { + return version[:i] + } + return version } // Subkey returns an action ID corresponding to mixing a parent // action ID with a string description of the subkey. func Subkey(parent ActionID, desc string) (ActionID, error) { h := sha256.New() - const subkeyPrefix = "subkey:" - if n, err := h.Write([]byte(subkeyPrefix)); n != len(subkeyPrefix) { - return ActionID{}, fmt.Errorf("wrote %d/%d bytes of subkey prefix with error %s", n, len(subkeyPrefix), err) - } - if n, err := h.Write(parent[:]); n != len(parent) { + h.Write([]byte(("subkey:"))) + n, err := h.Write(parent[:]) + if n != len(parent) { return ActionID{}, fmt.Errorf("wrote %d/%d bytes of parent with error %s", n, len(parent), err) } - if n, err := h.Write([]byte(desc)); n != len(desc) { + n, err = h.Write([]byte(desc)) + if n != len(desc) { return ActionID{}, fmt.Errorf("wrote %d/%d bytes of desc with error %s", n, len(desc), err) } @@ -75,7 +80,8 @@ func NewHash(name string) (*Hash, error) { if debugHash { fmt.Fprintf(os.Stderr, "HASH[%s]\n", h.name) } - if n, err := h.Write(hashSalt); n != len(hashSalt) { + n, err := h.Write(hashSalt) + if n != len(hashSalt) { return nil, fmt.Errorf("wrote %d/%d bytes of hash salt with error %s", n, len(hashSalt), err) } if verify { diff --git a/internal/cache/hash_gcil.go b/internal/cache/hash_gcil.go new file mode 100644 index 000000000000..08749036bd38 --- /dev/null +++ b/internal/cache/hash_gcil.go @@ -0,0 +1,5 @@ +package cache + +func SetSalt(b []byte) { + hashSalt = b +} diff --git a/internal/cache/prog.go b/internal/cache/prog.go new file mode 100644 index 000000000000..91ef831755a4 --- /dev/null +++ b/internal/cache/prog.go @@ -0,0 +1,428 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cache + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "log" + base "log" + "os" + "os/exec" + "sync" + "sync/atomic" + "time" + + "github.com/golangci/golangci-lint/internal/quoted" +) + +// ProgCache implements Cache via JSON messages over stdin/stdout to a child +// helper process which can then implement whatever caching policy/mechanism it +// wants. +// +// See https://github.com/golang/go/issues/59719 +type ProgCache struct { + cmd *exec.Cmd + stdout io.ReadCloser // from the child process + stdin io.WriteCloser // to the child process + bw *bufio.Writer // to stdin + jenc *json.Encoder // to bw + + // can are the commands that the child process declared that it supports. + // This is effectively the versioning mechanism. + can map[ProgCmd]bool + + // fuzzDirCache is another Cache implementation to use for the FuzzDir + // method. In practice this is the default GOCACHE disk-based + // implementation. + // + // TODO(bradfitz): maybe this isn't ideal. But we'd need to extend the Cache + // interface and the fuzzing callers to be less disk-y to do more here. + fuzzDirCache Cache + + closing atomic.Bool + ctx context.Context // valid until Close via ctxClose + ctxCancel context.CancelFunc // called on Close + readLoopDone chan struct{} // closed when readLoop returns + + mu sync.Mutex // guards following fields + nextID int64 + inFlight map[int64]chan<- *ProgResponse + outputFile map[OutputID]string // object => abs path on disk + + // writeMu serializes writing to the child process. + // It must never be held at the same time as mu. + writeMu sync.Mutex +} + +// ProgCmd is a command that can be issued to a child process. +// +// If the interface needs to grow, we can add new commands or new versioned +// commands like "get2". +type ProgCmd string + +const ( + cmdGet = ProgCmd("get") + cmdPut = ProgCmd("put") + cmdClose = ProgCmd("close") +) + +// ProgRequest is the JSON-encoded message that's sent from cmd/go to +// the GOLANGCI_LINT_CACHEPROG child process over stdin. Each JSON object is on its +// own line. A ProgRequest of Type "put" with BodySize > 0 will be followed +// by a line containing a base64-encoded JSON string literal of the body. +type ProgRequest struct { + // ID is a unique number per process across all requests. + // It must be echoed in the ProgResponse from the child. + ID int64 + + // Command is the type of request. + // The cmd/go tool will only send commands that were declared + // as supported by the child. + Command ProgCmd + + // ActionID is non-nil for get and puts. + ActionID []byte `json:",omitempty"` // or nil if not used + + // ObjectID is set for Type "put" and "output-file". + ObjectID []byte `json:",omitempty"` // or nil if not used + + // Body is the body for "put" requests. It's sent after the JSON object + // as a base64-encoded JSON string when BodySize is non-zero. + // It's sent as a separate JSON value instead of being a struct field + // send in this JSON object so large values can be streamed in both directions. + // The base64 string body of a ProgRequest will always be written + // immediately after the JSON object and a newline. + Body io.Reader `json:"-"` + + // BodySize is the number of bytes of Body. If zero, the body isn't written. + BodySize int64 `json:",omitempty"` +} + +// ProgResponse is the JSON response from the child process to cmd/go. +// +// With the exception of the first protocol message that the child writes to its +// stdout with ID==0 and KnownCommands populated, these are only sent in +// response to a ProgRequest from cmd/go. +// +// ProgResponses can be sent in any order. The ID must match the request they're +// replying to. +type ProgResponse struct { + ID int64 // that corresponds to ProgRequest; they can be answered out of order + Err string `json:",omitempty"` // if non-empty, the error + + // KnownCommands is included in the first message that cache helper program + // writes to stdout on startup (with ID==0). It includes the + // ProgRequest.Command types that are supported by the program. + // + // This lets us extend the protocol gracefully over time (adding "get2", + // etc), or fail gracefully when needed. It also lets us verify the program + // wants to be a cache helper. + KnownCommands []ProgCmd `json:",omitempty"` + + // For Get requests. + + Miss bool `json:",omitempty"` // cache miss + OutputID []byte `json:",omitempty"` + Size int64 `json:",omitempty"` // in bytes + Time *time.Time `json:",omitempty"` // an Entry.Time; when the object was added to the docs + + // DiskPath is the absolute path on disk of the ObjectID corresponding + // a "get" request's ActionID (on cache hit) or a "put" request's + // provided ObjectID. + DiskPath string `json:",omitempty"` +} + +// startCacheProg starts the prog binary (with optional space-separated flags) +// and returns a Cache implementation that talks to it. +// +// It blocks a few seconds to wait for the child process to successfully start +// and advertise its capabilities. +func startCacheProg(progAndArgs string, fuzzDirCache Cache) Cache { + if fuzzDirCache == nil { + panic("missing fuzzDirCache") + } + args, err := quoted.Split(progAndArgs) + if err != nil { + base.Fatalf("%s args: %v", envGolangciLintCacheProg, err) + } + var prog string + if len(args) > 0 { + prog = args[0] + args = args[1:] + } + + ctx, ctxCancel := context.WithCancel(context.Background()) + + cmd := exec.CommandContext(ctx, prog, args...) + out, err := cmd.StdoutPipe() + if err != nil { + base.Fatalf("StdoutPipe to %s: %v", envGolangciLintCacheProg, err) + } + in, err := cmd.StdinPipe() + if err != nil { + base.Fatalf("StdinPipe to %s: %v", envGolangciLintCacheProg, err) + } + cmd.Stderr = os.Stderr + cmd.Cancel = in.Close + + if err := cmd.Start(); err != nil { + base.Fatalf("error starting %s program %q: %v", envGolangciLintCacheProg, prog, err) + } + + pc := &ProgCache{ + ctx: ctx, + ctxCancel: ctxCancel, + fuzzDirCache: fuzzDirCache, + cmd: cmd, + stdout: out, + stdin: in, + bw: bufio.NewWriter(in), + inFlight: make(map[int64]chan<- *ProgResponse), + outputFile: make(map[OutputID]string), + readLoopDone: make(chan struct{}), + } + + // Register our interest in the initial protocol message from the child to + // us, saying what it can do. + capResc := make(chan *ProgResponse, 1) + pc.inFlight[0] = capResc + + pc.jenc = json.NewEncoder(pc.bw) + go pc.readLoop(pc.readLoopDone) + + // Give the child process a few seconds to report its capabilities. This + // should be instant and not require any slow work by the program. + timer := time.NewTicker(5 * time.Second) + defer timer.Stop() + for { + select { + case <-timer.C: + log.Printf("# still waiting for %s %v ...", envGolangciLintCacheProg, prog) + case capRes := <-capResc: + can := map[ProgCmd]bool{} + for _, cmd := range capRes.KnownCommands { + can[cmd] = true + } + if len(can) == 0 { + base.Fatalf("%s %v declared no supported commands", envGolangciLintCacheProg, prog) + } + pc.can = can + return pc + } + } +} + +func (c *ProgCache) readLoop(readLoopDone chan<- struct{}) { + defer close(readLoopDone) + jd := json.NewDecoder(c.stdout) + for { + res := new(ProgResponse) + if err := jd.Decode(res); err != nil { + if c.closing.Load() { + return // quietly + } + if err == io.EOF { + c.mu.Lock() + inFlight := len(c.inFlight) + c.mu.Unlock() + base.Fatalf("%s exited pre-Close with %v pending requests", envGolangciLintCacheProg, inFlight) + } + base.Fatalf("error reading JSON from %s: %v", envGolangciLintCacheProg, err) + } + c.mu.Lock() + ch, ok := c.inFlight[res.ID] + delete(c.inFlight, res.ID) + c.mu.Unlock() + if ok { + ch <- res + } else { + base.Fatalf("%s sent response for unknown request ID %v", envGolangciLintCacheProg, res.ID) + } + } +} + +func (c *ProgCache) send(ctx context.Context, req *ProgRequest) (*ProgResponse, error) { + resc := make(chan *ProgResponse, 1) + if err := c.writeToChild(req, resc); err != nil { + return nil, err + } + select { + case res := <-resc: + if res.Err != "" { + return nil, errors.New(res.Err) + } + return res, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (c *ProgCache) writeToChild(req *ProgRequest, resc chan<- *ProgResponse) (err error) { + c.mu.Lock() + c.nextID++ + req.ID = c.nextID + c.inFlight[req.ID] = resc + c.mu.Unlock() + + defer func() { + if err != nil { + c.mu.Lock() + delete(c.inFlight, req.ID) + c.mu.Unlock() + } + }() + + c.writeMu.Lock() + defer c.writeMu.Unlock() + + if err := c.jenc.Encode(req); err != nil { + return err + } + if err := c.bw.WriteByte('\n'); err != nil { + return err + } + if req.Body != nil && req.BodySize > 0 { + if err := c.bw.WriteByte('"'); err != nil { + return err + } + e := base64.NewEncoder(base64.StdEncoding, c.bw) + wrote, err := io.Copy(e, req.Body) + if err != nil { + return err + } + if err := e.Close(); err != nil { + return nil + } + if wrote != req.BodySize { + return fmt.Errorf("short write writing body to %s for action %x, object %x: wrote %v; expected %v", + envGolangciLintCacheProg, req.ActionID, req.ObjectID, wrote, req.BodySize) + } + if _, err := c.bw.WriteString("\"\n"); err != nil { + return err + } + } + if err := c.bw.Flush(); err != nil { + return err + } + return nil +} + +func (c *ProgCache) Get(a ActionID) (Entry, error) { + if !c.can[cmdGet] { + // They can't do a "get". Maybe they're a write-only cache. + // + // TODO(bradfitz,bcmills): figure out the proper error type here. Maybe + // errors.ErrUnsupported? Is entryNotFoundError even appropriate? There + // might be places where we rely on the fact that a recent Put can be + // read through a corresponding Get. Audit callers and check, and document + // error types on the Cache interface. + return Entry{}, &entryNotFoundError{} + } + res, err := c.send(c.ctx, &ProgRequest{ + Command: cmdGet, + ActionID: a[:], + }) + if err != nil { + return Entry{}, err // TODO(bradfitz): or entryNotFoundError? Audit callers. + } + if res.Miss { + return Entry{}, &entryNotFoundError{} + } + e := Entry{ + Size: res.Size, + } + if res.Time != nil { + e.Time = *res.Time + } else { + e.Time = time.Now() + } + if res.DiskPath == "" { + return Entry{}, &entryNotFoundError{fmt.Errorf("%s didn't populate DiskPath on get hit", envGolangciLintCacheProg)} + } + if copy(e.OutputID[:], res.OutputID) != len(res.OutputID) { + return Entry{}, &entryNotFoundError{errors.New("incomplete ProgResponse OutputID")} + } + c.noteOutputFile(e.OutputID, res.DiskPath) + return e, nil +} + +func (c *ProgCache) noteOutputFile(o OutputID, diskPath string) { + c.mu.Lock() + defer c.mu.Unlock() + c.outputFile[o] = diskPath +} + +func (c *ProgCache) OutputFile(o OutputID) string { + c.mu.Lock() + defer c.mu.Unlock() + return c.outputFile[o] +} + +func (c *ProgCache) Put(a ActionID, file io.ReadSeeker) (_ OutputID, size int64, _ error) { + // Compute output ID. + h := sha256.New() + if _, err := file.Seek(0, 0); err != nil { + return OutputID{}, 0, err + } + size, err := io.Copy(h, file) + if err != nil { + return OutputID{}, 0, err + } + var out OutputID + h.Sum(out[:0]) + + if _, err := file.Seek(0, 0); err != nil { + return OutputID{}, 0, err + } + + if !c.can[cmdPut] { + // Child is a read-only cache. Do nothing. + return out, size, nil + } + + res, err := c.send(c.ctx, &ProgRequest{ + Command: cmdPut, + ActionID: a[:], + ObjectID: out[:], + Body: file, + BodySize: size, + }) + if err != nil { + return OutputID{}, 0, err + } + if res.DiskPath == "" { + return OutputID{}, 0, fmt.Errorf("%s didn't return DiskPath in put response", envGolangciLintCacheProg) + } + c.noteOutputFile(out, res.DiskPath) + return out, size, err +} + +func (c *ProgCache) Close() error { + c.closing.Store(true) + var err error + + // First write a "close" message to the child so it can exit nicely + // and clean up if it wants. Only after that exchange do we cancel + // the context that kills the process. + if c.can[cmdClose] { + _, err = c.send(c.ctx, &ProgRequest{Command: cmdClose}) + } + c.ctxCancel() + <-c.readLoopDone + return err +} + +func (c *ProgCache) FuzzDir() string { + // TODO(bradfitz): figure out what to do here. For now just use the + // disk-based default. + return c.fuzzDirCache.FuzzDir() +} diff --git a/internal/cache/readme.md b/internal/cache/readme.md index b469711edd56..6a774536a610 100644 --- a/internal/cache/readme.md +++ b/internal/cache/readme.md @@ -1,18 +1,38 @@ # cache -Extracted from go/src/cmd/go/internal/cache/ -I don't know what version of Go this package was pulled from. +Extracted from `go/src/cmd/go/internal/cache/`. + +- sync with go1.23.2 +- sync with go1.22.8 +- sync with go1.21.13 +- sync with go1.20.14 +- sync with go1.19.13 +- sync with go1.18.10 +- sync with go1.17.13 +- sync with go1.16.15 +- sync with go1.15.15 +- sync with go1.14.15 + +# Previous History + +Based on the initial PR/commit the based in a mix between go1.12 and go1.13: +- cache.go (go1.13) +- cache_test.go (go1.12?) +- default.go (go1.12?) +- hash.go (go1.13 and go1.12 are identical) +- hash_test.go -> (go1.12?) Adapted for golangci-lint: -- https://github.com/golangci/golangci-lint/pull/699 -- https://github.com/golangci/golangci-lint/pull/779 -- https://github.com/golangci/golangci-lint/pull/788 -- https://github.com/golangci/golangci-lint/pull/808 -- https://github.com/golangci/golangci-lint/pull/1063 -- https://github.com/golangci/golangci-lint/pull/1070 -- https://github.com/golangci/golangci-lint/pull/1162 -- https://github.com/golangci/golangci-lint/pull/2318 -- https://github.com/golangci/golangci-lint/pull/2352 -- https://github.com/golangci/golangci-lint/pull/3012 -- https://github.com/golangci/golangci-lint/pull/3096 -- https://github.com/golangci/golangci-lint/pull/3204 +- https://github.com/golangci/golangci-lint/pull/699: initial code (contains modifications of the files) +- https://github.com/golangci/golangci-lint/pull/779: just a nolint (`cache.go`) +- https://github.com/golangci/golangci-lint/pull/788: only directory permissions changes (0777 -> 0744) (`cache.go`, `cache_test.go`, `default.go`) +- https://github.com/golangci/golangci-lint/pull/808: mainly related to logs and errors (`cache.go`, `default.go`, `hash.go`, `hash_test.go`) +- https://github.com/golangci/golangci-lint/pull/1063: `ioutil` -> `robustio` (`cache.go`) +- https://github.com/golangci/golangci-lint/pull/1070: add `t.Parallel()` inside `cache_test.go` +- https://github.com/golangci/golangci-lint/pull/1162: errors inside `cache.go` +- https://github.com/golangci/golangci-lint/pull/2318: `ioutil` -> `os` (`cache.go`, `cache_test.go`, `default.go`, `hash_test.go`) +- https://github.com/golangci/golangci-lint/pull/2352: Go doc typos +- https://github.com/golangci/golangci-lint/pull/3012: errors inside `cache.go` (`cache.go`, `default.go`) +- https://github.com/golangci/golangci-lint/pull/3196: constant for `GOLANGCI_LINT_CACHE` (`cache.go`) +- https://github.com/golangci/golangci-lint/pull/3204: add this file and `%w` in `fmt.Errorf` (`cache.go`) +- https://github.com/golangci/golangci-lint/pull/3604: remove `github.com/pkg/errors` (`cache.go`) diff --git a/internal/mmap/LICENSE b/internal/mmap/LICENSE new file mode 100644 index 000000000000..6a66aea5eafe --- /dev/null +++ b/internal/mmap/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/internal/mmap/mmap.go b/internal/mmap/mmap.go new file mode 100644 index 000000000000..fcbd3e08c1c5 --- /dev/null +++ b/internal/mmap/mmap.go @@ -0,0 +1,31 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This package is a lightly modified version of the mmap code +// in github.com/google/codesearch/index. + +// The mmap package provides an abstraction for memory mapping files +// on different platforms. +package mmap + +import ( + "os" +) + +// Data is mmap'ed read-only data from a file. +// The backing file is never closed, so Data +// remains valid for the lifetime of the process. +type Data struct { + f *os.File + Data []byte +} + +// Mmap maps the given file into memory. +func Mmap(file string) (Data, error) { + f, err := os.Open(file) + if err != nil { + return Data{}, err + } + return mmapFile(f) +} diff --git a/internal/mmap/mmap_other.go b/internal/mmap/mmap_other.go new file mode 100644 index 000000000000..4d2844fc3731 --- /dev/null +++ b/internal/mmap/mmap_other.go @@ -0,0 +1,21 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build (js && wasm) || wasip1 || plan9 + +package mmap + +import ( + "io" + "os" +) + +// mmapFile on other systems doesn't mmap the file. It just reads everything. +func mmapFile(f *os.File) (Data, error) { + b, err := io.ReadAll(f) + if err != nil { + return Data{}, err + } + return Data{f, b}, nil +} diff --git a/internal/mmap/mmap_unix.go b/internal/mmap/mmap_unix.go new file mode 100644 index 000000000000..5dce87236870 --- /dev/null +++ b/internal/mmap/mmap_unix.go @@ -0,0 +1,36 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix + +package mmap + +import ( + "fmt" + "io/fs" + "os" + "syscall" +) + +func mmapFile(f *os.File) (Data, error) { + st, err := f.Stat() + if err != nil { + return Data{}, err + } + size := st.Size() + pagesize := int64(os.Getpagesize()) + if int64(int(size+(pagesize-1))) != size+(pagesize-1) { + return Data{}, fmt.Errorf("%s: too large for mmap", f.Name()) + } + n := int(size) + if n == 0 { + return Data{f, nil}, nil + } + mmapLength := int(((size + pagesize - 1) / pagesize) * pagesize) // round up to page size + data, err := syscall.Mmap(int(f.Fd()), 0, mmapLength, syscall.PROT_READ, syscall.MAP_SHARED) + if err != nil { + return Data{}, &fs.PathError{Op: "mmap", Path: f.Name(), Err: err} + } + return Data{f, data[:n]}, nil +} diff --git a/internal/mmap/mmap_windows.go b/internal/mmap/mmap_windows.go new file mode 100644 index 000000000000..479ee307544f --- /dev/null +++ b/internal/mmap/mmap_windows.go @@ -0,0 +1,41 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mmap + +import ( + "fmt" + "os" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +func mmapFile(f *os.File) (Data, error) { + st, err := f.Stat() + if err != nil { + return Data{}, err + } + size := st.Size() + if size == 0 { + return Data{f, nil}, nil + } + h, err := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, 0, 0, nil) + if err != nil { + return Data{}, fmt.Errorf("CreateFileMapping %s: %w", f.Name(), err) + } + + addr, err := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, 0) + if err != nil { + return Data{}, fmt.Errorf("MapViewOfFile %s: %w", f.Name(), err) + } + var info windows.MemoryBasicInformation + err = windows.VirtualQuery(addr, &info, unsafe.Sizeof(info)) + if err != nil { + return Data{}, fmt.Errorf("VirtualQuery %s: %w", f.Name(), err) + } + data := unsafe.Slice((*byte)(unsafe.Pointer(addr)), int(info.RegionSize)) + return Data{f, data}, nil +} diff --git a/internal/mmap/readme.md b/internal/mmap/readme.md new file mode 100644 index 000000000000..b5ec2d36840d --- /dev/null +++ b/internal/mmap/readme.md @@ -0,0 +1,9 @@ +# mmap + +Extracted from `go/src/cmd/go/internal/mmap/` (related to `cache`). + +- sync with go1.23.2 +- sync with go1.22.8 +- sync with go1.21.13 +- sync with go1.20.14 +- sync with go1.19.13 diff --git a/internal/pkgcache/pkgcache.go b/internal/pkgcache/pkgcache.go index 3b3422eb7a8c..4a66ce5a8387 100644 --- a/internal/pkgcache/pkgcache.go +++ b/internal/pkgcache/pkgcache.go @@ -28,29 +28,28 @@ const ( // Cache is a per-package data cache. A cached data is invalidated when // package, or it's dependencies change. type Cache struct { - lowLevelCache *cache.Cache + lowLevelCache cache.Cache pkgHashes sync.Map sw *timeutils.Stopwatch - log logutils.Log // not used now, but may be needed for future debugging purposes + log logutils.Log ioSem chan struct{} // semaphore limiting parallel IO } func NewCache(sw *timeutils.Stopwatch, log logutils.Log) (*Cache, error) { - c, err := cache.Default() - if err != nil { - return nil, err - } return &Cache{ - lowLevelCache: c, + lowLevelCache: cache.Default(), sw: sw, log: log, ioSem: make(chan struct{}, runtime.GOMAXPROCS(-1)), }, nil } -func (c *Cache) Trim() { - c.sw.TrackStage("trim", func() { - c.lowLevelCache.Trim() +func (c *Cache) Close() { + c.sw.TrackStage("close", func() { + err := c.lowLevelCache.Close() + if err != nil { + c.log.Errorf("cache close: %v", err) + } }) } @@ -81,7 +80,7 @@ func (c *Cache) Put(pkg *packages.Package, mode HashMode, key string, data any) } c.ioSem <- struct{}{} c.sw.TrackStage("cache io", func() { - err = c.lowLevelCache.PutBytes(aID, buf.Bytes()) + err = cache.PutBytes(c.lowLevelCache, aID, buf.Bytes()) }) <-c.ioSem if err != nil { @@ -113,7 +112,7 @@ func (c *Cache) Get(pkg *packages.Package, mode HashMode, key string, data any) var b []byte c.ioSem <- struct{}{} c.sw.TrackStage("cache io", func() { - b, _, err = c.lowLevelCache.GetBytes(aID) + b, _, err = cache.GetBytes(c.lowLevelCache, aID) }) <-c.ioSem if err != nil { diff --git a/internal/quoted/LICENSE b/internal/quoted/LICENSE new file mode 100644 index 000000000000..6a66aea5eafe --- /dev/null +++ b/internal/quoted/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/internal/quoted/quoted.go b/internal/quoted/quoted.go new file mode 100644 index 000000000000..a81227507353 --- /dev/null +++ b/internal/quoted/quoted.go @@ -0,0 +1,129 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package quoted provides string manipulation utilities. +package quoted + +import ( + "flag" + "fmt" + "strings" + "unicode" +) + +func isSpaceByte(c byte) bool { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' +} + +// Split splits s into a list of fields, +// allowing single or double quotes around elements. +// There is no unescaping or other processing within +// quoted fields. +// +// Keep in sync with cmd/dist/quoted.go +func Split(s string) ([]string, error) { + // Split fields allowing '' or "" around elements. + // Quotes further inside the string do not count. + var f []string + for len(s) > 0 { + for len(s) > 0 && isSpaceByte(s[0]) { + s = s[1:] + } + if len(s) == 0 { + break + } + // Accepted quoted string. No unescaping inside. + if s[0] == '"' || s[0] == '\'' { + quote := s[0] + s = s[1:] + i := 0 + for i < len(s) && s[i] != quote { + i++ + } + if i >= len(s) { + return nil, fmt.Errorf("unterminated %c string", quote) + } + f = append(f, s[:i]) + s = s[i+1:] + continue + } + i := 0 + for i < len(s) && !isSpaceByte(s[i]) { + i++ + } + f = append(f, s[:i]) + s = s[i:] + } + return f, nil +} + +// Join joins a list of arguments into a string that can be parsed +// with Split. Arguments are quoted only if necessary; arguments +// without spaces or quotes are kept as-is. No argument may contain both +// single and double quotes. +func Join(args []string) (string, error) { + var buf []byte + for i, arg := range args { + if i > 0 { + buf = append(buf, ' ') + } + var sawSpace, sawSingleQuote, sawDoubleQuote bool + for _, c := range arg { + switch { + case c > unicode.MaxASCII: + continue + case isSpaceByte(byte(c)): + sawSpace = true + case c == '\'': + sawSingleQuote = true + case c == '"': + sawDoubleQuote = true + } + } + switch { + case !sawSpace && !sawSingleQuote && !sawDoubleQuote: + buf = append(buf, arg...) + + case !sawSingleQuote: + buf = append(buf, '\'') + buf = append(buf, arg...) + buf = append(buf, '\'') + + case !sawDoubleQuote: + buf = append(buf, '"') + buf = append(buf, arg...) + buf = append(buf, '"') + + default: + return "", fmt.Errorf("argument %q contains both single and double quotes and cannot be quoted", arg) + } + } + return string(buf), nil +} + +// A Flag parses a list of string arguments encoded with Join. +// It is useful for flags like cmd/link's -extldflags. +type Flag []string + +var _ flag.Value = (*Flag)(nil) + +func (f *Flag) Set(v string) error { + fs, err := Split(v) + if err != nil { + return err + } + *f = fs[:len(fs):len(fs)] + return nil +} + +func (f *Flag) String() string { + if f == nil { + return "" + } + s, err := Join(*f) + if err != nil { + return strings.Join(*f, " ") + } + return s +} diff --git a/internal/quoted/quoted_test.go b/internal/quoted/quoted_test.go new file mode 100644 index 000000000000..d76270c87b49 --- /dev/null +++ b/internal/quoted/quoted_test.go @@ -0,0 +1,88 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package quoted + +import ( + "reflect" + "strings" + "testing" +) + +func TestSplit(t *testing.T) { + for _, test := range []struct { + name string + value string + want []string + wantErr string + }{ + {name: "empty", value: "", want: nil}, + {name: "space", value: " ", want: nil}, + {name: "one", value: "a", want: []string{"a"}}, + {name: "leading_space", value: " a", want: []string{"a"}}, + {name: "trailing_space", value: "a ", want: []string{"a"}}, + {name: "two", value: "a b", want: []string{"a", "b"}}, + {name: "two_multi_space", value: "a b", want: []string{"a", "b"}}, + {name: "two_tab", value: "a\tb", want: []string{"a", "b"}}, + {name: "two_newline", value: "a\nb", want: []string{"a", "b"}}, + {name: "quote_single", value: `'a b'`, want: []string{"a b"}}, + {name: "quote_double", value: `"a b"`, want: []string{"a b"}}, + {name: "quote_both", value: `'a '"b "`, want: []string{"a ", "b "}}, + {name: "quote_contains", value: `'a "'"'b"`, want: []string{`a "`, `'b`}}, + {name: "escape", value: `\'`, want: []string{`\'`}}, + {name: "quote_unclosed", value: `'a`, wantErr: "unterminated ' string"}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := Split(test.value) + if err != nil { + if test.wantErr == "" { + t.Fatalf("unexpected error: %v", err) + } else if errMsg := err.Error(); !strings.Contains(errMsg, test.wantErr) { + t.Fatalf("error %q does not contain %q", errMsg, test.wantErr) + } + return + } + if test.wantErr != "" { + t.Fatalf("unexpected success; wanted error containing %q", test.wantErr) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("got %q; want %q", got, test.want) + } + }) + } +} + +func TestJoin(t *testing.T) { + for _, test := range []struct { + name string + args []string + want, wantErr string + }{ + {name: "empty", args: nil, want: ""}, + {name: "one", args: []string{"a"}, want: "a"}, + {name: "two", args: []string{"a", "b"}, want: "a b"}, + {name: "space", args: []string{"a ", "b"}, want: "'a ' b"}, + {name: "newline", args: []string{"a\n", "b"}, want: "'a\n' b"}, + {name: "quote", args: []string{`'a `, "b"}, want: `"'a " b`}, + {name: "unquoteable", args: []string{`'"`}, wantErr: "contains both single and double quotes and cannot be quoted"}, + } { + t.Run(test.name, func(t *testing.T) { + got, err := Join(test.args) + if err != nil { + if test.wantErr == "" { + t.Fatalf("unexpected error: %v", err) + } else if errMsg := err.Error(); !strings.Contains(errMsg, test.wantErr) { + t.Fatalf("error %q does not contain %q", errMsg, test.wantErr) + } + return + } + if test.wantErr != "" { + t.Fatalf("unexpected success; wanted error containing %q", test.wantErr) + } + if got != test.want { + t.Errorf("got %s; want %s", got, test.want) + } + }) + } +} diff --git a/internal/quoted/readme.md b/internal/quoted/readme.md new file mode 100644 index 000000000000..a3c33d624513 --- /dev/null +++ b/internal/quoted/readme.md @@ -0,0 +1,7 @@ +# quoted + +Extracted from `go/src/cmd/internal/quoted/` (related to `cache`). + +- sync go1.23.2 +- sync go1.22.8 +- sync go1.21.13 diff --git a/internal/renameio/readme.md b/internal/renameio/readme.md deleted file mode 100644 index 36ec6ed499d3..000000000000 --- a/internal/renameio/readme.md +++ /dev/null @@ -1,10 +0,0 @@ -# renameio - -Extracted from go/src/cmd/go/internal/renameio/ -I don't know what version of Go this package was pulled from. - -Adapted for golangci-lint: -- https://github.com/golangci/golangci-lint/pull/699 -- https://github.com/golangci/golangci-lint/pull/808 -- https://github.com/golangci/golangci-lint/pull/1063 -- https://github.com/golangci/golangci-lint/pull/3204 diff --git a/internal/renameio/renameio.go b/internal/renameio/renameio.go deleted file mode 100644 index 2f88f4f7cc5a..000000000000 --- a/internal/renameio/renameio.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package renameio writes files atomically by renaming temporary files. -package renameio - -import ( - "bytes" - "io" - "math/rand" - "os" - "path/filepath" - "strconv" - - "github.com/golangci/golangci-lint/internal/robustio" -) - -const patternSuffix = ".tmp" - -// Pattern returns a glob pattern that matches the unrenamed temporary files -// created when writing to filename. -func Pattern(filename string) string { - return filepath.Join(filepath.Dir(filename), filepath.Base(filename)+patternSuffix) -} - -// WriteFile is like os.WriteFile, but first writes data to an arbitrary -// file in the same directory as filename, then renames it atomically to the -// final name. -// -// That ensures that the final location, if it exists, is always a complete file. -func WriteFile(filename string, data []byte, perm os.FileMode) (err error) { - return WriteToFile(filename, bytes.NewReader(data), perm) -} - -// WriteToFile is a variant of WriteFile that accepts the data as an io.Reader -// instead of a slice. -func WriteToFile(filename string, data io.Reader, perm os.FileMode) (err error) { - f, err := tempFile(filepath.Dir(filename), filepath.Base(filename), perm) - if err != nil { - return err - } - defer func() { - // Only call os.Remove on f.Name() if we failed to rename it: otherwise, - // some other process may have created a new file with the same name after - // that. - if err != nil { - f.Close() - os.Remove(f.Name()) - } - }() - - if _, err := io.Copy(f, data); err != nil { - return err - } - // Sync the file before renaming it: otherwise, after a crash the reader may - // observe a 0-length file instead of the actual contents. - // See https://golang.org/issue/22397#issuecomment-380831736. - if err := f.Sync(); err != nil { - return err - } - if err := f.Close(); err != nil { - return err - } - - return robustio.Rename(f.Name(), filename) -} - -// tempFile creates a new temporary file with given permission bits. -func tempFile(dir, prefix string, perm os.FileMode) (f *os.File, err error) { - for i := 0; i < 10000; i++ { - name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+patternSuffix) - f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm) - if os.IsExist(err) { - continue - } - break - } - return -} - -// ReadFile is like os.ReadFile, but on Windows retries spurious errors that -// may occur if the file is concurrently replaced. -// -// Errors are classified heuristically and retries are bounded, so even this -// function may occasionally return a spurious error on Windows. -// If so, the error will likely wrap one of: -// - syscall.ERROR_ACCESS_DENIED -// - syscall.ERROR_FILE_NOT_FOUND -// - internal/syscall/windows.ERROR_SHARING_VIOLATION -func ReadFile(filename string) ([]byte, error) { - return robustio.ReadFile(filename) -} diff --git a/internal/renameio/renameio_test.go b/internal/renameio/renameio_test.go deleted file mode 100644 index 0f8dd3489760..000000000000 --- a/internal/renameio/renameio_test.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !plan9 - -package renameio - -import ( - "encoding/binary" - "math/rand" - "os" - "path/filepath" - "runtime" - "sync" - "sync/atomic" - "syscall" - "testing" - "time" - - "github.com/golangci/golangci-lint/internal/robustio" -) - -func TestConcurrentReadsAndWrites(t *testing.T) { - dir, err := os.MkdirTemp("", "renameio") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - path := filepath.Join(dir, "blob.bin") - - const chunkWords = 8 << 10 - buf := make([]byte, 2*chunkWords*8) - for i := uint64(0); i < 2*chunkWords; i++ { - binary.LittleEndian.PutUint64(buf[i*8:], i) - } - - var attempts int64 = 128 - if !testing.Short() { - attempts *= 16 - } - const parallel = 32 - - var sem = make(chan bool, parallel) - - var ( - writeSuccesses, readSuccesses int64 // atomic - writeErrnoSeen, readErrnoSeen sync.Map - ) - - for n := attempts; n > 0; n-- { - sem <- true - go func() { - defer func() { <-sem }() - - time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond) - offset := rand.Intn(chunkWords) - chunk := buf[offset*8 : (offset+chunkWords)*8] - if err := WriteFile(path, chunk, 0666); err == nil { - atomic.AddInt64(&writeSuccesses, 1) - } else if robustio.IsEphemeralError(err) { - var ( - dup bool - ) - if errno, ok := err.(syscall.Errno); ok { - _, dup = writeErrnoSeen.LoadOrStore(errno, true) - } - if !dup { - t.Logf("ephemeral error: %v", err) - } - } else { - t.Errorf("unexpected error: %v", err) - } - - time.Sleep(time.Duration(rand.Intn(100)) * time.Microsecond) - data, err := ReadFile(path) - if err == nil { - atomic.AddInt64(&readSuccesses, 1) - } else if robustio.IsEphemeralError(err) { - var ( - dup bool - ) - if errno, ok := err.(syscall.Errno); ok { - _, dup = readErrnoSeen.LoadOrStore(errno, true) - } - if !dup { - t.Logf("ephemeral error: %v", err) - } - return - } else { - t.Errorf("unexpected error: %v", err) - return - } - - if len(data) != 8*chunkWords { - t.Errorf("read %d bytes, but each write is a %d-byte file", len(data), 8*chunkWords) - return - } - - u := binary.LittleEndian.Uint64(data) - for i := 1; i < chunkWords; i++ { - next := binary.LittleEndian.Uint64(data[i*8:]) - if next != u+1 { - t.Errorf("wrote sequential integers, but read integer out of sequence at offset %d", i) - return - } - u = next - } - }() - } - - for n := parallel; n > 0; n-- { - sem <- true - } - - var minWriteSuccesses int64 = attempts - if runtime.GOOS == "windows" { - // Windows produces frequent "Access is denied" errors under heavy rename load. - // As long as those are the only errors and *some* of the writes succeed, we're happy. - minWriteSuccesses = attempts / 4 - } - - if writeSuccesses < minWriteSuccesses { - t.Errorf("%d (of %d) writes succeeded; want ≥ %d", writeSuccesses, attempts, minWriteSuccesses) - } else { - t.Logf("%d (of %d) writes succeeded (ok: ≥ %d)", writeSuccesses, attempts, minWriteSuccesses) - } - - var minReadSuccesses int64 = attempts - - switch runtime.GOOS { - case "windows": - // Windows produces frequent "Access is denied" errors under heavy rename load. - // As long as those are the only errors and *some* of the reads succeed, we're happy. - minReadSuccesses = attempts / 4 - - case "darwin": - // The filesystem on macOS 10.14 occasionally fails with "no such file or - // directory" errors. See https://golang.org/issue/33041. The flake rate is - // fairly low, so ensure that at least 75% of attempts succeed. - minReadSuccesses = attempts - (attempts / 4) - } - - if readSuccesses < minReadSuccesses { - t.Errorf("%d (of %d) reads succeeded; want ≥ %d", readSuccesses, attempts, minReadSuccesses) - } else { - t.Logf("%d (of %d) reads succeeded (ok: ≥ %d)", readSuccesses, attempts, minReadSuccesses) - } -} diff --git a/internal/renameio/umask_test.go b/internal/renameio/umask_test.go deleted file mode 100644 index 3f1795fbb0ee..000000000000 --- a/internal/renameio/umask_test.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !plan9 && !windows && !js - -package renameio - -import ( - "os" - "path/filepath" - "syscall" - "testing" -) - -func TestWriteFileModeAppliesUmask(t *testing.T) { - dir, err := os.MkdirTemp("", "renameio") - if err != nil { - t.Fatalf("Failed to create temporary directory: %v", err) - } - defer os.RemoveAll(dir) - - const mode = 0644 - const umask = 0007 - defer syscall.Umask(syscall.Umask(umask)) - - file := filepath.Join(dir, "testWrite") - err = WriteFile(file, []byte("go-build"), mode) - if err != nil { - t.Fatalf("Failed to write file: %v", err) - } - - fi, err := os.Stat(file) - if err != nil { - t.Fatalf("Stat %q (looking for mode %#o): %s", file, mode, err) - } - - if fi.Mode()&os.ModePerm != 0640 { - t.Errorf("Stat %q: mode %#o want %#o", file, fi.Mode()&os.ModePerm, 0640) - } -} diff --git a/internal/robustio/LICENSE b/internal/robustio/LICENSE new file mode 100644 index 000000000000..6a66aea5eafe --- /dev/null +++ b/internal/robustio/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/internal/testenv/LICENSE b/internal/testenv/LICENSE new file mode 100644 index 000000000000..6a66aea5eafe --- /dev/null +++ b/internal/testenv/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/internal/testenv/readme.md b/internal/testenv/readme.md new file mode 100644 index 000000000000..528890c26fa6 --- /dev/null +++ b/internal/testenv/readme.md @@ -0,0 +1,9 @@ +# testenv + +Extracted from `go/src/internal/testenv/`. + +Only the function `SyscallIsNotSupported` is extracted (related to `cache`). + +- sync with go1.23.2 +- sync with go1.22.8 +- sync with go1.21.13 diff --git a/internal/testenv/testenv.go b/internal/testenv/testenv.go new file mode 100644 index 000000000000..d1b5b7f04c66 --- /dev/null +++ b/internal/testenv/testenv.go @@ -0,0 +1,17 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package testenv provides information about what functionality +// is available in different testing environments run by the Go team. +// +// It is an internal package because these details are specific +// to the Go team's test setup (on build.golang.org) and not +// fundamental to tests in general. +package testenv + +// SyscallIsNotSupported reports whether err may indicate that a system call is +// not supported by the current platform or execution environment. +func SyscallIsNotSupported(err error) bool { + return syscallIsNotSupported(err) +} diff --git a/internal/testenv/testenv_notunix.go b/internal/testenv/testenv_notunix.go new file mode 100644 index 000000000000..a7df5f5ddcec --- /dev/null +++ b/internal/testenv/testenv_notunix.go @@ -0,0 +1,21 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build windows || plan9 || (js && wasm) || wasip1 + +package testenv + +import ( + "errors" + "io/fs" + "os" +) + +// Sigquit is the signal to send to kill a hanging subprocess. +// On Unix we send SIGQUIT, but on non-Unix we only have os.Kill. +var Sigquit = os.Kill + +func syscallIsNotSupported(err error) bool { + return errors.Is(err, fs.ErrPermission) || errors.Is(err, errors.ErrUnsupported) +} diff --git a/internal/testenv/testenv_notwin.go b/internal/testenv/testenv_notwin.go new file mode 100644 index 000000000000..30e159a6ecd4 --- /dev/null +++ b/internal/testenv/testenv_notwin.go @@ -0,0 +1,46 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !windows + +package testenv + +import ( + "fmt" + "os" + "path/filepath" + "runtime" +) + +func hasSymlink() (ok bool, reason string) { + switch runtime.GOOS { + case "plan9": + return false, "" + case "android", "wasip1": + // For wasip1, some runtimes forbid absolute symlinks, + // or symlinks that escape the current working directory. + // Perform a simple test to see whether the runtime + // supports symlinks or not. If we get a permission + // error, the runtime does not support symlinks. + dir, err := os.MkdirTemp("", "") + if err != nil { + return false, "" + } + defer func() { + _ = os.RemoveAll(dir) + }() + fpath := filepath.Join(dir, "testfile.txt") + if err := os.WriteFile(fpath, nil, 0644); err != nil { + return false, "" + } + if err := os.Symlink(fpath, filepath.Join(dir, "testlink")); err != nil { + if SyscallIsNotSupported(err) { + return false, fmt.Sprintf("symlinks unsupported: %s", err.Error()) + } + return false, "" + } + } + + return true, "" +} diff --git a/internal/testenv/testenv_unix.go b/internal/testenv/testenv_unix.go new file mode 100644 index 000000000000..a629078842ea --- /dev/null +++ b/internal/testenv/testenv_unix.go @@ -0,0 +1,43 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build unix + +package testenv + +import ( + "errors" + "io/fs" + "syscall" +) + +// Sigquit is the signal to send to kill a hanging subprocess. +// Send SIGQUIT to get a stack trace. +var Sigquit = syscall.SIGQUIT + +func syscallIsNotSupported(err error) bool { + if err == nil { + return false + } + + var errno syscall.Errno + if errors.As(err, &errno) { + switch errno { + case syscall.EPERM, syscall.EROFS: + // User lacks permission: either the call requires root permission and the + // user is not root, or the call is denied by a container security policy. + return true + case syscall.EINVAL: + // Some containers return EINVAL instead of EPERM if a system call is + // denied by security policy. + return true + } + } + + if errors.Is(err, fs.ErrPermission) || errors.Is(err, errors.ErrUnsupported) { + return true + } + + return false +} diff --git a/internal/testenv/testenv_windows.go b/internal/testenv/testenv_windows.go new file mode 100644 index 000000000000..4802b139518e --- /dev/null +++ b/internal/testenv/testenv_windows.go @@ -0,0 +1,47 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testenv + +import ( + "os" + "path/filepath" + "sync" + "syscall" +) + +var symlinkOnce sync.Once +var winSymlinkErr error + +func initWinHasSymlink() { + tmpdir, err := os.MkdirTemp("", "symtest") + if err != nil { + panic("failed to create temp directory: " + err.Error()) + } + defer os.RemoveAll(tmpdir) + + err = os.Symlink("target", filepath.Join(tmpdir, "symlink")) + if err != nil { + err = err.(*os.LinkError).Err + switch err { + case syscall.EWINDOWS, syscall.ERROR_PRIVILEGE_NOT_HELD: + winSymlinkErr = err + } + } +} + +func hasSymlink() (ok bool, reason string) { + symlinkOnce.Do(initWinHasSymlink) + + switch winSymlinkErr { + case nil: + return true, "" + case syscall.EWINDOWS: + return false, ": symlinks are not supported on your version of Windows" + case syscall.ERROR_PRIVILEGE_NOT_HELD: + return false, ": you don't have enough privileges to create symlinks" + } + + return false, "" +} diff --git a/pkg/commands/cache.go b/pkg/commands/cache.go index cc6c0eacd557..0af8cc2856e5 100644 --- a/pkg/commands/cache.go +++ b/pkg/commands/cache.go @@ -51,7 +51,7 @@ func newCacheCommand() *cacheCommand { } func (*cacheCommand) executeClean(_ *cobra.Command, _ []string) error { - cacheDir := cache.DefaultDir() + cacheDir, _ := cache.DefaultDir() if err := os.RemoveAll(cacheDir); err != nil { return fmt.Errorf("failed to remove dir %s: %w", cacheDir, err) @@ -61,7 +61,8 @@ func (*cacheCommand) executeClean(_ *cobra.Command, _ []string) error { } func (*cacheCommand) executeStatus(_ *cobra.Command, _ []string) { - cacheDir := cache.DefaultDir() + cacheDir, _ := cache.DefaultDir() + _, _ = fmt.Fprintf(logutils.StdOut, "Dir: %s\n", cacheDir) cacheSizeBytes, err := dirSizeBytes(cacheDir) diff --git a/pkg/goanalysis/runner.go b/pkg/goanalysis/runner.go index c1274ec09a7c..806aac81064c 100644 --- a/pkg/goanalysis/runner.go +++ b/pkg/goanalysis/runner.go @@ -84,7 +84,6 @@ func (r *runner) run(analyzers []*analysis.Analyzer, initialPackages []*packages []error, map[*analysis.Pass]*packages.Package, ) { debugf("Analyzing %d packages on load mode %s", len(initialPackages), r.loadMode) - defer r.pkgCache.Trim() roots := r.analyze(initialPackages, analyzers) diff --git a/pkg/goanalysis/runners.go b/pkg/goanalysis/runners.go index 79e52f52a3e0..698439f08196 100644 --- a/pkg/goanalysis/runners.go +++ b/pkg/goanalysis/runners.go @@ -182,6 +182,8 @@ func saveIssuesToCache(allPkgs []*packages.Package, pkgsFromCache map[*packages. close(pkgCh) wg.Wait() + lintCtx.PkgCache.Close() + issuesCacheDebugf("Saved %d issues from %d packages to cache in %s", savedIssuesCount, len(allPkgs), time.Since(startedAt)) } diff --git a/scripts/website/expand_templates/main.go b/scripts/website/expand_templates/main.go index 24e9e847c4bf..9a36acb1dcd2 100644 --- a/scripts/website/expand_templates/main.go +++ b/scripts/website/expand_templates/main.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "encoding/json" "fmt" "log" @@ -8,7 +9,8 @@ import ( "path/filepath" "strings" - "github.com/golangci/golangci-lint/internal/renameio" + "github.com/rogpeppe/go-internal/lockedfile" + "github.com/golangci/golangci-lint/scripts/website/github" "github.com/golangci/golangci-lint/scripts/website/types" ) @@ -80,7 +82,7 @@ func processDoc(path string, replacements map[string]string, madeReplacements ma } log.Printf("Expanded template in %s, saving it", path) - if err = renameio.WriteFile(path, []byte(content), os.ModePerm); err != nil { + if err = lockedfile.Write(path, bytes.NewBufferString(content), os.ModePerm); err != nil { return fmt.Errorf("write changes to file %s: %w", path, err) }