Sync upstream changes (#15)

Notably includes golang/go#66869 and `RemoveInsecurePaths()` for golang/go#55356

.github/workflows/go.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.16.x,1.17.x,1.18.x]
+        go-version: [1.21.x,1.22.x]
         os: [ubuntu-latest]
     steps:
       - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }}
@@ -30,6 +30,6 @@ jobs:
         env:
           GO111MODULE: on
         run: |
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.45.0
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.59.0
           $(go env GOPATH)/bin/golangci-lint run --timeout=5m --config ./.golangci.yml
           go test -race ./...

.github/workflows/vulncheck.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go-version: [ 1.19 ]
+        go-version: [ 1.22.x ]
     steps:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v3

.golangci.yml

@@ -12,11 +12,10 @@ linters:
     - goimports
     - misspell
     - govet
-    - golint
     - ineffassign
     - gosimple
-    - deadcode
-    - structcheck
+    - staticcheck
+    - unused
 
 issues:
   exclude-use-default: false
@@ -26,4 +25,4 @@ issues:
       - should have comment or be unexported
       - error strings should not be capitalized or end with punctuation or a newline
 service:
-  golangci-lint-version: 1.20.0 # use the fixed version to not introduce new linters unexpectedly
+  golangci-lint-version: 1.59.0 # use the fixed version to not introduce new linters unexpectedly

example_test.go

@@ -21,15 +21,14 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 
 	"github.com/minio/zipindex"
 )
 
 func ExampleReadDir() {
-	b, err := ioutil.ReadFile("testdata/big.zip")
+	b, err := os.ReadFile("testdata/big.zip")
 	if err != nil {
 		panic(err)
 	}
@@ -160,7 +159,7 @@ func ExampleDeserializeFiles() {
 		}
 	}
 
-	b, err := ioutil.ReadFile("testdata/big.zip")
+	b, err := os.ReadFile("testdata/big.zip")
 	exitOnErr(err)
 	// We only need the end of the file to parse the directory.
 	// Usually this should be at least 64K on initial try.
@@ -212,7 +211,7 @@ func ExampleDeserializeFiles() {
 	defer rc.Close()
 
 	// Read the zip file content.
-	content, err := ioutil.ReadAll(rc)
+	content, err := io.ReadAll(rc)
 	exitOnErr(err)
 
 	fmt.Printf("File content is '%s'\n", string(content))

file.go

@@ -22,7 +22,9 @@ import (
 	"fmt"
 	"hash/crc32"
 	"io"
+	"path/filepath"
 	"sort"
+	"strings"
 
 	"github.com/klauspost/compress/zstd"
 	"github.com/tinylib/msgp/msgp"
@@ -31,6 +33,7 @@ import (
 //go:generate msgp -file $GOFILE -unexported
 
 // File is a sparse representation of a File inside a zip file.
+//
 //msgp:tuple File
 type File struct {
 	Name               string // Name of the file as stored in the zip.
@@ -77,6 +80,7 @@ func (f *File) OpenRaw(r io.Reader) (io.Reader, error) {
 }
 
 // Files is a collection of files.
+//
 //msgp:ignore Files
 type Files []File
 
@@ -174,6 +178,24 @@ func (f Files) Serialize() ([]byte, error) {
 	return zstdEnc.EncodeAll(payload, res), nil
 }
 
+// RemoveInsecurePaths will remove any file with path deemed insecure.
+// This is files that fail either !filepath.IsLocal(file.Name) or contain a backslash.
+func (f *Files) RemoveInsecurePaths() {
+	files := *f
+	for i, file := range files {
+		if file.Name == "" {
+			// Zip permits an empty file name field.
+			continue
+		}
+		// The zip specification states that names must use forward slashes,
+		// so consider any backslashes in the name insecure.
+		if !filepath.IsLocal(file.Name) || strings.Contains(file.Name, `\`) {
+			files = append(files[:i], files[:i+1]...)
+		}
+	}
+	*f = files
+}
+
 // Sort files by offset in zip file.
 // Typically, directories are already sorted by offset.
 // This will usually provide the smallest possible serialized size.

fuzz_test.go

@@ -28,7 +28,6 @@ import (
 	"go/parser"
 	"go/token"
 	"io"
-	"io/ioutil"
 	"os"
 	"strconv"
 	"testing"
@@ -96,7 +95,7 @@ func FuzzRoundtrip(f *testing.F) {
 			defer rc.Close()
 
 			// Read the zip file content.
-			ioutil.ReadAll(rc)
+			io.ReadAll(rc)
 		}
 	})
 }

go.mod

@@ -1,8 +1,10 @@
 module github.com/minio/zipindex
 
-go 1.16
+go 1.20
 
 require (
-	github.com/klauspost/compress v1.15.9
-	github.com/tinylib/msgp v1.1.6
+	github.com/klauspost/compress v1.17.8
+	github.com/tinylib/msgp v1.1.9
 )
+
+require github.com/philhofer/fwd v1.1.2 // indirect

go.sum

@@ -1,27 +1,6 @@
-github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
-github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
-github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
-github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw=
-github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
+github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
+github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU=
+github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k=

reader.go

@@ -29,7 +29,6 @@ import (
 	"fmt"
 	"hash"
 	"io"
-	"io/ioutil"
 	"os"
 	"time"
 	"unicode/utf8"
@@ -206,7 +205,7 @@ func ReadFile(name string, filter FileFilter) (Files, error) {
 		if err != nil {
 			return nil, err
 		}
-		b, err := ioutil.ReadAll(f)
+		b, err := io.ReadAll(f)
 		if err != nil {
 			return nil, err
 		}
@@ -254,7 +253,7 @@ func (r *checksumReader) Read(b []byte) (n int, err error) {
 
 		if r.f.hasDataDescriptor() {
 			// If any compressed data remains, read it.
-			io.Copy(ioutil.Discard, r.compReader)
+			io.Copy(io.Discard, r.compReader)
 
 			if err1 := readDataDescriptor(r.raw, r.f); err1 != nil {
 				if err1 == io.EOF {
@@ -299,7 +298,7 @@ func (f *File) skipToBody(r io.Reader) error {
 	filenameLen := int(b.uint16())
 	extraLen := int(b.uint16())
 	// Skip extra...
-	_, err := io.CopyN(ioutil.Discard, r, int64(filenameLen+extraLen))
+	_, err := io.CopyN(io.Discard, r, int64(filenameLen+extraLen))
 	return err
 }
 
@@ -558,6 +557,11 @@ func readDirectoryEnd(buf []byte, size int64) (dir *directoryEnd, err error) {
 			return nil, err
 		}
 	}
+	maxInt64 := uint64(1<<63 - 1)
+	if d.directorySize > maxInt64 || d.directoryOffset > maxInt64 {
+		return nil, ErrFormat
+	}
+
 	// Make sure directoryOffset points to somewhere in our file.
 	if o := int64(d.directoryOffset); o < 0 || o >= size {
 		return nil, ErrFormat
@@ -617,9 +621,13 @@ func findSignatureInBlock(b []byte) int {
 		if b[i] == 'P' && b[i+1] == 'K' && b[i+2] == 0x05 && b[i+3] == 0x06 {
 			// n is length of comment
 			n := int(b[i+directoryEndLen-2]) | int(b[i+directoryEndLen-1])<<8
-			if n+directoryEndLen+i <= len(b) {
-				return i
+			if n+directoryEndLen+i > len(b) {
+				// Truncated comment.
+				// Some parsers (such as Info-ZIP) ignore the truncated comment
+				// rather than treating it as a hard error.
+				return -1
 			}
+			return i
 		}
 	}
 	return -1

register.go

@@ -25,7 +25,6 @@ package zipindex
 import (
 	"errors"
 	"io"
-	"io/ioutil"
 	"sync"
 
 	"github.com/klauspost/compress/flate"
@@ -82,7 +81,7 @@ var (
 )
 
 func init() {
-	RegisterDecompressor(Store, ioutil.NopCloser)
+	RegisterDecompressor(Store, io.NopCloser)
 	RegisterDecompressor(Deflate, newFlateReader)
 	RegisterDecompressor(Zstd, zstd.ZipDecompressor(zstd.WithDecoderLowmem(true)))
 }

struct.go

@@ -58,9 +58,6 @@ const (
 
 	// Version numbers.
 
-	// Limits for non zip64 files.
-	uint32max = (1 << 32) - 1
-
 	// Extra header IDs.
 	//
 	// IDs 0..31 are reserved for official use by PKWARE.
@@ -199,11 +196,6 @@ func (h *ZipDirEntry) Mode() (mode os.FileMode) {
 	return mode
 }
 
-// isZip64 reports whether the file size exceeds the 32 bit limit
-func (h *ZipDirEntry) isZip64() bool {
-	return h.CompressedSize64 >= uint32max || h.UncompressedSize64 >= uint32max
-}
-
 func msdosModeToFileMode(m uint32) (mode os.FileMode) {
 	if m&msdosDir != 0 {
 		mode = os.ModeDir | 0777

zipindex_test.go

@@ -20,7 +20,8 @@ import (
 	"archive/zip"
 	"bytes"
 	"errors"
-	"io/ioutil"
+	"io"
+	"os"
 	"path/filepath"
 	"reflect"
 	"testing"
@@ -60,12 +61,14 @@ func TestReadDir(t *testing.T) {
 		"zip64-2.zip",
 		"smallish.zip",
 		"zstd-compressed.zip",
+		"test-badbase.zip",
+		"comment-truncated.zip",
 		"fuzz/FuzzDeserializeFiles.zip",
 		"fuzz/FuzzRoundtrip.zip",
 	}
 	for _, test := range testSet {
 		t.Run(test, func(t *testing.T) {
-			input, err := ioutil.ReadFile(filepath.Join("testdata", test))
+			input, err := os.ReadFile(filepath.Join("testdata", test))
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -105,6 +108,7 @@ func TestReadDir(t *testing.T) {
 				sz = int(more.FromEnd)
 			}
 			files.OptimizeSize()
+			files.RemoveInsecurePaths()
 			ser, err := files.Serialize()
 			if err != nil {
 				t.Errorf("unexpected error: %v", err)
@@ -158,8 +162,8 @@ func TestReadDir(t *testing.T) {
 						t.Errorf("err mismatch: %v != %v", wantErr, gotErr)
 					}
 				}()
-				wantData, wantErr := ioutil.ReadAll(wantRC)
-				gotData, err := ioutil.ReadAll(rc)
+				wantData, wantErr := io.ReadAll(wantRC)
+				gotData, err := io.ReadAll(rc)
 				if err != nil {
 					if err == wantErr {
 						continue