From e827102a417b0d920a3ab725dd668248e7cfe6a1 Mon Sep 17 00:00:00 2001 From: "Force.Charlie" Date: Sat, 13 Jun 2020 22:35:06 +0800 Subject: [PATCH 1/9] support decompress zip(zstd,lzma,bzip2) --- zip.go | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/zip.go b/zip.go index 192bf607..c57c1789 100644 --- a/zip.go +++ b/zip.go @@ -6,11 +6,25 @@ import ( "compress/flate" "fmt" "io" + "io/ioutil" "log" "os" "path" "path/filepath" "strings" + + "github.com/dsnet/compress/bzip2" + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz/lzma" +) + +// Compression methods. +const ( + Store uint16 = 0 // no compression + Deflate uint16 = 8 // DEFLATE compressed + BZIP2 uint16 = 12 // bzip2 + LZMA uint16 = 14 //LZMA + ZSTD uint16 = 20 //see https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT. ) // Zip provides facilities for operating ZIP archives. @@ -145,7 +159,6 @@ func (z *Zip) Unarchive(source, destination string) error { return fmt.Errorf("opening zip archive for reading: %v", err) } defer z.Close() - // if the files in the archive do not all share a common // root, then make sure we extract to a single subfolder // rather than potentially littering the destination... @@ -376,6 +389,28 @@ func (z *Zip) Open(in io.Reader, size int64) error { if err != nil { return fmt.Errorf("creating reader: %v", err) } + // register zstd decompressor + z.zr.RegisterDecompressor(ZSTD, func(r io.Reader) io.ReadCloser { + zr, err := zstd.NewReader(r) + if err != nil { + return nil + } + return zr.IOReadCloser() + }) + z.zr.RegisterDecompressor(BZIP2, func(r io.Reader) io.ReadCloser { + bz2r, err := bzip2.NewReader(r, nil) + if err != nil { + return nil + } + return bz2r + }) + z.zr.RegisterDecompressor(LZMA, func(r io.Reader) io.ReadCloser { + lr, err := lzma.NewReader(r) + if err != nil { + return nil + } + return ioutil.NopCloser(lr) + }) z.ridx = 0 return nil } From 086abb57b376a37997944fed7d0b1988ae890056 Mon Sep 17 00:00:00 2001 From: "Force.Charlie" Date: Sun, 14 Jun 2020 10:37:09 +0800 Subject: [PATCH 2/9] support compress use zstd/lzma/bzip2 --- zip.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/zip.go b/zip.go index c57c1789..c6773e97 100644 --- a/zip.go +++ b/zip.go @@ -69,9 +69,11 @@ type Zip struct { // the operation will continue on remaining files. ContinueOnError bool - zw *zip.Writer - zr *zip.Reader - ridx int + //Compression algorithm + CompressionAlgorithm uint16 + zw *zip.Writer + zr *zip.Reader + ridx int } // CheckExt ensures the file extension matches the format. @@ -305,6 +307,20 @@ func (z *Zip) Create(out io.Writer) error { return flate.NewWriter(out, z.CompressionLevel) }) } + switch z.CompressionAlgorithm { + case BZIP2: + z.zw.RegisterCompressor(BZIP2, func(out io.Writer) (io.WriteCloser, error) { + return bzip2.NewWriter(out, &bzip2.WriterConfig{Level: z.CompressionLevel}) + }) + case LZMA: + z.zw.RegisterCompressor(LZMA, func(out io.Writer) (io.WriteCloser, error) { + return lzma.NewWriter(out) + }) + case ZSTD: + z.zw.RegisterCompressor(ZSTD, func(out io.Writer) (io.WriteCloser, error) { + return zstd.NewWriter(out) + }) + } return nil } @@ -333,7 +349,11 @@ func (z *Zip) Write(f File) error { if _, ok := compressedFormats[ext]; ok && z.SelectiveCompression { header.Method = zip.Store } else { - header.Method = zip.Deflate + if z.CompressionAlgorithm > 0 { + header.Method = z.CompressionAlgorithm // zip or zstd + } else { + header.Method = zip.Deflate + } } } From 86c3e868b6bdc834687039e0efa1432d50539f4d Mon Sep 17 00:00:00 2001 From: "Force.Charlie" Date: Sun, 14 Jun 2020 17:02:39 +0800 Subject: [PATCH 3/9] Update code --- go.mod | 10 +++++----- go.sum | 18 ++++++++++-------- zip.go | 17 +++++++---------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 67cb65c3..1ebdcdee 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,10 @@ require ( github.com/andybalholm/brotli v1.0.0 github.com/dsnet/compress v0.0.1 github.com/golang/snappy v0.0.1 - github.com/klauspost/compress v1.10.3 - github.com/klauspost/pgzip v1.2.2 - github.com/nwaples/rardecode v1.0.0 - github.com/pierrec/lz4/v3 v3.0.1 - github.com/ulikunitz/xz v0.5.6 + github.com/klauspost/compress v1.10.8 + github.com/klauspost/pgzip v1.2.4 + github.com/nwaples/rardecode v1.1.0 + github.com/pierrec/lz4/v3 v3.3.2 + github.com/ulikunitz/xz v0.5.7 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 ) diff --git a/go.sum b/go.sum index 23925155..6f4cf637 100644 --- a/go.sum +++ b/go.sum @@ -16,32 +16,34 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.3 h1:OP96hzwJVBIHYU52pVTI6CczrxPvrGfgqF9N5eTO0Q8= -github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.8 h1:eLeJ3dr/Y9+XRfJT4l+8ZjmtB5RPJhucH2HeCV5+IZY= +github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.2 h1:8d4I0LDiieuGngsqlqOih9ker/NS0LX4V0i+EhiFWg0= -github.com/klauspost/pgzip v1.2.2/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= +github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= -github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs= -github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= +github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pierrec/cmdflag v0.0.2/go.mod h1:a3zKGZ3cdQUfxjd0RGMLZr8xI3nvpJOB+m6o/1X5BmU= -github.com/pierrec/lz4/v3 v3.0.1 h1:VP/E0GE2MnyXUdS46vP8/JM5HU3bfDodAp9WTu9Gw7I= -github.com/pierrec/lz4/v3 v3.0.1/go.mod h1:280XNCGS8jAcG++AHdd6SeWnzyJ1w9oow2vbORyey8Q= +github.com/pierrec/lz4/v3 v3.3.2 h1:QTUOCbMNDbK4PYtkuHyOBd28C0UhPBw3T4OH4WpFDik= +github.com/pierrec/lz4/v3 v3.3.2/go.mod h1:280XNCGS8jAcG++AHdd6SeWnzyJ1w9oow2vbORyey8Q= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/schollz/progressbar/v2 v2.13.2/go.mod h1:6YZjqdthH6SCZKv2rqGryrxPtfmRB/DWZxSMfCXPyD8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= +github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/zip.go b/zip.go index c6773e97..04b9f126 100644 --- a/zip.go +++ b/zip.go @@ -70,10 +70,10 @@ type Zip struct { ContinueOnError bool //Compression algorithm - CompressionAlgorithm uint16 - zw *zip.Writer - zr *zip.Reader - ridx int + FileMethod uint16 + zw *zip.Writer + zr *zip.Reader + ridx int } // CheckExt ensures the file extension matches the format. @@ -307,7 +307,7 @@ func (z *Zip) Create(out io.Writer) error { return flate.NewWriter(out, z.CompressionLevel) }) } - switch z.CompressionAlgorithm { + switch z.FileMethod { case BZIP2: z.zw.RegisterCompressor(BZIP2, func(out io.Writer) (io.WriteCloser, error) { return bzip2.NewWriter(out, &bzip2.WriterConfig{Level: z.CompressionLevel}) @@ -349,11 +349,7 @@ func (z *Zip) Write(f File) error { if _, ok := compressedFormats[ext]; ok && z.SelectiveCompression { header.Method = zip.Store } else { - if z.CompressionAlgorithm > 0 { - header.Method = z.CompressionAlgorithm // zip or zstd - } else { - header.Method = zip.Deflate - } + header.Method = z.FileMethod } } @@ -602,6 +598,7 @@ func NewZip() *Zip { CompressionLevel: flate.DefaultCompression, MkdirAll: true, SelectiveCompression: true, + FileMethod: Deflate, } } From 5dd12515bc86d3a02e801fb7e51efa3e77c074f5 Mon Sep 17 00:00:00 2001 From: "Force.Charlie" Date: Sun, 14 Jun 2020 17:16:15 +0800 Subject: [PATCH 4/9] fix ls zip --- zip.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/zip.go b/zip.go index 04b9f126..b598545c 100644 --- a/zip.go +++ b/zip.go @@ -483,7 +483,27 @@ func (z *Zip) Walk(archive string, walkFn WalkFunc) error { return fmt.Errorf("opening zip reader: %v", err) } defer zr.Close() - + zr.RegisterDecompressor(ZSTD, func(r io.Reader) io.ReadCloser { + zr, err := zstd.NewReader(r) + if err != nil { + return nil + } + return zr.IOReadCloser() + }) + zr.RegisterDecompressor(BZIP2, func(r io.Reader) io.ReadCloser { + bz2r, err := bzip2.NewReader(r, nil) + if err != nil { + return nil + } + return bz2r + }) + zr.RegisterDecompressor(LZMA, func(r io.Reader) io.ReadCloser { + lr, err := lzma.NewReader(r) + if err != nil { + return nil + } + return ioutil.NopCloser(lr) + }) for _, zf := range zr.File { zfrc, err := zf.Open() if err != nil { From 3cc407ac3c5c414d13a9fea87bfbfc03fe9227b8 Mon Sep 17 00:00:00 2001 From: "Force.Charlie" Date: Sun, 14 Jun 2020 20:41:40 +0800 Subject: [PATCH 5/9] support WinZip zstd --- zip.go | 103 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 51 insertions(+), 52 deletions(-) diff --git a/zip.go b/zip.go index b598545c..3768d617 100644 --- a/zip.go +++ b/zip.go @@ -20,11 +20,12 @@ import ( // Compression methods. const ( - Store uint16 = 0 // no compression - Deflate uint16 = 8 // DEFLATE compressed - BZIP2 uint16 = 12 // bzip2 - LZMA uint16 = 14 //LZMA - ZSTD uint16 = 20 //see https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT. + Store uint16 = 0 // no compression + Deflate uint16 = 8 // DEFLATE compressed + BZIP2 uint16 = 12 // bzip2 + LZMA uint16 = 14 //LZMA + ZSTD uint16 = 20 //see https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT. + WINZIPZSTD uint16 = 93 ) // Zip provides facilities for operating ZIP archives. @@ -70,10 +71,11 @@ type Zip struct { ContinueOnError bool //Compression algorithm - FileMethod uint16 - zw *zip.Writer - zr *zip.Reader - ridx int + FileMethod uint16 + zw *zip.Writer + zr *zip.Reader + ridx int + decinitialized bool } // CheckExt ensures the file extension matches the format. @@ -84,6 +86,44 @@ func (*Zip) CheckExt(filename string) error { return nil } +// Registering a global decompressor is not reentrant and may panic +func (z *Zip) registerDecompressor() { + if z.decinitialized { + return + } + z.decinitialized = true + // register zstd decompressor + z.zr.RegisterDecompressor(ZSTD, func(r io.Reader) io.ReadCloser { + zr, err := zstd.NewReader(r) + if err != nil { + return nil + } + return zr.IOReadCloser() + }) + z.zr.RegisterDecompressor(BZIP2, func(r io.Reader) io.ReadCloser { + bz2r, err := bzip2.NewReader(r, nil) + if err != nil { + return nil + } + return bz2r + }) + z.zr.RegisterDecompressor(LZMA, func(r io.Reader) io.ReadCloser { + lr, err := lzma.NewReader(r) + if err != nil { + return nil + } + return ioutil.NopCloser(lr) + }) + // WinZip zstd support + z.zr.RegisterDecompressor(WINZIPZSTD, func(r io.Reader) io.ReadCloser { + zr, err := zstd.NewReader(r) + if err != nil { + return nil + } + return zr.IOReadCloser() + }) +} + // Archive creates a .zip file at destination containing // the files listed in sources. The destination must end // with ".zip". File paths can be those of regular files @@ -405,28 +445,7 @@ func (z *Zip) Open(in io.Reader, size int64) error { if err != nil { return fmt.Errorf("creating reader: %v", err) } - // register zstd decompressor - z.zr.RegisterDecompressor(ZSTD, func(r io.Reader) io.ReadCloser { - zr, err := zstd.NewReader(r) - if err != nil { - return nil - } - return zr.IOReadCloser() - }) - z.zr.RegisterDecompressor(BZIP2, func(r io.Reader) io.ReadCloser { - bz2r, err := bzip2.NewReader(r, nil) - if err != nil { - return nil - } - return bz2r - }) - z.zr.RegisterDecompressor(LZMA, func(r io.Reader) io.ReadCloser { - lr, err := lzma.NewReader(r) - if err != nil { - return nil - } - return ioutil.NopCloser(lr) - }) + z.registerDecompressor() z.ridx = 0 return nil } @@ -483,27 +502,7 @@ func (z *Zip) Walk(archive string, walkFn WalkFunc) error { return fmt.Errorf("opening zip reader: %v", err) } defer zr.Close() - zr.RegisterDecompressor(ZSTD, func(r io.Reader) io.ReadCloser { - zr, err := zstd.NewReader(r) - if err != nil { - return nil - } - return zr.IOReadCloser() - }) - zr.RegisterDecompressor(BZIP2, func(r io.Reader) io.ReadCloser { - bz2r, err := bzip2.NewReader(r, nil) - if err != nil { - return nil - } - return bz2r - }) - zr.RegisterDecompressor(LZMA, func(r io.Reader) io.ReadCloser { - lr, err := lzma.NewReader(r) - if err != nil { - return nil - } - return ioutil.NopCloser(lr) - }) + z.registerDecompressor() for _, zf := range zr.File { zfrc, err := zf.Open() if err != nil { From 5b6eca5919d8e624aa2915d2b601b1a17d14ce52 Mon Sep 17 00:00:00 2001 From: "Force.Charlie" Date: Tue, 16 Jun 2020 06:36:31 +0800 Subject: [PATCH 6/9] According to Review need to improve the code --- zip.go | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/zip.go b/zip.go index 3768d617..2d0e9445 100644 --- a/zip.go +++ b/zip.go @@ -18,14 +18,18 @@ import ( "github.com/ulikunitz/xz/lzma" ) +// ZipCompressionMethod Compression type +type ZipCompressionMethod uint16 + // Compression methods. +// see https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT. const ( - Store uint16 = 0 // no compression - Deflate uint16 = 8 // DEFLATE compressed - BZIP2 uint16 = 12 // bzip2 - LZMA uint16 = 14 //LZMA - ZSTD uint16 = 20 //see https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT. - WINZIPZSTD uint16 = 93 + Store ZipCompressionMethod = 0 + Deflate ZipCompressionMethod = 8 + BZIP2 ZipCompressionMethod = 12 + LZMA ZipCompressionMethod = 14 + ZSTD ZipCompressionMethod = 20 + WINZIPZSTD ZipCompressionMethod = 93 ) // Zip provides facilities for operating ZIP archives. @@ -70,8 +74,8 @@ type Zip struct { // the operation will continue on remaining files. ContinueOnError bool - //Compression algorithm - FileMethod uint16 + // Compression algorithm + FileMethod ZipCompressionMethod zw *zip.Writer zr *zip.Reader ridx int @@ -93,21 +97,21 @@ func (z *Zip) registerDecompressor() { } z.decinitialized = true // register zstd decompressor - z.zr.RegisterDecompressor(ZSTD, func(r io.Reader) io.ReadCloser { + z.zr.RegisterDecompressor(uint16(ZSTD), func(r io.Reader) io.ReadCloser { zr, err := zstd.NewReader(r) if err != nil { return nil } return zr.IOReadCloser() }) - z.zr.RegisterDecompressor(BZIP2, func(r io.Reader) io.ReadCloser { + z.zr.RegisterDecompressor(uint16(BZIP2), func(r io.Reader) io.ReadCloser { bz2r, err := bzip2.NewReader(r, nil) if err != nil { return nil } return bz2r }) - z.zr.RegisterDecompressor(LZMA, func(r io.Reader) io.ReadCloser { + z.zr.RegisterDecompressor(uint16(LZMA), func(r io.Reader) io.ReadCloser { lr, err := lzma.NewReader(r) if err != nil { return nil @@ -115,7 +119,7 @@ func (z *Zip) registerDecompressor() { return ioutil.NopCloser(lr) }) // WinZip zstd support - z.zr.RegisterDecompressor(WINZIPZSTD, func(r io.Reader) io.ReadCloser { + z.zr.RegisterDecompressor(uint16(WINZIPZSTD), func(r io.Reader) io.ReadCloser { zr, err := zstd.NewReader(r) if err != nil { return nil @@ -201,6 +205,7 @@ func (z *Zip) Unarchive(source, destination string) error { return fmt.Errorf("opening zip archive for reading: %v", err) } defer z.Close() + // if the files in the archive do not all share a common // root, then make sure we extract to a single subfolder // rather than potentially littering the destination... @@ -349,15 +354,15 @@ func (z *Zip) Create(out io.Writer) error { } switch z.FileMethod { case BZIP2: - z.zw.RegisterCompressor(BZIP2, func(out io.Writer) (io.WriteCloser, error) { + z.zw.RegisterCompressor(uint16(BZIP2), func(out io.Writer) (io.WriteCloser, error) { return bzip2.NewWriter(out, &bzip2.WriterConfig{Level: z.CompressionLevel}) }) case LZMA: - z.zw.RegisterCompressor(LZMA, func(out io.Writer) (io.WriteCloser, error) { + z.zw.RegisterCompressor(uint16(LZMA), func(out io.Writer) (io.WriteCloser, error) { return lzma.NewWriter(out) }) case ZSTD: - z.zw.RegisterCompressor(ZSTD, func(out io.Writer) (io.WriteCloser, error) { + z.zw.RegisterCompressor(uint16(ZSTD), func(out io.Writer) (io.WriteCloser, error) { return zstd.NewWriter(out) }) } @@ -389,7 +394,7 @@ func (z *Zip) Write(f File) error { if _, ok := compressedFormats[ext]; ok && z.SelectiveCompression { header.Method = zip.Store } else { - header.Method = z.FileMethod + header.Method = uint16(z.FileMethod) } } From 7d8190ef08f1d5a312c1ef6b3a4116374477f3fe Mon Sep 17 00:00:00 2001 From: "Force.Charlie" Date: Sat, 20 Jun 2020 13:35:45 +0800 Subject: [PATCH 7/9] ZSTD=20 deprecated --- zip.go | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/zip.go b/zip.go index 2d0e9445..7768b5fb 100644 --- a/zip.go +++ b/zip.go @@ -24,12 +24,11 @@ type ZipCompressionMethod uint16 // Compression methods. // see https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT. const ( - Store ZipCompressionMethod = 0 - Deflate ZipCompressionMethod = 8 - BZIP2 ZipCompressionMethod = 12 - LZMA ZipCompressionMethod = 14 - ZSTD ZipCompressionMethod = 20 - WINZIPZSTD ZipCompressionMethod = 93 + Store ZipCompressionMethod = 0 + Deflate ZipCompressionMethod = 8 + BZIP2 ZipCompressionMethod = 12 + LZMA ZipCompressionMethod = 14 + ZSTD ZipCompressionMethod = 93 ) // Zip provides facilities for operating ZIP archives. @@ -118,14 +117,6 @@ func (z *Zip) registerDecompressor() { } return ioutil.NopCloser(lr) }) - // WinZip zstd support - z.zr.RegisterDecompressor(uint16(WINZIPZSTD), func(r io.Reader) io.ReadCloser { - zr, err := zstd.NewReader(r) - if err != nil { - return nil - } - return zr.IOReadCloser() - }) } // Archive creates a .zip file at destination containing From 7f17b5b14c15562cdd15421b7bc634263f606012 Mon Sep 17 00:00:00 2001 From: "Force.Charlie" Date: Sat, 20 Jun 2020 13:37:52 +0800 Subject: [PATCH 8/9] Update github.com/klauspost/compress --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1ebdcdee..ff26ff5e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/andybalholm/brotli v1.0.0 github.com/dsnet/compress v0.0.1 github.com/golang/snappy v0.0.1 - github.com/klauspost/compress v1.10.8 + github.com/klauspost/compress v1.10.9 github.com/klauspost/pgzip v1.2.4 github.com/nwaples/rardecode v1.1.0 github.com/pierrec/lz4/v3 v3.3.2 diff --git a/go.sum b/go.sum index 6f4cf637..f183353e 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.8 h1:eLeJ3dr/Y9+XRfJT4l+8ZjmtB5RPJhucH2HeCV5+IZY= -github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.9 h1:pPRt1Z78crspaHISkpSSHjDlx+Tt9suHe519dsI0vF4= +github.com/klauspost/compress v1.10.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= From 6dc2d1ce36bec277e6c054f43d4a7a29f7d5fada Mon Sep 17 00:00:00 2001 From: "Force.Charlie" Date: Wed, 24 Jun 2020 22:55:43 +0800 Subject: [PATCH 9/9] Update klauspost/compress this module has a big performance improvement --- go.mod | 5 ++++- go.sum | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ff26ff5e..2ffa9222 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,12 @@ go 1.12 require ( github.com/andybalholm/brotli v1.0.0 github.com/dsnet/compress v0.0.1 + github.com/frankban/quicktest v1.10.0 // indirect github.com/golang/snappy v0.0.1 - github.com/klauspost/compress v1.10.9 + github.com/google/go-cmp v0.5.0 // indirect + github.com/klauspost/compress v1.10.10 github.com/klauspost/pgzip v1.2.4 + github.com/kr/text v0.2.0 // indirect github.com/nwaples/rardecode v1.1.0 github.com/pierrec/lz4/v3 v3.3.2 github.com/ulikunitz/xz v0.5.7 diff --git a/go.sum b/go.sum index f183353e..e94d4b4f 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= @@ -8,25 +9,34 @@ github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5Jflh github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/frankban/quicktest v1.4.0 h1:rCSCih1FnSWJEel/eub9wclBSqpF2F/PuvxUWGWnbO8= github.com/frankban/quicktest v1.4.0/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= +github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE= +github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.9 h1:pPRt1Z78crspaHISkpSSHjDlx+Tt9suHe519dsI0vF4= -github.com/klauspost/compress v1.10.9/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= +github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= @@ -50,6 +60,8 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=