Skip to content
This repository has been archived by the owner on Nov 19, 2024. It is now read-only.

Commit

Permalink
zip: Configurable algorithms: zstd/lzma/bzip2 (#223)
Browse files Browse the repository at this point in the history 
* support decompress zip(zstd,lzma,bzip2)

* support compress use zstd/lzma/bzip2

* Update code

* fix ls zip

* support WinZip zstd

* According to Review need to improve the code

* ZSTD=20 deprecated

* Update github.com/klauspost/compress

* Update klauspost/compress this module has a big performance improvement
  • Loading branch information
@fcharlie
fcharlie authored Jun 26, 2020
1 parent b44e6c6 commit d44471c
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 18 deletions.
13 changes: 8 additions & 5 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,14 @@ go 1.12
require (
github.com/andybalholm/brotli v1.0.0
github.com/dsnet/compress v0.0.1
github.com/frankban/quicktest v1.10.0 // indirect
github.com/golang/snappy v0.0.1
github.com/klauspost/compress v1.10.3
github.com/klauspost/pgzip v1.2.2
github.com/nwaples/rardecode v1.0.0
github.com/pierrec/lz4/v3 v3.0.1
github.com/ulikunitz/xz v0.5.6
github.com/google/go-cmp v0.5.0 // indirect
github.com/klauspost/compress v1.10.10
github.com/klauspost/pgzip v1.2.4
github.com/kr/text v0.2.0 // indirect
github.com/nwaples/rardecode v1.1.0
github.com/pierrec/lz4/v3 v3.3.2
github.com/ulikunitz/xz v0.5.7
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8
)
30 changes: 22 additions & 8 deletions go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,53 +1,67 @@
code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4=
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/frankban/quicktest v1.4.0 h1:rCSCih1FnSWJEel/eub9wclBSqpF2F/PuvxUWGWnbO8=
github.com/frankban/quicktest v1.4.0/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ=
github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE=
github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.10.3 h1:OP96hzwJVBIHYU52pVTI6CczrxPvrGfgqF9N5eTO0Q8=
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I=
github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/pgzip v1.2.2 h1:8d4I0LDiieuGngsqlqOih9ker/NS0LX4V0i+EhiFWg0=
github.com/klauspost/pgzip v1.2.2/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A=
github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/nwaples/rardecode v1.0.0 h1:r7vGuS5akxOnR4JQSkko62RJ1ReCMXxQRPtxsiFMBOs=
github.com/nwaples/rardecode v1.0.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ=
github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/pierrec/cmdflag v0.0.2/go.mod h1:a3zKGZ3cdQUfxjd0RGMLZr8xI3nvpJOB+m6o/1X5BmU=
github.com/pierrec/lz4/v3 v3.0.1 h1:VP/E0GE2MnyXUdS46vP8/JM5HU3bfDodAp9WTu9Gw7I=
github.com/pierrec/lz4/v3 v3.0.1/go.mod h1:280XNCGS8jAcG++AHdd6SeWnzyJ1w9oow2vbORyey8Q=
github.com/pierrec/lz4/v3 v3.3.2 h1:QTUOCbMNDbK4PYtkuHyOBd28C0UhPBw3T4OH4WpFDik=
github.com/pierrec/lz4/v3 v3.3.2/go.mod h1:280XNCGS8jAcG++AHdd6SeWnzyJ1w9oow2vbORyey8Q=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/schollz/progressbar/v2 v2.13.2/go.mod h1:6YZjqdthH6SCZKv2rqGryrxPtfmRB/DWZxSMfCXPyD8=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/ulikunitz/xz v0.5.6 h1:jGHAfXawEGZQ3blwU5wnWKQJvAraT7Ftq9EXjnXYgt8=
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4=
github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
Expand Down
77 changes: 72 additions & 5 deletions zip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,29 @@ import (
"compress/flate"
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path"
"path/filepath"
"strings"

"github.com/dsnet/compress/bzip2"
"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz/lzma"
)

// ZipCompressionMethod Compression type
type ZipCompressionMethod uint16

// Compression methods.
// see https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT.
const (
Store ZipCompressionMethod = 0
Deflate ZipCompressionMethod = 8
BZIP2 ZipCompressionMethod = 12
LZMA ZipCompressionMethod = 14
ZSTD ZipCompressionMethod = 93
)

// Zip provides facilities for operating ZIP archives.
Expand Down Expand Up @@ -55,9 +73,12 @@ type Zip struct {
// the operation will continue on remaining files.
ContinueOnError bool

zw *zip.Writer
zr *zip.Reader
ridx int
// Compression algorithm
FileMethod ZipCompressionMethod
zw *zip.Writer
zr *zip.Reader
ridx int
decinitialized bool
}

// CheckExt ensures the file extension matches the format.
Expand All @@ -68,6 +89,36 @@ func (*Zip) CheckExt(filename string) error {
return nil
}

// Registering a global decompressor is not reentrant and may panic
func (z *Zip) registerDecompressor() {
if z.decinitialized {
return
}
z.decinitialized = true
// register zstd decompressor
z.zr.RegisterDecompressor(uint16(ZSTD), func(r io.Reader) io.ReadCloser {
zr, err := zstd.NewReader(r)
if err != nil {
return nil
}
return zr.IOReadCloser()
})
z.zr.RegisterDecompressor(uint16(BZIP2), func(r io.Reader) io.ReadCloser {
bz2r, err := bzip2.NewReader(r, nil)
if err != nil {
return nil
}
return bz2r
})
z.zr.RegisterDecompressor(uint16(LZMA), func(r io.Reader) io.ReadCloser {
lr, err := lzma.NewReader(r)
if err != nil {
return nil
}
return ioutil.NopCloser(lr)
})
}

// Archive creates a .zip file at destination containing
// the files listed in sources. The destination must end
// with ".zip". File paths can be those of regular files
Expand Down Expand Up @@ -292,6 +343,20 @@ func (z *Zip) Create(out io.Writer) error {
return flate.NewWriter(out, z.CompressionLevel)
})
}
switch z.FileMethod {
case BZIP2:
z.zw.RegisterCompressor(uint16(BZIP2), func(out io.Writer) (io.WriteCloser, error) {
return bzip2.NewWriter(out, &bzip2.WriterConfig{Level: z.CompressionLevel})
})
case LZMA:
z.zw.RegisterCompressor(uint16(LZMA), func(out io.Writer) (io.WriteCloser, error) {
return lzma.NewWriter(out)
})
case ZSTD:
z.zw.RegisterCompressor(uint16(ZSTD), func(out io.Writer) (io.WriteCloser, error) {
return zstd.NewWriter(out)
})
}
return nil
}

Expand Down Expand Up @@ -320,7 +385,7 @@ func (z *Zip) Write(f File) error {
if _, ok := compressedFormats[ext]; ok && z.SelectiveCompression {
header.Method = zip.Store
} else {
header.Method = zip.Deflate
header.Method = uint16(z.FileMethod)
}
}

Expand Down Expand Up @@ -376,6 +441,7 @@ func (z *Zip) Open(in io.Reader, size int64) error {
if err != nil {
return fmt.Errorf("creating reader: %v", err)
}
z.registerDecompressor()
z.ridx = 0
return nil
}
Expand Down Expand Up @@ -432,7 +498,7 @@ func (z *Zip) Walk(archive string, walkFn WalkFunc) error {
return fmt.Errorf("opening zip reader: %v", err)
}
defer zr.Close()

z.registerDecompressor()
for _, zf := range zr.File {
zfrc, err := zf.Open()
if err != nil {
Expand Down Expand Up @@ -547,6 +613,7 @@ func NewZip() *Zip {
CompressionLevel: flate.DefaultCompression,
MkdirAll: true,
SelectiveCompression: true,
FileMethod: Deflate,
}
}

Expand Down

0 comments on commit d44471c

Please sign in to comment.