From 4846eaaf5d3e764ab643498f3cdbeb9cac0277fb Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 3 Dec 2018 00:15:18 -0700 Subject: [PATCH] Implement Walk and Extract package funcs and add examples to docs --- archiver.go | 86 ++++++++++++++--- doc_test.go | 260 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 333 insertions(+), 13 deletions(-) create mode 100644 doc_test.go diff --git a/archiver.go b/archiver.go index 786e1ca4..096eb036 100644 --- a/archiver.go +++ b/archiver.go @@ -1,16 +1,47 @@ -// Package archiver facilitates high-level archival and compression operations -// for a variety of formats and compression algorithms. There is a type -// definition for each format or algorithm that is supported. +// Package archiver facilitates convenient, cross-platform, high-level archival +// and compression operations for a variety of formats and compression algorithms. // -// Each format has an instance of the type that represents it already created -// for convenience, called `Default*` (replace the wildcard with the format's -// type name). They can be reused, but we recommend not changing their config -// between uses. There is no real performance benefit of reusing an instance. -// You can also use `New*()` to get a new instance of a type (such as -// `NewZip()`) with sane defaults, which is safer if your program is changing -// an instance's configuration. +// This package and its dependencies are written in pure Go (not cgo) and +// have no external dependencies, so they should run on all major platforms. +// (It also comes with a command for CLI use in the cmd/arc folder.) // -// The types and functions in this package are not safe for concurrent use. +// Each supported format or algorithm has a unique type definition that +// implements the interfaces corresponding to the tasks they perform. For +// example, the Tar type implements Reader, Writer, Archiver, Unarchiver, +// Walker, and several other interfaces. +// +// The most common functions are implemented at the package level for +// convenience: Archive, Unarchive, Walk, Extract, CompressFile, and +// DecompressFile. With these, the format type is chosen implicitly, +// and a sane default configuration is used. +// +// To customize a format's configuration, create an instance of its struct +// with its fields set to the desired values. You can also use and customize +// the handy Default* (replace the wildcard with the format's type name) +// for a quick, one-off instance of the format's type. +// +// To obtain a new instance of a format's struct with the default config, use +// the provided New*() functions. This is not required, however. An empty +// struct of any type, for example &Zip{} is perfectly valid, so you may +// create the structs manually, too. The examples on this page show how +// either may be done. +// +// See the examples in this package for an idea of how to wield this package +// for common tasks. Most of the examples which are specific to a certain +// format type, for example Zip, can be applied to other types that implement +// the same interfaces. For example, using Zip is very similar to using Tar +// or TarGz (etc), and using Gz is very similar to using Sz or Xz (etc). +// +// When creating archives or compressing files using a specific instance of +// the format's type, the name of the output file MUST match that of the +// format, to prevent confusion later on. If you absolutely need a different +// file extension, you may rename the file afterward. +// +// Values in this package are NOT safe for concurrent use. There is no +// performance benefit of reusing them, and since they may contain important +// state (especially while walking, reading, or writing), it is NOT +// recommended to reuse values from this package or change their configuration +// after they are in use. package archiver import ( @@ -155,7 +186,7 @@ func Archive(sources []string, destination string) error { } a, ok := aIface.(Archiver) if !ok { - return fmt.Errorf("format specified by destination filename is not an archive format: %s", destination) + return fmt.Errorf("format specified by destination filename is not an archive format: %s (%T)", destination, aIface) } return a.Archive(sources, destination) } @@ -175,11 +206,40 @@ func Unarchive(source, destination string) error { f.Close() u, ok := uaIface.(Unarchiver) if !ok { - return fmt.Errorf("format specified by destination filename is not an archive format: %s", destination) + return fmt.Errorf("format specified by destination filename is not an archive format: %s (%T)", destination, uaIface) } return u.Unarchive(source, destination) } +// Walk calls walkFn for each file within the given archive file. +// The archive format is chosen implicitly. +func Walk(archive string, walkFn WalkFunc) error { + wIface, err := ByExtension(archive) + if err != nil { + return err + } + w, ok := wIface.(Walker) + if !ok { + return fmt.Errorf("format specified by archive filename is not a walker format: %s (%T)", archive, wIface) + } + return w.Walk(archive, walkFn) +} + +// Extract extracts a single file from the given source archive. If the target +// is a directory, the entire folder will be extracted into destination. The +// archive format is chosen implicitly. +func Extract(source, target, destination string) error { + eIface, err := ByExtension(source) + if err != nil { + return err + } + e, ok := eIface.(Extractor) + if !ok { + return fmt.Errorf("format specified by source filename is not an extractor format: %s (%T)", source, eIface) + } + return e.Extract(source, target, destination) +} + // CompressFile is a convenience function to simply compress a file. // The compression algorithm is selected implicitly based on the // destination's extension. diff --git a/doc_test.go b/doc_test.go new file mode 100644 index 00000000..aadcc2d2 --- /dev/null +++ b/doc_test.go @@ -0,0 +1,260 @@ +package archiver + +import ( + "fmt" + "io" + "log" + "net/http" + "os" + "strconv" +) + +// The simplest use of this package: create an archive file +// from a list of filenames. This is the recommended way to +// do so using a default configuration, as it guarantees +// the file format matches the file extension, because the +// format to write is determined by the given extension. +func ExampleArchive() { + // any files in this list are added + // to the top level of the archive; + // directories are recursively added + files := []string{ + "index.html", + "photo.jpg", + "blog", // directory + "/home/website/copyright.txt", + } + + // archive format is determined by file extension + err := Archive(files, "blog_site.zip") + if err != nil { + log.Fatal(err) + } +} + +// The simplest use of this package: extract all of an archive's +// contents to a folder on disk using the default configuration. +// The archive format is determined automatically. +func ExampleUnarchive() { + err := Unarchive("blog_site.zip", "extracted/mysite") + if err != nil { + log.Fatal(err) + } +} + +// In this example, the DefaultZip is being customized so that +// all calls to its methods will use that configuration. +func ExampleZip_default() { + DefaultZip.OverwriteExisting = true + DefaultZip.ImplicitTopLevelFolder = true + // any subsequent use of DefaultZip uses + // this modified configuration +} + +// Here we create our own instance of the Zip format. No need +// to use the constructor function (NewZip) or the default +// instance (DefaultZip) if we do not want to. Instantiating +// the type like this allows us to easily be very explicit +// about our configuration. +func ExampleZip_custom() { + z := &Zip{ + CompressionLevel: 3, + OverwriteExisting: false, + MkdirAll: true, + SelectiveCompression: true, + ImplicitTopLevelFolder: true, + ContinueOnError: false, + } + // z is now ready to use for whatever (this is a dumb example) + fmt.Println(z.CheckExt("test.zip")) +} + +// Much like the package-level Archive function, this creates an +// archive using the configuration of the Zip instance it is called +// on. The output filename must match the format's recognized file +// extension(s). +func ExampleZip_Archive() { + err := DefaultZip.Archive([]string{"..."}, "example.zip") + if err != nil { + log.Fatal(err) + } +} + +// It's easy to list the items in an archive. This example +// prints the name and size of each file in the archive. Like +// other top-level functions in this package, the format is +// inferred automatically for you. +func ExampleWalk() { + err := Walk("example.tar.gz", func(f File) error { + fmt.Println(f.Name(), f.Size()) + // you could also read the contents; f is an io.Reader! + return nil + }) + if err != nil { + log.Fatal(err) + } +} + +// This example extracts target.txt from inside example.rar +// and puts it into a folder on disk called output/dir. +func ExampleExtract() { + err := Extract("example.rar", "target.txt", "output/dir") + if err != nil { + log.Fatal(err) + } +} + +// This example demonstrates how to read an +// archive in a streaming fashion. The idea +// is that you can stream the bytes of an +// archive from a stream, regardless of +// whether it is an actual file on disk. +// This means that you can read a huge +// archive file-by-file rather than having +// to store it all on disk first. In this +// example, we read a hypothetical archive +// from a (fake) HTTP request body and +// print its file names and sizes. The +// files can be read, of course, but they +// do not have to be. +func ExampleZip_streamingRead() { + // for the sake of the example compiling, pretend we have an HTTP request + req := new(http.Request) + contentLen, err := strconv.Atoi(req.Header.Get("Content-Length")) + if err != nil { + log.Fatal(err) + } + + // the Zip format requires knowing the length of the stream, + // but other formats don't generally require it, so it + // could be left as 0 when using those + err = DefaultZip.Open(req.Body, int64(contentLen)) + if err != nil { + log.Fatal(err) + } + defer DefaultZip.Close() + + // Note that DefaultZip now contains some state that + // is critical to reading the stream until it is closed, + // so do not reuse it until then. + + // iterate each file in the archive until EOF + for { + f, err := DefaultZip.Read() + if err == io.EOF { + break + } + if err != nil { + log.Fatal(err) + } + + // f is an io.ReadCloser, so you can read its contents + // if you wish; or you can access its header info through + // f.Header or the embedded os.FileInfo + fmt.Println("File name:", f.Name(), "File size:", f.Size()) + + // be sure to close f before moving on!! + err = f.Close() + if err != nil { + log.Fatal(err) + } + } +} + +// This example demonstrates how to write an +// archive in a streaming fashion. The idea +// is that you can stream the bytes of a new +// archive that is created on-the-fly from +// generic streams. Those streams could be +// actual files on disk, or they could be over +// a network, or standard output, or any other +// io.Reader/io.Writer. This example only adds +// one file to the archive and writes the +// resulting archive to standard output, but you +// could add as many files as needed with a loop. +func ExampleZip_streamingWrite() { + err := DefaultZip.Create(os.Stdout) + if err != nil { + log.Fatal(err) + } + defer DefaultZip.Close() + + // Note that DefaultZip now contains state + // critical to a successful write until it + // is closed, so don't reuse it for anything + // else until then. + + // At this point, you can open an actual file + // to add to the archive, or the "file" could + // come from any io.ReadCloser stream. If you + // only have an io.Reader, you can use + // ReadFakeCloser to make it into an + // io.ReadCloser. + + // The next part is a little tricky if you + // don't have an actual file because you will + // need an os.FileInfo. Fortunately, that's an + // interface! So go ahead and implement it in + // whatever way makes the most sense to you. + // You'll also need to give the file a name + // for within the archive. In this example, + // we'll open a real file. + + file, err := os.Open("foo.txt") + if err != nil { + log.Fatal(err) + } + defer file.Close() + fileInfo, err := file.Stat() + if err != nil { + log.Fatal(err) + } + + err = DefaultZip.Write(File{ + FileInfo: FileInfo{ + FileInfo: fileInfo, + CustomName: "name/in/archive.txt", + }, + ReadCloser: file, // does not have to be an actual file + }) + if err != nil { + log.Fatal(err) + } +} + +// This example compresses a standard tar file into a tar.gz file. +// Compression formats are selected by file extension. +func ExampleCompressFile() { + err := CompressFile("example.tar", "example.tar.gz") + if err != nil { + log.Fatal(err) + } +} + +// This example changes the default configuration for +// the Gz compression format. +func ExampleCompressFile_custom() { + DefaultGz.CompressionLevel = 5 + // any calls to DefaultGz now use the modified configuration +} + +// This example creates a new Gz instance and +// uses it to compress a stream, writing to +// another stream. This is sometimes preferable +// over modifying the DefaultGz. +func ExampleGz_Compress_custom() { + gz := &Gz{CompressionLevel: 5} + err := gz.Compress(os.Stdin, os.Stdout) + if err != nil { + log.Fatal(err) + } +} + +// This example decompresses a gzipped tarball and writes +// it to an adjacent file. +func ExampleDecompressFile() { + err := DecompressFile("example.tar.gz", "example.tar") + if err != nil { + log.Fatal(err) + } +}