Skip to content

Commit

Permalink
feat: Add decompression of file externals
Browse files Browse the repository at this point in the history
  • Loading branch information
twpayne committed Jul 4, 2024
1 parent 3ad9743 commit 676a9a9
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 18 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ Entries may have the following fields:
| Variable | Type | Default value | Description |
| ---------------------------- | -------- | ------------- | ---------------------------------------------------------------- |
| `type` | string | *none* | External type (`file`, `archive`, `archive-file`, or `git-repo`) |
| `decompress` | string | *none* | Decompression for file |
| `encrypted` | bool | `false` | Whether the external is encrypted |
| `exact` | bool | `false` | Add `exact_` attribute to directories in archive |
| `exclude` | []string | *none* | Patterns to exclude from archive |
Expand Down Expand Up @@ -51,6 +52,11 @@ has the given checksum.
The optional boolean `encrypted` field specifies whether the file or archive is
encrypted.

The optional string `decompress` specifies how the file should be decompressed.
Supported compression formats are `bzip2`, `gzip`, `xz`, and `zstd`. Note the
`.zip` files are archives and you must use the `archive-file` type to extract a
single file from a `.zip` archive.

If optional string `filter.command` and array of strings `filter.args` are
specified, the file or archive is filtered by piping it into the command's
standard input and reading the command's standard output.
Expand Down
53 changes: 53 additions & 0 deletions internal/chezmoi/compression.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package chezmoi

import (
"bytes"
"compress/bzip2"
"fmt"
"io"

"github.com/klauspost/compress/gzip"
"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
)

// A compressionFormat is a compression format.
type compressionFormat string

// Compression formats.
const (
compressionFormatNone compressionFormat = ""
compressionFormatBzip2 compressionFormat = "bzip2"
compressionFormatGzip compressionFormat = "gzip"
compressionFormatXz compressionFormat = "xz"
compressionFormatZstd compressionFormat = "zstd"
)

func decompress(compressionFormat compressionFormat, data []byte) ([]byte, error) {
switch compressionFormat {
case compressionFormatNone:
return data, nil
case compressionFormatBzip2:
return io.ReadAll(bzip2.NewReader(bytes.NewReader(data)))
case compressionFormatGzip:
gzipReader, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
return nil, err
}
return io.ReadAll(gzipReader)
case compressionFormatXz:
xzReader, err := xz.NewReader(bytes.NewReader(data))
if err != nil {
return nil, err
}
return io.ReadAll(xzReader)
case compressionFormatZstd:
zstdReader, err := zstd.NewReader(bytes.NewReader(data))
if err != nil {
return nil, err
}
return io.ReadAll(zstdReader)
default:
return nil, fmt.Errorf("%s: unknown compression format", compressionFormat)
}
}
42 changes: 24 additions & 18 deletions internal/chezmoi/sourcestate.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,24 +88,25 @@ type externalPull struct {

// An External is an external source.
type External struct {
Type ExternalType `json:"type" toml:"type" yaml:"type"`
Encrypted bool `json:"encrypted" toml:"encrypted" yaml:"encrypted"`
Exact bool `json:"exact" toml:"exact" yaml:"exact"`
Executable bool `json:"executable" toml:"executable" yaml:"executable"`
Private bool `json:"private" toml:"private" yaml:"private"`
ReadOnly bool `json:"readonly" toml:"readonly" yaml:"readonly"`
Checksum externalChecksum `json:"checksum" toml:"checksum" yaml:"checksum"`
Clone externalClone `json:"clone" toml:"clone" yaml:"clone"`
Exclude []string `json:"exclude" toml:"exclude" yaml:"exclude"`
Filter externalFilter `json:"filter" toml:"filter" yaml:"filter"`
Format ArchiveFormat `json:"format" toml:"format" yaml:"format"`
Archive externalArchive `json:"archive" toml:"archive" yaml:"archive"`
Include []string `json:"include" toml:"include" yaml:"include"`
ArchivePath string `json:"path" toml:"path" yaml:"path"`
Pull externalPull `json:"pull" toml:"pull" yaml:"pull"`
RefreshPeriod Duration `json:"refreshPeriod" toml:"refreshPeriod" yaml:"refreshPeriod"`
StripComponents int `json:"stripComponents" toml:"stripComponents" yaml:"stripComponents"`
URL string `json:"url" toml:"url" yaml:"url"`
Type ExternalType `json:"type" toml:"type" yaml:"type"`
Encrypted bool `json:"encrypted" toml:"encrypted" yaml:"encrypted"`
Exact bool `json:"exact" toml:"exact" yaml:"exact"`
Executable bool `json:"executable" toml:"executable" yaml:"executable"`
Private bool `json:"private" toml:"private" yaml:"private"`
ReadOnly bool `json:"readonly" toml:"readonly" yaml:"readonly"`
Checksum externalChecksum `json:"checksum" toml:"checksum" yaml:"checksum"`
Clone externalClone `json:"clone" toml:"clone" yaml:"clone"`
Decompress compressionFormat `json:"decompress" toml:"decompress" yaml:"decompress"`
Exclude []string `json:"exclude" toml:"exclude" yaml:"exclude"`
Filter externalFilter `json:"filter" toml:"filter" yaml:"filter"`
Format ArchiveFormat `json:"format" toml:"format" yaml:"format"`
Archive externalArchive `json:"archive" toml:"archive" yaml:"archive"`
Include []string `json:"include" toml:"include" yaml:"include"`
ArchivePath string `json:"path" toml:"path" yaml:"path"`
Pull externalPull `json:"pull" toml:"pull" yaml:"pull"`
RefreshPeriod Duration `json:"refreshPeriod" toml:"refreshPeriod" yaml:"refreshPeriod"`
StripComponents int `json:"stripComponents" toml:"stripComponents" yaml:"stripComponents"`
URL string `json:"url" toml:"url" yaml:"url"`
sourceAbsPath AbsPath
}

Expand Down Expand Up @@ -1685,6 +1686,11 @@ func (s *SourceState) getExternalData(
}
}

data, err = decompress(external.Decompress, data)
if err != nil {
return nil, fmt.Errorf("%s: %w", externalRelPath, err)
}

if external.Filter.Command != "" {
cmd := exec.Command(external.Filter.Command, external.Filter.Args...) //nolint:gosec
cmd.Stdin = bytes.NewReader(data)
Expand Down
55 changes: 55 additions & 0 deletions internal/cmd/testdata/scripts/externalcompression.txtar
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
[exec:bzip2] exec bzip2 www/file-bzip2
[exec:gzip] exec gzip www/file-gzip
[exec:xz] exec xz www/file-xz
[exec:zstd] exec zstd www/file-zstd

httpd www

# test that chezmoi apply decompresses files in multiple formats
exec chezmoi apply
[exec:bzip2] cmp $HOME/file-bzip2 golden/file-bzip2
[exec:gzip] cmp $HOME/file-gzip golden/file-gzip
[exec:xz] cmp $HOME/file-xz golden/file-xz
[exec:zstd] cmp $HOME/file-zstd golden/file-zstd

-- golden/file-bzip2 --
# contents of file-bzip2
-- golden/file-gzip --
# contents of file-gzip
-- golden/file-xz --
# contents of file-xz
-- golden/file-zstd --
# contents of file-zstd
-- home/user/.local/share/chezmoi/.chezmoiexternal.toml.tmpl --
{{ if lookPath "bzip2" }}
[file-bzip2]
type = "file"
url = "{{ env "HTTPD_URL" }}/file-bzip2.bz2"
decompress = "bzip2"
{{ end }}
{{ if lookPath "gzip" }}
[file-gzip]
type = "file"
url = "{{ env "HTTPD_URL" }}/file-gzip.gz"
decompress = "gzip"
{{ end }}
{{ if lookPath "xz" }}
[file-xz]
type = "file"
url = "{{ env "HTTPD_URL" }}/file-xz.xz"
decompress = "xz"
{{ end }}
{{ if lookPath "zstd" }}
[file-zstd]
type = "file"
url = "{{ env "HTTPD_URL" }}/file-zstd.zst"
decompress = "zstd"
{{ end }}
-- www/file-bzip2 --
# contents of file-bzip2
-- www/file-gzip --
# contents of file-gzip
-- www/file-xz --
# contents of file-xz
-- www/file-zstd --
# contents of file-zstd

0 comments on commit 676a9a9

Please sign in to comment.