Use zstd instead of gzip for embedded tarball

.gitignore

@@ -21,7 +21,7 @@
 /image/go_build_agent
 /image/main.squashfs
 /package/k3s
-/package/data.tar.gz
+/package/data.tar.zst
 /pkg/data/zz_generated_bindata.go
 __pycache__
 /tests/.pytest_cache/

Dockerfile.dapper

@@ -8,7 +8,7 @@ ENV https_proxy=$https_proxy
 ENV no_proxy=$no_proxy
 
 RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers zlib-dev tar zip squashfs-tools npm coreutils \
- python2 openssl-dev libffi-dev libseccomp libseccomp-dev make libuv-static sqlite-dev sqlite-static libselinux libselinux-dev zlib-dev zlib-static
+ python2 openssl-dev libffi-dev libseccomp libseccomp-dev make libuv-static sqlite-dev sqlite-static libselinux libselinux-dev zlib-dev zlib-static zstd
 RUN if [ "$(go env GOARCH)" = "arm64" ]; then \
  wget https://github.com/aquasecurity/trivy/releases/download/v0.11.0/trivy_0.11.0_Linux-ARM64.tar.gz && \
  tar -zxvf trivy_0.11.0_Linux-ARM64.tar.gz && \

go.mod

@@ -84,6 +84,7 @@ require (
  github.com/gorilla/websocket v1.4.2
  github.com/k3s-io/helm-controller v0.8.3
  github.com/k3s-io/kine v0.6.0
+ github.com/klauspost/compress v1.11.7
  github.com/kubernetes-sigs/cri-tools v0.0.0-00010101000000-000000000000
  github.com/lib/pq v1.8.0
  github.com/mattn/go-sqlite3 v1.14.4

go.sum

@@ -578,6 +578,8 @@ github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1q
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg=
+github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/knative/build v0.6.0/go.mod h1:/sU74ZQkwlYA5FwYDJhYTy61i/Kn+5eWfln2jDbw3Qo=
 github.com/knative/pkg v0.0.0-20190514205332-5e4512dcb2ca/go.mod h1:7Ijfhw7rfB+H9VtosIsDYvZQ+qYTz7auK3fHW/5z4ww=

package/Dockerfile

@@ -1,7 +1,8 @@
 FROM alpine:3.12 as base
-RUN apk add -U ca-certificates
-ADD build/out/data.tar.gz /image
+RUN apk add -U ca-certificates tar zstd
+COPY build/out/data.tar.zst /
 RUN mkdir -p /image/etc/ssl/certs /image/run /image/var/run /image/tmp /image/lib/modules /image/lib/firmware && \
+ tar -xa -C /image -f /data.tar.zst && \
  cp /etc/ssl/certs/ca-certificates.crt /image/etc/ssl/certs/ca-certificates.crt
 RUN cd image/bin && \
  rm -f k3s && \

pkg/untar/untar.go

@@ -7,14 +7,14 @@ package untar
 
 import (
  "archive/tar"
- "compress/gzip"
  "fmt"
  "io"
  "os"
  "path/filepath"
  "strings"
  "time"
 
+ "github.com/klauspost/compress/zstd"
  "github.com/sirupsen/logrus"
 )
 
@@ -23,7 +23,7 @@ import (
 // forked for now. Unfork and add some opts arguments here, so the
 // buildlet can use this code somehow.
 
-// Untar reads the gzip-compressed tar file from r and writes it into dir.
+// Untar reads the zstd-compressed tar file from r and writes it into dir.
 func Untar(r io.Reader, dir string) error {
  return untar(r, dir)
 }
@@ -38,10 +38,11 @@ func untar(r io.Reader, dir string) (err error) {
  logrus.Printf("error extracting tarball into %s after %d files, %d dirs, %v: %v", dir, nFiles, len(madeDir), td, err)
  }
  }()
- zr, err := gzip.NewReader(r)
+ zr, err := zstd.NewReader(r)
  if err != nil {
- return fmt.Errorf("requires gzip-compressed body: %v", err)
+ return fmt.Errorf("requires zstd-compressed body: %v", err)
  }
+ defer zr.Close()
  tr := tar.NewReader(zr)
  loggedChtimesError := false
  for {

scripts/package-cli

@@ -35,10 +35,11 @@ mkdir -p dist/artifacts
  set -x
 )
 
-tar cvzf ./build/out/data.tar.gz --exclude ./bin/hyperkube ./bin ./etc
-HASH=$(sha256sum ./build/out/data.tar.gz | awk '{print $1}')
+tar cvf ./build/out/data.tar --exclude ./bin/hyperkube ./bin ./etc 
+zstd -v -T0 -16 -f --long --rm ./build/out/data.tar -o ./build/out/data.tar.zst
+HASH=$(sha256sum ./build/out/data.tar.zst | awk '{print $1}')
 
-cp ./build/out/data.tar.gz ./build/data/${HASH}.tgz
+cp ./build/out/data.tar.zst ./build/data/${HASH}.tar.zst
 
 BIN_SUFFIX="-${ARCH}"
 if [ ${ARCH} = amd64 ]; then