Add a method to extract files from a docker image last layer

.github/workflows/secscan.yaml

@@ -19,7 +19,7 @@ jobs:
       - name: Checkout Source
         uses: actions/checkout@v4
       - name: Run Gosec Security Scanner
-        uses: securego/gosec@master
+        uses: securego/gosec@v2.21.0
         with:
           # we let the report trigger content trigger a failure using the GitHub Security features.
           args: '-no-fail -fmt sarif -out results.sarif ./...'

.github/workflows/unit-tests.yml

@@ -28,7 +28,7 @@ jobs:
   unit-tests:
     strategy:
       matrix:
-        go-version: [ "1.22" ]
+        go-version: [ "1.23" ]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -40,6 +40,9 @@ jobs:
         with:
           repository: quay.io/kairos/packages
           packages: utils/earthly
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@master
       - name: Run tests
         run: |
           earthly -P +test --GO_VERSION=${{ matrix.go-version }}

Earthfile

@@ -1,9 +1,9 @@
 VERSION 0.7
 
 # renovate: datasource=docker depName=golang
-ARG --global GO_VERSION=1.22
+ARG --global GO_VERSION=1.23
 # renovate: datasource=docker depName=golangci-lint
-ARG --global GOLINT_VERSION=1.59.1
+ARG --global GOLINT_VERSION=1.61.0
 # renovate: datasource=docker depName=quay.io/luet/base
 ARG --global LUET_VERSION=0.34.0
 
@@ -26,8 +26,10 @@ test:
     ENV CGO_ENABLED=0
     WORKDIR /build
     COPY +luet/luet /usr/bin/luet
-
-    RUN go run github.com/onsi/ginkgo/v2/ginkgo run --fail-fast --slow-spec-threshold 30s --covermode=atomic --coverprofile=coverage.out -p -r ./...
+    # Some tests need the docker client available
+    WITH DOCKER
+        RUN go run github.com/onsi/ginkgo/v2/ginkgo run --fail-fast --covermode=atomic --coverprofile=coverage.out -p -r ./...
+    END
     SAVE ARTIFACT coverage.out AS LOCAL coverage.out
 
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/kairos-io/kairos-sdk
 
-go 1.22.5
+go 1.23.1
 
 require (
 	github.com/avast/retry-go v2.7.0+incompatible
@@ -61,7 +61,6 @@ require (
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect

sysext/sysext.go

@@ -0,0 +1,97 @@
+package sysext
+
+import (
+	"archive/tar"
+	"errors"
+	"fmt"
+	"github.com/kairos-io/kairos-sdk/types"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+)
+
+var ErrorImageNoLayers = errors.New("image")
+
+// DefaultAllowListRegex provided for easy use of defaults for confext and sysext
+var DefaultAllowListRegex = regexp.MustCompile(`^usr/*|^/usr/*|^etc/*|^/etc/*`)
+
+// ExtractFilesFromLastLayer will get an image and a destination and extract the files from the last layer in the image
+// into that destination.
+// It will skip anything that doesn't start with /usr or /etc as its purpose is to get the files for creating a
+// sysextension or a confextension
+// Accepts an allowList in form of regexp.Regexp that will match the files and allow copying
+func ExtractFilesFromLastLayer(image v1.Image, dst string, log types.KairosLogger, allowList *regexp.Regexp) error {
+	layers, _ := image.Layers()
+	numLayers := len(layers)
+	if len(layers) <= 0 {
+		return ErrorImageNoLayers
+	}
+	return extractFilesFromLayer(image, dst, log, allowList, numLayers-1)
+}
+
+func extractFilesFromLayer(image v1.Image, dst string, log types.KairosLogger, allowList *regexp.Regexp, layerNumber int) error {
+	layers, _ := image.Layers()
+	layerToExtract := layers[layerNumber]
+	layerReader, _ := layerToExtract.Uncompressed()
+	defer func(layerReader io.ReadCloser) {
+		_ = layerReader.Close()
+	}(layerReader)
+	tr := tar.NewReader(layerReader)
+	// TODO: Support whiteout? https://github.com/opencontainers/image-spec/blob/79b036d80240ae530a8de15e1d21c7ab9292c693/layer.md#whiteouts
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("tar read: %w", err)
+		}
+
+		header.Name = filepath.Clean(header.Name)
+
+		path := filepath.Join(dst, header.Name)
+		fi := header.FileInfo()
+		mask := fi.Mode()
+		if !allowList.MatchString(header.Name) {
+			log.Debug("Skipping ", header.Name)
+			continue
+		}
+
+		switch header.Typeflag {
+		case tar.TypeDir:
+			log.Debugf("%s is a directory", header.Name)
+			if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) {
+				if err := os.MkdirAll(path, mask); err != nil {
+					return fmt.Errorf("mkdir: %w", err)
+				}
+			}
+		case tar.TypeReg:
+			log.Debugf("%s is a file", header.Name)
+			file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, mask)
+			if err != nil {
+				return fmt.Errorf("open: %w", err)
+			}
+			if _, err := io.Copy(file, tr); err != nil {
+				file.Close()
+				return fmt.Errorf("copy: %w", err)
+			}
+			file.Close()
+		case tar.TypeSymlink:
+			log.Debugf("%s is a symlink", header.Name)
+			targetPath := filepath.Join(filepath.Dir(path), header.Linkname)
+			if !strings.HasPrefix(targetPath, dst) {
+				return fmt.Errorf("symlink: %w", err)
+			}
+			if err := os.Symlink(header.Linkname, path); err != nil {
+				return fmt.Errorf("symlink: %w", err)
+			}
+		default:
+			return fmt.Errorf("unsupported type: %d", header.Typeflag)
+		}
+	}
+	return nil
+}

sysext/sysext_test.go

@@ -0,0 +1,158 @@
+package sysext
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	dockerImage "github.com/docker/docker/api/types/image"
+	"github.com/docker/docker/client"
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/daemon"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
+
+	"github.com/kairos-io/kairos-sdk/types"
+	"github.com/kairos-io/kairos-sdk/utils"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"regexp"
+	"testing"
+)
+
+func TestSuite(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "sysext Test Suite")
+}
+
+var _ = Describe("sysext", Label("sysext"), Ordered, func() {
+	var dest string
+	var image v1.Image
+	var imageTag string
+	var buf bytes.Buffer
+	var log types.KairosLogger
+	var err error
+
+	BeforeEach(func() {
+		buf = bytes.Buffer{}
+		log = types.NewBufferLogger(&buf)
+		dest, err = os.MkdirTemp("", "")
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		if CurrentSpecReport().Failed() {
+			_, _ = GinkgoWriter.Write(buf.Bytes())
+		}
+		Expect(os.RemoveAll(dest)).To(Succeed())
+	})
+
+	When("Using a normal image", func() {
+		BeforeEach(func() {
+			imageTag = createTestDockerImage()
+			By(fmt.Sprintf("Created image %s", imageTag))
+			image, err = utils.GetImage(imageTag, utils.GetCurrentPlatform(), nil, nil)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		AfterEach(func() {
+			cli, _ := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+			_, _ = cli.ImageRemove(context.Background(), imageTag, dockerImage.RemoveOptions{Force: true})
+			By(fmt.Sprintf("Removed image %s", imageTag))
+		})
+		It("should extract the files into the dir", func() {
+			err = ExtractFilesFromLastLayer(image, dest, log, DefaultAllowListRegex)
+			Expect(err).ToNot(HaveOccurred())
+			_, err := os.Stat(filepath.Join(dest, "usr", "yes"))
+			Expect(err).ToNot(HaveOccurred())
+			_, err = os.Stat(filepath.Join(dest, "etc", "yes"))
+			Expect(err).ToNot(HaveOccurred())
+			_, err = os.Stat(filepath.Join(dest, "opt", "nope"))
+			Expect(err).To(HaveOccurred())
+			_, err = os.Stat(filepath.Join(dest, "var", "nope"))
+			Expect(err).To(HaveOccurred())
+		})
+		It("properly uses the allowList", func() {
+			allowList := regexp.MustCompile(`^var|^/var`)
+			err = ExtractFilesFromLastLayer(image, dest, log, allowList)
+			Expect(err).ToNot(HaveOccurred())
+			_, err := os.Stat(filepath.Join(dest, "usr", "yes"))
+			Expect(err).To(HaveOccurred())
+			_, err = os.Stat(filepath.Join(dest, "etc", "yes"))
+			Expect(err).To(HaveOccurred())
+			_, err = os.Stat(filepath.Join(dest, "opt", "nope"))
+			Expect(err).To(HaveOccurred())
+			_, err = os.Stat(filepath.Join(dest, "var", "nope"))
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+
+	When("Using an empty image", func() {
+		BeforeEach(func() {
+			imageTag = createEmptyDockerImage()
+			By(fmt.Sprintf("Created image %s", imageTag))
+			image, err = utils.GetImage(imageTag, utils.GetCurrentPlatform(), nil, nil)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		AfterEach(func() {
+			cli, _ := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+			_, _ = cli.ImageRemove(context.Background(), imageTag, dockerImage.RemoveOptions{Force: true})
+			By(fmt.Sprintf("Removed image %s", imageTag))
+		})
+		It("Fails with no layers image", func() {
+			// Cleanup existing image before creating a new one
+
+			err = ExtractFilesFromLastLayer(image, dest, log, DefaultAllowListRegex)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(Equal(ErrorImageNoLayers))
+		})
+	})
+})
+
+func createEmptyDockerImage() string {
+	var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
+
+	b := make([]rune, 8)
+	for i := range b {
+		b[i] = letterRunes[rand.Intn(len(letterRunes))]
+	}
+
+	img, err := mutate.AppendLayers(empty.Image)
+	Expect(err).ToNot(HaveOccurred())
+	tag, err := name.NewTag(fmt.Sprintf("kairos-empty-%s:latest", string(b)))
+	Expect(err).ToNot(HaveOccurred())
+	_, err = daemon.Write(tag, img)
+	Expect(err).ToNot(HaveOccurred())
+
+	return tag.String()
+}
+
+func createTestDockerImage() string {
+	var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
+
+	b := make([]rune, 8)
+	for i := range b {
+		b[i] = letterRunes[rand.Intn(len(letterRunes))]
+	}
+
+	// We don't care about this layer so make it a bit fake
+	fistLayer, _ := crane.Layer(map[string][]byte{
+		"/etc/one":     []byte("hello"),
+		"/etc/another": []byte("world"),
+	})
+
+	secondLayer, err := tarball.LayerFromFile("testdata/test.tar")
+	Expect(err).ToNot(HaveOccurred())
+	img, err := mutate.AppendLayers(empty.Image, fistLayer, secondLayer)
+	Expect(err).ToNot(HaveOccurred())
+	tag, err := name.NewTag(fmt.Sprintf("kairos-test-%s:latest", string(b)))
+	Expect(err).ToNot(HaveOccurred())
+	_, err = daemon.Write(tag, img)
+	Expect(err).ToNot(HaveOccurred())
+
+	return tag.String()
+}