Add auto import images

Signed-off-by: Vitor Savian <vitor.savian@suse.com>

.github/workflows/e2e.yaml

@@ -38,7 +38,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        etest: [startup, s3, btrfs, externalip, privateregistry, embeddedmirror, wasm]
+        etest: [autoimport, startup, s3, btrfs, externalip, privateregistry, embeddedmirror, wasm]
       max-parallel: 3
     steps:
       - name: "Checkout"

docs/adrs/add-auto-import-embedded-registry.md

@@ -0,0 +1,54 @@
+# Easy way for auto adding images to k3s
+
+Date: 2024-10-2
+
+## Status
+
+Proposed
+
+## Context
+
+Since the feature for embedded registry, the users appeared with a question about having to manually import images, specially in edge environments.
+
+As a result, there is a need for a folder who can handle this action, where every image there will be watched by a controller for changes or new images, this new images or new changes will be added to the containerd image store.
+
+The controller will watch the agent/images folder that is the default folder for the images, as the first iteration about the controller he will mainly work with the default image folder, but in the future we can set to watch more folders.
+
+The main idea for the controller is to create a map for the file infos maintaining the state for the files, with that we can see if a file was modified and if the size changed.
+
+### Map to handle the state from the files
+
+This map will have the entire filepath of the file in the `key` value, since we can get the value from the key with only the `event.Name`
+
+```go
+    map[string]fs.FileInfo
+```
+
+### Why use fsnotify
+
+With this library we can easily use for any linux distros without the need to port for a specify distro and can also run in windows.
+
+The main idea for the watch will be taking care of the last time that was modified the image file.
+
+fsnotify has a great toolset for handling changes in files, since the code will have a channel to receive events such as CREATE, RENAME, REMOVE and WRITE.
+
+### How the controller will work with the events
+
+When the controller receive a event saying that a file was created, he will add to the map and import the images if the event that he has received is not a directory and then import the image.
+
+When the controller receive a event saying that a file was writen, he will verify if the file has the size changed and if the file has the time modified based on the time and size from the state.
+
+When the controller receive a event saying that a file was renamed, or removed, he will delete this file from the state. when a file is renamed, it is created a new file with the same infos but with a the new name, so the watcher will sent for the controller a event saying that a file was created.
+
+## Decision
+
+- Not decided yet
+
+## Consequences
+
+Good:
+- Better use of embedded containerd image store.
+- Fsnotify it's a indirect dependency that upstream uses
+
+Bad:
+- The need for another dependency

go.mod

@@ -89,6 +89,7 @@ require (
 	github.com/docker/docker v27.1.1+incompatible
 	github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83
 	github.com/flannel-io/flannel v0.25.6
+	github.com/fsnotify/fsnotify v1.7.0
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-logr/logr v1.4.2
 	github.com/go-logr/stdr v1.2.3-0.20220714215716-96bad1d688c5
@@ -248,7 +249,6 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/flynn/noise v1.1.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.2 // indirect

pkg/agent/containerd/containerd.go

@@ -34,8 +34,9 @@ var (
 	// In addition to using the CRI pinned label, we add our own label to indicate that
 	// the image was pinned by the import process, so that we can clear the pin on subsequent startups.
 	// ref: https://github.com/containerd/containerd/blob/release/1.7/pkg/cri/labels/labels.go
-	k3sPinnedImageLabelKey   = "io.cattle." + version.Program + ".pinned"
-	k3sPinnedImageLabelValue = "pinned"
+	k3sPinnedImageLabelKey     = "io.cattle." + version.Program + ".pinned"
+	k3sAutoImportImageLabelKey = "io.cattle." + version.Program + ".import"
+	k3sPinnedImageLabelValue   = "pinned"
 )
 
 // Run configures and starts containerd as a child process. Once it is up, images are preloaded
@@ -115,10 +116,14 @@ func Run(ctx context.Context, cfg *config.Node) error {
 // any .txt files are processed as a list of images that should be pre-pulled from remote registries.
 // If configured, imported images are retagged as being pulled from additional registries.
 func PreloadImages(ctx context.Context, cfg *config.Node) error {
-	fileInfo, err := os.Stat(cfg.Images)
-	if os.IsNotExist(err) {
+	err := os.MkdirAll(cfg.Images, 0744)
+	if err != nil {
+		logrus.Errorf("Unable to create agent/images folder in %s: %v", cfg.Images, err)
 		return nil
-	} else if err != nil {
+	}
+
+	fileInfo, err := os.Stat(cfg.Images)
+	if err != nil {
 		logrus.Errorf("Unable to find images in %s: %v", cfg.Images, err)
 		return nil
 	}
@@ -176,6 +181,9 @@ func PreloadImages(ctx context.Context, cfg *config.Node) error {
 		}
 		logrus.Infof("Imported images from %s in %s", filePath, time.Since(start))
 	}
+
+	go watchImages(ctx, cfg)
+
 	return nil
 }
 
@@ -214,7 +222,7 @@ func preloadFile(ctx context.Context, cfg *config.Node, client *containerd.Clien
 		}
 	}
 
-	if err := labelImages(ctx, client, images); err != nil {
+	if err := labelImages(ctx, client, images, filepath.Base(filePath)); err != nil {
 		return errors.Wrap(err, "failed to add pinned label to images")
 	}
 	if err := retagImages(ctx, client, images, cfg.AgentConfig.AirgapExtraRegistry); err != nil {
@@ -245,6 +253,25 @@ func clearLeases(ctx context.Context, client *containerd.Client) error {
 	return nil
 }
 
+// clearLabelFromFile removes the auto import label on images with the value based on the file name
+func clearLabelFromAutoImport(ctx context.Context, client *containerd.Client, filePath string) error {
+	var errs []error
+	imageService := client.ImageService()
+	images, err := imageService.List(ctx, fmt.Sprintf("labels.%q==%s", k3sAutoImportImageLabelKey, filepath.Base(filePath)))
+	if err != nil {
+		return err
+	}
+	for _, image := range images {
+		if image.Labels[k3sAutoImportImageLabelKey] == filepath.Base(filePath) {
+			delete(image.Labels, k3sAutoImportImageLabelKey)
+			if _, err := imageService.Update(ctx, image, "labels"); err != nil {
+				errs = append(errs, errors.Wrap(err, "failed to delete auto import label from image "+image.Name))
+			}
+		}
+	}
+	return merr.NewErrors(errs...)
+}
+
 // clearLabels removes the pinned labels on all images in the image store that were previously pinned by k3s
 func clearLabels(ctx context.Context, client *containerd.Client) error {
 	var errs []error
@@ -265,7 +292,7 @@ func clearLabels(ctx context.Context, client *containerd.Client) error {
 
 // labelImages adds labels to the listed images, indicating that they
 // are pinned by k3s and should not be pruned.
-func labelImages(ctx context.Context, client *containerd.Client, images []images.Image) error {
+func labelImages(ctx context.Context, client *containerd.Client, images []images.Image, fileName string) error {
 	var errs []error
 	imageService := client.ImageService()
 	for i, image := range images {
@@ -277,6 +304,7 @@ func labelImages(ctx context.Context, client *containerd.Client, images []images
 		if image.Labels == nil {
 			image.Labels = map[string]string{}
 		}
+		image.Labels[k3sAutoImportImageLabelKey] = fileName
 		image.Labels[k3sPinnedImageLabelKey] = k3sPinnedImageLabelValue
 		image.Labels[labels.PinnedImageLabelKey] = labels.PinnedImageLabelValue
 		updatedImage, err := imageService.Update(ctx, image, "labels")