Merge branch 'main' into ROCKS-1471/symmetric-hardlinks-in-slicer

README.md

@@ -269,11 +269,16 @@ have additional information for identifying the kind of content to expect:
  which are only available for certain architectures. Example:
  `/usr/bin/hello: {arch: amd64}` will instruct Chisel to extract and install
  the "/usr/bin/hello" file only when chiselling an amd64 filesystem.
+ - **generate**: accepts a `manifest` value to instruct Chisel to generate the
+ manifest files in the directory. Example: `/var/lib/chisel/**:{generate:
+ manifest}`. NOTE: the provided path has to be of the form
+ `/slashed/path/to/dir/**` and no wildcards can appear apart from the trailing
+ `**`.
 
 ## TODO
 
 - [ ] Preserve ownerships when possible
-- [ ] GPG signature checking for archives
+- [x] GPG signature checking for archives
 - [ ] Use a fake server for the archive tests
 - [ ] Functional tests
 

go.mod

@@ -12,7 +12,7 @@ require (
 	golang.org/x/crypto v0.21.0
 	golang.org/x/term v0.18.0
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
-	gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (

go.sum

@@ -90,7 +90,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 h1:dbuHpmKjkDzSOMKAWl10QNlgaZUd3V1q99xc81tt2Kc=
-gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/setup/setup.go

@@ -60,11 +60,12 @@ type SliceScripts struct {
 type PathKind string
 
 const (
-	DirPath     PathKind = "dir"
-	CopyPath    PathKind = "copy"
-	GlobPath    PathKind = "glob"
-	TextPath    PathKind = "text"
-	SymlinkPath PathKind = "symlink"
+	DirPath      PathKind = "dir"
+	CopyPath     PathKind = "copy"
+	GlobPath     PathKind = "glob"
+	TextPath     PathKind = "text"
+	SymlinkPath  PathKind = "symlink"
+	GeneratePath PathKind = "generate"
 
 	// TODO Maybe in the future, for binary support.
 	//Base64Path PathKind = "base64"
@@ -77,14 +78,22 @@ const (
 	UntilMutate PathUntil = "mutate"
 )
 
+type GenerateKind string
+
+const (
+	GenerateNone     GenerateKind = ""
+	GenerateManifest GenerateKind = "manifest"
+)
+
 type PathInfo struct {
 	Kind PathKind
 	Info string
 	Mode uint
 
-	Mutable bool
-	Until   PathUntil
-	Arch    []string
+	Mutable  bool
+	Until    PathUntil
+	Arch     []string
+	Generate GenerateKind
 }
 
 // SameContent returns whether the path has the same content properties as some
@@ -95,7 +104,8 @@ func (pi *PathInfo) SameContent(other *PathInfo) bool {
 	return (pi.Kind == other.Kind &&
 		pi.Info == other.Info &&
 		pi.Mode == other.Mode &&
-		pi.Mutable == other.Mutable)
+		pi.Mutable == other.Mutable &&
+		pi.Generate == other.Generate)
 }
 
 type SliceKey struct {
@@ -141,10 +151,20 @@ func ReadRelease(dir string) (*Release, error) {
 
 func (r *Release) validate() error {
 	keys := []SliceKey(nil)
+
+	// Check for info conflicts and prepare for following checks. A conflict
+	// means that two slices attempt to extract different files or directories
+	// to the same location.
+	// Conflict validation is done without downloading packages which means that
+	// if we are extracting content from different packages to the same location
+	// we cannot be sure that it will be the same. On the contrary, content
+	// extracted from the same package will never conflict because it is
+	// guaranteed to be the same.
+	// The above also means that generated content (e.g. text files, directories
+	// with make:true) will always conflict with extracted content, because we
+	// cannot validate that they are the same without downloading the package.
 	paths := make(map[string]*Slice)
 	globs := make(map[string]*Slice)
-
-	// Check for info conflicts and prepare for following checks.
 	for _, pkg := range r.Packages {
 		for _, new := range pkg.Slices {
 			keys = append(keys, SliceKey{pkg.Name, new.Name})
@@ -157,36 +177,50 @@ func (r *Release) validate() error {
 						}
 						return fmt.Errorf("slices %s and %s conflict on %s", old, new, newPath)
 					}
+					// Note: Because for conflict resolution we only check that
+					// the created file would be the same and we know newInfo and
+					// oldInfo produce the same one, we do not have to record
+					// newInfo.
 				} else {
-					if newInfo.Kind == GlobPath {
+					paths[newPath] = new
+					if newInfo.Kind == GeneratePath || newInfo.Kind == GlobPath {
 						globs[newPath] = new
 					}
-					paths[newPath] = new
 				}
 			}
 		}
 	}
 
-	// Check for cycles.
-	_, err := order(r.Packages, keys)
-	if err != nil {
-		return err
-	}
-
-	// Check for glob conflicts.
-	for newPath, new := range globs {
-		for oldPath, old := range paths {
-			if new.Package == old.Package {
+	// Check for glob and generate conflicts.
+	for oldPath, old := range globs {
+		oldInfo := old.Contents[oldPath]
+		for newPath, new := range paths {
+			if oldPath == newPath {
+				// Identical paths have been filtered earlier. This must be the
+				// exact same entry.
 				continue
 			}
+			newInfo := new.Contents[newPath]
+			if oldInfo.Kind == GlobPath && (newInfo.Kind == GlobPath || newInfo.Kind == CopyPath) {
+				if new.Package == old.Package {
+					continue
+				}
+			}
 			if strdist.GlobPath(newPath, oldPath) {
-				if old.Package > new.Package || old.Package == new.Package && old.Name > new.Name {
-					old, oldPath, new, newPath = new, newPath, old, oldPath
+				if (old.Package > new.Package) || (old.Package == new.Package && old.Name > new.Name) ||
+					(old.Package == new.Package && old.Name == new.Name && oldPath > newPath) {
+					old, new = new, old
+					oldPath, newPath = newPath, oldPath
 				}
 				return fmt.Errorf("slices %s and %s conflict on %s and %s", old, new, oldPath, newPath)
 			}
 		}
-		paths[newPath] = new
+	}
+
+	// Check for cycles.
+	_, err := order(r.Packages, keys)
+	if err != nil {
+		return err
 	}
 
 	return nil
@@ -357,8 +391,9 @@ type yamlPath struct {
 	Symlink string  `yaml:"symlink"`
 	Mutable bool    `yaml:"mutable"`
 
-	Until PathUntil `yaml:"until"`
-	Arch  yamlArch  `yaml:"arch"`
+	Until    PathUntil    `yaml:"until"`
+	Arch     yamlArch     `yaml:"arch"`
+	Generate GenerateKind `yaml:"generate"`
 }
 
 // SameContent returns whether the path has the same content properties as some
@@ -583,7 +618,19 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro
 			var mutable bool
 			var until PathUntil
 			var arch []string
-			if strings.ContainsAny(contPath, "*?") {
+			var generate GenerateKind
+			if yamlPath != nil && yamlPath.Generate != "" {
+				zeroPathGenerate := zeroPath
+				zeroPathGenerate.Generate = yamlPath.Generate
+				if !yamlPath.SameContent(&zeroPathGenerate) || yamlPath.Until != UntilNone {
+					return nil, fmt.Errorf("slice %s_%s path %s has invalid generate options",
+						pkgName, sliceName, contPath)
+				}
+				if _, err := validateGeneratePath(contPath); err != nil {
+					return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkgName, sliceName, err)
+				}
+				kinds = append(kinds, GeneratePath)
+			} else if strings.ContainsAny(contPath, "*?") {
 				if yamlPath != nil {
 					if !yamlPath.SameContent(&zeroPath) {
 						return nil, fmt.Errorf("slice %s_%s path %s has invalid wildcard options",
@@ -595,6 +642,7 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro
 			if yamlPath != nil {
 				mode = yamlPath.Mode
 				mutable = yamlPath.Mutable
+				generate = yamlPath.Generate
 				if yamlPath.Dir {
 					if !strings.HasSuffix(contPath, "/") {
 						return nil, fmt.Errorf("slice %s_%s path %s must end in / for 'make' to be valid",
@@ -644,12 +692,13 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro
 				return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkgName, sliceName, contPath)
 			}
 			slice.Contents[contPath] = PathInfo{
-				Kind:    kinds[0],
-				Info:    info,
-				Mode:    mode,
-				Mutable: mutable,
-				Until:   until,
-				Arch:    arch,
+				Kind:     kinds[0],
+				Info:     info,
+				Mode:     mode,
+				Mutable:  mutable,
+				Until:    until,
+				Arch:     arch,
+				Generate: generate,
 			}
 		}
 
@@ -659,6 +708,22 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro
 	return &pkg, err
 }
 
+// validateGeneratePath validates that the path follows the following format:
+//   - /slashed/path/to/dir/**
+//
+// Wildcard characters can only appear at the end as **, and the path before
+// those wildcards must be a directory.
+func validateGeneratePath(path string) (string, error) {
+	if !strings.HasSuffix(path, "/**") {
+		return "", fmt.Errorf("%s does not end with /**", path)
+	}
+	dirPath := strings.TrimSuffix(path, "**")
+	if strings.ContainsAny(dirPath, "*?") {
+		return "", fmt.Errorf("%s contains wildcard characters in addition to trailing **", path)
+	}
+	return dirPath, nil
+}
+
 func stripBase(baseDir, path string) string {
 	// Paths must be clean for this to work correctly.
 	return strings.TrimPrefix(path, baseDir+string(filepath.Separator))
@@ -691,9 +756,17 @@ func Select(release *Release, slices []SliceKey) (*Selection, error) {
 					}
 					return nil, fmt.Errorf("slices %s and %s conflict on %s", old, new, newPath)
 				}
-				continue
+			} else {
+				paths[newPath] = new
+			}
+			// An invalid "generate" value should only throw an error if that
+			// particular slice is selected. Hence, the check is here.
+			switch newInfo.Generate {
+			case GenerateNone, GenerateManifest:
+			default:
+				return nil, fmt.Errorf("slice %s has invalid 'generate' for path %s: %q, consider an update if available",
+					new, newPath, newInfo.Generate)
 			}
-			paths[newPath] = new
 		}
 	}