Implementing logic split flattened modules into different tar files, and

calculate which buildpacks must be added into the ephemeral builder

Signed-off-by: Juan Bustamante <jbustamante@vmware.com>

acceptance/acceptance_test.go

@@ -2778,7 +2778,7 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
 						return packageTomlFile.Name()
 					}
 
-					when.Focus("--flatten", func() {
+					when("--flatten", func() {
 						it("creates the package as a single layer", func() {
 							//h.SkipIf(t, !pack.SupportsFeature(invoke.BuildpackFlatten), "")
 
@@ -2816,7 +2816,7 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
 								"--builder", builderName,
 							)
 
-							h.AssertContains(t, output, "Buildpack 'simple/layers@simple-layers-version' is a component of a flattened buildpack that will be added elsewhere, skipping...")
+							h.AssertContainsMatch(t, output, "Buildpack '(simple/layers@simple-layers-version|simple/layers/parent@simple-layers-parent-version)' is a component of a flattened buildpack that will be added elsewhere, skipping...")
 						})
 					})
 				})

internal/builder/builder.go

@@ -581,27 +581,51 @@ func (b *Builder) addExplodedModules(kind string, logger logging.Logger, tmpDir
 				lids[i] <- modInfo{err: errors.Wrapf(err, "creating %s temp dir", kind)}
 			}
 
-			// create tar file
-			//layerTar, err := buildpack.ToLayerTar(modTmpDir, module)
-			//if err != nil {
-			//	lids[i] <- modInfo{err: err}
-			//}
-			layerTars, _ := buildpack.ToNLayerTar(modTmpDir, module)
 			var diffIDs []v1.Hash
 			var infos []dist.ModuleInfo // only id & version will be populated
-			for _, lt := range layerTars {
+			var layerTars []string
+
+			if kind == buildpack.KindBuildpack {
+				moduleTars, err := buildpack.ToNLayerTar(modTmpDir, module)
+				if err != nil {
+					lids[i] <- modInfo{err: errors.Wrapf(err, "creating %s tar file", module.Descriptor().Info().FullName())}
+				}
+
+				for _, lt := range moduleTars {
+					// generate diff id
+					diffID, err := dist.LayerDiffID(lt.Path())
+					diffIDs = append(diffIDs, diffID)
+					infos = append(infos, lt.Info())
+					layerTars = append(layerTars, lt.Path())
+					if err != nil {
+						lids[i] <- modInfo{err: errors.Wrapf(err,
+							"getting content hashes for %s %s",
+							kind,
+							style.Symbol(lt.Info().FullName()),
+						)}
+					}
+				}
+			} else if kind == buildpack.KindExtension {
+				// create tar file
+				layerTar, err := buildpack.ToLayerTar(modTmpDir, module)
+				if err != nil {
+					lids[i] <- modInfo{err: err}
+				}
+
 				// generate diff id
-				diffID, err := dist.LayerDiffID(lt)
-				diffIDs = append(diffIDs, diffID)
-				info := parseSparseModuleInfo(lt)
-				infos = append(infos, info)
+				diffID, err := dist.LayerDiffID(layerTar)
+				info := module.Descriptor().Info()
 				if err != nil {
 					lids[i] <- modInfo{err: errors.Wrapf(err,
 						"getting content hashes for %s %s",
 						kind,
 						style.Symbol(info.FullName()),
 					)}
 				}
+
+				diffIDs = append(diffIDs, diffID)
+				infos = append(infos, info)
+				layerTars = append(layerTars, layerTar)
 			}
 
 			lids[i] <- modInfo{
@@ -618,15 +642,33 @@ func (b *Builder) addExplodedModules(kind string, logger logging.Logger, tmpDir
 			return mi.err
 		}
 		// maybe we got multiple modules back
+		if len(mi.layerTars) == 0 {
+			// skip if empty
+			logger.Debugf("%s %s is a component of a flattened buildpack that will be added elsewhere, skipping...", istrings.Title(kind), style.Symbol(module.Descriptor().Info().FullName()))
+			continue
+		}
 
 		for idx := range mi.layerTars {
 			info, diffID, layerTar := mi.infos[idx], mi.diffIDs[idx], mi.layerTars[idx]
 
-			// skip if empty
-			if diffID.String() == "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
-				// tar is empty
-				logger.Debugf("%s %s is a component of a flattened buildpack that will be added elsewhere, skipping...", istrings.Title(kind), style.Symbol(info.FullName()))
-				continue
+			if info.FullName() != fmt.Sprintf("%s@%s", module.Descriptor().EscapedID(), module.Descriptor().Info().Version) && info.FullName() != module.Descriptor().Info().FullName() {
+				// we need to look for the correct module
+				found := false
+				for _, m := range additionalModules {
+					if info.FullName() == fmt.Sprintf("%s@%s", m.Descriptor().EscapedID(), m.Descriptor().Info().Version) || info.FullName() == module.Descriptor().Info().FullName() {
+						module = m
+						found = true
+						break
+					}
+				}
+				info = module.Descriptor().Info()
+				if !found {
+					return errors.Errorf(
+						"module %s %s could not be found",
+						kind,
+						style.Symbol(info.FullName()),
+					)
+				}
 			}
 
 			// check against builder layers

pkg/buildpack/buildpack.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 
 	"github.com/BurntSushi/toml"
 	"github.com/buildpacks/lifecycle/api"
@@ -326,44 +327,104 @@ func ToLayerTar(dest string, module BuildModule) (string, error) {
 	return layerTar, nil
 }
 
-func ToNLayerTar(dest string, module BuildModule) ([]string, error) {
-	descriptor := module.Descriptor()
+func ToNLayerTar(dest string, module BuildModule) ([]ModuleTar, error) {
 	modReader, err := module.Open()
 	if err != nil {
-		return "", errors.Wrap(err, "opening blob")
+		return nil, errors.Wrap(err, "opening blob")
 	}
 	defer modReader.Close()
 
-	layerTar := filepath.Join(dest, fmt.Sprintf("%s.%s.tar", descriptor.EscapedID(), descriptor.Info().Version))
-	fh, err := os.Create(layerTar)
-	if err != nil {
-		return "", errors.Wrap(err, "create file for tar")
-	}
-	defer fh.Close()
+	tarCollection := newModuleTarCollection(dest)
 
 	// the build module __could__ have more than one buildpack on it,
 	// if we detect multiple buildpacks, explode them
-	tr := tar.NewReader(fh)
-	var bps []string                // id:version
-	layerTars := []string{layerTar} // maybe more will be added
+	tr := tar.NewReader(modReader)
 	for {
-		hdr, err := tr.Next()
+		header, err := tr.Next()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, errors.Wrapf(err, "failed to read next data '%s'", header.Name)
+		}
+
+		id := module.Descriptor().EscapedID()
+		version := module.Descriptor().Info().Version
+		if mustBeSplit(header, module.Descriptor()) {
+			id, version = parseBpIDAndVersion(header)
+			if version == "" {
+				continue
+			}
+		}
+
+		bpTar, err := tarCollection.get(id, version)
+		if err != nil {
+			return nil, err
+		}
+
+		err = bpTar.writer.WriteHeader(header)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to write header for '%s'", header.Name)
+		}
+
+		buf, err := io.ReadAll(tr)
 		if err != nil {
-			panic(err) // TODO
+			return nil, errors.Wrapf(err, "failed to read contents of '%s'", header.Name)
 		}
-		bp := parseBpIDAndVersion(hdr) // looks for /cnb/buildpacks/bp-id/bp-version header
-		if newBp(bp) {
-			// start a new tar file `layerTar2`
-		} else {
-			// write to `layerTar`
+
+		_, err = bpTar.writer.Write(buf)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to write contents to '%s'", header.Name)
 		}
 	}
 
-	//if _, err := io.Copy(fh, modReader); err != nil {
-	//	return "", errors.Wrap(err, "writing blob to tar")
-	//}
+	errs := tarCollection.close()
+	if len(errs) > 0 {
+		// TODO add errors
+		return nil, errors.New("closing file")
+	}
+
+	return tarCollection.getModules(), nil
+}
+
+func mustBeSplit(hdr *tar.Header, desc Descriptor) bool {
+	if hdr.Typeflag == tar.TypeDir {
+		rootFolder := filepath.Join(dist.BuildpacksDir, desc.EscapedID())
+		if strings.HasPrefix(hdr.Name, rootFolder) {
+			if path.Dir(hdr.Name) != dist.BuildpacksDir {
+				versionFolder := filepath.Join(rootFolder, desc.Info().Version)
+				return !strings.HasPrefix(hdr.Name, versionFolder)
+			}
+			return false // it's buildpack root folder, do not split, but maybe we need to write?
+		}
+		return true // it's a different buildpack root folder
+	}
+	return !strings.HasPrefix(hdr.Name, filepath.Join(dist.BuildpacksDir, desc.EscapedID(), desc.Info().Version))
+}
 
-	return layerTars, nil
+func parseBpIDAndVersion(hdr *tar.Header) (id, version string) {
+	// if !strings.HasPrefix(hdr.Name, dist.BuildpacksDir) {
+	// error ?
+	// }
+	// splitting "/cnb/buildpacks/{ID}/{version}/*" returns
+	// [0] = "" -> first element is empty
+	// [1] = "cnb"
+	// [2] = "buildpacks"
+	// [3] = "{ID}"
+	// [4] = "{version}"
+	// ...
+	parts := strings.Split(hdr.Name, "/")
+	size := len(parts)
+	switch {
+	case size < 4:
+		// error
+	case size == 4:
+		id = parts[3]
+	case size >= 5:
+		id = parts[3]
+		version = parts[4]
+	}
+	return id, version
 }
 
 // Set returns a set of the given string slice.
@@ -376,3 +437,82 @@ func Set(exclude []string) map[string]struct{} {
 	}
 	return excludedModules
 }
+
+type ModuleTar interface {
+	Info() dist.ModuleInfo
+	Path() string
+}
+
+type moduleTar struct {
+	info   dist.ModuleInfo
+	path   string
+	writer archive.TarWriter
+}
+
+func (t moduleTar) Info() dist.ModuleInfo {
+	return t.info
+}
+
+func (t moduleTar) Path() string {
+	return t.path
+}
+
+func newModuleTar(dest, id, version string) (moduleTar, error) {
+	layerTar := filepath.Join(dest, fmt.Sprintf("%s.%s.tar", id, version))
+	fh, err := os.Create(layerTar)
+	if err != nil {
+		return moduleTar{}, errors.Wrapf(err, "creating file at path %s", layerTar)
+	}
+	return moduleTar{
+		info: dist.ModuleInfo{
+			ID:      id,
+			Version: version,
+		},
+		path:   layerTar,
+		writer: tar.NewWriter(fh),
+	}, nil
+}
+
+type moduleTarCollection struct {
+	rootPath string
+	modules  map[string]moduleTar
+}
+
+func newModuleTarCollection(rootPath string) *moduleTarCollection {
+	return &moduleTarCollection{
+		rootPath: rootPath,
+		modules:  map[string]moduleTar{},
+	}
+}
+
+func (m *moduleTarCollection) get(id, version string) (moduleTar, error) {
+	key := fmt.Sprintf("%s@%s", id, version)
+	if _, ok := m.modules[key]; !ok {
+		module, err := newModuleTar(m.rootPath, id, version)
+		if err != nil {
+			return moduleTar{}, err
+		}
+		m.modules[key] = module
+	}
+	return m.modules[key], nil
+}
+
+func (m *moduleTarCollection) getModules() []ModuleTar {
+	var modulesTar []ModuleTar
+	for _, v := range m.modules {
+		v.writer.Close()
+		modulesTar = append(modulesTar, v)
+	}
+	return modulesTar
+}
+
+func (m *moduleTarCollection) close() []error {
+	var errors []error
+	for _, v := range m.modules {
+		err := v.writer.Close()
+		if err != nil {
+			errors = append(errors, err)
+		}
+	}
+	return errors
+}