Skip to content

Commit

Permalink
Fixing condition to run pack builder create --flatten acceptance tests
Browse files Browse the repository at this point in the history 
Signed-off-by: Juan Bustamante <jbustamante@vmware.com>
  • Loading branch information
@jjbustamante
jjbustamante committed Jun 9, 2023
1 parent 6f76a88 commit c8041e7
Show file tree
Hide file tree
Showing 5 changed files with 149 additions and 157 deletions.
2 changes: 1 addition & 1 deletion acceptance/acceptance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2780,7 +2780,7 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]

when("--flatten", func() {
it("creates the package as a single layer", func() {
//h.SkipIf(t, !pack.SupportsFeature(invoke.BuildpackFlatten), "")
h.SkipIf(t, !pack.SupportsFeature(invoke.BuildpackFlatten), "")

packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
nestedPackageName := "test/package-" + h.RandString(10)
Expand Down
3 changes: 1 addition & 2 deletions acceptance/invoke/pack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ var featureTests = map[Feature]func(i *PackInvoker) bool{
return !i.atLeast("v0.30.0")
},
BuildpackFlatten: func(i *PackInvoker) bool {
return !i.atLeast("v0.30.0")
return i.atLeast("v0.30.0")
},
}

Expand All @@ -264,7 +264,6 @@ func (i *PackInvoker) semanticVersion() *semver.Version {
version := i.Version()
semanticVersion, err := semver.NewVersion(strings.TrimPrefix(strings.Split(version, " ")[0], "v"))
i.assert.Nil(err)

return semanticVersion
}

Expand Down
247 changes: 109 additions & 138 deletions internal/builder/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,13 @@ type toAdd struct {
module buildpack.BuildModule
}

type explodedBuildModule struct {
module buildpack.BuildModule
moduleTar buildpack.ModuleTar
diffIDs v1.Hash
err error
}

type BuilderOption func(*options) error

type options struct {
Expand Down Expand Up @@ -560,158 +567,48 @@ func (b *Builder) Save(logger logging.Logger, creatorMetadata CreatorMetadata) e

func (b *Builder) addExplodedModules(kind string, logger logging.Logger, tmpDir string, image imgutil.Image, additionalModules []buildpack.BuildModule, layers dist.ModuleLayers) error {
collectionToAdd := map[string]toAdd{}
explodedAdditionalModules := explodeBuildModules(kind, tmpDir, additionalModules, logger)

type modInfo struct {
infos []dist.ModuleInfo
layerTars []string
diffIDs []v1.Hash
err error
}

lids := make([]chan modInfo, len(additionalModules))
for i := range lids {
lids[i] = make(chan modInfo, 1)
}

for i, module := range additionalModules {
go func(i int, module buildpack.BuildModule) {
// create directory
modTmpDir := filepath.Join(tmpDir, fmt.Sprintf("%s-%s", kind, strconv.Itoa(i)))
if err := os.MkdirAll(modTmpDir, os.ModePerm); err != nil {
lids[i] <- modInfo{err: errors.Wrapf(err, "creating %s temp dir", kind)}
}

var diffIDs []v1.Hash
var infos []dist.ModuleInfo // only id & version will be populated
var layerTars []string

if kind == buildpack.KindBuildpack {
moduleTars, err := buildpack.ToNLayerTar(modTmpDir, module)
if err != nil {
lids[i] <- modInfo{err: errors.Wrapf(err, "creating %s tar file", module.Descriptor().Info().FullName())}
}

for _, lt := range moduleTars {
// generate diff id
diffID, err := dist.LayerDiffID(lt.Path())
diffIDs = append(diffIDs, diffID)
infos = append(infos, lt.Info())
layerTars = append(layerTars, lt.Path())
if err != nil {
lids[i] <- modInfo{err: errors.Wrapf(err,
"getting content hashes for %s %s",
kind,
style.Symbol(lt.Info().FullName()),
)}
}
}
} else if kind == buildpack.KindExtension {
// create tar file
layerTar, err := buildpack.ToLayerTar(modTmpDir, module)
if err != nil {
lids[i] <- modInfo{err: err}
}

// generate diff id
diffID, err := dist.LayerDiffID(layerTar)
info := module.Descriptor().Info()
if err != nil {
lids[i] <- modInfo{err: errors.Wrapf(err,
"getting content hashes for %s %s",
kind,
style.Symbol(info.FullName()),
)}
}

diffIDs = append(diffIDs, diffID)
infos = append(infos, info)
layerTars = append(layerTars, layerTar)
}

lids[i] <- modInfo{
infos: infos,
layerTars: layerTars,
diffIDs: diffIDs,
}
}(i, module)
}

for i, module := range additionalModules {
mi := <-lids[i]
for i, mi := range explodedAdditionalModules {
if mi.err != nil {
return mi.err
}
// maybe we got multiple modules back
if len(mi.layerTars) == 0 {
// skip if empty
logger.Debugf("%s %s is a component of a flattened buildpack that will be added elsewhere, skipping...", istrings.Title(kind), style.Symbol(module.Descriptor().Info().FullName()))
continue
}

for idx := range mi.layerTars {
info, diffID, layerTar := mi.infos[idx], mi.diffIDs[idx], mi.layerTars[idx]
info, diffID, layerTar, module := mi.module.Descriptor().Info(), mi.diffIDs, mi.moduleTar.Path(), mi.module

if diffID.String() == "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
// tar is empty
logger.Debugf("%s %s is a component of a flattened buildpack that will be added elsewhere, skipping...", istrings.Title(kind), style.Symbol(info.FullName()))
// check against builder layers
if existingInfo, ok := layers[info.ID][info.Version]; ok {
if existingInfo.LayerDiffID == diffID.String() {
logger.Debugf("%s %s already exists on builder with same contents, skipping...", istrings.Title(kind), style.Symbol(info.FullName()))
continue
}

if info.FullName() != fmt.Sprintf("%s@%s", module.Descriptor().EscapedID(), module.Descriptor().Info().Version) && info.FullName() != module.Descriptor().Info().FullName() {
// we need to look for the correct module
found := false
for _, m := range additionalModules {
if info.FullName() == fmt.Sprintf("%s@%s", m.Descriptor().EscapedID(), m.Descriptor().Info().Version) || info.FullName() == module.Descriptor().Info().FullName() {
module = m
found = true
break
}
}
info = module.Descriptor().Info()
if !found {
return errors.Errorf(
"module %s %s could not be found",
kind,
style.Symbol(info.FullName()),
)
} else {
whiteoutsTar, err := b.whiteoutLayer(tmpDir, i, info)
if err != nil {
return err
}
}

// check against builder layers
if existingInfo, ok := layers[info.ID][info.Version]; ok {
if existingInfo.LayerDiffID == diffID.String() {
logger.Debugf("%s %s already exists on builder with same contents, skipping...", istrings.Title(kind), style.Symbol(info.FullName()))
continue
} else {
whiteoutsTar, err := b.whiteoutLayer(tmpDir, i, info)
if err != nil {
return err
}

if err := image.AddLayer(whiteoutsTar); err != nil {
return errors.Wrap(err, "adding whiteout layer tar")
}
if err := image.AddLayer(whiteoutsTar); err != nil {
return errors.Wrap(err, "adding whiteout layer tar")
}

logger.Debugf(ModuleOnBuilderMessage, kind, style.Symbol(info.FullName()), style.Symbol(existingInfo.LayerDiffID), style.Symbol(diffID.String()))
}

// check against other modules to be added
if otherAdditionalMod, ok := collectionToAdd[info.FullName()]; ok {
if otherAdditionalMod.diffID == diffID.String() {
logger.Debugf("%s %s with same contents is already being added, skipping...", istrings.Title(kind), style.Symbol(info.FullName()))
continue
}
logger.Debugf(ModuleOnBuilderMessage, kind, style.Symbol(info.FullName()), style.Symbol(existingInfo.LayerDiffID), style.Symbol(diffID.String()))
}

logger.Debugf(ModulePreviouslyDefinedMessage, kind, style.Symbol(info.FullName()), style.Symbol(otherAdditionalMod.diffID), style.Symbol(diffID.String()))
// check against other modules to be added
if otherAdditionalMod, ok := collectionToAdd[info.FullName()]; ok {
if otherAdditionalMod.diffID == diffID.String() {
logger.Debugf("%s %s with same contents is already being added, skipping...", istrings.Title(kind), style.Symbol(info.FullName()))
continue
}

// note: if same id@version is in additionalModules, last one wins (see warnings above)
collectionToAdd[info.FullName()] = toAdd{
tarPath: layerTar,
diffID: diffID.String(),
module: module,
}
logger.Debugf(ModulePreviouslyDefinedMessage, kind, style.Symbol(info.FullName()), style.Symbol(otherAdditionalMod.diffID), style.Symbol(diffID.String()))
}

// note: if same id@version is in additionalModules, last one wins (see warnings above)
collectionToAdd[info.FullName()] = toAdd{
tarPath: layerTar,
diffID: diffID.String(),
module: module,
}
}

Expand Down Expand Up @@ -1249,3 +1146,77 @@ func sortKeys(collection map[string]toAdd) []string {
sort.Strings(keys)
return keys
}

// explodeBuildModules takes the given Build Modules and parallelize the reading operation of their internal tar files. In
// case a flattened module if found, it will split each flattened buildpack into an individual module and skip all the empty ones
// returns an explodedBuildModule array with the Build Module information but also the diffId and the tar file on disk
func explodeBuildModules(kind, tmpDir string, additionalModules []buildpack.BuildModule, logger logging.Logger) []explodedBuildModule {
type modInfo struct {
moduleTars []buildpack.ModuleTar
err error
}

lids := make([]chan modInfo, len(additionalModules))
for i := range lids {
lids[i] = make(chan modInfo, 1)
}

for i, module := range additionalModules {
go func(i int, module buildpack.BuildModule) {
// create directory
modTmpDir := filepath.Join(tmpDir, fmt.Sprintf("%s-%s", kind, strconv.Itoa(i)))
if err := os.MkdirAll(modTmpDir, os.ModePerm); err != nil {
lids[i] <- modInfo{err: errors.Wrapf(err, "creating %s temp dir", kind)}
}
moduleTars, err := buildpack.ToNLayerTar(modTmpDir, module, logger)
if err != nil {
err = errors.Wrapf(err, "creating %s tar file", module.Descriptor().Info().FullName())
}
lids[i] <- modInfo{moduleTars: moduleTars, err: err}
}(i, module)
}

var result []explodedBuildModule

// maybe we got multiple modules back, we need to skip the empty ones
for i := 0; i < len(lids); i++ {
mi := <-lids[i]
if mi.err != nil {
eBM := explodedBuildModule{err: mi.err}
result = append(result, eBM)
}
for _, moduleTar := range mi.moduleTars {
// it could be an individual buildpack or flattened buildpack that writes an empty tar on disk
eBM := explodedBuildModule{moduleTar: moduleTar}
diffID, err := dist.LayerDiffID(moduleTar.Path())
if err != nil {
eBM.err = err
}
if diffID.String() == "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
logger.Debugf("%s %s is a component of a flattened buildpack that will be added elsewhere, skipping...", istrings.Title(kind), style.Symbol(moduleTar.Info().FullName()))
continue // we don't need to keep empty tars
}
// it is an individual buildpack
eBM.diffIDs = diffID
result = append(result, eBM)
}
}

// we need to match the exploded modules with its corresponding BuildModule.
// this is important when flattened modules where included
for i, eBM := range result {
if eBM.err != nil {
continue
}
for _, module := range additionalModules {
if eBM.moduleTar.Info().FullName() == fmt.Sprintf("%s@%s", module.Descriptor().EscapedID(), module.Descriptor().Info().Version) ||
eBM.moduleTar.Info().FullName() == module.Descriptor().Info().FullName() {
eBM.module = module
result[i] = eBM
break
}
}
}

return result
}
Loading

0 comments on commit c8041e7

Please sign in to comment.