Skip to content

Commit

Permalink
skipping flatten acceptance on windows based containers
Browse files Browse the repository at this point in the history 
Signed-off-by: Juan Bustamante <jbustamante@vmware.com>
  • Loading branch information
@jjbustamante
jjbustamante committed Jun 13, 2023
1 parent 325c400 commit da011d3
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 59 deletions.
1 change: 1 addition & 0 deletions acceptance/acceptance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2781,6 +2781,7 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
when("--flatten", func() {
it("creates the package as a single layer", func() {
h.SkipIf(t, !pack.SupportsFeature(invoke.BuildpackFlatten), "")
h.SkipIf(t, imageManager.HostOS() == "windows", "These tests are not yet compatible with Windows-based containers")

packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
nestedPackageName := "test/package-" + h.RandString(10)
Expand Down
72 changes: 50 additions & 22 deletions internal/builder/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1166,13 +1166,19 @@ func explodeBuildModules(kind, tmpDir string, additionalModules []buildpack.Buil
// create directory
modTmpDir := filepath.Join(tmpDir, fmt.Sprintf("%s-%s", kind, strconv.Itoa(i)))
if err := os.MkdirAll(modTmpDir, os.ModePerm); err != nil {
lids[i] <- modInfo{err: errors.Wrapf(err, "creating %s temp dir", kind)}
moduleTar := &errModuleTar{
module: module,
}
lids[i] <- modInfo{moduleTars: []buildpack.ModuleTar{moduleTar}, err: errors.Wrapf(err, "creating %s temp dir", kind)}
}
moduleTars, err := buildpack.ToNLayerTar(modTmpDir, module, logger)
moduleTars, err := buildpack.ToNLayerTar(kind, modTmpDir, module, logger)
if err != nil {
err = errors.Wrapf(err, "creating %s tar file", module.Descriptor().Info().FullName())
moduleTar := &errModuleTar{
module: module,
}
lids[i] <- modInfo{moduleTars: []buildpack.ModuleTar{moduleTar}, err: errors.Wrapf(err, "creating %s tar file", module.Descriptor().Info().FullName())}
}
lids[i] <- modInfo{moduleTars: moduleTars, err: err}
lids[i] <- modInfo{moduleTars: moduleTars}
}(i, module)
}

Expand All @@ -1182,32 +1188,34 @@ func explodeBuildModules(kind, tmpDir string, additionalModules []buildpack.Buil
for i := 0; i < len(lids); i++ {
mi := <-lids[i]
if mi.err != nil {
eBM := explodedBuildModule{err: mi.err}
result = append(result, eBM)
}
for _, moduleTar := range mi.moduleTars {
// it could be an individual buildpack or flattened buildpack that writes an empty tar on disk
eBM := explodedBuildModule{moduleTar: moduleTar}
diffID, err := dist.LayerDiffID(moduleTar.Path())
if err != nil {
eBM.err = err
}
if diffID.String() == "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
logger.Debugf("%s %s is a component of a flattened buildpack that will be added elsewhere, skipping...", istrings.Title(kind), style.Symbol(moduleTar.Info().FullName()))
continue // we don't need to keep empty tars
// it means an individual or flattened Buildpack throw an error
eBM := explodedBuildModule{
moduleTar: mi.moduleTars[0], // we know it is associated with just one
err: mi.err,
}
// it is an individual buildpack
eBM.diffIDs = diffID
result = append(result, eBM)
} else {
for _, moduleTar := range mi.moduleTars {
// it could be an individual Buildpack or flattened Buildpack that writes an empty tar on disk
eBM := explodedBuildModule{moduleTar: moduleTar}
diffID, err := dist.LayerDiffID(moduleTar.Path())
if err != nil {
eBM.err = err
}
if diffID.String() == "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
logger.Debugf("%s %s is a component of a flattened buildpack that will be added elsewhere, skipping...", istrings.Title(kind), style.Symbol(moduleTar.Info().FullName()))
continue // we don't need to keep empty tars
}
// it is an individual buildpack
eBM.diffIDs = diffID
result = append(result, eBM)
}
}
}

// we need to match the exploded modules with its corresponding BuildModule.
// this is important when flattened modules where included
for i, eBM := range result {
if eBM.err != nil {
continue
}
for _, module := range additionalModules {
if eBM.moduleTar.Info().FullName() == fmt.Sprintf("%s@%s", module.Descriptor().EscapedID(), module.Descriptor().Info().Version) ||
eBM.moduleTar.Info().FullName() == module.Descriptor().Info().FullName() {
Expand All @@ -1218,5 +1226,25 @@ func explodeBuildModules(kind, tmpDir string, additionalModules []buildpack.Buil
}
}

logger.Debug("Returning exploded modules:")
for _, eBM := range result {
if eBM.err != nil {
continue
}
logger.Debugf("%s", eBM.module.Descriptor().Info().FullName())
}

return result
}

type errModuleTar struct {
module buildpack.BuildModule
}

func (e *errModuleTar) Info() dist.ModuleInfo {
return e.module.Descriptor().Info()
}

func (e *errModuleTar) Path() string {
return ""
}
59 changes: 23 additions & 36 deletions pkg/buildpack/buildpack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,8 +329,8 @@ func ToLayerTar(dest string, module BuildModule) (string, error) {
return layerTar, nil
}

func ToNLayerTar(dest string, module BuildModule, logger logging.Logger) ([]ModuleTar, error) {
mustSplit, err := split(module, logger)
func ToNLayerTar(kind, dest string, module BuildModule, logger logging.Logger) ([]ModuleTar, error) {
mustSplit, err := split(kind, module, logger)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -368,17 +368,18 @@ func ToNLayerTar(dest string, module BuildModule, logger logging.Logger) ([]Modu
return nil, errors.Wrapf(err, "failed to read next data '%s'", header.Name)
}

id := module.Descriptor().EscapedID()
version := module.Descriptor().Info().Version
if mustBeSplit(header, module.Descriptor(), logger) {
id, version = parseBpIDAndVersion(header)
id, version := parseBpIDAndVersion(header)
if !matchIDVersion(module.Descriptor(), id, version) {
if version == "" && header.Typeflag == tar.TypeDir {
// we need to save it and write it later
if _, ok := rootFolderHeaders[id]; !ok {
rootFolderHeaders[id] = header
continue
}
}
} else {
id = module.Descriptor().EscapedID()
version = module.Descriptor().Info().Version
}

bpTar, err := tarCollection.get(id, version)
Expand All @@ -388,14 +389,14 @@ func ToNLayerTar(dest string, module BuildModule, logger logging.Logger) ([]Modu

if rootHeader, ok := rootFolderHeaders[id]; ok {
err = bpTar.writer.WriteHeader(rootHeader)
logger.Debugf("writing root folder %s into module tar with id=s%, version=%", rootHeader.Name, id, version)
logger.Debugf("writing root folder %s into module tar with id=%s", rootHeader.Name, id)
if err != nil {
return nil, errors.Wrapf(err, "failed to write root header for '%s'", rootHeader.Name)
}
delete(rootFolderHeaders, id)
}

logger.Debugf("writing %s it module tar with id=%s, version=%s", header.Name, id, version)
logger.Debugf("writing %s into module tar with id=%s, version=%s", header.Name, id, version)
err = bpTar.writer.WriteHeader(header)
if err != nil {
return nil, errors.Wrapf(err, "failed to write header for '%s'", header.Name)
Expand All @@ -421,7 +422,10 @@ func ToNLayerTar(dest string, module BuildModule, logger logging.Logger) ([]Modu
return tarCollection.getModules(), nil
}

func split(module BuildModule, logger logging.Logger) (bool, error) {
func split(kind string, module BuildModule, logger logging.Logger) (bool, error) {
if kind == KindExtension {
return false, nil
}
modReader, err := module.Open()
if err != nil {
return false, errors.Wrap(err, "opening blob")
Expand All @@ -437,42 +441,25 @@ func split(module BuildModule, logger logging.Logger) (bool, error) {
}
return false, errors.Wrapf(err, "failed to read next data '%s'", header.Name)
}
if ok := mustBeSplit(header, module.Descriptor(), logger); ok {
return true, nil
id, version := parseBpIDAndVersion(header)
if id != "" && version != "" {
if ok := matchIDVersion(module.Descriptor(), id, version); !ok {
return true, nil
}
}
}
return false, nil
}

func mustBeSplit(hdr *tar.Header, desc Descriptor, logger logging.Logger) bool {
// This is a hack, see: internal/fake/fake_buildpack_blob.go
if !strings.HasPrefix(hdr.Name, dist.BuildpacksDir) {
logger.Debugf("header %s to not have /cnb/buildpacks prefix", hdr.Name)
return false
}
if hdr.Typeflag == tar.TypeDir {
rootFolder := filepath.Join(dist.BuildpacksDir, desc.EscapedID())
if strings.Contains(rootFolder, hdr.Name) {
if path.Dir(hdr.Name) != dist.BuildpacksDir {
versionFolder := filepath.Join(rootFolder, desc.Info().Version)
logger.Debugf("checking if %s has prefix version folder %s", hdr.Name, versionFolder)
return !strings.HasPrefix(hdr.Name, versionFolder)
}
logger.Debugf("%s is a module root folder %s, do not split it", hdr.Name, rootFolder)
return false
}
logger.Debugf("%s it's a different buildpack root folder %s", hdr.Name, rootFolder)
return true
func matchIDVersion(desc Descriptor, id, version string) bool {
sameID := id == desc.EscapedID() || id == desc.Info().ID
if version == "" {
return sameID
}
modulePath := filepath.Join(dist.BuildpacksDir, desc.EscapedID(), desc.Info().Version)
logger.Debugf("checking if %s has prefix version folder %s", hdr.Name, modulePath)
return !strings.HasPrefix(hdr.Name, modulePath)
return sameID && version == desc.Info().Version
}

func parseBpIDAndVersion(hdr *tar.Header) (id, version string) {
// if !strings.HasPrefix(hdr.Name, dist.BuildpacksDir) {
// error ?
// }
// splitting "/cnb/buildpacks/{ID}/{version}/*" returns
// [0] = "" -> first element is empty
// [1] = "cnb"
Expand Down
2 changes: 1 addition & 1 deletion pkg/buildpack/buildpack_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -632,7 +632,7 @@ version = "buildpack-2-version-1"
},
)

tarPaths, err := buildpack.ToNLayerTar(tmpDir, bp, logger)
tarPaths, err := buildpack.ToNLayerTar(buildpack.KindBuildpack, tmpDir, bp, logger)
h.AssertNil(t, err)
h.AssertEq(t, len(tarPaths), 2)
for _, tarPath := range tarPaths {
Expand Down

0 comments on commit da011d3

Please sign in to comment.