Skip to content

Commit

Permalink
skipping flatten acceptance on windows based containers
Browse files Browse the repository at this point in the history 
Signed-off-by: Juan Bustamante <jbustamante@vmware.com>
  • Loading branch information
@jjbustamante
jjbustamante committed Jun 15, 2023
1 parent 325c400 commit d4441ca
Show file tree
Hide file tree
Showing 8 changed files with 312 additions and 167 deletions.
1 change: 1 addition & 0 deletions acceptance/acceptance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2781,6 +2781,7 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
when("--flatten", func() {
it("creates the package as a single layer", func() {
h.SkipIf(t, !pack.SupportsFeature(invoke.BuildpackFlatten), "")
h.SkipIf(t, imageManager.HostOS() == "windows", "These tests are not yet compatible with Windows-based containers")

packageTomlPath := generatePackageTomlWithOS(t, assert, pack, tmpDir, simplePackageConfigFixtureName, imageManager.HostOS())
nestedPackageName := "test/package-" + h.RandString(10)
Expand Down
82 changes: 54 additions & 28 deletions internal/builder/builder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1166,57 +1166,83 @@ func explodeBuildModules(kind, tmpDir string, additionalModules []buildpack.Buil
// create directory
modTmpDir := filepath.Join(tmpDir, fmt.Sprintf("%s-%s", kind, strconv.Itoa(i)))
if err := os.MkdirAll(modTmpDir, os.ModePerm); err != nil {
lids[i] <- modInfo{err: errors.Wrapf(err, "creating %s temp dir", kind)}
moduleTar := &errModuleTar{
module: module,
}
lids[i] <- modInfo{moduleTars: []buildpack.ModuleTar{moduleTar}, err: errors.Wrapf(err, "creating %s temp dir", kind)}
}
moduleTars, err := buildpack.ToNLayerTar(modTmpDir, module, logger)
if err != nil {
err = errors.Wrapf(err, "creating %s tar file", module.Descriptor().Info().FullName())
moduleTar := &errModuleTar{
module: module,
}
lids[i] <- modInfo{moduleTars: []buildpack.ModuleTar{moduleTar}, err: errors.Wrapf(err, "creating %s tar file", module.Descriptor().Info().FullName())}
}
lids[i] <- modInfo{moduleTars: moduleTars, err: err}
lids[i] <- modInfo{moduleTars: moduleTars}
}(i, module)
}

var result []explodedBuildModule

// maybe we got multiple modules back, we need to skip the empty ones
for i := 0; i < len(lids); i++ {
for i, module := range additionalModules {
mi := <-lids[i]
if mi.err != nil {
eBM := explodedBuildModule{err: mi.err}
result = append(result, eBM)
}
for _, moduleTar := range mi.moduleTars {
// it could be an individual buildpack or flattened buildpack that writes an empty tar on disk
eBM := explodedBuildModule{moduleTar: moduleTar}
diffID, err := dist.LayerDiffID(moduleTar.Path())
if err != nil {
eBM.err = err
// it means an individual or flattened Buildpack throw an error
eBM := explodedBuildModule{
moduleTar: mi.moduleTars[0], // we know it is associated with just one
err: mi.err,
}
if diffID.String() == "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
logger.Debugf("%s %s is a component of a flattened buildpack that will be added elsewhere, skipping...", istrings.Title(kind), style.Symbol(moduleTar.Info().FullName()))
continue // we don't need to keep empty tars
}
// it is an individual buildpack
eBM.diffIDs = diffID
result = append(result, eBM)
} else {
for _, moduleTar := range mi.moduleTars {
// it could be an individual Buildpack or flattened Buildpack that writes an empty tar on disk
eBM := explodedBuildModule{moduleTar: moduleTar}
diffID, err := dist.LayerDiffID(moduleTar.Path())
if err != nil {
eBM.err = err
}
if diffID.String() == "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" {
logger.Debugf("%s %s is a component of a flattened buildpack that will be added elsewhere, skipping...", istrings.Title(kind), style.Symbol(moduleTar.Info().FullName()))
continue // we don't need to keep empty tars
}
// it is an individual buildpack
eBM.diffIDs = diffID
if moduleTar.Info().FullName() == fmt.Sprintf("%s@%s", module.Descriptor().EscapedID(), module.Descriptor().Info().Version) ||
moduleTar.Info().FullName() == module.Descriptor().Info().FullName() {
eBM.module = module
}
result = append(result, eBM)
}
}
}

// we need to match the exploded modules with its corresponding BuildModule.
// this is important when flattened modules where included
for i, eBM := range result {
if eBM.err != nil {
continue
}
for _, module := range additionalModules {
if eBM.moduleTar.Info().FullName() == fmt.Sprintf("%s@%s", module.Descriptor().EscapedID(), module.Descriptor().Info().Version) ||
eBM.moduleTar.Info().FullName() == module.Descriptor().Info().FullName() {
eBM.module = module
result[i] = eBM
break
if eBM.module == nil {
for _, module := range additionalModules {
if eBM.moduleTar.Info().FullName() == fmt.Sprintf("%s@%s", module.Descriptor().EscapedID(), module.Descriptor().Info().Version) ||
eBM.moduleTar.Info().FullName() == module.Descriptor().Info().FullName() {
eBM.module = module
result[i] = eBM
break
}
}
}
}

return result
}

type errModuleTar struct {
module buildpack.BuildModule
}

func (e *errModuleTar) Info() dist.ModuleInfo {
return e.module.Descriptor().Info()
}

func (e *errModuleTar) Path() string {
return ""
}
4 changes: 4 additions & 0 deletions internal/fakes/fake_buildpack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,3 +99,7 @@ func (b *fakeBuildpack) Open() (io.ReadCloser, error) {

return tarBuilder.Reader(archive.DefaultTarWriterFactory()), nil
}

func (b *fakeBuildpack) ContainsFlattenedModules() bool {
return false
}
4 changes: 4 additions & 0 deletions internal/fakes/fake_extension.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,3 +97,7 @@ func (b *fakeExtension) Open() (io.ReadCloser, error) {

return tarBuilder.Reader(archive.DefaultTarWriterFactory()), nil
}

func (b *fakeExtension) ContainsFlattenedModules() bool {
return false
}
134 changes: 61 additions & 73 deletions pkg/buildpack/buildpack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ type BuildModule interface {
// timestamp and root UID/GID).
Open() (io.ReadCloser, error)
Descriptor() Descriptor
ContainsFlattenedModules() bool
}

type Descriptor interface {
Expand All @@ -51,22 +52,48 @@ type Blob interface {
Open() (io.ReadCloser, error)
}

type BlobOption func(*blobOption) error

type blobOption struct {
flattened bool
}

func Flattened() BlobOption {
return func(o *blobOption) error {
o.flattened = true
return nil
}
}

type buildModule struct {
descriptor Descriptor
Blob `toml:"-"`
flattened bool
}

func (b *buildModule) Descriptor() Descriptor {
return b.descriptor
}

func (b *buildModule) ContainsFlattenedModules() bool {
return b.flattened
}

// FromBlob constructs a buildpack or extension from a blob. It is assumed that the buildpack
// contents are structured as per the distribution spec (currently '/cnb/buildpacks/{ID}/{version}/*' or
// '/cnb/extensions/{ID}/{version}/*').
func FromBlob(descriptor Descriptor, blob Blob) BuildModule {
func FromBlob(descriptor Descriptor, blob Blob, ops ...BlobOption) BuildModule {
blobOpts := &blobOption{}
for _, op := range ops {
if err := op(blobOpts); err != nil {
return nil
}
}

return &buildModule{
Blob: blob,
descriptor: descriptor,
flattened: blobOpts.flattened,
}
}

Expand Down Expand Up @@ -330,21 +357,8 @@ func ToLayerTar(dest string, module BuildModule) (string, error) {
}

func ToNLayerTar(dest string, module BuildModule, logger logging.Logger) ([]ModuleTar, error) {
mustSplit, err := split(module, logger)
if err != nil {
return nil, err
}

if !mustSplit {
tarFile, err := ToLayerTar(dest, module)
if err != nil {
return nil, err
}
layerTar := &moduleTar{
info: module.Descriptor().Info(),
path: tarFile,
}
return []ModuleTar{layerTar}, nil
if !module.ContainsFlattenedModules() {
return handleSingleOrEmptyModule(dest, module)
}

modReader, err := module.Open()
Expand All @@ -359,26 +373,32 @@ func ToNLayerTar(dest string, module BuildModule, logger logging.Logger) ([]Modu
// the build module __could__ have more than one buildpack on it,
// if we detect multiple buildpacks, explode them
tr := tar.NewReader(modReader)
headersProcessed := 0
for {
header, err := tr.Next()
if err != nil {
if err == io.EOF {
if headersProcessed == 0 {
// If the reader was empty, we need to write an empty tar on disk
return handleSingleOrEmptyModule(dest, module)
}
break
}
return nil, errors.Wrapf(err, "failed to read next data '%s'", header.Name)
}

id := module.Descriptor().EscapedID()
version := module.Descriptor().Info().Version
if mustBeSplit(header, module.Descriptor(), logger) {
id, version = parseBpIDAndVersion(header)
id, version := parseBpIDAndVersion(header)
if !matchIDVersion(module.Descriptor(), id, version) {
if version == "" && header.Typeflag == tar.TypeDir {
// we need to save it and write it later
if _, ok := rootFolderHeaders[id]; !ok {
rootFolderHeaders[id] = header
continue
}
}
} else {
id = module.Descriptor().EscapedID()
version = module.Descriptor().Info().Version
}

bpTar, err := tarCollection.get(id, version)
Expand All @@ -388,14 +408,14 @@ func ToNLayerTar(dest string, module BuildModule, logger logging.Logger) ([]Modu

if rootHeader, ok := rootFolderHeaders[id]; ok {
err = bpTar.writer.WriteHeader(rootHeader)
logger.Debugf("writing root folder %s into module tar with id=s%, version=%", rootHeader.Name, id, version)
logger.Debugf("writing root folder %s into module tar with id=%s", rootHeader.Name, id)
if err != nil {
return nil, errors.Wrapf(err, "failed to write root header for '%s'", rootHeader.Name)
}
delete(rootFolderHeaders, id)
}

logger.Debugf("writing %s it module tar with id=%s, version=%s", header.Name, id, version)
logger.Debugf("writing %s into module tar with id=%s, version=%s", header.Name, id, version)
err = bpTar.writer.WriteHeader(header)
if err != nil {
return nil, errors.Wrapf(err, "failed to write header for '%s'", header.Name)
Expand All @@ -410,69 +430,26 @@ func ToNLayerTar(dest string, module BuildModule, logger logging.Logger) ([]Modu
if err != nil {
return nil, errors.Wrapf(err, "failed to write contents to '%s'", header.Name)
}
headersProcessed++
}

errs := tarCollection.close()
if len(errs) > 0 {
// TODO add errors
return nil, errors.New("closing file")
return nil, errors.New("closing files")
}

return tarCollection.getModules(), nil
}

func split(module BuildModule, logger logging.Logger) (bool, error) {
modReader, err := module.Open()
if err != nil {
return false, errors.Wrap(err, "opening blob")
}
defer modReader.Close()

tr := tar.NewReader(modReader)
for {
header, err := tr.Next()
if err != nil {
if err == io.EOF {
break
}
return false, errors.Wrapf(err, "failed to read next data '%s'", header.Name)
}
if ok := mustBeSplit(header, module.Descriptor(), logger); ok {
return true, nil
}
}
return false, nil
}

func mustBeSplit(hdr *tar.Header, desc Descriptor, logger logging.Logger) bool {
// This is a hack, see: internal/fake/fake_buildpack_blob.go
if !strings.HasPrefix(hdr.Name, dist.BuildpacksDir) {
logger.Debugf("header %s to not have /cnb/buildpacks prefix", hdr.Name)
return false
}
if hdr.Typeflag == tar.TypeDir {
rootFolder := filepath.Join(dist.BuildpacksDir, desc.EscapedID())
if strings.Contains(rootFolder, hdr.Name) {
if path.Dir(hdr.Name) != dist.BuildpacksDir {
versionFolder := filepath.Join(rootFolder, desc.Info().Version)
logger.Debugf("checking if %s has prefix version folder %s", hdr.Name, versionFolder)
return !strings.HasPrefix(hdr.Name, versionFolder)
}
logger.Debugf("%s is a module root folder %s, do not split it", hdr.Name, rootFolder)
return false
}
logger.Debugf("%s it's a different buildpack root folder %s", hdr.Name, rootFolder)
return true
func matchIDVersion(desc Descriptor, id, version string) bool {
sameID := id == desc.EscapedID() || id == desc.Info().ID
if version == "" {
return sameID
}
modulePath := filepath.Join(dist.BuildpacksDir, desc.EscapedID(), desc.Info().Version)
logger.Debugf("checking if %s has prefix version folder %s", hdr.Name, modulePath)
return !strings.HasPrefix(hdr.Name, modulePath)
return sameID && version == desc.Info().Version
}

func parseBpIDAndVersion(hdr *tar.Header) (id, version string) {
// if !strings.HasPrefix(hdr.Name, dist.BuildpacksDir) {
// error ?
// }
// splitting "/cnb/buildpacks/{ID}/{version}/*" returns
// [0] = "" -> first element is empty
// [1] = "cnb"
Expand All @@ -494,6 +471,18 @@ func parseBpIDAndVersion(hdr *tar.Header) (id, version string) {
return id, version
}

func handleSingleOrEmptyModule(dest string, module BuildModule) ([]ModuleTar, error) {
tarFile, err := ToLayerTar(dest, module)
if err != nil {
return nil, err
}
layerTar := &moduleTar{
info: module.Descriptor().Info(),
path: tarFile,
}
return []ModuleTar{layerTar}, nil
}

// Set returns a set of the given string slice.
func Set(exclude []string) map[string]struct{} {
type void struct{}
Expand Down Expand Up @@ -567,7 +556,6 @@ func (m *moduleTarCollection) get(id, version string) (moduleTar, error) {
func (m *moduleTarCollection) getModules() []ModuleTar {
var modulesTar []ModuleTar
for _, v := range m.modules {
v.writer.Close()
vv := &v // TODO check this, its a hack for lint
modulesTar = append(modulesTar, vv)
}
Expand Down
Loading

0 comments on commit d4441ca

Please sign in to comment.