Merge pull request #1757 from itsdarshankumar/extension-downloader

Add support for extensions in pack Buildpack downloader

pkg/blob/blob.go

@@ -58,7 +58,6 @@ func (b blob) Open() (r io.ReadCloser, err error) {
 		defer fh.Close()
 		return gzr.Close()
 	})
-
 	return rc, nil
 }
 

pkg/buildpack/downloader.go

@@ -96,7 +96,6 @@ func (c *buildpackDownloader) Download(ctx context.Context, moduleURI string, op
 			return nil, nil, err
 		}
 	}
-
 	var mainBP BuildModule
 	var depBPs []BuildModule
 	switch locatorType {
@@ -198,13 +197,12 @@ func extractPackaged(ctx context.Context, kind string, pkgImageRef string, fetch
 	case KindBuildpack:
 		mainModule, depModules, err = extractBuildpacks(pkgImage)
 	case KindExtension:
-		return nil, nil, nil // TODO: add extractExtensions when `pack extension package` is supported in https://github.com/buildpacks/pack/issues/1489
+		mainModule, err = extractExtensions(pkgImage)
 	default:
 		return nil, nil, fmt.Errorf("unknown module kind: %s", kind)
 	}
 	if err != nil {
 		return nil, nil, errors.Wrapf(err, "extracting %ss from %s", kind, style.Symbol(pkgImageRef))
 	}
-
 	return mainModule, depModules, nil
 }

pkg/buildpack/downloader_test.go

@@ -310,6 +310,23 @@ func testBuildpackDownloader(t *testing.T, when spec.G, it spec.S) {
 					h.AssertEq(t, mainExt.Descriptor().Info().ID, "ext.one")
 				})
 			})
+
+			when("kind == packagedExtension", func() {
+				it("succeeds", func() {
+					packagedExtensionPath := filepath.Join("testdata", "tree-extension.cnb")
+					packagedExtensionURI, _ := paths.FilePathToURI(packagedExtensionPath, "")
+					mockDownloader.EXPECT().Download(gomock.Any(), packagedExtensionURI).Return(blob.NewBlob(packagedExtensionPath), nil).AnyTimes()
+					downloadOptions = buildpack.DownloadOptions{
+						ImageOS:         "linux",
+						ModuleKind:      "extension",
+						RelativeBaseDir: "testdata",
+						Daemon:          true,
+						PullPolicy:      image.PullAlways,
+					}
+					mainExt, _, _ := buildpackDownloader.Download(context.TODO(), "tree-extension.cnb", downloadOptions)
+					h.AssertEq(t, mainExt.Descriptor().Info().ID, "samples-tree")
+				})
+			})
 		})
 
 		when("package image is not a valid package", func() {

pkg/buildpack/locator_type.go

@@ -92,7 +92,6 @@ func parseNakedLocator(locator, relativeBaseDir string, buildpacksFromBuilder []
 	//   2. Does it match a buildpack ID in the builder
 	//   3. Does it look like a Buildpack Registry ID
 	//   4. Does it look like a Docker ref
-
 	if isLocalFile(locator, relativeBaseDir) {
 		return URILocator
 	}

pkg/buildpack/oci_layout_package.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"compress/gzip"
 	"encoding/json"
+	"fmt"
 	"io"
 	"path"
 	"strings"
@@ -41,7 +42,7 @@ func IsOCILayoutBlob(blob blob2.Blob) (bool, error) {
 
 // BuildpacksFromOCILayoutBlob constructs buildpacks from a blob in OCI layout format.
 func BuildpacksFromOCILayoutBlob(blob Blob) (mainBP BuildModule, dependencies []BuildModule, err error) {
-	layoutPackage, err := newOCILayoutPackage(blob)
+	layoutPackage, err := newOCILayoutPackage(blob, KindBuildpack)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -51,11 +52,16 @@ func BuildpacksFromOCILayoutBlob(blob Blob) (mainBP BuildModule, dependencies []
 
 // ExtensionsFromOCILayoutBlob constructs extensions from a blob in OCI layout format.
 func ExtensionsFromOCILayoutBlob(blob Blob) (mainExt BuildModule, err error) {
-	return nil, nil // TODO: add extractExtensions when `pack extension package` is supported in https://github.com/buildpacks/pack/issues/1489
+	layoutPackage, err := newOCILayoutPackage(blob, KindExtension)
+	if err != nil {
+		return nil, err
+	}
+
+	return extractExtensions(layoutPackage)
 }
 
 func ConfigFromOCILayoutBlob(blob Blob) (config v1.ImageConfig, err error) {
-	layoutPackage, err := newOCILayoutPackage(blob)
+	layoutPackage, err := newOCILayoutPackage(blob, KindBuildpack)
 	if err != nil {
 		return v1.ImageConfig{}, err
 	}
@@ -68,7 +74,7 @@ type ociLayoutPackage struct {
 	blob      Blob
 }
 
-func newOCILayoutPackage(blob Blob) (*ociLayoutPackage, error) {
+func newOCILayoutPackage(blob Blob, kind string) (*ociLayoutPackage, error) {
 	index := &v1.Index{}
 
 	if err := unmarshalJSONFromBlob(blob, "/index.json", index); err != nil {
@@ -96,10 +102,20 @@ func newOCILayoutPackage(blob Blob) (*ociLayoutPackage, error) {
 	if err := unmarshalJSONFromBlob(blob, pathFromDescriptor(manifest.Config), imageInfo); err != nil {
 		return nil, err
 	}
-
-	layersLabel := imageInfo.Config.Labels[dist.BuildpackLayersLabel]
-	if layersLabel == "" {
-		return nil, errors.Errorf("label %s not found", style.Symbol(dist.BuildpackLayersLabel))
+	var layersLabel string
+	switch kind {
+	case KindBuildpack:
+		layersLabel = imageInfo.Config.Labels[dist.BuildpackLayersLabel]
+		if layersLabel == "" {
+			return nil, errors.Errorf("label %s not found", style.Symbol(dist.BuildpackLayersLabel))
+		}
+	case KindExtension:
+		layersLabel = imageInfo.Config.Labels[dist.ExtensionLayersLabel]
+		if layersLabel == "" {
+			return nil, errors.Errorf("label %s not found", style.Symbol(dist.ExtensionLayersLabel))
+		}
+	default:
+		return nil, fmt.Errorf("unknown module kind: %s", kind)
 	}
 
 	bpLayers := dist.ModuleLayers{}

pkg/buildpack/oci_layout_package_test.go

@@ -57,19 +57,18 @@ func testOCILayoutPackage(t *testing.T, when spec.G, it spec.S) {
 		})
 	})
 
-	when.Pend("#ExtensionsFromOCILayoutBlob", func() { // TODO: add fixture when `pack extension package` is supported in https://github.com/buildpacks/pack/issues/1489
+	when("#ExtensionsFromOCILayoutBlob", func() {
 		it("extracts buildpacks", func() {
-			ext, err := buildpack.ExtensionsFromOCILayoutBlob(blob.NewBlob(filepath.Join("testdata", "hello-extensions.cnb")))
+			ext, err := buildpack.ExtensionsFromOCILayoutBlob(blob.NewBlob(filepath.Join("testdata", "tree-extension.cnb")))
 			h.AssertNil(t, err)
 
-			h.AssertEq(t, ext.Descriptor().Info().ID, "io.buildpacks.samples.hello-extensions")
+			h.AssertEq(t, ext.Descriptor().Info().ID, "samples-tree")
 			h.AssertEq(t, ext.Descriptor().Info().Version, "0.0.1")
 		})
 
 		it("provides readable blobs", func() {
-			ext, err := buildpack.ExtensionsFromOCILayoutBlob(blob.NewBlob(filepath.Join("testdata", "hello-extensions.cnb")))
+			ext, err := buildpack.ExtensionsFromOCILayoutBlob(blob.NewBlob(filepath.Join("testdata", "tree-extension.cnb")))
 			h.AssertNil(t, err)
-
 			reader, err := ext.Open()
 			h.AssertNil(t, err)
 

pkg/buildpack/package.go

@@ -98,6 +98,63 @@ func extractBuildpacks(pkg Package) (mainBP BuildModule, depBPs []BuildModule, e
 	return mainBP, depBPs, nil
 }
 
+func extractExtensions(pkg Package) (mainExt BuildModule, err error) {
+	pkg = &syncPkg{pkg: pkg}
+	md := &Metadata{}
+	if found, err := dist.GetLabel(pkg, MetadataLabel, md); err != nil {
+		return nil, err
+	} else if !found {
+		return nil, errors.Errorf(
+			"could not find label %s",
+			style.Symbol(MetadataLabel),
+		)
+	}
+
+	pkgLayers := dist.ModuleLayers{}
+	ok, err := dist.GetLabel(pkg, dist.ExtensionLayersLabel, &pkgLayers)
+	if err != nil {
+		return nil, err
+	}
+
+	if !ok {
+		return nil, errors.Errorf(
+			"could not find label %s",
+			style.Symbol(dist.ExtensionLayersLabel),
+		)
+	}
+	for extID, v := range pkgLayers {
+		for extVersion, extInfo := range v {
+			desc := dist.ExtensionDescriptor{
+				WithAPI: extInfo.API,
+				WithInfo: dist.ModuleInfo{
+					ID:       extID,
+					Version:  extVersion,
+					Homepage: extInfo.Homepage,
+					Name:     extInfo.Name,
+				},
+			}
+
+			diffID := extInfo.LayerDiffID // Allow use in closure
+			b := &openerBlob{
+				opener: func() (io.ReadCloser, error) {
+					rc, err := pkg.GetLayer(diffID)
+					if err != nil {
+						return nil, errors.Wrapf(err,
+							"extracting extension %s layer (diffID %s)",
+							style.Symbol(desc.Info().FullName()),
+							style.Symbol(diffID),
+						)
+					}
+					return rc, nil
+				},
+			}
+
+			mainExt = FromBlob(&desc, b)
+		}
+	}
+	return mainExt, nil
+}
+
 type openerBlob struct {
 	opener func() (io.ReadCloser, error)
 }

pkg/client/create_builder.go

@@ -260,7 +260,6 @@ func (c *Client) addConfig(ctx context.Context, kind string, config pubbldr.Modu
 	if err != nil {
 		return errors.Wrapf(err, "getting OS from %s", style.Symbol(bldr.Image().Name()))
 	}
-
 	mainBP, depBPs, err := c.buildpackDownloader.Download(ctx, config.URI, buildpack.DownloadOptions{
 		Daemon:          !opts.Publish,
 		ImageName:       config.ImageName,
@@ -273,7 +272,6 @@ func (c *Client) addConfig(ctx context.Context, kind string, config pubbldr.Modu
 	if err != nil {
 		return errors.Wrapf(err, "downloading %s", kind)
 	}
-
 	err = validateModule(kind, mainBP, config.URI, config.ID, config.Version)
 	if err != nil {
 		return errors.Wrapf(err, "invalid %s", kind)