diff --git a/pkg/contexts/oci/artdesc/utils.go b/pkg/contexts/oci/artdesc/utils.go index f4f3654798..767137ea2f 100644 --- a/pkg/contexts/oci/artdesc/utils.go +++ b/pkg/contexts/oci/artdesc/utils.go @@ -60,6 +60,11 @@ func ToDescriptorMediaType(media string) string { return ToContentMediaType(media) + "+json" } +func ToArchiveMediaTypes(media string) []string { + base := ToContentMediaType(media) + return []string{base + "+tar", base + "+tar+gzip"} +} + func IsOCIMediaType(media string) bool { c := ToContentMediaType(media) for _, t := range ContentTypes() { @@ -90,8 +95,7 @@ func DescriptorTypes() []string { func ArchiveBlobTypes() []string { r := []string{} for _, t := range ContentTypes() { - t = ToContentMediaType(t) - r = append(r, t+"+tar", t+"+tar+gzip") + r = append(r, ToArchiveMediaTypes(t)...) } return r } diff --git a/pkg/contexts/ocm/download/handlers/blueprint/blueprint_test.go b/pkg/contexts/ocm/download/handlers/blueprint/blueprint_test.go new file mode 100644 index 0000000000..bd99094037 --- /dev/null +++ b/pkg/contexts/ocm/download/handlers/blueprint/blueprint_test.go @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package blueprint_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/env/builder" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/mandelsoft/vfs/pkg/projectionfs" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/common/accessio" + "github.com/open-component-model/ocm/pkg/common/accessobj" + "github.com/open-component-model/ocm/pkg/contexts/oci/testhelper" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" + v1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" + "github.com/open-component-model/ocm/pkg/contexts/ocm/download" + "github.com/open-component-model/ocm/pkg/contexts/ocm/download/handlers/blueprint" + ctfocm "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/ctf" + tenv "github.com/open-component-model/ocm/pkg/env" + "github.com/open-component-model/ocm/pkg/utils/tarutils" +) + +const ( + COMPONENT = "github.com/compa" + VERSION = "1.0.0" + CTF = "ctf" + OCI = "oci" + + OCIHOST = "source" + OCINAMESPACE = "ocm/value" + OCIVERSION = "v2.0" + + MIMETYPE = "testmimetype" + ARTIFACT_TYPE = "testartifacttype" + OCI_ARTIFACT_NAME = "ociblueprint" + LOCAL_ARTIFACT_NAME = "localblueprint" + ARTIFACT_VERSION = "v1.0.0" + + TESTDATA_PATH = "testdata/blueprint" + ARCHIVE_PATH = "archive" + DOWNLOAD_PATH = "download" +) + +var _ = Describe("download blueprint", func() { + var env *Builder + + BeforeEach(func() { + env = NewBuilder(tenv.NewEnvironment(tenv.TestData())) + + MustBeSuccessful(tarutils.CreateTarFromFs(Must(projectionfs.New(env, TESTDATA_PATH)), ARCHIVE_PATH, tarutils.Gzip, env)) + + env.OCICommonTransport(OCI, accessio.FormatDirectory, func() { + env.Namespace(OCINAMESPACE, func() { + env.Manifest(OCIVERSION, func() { + env.Config(func() { + env.BlobStringData(blueprint.CONFIG_MIME_TYPE, "{}") + }) + env.Layer(func() { + env.BlobFromFile(blueprint.BLUEPRINT_MIMETYPE, ARCHIVE_PATH) + }) + }) + }) + }) + + testhelper.FakeOCIRepo(env, OCI, OCIHOST) + env.OCMCommonTransport(CTF, accessio.FormatDirectory, func() { + env.ComponentVersion(COMPONENT, VERSION, func() { + env.Resource(OCI_ARTIFACT_NAME, ARTIFACT_VERSION, blueprint.TYPE, v1.ExternalRelation, func() { + env.Access(ociartifact.New(OCIHOST + ".alias/" + OCINAMESPACE + ":" + OCIVERSION)) + }) + env.Resource(LOCAL_ARTIFACT_NAME, ARTIFACT_VERSION, blueprint.TYPE, v1.LocalRelation, func() { + env.BlobFromFile(blueprint.BLUEPRINT_MIMETYPE, ARCHIVE_PATH) + }) + }) + }) + }) + + AfterEach(func() { + env.Cleanup() + }) + DescribeTable("download blueprints", func(index int) { + src := Must(ctfocm.Open(env.OCMContext(), accessobj.ACC_READONLY, CTF, 0, env)) + defer Close(src, "source ctf") + + cv := Must(src.LookupComponentVersion(COMPONENT, VERSION)) + defer Close(cv) + + racc := Must(cv.GetResourceByIndex(index)) + + p, buf := common.NewBufferedPrinter() + ok, path := Must2(download.For(env).Download(p, racc, DOWNLOAD_PATH, env)) + Expect(ok).To(BeTrue()) + Expect(path).To(Equal(DOWNLOAD_PATH)) + Expect(env.FileExists(DOWNLOAD_PATH + "/blueprint.yaml")).To(BeTrue()) + Expect(env.FileExists(DOWNLOAD_PATH + "/test/README.md")).To(BeTrue()) + Expect(buf.String()).To(StringEqualTrimmedWithContext(DOWNLOAD_PATH + ": 2 file(s) with 390 byte(s) written")) + }, + Entry("oci artifact", 0), + Entry("local resource", 1), + ) +}) diff --git a/pkg/contexts/ocm/download/handlers/blueprint/extrator.go b/pkg/contexts/ocm/download/handlers/blueprint/extrator.go new file mode 100644 index 0000000000..155e9da8f5 --- /dev/null +++ b/pkg/contexts/ocm/download/handlers/blueprint/extrator.go @@ -0,0 +1,92 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package blueprint + +import ( + "github.com/mandelsoft/vfs/pkg/projectionfs" + "github.com/mandelsoft/vfs/pkg/vfs" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/common/accessio" + "github.com/open-component-model/ocm/pkg/common/accessobj" + "github.com/open-component-model/ocm/pkg/common/compression" + "github.com/open-component-model/ocm/pkg/contexts/oci/repositories/artifactset" + "github.com/open-component-model/ocm/pkg/finalizer" + "github.com/open-component-model/ocm/pkg/utils/tarutils" +) + +const ( + BLUEPRINT_MIMETYPE_LEGACY = "application/vnd.gardener.landscaper.blueprint.layer.v1.tar" + BLUEPRINT_MIMETYPE = "application/vnd.gardener.landscaper.blueprint.v1+tar+gzip" +) + +func ExtractArchive(pr common.Printer, _ *Handler, access accessio.DataAccess, path string, fs vfs.FileSystem) (_ bool, rerr error) { + var finalize finalizer.Finalizer + defer finalize.FinalizeWithErrorPropagationf(&rerr, "extracting archived (and compressed) blueprint") + + rawReader, err := access.Reader() + if err != nil { + return true, err + } + finalize.Close(rawReader) + + reader, _, err := compression.AutoDecompress(rawReader) + if err != nil { + return true, err + } + finalize.Close(reader) + + err = fs.MkdirAll(path, 0o700) + if err != nil { + return true, err + } + + pfs, err := projectionfs.New(fs, path) + if err != nil { + return true, err + } + fcnt, bcnt, err := tarutils.ExtractTarToFsWithInfo(pfs, reader) + if err != nil { + return true, err + } + pr.Printf("%s: %d file(s) with %d byte(s) written\n", path, fcnt, bcnt) + return true, nil +} + +func ExtractArtifact(pr common.Printer, handler *Handler, access accessio.DataAccess, path string, fs vfs.FileSystem) (_ bool, rerr error) { + var finalize finalizer.Finalizer + defer finalize.FinalizeWithErrorPropagationf(&rerr, "extracting oci artifact containing a blueprint") + + rd, err := access.Reader() + if err != nil { + return true, err + } + finalize.Close(rd) + + set, err := artifactset.Open(accessobj.ACC_READONLY, "", 0, accessio.Reader(rd)) + if err != nil { + return true, err + } + finalize.Close(set) + + art, err := set.GetArtifact(set.GetMain().String()) + if err != nil { + return true, err + } + finalize.Close(art) + + desc := art.ManifestAccess().GetDescriptor().Layers[0] + if !handler.ociConfigMimeTypes.Contains(art.ManifestAccess().GetDescriptor().Config.MediaType) { + if desc.MediaType != BLUEPRINT_MIMETYPE && desc.MediaType != BLUEPRINT_MIMETYPE_LEGACY { + return false, nil + } + } + + blob, err := art.GetBlob(desc.Digest) + if err != nil { + return true, err + } + return ExtractArchive(pr, handler, blob, path, fs) +} diff --git a/pkg/contexts/ocm/download/handlers/blueprint/handler.go b/pkg/contexts/ocm/download/handlers/blueprint/handler.go new file mode 100644 index 0000000000..1dc0c9316f --- /dev/null +++ b/pkg/contexts/ocm/download/handlers/blueprint/handler.go @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: 2022 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package blueprint + +import ( + "github.com/mandelsoft/vfs/pkg/vfs" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/common/accessio" + "github.com/open-component-model/ocm/pkg/contexts/oci/artdesc" + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + registry "github.com/open-component-model/ocm/pkg/contexts/ocm/download" + "github.com/open-component-model/ocm/pkg/contexts/ocm/resourcetypes" + "github.com/open-component-model/ocm/pkg/finalizer" + "github.com/open-component-model/ocm/pkg/generics" + "github.com/open-component-model/ocm/pkg/mime" + "github.com/open-component-model/ocm/pkg/utils" +) + +const ( + TYPE = resourcetypes.BLUEPRINT + LEGACY_TYPE = resourcetypes.BLUEPRINT_LEGACY + CONFIG_MIME_TYPE = "application/vnd.gardener.landscaper.blueprint.config.v1" +) + +type Extractor func(pr common.Printer, handler *Handler, access accessio.DataAccess, path string, fs vfs.FileSystem) (bool, error) + +var ( + supportedArtifactTypes []string + mimeTypeExtractorRegistry map[string]Extractor +) + +type Handler struct { + ociConfigMimeTypes generics.Set[string] +} + +func init() { + supportedArtifactTypes = []string{TYPE, LEGACY_TYPE} + mimeTypeExtractorRegistry = map[string]Extractor{ + mime.MIME_TAR: ExtractArchive, + mime.MIME_TGZ: ExtractArchive, + mime.MIME_TGZ_ALT: ExtractArchive, + BLUEPRINT_MIMETYPE: ExtractArchive, + } + for _, t := range append(artdesc.ToArchiveMediaTypes(artdesc.MediaTypeImageManifest), artdesc.ToArchiveMediaTypes(artdesc.MediaTypeDockerSchema2Manifest)...) { + mimeTypeExtractorRegistry[t] = ExtractArtifact + } + + h := New() + + registry.Register(h, registry.ForArtifactType(TYPE)) + registry.Register(h, registry.ForArtifactType(LEGACY_TYPE)) +} + +func New(configmimetypes ...string) *Handler { + if len(configmimetypes) == 0 || utils.Optional(configmimetypes...) == "" { + configmimetypes = []string{CONFIG_MIME_TYPE} + } + return &Handler{ + ociConfigMimeTypes: generics.NewSet[string](configmimetypes...), + } +} + +func (h *Handler) Download(pr common.Printer, racc cpi.ResourceAccess, path string, fs vfs.FileSystem) (_ bool, _ string, err error) { + var finalize finalizer.Finalizer + defer finalize.FinalizeWithErrorPropagationf(&err, "downloading blueprint") + + meth, err := racc.AccessMethod() + if err != nil { + return false, "", err + } + finalize.Close(meth) + + ex := mimeTypeExtractorRegistry[meth.MimeType()] + if ex == nil { + return false, "", nil + } + + ok, err := ex(pr, h, meth, path, fs) + if err != nil || !ok { + return ok, "", err + } + return true, path, nil +} diff --git a/pkg/contexts/ocm/download/handlers/blueprint/registration.go b/pkg/contexts/ocm/download/handlers/blueprint/registration.go new file mode 100644 index 0000000000..cfd4168e63 --- /dev/null +++ b/pkg/contexts/ocm/download/handlers/blueprint/registration.go @@ -0,0 +1,98 @@ +// // SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// // +// // SPDX-License-Identifier: Apache-2.0 +package blueprint + +import ( + "fmt" + "strings" + + "golang.org/x/exp/slices" + + "github.com/open-component-model/ocm/pkg/contexts/ocm/cpi" + "github.com/open-component-model/ocm/pkg/contexts/ocm/download" + "github.com/open-component-model/ocm/pkg/errors" + "github.com/open-component-model/ocm/pkg/listformat" + "github.com/open-component-model/ocm/pkg/registrations" + "github.com/open-component-model/ocm/pkg/utils" +) + +const PATH = "landscaper/blueprint" + +func init() { + download.RegisterHandlerRegistrationHandler(PATH, &RegistrationHandler{}) +} + +type Config struct { + OCIConfigTypes []string `json:"ociConfigTypes"` +} + +func AttributeDescription() map[string]string { + return map[string]string{ + "ociConfigTypes": "a list of accepted OCI config archive mime types\n" + + "defaulted by <code>" + CONFIG_MIME_TYPE + "</code>.", + } +} + +type RegistrationHandler struct{} + +var _ download.HandlerRegistrationHandler = (*RegistrationHandler)(nil) + +func (r *RegistrationHandler) RegisterByName(handler string, ctx download.Target, config download.HandlerConfig, olist ...download.HandlerOption) (bool, error) { + var err error + + if handler != "" { + return true, fmt.Errorf("invalid blueprint handler %q", handler) + } + + opts := download.NewHandlerOptions(olist...) + if opts.MimeType != "" && !slices.Contains(supportedArtifactTypes, opts.ArtifactType) { + return false, errors.Newf("artifact type %s not supported", opts.ArtifactType) + } + + if opts.MimeType != "" { + if _, ok := mimeTypeExtractorRegistry[opts.MimeType]; !ok { + return false, errors.Newf("mime type %s not supported", opts.MimeType) + } + } + + attr, err := registrations.DecodeDefaultedConfig[Config](config) + if err != nil { + return true, errors.Wrapf(err, "cannot unmarshal download handler configuration") + } + + h := New(attr.OCIConfigTypes...) + if opts.MimeType == "" { + for m := range mimeTypeExtractorRegistry { + opts.MimeType = m + download.For(ctx).Register(h, opts) + } + } else { + download.For(ctx).Register(h, opts) + } + + return true, nil +} + +func (r *RegistrationHandler) GetHandlers(ctx cpi.Context) registrations.HandlerInfos { + return registrations.NewLeafHandlerInfo("uploading an OCI artifact to an OCI registry", ` +The <code>artifact</code> downloader is able to transfer OCI artifact-like resources +into an OCI registry given by the combination of the download target and the +registration config. + +If no config is given, the target must be an OCI reference with a potentially +omitted repository. The repo part is derived from the reference hint provided +by the resource's access specification. + +If the config is given, the target is used as repository name prefixed with an +optional repository prefix given by the configuration. + +The following artifact media types are supported: +`+listformat.FormatList("", utils.StringMapKeys(mimeTypeExtractorRegistry)...)+` +It accepts a config with the following fields: +`+listformat.FormatMapElements("", AttributeDescription())+` + +This handler is by default registered for the following artifact types: +`+strings.Join(supportedArtifactTypes, ","), + ) +} diff --git a/pkg/contexts/ocm/download/handlers/blueprint/registration_test.go b/pkg/contexts/ocm/download/handlers/blueprint/registration_test.go new file mode 100644 index 0000000000..04df997d75 --- /dev/null +++ b/pkg/contexts/ocm/download/handlers/blueprint/registration_test.go @@ -0,0 +1,78 @@ +package blueprint_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "github.com/open-component-model/ocm/pkg/env/builder" + . "github.com/open-component-model/ocm/pkg/testutils" + + "github.com/mandelsoft/vfs/pkg/projectionfs" + + "github.com/open-component-model/ocm/pkg/common" + "github.com/open-component-model/ocm/pkg/common/accessio" + "github.com/open-component-model/ocm/pkg/common/accessobj" + "github.com/open-component-model/ocm/pkg/contexts/oci/testhelper" + "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/ociartifact" + v1 "github.com/open-component-model/ocm/pkg/contexts/ocm/compdesc/meta/v1" + "github.com/open-component-model/ocm/pkg/contexts/ocm/download" + "github.com/open-component-model/ocm/pkg/contexts/ocm/download/handlers/blueprint" + "github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/ctf" + tenv "github.com/open-component-model/ocm/pkg/env" + "github.com/open-component-model/ocm/pkg/utils/tarutils" +) + +var _ = Describe("blueprint downloader registration", func() { + var env *Builder + + BeforeEach(func() { + env = NewBuilder(tenv.NewEnvironment(tenv.TestData())) + + MustBeSuccessful(tarutils.CreateTarFromFs(Must(projectionfs.New(env, TESTDATA_PATH)), ARCHIVE_PATH, tarutils.Gzip, env)) + + env.OCICommonTransport(OCI, accessio.FormatDirectory, func() { + env.Namespace(OCINAMESPACE, func() { + env.Manifest(OCIVERSION, func() { + env.Config(func() { + env.BlobStringData(MIMETYPE, "{}") + }) + env.Layer(func() { + env.BlobFromFile(MIMETYPE, ARCHIVE_PATH) + }) + }) + }) + }) + + testhelper.FakeOCIRepo(env, OCI, OCIHOST) + env.OCMCommonTransport(CTF, accessio.FormatDirectory, func() { + env.ComponentVersion(COMPONENT, VERSION, func() { + env.Resource(OCI_ARTIFACT_NAME, ARTIFACT_VERSION, ARTIFACT_TYPE, v1.ExternalRelation, func() { + env.Access(ociartifact.New(OCIHOST + ".alias/" + OCINAMESPACE + ":" + OCIVERSION)) + }) + }) + }) + }) + + AfterEach(func() { + env.Cleanup() + }) + + It("register and use blueprint downloader for artifact type \"testartifacttype\"", func() { + // As the handler is not registered for the artifact type "testartifacttype" per default (thus, in the + // init-function of handler.go), this test fails if the registration does not work. + Expect(download.For(env).RegisterByName(blueprint.PATH, env.OCMContext(), &blueprint.Config{[]string{MIMETYPE}}, download.ForArtifactType(ARTIFACT_TYPE))).To(BeTrue()) + + repo := Must(ctf.Open(env.OCMContext(), accessobj.ACC_READONLY, CTF, 0, env)) + defer Close(repo) + cv := Must(repo.LookupComponentVersion(COMPONENT, VERSION)) + defer Close(cv) + racc := Must(cv.GetResourceByIndex(0)) + + p, buf := common.NewBufferedPrinter() + ok, path := Must2(download.For(env).Download(p, racc, DOWNLOAD_PATH, env)) + Expect(ok).To(BeTrue()) + Expect(path).To(Equal(DOWNLOAD_PATH)) + Expect(env.FileExists(DOWNLOAD_PATH + "/blueprint.yaml")).To(BeTrue()) + Expect(env.FileExists(DOWNLOAD_PATH + "/test/README.md")).To(BeTrue()) + Expect(buf.String()).To(StringEqualTrimmedWithContext(DOWNLOAD_PATH + ": 2 file(s) with 390 byte(s) written")) + }) +}) diff --git a/pkg/contexts/ocm/download/handlers/blueprint/suite_test.go b/pkg/contexts/ocm/download/handlers/blueprint/suite_test.go new file mode 100644 index 0000000000..265023d37d --- /dev/null +++ b/pkg/contexts/ocm/download/handlers/blueprint/suite_test.go @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and Open Component Model contributors. +// +// SPDX-License-Identifier: Apache-2.0 + +package blueprint_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Download Blueprint Test Suite") +} diff --git a/pkg/contexts/ocm/download/handlers/blueprint/testdata/blueprint/blueprint.yaml b/pkg/contexts/ocm/download/handlers/blueprint/testdata/blueprint/blueprint.yaml new file mode 100644 index 0000000000..bc42394195 --- /dev/null +++ b/pkg/contexts/ocm/download/handlers/blueprint/testdata/blueprint/blueprint.yaml @@ -0,0 +1,12 @@ +apiVersion: landscaper.gardener.cloud/v1alpha1 +kind: Blueprint +jsonSchema: "https://json-schema.org/draft/2019-09/schema" + +imports: + - name: cluster # name of the import parameter + targetType: landscaper.gardener.cloud/kubernetes-cluster # type of the import parameter + +deployExecutions: + - name: default + type: GoTemplate + template: diff --git a/pkg/contexts/ocm/download/handlers/blueprint/testdata/blueprint/test/README.md b/pkg/contexts/ocm/download/handlers/blueprint/testdata/blueprint/test/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pkg/contexts/ocm/download/handlers/dirtree/register_test.go b/pkg/contexts/ocm/download/handlers/dirtree/registration_test.go similarity index 100% rename from pkg/contexts/ocm/download/handlers/dirtree/register_test.go rename to pkg/contexts/ocm/download/handlers/dirtree/registration_test.go diff --git a/pkg/contexts/ocm/download/handlers/init.go b/pkg/contexts/ocm/download/handlers/init.go index 5b3c41dc6b..daaf062dbd 100644 --- a/pkg/contexts/ocm/download/handlers/init.go +++ b/pkg/contexts/ocm/download/handlers/init.go @@ -6,6 +6,7 @@ package handlers import ( _ "github.com/open-component-model/ocm/pkg/contexts/ocm/download/handlers/blob" + _ "github.com/open-component-model/ocm/pkg/contexts/ocm/download/handlers/blueprint" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/download/handlers/dirtree" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/download/handlers/executable" _ "github.com/open-component-model/ocm/pkg/contexts/ocm/download/handlers/helm" diff --git a/pkg/contexts/ocm/resourcetypes/const.go b/pkg/contexts/ocm/resourcetypes/const.go index 3e024a1de6..afcc720bb8 100644 --- a/pkg/contexts/ocm/resourcetypes/const.go +++ b/pkg/contexts/ocm/resourcetypes/const.go @@ -18,6 +18,10 @@ const ( // HELM_CHART describes a helm chart, either stored as OCI artifact or as tar // blob (tar media type). HELM_CHART = "helmChart" + // BLUEPRINT describes a Gardener Landscaper blueprint which is an artifact used in its installations describing + // how to deploy a software component. + BLUEPRINT = "landscaper.gardener.cloud/blueprint" + BLUEPRINT_LEGACY = "blueprint" // BLOB describes any anonymous untyped blob data. BLOB = "blob" // FILESYSTEM describes a directory structure stored as archive (tar, tgz).