diff --git a/.github/workflows/components.yaml b/.github/workflows/components.yaml index c6f8d53bde..520bc75639 100644 --- a/.github/workflows/components.yaml +++ b/.github/workflows/components.yaml @@ -30,7 +30,7 @@ permissions: env: REF: ${{ inputs.ref == '' && github.ref || inputs.ref }} CTF_TYPE: directory - components: '["ocmcli", "helminstaller", "helmdemo", "subchartsdemo", "ecrplugin"]' + components: '["ocmcli", "helminstaller", "helmdemo", "subchartsdemo", "ecrplugin", "jfrogplugin"]' IMAGE_PLATFORMS: 'linux/amd64 linux/arm64' PLATFORMS: 'windows/amd64 darwin/arm64 darwin/amd64 linux/amd64 linux/arm64' BUILDX_CACHE_PUSH: false diff --git a/api/ocm/extensions/blobhandler/handlers/generic/plugin/blobhandler.go b/api/ocm/extensions/blobhandler/handlers/generic/plugin/blobhandler.go index c9ae369c4e..a576bd7f09 100644 --- a/api/ocm/extensions/blobhandler/handlers/generic/plugin/blobhandler.go +++ b/api/ocm/extensions/blobhandler/handlers/generic/plugin/blobhandler.go @@ -70,6 +70,7 @@ func (b *pluginHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, glo "uploader", b.name, "arttype", artType, "mediatype", blob.MimeType(), + "digest", blob.Digest(), "hint", hint, "target", string(target), ) @@ -85,5 +86,5 @@ func (b *pluginHandler) StoreBlob(blob cpi.BlobAccess, artType, hint string, glo r := accessio.NewOndemandReader(blob) defer errors.PropagateError(&err, r.Close) - return b.plugin.Put(b.name, r, artType, blob.MimeType(), hint, creddata, target) + return b.plugin.Put(b.name, r, artType, blob.MimeType(), hint, string(blob.Digest()), creddata, target) } diff --git a/api/ocm/extensions/blobhandler/handlers/generic/plugin/testdata/test b/api/ocm/extensions/blobhandler/handlers/generic/plugin/testdata/test index 7b08e5d838..87e3556cc3 100755 --- a/api/ocm/extensions/blobhandler/handlers/generic/plugin/testdata/test +++ b/api/ocm/extensions/blobhandler/handlers/generic/plugin/testdata/test @@ -13,6 +13,7 @@ media= artifact= hint= creds= +digest= args=( ) parseArgs() { @@ -21,6 +22,9 @@ parseArgs() { --mediaType|-m) media="$2" shift 2;; + --digest|-d) + digest="$2" + shift 2;; --artifactType|-a) artifact="$2" shift 2;; diff --git a/api/ocm/plugin/cache/updater.go b/api/ocm/plugin/cache/updater.go index 17978059e3..22441765b7 100644 --- a/api/ocm/plugin/cache/updater.go +++ b/api/ocm/plugin/cache/updater.go @@ -296,44 +296,57 @@ func (o *PluginUpdater) download(session ocm.Session, cv ocm.ComponentVersionAcc return nil } } + dir := plugindirattr.Get(o.Context) - if dir != "" { - lock, err := filelock.LockDir(dir) + if dir == "" { + home, err := os.UserHomeDir() // use home if provided if err != nil { - return err + return fmt.Errorf("failed to determine home directory to determine default plugin directory: %w", err) } - defer lock.Close() - - target := filepath.Join(dir, desc.PluginName) - - verb := "installing" - if ok, _ := vfs.FileExists(fs, target); ok { - if !o.Force && (cv.GetVersion() == o.Current || !o.UpdateMode) { - return fmt.Errorf("plugin %s already found in %s", desc.PluginName, dir) - } - if o.UpdateMode { - verb = "updating" - } - fs.Remove(target) + dir = filepath.Join(home, plugindirattr.DEFAULT_PLUGIN_DIR) + if err := os.Mkdir(dir, os.ModePerm|os.ModeDir); err != nil { + return fmt.Errorf("failed to create default plugin directory: %w", err) } - o.Printer.Printf("%s plugin %s[%s] in %s...\n", verb, desc.PluginName, desc.PluginVersion, dir) - dst, err := fs.OpenFile(target, vfs.O_CREATE|vfs.O_TRUNC|vfs.O_WRONLY, 0o755) - if err != nil { - return errors.Wrapf(err, "cannot create plugin file %s", target) + if err := plugindirattr.Set(o.Context, dir); err != nil { + return fmt.Errorf("failed to set plugin dir after defaulting: %w", err) } - src, err := fs.OpenFile(file.Name(), vfs.O_RDONLY, 0) - if err != nil { - dst.Close() - return errors.Wrapf(err, "cannot open plugin executable %s", file.Name()) + } + + lock, err := filelock.LockDir(dir) + if err != nil { + return err + } + defer lock.Close() + + target := filepath.Join(dir, desc.PluginName) + + verb := "installing" + if ok, _ := vfs.FileExists(fs, target); ok { + if !o.Force && (cv.GetVersion() == o.Current || !o.UpdateMode) { + return fmt.Errorf("plugin %s already found in %s", desc.PluginName, dir) } - _, err = io.Copy(dst, src) - dst.Close() - utils.IgnoreError(src.Close()) - utils.IgnoreError(os.Remove(file.Name())) - utils.IgnoreError(SetPluginSourceInfo(dir, cv, found.Meta().Name, desc.PluginName)) - if err != nil { - return errors.Wrapf(err, "cannot copy plugin file %s", target) + if o.UpdateMode { + verb = "updating" } + fs.Remove(target) + } + o.Printer.Printf("%s plugin %s[%s] in %s...\n", verb, desc.PluginName, desc.PluginVersion, dir) + dst, err := fs.OpenFile(target, vfs.O_CREATE|vfs.O_TRUNC|vfs.O_WRONLY, 0o755) + if err != nil { + return errors.Wrapf(err, "cannot create plugin file %s", target) + } + src, err := fs.OpenFile(file.Name(), vfs.O_RDONLY, 0) + if err != nil { + dst.Close() + return errors.Wrapf(err, "cannot open plugin executable %s", file.Name()) + } + _, err = io.Copy(dst, src) + dst.Close() + utils.IgnoreError(src.Close()) + utils.IgnoreError(os.Remove(file.Name())) + utils.IgnoreError(SetPluginSourceInfo(dir, cv, found.Meta().Name, desc.PluginName)) + if err != nil { + return errors.Wrapf(err, "cannot copy plugin file %s", target) } } return nil diff --git a/api/ocm/plugin/plugin.go b/api/ocm/plugin/plugin.go index f546a0781a..5d4cb2524c 100644 --- a/api/ocm/plugin/plugin.go +++ b/api/ocm/plugin/plugin.go @@ -6,10 +6,12 @@ import ( "fmt" "io" "os" + "strings" "sync" "github.com/mandelsoft/goutils/errors" "github.com/mandelsoft/goutils/finalizer" + mlog "github.com/mandelsoft/logging" "github.com/mandelsoft/vfs/pkg/vfs" "ocm.software/ocm/api/credentials" @@ -112,11 +114,32 @@ func (p *pluginImpl) Exec(r io.Reader, w io.Writer, args ...string) (result []by args = append([]string{"--" + ppi.OptPlugingLogConfig, string(data)}, args...) } - if len(p.config) == 0 { - p.ctx.Logger(TAG).Debug("execute plugin action", "path", p.Path(), "args", args) - } else { - p.ctx.Logger(TAG).Debug("execute plugin action", "path", p.Path(), "args", args, "config", p.config) + if p.ctx.Logger(TAG).Enabled(mlog.DebugLevel) { + // Plainly kill any credentials found in the logger. + // Stupidly match for "credentials" arg. + // Not totally safe, but better than nothing. + logargs := make([]string, len(args)) + for i, arg := range args { + if logargs[i] != "" { + continue + } + if strings.Contains(arg, "credentials") { + if strings.Contains(arg, "=") { + logargs[i] = "***" + } else if i+1 < len(args)-1 { + logargs[i+1] = "***" + } + } + logargs[i] = arg + } + + if len(p.config) == 0 { + p.ctx.Logger(TAG).Debug("execute plugin action", "path", p.Path(), "args", logargs) + } else { + p.ctx.Logger(TAG).Debug("execute plugin action", "path", p.Path(), "args", logargs, "config", p.config) + } } + data, err := cache.Exec(p.Path(), p.config, r, w, args...) if logfile != nil { @@ -293,7 +316,7 @@ func (p *pluginImpl) Get(w io.Writer, creds, spec json.RawMessage) error { return err } -func (p *pluginImpl) Put(name string, r io.Reader, artType, mimeType, hint string, creds, target json.RawMessage) (ocm.AccessSpec, error) { +func (p *pluginImpl) Put(name string, r io.Reader, artType, mimeType, hint, digest string, creds, target json.RawMessage) (ocm.AccessSpec, error) { args := []string{upload.Name, put.Name, name, string(target)} if creds != nil { @@ -308,6 +331,9 @@ func (p *pluginImpl) Put(name string, r io.Reader, artType, mimeType, hint strin if artType != "" { args = append(args, "--"+put.OptArt, artType) } + if digest != "" { + args = append(args, "--"+put.OptDigest, digest) + } result, err := p.Exec(r, nil, args...) if err != nil { return nil, err diff --git a/api/ocm/plugin/ppi/cmds/common/const.go b/api/ocm/plugin/ppi/cmds/common/const.go index e58eaf96c6..622896d369 100644 --- a/api/ocm/plugin/ppi/cmds/common/const.go +++ b/api/ocm/plugin/ppi/cmds/common/const.go @@ -7,4 +7,5 @@ const ( OptArt = "artifactType" OptConfig = "config" OptCliConfig = "cli-config" + OptDigest = "digest" ) diff --git a/api/ocm/plugin/ppi/cmds/upload/put/cmd.go b/api/ocm/plugin/ppi/cmds/upload/put/cmd.go index 7bdac8633a..dfcef8844f 100644 --- a/api/ocm/plugin/ppi/cmds/upload/put/cmd.go +++ b/api/ocm/plugin/ppi/cmds/upload/put/cmd.go @@ -3,8 +3,6 @@ package put import ( "encoding/json" "fmt" - "io" - "os" "github.com/mandelsoft/goutils/errors" "github.com/spf13/cobra" @@ -19,11 +17,12 @@ import ( ) const ( - Name = "put" - OptCreds = common.OptCreds - OptHint = common.OptHint - OptMedia = common.OptMedia - OptArt = common.OptArt + Name = "put" + OptCreds = common.OptCreds + OptHint = common.OptHint + OptMedia = common.OptMedia + OptArt = common.OptArt + OptDigest = common.OptDigest ) func New(p ppi.Plugin) *cobra.Command { @@ -58,6 +57,7 @@ type Options struct { Credentials credentials.DirectCredentials MediaType string ArtifactType string + Digest string Hint string } @@ -68,42 +68,49 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVarP(&o.MediaType, OptMedia, "m", "", "media type of input blob") fs.StringVarP(&o.ArtifactType, OptArt, "a", "", "artifact type of input blob") fs.StringVarP(&o.Hint, OptHint, "H", "", "reference hint for storing blob") + fs.StringVarP(&o.Digest, OptDigest, "d", "", "digest of the blob") } func (o *Options) Complete(args []string) error { o.Name = args[0] if err := runtime.DefaultYAMLEncoding.Unmarshal([]byte(args[1]), &o.Specification); err != nil { - return errors.Wrapf(err, "invalid repository specification") + return fmt.Errorf("invalid repository specification: %w", err) } return nil } -func Command(p ppi.Plugin, cmd *cobra.Command, opts *Options) error { - spec, err := p.DecodeUploadTargetSpecification(opts.Specification) - if err != nil { - return errors.Wrapf(err, "target specification") +func Command(p ppi.Plugin, cmd *cobra.Command, opts *Options) (err error) { + var spec ppi.UploadTargetSpec + if spec, err = p.DecodeUploadTargetSpecification(opts.Specification); err != nil { + return fmt.Errorf("error decoding upload target specification: %w", err) } u := p.GetUploader(opts.Name) if u == nil { return errors.ErrNotFound(descriptor.KIND_UPLOADER, fmt.Sprintf("%s:%s", opts.ArtifactType, opts.MediaType)) } - w, h, err := u.Writer(p, opts.ArtifactType, opts.MediaType, opts.Hint, spec, opts.Credentials) - if err != nil { - return err - } - _, err = io.Copy(w, os.Stdin) - if err != nil { - w.Close() - return err - } - err = w.Close() - if err != nil { - return err + + reader := cmd.InOrStdin() + + var provider ppi.AccessSpecProvider + if provider, err = u.Upload( + cmd.Context(), + p, + opts.ArtifactType, + opts.MediaType, + opts.Hint, + opts.Digest, + spec, + opts.Credentials, + reader, + ); err != nil { + return fmt.Errorf("upload failed: %w", err) } - acc := h() - data, err := json.Marshal(acc) - if err == nil { + + acc := provider() + + var data []byte + if data, err = json.Marshal(acc); err == nil { cmd.Printf("%s\n", string(data)) } return err diff --git a/api/ocm/plugin/ppi/interface.go b/api/ocm/plugin/ppi/interface.go index bb35c71b04..27f9048ee5 100644 --- a/api/ocm/plugin/ppi/interface.go +++ b/api/ocm/plugin/ppi/interface.go @@ -1,6 +1,7 @@ package ppi import ( + "context" "encoding/json" "io" @@ -113,7 +114,7 @@ type Uploader interface { Description() string ValidateSpecification(p Plugin, spec UploadTargetSpec) (info *UploadTargetSpecInfo, err error) - Writer(p Plugin, arttype, mediatype string, hint string, spec UploadTargetSpec, creds credentials.Credentials) (io.WriteCloser, AccessSpecProvider, error) + Upload(ctx context.Context, p Plugin, arttype, mediatype, hint, digest string, spec UploadTargetSpec, creds credentials.Credentials, reader io.Reader) (AccessSpecProvider, error) } type UploadTargetSpec = runtime.TypedObject diff --git a/cmds/demoplugin/uploaders/demo.go b/cmds/demoplugin/uploaders/demo.go index 706d7ffd5d..3073bc0fb6 100644 --- a/cmds/demoplugin/uploaders/demo.go +++ b/cmds/demoplugin/uploaders/demo.go @@ -1,6 +1,7 @@ package uploaders import ( + "context" "fmt" "io" "os" @@ -56,7 +57,7 @@ func (a *Uploader) Decoders() ppi.UploadFormats { return types } -func (a *Uploader) ValidateSpecification(p ppi.Plugin, spec ppi.UploadTargetSpec) (*ppi.UploadTargetSpecInfo, error) { +func (a *Uploader) ValidateSpecification(_ ppi.Plugin, spec ppi.UploadTargetSpec) (*ppi.UploadTargetSpecInfo, error) { var info ppi.UploadTargetSpecInfo my := spec.(*TargetSpec) @@ -72,13 +73,13 @@ func (a *Uploader) ValidateSpecification(p ppi.Plugin, spec ppi.UploadTargetSpec return &info, nil } -func (a *Uploader) Writer(p ppi.Plugin, arttype, mediatype, hint string, repo ppi.UploadTargetSpec, creds credentials.Credentials) (io.WriteCloser, ppi.AccessSpecProvider, error) { +func (a *Uploader) Upload(_ context.Context, p ppi.Plugin, arttype, mediatype, hint, digest string, spec ppi.UploadTargetSpec, creds credentials.Credentials, reader io.Reader) (ppi.AccessSpecProvider, error) { var file *os.File var err error cfg, err := p.GetConfig() if err != nil { - return nil, nil, errors.Wrapf(err, "can't get config for access method %s", mediatype) + return nil, errors.Wrapf(err, "can't get config for access method %s", mediatype) } root := os.TempDir() @@ -86,12 +87,12 @@ func (a *Uploader) Writer(p ppi.Plugin, arttype, mediatype, hint string, repo pp root = cfg.(*config.Config).Uploaders.Path err := os.MkdirAll(root, 0o700) if err != nil { - return nil, nil, errors.Wrapf(err, "cannot create root dir") + return nil, errors.Wrapf(err, "cannot create root dir") } } path := hint - my := repo.(*TargetSpec) + my := spec.(*TargetSpec) dir := root if my.Path != "" { root = filepath.Join(root, my.Path) @@ -106,7 +107,7 @@ func (a *Uploader) Writer(p ppi.Plugin, arttype, mediatype, hint string, repo pp err = os.MkdirAll(dir, 0o700) if err != nil { - return nil, nil, err + return nil, err } if hint == "" { @@ -115,8 +116,13 @@ func (a *Uploader) Writer(p ppi.Plugin, arttype, mediatype, hint string, repo pp file, err = os.OpenFile(filepath.Join(os.TempDir(), path), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) } if err != nil { - return nil, nil, err + return nil, err } writer := NewWriter(file, path, mediatype, hint == "", accessmethods.NAME, accessmethods.VERSION) - return writer, writer.Specification, nil + + if _, err = io.Copy(writer, reader); err != nil { + return nil, fmt.Errorf("cannot write to %q: %w", file.Name(), err) + } + + return writer.Specification, nil } diff --git a/cmds/jfrogplugin/config/config.go b/cmds/jfrogplugin/config/config.go new file mode 100644 index 0000000000..27d735f880 --- /dev/null +++ b/cmds/jfrogplugin/config/config.go @@ -0,0 +1,21 @@ +package config + +import ( + "encoding/json" + "fmt" +) + +type Config struct { +} + +// GetConfig returns the config from the raw json message. +// any return is required for the plugin interface. +func GetConfig(raw json.RawMessage) (interface{}, error) { + var cfg Config + + if err := json.Unmarshal(raw, &cfg); err != nil { + return nil, fmt.Errorf("could not get config: %w", err) + } + return &cfg, nil + +} diff --git a/cmds/jfrogplugin/main.go b/cmds/jfrogplugin/main.go new file mode 100644 index 0000000000..ab5cdb49ca --- /dev/null +++ b/cmds/jfrogplugin/main.go @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + "os" + + "ocm.software/ocm/api/ocm/plugin/ppi/cmds" + jfrogppi "ocm.software/ocm/cmds/jfrogplugin/ppi" +) + +func main() { + plugin, err := jfrogppi.Plugin() + if err != nil { + fmt.Fprintf(os.Stderr, "error while creating plugin: %v\n", err) + os.Exit(1) + } + if err := cmds.NewPluginCommand(plugin).Execute(os.Args[1:]); err != nil { + fmt.Fprintf(os.Stderr, "error while running plugin: %v\n", err) + os.Exit(1) + } +} diff --git a/cmds/jfrogplugin/main_test.go b/cmds/jfrogplugin/main_test.go new file mode 100644 index 0000000000..d279c9168a --- /dev/null +++ b/cmds/jfrogplugin/main_test.go @@ -0,0 +1,244 @@ +package main_test + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + . "github.com/mandelsoft/goutils/testutils" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/oci/artdesc" + "ocm.software/ocm/api/oci/extensions/repositories/artifactset" + helmaccess "ocm.software/ocm/api/ocm/extensions/accessmethods/helm" + "ocm.software/ocm/api/ocm/extensions/artifacttypes" + "ocm.software/ocm/api/ocm/plugin/ppi" + "ocm.software/ocm/api/tech/helm/loader" + "ocm.software/ocm/api/utils/runtime" + . "ocm.software/ocm/cmds/jfrogplugin/testhelper" + "ocm.software/ocm/cmds/jfrogplugin/uploaders/helm" +) + +var _ = Describe(helm.VERSIONED_NAME, func() { + var ( + env *TestEnv + server *httptest.Server + reindexedAfterUpload bool + user, pass = "foo", "bar" + creds = string(Must(json.Marshal(credentials.DirectCredentials{ + credentials.ATTR_USERNAME: user, + credentials.ATTR_PASSWORD: pass, + }))) + ) + + BeforeEach(func() { + env = Must(NewTestEnv()) + DeferCleanup(env.Cleanup) + }) + + It("Upload Validate Invalid Spec", func(ctx SpecContext) { + env.CLI.Command().SetContext(ctx) + err := env.Execute("upload", "validate", "--artifactType=bla", "abc", "def") + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(ContainSubstring("error unmarshaling JSON"))) + }) + + It("Validate Spec OK and empty", func(ctx SpecContext) { + env.CLI.Command().SetContext(ctx) + + uploadSpec := &helm.JFrogHelmUploaderSpec{ + ObjectVersionedType: runtime.ObjectVersionedType{Type: helm.VERSIONED_NAME}, + } + + Expect(env.Execute( + "upload", "validate", "--artifactType", artifacttypes.HELM_CHART, + helm.NAME, + string(Must(json.Marshal(uploadSpec))), + )).To(Succeed()) + + Expect(env.CLI.GetOutput()).To(Not(BeEmpty())) + var info ppi.UploadTargetSpecInfo + Expect(json.Unmarshal(env.CLI.GetOutput(), &info)).To(Succeed()) + Expect(info.ConsumerId.Type()).To(Equal(helm.NAME)) + }) + + It("Validate Upload Spec OK with full identity based on Artifact Set containing OCI Image", func(ctx SpecContext) { + env.CLI.Command().SetContext(ctx) + + purl := Must(url.Parse("https://ocm.software:5501/my-artifactory")) + uploadSpec := &helm.JFrogHelmUploaderSpec{ + ObjectVersionedType: runtime.ObjectVersionedType{Type: helm.VERSIONED_NAME}, + URL: purl.String(), + Repository: "my-repo", + } + + Expect(env.Execute("upload", "validate", + "--artifactType", artifacttypes.HELM_CHART, + helm.NAME, + string(Must(json.Marshal(uploadSpec))), + )).To(Succeed()) + + var info ppi.UploadTargetSpecInfo + output := env.CLI.GetOutput() + Expect(output).To(Not(BeEmpty())) + Expect(json.Unmarshal(output, &info)).To(Succeed()) + + Expect(info.ConsumerId.Type()).To(Equal(helm.NAME)) + Expect(info.ConsumerId.Match(credentials.ConsumerIdentity{ + helm.ID_TYPE: helm.NAME, + helm.ID_HOSTNAME: purl.Hostname(), + helm.ID_PORT: purl.Port(), + helm.ID_REPOSITORY: "my-repo", + })).To(BeTrue(), "the identity should contain all attributes relevant to"+ + " match the correct repository for a resource transfer") + }) + + BeforeEach(func() { + reindexedAfterUpload = false + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.String(), "reindex") { + w.WriteHeader(http.StatusOK) + reindexedAfterUpload = true + return + } + + if r.Method != http.MethodPut { + http.Error(w, fmt.Sprintf("expected %s request, got %s", http.MethodPut, r.Method), http.StatusBadRequest) + } + + u, p, ok := r.BasicAuth() + if !ok { + http.Error(w, fmt.Sprintf("expected basic auth header, got %s,%s but expected %s,%s", u, p, user, pass), http.StatusBadRequest) + } + + data := Must(io.ReadAll(r.Body)) + unzipped, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + http.Error(w, fmt.Sprintf("failed to unzip data: %v", err), http.StatusBadRequest) + } + var buf bytes.Buffer + io.Copy(&buf, unzipped) + unzipped.Close() + bufBytes := buf.Bytes() + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + io.Copy(writer, bytes.NewReader(bufBytes)) + writer.Close() + data = compressed.Bytes() + + chart, err := loader.LoadArchive(bytes.NewReader(data)) + if err != nil { + http.Error(w, fmt.Sprintf("failed to load chart: %s", err.Error()), http.StatusBadRequest) + } + + resData, err := json.Marshal(&helm.ArtifactoryUploadResponse{ + DownloadUri: fmt.Sprintf("%s/my-repo/%s-%s.tgz", server.URL, chart.Name(), chart.Metadata.Version), + Path: "/mocked/chart.tgz", + CreatedBy: "mocked", + Created: time.Now().Format(time.RFC3339), + Repo: "my-repo", + MimeType: helm.MEDIA_TYPE, + Checksums: helm.ArtifactoryUploadChecksums{ + Sha256: r.Header.Get("X-Checksum-Sha256"), + }, + Size: strconv.Itoa(len(data)), + }) + if err != nil { + http.Error(w, fmt.Sprintf("failed to marshal response: %v", err), http.StatusInternalServerError) + } + + // mimick the upload response + w.WriteHeader(http.StatusCreated) + if _, err := io.Copy(w, bytes.NewReader(resData)); err != nil { + Fail(fmt.Sprintf("failed to write response: %v", err)) + } + })) + + DeferCleanup(server.Close) + }) + + It("Upload Artifact Set to Server (no reindex) with Basic Auth", func(ctx SpecContext) { + env.CLI.Command().SetContext(ctx) + + testDataPath := Must(filepath.Abs("../../api/ocm/extensions/download/handlers/helm/testdata/test-chart-oci-artifact.tgz")) + testDataFile := Must(os.OpenFile(testDataPath, os.O_RDONLY, 0o400)) + DeferCleanup(testDataFile.Close) + testData := Must(io.ReadAll(testDataFile)) + + purl := Must(helm.ParseURLAllowNoScheme(server.URL)) + uploadSpec := &helm.JFrogHelmUploaderSpec{ + ObjectVersionedType: runtime.ObjectVersionedType{Type: helm.VERSIONED_NAME}, + URL: purl.String(), + Repository: "my-repo", + } + + env.CLI.SetInput(io.NopCloser(bytes.NewReader(testData))) + Expect(env.Execute("upload", "put", + "--artifactType", artifacttypes.HELM_CHART, + "--mediaType", artifactset.MediaType(artdesc.MediaTypeImageManifest), + "--credentials", creds, + helm.NAME, + string(Must(json.Marshal(uploadSpec)))), + ).To(Succeed()) + + var spec helmaccess.AccessSpec + Expect(json.Unmarshal(env.GetOutput(), &spec)).To(Succeed()) + Expect(spec).To(Not(BeNil())) + Expect(spec.HelmRepository).To(Equal(fmt.Sprintf("%s/artifactory/api/helm/%s", server.URL, "my-repo"))) + Expect(spec.HelmChart).To(ContainSubstring(":"), "helm chart is separated with version") + splitChart := strings.Split(spec.HelmChart, ":") + Expect(splitChart).To(HaveLen(2), "helm chart is separated with version") + Expect(splitChart[0]).To(Equal("test-chart"), "the chart name should be test-chart") + Expect(splitChart[1]).To(Equal("0.1.0"), "the chart version should be 0.1.0") + Expect(reindexedAfterUpload).To(BeFalse(), "the server should not have been reindexed as it wasnt requested explicitly") + }) + + It("Upload Artifact Set to Server (with reindex) with Basic Auth", func(ctx SpecContext) { + env.CLI.Command().SetContext(ctx) + + testDataPath := Must(filepath.Abs("../../api/ocm/extensions/download/handlers/helm/testdata/test-chart-oci-artifact.tgz")) + testDataFile := Must(os.OpenFile(testDataPath, os.O_RDONLY, 0o400)) + DeferCleanup(testDataFile.Close) + testData := Must(io.ReadAll(testDataFile)) + + purl := Must(helm.ParseURLAllowNoScheme(server.URL)) + uploadSpec := &helm.JFrogHelmUploaderSpec{ + ObjectVersionedType: runtime.ObjectVersionedType{Type: helm.VERSIONED_NAME}, + URL: purl.String(), + Repository: "my-repo", + ReIndexAfterUpload: true, + } + + env.CLI.SetInput(io.NopCloser(bytes.NewReader(testData))) + Expect(env.Execute("upload", "put", + "--artifactType", artifacttypes.HELM_CHART, + "--mediaType", artifactset.MediaType(artdesc.MediaTypeImageManifest), + "--credentials", creds, + helm.NAME, + string(Must(json.Marshal(uploadSpec)))), + ).To(Succeed()) + + var spec helmaccess.AccessSpec + Expect(json.Unmarshal(env.GetOutput(), &spec)).To(Succeed()) + Expect(spec).To(Not(BeNil())) + Expect(spec.HelmRepository).To(Equal(fmt.Sprintf("%s/artifactory/api/helm/%s", server.URL, "my-repo"))) + Expect(spec.HelmChart).To(ContainSubstring(":"), "helm chart is separated with version") + splitChart := strings.Split(spec.HelmChart, ":") + Expect(splitChart).To(HaveLen(2), "helm chart is separated with version") + Expect(splitChart[0]).To(Equal("test-chart"), "the chart name should be test-chart") + Expect(splitChart[1]).To(Equal("0.1.0"), "the chart version should be 0.1.0") + Expect(reindexedAfterUpload).To(BeTrue(), "the server should not have been reindexed as it wasnt requested explicitly") + }) +}) diff --git a/cmds/jfrogplugin/ppi/ppi.go b/cmds/jfrogplugin/ppi/ppi.go new file mode 100644 index 0000000000..3ce9018181 --- /dev/null +++ b/cmds/jfrogplugin/ppi/ppi.go @@ -0,0 +1,63 @@ +package ppi + +import ( + "fmt" + "strconv" + + ocmconfig "ocm.software/ocm/api/config" + "ocm.software/ocm/api/ocm/extensions/artifacttypes" + "ocm.software/ocm/api/ocm/extensions/blobhandler" + "ocm.software/ocm/api/ocm/plugin" + "ocm.software/ocm/api/ocm/plugin/ppi" + "ocm.software/ocm/api/version" + "ocm.software/ocm/cmds/jfrogplugin/config" + "ocm.software/ocm/cmds/jfrogplugin/uploaders/helm" +) + +const NAME = "jfrog" + +func Plugin() (ppi.Plugin, error) { + p := ppi.NewPlugin(NAME, version.Get().String()) + + p.SetShort(NAME + " plugin") + p.SetLong(`ALPHA GRADE plugin providing custom functions related to interacting with JFrog Repositories (e.g. Artifactory). + +This plugin is solely for interacting with JFrog Servers and cannot be used for generic repository types. +Thus, you should only consider this plugin if +- You need to use a JFrog specific API +- You cannot use any of the generic (non-jfrog) implementations. + +If given an OCI Artifact Set (for example by using it on a resource with a Helm Chart backed by an OCI registry), +it will do a best effort conversion to a normal helm chart and upload that in its stead. Note that this conversion +is not perfect however, since the Upload will inevitably strip provenance information from the chart. +This can lead to unintended side effects such as +- Having the wrong digest in the resource access +- Losing the ability to convert back to an OCI artifact set without changing digests and losing provenance information. +This means that effectively you should try to migrate to pure OCI registries instead of JFrog HELM repositories as soon +as possible (this uploader is just a stop gap). + +Examples: + +You can configure the JFrog plugin as an Uploader in an ocm config file with: + +- type: ` + fmt.Sprintf("%s.ocm.%s", plugin.KIND_UPLOADER, ocmconfig.OCM_CONFIG_TYPE_SUFFIX) + ` + registrations: + - name: ` + fmt.Sprintf("%s/%s/%s", plugin.KIND_PLUGIN, NAME, helm.NAME) + ` + artifactType: ` + artifacttypes.HELM_CHART + ` + priority: 200 # must be > ` + strconv.Itoa(blobhandler.DEFAULT_BLOBHANDLER_PRIO) + ` to be used over the default handler + config: + type: ` + fmt.Sprintf("%s/%s", helm.NAME, helm.VERSION) + ` + # this is only a sample JFrog Server URL, do NOT append /artifactory + url: int.repositories.ocm.software + repository: ocm-helm-test +`) + p.SetConfigParser(config.GetConfig) + p.ForwardLogging(true) + + u := helm.New() + if err := p.RegisterUploader(artifacttypes.HELM_CHART, "", u); err != nil { + return nil, err + } + + return p, nil +} diff --git a/cmds/jfrogplugin/suite_test.go b/cmds/jfrogplugin/suite_test.go new file mode 100644 index 0000000000..27381dfaa8 --- /dev/null +++ b/cmds/jfrogplugin/suite_test.go @@ -0,0 +1,13 @@ +package main_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "JFrog Helm uploader plugin tests") +} diff --git a/cmds/jfrogplugin/testhelper/env.go b/cmds/jfrogplugin/testhelper/env.go new file mode 100644 index 0000000000..ce1c8ef80d --- /dev/null +++ b/cmds/jfrogplugin/testhelper/env.go @@ -0,0 +1,81 @@ +package testhelper + +import ( + "bytes" + "io" + + "gopkg.in/yaml.v3" + + "ocm.software/ocm/api/helper/builder" + "ocm.software/ocm/api/helper/env" + "ocm.software/ocm/api/ocm/plugin/ppi" + "ocm.software/ocm/api/ocm/plugin/ppi/cmds" + "ocm.software/ocm/cmds/jfrogplugin/config" + jfrogppi "ocm.software/ocm/cmds/jfrogplugin/ppi" +) + +type CLI struct { + ppi.Plugin + *cmds.PluginCommand + + config string + + output *bytes.Buffer +} + +func NewCLI() (*CLI, error) { + plugin, err := jfrogppi.Plugin() + if err != nil { + return nil, err + } + cmd := cmds.NewPluginCommand(plugin) + cli := &CLI{Plugin: plugin, PluginCommand: cmd, output: &bytes.Buffer{}} + cmd.Command().SetOut(cli.output) + return cli, nil +} + +func (cli *CLI) Execute(args ...string) error { + cli.output.Reset() + if cli.config != "" { + args = append(args, "--config", cli.config) + } + return cli.PluginCommand.Execute(args) +} + +func (cli *CLI) SetConfig(cfg *config.Config) error { + var data bytes.Buffer + if err := yaml.NewEncoder(&data).Encode(cfg); err != nil { + return err + } + + cli.config = data.String() + + return nil +} + +func (cli *CLI) GetOutput() []byte { + return cli.output.Bytes() +} + +func (cli *CLI) SetInput(data io.Reader) { + cli.PluginCommand.Command().SetIn(data) +} + +type TestEnv struct { + *builder.Builder + *CLI +} + +func NewTestEnv(opts ...env.Option) (*TestEnv, error) { + b := builder.NewBuilder(opts...) + + cli, err := NewCLI() + if err != nil { + return nil, err + } + + return &TestEnv{ + Builder: b, + CLI: cli, + }, nil +} diff --git a/cmds/jfrogplugin/uploaders/helm/convert.go b/cmds/jfrogplugin/uploaders/helm/convert.go new file mode 100644 index 0000000000..fedcf2ce90 --- /dev/null +++ b/cmds/jfrogplugin/uploaders/helm/convert.go @@ -0,0 +1,77 @@ +package helm + +import ( + "errors" + "fmt" + "io" + + "ocm.software/ocm/api/oci" + "ocm.software/ocm/api/oci/extensions/repositories/artifactset" + "ocm.software/ocm/api/utils/accessio" + "ocm.software/ocm/api/utils/accessobj" +) + +// ConvertArtifactSetWithOCIImageHelmChartToPlainTGZChart converts an artifact set with a single layer helm OCI image to a plain tgz chart. +// Note that this transformation is not completely reversible because an OCI artifact contains provenance data, while +// a plain tgz chart does not. +// This means converting back from a signed tgz chart to an OCI image will lose the provenance data, and also change digests. +// The returned digest is the digest of the tgz chart. +func ConvertArtifactSetWithOCIImageHelmChartToPlainTGZChart(reader io.Reader) (_ io.ReadCloser, _ string, err error) { + set, err := artifactset.Open(accessobj.ACC_READONLY, "", 0, accessio.Reader(io.NopCloser(reader))) + if err != nil { + return nil, "", fmt.Errorf("failed to open helm OCI artifact as artifact set: %w", err) + } + defer func() { + err = errors.Join(err, set.Close()) + }() + + art, err := set.GetArtifact(set.GetMain().String()) + if err != nil { + return nil, "", fmt.Errorf("failed to get artifact from set: %w", err) + } + defer func() { + err = errors.Join(err, art.Close()) + }() + + chartTgz, provenance, err := accessSingleLayerOCIHelmChart(art) + if err != nil { + return nil, "", fmt.Errorf("failed to access OCI artifact as a single layer helm OCI image: %w", err) + } + defer func() { + err = errors.Join(err, chartTgz.Close()) + if provenance != nil { + err = errors.Join(err, provenance.Close()) + } + }() + + chartReader, err := chartTgz.Reader() + if err != nil { + return nil, "", fmt.Errorf("failed to get reader for chart tgz: %w", err) + } + + digest := chartTgz.Digest().String() + + return chartReader, digest, nil +} + +func accessSingleLayerOCIHelmChart(art oci.ArtifactAccess) (chart oci.BlobAccess, prov oci.BlobAccess, err error) { + m := art.ManifestAccess() + if m == nil { + return nil, nil, errors.New("artifact is no image manifest") + } + if len(m.GetDescriptor().Layers) < 1 { + return nil, nil, errors.New("no layers found") + } + + if chart, err = m.GetBlob(m.GetDescriptor().Layers[0].Digest); err != nil { + return nil, nil, err + } + + if len(m.GetDescriptor().Layers) > 1 { + if prov, err = m.GetBlob(m.GetDescriptor().Layers[1].Digest); err != nil { + return nil, nil, err + } + } + + return chart, prov, nil +} diff --git a/cmds/jfrogplugin/uploaders/helm/headers.go b/cmds/jfrogplugin/uploaders/helm/headers.go new file mode 100644 index 0000000000..7c677cf681 --- /dev/null +++ b/cmds/jfrogplugin/uploaders/helm/headers.go @@ -0,0 +1,25 @@ +package helm + +import ( + "net/http" + + "ocm.software/ocm/api/credentials" +) + +func SetHeadersFromCredentials(req *http.Request, creds credentials.Credentials) { + if creds == nil { + return + } + if creds.ExistsProperty(credentials.ATTR_TOKEN) { + req.Header.Set("Authorization", "Bearer "+creds.GetProperty(credentials.ATTR_TOKEN)) + } else { + var user, pass string + if creds.ExistsProperty(credentials.ATTR_USERNAME) { + user = creds.GetProperty(credentials.ATTR_USERNAME) + } + if creds.ExistsProperty(credentials.ATTR_PASSWORD) { + pass = creds.GetProperty(credentials.ATTR_PASSWORD) + } + req.SetBasicAuth(user, pass) + } +} diff --git a/cmds/jfrogplugin/uploaders/helm/helm.go b/cmds/jfrogplugin/uploaders/helm/helm.go new file mode 100644 index 0000000000..da1f4d30fe --- /dev/null +++ b/cmds/jfrogplugin/uploaders/helm/helm.go @@ -0,0 +1,287 @@ +package helm + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + "time" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/credentials/cpi" + "ocm.software/ocm/api/credentials/identity/hostpath" + "ocm.software/ocm/api/oci/artdesc" + "ocm.software/ocm/api/oci/extensions/repositories/artifactset" + "ocm.software/ocm/api/ocm/extensions/artifacttypes" + "ocm.software/ocm/api/ocm/plugin/ppi" + "ocm.software/ocm/api/tech/helm" + "ocm.software/ocm/api/tech/helm/loader" + "ocm.software/ocm/api/utils/runtime" +) + +const ( + // MEDIA_TYPE is the media type of the HELM Chart artifact as tgz. + // It is the definitive format for JFrog Uploads + MEDIA_TYPE = helm.ChartMediaType + + // NAME of the Uploader and the Configuration + NAME = "JFrogHelm" + + // VERSION of the Uploader TODO Increment once stable + VERSION = "v1alpha1" + + // VERSIONED_NAME is the name of the Uploader including the version + VERSIONED_NAME = NAME + runtime.VersionSeparator + VERSION + + // ID_TYPE is the type of the JFrog Artifactory credentials + ID_TYPE = cpi.ID_TYPE + // ID_HOSTNAME is the hostname of the artifactory server to upload to + ID_HOSTNAME = hostpath.ID_HOSTNAME + // ID_PORT is the port of the artifactory server to upload to + ID_PORT = hostpath.ID_PORT + // ID_REPOSITORY is the repository name in JFrog Artifactory to upload to + ID_REPOSITORY = "repository" + + // DEFAULT_TIMEOUT is the default timeout for http requests issued by the uploader. + DEFAULT_TIMEOUT = time.Minute +) + +type JFrogHelmUploaderSpec struct { + runtime.ObjectVersionedType `json:",inline"` + + // URL is the hostname of the JFrog instance. + // Required for correct reference to Artifactory. + URL string `json:"url"` + + // Repository is the repository to upload to. + // Required for correct reference to Artifactory. + Repository string `json:"repository"` + + JFrogHelmChart `json:",inline"` + + // Timeout is the maximum duration the upload of the chart can take + // before aborting and failing. + // OPTIONAL: If not set, set to the internal DEFAULT_TIMEOUT. + Timeout *time.Duration `json:"timeout,omitempty"` + + // ReIndexAfterUpload is a flag to indicate if the chart should be reindexed after upload or not. + // OPTIONAL: If not set, defaulted to false. + ReIndexAfterUpload bool `json:"reindexAfterUpload,omitempty"` +} + +type JFrogHelmChart struct { + // ChartName is the desired name of the chart in the repository. + // OPTIONAL: If not set, defaulted from the passed Hint. + Name string `json:"name,omitempty"` + // Version is the desired version of the chart + // OPTIONAL: If not set, defaulted from the passed Hint. + Version string `json:"version,omitempty"` +} + +func (s *JFrogHelmUploaderSpec) GetTimeout() time.Duration { + if s.Timeout == nil { + return DEFAULT_TIMEOUT + } + return *s.Timeout +} + +var types ppi.UploadFormats + +func init() { + decoder, err := runtime.NewDirectDecoder[runtime.TypedObject](&JFrogHelmUploaderSpec{}) + if err != nil { + panic(err) + } + types = ppi.UploadFormats{VERSIONED_NAME: decoder} +} + +func (a *Uploader) Decoders() ppi.UploadFormats { + return types +} + +type Uploader struct { + ppi.UploaderBase + *http.Client +} + +var _ ppi.Uploader = (*Uploader)(nil) + +func New() ppi.Uploader { + client := &http.Client{} + // we do not want to double compress helm tgz files + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DisableCompression = true + client.Transport = transport + return &Uploader{ + UploaderBase: ppi.MustNewUploaderBase(NAME, "upload artifacts to JFrog HELM repositories by using the JFrog REST API."), + Client: client, + } +} + +func (a *Uploader) ValidateSpecification(_ ppi.Plugin, spec ppi.UploadTargetSpec) (*ppi.UploadTargetSpecInfo, error) { + targetSpec, ok := spec.(*JFrogHelmUploaderSpec) + if !ok { + return nil, fmt.Errorf("invalid spec type %T", spec) + } + + info, err := ConvertTargetSpecToInfo(targetSpec) + if err != nil { + return nil, fmt.Errorf("failed to convert target spec to info: %w", err) + } + + return info, nil +} + +// Upload uploads any artifact that is of type artifacttypes.HELM_CHART. +// Process: +// 1. introspect the JFrogHelmUploaderSpec (cast from ppi.UploadTargetSpec) and hint parameter +// (the hint is expected to be an OCI style reference, such as `repo/comp:version`) +// 2. building an Artifactory Style JFrog Upload URL out of it (see ConvertTargetSpecToHelmUploadURL), +// 3. creating a request respecting the passed credentials based on SetHeadersFromCredentials +// 4. uploading the passed blob as is (expected to be a tgz byte stream) +// 5. intepreting the JFrog API response, and converting it from ArtifactoryUploadResponse to ppi.AccessSpec +func (a *Uploader) Upload( + ctx context.Context, + p ppi.Plugin, + arttype, mediaType, _, digest string, + targetSpec ppi.UploadTargetSpec, + creds credentials.Credentials, + reader io.Reader, +) (ppi.AccessSpecProvider, error) { + if arttype != artifacttypes.HELM_CHART { + return nil, fmt.Errorf("unsupported artifact type %s", arttype) + } + + spec, ok := targetSpec.(*JFrogHelmUploaderSpec) + if !ok { + return nil, fmt.Errorf("the type %T is not a valid target spec type", spec) + } + + switch mediaType { + case helm.ChartMediaType: + // if it is a native chart tgz we can pass it on as is + case artifactset.MediaType(artdesc.MediaTypeImageManifest): + // if we have an artifact set (ocm custom version of index + locally colocated blobs as files, we need + // to translate it. This translation is not perfect because the ociArtifactDigest that might be + // generated in OCM is not the same as the one that is used within Artifactory, but Uploaders + // do not have a way of providing back digest information to the caller. + // TODO: At some point consider this for a plugin rework. + var err error + if reader, digest, err = ConvertArtifactSetWithOCIImageHelmChartToPlainTGZChart(reader); err != nil { + return nil, fmt.Errorf("failed to convert OCI Helm Chart to plain TGZ: %w", err) + } + default: + return nil, fmt.Errorf("unsupported media type %s", mediaType) + } + + var buf bytes.Buffer + chart, err := loader.LoadArchive(io.TeeReader(reader, &buf)) + if err != nil { + return nil, fmt.Errorf("failed to load chart: %w", err) + } + spec.Name = chart.Metadata.Name + spec.Version = chart.Metadata.Version + reader = &buf + + // now based on the chart and repository we can upload it to the correct location. + targetURL, err := ConvertTargetSpecToHelmUploadURL(spec) + if err != nil { + return nil, fmt.Errorf("failed to convert target spec to URL: %w", err) + } + + ctx, cancel := context.WithTimeout(ctx, spec.GetTimeout()) + defer cancel() + + access, err := Upload(ctx, reader, a.Client, targetURL, creds, digest) + if err != nil { + return nil, fmt.Errorf("failed to upload: %w", err) + } + + if spec.ReIndexAfterUpload { + if err := ReindexChart(ctx, a.Client, spec.URL, spec.Repository, creds); err != nil { + return nil, fmt.Errorf("failed to reindex chart: %w", err) + } + } + + return func() ppi.AccessSpec { + return access + }, nil +} + +// ConvertTargetSpecToInfo converts the JFrogHelmUploaderSpec +// to a valid info block containing the consumer ID used +// in the library to identify the correct credentials that need to +// be passed to it. +func ConvertTargetSpecToInfo(spec *JFrogHelmUploaderSpec) (*ppi.UploadTargetSpecInfo, error) { + purl, err := ParseURLAllowNoScheme(spec.URL) + if err != nil { + return nil, fmt.Errorf("failed to parse URL: %w", err) + } + + var info ppi.UploadTargetSpecInfo + + // By default, we identify an artifactory repository as a combination + // of Host & Repository + info.ConsumerId = credentials.ConsumerIdentity{ + ID_TYPE: NAME, + ID_HOSTNAME: purl.Hostname(), + ID_REPOSITORY: spec.Repository, + } + if purl.Port() != "" { + info.ConsumerId.SetNonEmptyValue(ID_PORT, purl.Port()) + } + + return &info, nil +} + +// ConvertTargetSpecToHelmUploadURL interprets the JFrogHelmUploaderSpec into a valid REST API Endpoint URL to upload to. +// It requires a valid ChartName and ChartVersion to determine the correct URL endpoint. +// +// See https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact for the URL endpoint +// See https://jfrog.com/help/r/jfrog-artifactory-documentation/deploying-artifacts for artifact deployment reference +// See https://jfrog.com/help/r/jfrog-artifactory-documentation/use-the-jfrog-helm-client for the HELM Client reference. +// +// Example: +// +// JFrogHelmUploaderSpec.URL => demo.jfrog.ocm.software +// JFrogHelmUploaderSpec.Repository => my-charts +// JFrogHelmUploaderSpec.ChartName => podinfo +// JFrogHelmUploaderSpec.ChartVersion => 0.0.1 +// +// will result in +// +// url.URL => https://demo.jfrog.ocm.software/artifactory/my-charts/podinfo-0.0.1.tgz +func ConvertTargetSpecToHelmUploadURL(spec *JFrogHelmUploaderSpec) (*url.URL, error) { + requestURLParsed, err := ParseURLAllowNoScheme(spec.URL) + if err != nil { + return nil, fmt.Errorf("failed to parse full request URL: %w", err) + } + requestURLParsed.Path = path.Join("artifactory", spec.Repository, fmt.Sprintf("%s-%s.tgz", spec.Name, spec.Version)) + return requestURLParsed, nil +} + +// ParseURLAllowNoScheme is an adaptation / hack on url.Parse because +// url.Parse does not support parsing a URL without a prefixed scheme. +// However, we would like to accept these kind of URLs because we default them +// to "https://" out of convenience. +func ParseURLAllowNoScheme(urlToParse string) (*url.URL, error) { + const dummyScheme = "dummy" + if !strings.Contains(urlToParse, "://") { + urlToParse = dummyScheme + "://" + urlToParse + } + parsedURL, err := url.Parse(urlToParse) + if err != nil { + return nil, err + } + if parsedURL.Scheme == dummyScheme { + parsedURL.Scheme = "" + } + if parsedURL.Scheme == "" { + parsedURL.Scheme = "https" + } + return parsedURL, nil +} diff --git a/cmds/jfrogplugin/uploaders/helm/reindex.go b/cmds/jfrogplugin/uploaders/helm/reindex.go new file mode 100644 index 0000000000..1fd1f90dd2 --- /dev/null +++ b/cmds/jfrogplugin/uploaders/helm/reindex.go @@ -0,0 +1,57 @@ +package helm + +import ( + "fmt" + "io" + "net/http" + "path" + + "golang.org/x/net/context" + + "ocm.software/ocm/api/credentials" +) + +func ReindexChart(ctx context.Context, client *http.Client, artifactoryURL string, + repository string, + creds credentials.Credentials) (err error) { + reindexURL, err := convertToReindexURL(artifactoryURL, repository) + if err != nil { + return fmt.Errorf("failed to convert to reindex URL: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, reindexURL, nil) + if err != nil { + return fmt.Errorf("failed to create reindex request: %w", err) + } + SetHeadersFromCredentials(req, creds) + req.Header = req.Header.Clone() + + res, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to reindex chart: %w", err) + } + + if res.StatusCode != http.StatusOK { + responseBytes, err := io.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("failed to read response body but server returned %v: %w", res.StatusCode, err) + } + var body string + if len(responseBytes) > 0 { + body = fmt.Sprintf(": %s", string(responseBytes)) + } + return fmt.Errorf("invalid response (status %v) while reindexing at %q: %s", res.StatusCode, reindexURL, body) + } + + return nil +} + +// convertToReindexURL converts the base URL and repository to a reindex URL. +// see https://jfrog.com/help/r/jfrog-rest-apis/calculate-helm-chart-index for the reindex API +func convertToReindexURL(baseURL string, repository string) (string, error) { + u, err := ParseURLAllowNoScheme(baseURL) + if err != nil { + return "", fmt.Errorf("failed to parse url: %w", err) + } + u.Path = path.Join(u.Path, "api", "helm", repository, "reindex") + return u.String(), nil +} diff --git a/cmds/jfrogplugin/uploaders/helm/upload.go b/cmds/jfrogplugin/uploaders/helm/upload.go new file mode 100644 index 0000000000..5edf9e5350 --- /dev/null +++ b/cmds/jfrogplugin/uploaders/helm/upload.go @@ -0,0 +1,135 @@ +package helm + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + + godigest "github.com/opencontainers/go-digest" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/ocm/extensions/accessmethods/helm" + "ocm.software/ocm/api/ocm/plugin/ppi" +) + +func Upload( + ctx context.Context, + data io.Reader, + client *http.Client, + url *url.URL, + creds credentials.Credentials, + digest string, +) (_ ppi.AccessSpec, err error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var req *http.Request + var res *http.Response + if req, err = http.NewRequestWithContext(ctx, http.MethodPut, url.String(), data); err != nil { + return nil, fmt.Errorf("failed to create HTTP request for upload: %w", err) + } + + // if there is no digest information, we skip the digest headers. + // note that this will cause insecure uploads and should be avoided where possible + if digest != "" { + parsedDigest, err := godigest.Parse(digest) + if err != nil { + return nil, fmt.Errorf("failed to parse digest: %w", err) + } + + // see https://jfrog.com/help/r/jfrog-rest-apis/deploy-artifact-by-checksum for the checksum headers + switch parsedDigest.Algorithm() { + case godigest.SHA256: + req.Header.Set("X-Checksum-Sha256", parsedDigest.Encoded()) + default: + return nil, fmt.Errorf("unsupported digest algorithm, must be %s to allow upload to jfrog: %s", godigest.SHA256, parsedDigest.Algorithm()) + } + req.Header.Set("X-Checksum-Deploy", "false") + } + + req.Header.Set("Accept-Encoding", "gzip") + + SetHeadersFromCredentials(req, creds) + + if res, err = client.Do(req); err != nil { + return nil, fmt.Errorf("failed to store blob in artifactory: %w", err) + } + defer func() { + err = errors.Join(err, res.Body.Close()) + }() + + if res.StatusCode != http.StatusCreated { + responseBytes, err := io.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body but server returned %v: %w", res.StatusCode, err) + } + var body string + if len(responseBytes) > 0 { + body = fmt.Sprintf(": %s", string(responseBytes)) + } + return nil, fmt.Errorf("invalid response (status %v)%s", res.StatusCode, body) + } + + var buf bytes.Buffer + body := io.TeeReader(res.Body, &buf) + uploadResponse := &ArtifactoryUploadResponse{} + if err = json.NewDecoder(body).Decode(uploadResponse); err != nil { + return nil, fmt.Errorf("failed to decode response (original %q): %w", buf.String(), err) + } + + return uploadResponse.ToHelmAccessSpec() +} + +type ArtifactoryUploadResponse struct { + Repo string `json:"repo,omitempty"` + Path string `json:"path,omitempty"` + Created string `json:"created,omitempty"` + CreatedBy string `json:"createdBy,omitempty"` + DownloadUri string `json:"downloadUri,omitempty"` + MimeType string `json:"mimeType,omitempty"` + Size string `json:"size,omitempty"` + Checksums ArtifactoryUploadChecksums `json:"checksums,omitempty"` + Uri string `json:"uri"` +} + +type ArtifactoryUploadChecksums struct { + Sha1 string `json:"sha1,omitempty"` + Sha256 string `json:"sha256,omitempty"` + Md5 string `json:"md5,omitempty"` +} + +func (r *ArtifactoryUploadResponse) URL() string { + if r.DownloadUri != "" { + return r.DownloadUri + } + return r.Uri +} + +func (r *ArtifactoryUploadResponse) ToHelmAccessSpec() (ppi.AccessSpec, error) { + u := r.URL() + urlp, err := url.Parse(u) + if err != nil { + return nil, err + } + chart := path.Base(urlp.Path) + chart = strings.TrimSuffix(chart, path.Ext(chart)) + + // this is needed so that the chart version constructor for OCM is happy + // OCM encodes helm charts with a ":"... + if idx := strings.LastIndex(chart, "-"); idx > 0 { + chart = chart[:idx] + ":" + chart[idx+1:] + } + + urlp.Path = "" + urlp = urlp.JoinPath("artifactory", "api", "helm", r.Repo) + repo := urlp.String() + + return helm.New(chart, repo), nil +} diff --git a/cmds/jfrogplugin/uploaders/helm/upload_test.go b/cmds/jfrogplugin/uploaders/helm/upload_test.go new file mode 100644 index 0000000000..e5737d16fb --- /dev/null +++ b/cmds/jfrogplugin/uploaders/helm/upload_test.go @@ -0,0 +1,82 @@ +package helm + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + neturl "net/url" + "strings" + "testing" + + "ocm.software/ocm/api/credentials" + "ocm.software/ocm/api/ocm/extensions/accessmethods/helm" +) + +func TestUpload(t *testing.T) { + const artifactory = "https://mocked.artifactory.localhost:9999" + const chartName, chartVersion, repo = "chart", "1.0.0", "repo" + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + t.Fatalf("method is not PUT as expected by JFrog") + } + if user, pass, ok := r.BasicAuth(); !ok { + t.Fatalf("invalid basic auth: %s - %s", user, pass) + } + res := ArtifactoryUploadResponse{ + Repo: repo, + DownloadUri: fmt.Sprintf("%s/path/to/%s/%s-%s.tgz", artifactory, repo, chartName, chartVersion), + } + data, err := json.Marshal(res) + if err != nil { + t.Fatalf("failed to marshal response: %v", err) + } + if _, err := w.Write(data); err != nil { + t.Fatalf("failed to write response: %v", err) + } + })) + t.Cleanup(func() { + srv.Close() + }) + client := srv.Client() + + url, err := neturl.Parse(srv.URL) + if err != nil { + t.Fatalf("unexpected test client URL: %v", err) + } + + ctx := context.Background() + + data := strings.NewReader("testdata") + + accessSpec, err := Upload(ctx, data, client, url, credentials.DirectCredentials{ + credentials.ATTR_USERNAME: "foo", + credentials.ATTR_PASSWORD: "bar", + }, "") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if typ := accessSpec.GetType(); typ != helm.Type { + t.Fatalf("unexpected type: %v", typ) + } + + helmAccessSpec, ok := accessSpec.(*helm.AccessSpec) + if !ok { + t.Fatalf("unexpected cast failure to helm access spec") + } + + if specChart := helmAccessSpec.GetChartName(); specChart != chartName { + t.Fatalf("unexpected chart name: %v", specChart) + } + if specVersion := helmAccessSpec.GetVersion(); specVersion != chartVersion { + t.Fatalf("unexpected chart version: %v", specVersion) + } + + if helmAccessSpec.HelmRepository != fmt.Sprintf("%s/artifactory/api/helm/%s", artifactory, repo) { + t.Fatalf("expected an injected helm api reference to artifactory") + } + +} diff --git a/components/jfrogplugin/Makefile b/components/jfrogplugin/Makefile new file mode 100644 index 0000000000..ef51b171e2 --- /dev/null +++ b/components/jfrogplugin/Makefile @@ -0,0 +1,127 @@ +NAME = jfrogplugin +PROVIDER ?= ocm.software +GITHUBORG ?= open-component-model +COMPONENT = $(PROVIDER)/plugins/$(NAME) +OCMREPO ?= ghcr.io/$(GITHUBORG)/ocm +PLATFORMS ?= linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 +CTF_TYPE ?= directory + + +REPO_ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))/../.. +ifeq ($(VERSION),) +VERSION := $(shell go run ../../api/version/generate/release_generate.go print-rc-version $(CANDIDATE)) +endif +COMMIT = $(shell git rev-parse --verify HEAD) +# if EFFECTIVE_VERSION is not set, set it to VERSION+COMMIT +# this is not the same as '?=' because it will also set the value if EFFECTIVE_VERSION is set to an empty string +ifeq ($(EFFECTIVE_VERSION),) +EFFECTIVE_VERSION := $(VERSION)+$(COMMIT) +endif +GIT_TREE_STATE := $(shell [ -z "$$(git status --porcelain 2>/dev/null)" ] && echo clean || echo dirty) + +CMDSRCS=$(shell find $(REPO_ROOT)/cmds/$(NAME) -type f) +OCMSRCS=$(shell find $(REPO_ROOT)/api -type f) $(REPO_ROOT)/go.* + +CREDS ?= +# Define the path to the binary +OCM_BIN = $(REPO_ROOT)/bin/ocm + +# Rule to build the binary if it doesn't exist or if the source code has changed +$(OCM_BIN): $(REPO_ROOT)/cmds/ocm/main.go + mkdir -p $(REPO_ROOT)/bin + go build -ldflags $(BUILD_FLAGS) -o $(OCM_BIN) $(REPO_ROOT)/cmds/ocm + +# Use the binary for the OCM command +OCM = $(OCM_BIN) $(CREDS) + +GEN = $(REPO_ROOT)/gen/$(NAME) + + $(GEN): + @mkdir -p $(GEN) + +NOW := $(shell date -u +%FT%T%z) +BUILD_FLAGS := "-s -w \ + -X ocm.software/ocm/api/version.gitVersion=$(EFFECTIVE_VERSION) \ + -X ocm.software/ocm/api/version.gitTreeState=$(GIT_TREE_STATE) \ + -X ocm.software/ocm/api/version.gitCommit=$(COMMIT) \ + -X ocm.software/ocm/api/version.buildDate=$(NOW)" + +.PHONY: build +build: $(GEN)/build + +$(GEN)/build: $(GEN) $(CMDSRCS) $(OCMSRCS) + @for i in $(PLATFORMS); do \ + tag=$$(echo $$i | sed -e s:/:-:g); \ + echo GOARCH=$$(basename $$i) GOOS=$$(dirname $$i) CGO_ENABLED=0 go build -ldflags $(BUILD_FLAGS) -o $(GEN)/$(NAME).$$tag ../../cmds/$(NAME); \ + GOARCH=$$(basename $$i) GOOS=$$(dirname $$i) CGO_ENABLED=0 go build -ldflags $(BUILD_FLAGS) -o $(GEN)/$(NAME).$$tag ../../cmds/$(NAME) & \ + done; \ + wait + @touch $(GEN)/build + + +.PHONY: ctf +ctf: $(GEN)/ctf + +$(GEN)/ctf: $(OCM_BIN) $(GEN)/.exists $(GEN)/build component-constructor.yaml $(CHARTSRCS) + @rm -rf "$(GEN)/ctf" + $(OCM) add componentversions \ + --create \ + --file $(GEN)/ctf \ + --type $(CTF_TYPE) \ + --templater=spiff \ + COMPONENT="$(COMPONENT)" \ + NAME="$(NAME)" \ + VERSION="$(VERSION)" \ + PROVIDER="$(PROVIDER)" \ + COMMIT="$(COMMIT)" \ + GEN="$(GEN)" \ + PLATFORMS="$(PLATFORMS)" \ + component-constructor.yaml + touch "$(GEN)/ctf" + +.PHONY: version +version: + @echo $(VERSION) + +.PHONY: push +push: $(GEN)/ctf $(GEN)/push.$(NAME) + +$(GEN)/push.$(NAME): $(GEN)/ctf $(OCM_BIN) + $(OCM) transfer ctf -f $(GEN)/ctf $(OCMREPO) + @touch $(GEN)/push.$(NAME) + +.PHONY: plain-push +plain-push: $(GEN) $(OCM_BIN) + $(OCM) transfer ctf -f $(GEN)/ctf $(OCMREPO) + @touch $(GEN)/push.$(NAME) + +.PHONY: transport +transport: $(OCM_BIN) +ifneq ($(TARGETREPO),) + $(OCM) transfer component -Vc $(OCMREPO)//$(COMPONENT):$(VERSION) $(TARGETREPO) +endif + +$(GEN)/.exists: + @mkdir -p $(GEN) + @touch $@ + +.PHONY: info +info: + @echo "ROOT: $(REPO_ROOT)" + @echo "VERSION: $(VERSION)" + @echo "COMMIT; $(COMMIT)" + +.PHONY: describe +describe: $(GEN)/ctf $(OCM_BIN) + $(OCM) get resources --lookup $(OCMREPO) -r -o treewide $(GEN)/ctf + +.PHONY: descriptor +descriptor: $(GEN)/ctf $(OCM_BIN) + $(OCM) get component -S v3alpha1 -o yaml $(GEN)/ctf + +.PHONY: clean +clean: + rm -rf $(GEN) + +install: $(GEN)/ctf $(OCM_BIN) + $(OCM) install plugin -f $(GEN)/ctf \ No newline at end of file diff --git a/components/jfrogplugin/bindings.yaml b/components/jfrogplugin/bindings.yaml new file mode 100644 index 0000000000..ae8fb3bd18 --- /dev/null +++ b/components/jfrogplugin/bindings.yaml @@ -0,0 +1,4 @@ +values: + GEN: ../../gen/jfrogplugin + PLATFORMS: linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 + VERSION: "1.0" diff --git a/components/jfrogplugin/component-constructor.yaml b/components/jfrogplugin/component-constructor.yaml new file mode 100644 index 0000000000..b5b7d3f51b --- /dev/null +++ b/components/jfrogplugin/component-constructor.yaml @@ -0,0 +1,23 @@ +--- +helper: + <<<: (( &temporary )) + executable: + <<<: (( &template )) + name: jfrog + type: ocmPlugin + version: (( values.VERSION )) + extraIdentity: + os: ((dirname(p) )) + architecture: (( basename(p) )) + input: + type: file + # Generate the path to the plugin binary by looking into the base path and encoding the platform + path: (( values.GEN "/" values.NAME "." replace(p,"/","-") )) + +components: + - name: (( values.COMPONENT)) + version: (( values.VERSION)) + provider: + name: (( values.PROVIDER)) + # use all platforms and create a resource for each + resources: (( map[split(" ", values.PLATFORMS)|p|-> *helper.executable] )) \ No newline at end of file diff --git a/components/jfrogplugin/sources.yaml b/components/jfrogplugin/sources.yaml new file mode 100644 index 0000000000..dcc9c5f896 --- /dev/null +++ b/components/jfrogplugin/sources.yaml @@ -0,0 +1,7 @@ +name: source +type: filesytem +access: + type: github + repoUrl: github.com/open-component-model/ocm + commit: ${COMMIT} +version: ${VERSION} diff --git a/docs/pluginreference/plugin_upload_put.md b/docs/pluginreference/plugin_upload_put.md index a0bf9df648..6c5723b2b0 100644 --- a/docs/pluginreference/plugin_upload_put.md +++ b/docs/pluginreference/plugin_upload_put.md @@ -12,6 +12,7 @@ plugin upload put [] [] -a, --artifactType string artifact type of input blob -C, --credential = dedicated credential value (default []) -c, --credentials YAML credentials + -d, --digest string digest of the blob -h, --help help for put -H, --hint string reference hint for storing blob -m, --mediaType string media type of input blob