From 7535e39e6901bff7a4b0bb28cc7b9f06c10e5be8 Mon Sep 17 00:00:00 2001 From: Kalman Speier Date: Mon, 2 Oct 2023 14:32:13 +0200 Subject: [PATCH 1/4] add support GITHUB_TOKEN env var --- pkg/providers/github.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/providers/github.go b/pkg/providers/github.go index ee80adb..87cda5f 100644 --- a/pkg/providers/github.go +++ b/pkg/providers/github.go @@ -114,6 +114,9 @@ func newGitHub(u *url.URL) (Provider, error) { } token := os.Getenv("GITHUB_AUTH_TOKEN") + if len(token) == 0 { + token = os.Getenv("GITHUB_TOKEN") + } // GHES client gbu := os.Getenv("GHES_BASE_URL") From 34bf3597ad71a43ac8ebc1615f0984db982d4670 Mon Sep 17 00:00:00 2001 From: Kalman Speier Date: Fri, 3 Nov 2023 12:38:55 +0100 Subject: [PATCH 2/4] helm provider --- pkg/providers/helm.go | 124 +++++++++++++++++++++++++++++++++++++ pkg/providers/providers.go | 4 ++ 2 files changed, 128 insertions(+) create mode 100644 pkg/providers/helm.go diff --git a/pkg/providers/helm.go b/pkg/providers/helm.go new file mode 100644 index 0000000..9b5dd23 --- /dev/null +++ b/pkg/providers/helm.go @@ -0,0 +1,124 @@ +package providers + +import ( + "crypto/sha256" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/apex/log" + + "github.com/marcosnils/bin/pkg/assets" +) + +const ( + repo = "helm" + helmBaseURL = "https://get.helm.sh" + helmLatestPath = "/helm-latest-version" +) + +var ( + helmOSArchSuffixes = []string{ + "darwin-amd64.tar.gz", + "darwin-arm64.tar.gz", + "linux-amd64.tar.gz", + "linux-arm.tar.gz", + "linux-arm64.tar.gz", + "linux-386.tar.gz", + "linux-ppc64le.tar.gz", + "linux-s390x.tar.gz", + "windows-amd64.zip", + } +) + +type helm struct { + tag string +} + +func (h *helm) GetID() string { + return repo +} + +func (h *helm) GetLatestVersion() (string, string, error) { + log.Debugf("Getting latest Helm release") + + resp, err := http.Get(helmBaseURL + helmLatestPath) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", "", fmt.Errorf("HTTP response error: %v", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", "", fmt.Errorf("Error reading response body: %v", err) + } + + return strings.TrimSuffix(string(body), "\n"), helmBaseURL, nil +} + +func (h *helm) Fetch(opts *FetchOpts) (*File, error) { + var version string + if len(h.tag) > 0 { + log.Infof("Getting release %s for Helm", h.tag) + version = h.tag + } else { + latest, _, err := h.GetLatestVersion() + if err != nil { + return nil, err + } + version = latest + } + + candidates := []*assets.Asset{} + for _, suffix := range helmOSArchSuffixes { + fname := fmt.Sprintf("%s-%s-%s", repo, version, suffix) + link := fmt.Sprintf("%s/%s", helmBaseURL, fname) + candidates = append(candidates, &assets.Asset{Name: fname, URL: link}) + } + + f := assets.NewFilter(&assets.FilterOpts{SkipScoring: opts.All, PackagePath: opts.PackagePath, SkipPathCheck: opts.SkipPatchCheck}) + gf, err := f.FilterAssets(repo, candidates) + if err != nil { + return nil, err + } + + outFile, err := f.ProcessURL(gf) + if err != nil { + return nil, err + } + + return &File{Data: outFile.Source, Name: assets.SanitizeName(outFile.Name, version), Hash: sha256.New(), Version: version}, nil +} + +func newHelm(u *url.URL) (Provider, error) { + s := strings.Split(u.Path, "/") + + // it's a specific releases URL + var tag string + if len(s) >= 2 { + tag = parseTagVersion(s[1]) + } + + return &helm{tag: tag}, nil +} + +func parseTagVersion(s string) (v string) { + r, err := regexp.Compile(`^.*(v((\d*)\.(\d*)\.(\d*))).*$`) + if err != nil { + return + } + + m := r.FindStringSubmatch(s) + if len(m) > 0 { + v = m[1] + } + + return +} diff --git a/pkg/providers/providers.go b/pkg/providers/providers.go index 32d3996..ccbfdf9 100644 --- a/pkg/providers/providers.go +++ b/pkg/providers/providers.go @@ -70,5 +70,9 @@ func New(u, provider string) (Provider, error) { return newHashiCorp(purl) } + if strings.Contains(purl.Host, "get.helm.sh") || provider == "helm" { + return newHelm(purl) + } + return nil, fmt.Errorf("Can't find provider for url %s", u) } From 195131f81772da94161e31711de2f2eadb7989da Mon Sep 17 00:00:00 2001 From: Kalman Speier Date: Sun, 5 Nov 2023 13:16:15 +0100 Subject: [PATCH 3/4] generic provider, drop helm provider --- cmd/ensure.go | 3 +- cmd/install.go | 13 ++-- cmd/update.go | 6 +- pkg/config/config.go | 1 + pkg/providers/generic.go | 106 +++++++++++++++++++++++++++++++ pkg/providers/helm.go | 124 ------------------------------------- pkg/providers/providers.go | 9 +-- 7 files changed, 125 insertions(+), 137 deletions(-) create mode 100644 pkg/providers/generic.go delete mode 100644 pkg/providers/helm.go diff --git a/cmd/ensure.go b/cmd/ensure.go index 018a6b1..7797f26 100644 --- a/cmd/ensure.go +++ b/cmd/ensure.go @@ -51,7 +51,7 @@ func newEnsureCmd() *ensureCmd { continue } - p, err := providers.New(binCfg.URL, binCfg.Provider) + p, err := providers.New(binCfg.URL, binCfg.Provider, binCfg.LatestURL) if err != nil { return err } @@ -71,6 +71,7 @@ func newEnsureCmd() *ensureCmd { Version: pResult.Version, Hash: fmt.Sprintf("%x", pResult.Hash.Sum(nil)), URL: binCfg.URL, + LatestURL: binCfg.LatestURL, }) if err != nil { return err diff --git a/cmd/install.go b/cmd/install.go index 38e9c84..6d5773d 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -18,9 +18,10 @@ type installCmd struct { } type installOpts struct { - force bool - provider string - all bool + all bool + force bool + provider string + latestURL string } func newInstallCmd() *installCmd { @@ -57,7 +58,7 @@ func newInstallCmd() *installCmd { // TODO check if binary already exists in config // and triger the update process if that's the case - p, err := providers.New(u, root.opts.provider) + p, err := providers.New(u, root.opts.provider, root.opts.latestURL) if err != nil { return err } @@ -87,6 +88,7 @@ func newInstallCmd() *installCmd { Hash: fmt.Sprintf("%x", pResult.Hash.Sum(nil)), URL: u, Provider: p.GetID(), + LatestURL: root.opts.latestURL, PackagePath: pResult.PackagePath, }) @@ -101,9 +103,10 @@ func newInstallCmd() *installCmd { } root.cmd = cmd - root.cmd.Flags().BoolVarP(&root.opts.force, "force", "f", false, "Force the installation even if the file already exists") root.cmd.Flags().BoolVarP(&root.opts.all, "all", "a", false, "Show all possible download options (skip scoring & filtering)") + root.cmd.Flags().BoolVarP(&root.opts.force, "force", "f", false, "Force the installation even if the file already exists") root.cmd.Flags().StringVarP(&root.opts.provider, "provider", "p", "", "Forces to use a specific provider") + root.cmd.Flags().StringVarP(&root.opts.latestURL, "latest-url", "u", "", "Specify a latest version URL for generic provider") return root } diff --git a/cmd/update.go b/cmd/update.go index c233041..a293931 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -68,7 +68,7 @@ func newUpdateCmd() *updateCmd { updateFailures := map[*config.Binary]error{} for _, b := range binsToProcess { - p, err := providers.New(b.URL, b.Provider) + p, err := providers.New(b.URL, b.Provider, b.LatestURL) if err != nil { return err } @@ -108,8 +108,7 @@ func newUpdateCmd() *updateCmd { // the same thing as install logic. Refactor to // use the same code in both places for ui, b := range toUpdate { - - p, err := providers.New(ui.url, b.Provider) + p, err := providers.New(ui.url, b.Provider, b.LatestURL) if err != nil { return err } @@ -133,6 +132,7 @@ func newUpdateCmd() *updateCmd { Version: pResult.Version, Hash: fmt.Sprintf("%x", pResult.Hash.Sum(nil)), URL: ui.url, + LatestURL: b.LatestURL, PackagePath: pResult.PackagePath, }) if err != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index 7c54b31..8e7ff90 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -31,6 +31,7 @@ type Binary struct { Hash string `json:"hash"` URL string `json:"url"` Provider string `json:"provider"` + LatestURL string `json:"latest_url"` // for generic provider // if file is installed from a package format (zip, tar, etc) store // the package path in config so we don't ask the user to select // the path again when upgrading diff --git a/pkg/providers/generic.go b/pkg/providers/generic.go new file mode 100644 index 0000000..6303b89 --- /dev/null +++ b/pkg/providers/generic.go @@ -0,0 +1,106 @@ +package providers + +import ( + "crypto/sha256" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/apex/log" + "github.com/marcosnils/bin/pkg/assets" +) + +type generic struct { + baseURL string + name string + version string + os string + arch string + ext string + latestURL string +} + +func (g *generic) GetID() string { + return "generic" +} + +func (g *generic) GetLatestVersion() (string, string, error) { + log.Debugf("Getting latest release for %s", g.name) + + resp, err := http.Get(g.latestURL) + if err != nil { + return "", "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", "", fmt.Errorf("HTTP response error: %v", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", "", fmt.Errorf("Error reading response body: %v", err) + } + + version := strings.TrimSuffix(string(body), "\n") + fname := fmt.Sprintf("%s-%s-%s-%s.%s", g.name, version, g.os, g.arch, g.ext) + link := fmt.Sprintf("%s/%s", g.baseURL, fname) + + return version, link, nil +} + +func (g *generic) Fetch(opts *FetchOpts) (*File, error) { + var version string + if len(g.version) > 0 { + log.Infof("Getting release %s for %s", g.version, g.name) + version = g.version + } else { + latest, _, err := g.GetLatestVersion() + if err != nil { + return nil, err + } + version = latest + } + + fname := fmt.Sprintf("%s-%s-%s-%s.%s", g.name, version, g.os, g.arch, g.ext) + link := fmt.Sprintf("%s/%s", g.baseURL, fname) + + candidates := []*assets.Asset{ + { + Name: fname, + URL: link, + }, + } + + f := assets.NewFilter(&assets.FilterOpts{SkipScoring: opts.All, PackagePath: opts.PackagePath, SkipPathCheck: opts.SkipPatchCheck}) + gf, err := f.FilterAssets(g.name, candidates) + if err != nil { + return nil, err + } + + outFile, err := f.ProcessURL(gf) + if err != nil { + return nil, err + } + + return &File{Data: outFile.Source, Name: assets.SanitizeName(outFile.Name, version), Hash: sha256.New(), Version: version}, nil +} + +func newGeneric(u *url.URL, latestURL string) (Provider, error) { + r := regexp.MustCompile(`^(.+)\/(.+)-(v\d*\.\d*\.\d*)-([a-z]+)-([a-z0-9]+).(.*)$`) + + m := r.FindStringSubmatch(u.String()) + if len(m) != 7 { + return nil, errors.New("Failed to parse specified URL") + } + + if len(latestURL) == 0 { + return nil, errors.New("Latest URL must be specified for generic provider") + } + + return &generic{baseURL: m[1], name: m[2], version: m[3], os: m[4], arch: m[5], ext: m[6], latestURL: latestURL}, nil +} diff --git a/pkg/providers/helm.go b/pkg/providers/helm.go deleted file mode 100644 index 9b5dd23..0000000 --- a/pkg/providers/helm.go +++ /dev/null @@ -1,124 +0,0 @@ -package providers - -import ( - "crypto/sha256" - "fmt" - "io" - "net/http" - "net/url" - "regexp" - "strings" - - "github.com/apex/log" - - "github.com/marcosnils/bin/pkg/assets" -) - -const ( - repo = "helm" - helmBaseURL = "https://get.helm.sh" - helmLatestPath = "/helm-latest-version" -) - -var ( - helmOSArchSuffixes = []string{ - "darwin-amd64.tar.gz", - "darwin-arm64.tar.gz", - "linux-amd64.tar.gz", - "linux-arm.tar.gz", - "linux-arm64.tar.gz", - "linux-386.tar.gz", - "linux-ppc64le.tar.gz", - "linux-s390x.tar.gz", - "windows-amd64.zip", - } -) - -type helm struct { - tag string -} - -func (h *helm) GetID() string { - return repo -} - -func (h *helm) GetLatestVersion() (string, string, error) { - log.Debugf("Getting latest Helm release") - - resp, err := http.Get(helmBaseURL + helmLatestPath) - if err != nil { - return "", "", err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return "", "", fmt.Errorf("HTTP response error: %v", resp.StatusCode) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", "", fmt.Errorf("Error reading response body: %v", err) - } - - return strings.TrimSuffix(string(body), "\n"), helmBaseURL, nil -} - -func (h *helm) Fetch(opts *FetchOpts) (*File, error) { - var version string - if len(h.tag) > 0 { - log.Infof("Getting release %s for Helm", h.tag) - version = h.tag - } else { - latest, _, err := h.GetLatestVersion() - if err != nil { - return nil, err - } - version = latest - } - - candidates := []*assets.Asset{} - for _, suffix := range helmOSArchSuffixes { - fname := fmt.Sprintf("%s-%s-%s", repo, version, suffix) - link := fmt.Sprintf("%s/%s", helmBaseURL, fname) - candidates = append(candidates, &assets.Asset{Name: fname, URL: link}) - } - - f := assets.NewFilter(&assets.FilterOpts{SkipScoring: opts.All, PackagePath: opts.PackagePath, SkipPathCheck: opts.SkipPatchCheck}) - gf, err := f.FilterAssets(repo, candidates) - if err != nil { - return nil, err - } - - outFile, err := f.ProcessURL(gf) - if err != nil { - return nil, err - } - - return &File{Data: outFile.Source, Name: assets.SanitizeName(outFile.Name, version), Hash: sha256.New(), Version: version}, nil -} - -func newHelm(u *url.URL) (Provider, error) { - s := strings.Split(u.Path, "/") - - // it's a specific releases URL - var tag string - if len(s) >= 2 { - tag = parseTagVersion(s[1]) - } - - return &helm{tag: tag}, nil -} - -func parseTagVersion(s string) (v string) { - r, err := regexp.Compile(`^.*(v((\d*)\.(\d*)\.(\d*))).*$`) - if err != nil { - return - } - - m := r.FindStringSubmatch(s) - if len(m) > 0 { - v = m[1] - } - - return -} diff --git a/pkg/providers/providers.go b/pkg/providers/providers.go index ccbfdf9..625413c 100644 --- a/pkg/providers/providers.go +++ b/pkg/providers/providers.go @@ -31,6 +31,7 @@ type Provider interface { // Fetch returns the file metadata to retrieve a specific binary given // for a provider Fetch(*FetchOpts) (*File, error) + // GetLatestVersion returns the version and the URL of the // latest version for this binary GetLatestVersion() (string, string, error) @@ -44,16 +45,16 @@ var ( dockerUrlPrefix = regexp.MustCompile("^docker://") ) -func New(u, provider string) (Provider, error) { +func New(u, provider, latestURL string) (Provider, error) { if dockerUrlPrefix.MatchString(u) { return newDocker(u) } + if !httpUrlPrefix.MatchString(u) { u = fmt.Sprintf("https://%s", u) } purl, err := url.Parse(u) - if err != nil { return nil, err } @@ -70,8 +71,8 @@ func New(u, provider string) (Provider, error) { return newHashiCorp(purl) } - if strings.Contains(purl.Host, "get.helm.sh") || provider == "helm" { - return newHelm(purl) + if len(latestURL) != 0 && (provider == "generic" || len(provider) == 0) { + return newGeneric(purl, latestURL) } return nil, fmt.Errorf("Can't find provider for url %s", u) From a775ef36ee66452ae17e001a49c56ae466af7bd5 Mon Sep 17 00:00:00 2001 From: Kalman Speier Date: Sun, 5 Nov 2023 13:52:34 +0100 Subject: [PATCH 4/4] add name and link format function to generic provider --- pkg/providers/generic.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/pkg/providers/generic.go b/pkg/providers/generic.go index 6303b89..ea63ff6 100644 --- a/pkg/providers/generic.go +++ b/pkg/providers/generic.go @@ -47,8 +47,7 @@ func (g *generic) GetLatestVersion() (string, string, error) { } version := strings.TrimSuffix(string(body), "\n") - fname := fmt.Sprintf("%s-%s-%s-%s.%s", g.name, version, g.os, g.arch, g.ext) - link := fmt.Sprintf("%s/%s", g.baseURL, fname) + _, link := g.fmtNameLink(version) return version, link, nil } @@ -66,15 +65,8 @@ func (g *generic) Fetch(opts *FetchOpts) (*File, error) { version = latest } - fname := fmt.Sprintf("%s-%s-%s-%s.%s", g.name, version, g.os, g.arch, g.ext) - link := fmt.Sprintf("%s/%s", g.baseURL, fname) - - candidates := []*assets.Asset{ - { - Name: fname, - URL: link, - }, - } + name, link := g.fmtNameLink(version) + candidates := []*assets.Asset{{Name: name, URL: link}} f := assets.NewFilter(&assets.FilterOpts{SkipScoring: opts.All, PackagePath: opts.PackagePath, SkipPathCheck: opts.SkipPatchCheck}) gf, err := f.FilterAssets(g.name, candidates) @@ -90,6 +82,11 @@ func (g *generic) Fetch(opts *FetchOpts) (*File, error) { return &File{Data: outFile.Source, Name: assets.SanitizeName(outFile.Name, version), Hash: sha256.New(), Version: version}, nil } +func (g *generic) fmtNameLink(version string) (string, string) { + fname := fmt.Sprintf("%s-%s-%s-%s.%s", g.name, version, g.os, g.arch, g.ext) + return fname, fmt.Sprintf("%s/%s", g.baseURL, fname) +} + func newGeneric(u *url.URL, latestURL string) (Provider, error) { r := regexp.MustCompile(`^(.+)\/(.+)-(v\d*\.\d*\.\d*)-([a-z]+)-([a-z0-9]+).(.*)$`)