Skip to content
This repository was archived by the owner on Oct 29, 2023. It is now read-only.

Commit

Permalink
Add downloads for arm64 (Issue #145)
Browse files Browse the repository at this point in the history
  • Loading branch information
@datacharmer
datacharmer committed Aug 15, 2022
1 parent bdf0e2a commit aa25fed
Show file tree
Hide file tree
Showing 10 changed files with 310 additions and 52 deletions.
9 changes: 9 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,12 @@
## 1.69.0 15-Aug-2022

## NEW FEATURES

* Add downloads for arm64 (Issue #145)
* **WARNING** potentially breaking feature:
* Add option `--arch` to `downloads` commands: `get-by-version`, `add` (mandatory), `list`, `tree`


## 1.68.0 12-Aug-2022

### NEW FEATURES
Expand Down
54 changes: 47 additions & 7 deletions cmd/downloads.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import (
func treeRemoteTarballs(cmd *cobra.Command, args []string) {
flavor, _ := cmd.Flags().GetString(globals.FlavorLabel)
OS, _ := cmd.Flags().GetString(globals.OSLabel)
arch, _ := cmd.Flags().GetString(globals.ArchLabel)
version, _ := cmd.Flags().GetString(globals.VersionLabel)
maxItemsPerVersion, _ := cmd.Flags().GetInt(globals.MaxItemsLabel)
showUrl, _ := cmd.Flags().GetBool(globals.ShowUrlLabel)
Expand All @@ -49,6 +50,9 @@ func treeRemoteTarballs(cmd *cobra.Command, args []string) {
if OS == "" {
OS = strings.ToLower(runtime.GOOS)
}
if arch == "" {
arch = strings.ToLower(runtime.GOARCH)
}
if OS == "macos" || OS == "osx" {
OS = "darwin"
}
Expand All @@ -58,6 +62,9 @@ func treeRemoteTarballs(cmd *cobra.Command, args []string) {
if !strings.EqualFold(OS, tb.OperatingSystem) {
continue
}
if !strings.EqualFold(arch, tb.Arch) {
continue
}
if flavor != tb.Flavor {
continue
}
Expand Down Expand Up @@ -141,6 +148,7 @@ func listRemoteTarballs(cmd *cobra.Command, args []string) {

flavor, _ := cmd.Flags().GetString(globals.FlavorLabel)
OS, _ := cmd.Flags().GetString(globals.OSLabel)
arch, _ := cmd.Flags().GetString(globals.ArchLabel)
version, _ := cmd.Flags().GetString(globals.VersionLabel)
sortBy, _ := cmd.Flags().GetString(globals.SortByLabel)
OS = strings.ToLower(OS)
Expand All @@ -161,20 +169,31 @@ func listRemoteTarballs(cmd *cobra.Command, args []string) {
table.Header = &simpletable.Header{
Cells: []*simpletable.Cell{
{Align: simpletable.AlignCenter, Text: headerName},
{Align: simpletable.AlignCenter, Text: "OS"},
{Align: simpletable.AlignCenter, Text: "OS-arch"},
{Align: simpletable.AlignRight, Text: "version"},
{Align: simpletable.AlignCenter, Text: "flavor"},
{Align: simpletable.AlignRight, Text: "size"},
{Align: simpletable.AlignCenter, Text: "minimal"},
},
}

var tarballList = downloads.DefaultTarballRegistry.Tarballs
notes := ""
if downloads.TarballRegistryFileExist() {
notes = fmt.Sprintf("[loaded from %s]", downloads.TarballFileRegistry)
data, err := os.ReadFile(downloads.TarballFileRegistry)
if err != nil {
common.Exitf(1, "error reading from file %s: %s", downloads.TarballFileRegistry, err)
}
var tarballObj downloads.TarballCollection
err = json.Unmarshal(data, &tarballObj)
if err != nil {
common.Exitf(1, "error decoding JSON from file %s: %s", downloads.TarballFileRegistry, err)
}
tarballList = tarballObj.Tarballs
}
fmt.Printf("Available tarballs %s (%s)\n", notes, downloads.DefaultTarballRegistry.UpdatedOn)
tarballList := downloads.DefaultTarballRegistry.Tarballs
//tarballList := downloads.DefaultTarballRegistry.Tarballs
tarballList = downloads.SortedTarballList(tarballList, sortBy)
for _, tb := range tarballList {
var cells []*simpletable.Cell
Expand All @@ -187,7 +206,7 @@ func listRemoteTarballs(cmd *cobra.Command, args []string) {
} else {
cells = append(cells, &simpletable.Cell{Text: tb.Name})
}
cells = append(cells, &simpletable.Cell{Text: tb.OperatingSystem})
cells = append(cells, &simpletable.Cell{Text: tb.OperatingSystem + "-" + tb.Arch})
cells = append(cells, &simpletable.Cell{Align: simpletable.AlignRight, Text: tb.Version})
cells = append(cells, &simpletable.Cell{Text: tb.Flavor})
cells = append(cells, &simpletable.Cell{Align: simpletable.AlignRight, Text: humanize.Bytes(uint64(tb.Size))})
Expand All @@ -201,6 +220,9 @@ func listRemoteTarballs(cmd *cobra.Command, args []string) {
if OS != "" && strings.ToLower(OS) != "all" && OS != strings.ToLower(tb.OperatingSystem) {
continue
}
if arch != "" && !strings.EqualFold(arch, tb.Arch) {
continue
}
table.Body.Cells = append(table.Body.Cells, cells)
}
table.SetStyle(simpletable.StyleCompactLite)
Expand All @@ -214,7 +236,14 @@ func getRemoteTarball(cmd *cobra.Command, args []string) error {
}
options := getCommonFlags(cmd)
options.TarballOS, _ = cmd.Flags().GetString(globals.OSLabel)
options.TarballArch, _ = cmd.Flags().GetString(globals.ArchLabel)
options.Unpack, _ = cmd.Flags().GetBool(globals.UnpackLabel)
if options.TarballOS == "" {
options.TarballOS = strings.ToLower(runtime.GOOS)
}
if options.TarballArch == "" {
options.TarballArch = strings.ToLower(runtime.GOARCH)
}
if common.IsUrl(args[0]) {
options.TarballUrl = args[0]
} else {
Expand Down Expand Up @@ -248,8 +277,12 @@ func getRemoteTarballByVersion(cmd *cobra.Command, args []string) error {
options.Newest, _ = cmd.Flags().GetBool(globals.NewestLabel)
options.GuessLatest, _ = cmd.Flags().GetBool(globals.GuessLatestLabel)
options.TarballOS, _ = cmd.Flags().GetString(globals.OSLabel)
options.TarballArch, _ = cmd.Flags().GetString(globals.ArchLabel)
options.Unpack, _ = cmd.Flags().GetBool(globals.UnpackLabel)
options.Version = args[0]
if options.TarballArch == "" {
options.TarballArch = runtime.GOARCH
}

return ops.GetRemoteTarball(options)
}
Expand Down Expand Up @@ -410,7 +443,7 @@ func addRemoteTarballToCollection(cmd *cobra.Command, args []string) {
if err == nil {
if overwrite {
var newList []downloads.TarballDescription
newList, err = downloads.DeleteTarball(tb.Name)
newList, err = downloads.DeleteTarball(tarballCollection.Tarballs, tb.Name)
if err != nil {
common.Exitf(1, "error removing tarball %s from list", tb.Name)
}
Expand Down Expand Up @@ -460,6 +493,7 @@ func addTarballToCollection(cmd *cobra.Command, args []string) {
baseName := common.BaseName(fileName)

OS, _ := flags.GetString(globals.OSLabel)
arch, _ := flags.GetString(globals.ArchLabel)
flavor, _ := flags.GetString(globals.FlavorLabel)
tarballUrl, _ := flags.GetString(globals.UrlLabel)
version, _ := flags.GetString(globals.VersionLabel)
Expand All @@ -470,7 +504,7 @@ func addTarballToCollection(cmd *cobra.Command, args []string) {
if err == nil {
if overwrite {
var newList []downloads.TarballDescription
newList, err = downloads.DeleteTarball(baseName)
newList, err = downloads.DeleteTarball(tarballCollection.Tarballs, baseName)
if err != nil {
common.Exitf(1, "error removing tarball %s from list", baseName)
}
Expand All @@ -483,6 +517,7 @@ func addTarballToCollection(cmd *cobra.Command, args []string) {
}
var tarballDesc = downloads.TarballDescription{
OperatingSystem: OS,
Arch: arch,
Minimal: minimal,
ShortVersion: shortVersion,
Version: version,
Expand Down Expand Up @@ -525,7 +560,7 @@ func addTarballToCollectionFromStdin(cmd *cobra.Command, args []string) {
if err == nil {
if overwrite {
var newList []downloads.TarballDescription
newList, err = downloads.DeleteTarball(tarballDesc.Name)
newList, err = downloads.DeleteTarball(downloads.DefaultTarballRegistry.Tarballs, tarballDesc.Name)
if err != nil {
common.Exitf(1, "error removing tarball %s from list", tarballDesc.Name)
}
Expand Down Expand Up @@ -557,7 +592,7 @@ func removeTarballFromCollection(cmd *cobra.Command, args []string) {
var tarballCollection = downloads.DefaultTarballRegistry

var newList []downloads.TarballDescription
newList, err = downloads.DeleteTarball(fileName)
newList, err = downloads.DeleteTarball(tarballCollection.Tarballs, fileName)
if err != nil {
common.Exitf(1, "error removing tarball %s from list", fileName)
}
Expand Down Expand Up @@ -743,12 +778,14 @@ func init() {
downloadsListCmd.Flags().BoolP(globals.ShowUrlLabel, "", false, "Show the URL")
downloadsListCmd.Flags().String(globals.FlavorLabel, "", "Which flavor will be listed")
downloadsListCmd.Flags().String(globals.OSLabel, "", "Which OS will be listed")
downloadsListCmd.Flags().String(globals.ArchLabel, "", "Which architecture will be listed")
downloadsListCmd.Flags().String(globals.VersionLabel, "", "Which version will be listed")
downloadsListCmd.Flags().String(globals.SortByLabel, "name", "Sort by field {name/date/version}")

downloadsTreeCmd.Flags().String(globals.FlavorLabel, "", "Which flavor will be listed")
downloadsTreeCmd.Flags().BoolP(globals.ShowUrlLabel, "", false, "Show the URL")
downloadsTreeCmd.Flags().String(globals.OSLabel, "", "Which OS will be listed")
downloadsTreeCmd.Flags().String(globals.ArchLabel, "", "Which OS will be listed")
downloadsTreeCmd.Flags().String(globals.VersionLabel, "", "Which version will be listed")
downloadsTreeCmd.Flags().IntP(globals.MaxItemsLabel, "", 3, "Show a maximum of items for each Short version (0 = ALL)")
_ = downloadsTreeCmd.MarkFlagRequired(globals.FlavorLabel)
Expand All @@ -757,6 +794,7 @@ func init() {
downloadsGetByVersionCmd.Flags().BoolP(globals.MinimalLabel, "", false, "Choose only minimal tarballs")
downloadsGetByVersionCmd.Flags().String(globals.FlavorLabel, "", "Choose only the given flavor")
downloadsGetByVersionCmd.Flags().String(globals.OSLabel, "", "Choose only the given OS")
downloadsGetByVersionCmd.Flags().String(globals.ArchLabel, "", "Choose only the given arch")
downloadsGetByVersionCmd.Flags().BoolP(globals.GuessLatestLabel, "", false, "Guess the latest version (highest version w/ increased revision number)")
downloadsGetByVersionCmd.Flags().BoolP(globals.UnpackLabel, "", false, "Unpack after downloading")
addCommonDownloadsFlags(downloadsGetByVersionCmd)
Expand All @@ -768,6 +806,7 @@ func init() {
addCommonDownloadsFlags(downloadsGetUnpackCmd)

downloadsAddCmd.Flags().String(globals.OSLabel, "", "Define the tarball OS")
downloadsAddCmd.Flags().String(globals.ArchLabel, "", "Define the tarball architecture")
downloadsAddCmd.Flags().String(globals.FlavorLabel, "", "Define the tarball flavor")
downloadsAddCmd.Flags().String(globals.VersionLabel, "", "Define the tarball version")
downloadsAddCmd.Flags().String(globals.ShortVersionLabel, "", "Define the tarball short version")
Expand All @@ -776,6 +815,7 @@ func init() {
downloadsAddCmd.Flags().BoolP(globals.OverwriteLabel, "", false, "Overwrite existing entry")
_ = downloadsAddCmd.MarkFlagRequired(globals.UrlLabel)
_ = downloadsAddCmd.MarkFlagRequired(globals.OSLabel)
_ = downloadsAddCmd.MarkFlagRequired(globals.ArchLabel)

downloadsAddRemoteCmd.Flags().BoolP(globals.OverwriteLabel, "", false, "Overwrite existing entry")
downloadsAddRemoteCmd.Flags().BoolP(globals.MinimalLabel, "", false, "Define whether the wanted tarball is a minimal one")
Expand Down
28 changes: 19 additions & 9 deletions downloads/remote_registry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type TarballDescription struct {
Name string `json:"name"`
Checksum string `json:"checksum,omitempty"`
OperatingSystem string `json:"OS"`
Arch string `json:"arch"`
Url string `json:"url"`
Flavor string `json:"flavor"`
Minimal bool `json:"minimal"`
Expand Down Expand Up @@ -167,10 +168,10 @@ func FindTarballByName(tarballName string) (TarballDescription, error) {
}
return TarballDescription{}, fmt.Errorf("tarball with name %s not found", tarballName)
}
func DeleteTarball(tarballName string) ([]TarballDescription, error) {
func DeleteTarball(tarballs []TarballDescription, tarballName string) ([]TarballDescription, error) {
var newList []TarballDescription
found := false
for _, tb := range DefaultTarballRegistry.Tarballs {
for _, tb := range tarballs {
if tb.Name == tarballName {
found = true
} else {
Expand Down Expand Up @@ -214,25 +215,34 @@ func CompareTarballChecksum(tarball TarballDescription, fileName string) error {
return nil
}

func FindTarballByVersionFlavorOS(version, flavor, OS string, minimal, newest bool) (TarballDescription, error) {
return FindOrGuessTarballByVersionFlavorOS(version, flavor, OS, minimal, newest, false)
func FindTarballByVersionFlavorOS(version, flavor, OS, arch string, minimal, newest bool) (TarballDescription, error) {
return FindOrGuessTarballByVersionFlavorOS(version, flavor, OS, arch, minimal, newest, false)
}

func FindOrGuessTarballByVersionFlavorOS(version, flavor, OS string, minimal, newest, guess bool) (TarballDescription, error) {
func FindOrGuessTarballByVersionFlavorOS(version, flavor, OS, arch string, minimal, newest, guess bool) (TarballDescription, error) {
flavor = strings.ToLower(flavor)
OS = strings.ToLower(OS)
arch = strings.ToLower(arch)
if OS == "osx" || OS == "macos" || OS == "os x" {
OS = "darwin"
}
if arch == "x86_64" || arch == "x86-64" {
arch = "amd64"
}
if guess {
minimal = false
}
var tbd []TarballDescription
newestVersionList := []int{0, 0, 0}
for _, tb := range DefaultTarballRegistry.Tarballs {
archMatch := true
if tb.Arch != "" {
archMatch = strings.ToLower(tb.Arch) == arch
}
if (tb.Version == version || tb.ShortVersion == version) &&
strings.ToLower(tb.Flavor) == flavor &&
strings.ToLower(tb.OperatingSystem) == OS &&
archMatch &&
(!minimal || minimal == tb.Minimal) {

if guess {
Expand Down Expand Up @@ -521,7 +531,7 @@ func CheckTarballList(tarballList []TarballDescription) error {
uniqueNames := make(map[string]bool)
uniqueCombinations := make(map[string]bool)
for _, tb := range tarballList {
key := fmt.Sprintf("%s-%s-%s-%v", tb.OperatingSystem, tb.Flavor, tb.Version, tb.Minimal)
key := fmt.Sprintf("%s-%s-%s-%s-%v", tb.OperatingSystem, tb.Arch, tb.Flavor, tb.Version, tb.Minimal)

// Makes sure that we don't have duplicate names in the list
_, seen := uniqueNames[tb.Name]
Expand All @@ -530,11 +540,11 @@ func CheckTarballList(tarballList []TarballDescription) error {
}
uniqueNames[tb.Name] = true

// Makes sure that we don't have duplicate combinations of OS+Flavor+Version+Minimal in the list
// Makes sure that we don't have duplicate combinations of OS+arch+Flavor+Version+Minimal in the list
_, seen = uniqueCombinations[key]
if seen {
return fmt.Errorf("tarball with OS %s, flavor %s, version %s, and minimal %v listed more than once",
tb.OperatingSystem, tb.Flavor, tb.Version, tb.Minimal)
return fmt.Errorf("tarball with OS %s-%s, flavor %s, version %s, and minimal %v listed more than once",
tb.OperatingSystem, tb.Arch, tb.Flavor, tb.Version, tb.Minimal)
}
uniqueCombinations[key] = true
}
Expand Down
13 changes: 11 additions & 2 deletions downloads/remote_tarballs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,8 @@ func GetRemoteTarballList(tarballType TarballType, version, OS string, withSize
`\s*` + // optional spaces
`(\w+)`)) // Capture a string of alphanumeric characters (the checksum)
*/
reLine := regexp.MustCompile(fmt.Sprintf(`\((%s-\d+\.\d+\.\d+[^\)]+x86.64[^\)]+z)\).*?MD5:\s*(\w+)`, downloadsSettings[tarballType].NameInFile))
//reLine := regexp.MustCompile(fmt.Sprintf(`\((%s-\d+\.\d+\.\d+[^\)]+x86.64[^\)]+z)\).*?MD5:\s*(\w+)`, downloadsSettings[tarballType].NameInFile))
reLine := regexp.MustCompile(fmt.Sprintf(`\((%s-\d+\.\d+\.\d+[^\)]+-(\w+64)[^\)]+z)\).*?MD5:\s*(\w+)`, downloadsSettings[tarballType].NameInFile))
matches := reLine.FindAllStringSubmatch(text, -1)
if len(matches) == 0 {
return nil, fmt.Errorf("no %s tarballs found for %s", tarballType, version)
Expand All @@ -195,8 +196,9 @@ func GetRemoteTarballList(tarballType TarballType, version, OS string, withSize

tbd := TarballDescription{
Name: m[1],
Checksum: "MD5:" + m[2],
Checksum: "MD5:" + m[3],
OperatingSystem: internalOsName[OS],
Arch: archNormalize(m[2]),
ShortVersion: shortVersion,
Version: longVersion,
Flavor: downloadsSettings[tarballType].Flavor,
Expand All @@ -215,3 +217,10 @@ func GetRemoteTarballList(tarballType TarballType, version, OS string, withSize
}
return result, nil
}

func archNormalize(s string) string {
if s == "x86_64" {
return "amd64"
}
return s
}
Loading

0 comments on commit aa25fed

Please sign in to comment.