list s3 objects

cmd/release/cmd/generate.go

@@ -8,6 +8,9 @@ import (
 	"os"
 	"strings"
 
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/rancher/ecm-distro-tools/release"
 	"github.com/rancher/ecm-distro-tools/release/k3s"
 	"github.com/rancher/ecm-distro-tools/release/rancher"
@@ -113,7 +116,13 @@ var rancherGenerateArtifactsIndexSubCmd = &cobra.Command{
 	Use:   "artifacts-index",
 	Short: "Generate artifacts index page",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return rancher.GeneratePrimeArtifactsIndex(rancherArtifactsIndexWriteToPath, rancherArtifactsIndexIgnoreVersions)
+		ctx := context.Background()
+		cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(rootConfig.Auth.AWSAccessKeyID, rootConfig.Auth.AWSSecretAccessKey, rootConfig.Auth.AWSSessionToken)), config.WithDefaultRegion(rootConfig.Auth.AWSDefaultRegion))
+		if err != nil {
+			return err
+		}
+		client := s3.NewFromConfig(cfg)
+		return rancher.GeneratePrimeArtifactsIndex(ctx, rancherArtifactsIndexWriteToPath, rancherArtifactsIndexIgnoreVersions, client)
 	},
 }
 

cmd/release/cmd/root.go

@@ -55,9 +55,6 @@ func initConfig() {
 		if os.Args[1] == "config" && os.Args[2] == "gen" {
 			return
 		}
-		if os.Args[1] == "config" && os.Args[2] == "edit" {
-			ignoreValidate = true
-		}
 	}
 	var conf *config.Config
 	var err error

cmd/release/config/config.go

@@ -76,6 +76,7 @@ type Auth struct {
 	SSHKeyPath         string `json:"ssh_key_path" validate:"filepath"`
 	AWSAccessKeyID     string `json:"aws_access_key_id"`
 	AWSSecretAccessKey string `json:"aws_secret_access_key"`
+	AWSSessionToken    string `json:"aws_session_token"`
 	AWSDefaultRegion   string `json:"aws_default_region"`
 }
 
@@ -184,6 +185,7 @@ func ExampleConfig() (string, error) {
 			SSHKeyPath:         "path/to/your/ssh/key",
 			AWSAccessKeyID:     "XXXXXXXXXXXXXXXXXXX",
 			AWSSecretAccessKey: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+			AWSSessionToken:    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
 			AWSDefaultRegion:   "us-east-1",
 		},
 	}

go.mod

@@ -17,6 +17,9 @@ require (
 
 require (
 	github.com/MetalBlueberry/go-plotly v0.4.0
+	github.com/aws/aws-sdk-go-v2/config v1.27.29
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.29
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1
 	github.com/go-playground/validator/v10 v10.22.0
 	github.com/spf13/cobra v1.8.0
 	github.com/urfave/cli/v2 v2.25.7
@@ -28,6 +31,21 @@ require (
 
 require (
 	dario.cat/mergo v1.0.0 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.30.4 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 // indirect
+	github.com/aws/smithy-go v1.20.4 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect

go.sum

@@ -14,6 +14,42 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI
 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8=
+github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw=
+github.com/aws/aws-sdk-go-v2/config v1.27.29 h1:+ZPKb3u9Up4KZWLGTtpTmC5T3XmRD1ZQ8XQjRCHUvJw=
+github.com/aws/aws-sdk-go-v2/config v1.27.29/go.mod h1:yxqvuubha9Vw8stEgNiStO+yZpP68Wm9hLmcm+R/Qk4=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.29 h1:CwGsupsXIlAFYuDVHv1nnK0wnxO0wZ/g1L8DSK/xiIw=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.29/go.mod h1:BPJ/yXV92ZVq6G8uYvbU0gSl8q94UB63nMT5ctNO38g=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 h1:GckUnpm4EJOAio1c8o25a+b3lVfwVzC9gnSBqiiNmZM=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18/go.mod h1:Br6+bxfG33Dk3ynmkhsW2Z/t9D4+lRqdLDNCKi85w0U=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 h1:jg16PhLPUiHIj8zYIW6bqzeQSuHVEiWnGA0Brz5Xv2I=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16/go.mod h1:Uyk1zE1VVdsHSU7096h/rwnXDzOzYQVl+FNPhPw7ShY=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1 h1:mx2ucgtv+MWzJesJY9Ig/8AFHgoE5FwLXwUVgW/FGdI=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.60.1/go.mod h1:BSPI0EfnYUuNHPS0uqIo5VrRwzie+Fp+YhQOUs16sKI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 h1:zCsFCKvbj25i7p1u94imVoO447I/sFv8qq+lGJhRN0c=
+github.com/aws/aws-sdk-go-v2/service/sso v1.22.5/go.mod h1:ZeDX1SnKsVlejeuz41GiajjZpRSWR7/42q/EyA/QEiM=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 h1:SKvPgvdvmiTWoi0GAJ7AsJfOz3ngVkD/ERbs5pUnHNI=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5/go.mod h1:20sz31hv/WsPa3HhU3hfrIet2kxM4Pe0r20eBZ20Tac=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 h1:OMsEmCyz2i89XwRwPouAJvhj81wINh+4UK+k/0Yo/q8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.30.5/go.mod h1:vmSqFK+BVIwVpDAGZB3CoCXHzurt4qBE8lf+I/kRTh0=
+github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4=
+github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=

release/rancher/rancher.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"encoding/xml"
 	"errors"
 	"fmt"
 	htmlTemplate "html/template"
@@ -24,6 +23,7 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/google/go-github/v39/github"
 	ecmConfig "github.com/rancher/ecm-distro-tools/cmd/release/config"
 	ecmHTTP "github.com/rancher/ecm-distro-tools/http"
@@ -40,7 +40,8 @@ const (
 	rancherImagesBaseURL          = "https://github.com/rancher/rancher/releases/download/"
 	rancherImagesFileName         = "/rancher-images.txt"
 	rancherHelmRepositoryURL      = "https://releases.rancher.com/server-charts/latest/index.yaml"
-	rancherArtifactsListURL       = "https://prime-artifacts.s3.amazonaws.com"
+	rancherArtifactsBucket        = "prime-artifacts"
+	rancherArtifactsPrefix        = "rancher/v"
 	rancherArtifactsBaseURL       = "https://prime.ribs.rancher.io"
 	rancherRegistryBaseURL        = "https://registry.rancher.com"
 	stagingRancherRegistryBaseURL = "https://stgregistry.suse.com"
@@ -92,12 +93,6 @@ type RancherRCDeps struct {
 	KDMWithDev     []RancherRCDepsLine `json:"kdmWithDev"`
 }
 
-type ListBucketResult struct {
-	Contents []struct {
-		Key string `xml:"Key"`
-	} `xml:"Contents"`
-}
-
 type ArtifactsIndexContent struct {
 	GA         ArtifactsIndexContentGroup `json:"ga"`
 	PreRelease ArtifactsIndexContentGroup `json:"preRelease"`
@@ -113,28 +108,44 @@ type registryAuthToken struct {
 	Token string `json:"token"`
 }
 
-func GeneratePrimeArtifactsIndex(path string, ignoreVersions []string) error {
-	client := ecmHTTP.NewClient(time.Second * 15)
-	resp, err := client.Get(rancherArtifactsListURL)
-	if err != nil {
-		return err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return errors.New("unexpected status code on " + rancherArtifactsListURL + " expected 200, got " + strconv.Itoa(resp.StatusCode))
-	}
-	defer resp.Body.Close()
-	contentDecoder := xml.NewDecoder(resp.Body)
-	var listBucket ListBucketResult
-	if err := contentDecoder.Decode(&listBucket); err != nil {
-		return err
+func listS3Objects(ctx context.Context, s3Client *s3.Client, bucketName string, prefix string) ([]string, error) {
+	fmt.Println(bucketName, prefix)
+	keys := []string{}
+	var continuationToken *string
+	isTruncated := true
+	for isTruncated {
+		fmt.Println("getting objects")
+		objects, err := s3Client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
+			Bucket:            &bucketName,
+			Prefix:            &prefix,
+			ContinuationToken: continuationToken,
+		})
+		if err != nil {
+			return nil, err
+		}
+		for _, object := range objects.Contents {
+			keys = append(keys, *object.Key)
+		}
+		// used for pagination
+		continuationToken = objects.NextContinuationToken
+		// if the bucket has more keys
+		if objects.IsTruncated != nil && !*objects.IsTruncated {
+			isTruncated = false
+		}
 	}
+	return keys, nil
+}
 
+func GeneratePrimeArtifactsIndex(ctx context.Context, path string, ignoreVersions []string, s3Client *s3.Client) error {
 	ignore := make(map[string]bool, len(ignoreVersions))
 	for _, v := range ignoreVersions {
 		ignore[v] = true
 	}
-
-	content := generateArtifactsIndexContent(listBucket, ignore)
+	keys, err := listS3Objects(ctx, s3Client, rancherArtifactsBucket, rancherArtifactsPrefix)
+	if err != nil {
+		return err
+	}
+	content := generateArtifactsIndexContent(keys, ignore)
 	gaIndex, err := generatePrimeArtifactsHTML(content.GA)
 	if err != nil {
 		return err
@@ -149,7 +160,7 @@ func GeneratePrimeArtifactsIndex(path string, ignoreVersions []string) error {
 	return os.WriteFile(filepath.Join(path, "index-prerelease.html"), preReleaseIndex, 0644)
 }
 
-func generateArtifactsIndexContent(listBucket ListBucketResult, ignoreVersions map[string]bool) ArtifactsIndexContent {
+func generateArtifactsIndexContent(keys []string, ignoreVersions map[string]bool) ArtifactsIndexContent {
 	indexContent := ArtifactsIndexContent{
 		GA: ArtifactsIndexContentGroup{
 			Versions:      []string{},
@@ -165,11 +176,11 @@ func generateArtifactsIndexContent(listBucket ListBucketResult, ignoreVersions m
 	var versions []string
 	versionsFiles := make(map[string][]string)
 
-	for _, content := range listBucket.Contents {
-		if !strings.Contains(content.Key, "rancher/") {
+	for _, key := range keys {
+		if !strings.Contains(key, "rancher/") {
 			continue
 		}
-		keyFile := strings.Split(strings.TrimPrefix(content.Key, "rancher/"), "/")
+		keyFile := strings.Split(strings.TrimPrefix(key, "rancher/"), "/")
 		if len(keyFile) < 2 || keyFile[1] == "" {
 			continue
 		}