refactor: use the latest version info to track latest artifacts of cn region

pkg/artifacts/constants.go

@@ -21,14 +21,17 @@ const (
 	// GreptimeChartReleaseDownloadURL is the URL of the Greptime charts that stored in the GitHub release.
 	GreptimeChartReleaseDownloadURL = "https://github.com/GreptimeTeam/helm-charts/releases/download"
 
+	// Greptime release bucket public endpoint in CN region.
+	GreptimeReleaseBucketCN = "https://downloads.greptime.cn/releases"
+
 	// GreptimeCNCharts is the URL of the Greptime charts that stored in the S3 bucket of the CN region.
-	GreptimeCNCharts = "https://downloads.greptime.cn/releases/charts"
+	GreptimeCNCharts = GreptimeReleaseBucketCN + "/charts"
 
 	// GreptimeDBCNBinaries is the URL of the GreptimeDB binaries that stored in the S3 bucket of the CN region.
-	GreptimeDBCNBinaries = "https://downloads.greptime.cn/releases/greptimedb"
+	GreptimeDBCNBinaries = GreptimeReleaseBucketCN + "/greptimedb"
 
 	// EtcdCNBinaries is the URL of the etcd binaries that stored in the S3 bucket of the CN region.
-	EtcdCNBinaries = "https://downloads.greptime.cn/releases/etcd"
+	EtcdCNBinaries = GreptimeReleaseBucketCN + "/etcd"
 
 	// LatestVersionTag is the tag of the latest version.
 	LatestVersionTag = "latest"
@@ -54,8 +57,8 @@ const (
 	// EtcdBinName is the artifact name of etcd.
 	EtcdBinName = "etcd"
 
-	// GreptimeDBChartName is the chart name of GreptimeDB.
-	GreptimeDBChartName = "greptimedb"
+	// GreptimeDBClusterChartName is the chart name of GreptimeDB.
+	GreptimeDBClusterChartName = "greptimedb-cluster"
 
 	// GreptimeDBOperatorChartName is the chart name of GreptimeDB operator.
 	GreptimeDBOperatorChartName = "greptimedb-operator"

pkg/artifacts/manager.go

@@ -119,32 +119,24 @@ func (m *manager) NewSource(name, version string, typ ArtifactType, fromCNRegion
 		FromCNRegion: fromCNRegion,
 	}
 
+	if version == LatestVersionTag || len(version) == 0 {
+		latestVersion, err := m.resolveLatestVersion(typ, name, fromCNRegion)
+		if err != nil {
+			return nil, err
+		}
+		src.Version = latestVersion
+	}
+
 	if src.Type == ArtifactTypeChart {
 		src.FileName = m.chartFileName(src.Name, src.Version)
 		if src.FromCNRegion {
 			// The download URL example: 'https://downloads.greptime.cn/releases/charts/etcd/9.2.0/etcd-9.2.0.tgz'.
-			src.URL = fmt.Sprintf("%s/%s/%s/%s", GreptimeCNCharts, src.Name, version, src.FileName)
+			src.URL = fmt.Sprintf("%s/%s/%s/%s", GreptimeCNCharts, src.Name, src.Version, src.FileName)
 		} else {
 			// Specify the OCI registry URL for the etcd chart.
 			if src.Name == EtcdChartName {
 				// The download URL example: 'oci://registry-1.docker.io/bitnamicharts/etcd:9.2.0'.
 				src.URL = EtcdOCIRegistry
-				return src, nil
-			}
-
-			if src.Version == LatestVersionTag {
-				// Use chart index file to locate the latest chart version.
-				indexFile, err := m.chartIndexFile(context.TODO(), GreptimeChartIndexURL)
-				if err != nil {
-					return nil, err
-				}
-
-				chartVersion, err := m.latestChartVersion(indexFile, src.Name)
-				if err != nil {
-					return nil, err
-				}
-
-				src.URL = chartVersion.URLs[0]
 			} else {
 				// The download URL example: 'https://github.com/GreptimeTeam/helm-charts/releases/download/greptimedb-0.1.1-alpha.3/greptimedb-0.1.1-alpha.3.tgz'.
 				src.URL = fmt.Sprintf("%s/%s/%s", GreptimeChartReleaseDownloadURL, strings.TrimSuffix(src.FileName, fileutils.TgzExtension), src.FileName)
@@ -382,14 +374,6 @@ func (m *manager) latestGitHubReleaseVersion(org, repo string) (string, error) {
 }
 
 func (m *manager) etcdBinaryDownloadURL(version string, fromCNRegion bool) (string, error) {
-	if version != DefaultEtcdBinVersion && fromCNRegion {
-		return "", fmt.Errorf("only support %s in cn region", DefaultEtcdBinVersion)
-	}
-
-	if version == LatestVersionTag {
-		return "", fmt.Errorf("can't support latest version")
-	}
-
 	var ext string
 
 	switch runtime.GOOS {
@@ -413,11 +397,6 @@ func (m *manager) etcdBinaryDownloadURL(version string, fromCNRegion bool) (stri
 }
 
 func (m *manager) greptimeBinaryDownloadURL(version string, fromCNRegion bool) (string, error) {
-	// FIXME(zyy17): we should support the latest version in the future.
-	if version == LatestVersionTag && fromCNRegion {
-		return "", fmt.Errorf("can't support latest version in cn region")
-	}
-
 	newVersion, err := isBreakingVersion(version)
 	if err != nil {
 		return "", err
@@ -480,6 +459,80 @@ func (m *manager) installBinaries(downloadFile, installDir string) error {
 	return nil
 }
 
+// resolveLatestVersion resolves the latest tag to the specific version.
+func (m *manager) resolveLatestVersion(typ ArtifactType, name string, fromCNRegion bool) (string, error) {
+	if fromCNRegion {
+		return m.getVersionInfoFromS3(typ, name, false)
+	}
+
+	switch typ {
+	case ArtifactTypeChart:
+		// Use chart index file to locate the latest chart version.
+		indexFile, err := m.chartIndexFile(context.TODO(), GreptimeChartIndexURL)
+		if err != nil {
+			return "", err
+		}
+
+		chartVersion, err := m.latestChartVersion(indexFile, name)
+		if err != nil {
+			return "", err
+		}
+		return chartVersion.Version, nil
+	case ArtifactTypeBinary:
+		// Get the latest version of the latest greptime binary.
+		latestVersion, err := m.latestGitHubReleaseVersion(GreptimeGitHubOrg, GreptimeDBGithubRepo)
+		if err != nil {
+			return "", err
+		}
+		return latestVersion, nil
+	default:
+		return "", fmt.Errorf("unsupported artifact type: %s", string(typ))
+	}
+}
+
+// getVersionInfoFromS3 gets the latest version info from S3.
+func (m *manager) getVersionInfoFromS3(typ ArtifactType, name string, nightly bool) (string, error) {
+	// Note: it uses 'greptimedb' directory to store the greptime binary.
+	if name == GreptimeBinName {
+		name = "greptimedb"
+	}
+	var latestVersionInfoURL string
+	switch typ {
+	case ArtifactTypeChart:
+		latestVersionInfoURL = fmt.Sprintf("%s/charts/%s/latest-version.txt", GreptimeReleaseBucketCN, name)
+	case ArtifactTypeBinary:
+		if nightly {
+			latestVersionInfoURL = fmt.Sprintf("%s/%s/latest-nightly-version.txt", GreptimeReleaseBucketCN, name)
+		} else {
+			latestVersionInfoURL = fmt.Sprintf("%s/%s/latest-version.txt", GreptimeReleaseBucketCN, name)
+		}
+	default:
+		return "", fmt.Errorf("unsupported artifact type: %s", string(typ))
+	}
+
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, latestVersionInfoURL, nil)
+	if err != nil {
+		return "", err
+	}
+
+	httpClient := &http.Client{}
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("get latest info from '%s' failed, status code: %d", latestVersionInfoURL, resp.StatusCode)
+	}
+	defer resp.Body.Close()
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimRight(string(data), "\n"), nil
+}
+
 // BreakingChangeVersion is the version that the download URL of the greptime binary is changed.
 const BreakingChangeVersion = "v0.4.0-nightly-20230802"
 

pkg/artifacts/manager_test.go

@@ -46,9 +46,9 @@ func TestDownloadCharts(t *testing.T) {
 		typ          ArtifactType
 		fromCNRegion bool
 	}{
-		{GreptimeDBChartName, "latest", ArtifactTypeChart, false},
+		{GreptimeDBClusterChartName, "latest", ArtifactTypeChart, false},
 		{GreptimeDBOperatorChartName, "latest", ArtifactTypeChart, false},
-		{GreptimeDBChartName, "0.1.1-alpha.13", ArtifactTypeChart, false},
+		{GreptimeDBClusterChartName, "0.1.2", ArtifactTypeChart, false},
 		{GreptimeDBOperatorChartName, "0.1.1-alpha.12", ArtifactTypeChart, false},
 		{EtcdChartName, DefaultEtcdChartVersion, ArtifactTypeChart, false},
 	}
@@ -92,9 +92,9 @@ func TestDownloadChartsFromCNRegion(t *testing.T) {
 		typ          ArtifactType
 		fromCNRegion bool
 	}{
-		{GreptimeDBChartName, LatestVersionTag, ArtifactTypeChart, true},
+		{GreptimeDBClusterChartName, LatestVersionTag, ArtifactTypeChart, true},
 		{GreptimeDBOperatorChartName, LatestVersionTag, ArtifactTypeChart, true},
-		{GreptimeDBChartName, "0.1.1-alpha.13", ArtifactTypeChart, true},
+		{GreptimeDBClusterChartName, "0.1.2", ArtifactTypeChart, true},
 		{GreptimeDBOperatorChartName, "0.1.1-alpha.12", ArtifactTypeChart, true},
 		{EtcdChartName, DefaultEtcdChartVersion, ArtifactTypeChart, true},
 	}
@@ -225,7 +225,7 @@ func TestArtifactsCache(t *testing.T) {
 
 	ctx := context.Background()
 
-	src, err := m.NewSource(GreptimeDBChartName, LatestVersionTag, ArtifactTypeChart, false)
+	src, err := m.NewSource(GreptimeDBClusterChartName, LatestVersionTag, ArtifactTypeChart, false)
 	if err != nil {
 		t.Errorf("failed to create source: %v", err)
 	}

pkg/cmd/gtctl/cluster/create/create.go

@@ -315,7 +315,7 @@ func deployGreptimeDBCluster(ctx context.Context, l logger.Logger, options *Clus
 		DatanodeStorageClassName:    options.StorageClassName,
 		DatanodeStorageSize:         options.StorageSize,
 		DatanodeStorageRetainPolicy: options.StorageRetainPolicy,
-		EtcdEndPoint:                fmt.Sprintf("%s.%s:2379", common.EtcdClusterName(clusterName), options.EtcdNamespace),
+		EtcdEndPoints:               fmt.Sprintf("%s.%s:2379", common.EtcdClusterName(clusterName), options.EtcdNamespace),
 		ConfigValues:                options.Set.clusterConfig,
 		UseGreptimeCNArtifacts:      options.UseGreptimeCNArtifacts,
 		ValuesFile:                  options.GreptimeDBClusterValuesFile,

pkg/deployer/baremetal/deployer.go

@@ -300,7 +300,7 @@ func (d *Deployer) CreateGreptimeDBCluster(ctx context.Context, clusterName stri
 		if d.config.Cluster.Artifact.Local != "" {
 			binPath = d.config.Cluster.Artifact.Local
 		} else {
-			src, err := d.am.NewSource(artifacts.GreptimeBinName, d.config.Cluster.Artifact.Version, artifacts.ArtifactTypeBinary, false)
+			src, err := d.am.NewSource(artifacts.GreptimeBinName, d.config.Cluster.Artifact.Version, artifacts.ArtifactTypeBinary, options.UseGreptimeCNArtifacts)
 			if err != nil {
 				return err
 			}
@@ -379,7 +379,7 @@ func (d *Deployer) CreateEtcdCluster(ctx context.Context, clusterName string, op
 		if d.config.Etcd.Artifact.Local != "" {
 			binPath = d.config.Etcd.Artifact.Local
 		} else {
-			src, err := d.am.NewSource(artifacts.EtcdBinName, d.config.Etcd.Artifact.Version, artifacts.ArtifactTypeBinary, false)
+			src, err := d.am.NewSource(artifacts.EtcdBinName, d.config.Etcd.Artifact.Version, artifacts.ArtifactTypeBinary, options.UseGreptimeCNArtifacts)
 			if err != nil {
 				return err
 			}

pkg/deployer/k8s/deployer.go

@@ -130,7 +130,7 @@ func (d *deployer) CreateGreptimeDBCluster(ctx context.Context, name string, opt
 	opts := &helm.LoadOptions{
 		ReleaseName:   resourceName,
 		Namespace:     resourceNamespace,
-		ChartName:     artifacts.GreptimeDBChartName,
+		ChartName:     artifacts.GreptimeDBClusterChartName,
 		ChartVersion:  options.GreptimeDBChartVersion,
 		FromCNRegion:  options.UseGreptimeCNArtifacts,
 		ValuesOptions: *options,

pkg/deployer/types.go

@@ -77,7 +77,7 @@ type CreateGreptimeDBClusterOptions struct {
 	DatanodeStorageClassName    string `helm:"datanode.storage.storageClassName"`
 	DatanodeStorageSize         string `helm:"datanode.storage.storageSize"`
 	DatanodeStorageRetainPolicy string `helm:"datanode.storage.storageRetainPolicy"`
-	EtcdEndPoint                string `helm:"etcdEndpoints"`
+	EtcdEndPoints               string `helm:"meta.etcdEndpoints"`
 	ConfigValues                string `helm:"*"`
 }
 

pkg/helm/loader_test.go

@@ -40,15 +40,15 @@ func TestLoadAndRenderChart(t *testing.T) {
 	opts := &LoadOptions{
 		ReleaseName:  "gtctl-ut",
 		Namespace:    "default",
-		ChartName:    artifacts.GreptimeDBChartName,
-		ChartVersion: artifacts.LatestVersionTag,
+		ChartName:    artifacts.GreptimeDBClusterChartName,
+		ChartVersion: "0.1.2",
 		FromCNRegion: false,
 		ValuesOptions: deployer.CreateGreptimeDBClusterOptions{
 			ImageRegistry:               "registry.cn-hangzhou.aliyuncs.com",
 			DatanodeStorageClassName:    "ebs-sc",
 			DatanodeStorageSize:         "11Gi",
 			DatanodeStorageRetainPolicy: "Delete",
-			EtcdEndPoint:                "127.0.0.1:2379",
+			EtcdEndPoints:               "mycluster-etcd.default:2379",
 			InitializerImageRegistry:    "registry.cn-hangzhou.aliyuncs.com",
 			ConfigValues:                "meta.replicas=3",
 		},

pkg/helm/testdata/db-manifests.yaml

@@ -1,5 +1,5 @@
 ---
-# Source: greptimedb/templates/cluster.yaml
+# Source: greptimedb-cluster/templates/cluster.yaml
 apiVersion: greptime.io/v1alpha1
 kind: GreptimeDBCluster
 metadata:
@@ -8,7 +8,7 @@ metadata:
 spec:
   base:
     main:
-      image: 'registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb:v0.4.0'
+      image: 'registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb:v0.4.1'
       resources:
         limits:
           cpu: 500m
@@ -21,19 +21,20 @@ spec:
   meta:
     replicas: 3
     etcdEndpoints:
-      - 127.0.0.1:2379
+      - mycluster-etcd.default:2379
   datanode:
     replicas: 3
     storage:
       storageClassName: ebs-sc
       storageSize: 11Gi
       storageRetainPolicy: Delete
+      walDir: /tmp/greptimedb/wal
   httpServicePort: 4000
   grpcServicePort: 4001
   mysqlServicePort: 4002
   postgresServicePort: 4003
   openTSDBServicePort: 4242
   initializer:
-    image: 'registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb-initializer:0.1.0-alpha.16'
+    image: 'registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb-initializer:0.1.0-alpha.17'
   storage:
     {}

pkg/metadata/manager_test.go

@@ -41,11 +41,11 @@ func TestMetadataManager(t *testing.T) {
 	}{
 		{
 			src: &artifacts.Source{
-				Name:    artifacts.GreptimeDBChartName,
+				Name:    artifacts.GreptimeDBClusterChartName,
 				Version: artifacts.LatestVersionTag,
 				Type:    artifacts.ArtifactTypeChart,
 			},
-			wantedDestDir: filepath.Join(tempDir, BaseDir, "artifacts", "charts", artifacts.GreptimeDBChartName, artifacts.LatestVersionTag, "pkg"),
+			wantedDestDir: filepath.Join(tempDir, BaseDir, "artifacts", "charts", artifacts.GreptimeDBClusterChartName, artifacts.LatestVersionTag, "pkg"),
 		},
 		{
 			src: &artifacts.Source{