diff --git a/pkg/artifacts/constants.go b/pkg/artifacts/constants.go index e315bd42..463f4676 100644 --- a/pkg/artifacts/constants.go +++ b/pkg/artifacts/constants.go @@ -21,14 +21,17 @@ const ( // GreptimeChartReleaseDownloadURL is the URL of the Greptime charts that stored in the GitHub release. GreptimeChartReleaseDownloadURL = "https://github.com/GreptimeTeam/helm-charts/releases/download" + // Greptime release bucket public endpoint in CN region. + GreptimeReleaseBucketCN = "https://downloads.greptime.cn/releases" + // GreptimeCNCharts is the URL of the Greptime charts that stored in the S3 bucket of the CN region. - GreptimeCNCharts = "https://downloads.greptime.cn/releases/charts" + GreptimeCNCharts = GreptimeReleaseBucketCN + "/charts" // GreptimeDBCNBinaries is the URL of the GreptimeDB binaries that stored in the S3 bucket of the CN region. - GreptimeDBCNBinaries = "https://downloads.greptime.cn/releases/greptimedb" + GreptimeDBCNBinaries = GreptimeReleaseBucketCN + "/greptimedb" // EtcdCNBinaries is the URL of the etcd binaries that stored in the S3 bucket of the CN region. - EtcdCNBinaries = "https://downloads.greptime.cn/releases/etcd" + EtcdCNBinaries = GreptimeReleaseBucketCN + "/etcd" // LatestVersionTag is the tag of the latest version. LatestVersionTag = "latest" @@ -54,8 +57,8 @@ const ( // EtcdBinName is the artifact name of etcd. EtcdBinName = "etcd" - // GreptimeDBChartName is the chart name of GreptimeDB. - GreptimeDBChartName = "greptimedb" + // GreptimeDBClusterChartName is the chart name of GreptimeDB. + GreptimeDBClusterChartName = "greptimedb-cluster" // GreptimeDBOperatorChartName is the chart name of GreptimeDB operator. GreptimeDBOperatorChartName = "greptimedb-operator" diff --git a/pkg/artifacts/manager.go b/pkg/artifacts/manager.go index aa4ac19c..859d17f5 100644 --- a/pkg/artifacts/manager.go +++ b/pkg/artifacts/manager.go @@ -119,32 +119,24 @@ func (m *manager) NewSource(name, version string, typ ArtifactType, fromCNRegion FromCNRegion: fromCNRegion, } + if version == LatestVersionTag || len(version) == 0 { + latestVersion, err := m.resolveLatestVersion(typ, name, fromCNRegion) + if err != nil { + return nil, err + } + src.Version = latestVersion + } + if src.Type == ArtifactTypeChart { src.FileName = m.chartFileName(src.Name, src.Version) if src.FromCNRegion { // The download URL example: 'https://downloads.greptime.cn/releases/charts/etcd/9.2.0/etcd-9.2.0.tgz'. - src.URL = fmt.Sprintf("%s/%s/%s/%s", GreptimeCNCharts, src.Name, version, src.FileName) + src.URL = fmt.Sprintf("%s/%s/%s/%s", GreptimeCNCharts, src.Name, src.Version, src.FileName) } else { // Specify the OCI registry URL for the etcd chart. if src.Name == EtcdChartName { // The download URL example: 'oci://registry-1.docker.io/bitnamicharts/etcd:9.2.0'. src.URL = EtcdOCIRegistry - return src, nil - } - - if src.Version == LatestVersionTag { - // Use chart index file to locate the latest chart version. - indexFile, err := m.chartIndexFile(context.TODO(), GreptimeChartIndexURL) - if err != nil { - return nil, err - } - - chartVersion, err := m.latestChartVersion(indexFile, src.Name) - if err != nil { - return nil, err - } - - src.URL = chartVersion.URLs[0] } else { // The download URL example: 'https://github.com/GreptimeTeam/helm-charts/releases/download/greptimedb-0.1.1-alpha.3/greptimedb-0.1.1-alpha.3.tgz'. src.URL = fmt.Sprintf("%s/%s/%s", GreptimeChartReleaseDownloadURL, strings.TrimSuffix(src.FileName, fileutils.TgzExtension), src.FileName) @@ -382,14 +374,6 @@ func (m *manager) latestGitHubReleaseVersion(org, repo string) (string, error) { } func (m *manager) etcdBinaryDownloadURL(version string, fromCNRegion bool) (string, error) { - if version != DefaultEtcdBinVersion && fromCNRegion { - return "", fmt.Errorf("only support %s in cn region", DefaultEtcdBinVersion) - } - - if version == LatestVersionTag { - return "", fmt.Errorf("can't support latest version") - } - var ext string switch runtime.GOOS { @@ -413,11 +397,6 @@ func (m *manager) etcdBinaryDownloadURL(version string, fromCNRegion bool) (stri } func (m *manager) greptimeBinaryDownloadURL(version string, fromCNRegion bool) (string, error) { - // FIXME(zyy17): we should support the latest version in the future. - if version == LatestVersionTag && fromCNRegion { - return "", fmt.Errorf("can't support latest version in cn region") - } - newVersion, err := isBreakingVersion(version) if err != nil { return "", err @@ -480,6 +459,80 @@ func (m *manager) installBinaries(downloadFile, installDir string) error { return nil } +// resolveLatestVersion resolves the latest tag to the specific version. +func (m *manager) resolveLatestVersion(typ ArtifactType, name string, fromCNRegion bool) (string, error) { + if fromCNRegion { + return m.getVersionInfoFromS3(typ, name, false) + } + + switch typ { + case ArtifactTypeChart: + // Use chart index file to locate the latest chart version. + indexFile, err := m.chartIndexFile(context.TODO(), GreptimeChartIndexURL) + if err != nil { + return "", err + } + + chartVersion, err := m.latestChartVersion(indexFile, name) + if err != nil { + return "", err + } + return chartVersion.Version, nil + case ArtifactTypeBinary: + // Get the latest version of the latest greptime binary. + latestVersion, err := m.latestGitHubReleaseVersion(GreptimeGitHubOrg, GreptimeDBGithubRepo) + if err != nil { + return "", err + } + return latestVersion, nil + default: + return "", fmt.Errorf("unsupported artifact type: %s", string(typ)) + } +} + +// getVersionInfoFromS3 gets the latest version info from S3. +func (m *manager) getVersionInfoFromS3(typ ArtifactType, name string, nightly bool) (string, error) { + // Note: it uses 'greptimedb' directory to store the greptime binary. + if name == GreptimeBinName { + name = "greptimedb" + } + var latestVersionInfoURL string + switch typ { + case ArtifactTypeChart: + latestVersionInfoURL = fmt.Sprintf("%s/charts/%s/latest-version.txt", GreptimeReleaseBucketCN, name) + case ArtifactTypeBinary: + if nightly { + latestVersionInfoURL = fmt.Sprintf("%s/%s/latest-nightly-version.txt", GreptimeReleaseBucketCN, name) + } else { + latestVersionInfoURL = fmt.Sprintf("%s/%s/latest-version.txt", GreptimeReleaseBucketCN, name) + } + default: + return "", fmt.Errorf("unsupported artifact type: %s", string(typ)) + } + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, latestVersionInfoURL, nil) + if err != nil { + return "", err + } + + httpClient := &http.Client{} + resp, err := httpClient.Do(req) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("get latest info from '%s' failed, status code: %d", latestVersionInfoURL, resp.StatusCode) + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return strings.TrimRight(string(data), "\n"), nil +} + // BreakingChangeVersion is the version that the download URL of the greptime binary is changed. const BreakingChangeVersion = "v0.4.0-nightly-20230802" diff --git a/pkg/artifacts/manager_test.go b/pkg/artifacts/manager_test.go index 1946e7f2..47126c93 100644 --- a/pkg/artifacts/manager_test.go +++ b/pkg/artifacts/manager_test.go @@ -46,9 +46,9 @@ func TestDownloadCharts(t *testing.T) { typ ArtifactType fromCNRegion bool }{ - {GreptimeDBChartName, "latest", ArtifactTypeChart, false}, + {GreptimeDBClusterChartName, "latest", ArtifactTypeChart, false}, {GreptimeDBOperatorChartName, "latest", ArtifactTypeChart, false}, - {GreptimeDBChartName, "0.1.1-alpha.13", ArtifactTypeChart, false}, + {GreptimeDBClusterChartName, "0.1.2", ArtifactTypeChart, false}, {GreptimeDBOperatorChartName, "0.1.1-alpha.12", ArtifactTypeChart, false}, {EtcdChartName, DefaultEtcdChartVersion, ArtifactTypeChart, false}, } @@ -92,9 +92,9 @@ func TestDownloadChartsFromCNRegion(t *testing.T) { typ ArtifactType fromCNRegion bool }{ - {GreptimeDBChartName, LatestVersionTag, ArtifactTypeChart, true}, + {GreptimeDBClusterChartName, LatestVersionTag, ArtifactTypeChart, true}, {GreptimeDBOperatorChartName, LatestVersionTag, ArtifactTypeChart, true}, - {GreptimeDBChartName, "0.1.1-alpha.13", ArtifactTypeChart, true}, + {GreptimeDBClusterChartName, "0.1.2", ArtifactTypeChart, true}, {GreptimeDBOperatorChartName, "0.1.1-alpha.12", ArtifactTypeChart, true}, {EtcdChartName, DefaultEtcdChartVersion, ArtifactTypeChart, true}, } @@ -225,7 +225,7 @@ func TestArtifactsCache(t *testing.T) { ctx := context.Background() - src, err := m.NewSource(GreptimeDBChartName, LatestVersionTag, ArtifactTypeChart, false) + src, err := m.NewSource(GreptimeDBClusterChartName, LatestVersionTag, ArtifactTypeChart, false) if err != nil { t.Errorf("failed to create source: %v", err) } diff --git a/pkg/cmd/gtctl/cluster/create/create.go b/pkg/cmd/gtctl/cluster/create/create.go index 4775f16f..9c5c0c5d 100644 --- a/pkg/cmd/gtctl/cluster/create/create.go +++ b/pkg/cmd/gtctl/cluster/create/create.go @@ -315,7 +315,7 @@ func deployGreptimeDBCluster(ctx context.Context, l logger.Logger, options *Clus DatanodeStorageClassName: options.StorageClassName, DatanodeStorageSize: options.StorageSize, DatanodeStorageRetainPolicy: options.StorageRetainPolicy, - EtcdEndPoint: fmt.Sprintf("%s.%s:2379", common.EtcdClusterName(clusterName), options.EtcdNamespace), + EtcdEndPoints: fmt.Sprintf("%s.%s:2379", common.EtcdClusterName(clusterName), options.EtcdNamespace), ConfigValues: options.Set.clusterConfig, UseGreptimeCNArtifacts: options.UseGreptimeCNArtifacts, ValuesFile: options.GreptimeDBClusterValuesFile, diff --git a/pkg/deployer/baremetal/deployer.go b/pkg/deployer/baremetal/deployer.go index 6af144d2..8ead9541 100644 --- a/pkg/deployer/baremetal/deployer.go +++ b/pkg/deployer/baremetal/deployer.go @@ -300,7 +300,7 @@ func (d *Deployer) CreateGreptimeDBCluster(ctx context.Context, clusterName stri if d.config.Cluster.Artifact.Local != "" { binPath = d.config.Cluster.Artifact.Local } else { - src, err := d.am.NewSource(artifacts.GreptimeBinName, d.config.Cluster.Artifact.Version, artifacts.ArtifactTypeBinary, false) + src, err := d.am.NewSource(artifacts.GreptimeBinName, d.config.Cluster.Artifact.Version, artifacts.ArtifactTypeBinary, options.UseGreptimeCNArtifacts) if err != nil { return err } @@ -379,7 +379,7 @@ func (d *Deployer) CreateEtcdCluster(ctx context.Context, clusterName string, op if d.config.Etcd.Artifact.Local != "" { binPath = d.config.Etcd.Artifact.Local } else { - src, err := d.am.NewSource(artifacts.EtcdBinName, d.config.Etcd.Artifact.Version, artifacts.ArtifactTypeBinary, false) + src, err := d.am.NewSource(artifacts.EtcdBinName, d.config.Etcd.Artifact.Version, artifacts.ArtifactTypeBinary, options.UseGreptimeCNArtifacts) if err != nil { return err } diff --git a/pkg/deployer/k8s/deployer.go b/pkg/deployer/k8s/deployer.go index 017189af..f60fe5a6 100644 --- a/pkg/deployer/k8s/deployer.go +++ b/pkg/deployer/k8s/deployer.go @@ -130,7 +130,7 @@ func (d *deployer) CreateGreptimeDBCluster(ctx context.Context, name string, opt opts := &helm.LoadOptions{ ReleaseName: resourceName, Namespace: resourceNamespace, - ChartName: artifacts.GreptimeDBChartName, + ChartName: artifacts.GreptimeDBClusterChartName, ChartVersion: options.GreptimeDBChartVersion, FromCNRegion: options.UseGreptimeCNArtifacts, ValuesOptions: *options, diff --git a/pkg/deployer/types.go b/pkg/deployer/types.go index 59a92c93..7dd7d21d 100644 --- a/pkg/deployer/types.go +++ b/pkg/deployer/types.go @@ -77,7 +77,7 @@ type CreateGreptimeDBClusterOptions struct { DatanodeStorageClassName string `helm:"datanode.storage.storageClassName"` DatanodeStorageSize string `helm:"datanode.storage.storageSize"` DatanodeStorageRetainPolicy string `helm:"datanode.storage.storageRetainPolicy"` - EtcdEndPoint string `helm:"etcdEndpoints"` + EtcdEndPoints string `helm:"meta.etcdEndpoints"` ConfigValues string `helm:"*"` } diff --git a/pkg/helm/loader_test.go b/pkg/helm/loader_test.go index 908b76f6..8a25e966 100644 --- a/pkg/helm/loader_test.go +++ b/pkg/helm/loader_test.go @@ -40,15 +40,15 @@ func TestLoadAndRenderChart(t *testing.T) { opts := &LoadOptions{ ReleaseName: "gtctl-ut", Namespace: "default", - ChartName: artifacts.GreptimeDBChartName, - ChartVersion: artifacts.LatestVersionTag, + ChartName: artifacts.GreptimeDBClusterChartName, + ChartVersion: "0.1.2", FromCNRegion: false, ValuesOptions: deployer.CreateGreptimeDBClusterOptions{ ImageRegistry: "registry.cn-hangzhou.aliyuncs.com", DatanodeStorageClassName: "ebs-sc", DatanodeStorageSize: "11Gi", DatanodeStorageRetainPolicy: "Delete", - EtcdEndPoint: "127.0.0.1:2379", + EtcdEndPoints: "mycluster-etcd.default:2379", InitializerImageRegistry: "registry.cn-hangzhou.aliyuncs.com", ConfigValues: "meta.replicas=3", }, diff --git a/pkg/helm/testdata/db-manifests.yaml b/pkg/helm/testdata/db-manifests.yaml index 50ea02d2..ebfc7b7a 100644 --- a/pkg/helm/testdata/db-manifests.yaml +++ b/pkg/helm/testdata/db-manifests.yaml @@ -1,5 +1,5 @@ --- -# Source: greptimedb/templates/cluster.yaml +# Source: greptimedb-cluster/templates/cluster.yaml apiVersion: greptime.io/v1alpha1 kind: GreptimeDBCluster metadata: @@ -8,7 +8,7 @@ metadata: spec: base: main: - image: 'registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb:v0.4.0' + image: 'registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb:v0.4.1' resources: limits: cpu: 500m @@ -21,19 +21,20 @@ spec: meta: replicas: 3 etcdEndpoints: - - 127.0.0.1:2379 + - mycluster-etcd.default:2379 datanode: replicas: 3 storage: storageClassName: ebs-sc storageSize: 11Gi storageRetainPolicy: Delete + walDir: /tmp/greptimedb/wal httpServicePort: 4000 grpcServicePort: 4001 mysqlServicePort: 4002 postgresServicePort: 4003 openTSDBServicePort: 4242 initializer: - image: 'registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb-initializer:0.1.0-alpha.16' + image: 'registry.cn-hangzhou.aliyuncs.com/greptime/greptimedb-initializer:0.1.0-alpha.17' storage: {} diff --git a/pkg/metadata/manager_test.go b/pkg/metadata/manager_test.go index 52ac9c9a..a2b1e32a 100644 --- a/pkg/metadata/manager_test.go +++ b/pkg/metadata/manager_test.go @@ -41,11 +41,11 @@ func TestMetadataManager(t *testing.T) { }{ { src: &artifacts.Source{ - Name: artifacts.GreptimeDBChartName, + Name: artifacts.GreptimeDBClusterChartName, Version: artifacts.LatestVersionTag, Type: artifacts.ArtifactTypeChart, }, - wantedDestDir: filepath.Join(tempDir, BaseDir, "artifacts", "charts", artifacts.GreptimeDBChartName, artifacts.LatestVersionTag, "pkg"), + wantedDestDir: filepath.Join(tempDir, BaseDir, "artifacts", "charts", artifacts.GreptimeDBClusterChartName, artifacts.LatestVersionTag, "pkg"), }, { src: &artifacts.Source{