diff --git a/pkg/artifacts/constants.go b/pkg/artifacts/constants.go new file mode 100644 index 00000000..aea03547 --- /dev/null +++ b/pkg/artifacts/constants.go @@ -0,0 +1,68 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package artifacts + +const ( + // GreptimeChartIndexURL is the URL of the Greptime chart index. + GreptimeChartIndexURL = "https://raw.githubusercontent.com/GreptimeTeam/helm-charts/gh-pages/index.yaml" + + // GreptimeChartReleaseDownloadURL is the URL of the Greptime charts that stored in the GitHub release. + GreptimeChartReleaseDownloadURL = "https://github.com/GreptimeTeam/helm-charts/releases/download" + + // GreptimeCNCharts is the URL of the Greptime charts that stored in the S3 bucket of the CN region. + GreptimeCNCharts = "https://downloads.greptime.cn/releases/charts" + + // GreptimeDBCNBinaries is the URL of the GreptimeDB binaries that stored in the S3 bucket of the CN region. + GreptimeDBCNBinaries = "https://downloads.greptime.cn/releases/greptimedb" + + // LatestVersionTag is the tag of the latest version. + LatestVersionTag = "latest" + + // EtcdOCIRegistry is the OCI registry of the etcd chart. + EtcdOCIRegistry = "oci://registry-1.docker.io/bitnamicharts/etcd" + + // GreptimeGitHubOrg is the GitHub organization of Greptime. + GreptimeGitHubOrg = "GreptimeTeam" + + // GreptimeDBGithubRepo is the GitHub repository of GreptimeDB. + GreptimeDBGithubRepo = "greptimedb" + + // EtcdGitHubOrg is the GitHub organization of etcd. + EtcdGitHubOrg = "etcd-io" + + // EtcdGithubRepo is the GitHub repository of etcd. + EtcdGithubRepo = "etcd" + + // GreptimeBinName is the artifact name of greptime. + GreptimeBinName = "greptime" + + // EtcdBinName is the artifact name of etcd. + EtcdBinName = "etcd" + + // GreptimeDBChartName is the chart name of GreptimeDB. + GreptimeDBChartName = "greptimedb" + + // GreptimeDBOperatorChartName is the chart name of GreptimeDB operator. + GreptimeDBOperatorChartName = "greptimedb-operator" + + // EtcdChartName is the chart name of etcd. + EtcdChartName = "etcd" + + // DefaultEtcdChartVersion is the default etcd chart version. + DefaultEtcdChartVersion = "9.2.0" + + // DefaultEtcdBinVersion is the default etcd binary version. + DefaultEtcdBinVersion = "v3.5.7" +) diff --git a/pkg/artifacts/manager.go b/pkg/artifacts/manager.go new file mode 100644 index 00000000..0a5beead --- /dev/null +++ b/pkg/artifacts/manager.go @@ -0,0 +1,466 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package artifacts + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "runtime" + "strings" + + "github.com/google/go-github/v53/github" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/registry" + "helm.sh/helm/v3/pkg/repo" + "sigs.k8s.io/yaml" + + "github.com/GreptimeTeam/gtctl/pkg/logger" + fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" + semverutils "github.com/GreptimeTeam/gtctl/pkg/utils/semver" +) + +// Manager is the interface for managing artifacts. +// For now, the artifacts can be helm charts and binaries. +type Manager interface { + // NewSource creates an artifact source with name, version, type and fromCNRegion. + NewSource(name, version string, typ ArtifactType, fromCNRegion bool) (*Source, error) + + // DownloadTo downloads the artifact from the source to the dest and returns the path of the artifact. + DownloadTo(ctx context.Context, from *Source, destDir string, opts *DownloadOptions) (string, error) +} + +// ArtifactType is the type of the artifact. +type ArtifactType string + +const ( + // ArtifactTypeChart indicates the artifact is a helm chart. + ArtifactTypeChart ArtifactType = "chart" + + // ArtifactTypeBinary indicates the artifact is a binary. + ArtifactTypeBinary ArtifactType = "binary" +) + +// Source is the source of the artifact. +type Source struct { + // The Name of the artifact. + Name string + + // The FileName of the artifact. + FileName string + + // The URL of the artifact. It can be the normal http/https URL or the OCI URL. + URL string + + // The Version of the artifact. + Version string + + // The type of the artifact. + Type ArtifactType + + // Indicates whether the artifact is from the CN region. + FromCNRegion bool +} + +// DownloadOptions is the options for downloading the artifact. +type DownloadOptions struct { + // If UseCache is true, the manager will use the cache if the artifact already exists. + UseCache bool +} + +// manager is the implementation of Manager interface. +type manager struct { + logger logger.Logger +} + +var _ Manager = &manager{} + +type Option func(*manager) + +// NewManager creates a new Manager with workingDir, logger and other options. +func NewManager(logger logger.Logger, opts ...Option) (Manager, error) { + m := &manager{ + logger: logger, + } + + for _, opt := range opts { + opt(m) + } + + return m, nil +} + +func (m *manager) NewSource(name, version string, typ ArtifactType, fromCNRegion bool) (*Source, error) { + src := &Source{ + Name: name, + Type: typ, + Version: version, + FromCNRegion: fromCNRegion, + } + + if src.Type == ArtifactTypeChart { + src.FileName = m.chartFileName(src.Name, src.Version) + if src.FromCNRegion { + // The download URL example: 'https://downloads.greptime.cn/releases/charts/etcd/9.2.0/etcd-9.2.0.tgz'. + src.URL = fmt.Sprintf("%s/%s/%s/%s", GreptimeCNCharts, src.Name, version, src.FileName) + } else { + // Specify the OCI registry URL for the etcd chart. + if src.Name == EtcdChartName { + // The download URL example: 'oci://registry-1.docker.io/bitnamicharts/etcd:9.2.0'. + src.URL = EtcdOCIRegistry + return src, nil + } + + if src.Version == LatestVersionTag { + // Use chart index file to locate the latest chart version. + indexFile, err := m.chartIndexFile(context.TODO(), GreptimeChartIndexURL) + if err != nil { + return nil, err + } + + chartVersion, err := m.latestChartVersion(indexFile, src.Name) + if err != nil { + return nil, err + } + + src.URL = chartVersion.URLs[0] + } else { + // The download URL example: 'https://github.com/GreptimeTeam/helm-charts/releases/download/greptimedb-0.1.1-alpha.3/greptimedb-0.1.1-alpha.3.tgz'. + src.URL = fmt.Sprintf("%s/%s/%s", GreptimeChartReleaseDownloadURL, strings.TrimSuffix(src.FileName, fileutils.TgzExtension), src.FileName) + } + } + } + + if src.Type == ArtifactTypeBinary { + if src.Name == EtcdBinName { + downloadURL, err := m.etcdBinaryDownloadURL(src.Version) + if err != nil { + return nil, err + } + src.URL = downloadURL + src.FileName = path.Base(src.URL) + } + + if src.Name == GreptimeBinName { + specificVersion := src.Version + if specificVersion == LatestVersionTag && !src.FromCNRegion { + // Get the latest version of the latest greptime binary. + latestVersion, err := m.latestGitHubReleaseVersion(GreptimeGitHubOrg, GreptimeDBGithubRepo) + if err != nil { + return nil, err + } + specificVersion = latestVersion + } + + downloadURL, err := m.greptimeBinaryDownloadURL(specificVersion) + if err != nil { + return nil, err + } + src.URL = downloadURL + src.FileName = path.Base(src.URL) + } + } + + return src, nil +} + +func (m *manager) DownloadTo(ctx context.Context, from *Source, destDir string, opts *DownloadOptions) (string, error) { + artifactFile := filepath.Join(destDir, from.FileName) + shouldDownload := true + if opts.UseCache { + _, err := os.Stat(artifactFile) + + // If the file exists, skip downloading. + if err == nil { + m.logger.V(3).Infof("The artifact file '%s' already exists, skip downloading.", artifactFile) + shouldDownload = false + } + + // Other error happened, return it. + if err != nil && !os.IsNotExist(err) { + return "", err + } + } + + if shouldDownload { + m.logger.V(3).Infof("Downloading artifact from '%s' to '%s'", from.URL, destDir) + + // Ensure the directories of the destDir exist. + if err := fileutils.EnsureDir(destDir); err != nil { + return "", err + } + + // Download the helm chart from OCI registry. + if registry.IsOCI(from.URL) && from.Type == ArtifactTypeChart { + if err := m.downloadFromOCI(from.URL, from.Version, destDir); err != nil { + return "", err + } + return artifactFile, nil + } + + if err := m.downloadFromHTTP(ctx, from.URL, artifactFile); err != nil { + return "", err + } + } + + if from.Type == ArtifactTypeBinary { + installDir := filepath.Join(filepath.Dir(destDir), "bin") + if err := m.installBinaries(artifactFile, installDir); err != nil { + return "", err + } + return filepath.Join(filepath.Dir(destDir), "bin", from.Name), nil + } + + return artifactFile, nil +} + +func (m *manager) downloadFromHTTP(ctx context.Context, httpURL string, dest string) error { + httpClient := &http.Client{} + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, httpURL, nil) + if err != nil { + return err + } + + resp, err := httpClient.Do(req) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download failed, status code: %d", resp.StatusCode) + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + file, err := os.Create(dest) + if err != nil { + return err + } + + _, err = file.Write(data) + if err != nil { + return err + } + + return nil +} + +func (m *manager) downloadFromOCI(registryURL, version, dest string) error { + registryClient, err := registry.NewClient( + registry.ClientOptDebug(false), + registry.ClientOptEnableCache(false), + registry.ClientOptCredentialsFile(""), + ) + if err != nil { + return err + } + + cfg := new(action.Configuration) + cfg.RegistryClient = registryClient + + // Create a pull action + client := action.NewPullWithOpts(action.WithConfig(cfg)) + client.Settings = cli.New() + client.Version = version + client.DestDir = dest + + m.logger.V(3).Infof("Pulling chart '%s', version: '%s' from OCI registry", registryURL, version) + + // Execute the pull action + if _, err := client.Run(registryURL); err != nil { + return err + } + + return nil +} + +// chartIndexFile returns the index file of the chart. We use the index file to get the specific version of the latest chart. +func (m *manager) chartIndexFile(ctx context.Context, indexURL string) (*repo.IndexFile, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, indexURL, nil) + if err != nil { + return nil, err + } + + rsp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer rsp.Body.Close() + + data, err := io.ReadAll(rsp.Body) + if err != nil { + return nil, err + } + + if len(data) == 0 { + return nil, repo.ErrEmptyIndexYaml + } + + indexFile := &repo.IndexFile{} + if err := yaml.UnmarshalStrict(data, &indexFile); err != nil { + return nil, err + } + + for _, cvs := range indexFile.Entries { + for idx := len(cvs) - 1; idx >= 0; idx-- { + if cvs[idx] == nil { + continue + } + if cvs[idx].APIVersion == "" { + cvs[idx].APIVersion = chart.APIVersionV1 + } + if err := cvs[idx].Validate(); err != nil { + cvs = append(cvs[:idx], cvs[idx+1:]...) + } + } + } + + indexFile.SortEntries() + if indexFile.APIVersion == "" { + return indexFile, repo.ErrNoAPIVersion + } + + return indexFile, nil +} + +// latestChartVersion returns the latest chart version. +func (m *manager) latestChartVersion(indexFile *repo.IndexFile, chartName string) (*repo.ChartVersion, error) { + if versions, ok := indexFile.Entries[chartName]; ok { + if versions.Len() > 0 { + // The Entries are already sorted by version so the position 0 always point to the latest version. + v := []*repo.ChartVersion(versions) + if len(v[0].URLs) == 0 { + return nil, fmt.Errorf("no download URLs found for %s-%s", chartName, v[0].Version) + } + return v[0], nil + } + return nil, fmt.Errorf("chart %s has empty versions", chartName) + } + + return nil, fmt.Errorf("chart %s not found", chartName) +} + +func (m *manager) chartFileName(chartName, version string) string { + return fmt.Sprintf("%s-%s.tgz", chartName, version) +} + +// latestGitHubReleaseVersion returns the latest GitHub release version. It's used to locate the latest version of the latest greptime binary. +func (m *manager) latestGitHubReleaseVersion(org, repo string) (string, error) { + client := github.NewClient(nil) + release, _, err := client.Repositories.GetLatestRelease(context.Background(), org, repo) + if err != nil { + return "", err + } + return *release.TagName, nil +} + +func (m *manager) etcdBinaryDownloadURL(version string) (string, error) { + var ext string + + switch runtime.GOOS { + case "darwin": + ext = fileutils.ZipExtension + case "linux": + ext = fileutils.TarGzExtension + default: + return "", fmt.Errorf("unsupported OS: %s", runtime.GOOS) + } + + // For the function stability, we always use the specific version of etcd. + downloadURL := fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/etcd-%s-%s-%s%s", + EtcdGitHubOrg, EtcdGithubRepo, version, version, runtime.GOOS, runtime.GOARCH, ext) + + return downloadURL, nil +} + +func (m *manager) greptimeBinaryDownloadURL(version string) (string, error) { + newVersion, err := isBreakingVersion(version) + if err != nil { + return "", err + } + + var packageName string + if newVersion { + packageName = fmt.Sprintf("greptime-%s-%s-%s.tar.gz", runtime.GOOS, runtime.GOARCH, version) + } else { + packageName = fmt.Sprintf("greptime-%s-%s.tgz", runtime.GOOS, runtime.GOARCH) + } + + return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", + GreptimeGitHubOrg, GreptimeDBGithubRepo, version, packageName), nil +} + +// installBinaries installs the binaries to the installDir. +func (m *manager) installBinaries(downloadFile, installDir string) error { + if err := fileutils.EnsureDir(installDir); err != nil { + return err + } + + tempDir, err := os.MkdirTemp("/tmp", "gtctl-") + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + + if err := fileutils.Uncompress(downloadFile, tempDir); err != nil { + return err + } + + m.logger.V(3).Infof("Installing binaries '%s' to '%s'", downloadFile, installDir) + + if err := filepath.Walk(tempDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.Mode().IsRegular() && (info.Mode()&0111 != 0) { // Move the executable file to the installDir. + newFilePath := filepath.Join(installDir, info.Name()) + if path != newFilePath { + if err := os.Rename(path, newFilePath); err != nil { + return err + } + } + } + + return nil + }); err != nil { + return err + } + + return nil +} + +// BreakingChangeVersion is the version that the download URL of the greptime binary is changed. +const BreakingChangeVersion = "v0.4.0-nightly-20230802" + +// TODO(zyy17): This function is just a temporary solution. We will remove it after the download URL of the greptime binary is stable. +func isBreakingVersion(version string) (bool, error) { + newVersion, err := semverutils.Compare(version, BreakingChangeVersion) + if err != nil { + return false, err + } + + return newVersion || version == BreakingChangeVersion, nil +} diff --git a/pkg/artifacts/manager_test.go b/pkg/artifacts/manager_test.go new file mode 100644 index 00000000..a8f54143 --- /dev/null +++ b/pkg/artifacts/manager_test.go @@ -0,0 +1,235 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package artifacts + +import ( + "context" + "fmt" + "os" + "testing" + + "sigs.k8s.io/kind/pkg/log" + + "github.com/GreptimeTeam/gtctl/pkg/logger" +) + +func TestDownloadCharts(t *testing.T) { + tempDir, err := os.MkdirTemp("/tmp", "gtctl-ut-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + m, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored())) + if err != nil { + t.Fatalf("failed to create artifacts manager: %v", err) + } + + ctx := context.Background() + + tests := []struct { + name string + version string + typ ArtifactType + fromCNRegion bool + }{ + {GreptimeDBChartName, "latest", ArtifactTypeChart, false}, + {GreptimeDBOperatorChartName, "latest", ArtifactTypeChart, false}, + {GreptimeDBChartName, "0.1.1-alpha.13", ArtifactTypeChart, false}, + {GreptimeDBOperatorChartName, "0.1.1-alpha.12", ArtifactTypeChart, false}, + {EtcdChartName, DefaultEtcdChartVersion, ArtifactTypeChart, false}, + } + for _, tt := range tests { + src, err := m.NewSource(tt.name, tt.version, tt.typ, tt.fromCNRegion) + if err != nil { + t.Errorf("failed to create source: %v", err) + } + artifactFile, err := m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{UseCache: false}) + if err != nil { + t.Errorf("failed to download: %v", err) + } + + _, err = os.Stat(artifactFile) + if os.IsNotExist(err) { + t.Errorf("artifact file does not exist: %v", err) + } + if err != nil { + t.Errorf("failed to stat artifact file: %v", err) + } + } +} + +func TestDownloadChartsFromCNRegion(t *testing.T) { + tempDir, err := os.MkdirTemp("/tmp", "gtctl-ut-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + m, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored())) + if err != nil { + t.Fatalf("failed to create artifacts manager: %v", err) + } + + ctx := context.Background() + + tests := []struct { + name string + version string + typ ArtifactType + fromCNRegion bool + }{ + {GreptimeDBChartName, LatestVersionTag, ArtifactTypeChart, true}, + {GreptimeDBOperatorChartName, LatestVersionTag, ArtifactTypeChart, true}, + {GreptimeDBChartName, "0.1.1-alpha.13", ArtifactTypeChart, true}, + {GreptimeDBOperatorChartName, "0.1.1-alpha.12", ArtifactTypeChart, true}, + {EtcdChartName, DefaultEtcdChartVersion, ArtifactTypeChart, true}, + } + for _, tt := range tests { + src, err := m.NewSource(tt.name, tt.version, tt.typ, tt.fromCNRegion) + if err != nil { + t.Errorf("failed to create source: %v", err) + } + artifactFile, err := m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{UseCache: false}) + if err != nil { + t.Errorf("failed to download: %v", err) + } + + _, err = os.Stat(artifactFile) + if os.IsNotExist(err) { + t.Errorf("artifact file does not exist: %v", err) + } + if err != nil { + t.Errorf("failed to stat artifact file: %v", err) + } + } +} + +func TestDownloadBinaries(t *testing.T) { + tempDir, err := os.MkdirTemp("/tmp", "gtctl-ut-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + m, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored())) + if err != nil { + t.Fatalf("failed to create artifacts manager: %v", err) + } + + ctx := context.Background() + + tests := []struct { + name string + version string + typ ArtifactType + fromCNRegion bool + }{ + {GreptimeBinName, LatestVersionTag, ArtifactTypeBinary, false}, + {GreptimeBinName, "v0.4.0-nightly-20231002", ArtifactTypeBinary, false}, + {EtcdBinName, DefaultEtcdBinVersion, ArtifactTypeBinary, false}, + } + for _, tt := range tests { + src, err := m.NewSource(tt.name, tt.version, tt.typ, tt.fromCNRegion) + if err != nil { + t.Errorf("failed to create source: %v", err) + } + artifactFile, err := m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{UseCache: false}) + if err != nil { + t.Errorf("failed to download: %v", err) + } + + info, err := os.Stat(artifactFile) + if os.IsNotExist(err) { + t.Errorf("artifact file does not exist: %v", err) + } + if info.Mode()&0111 == 0 { + t.Errorf("binary file is not executable") + } + if err != nil { + t.Errorf("failed to stat artifact file: %v", err) + } + } +} + +func TestArtifactsCache(t *testing.T) { + tempDir, err := os.MkdirTemp("/tmp", "gtctl-ut-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + m, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored())) + if err != nil { + t.Fatalf("failed to create artifacts manager: %v", err) + } + + ctx := context.Background() + + src, err := m.NewSource(GreptimeDBChartName, LatestVersionTag, ArtifactTypeChart, false) + if err != nil { + t.Errorf("failed to create source: %v", err) + } + artifactFile, err := m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{UseCache: false}) + if err != nil { + t.Errorf("failed to download: %v", err) + } + + firstTimeInfo, err := os.Stat(artifactFile) + if os.IsNotExist(err) { + t.Errorf("artifact file does not exist: %v", err) + } + if err != nil { + t.Errorf("failed to stat artifact file: %v", err) + } + + // Download again with cache. + artifactFile, err = m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{UseCache: true}) + if err != nil { + t.Errorf("failed to download: %v", err) + } + secondTimeInfo, err := os.Stat(artifactFile) + if os.IsNotExist(err) { + t.Errorf("artifact file does not exist: %v", err) + } + if err != nil { + t.Errorf("failed to stat artifact file: %v", err) + } + if os.IsNotExist(err) { + t.Errorf("artifact file does not exist: %v", err) + } + if err != nil { + t.Errorf("failed to stat artifact file: %v", err) + } + + if firstTimeInfo.ModTime() != secondTimeInfo.ModTime() { + t.Errorf("artifact file is not cached") + } +} + +func destDir(workingDir string, src *Source) string { + var artifactsDir string + + switch src.Type { + case ArtifactTypeBinary: + artifactsDir = "binaries" + case ArtifactTypeChart: + artifactsDir = "charts" + default: + panic(fmt.Sprintf("unknown artifact type: %s", src.Type)) + } + + return fmt.Sprintf("%s/artifacts/%s/%s/%s/pkg", workingDir, artifactsDir, src.Name, src.Version) +} diff --git a/pkg/deployer/baremetal/artifacts.go b/pkg/deployer/baremetal/artifacts.go deleted file mode 100644 index bc4fc77c..00000000 --- a/pkg/deployer/baremetal/artifacts.go +++ /dev/null @@ -1,343 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package baremetal - -import ( - "context" - "fmt" - "io" - "net/http" - "os" - "path" - "path/filepath" - "runtime" - "strings" - - "github.com/google/go-github/v53/github" - - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" - "github.com/GreptimeTeam/gtctl/pkg/logger" - fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" - semverutils "github.com/GreptimeTeam/gtctl/pkg/utils/semver" -) - -const ( - GreptimeGitHubOrg = "GreptimeTeam" - GreptimeDBGithubRepo = "greptimedb" - GreptimeBinName = "greptime" - - EtcdGitHubOrg = "etcd-io" - EtcdGithubRepo = "etcd" - - GOOSDarwin = "darwin" - GOOSLinux = "linux" - - BreakingChangeVersion = "v0.4.0-nightly-20230802" -) - -// ArtifactManager is responsible for managing the artifacts of a GreptimeDB cluster. -type ArtifactManager struct { - // dir is the global directory that contains all the artifacts. - dir string - - // If alwaysDownload is false, the manager will not download the artifact if it already exists. - alwaysDownload bool - - logger logger.Logger -} - -type ArtifactType string - -const ( - GreptimeArtifactType ArtifactType = "greptime" - EtcdArtifactType ArtifactType = "etcd" -) - -func (t ArtifactType) String() string { - return string(t) -} - -func NewArtifactManager(workingDir string, l logger.Logger, alwaysDownload bool) (*ArtifactManager, error) { - dir := path.Join(workingDir, "artifacts") - if err := fileutils.CreateDirIfNotExists(dir); err != nil { - return nil, err - } - - return &ArtifactManager{dir: dir, alwaysDownload: alwaysDownload, logger: l}, nil -} - -// BinaryPath returns the path of the binary of the given type and version. -func (am *ArtifactManager) BinaryPath(typ ArtifactType, artifact *config.Artifact) (string, error) { - if artifact.Local != "" { - return artifact.Local, nil - } - - bin := path.Join(am.dir, typ.String(), artifact.Version, "bin", typ.String()) - if _, err := os.Stat(bin); os.IsNotExist(err) { - return "", fmt.Errorf("binary not found: %s", bin) - } - return bin, nil -} - -// PrepareArtifact will download the artifact from the given URL and uncompressed it. -func (am *ArtifactManager) PrepareArtifact(ctx context.Context, typ ArtifactType, artifact *config.Artifact) error { - // If you use the local artifact, we don't need to download it. - if artifact.Local != "" { - return nil - } - - var ( - version = artifact.Version - pkgDir = path.Join(am.dir, typ.String(), version, "pkg") - binDir = path.Join(am.dir, typ.String(), version, "bin") - ) - - artifactFile, err := am.download(ctx, typ, version, pkgDir) - if err != nil { - return err - } - - // Normalize the directory structure. - // The directory of artifacts looks like('tree -L 5 ~/.gtctl | sed 's/\xc2\xa0/ /g'): - // ${HOME}/.gtctl - // └── artifacts - // ├── etcd - // │ └── v3.5.7 - // │ ├── bin - // │ │ ├── etcd - // │ │ ├── etcdctl - // │ │ └── etcdutl - // │ └── pkg - // │ ├── etcd-v3.5.7-darwin-arm64 - // │ └── etcd-v3.5.7-darwin-arm64.zip - // └── greptime - // ├── latest - // │ ├── bin - // │ │ └── greptime - // │ └── pkg - // │ └── greptime-darwin-arm64.tgz - // └── v0.1.2 - // ├── bin - // │ └── greptime - // └── pkg - // └── greptime-darwin-arm64.tgz - switch typ { - case GreptimeArtifactType: - return am.installGreptime(artifactFile, binDir, version) - case EtcdArtifactType: - return am.installEtcd(artifactFile, pkgDir, binDir) - default: - return fmt.Errorf("unsupported artifact type: %s", typ) - } -} - -func (am *ArtifactManager) installEtcd(artifactFile, pkgDir, binDir string) error { - if err := fileutils.Uncompress(artifactFile, pkgDir); err != nil { - return err - } - - if err := fileutils.CreateDirIfNotExists(binDir); err != nil { - return err - } - - artifactFile = path.Base(artifactFile) - // If the artifactFile is '${pkgDir}/etcd-v3.5.7-darwin-arm64.zip', it will get '${pkgDir}/etcd-v3.5.7-darwin-arm64'. - uncompressedDir := path.Join(pkgDir, artifactFile[:len(artifactFile)-len(filepath.Ext(artifactFile))]) - uncompressedDir = strings.TrimSuffix(uncompressedDir, fileutils.TarExtension) - binaries := []string{"etcd", "etcdctl", "etcdutl"} - for _, binary := range binaries { - if err := fileutils.CopyFile(path.Join(uncompressedDir, binary), path.Join(binDir, binary)); err != nil { - return err - } - if err := os.Chmod(path.Join(binDir, binary), 0755); err != nil { - return err - } - } - return nil -} - -func (am *ArtifactManager) installGreptime(artifactFile, binDir, version string) error { - if err := fileutils.CreateDirIfNotExists(binDir); err != nil { - return err - } - - if err := fileutils.Uncompress(artifactFile, binDir); err != nil { - return err - } - - newVersion, err := am.isBreakingVersion(version) - if err != nil { - return err - } - - // If it's the breaking version, adapt to the new directory layout. - if newVersion { - originalBinDir := path.Join(binDir, strings.TrimSuffix(path.Base(artifactFile), fileutils.TarGzExtension)) - if err := os.Rename(path.Join(originalBinDir, GreptimeBinName), path.Join(binDir, GreptimeBinName)); err != nil { - return err - } - if err := os.Remove(originalBinDir); err != nil { - return err - } - } - - if err := os.Chmod(path.Join(binDir, GreptimeBinName), 0755); err != nil { - return err - } - - return nil -} - -func (am *ArtifactManager) download(ctx context.Context, typ ArtifactType, version, pkgDir string) (string, error) { - downloadURL, err := am.artifactURL(typ, version) - if err != nil { - return "", err - } - - if err := fileutils.CreateDirIfNotExists(pkgDir); err != nil { - return "", err - } - - artifactFile := path.Join(pkgDir, path.Base(downloadURL)) - if !am.alwaysDownload { - // The artifact file already exists, skip downloading. - if _, err := os.Stat(artifactFile); err == nil { - am.logger.V(3).Infof("The artifact file '%s' already exists, skip downloading.", artifactFile) - return artifactFile, nil - } - - // Other error happened, return it. - if err != nil && !os.IsNotExist(err) { - return "", err - } - } - - httpClient := &http.Client{} - - am.logger.V(3).Infof("Downloading artifact from '%s' to '%s'", downloadURL, artifactFile) - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) - if err != nil { - return "", err - } - resp, err := httpClient.Do(req) - if err != nil { - return "", err - } - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("download failed, status code: %d", resp.StatusCode) - } - defer resp.Body.Close() - - data, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - - file, err := os.Create(artifactFile) - if err != nil { - return "", err - } - - _, err = file.Write(data) - if err != nil { - return "", err - } - - return artifactFile, nil -} - -func (am *ArtifactManager) artifactURL(typ ArtifactType, version string) (string, error) { - switch typ { - case GreptimeArtifactType: - return am.greptimeDownloadURL(version) - case EtcdArtifactType: - return am.etcdDownloadURL(version) - default: - return "", fmt.Errorf("unsupported artifact type: %v", typ) - } -} - -func (am *ArtifactManager) getGreptimeLatestVersion() (string, error) { - client := github.NewClient(nil) - release, _, err := client.Repositories.GetLatestRelease(context.Background(), GreptimeGitHubOrg, GreptimeDBGithubRepo) - if err != nil { - return "", err - } - return *release.TagName, nil -} - -func (am *ArtifactManager) greptimeDownloadURL(version string) (string, error) { - if version == "latest" { - // Get the latest greptime released version. - latestVersion, err := am.getGreptimeLatestVersion() - if err != nil { - return "", err - } - version = latestVersion - } - - newVersion, err := am.isBreakingVersion(version) - if err != nil { - return "", err - } - - // If version >= BreakingChangeVersion, use the new download URL. - if newVersion { - return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s-%s-%s-%s.tar.gz", - GreptimeGitHubOrg, GreptimeDBGithubRepo, version, string(GreptimeArtifactType), runtime.GOOS, runtime.GOARCH, version), nil - } - - // If version < BreakingChangeVersion, use the old download URL. - return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s-%s-%s.tgz", - GreptimeGitHubOrg, GreptimeDBGithubRepo, version, string(GreptimeArtifactType), runtime.GOOS, runtime.GOARCH), nil -} - -func (am *ArtifactManager) etcdDownloadURL(version string) (string, error) { - var ext string - - switch runtime.GOOS { - case GOOSDarwin: - ext = fileutils.ZipExtension - case GOOSLinux: - ext = fileutils.TarGzExtension - default: - return "", fmt.Errorf("unsupported OS: %s", runtime.GOOS) - } - - // For the function stability, we use the specific version of etcd. - downloadURL := fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s-%s-%s-%s%s", - EtcdGitHubOrg, EtcdGithubRepo, version, string(EtcdArtifactType), version, runtime.GOOS, runtime.GOARCH, ext) - - return downloadURL, nil -} - -func (am *ArtifactManager) isBreakingVersion(version string) (bool, error) { - if version == "latest" { - // Get the latest greptime released version. - latestVersion, err := am.getGreptimeLatestVersion() - if err != nil { - return false, err - } - version = latestVersion - } - - newVersion, err := semverutils.Compare(version, BreakingChangeVersion) - if err != nil { - return false, err - } - - return newVersion || version == BreakingChangeVersion, nil -} diff --git a/pkg/deployer/baremetal/artifacts_test.go b/pkg/deployer/baremetal/artifacts_test.go deleted file mode 100644 index 316ef36a..00000000 --- a/pkg/deployer/baremetal/artifacts_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package baremetal - -import ( - "context" - "os" - "testing" - - "sigs.k8s.io/kind/pkg/log" - - "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" - "github.com/GreptimeTeam/gtctl/pkg/logger" -) - -const ( - testDir = "/tmp/gtctl-test-am" -) - -func TestArtifactManager(t *testing.T) { - am, err := NewArtifactManager(testDir, logger.New(os.Stdout, log.Level(4), logger.WithColored()), false) - if err != nil { - t.Errorf("failed to create artifact manager: %v", err) - } - - // Cleanup test directory. - defer func() { - os.RemoveAll(testDir) - }() - - testConfigs := []*config.Artifact{ - { - Version: "latest", - }, - { - Version: BreakingChangeVersion, - }, - } - - ctx := context.Background() - for _, tc := range testConfigs { - if err := am.PrepareArtifact(ctx, GreptimeArtifactType, tc); err != nil { - t.Errorf("failed to prepare artifact: %v", err) - } - - _, err := am.BinaryPath(GreptimeArtifactType, tc) - if err != nil { - t.Errorf("failed to get binary path: %v", err) - } - } -} diff --git a/pkg/deployer/baremetal/component/datanode.go b/pkg/deployer/baremetal/component/datanode.go index 0a169cfa..1e3ff619 100644 --- a/pkg/deployer/baremetal/component/datanode.go +++ b/pkg/deployer/baremetal/component/datanode.go @@ -61,25 +61,25 @@ func (d *datanode) Start(ctx context.Context, binary string) error { dirName := fmt.Sprintf("%s.%d", d.Name(), i) dataHomeDir := path.Join(d.workingDirs.DataDir, config.DataHomeDir) - if err := fileutils.CreateDirIfNotExists(dataHomeDir); err != nil { + if err := fileutils.EnsureDir(dataHomeDir); err != nil { return err } d.dataHomeDirs = append(d.dataHomeDirs, dataHomeDir) datanodeLogDir := path.Join(d.workingDirs.LogsDir, dirName) - if err := fileutils.CreateDirIfNotExists(datanodeLogDir); err != nil { + if err := fileutils.EnsureDir(datanodeLogDir); err != nil { return err } d.logsDirs = append(d.logsDirs, datanodeLogDir) datanodePidDir := path.Join(d.workingDirs.PidsDir, dirName) - if err := fileutils.CreateDirIfNotExists(datanodePidDir); err != nil { + if err := fileutils.EnsureDir(datanodePidDir); err != nil { return err } d.pidsDirs = append(d.pidsDirs, datanodePidDir) walDir := path.Join(d.workingDirs.DataDir, dirName, config.DataWalDir) - if err := fileutils.CreateDirIfNotExists(walDir); err != nil { + if err := fileutils.EnsureDir(walDir); err != nil { return err } d.dataDirs = append(d.dataDirs, path.Join(d.workingDirs.DataDir, dirName)) diff --git a/pkg/deployer/baremetal/component/etcd.go b/pkg/deployer/baremetal/component/etcd.go index 803a3022..2e122f9e 100644 --- a/pkg/deployer/baremetal/component/etcd.go +++ b/pkg/deployer/baremetal/component/etcd.go @@ -51,7 +51,7 @@ func (e *etcd) Start(ctx context.Context, binary string) error { etcdDirs = []string{etcdDataDir, etcdLogDir, etcdPidDir} ) for _, dir := range etcdDirs { - if err := fileutils.CreateDirIfNotExists(dir); err != nil { + if err := fileutils.EnsureDir(dir); err != nil { return err } } diff --git a/pkg/deployer/baremetal/component/frontend.go b/pkg/deployer/baremetal/component/frontend.go index 623ced5f..80956ae2 100644 --- a/pkg/deployer/baremetal/component/frontend.go +++ b/pkg/deployer/baremetal/component/frontend.go @@ -56,13 +56,13 @@ func (f *frontend) Start(ctx context.Context, binary string) error { dirName := fmt.Sprintf("%s.%d", f.Name(), i) frontendLogDir := path.Join(f.workingDirs.LogsDir, dirName) - if err := fileutils.CreateDirIfNotExists(frontendLogDir); err != nil { + if err := fileutils.EnsureDir(frontendLogDir); err != nil { return err } f.logsDirs = append(f.logsDirs, frontendLogDir) frontendPidDir := path.Join(f.workingDirs.PidsDir, dirName) - if err := fileutils.CreateDirIfNotExists(frontendPidDir); err != nil { + if err := fileutils.EnsureDir(frontendPidDir); err != nil { return err } f.pidsDirs = append(f.pidsDirs, frontendPidDir) diff --git a/pkg/deployer/baremetal/component/metasrv.go b/pkg/deployer/baremetal/component/metasrv.go index 61e05786..e29b62d9 100644 --- a/pkg/deployer/baremetal/component/metasrv.go +++ b/pkg/deployer/baremetal/component/metasrv.go @@ -64,13 +64,13 @@ func (m *metaSrv) Start(ctx context.Context, binary string) error { dirName := fmt.Sprintf("%s.%d", m.Name(), i) metaSrvLogDir := path.Join(m.workingDirs.LogsDir, dirName) - if err := fileutils.CreateDirIfNotExists(metaSrvLogDir); err != nil { + if err := fileutils.EnsureDir(metaSrvLogDir); err != nil { return err } m.logsDirs = append(m.logsDirs, metaSrvLogDir) metaSrvPidDir := path.Join(m.workingDirs.PidsDir, dirName) - if err := fileutils.CreateDirIfNotExists(metaSrvPidDir); err != nil { + if err := fileutils.EnsureDir(metaSrvPidDir); err != nil { return err } m.pidsDirs = append(m.pidsDirs, metaSrvPidDir) diff --git a/pkg/deployer/baremetal/config/common.go b/pkg/deployer/baremetal/config/common.go index 43642e83..3c5a13f1 100644 --- a/pkg/deployer/baremetal/config/common.go +++ b/pkg/deployer/baremetal/config/common.go @@ -16,6 +16,8 @@ package config import ( "time" + + "github.com/GreptimeTeam/gtctl/pkg/artifacts" ) const ( @@ -28,9 +30,6 @@ const ( DataHomeDir = "home" DataWalDir = "wal" - DefaultEtcdVersion = "v3.5.7" - DefaultGreptimeVersion = "latest" - DefaultLogLevel = "info" ) @@ -74,7 +73,7 @@ func DefaultConfig() *Config { return &Config{ Cluster: &Cluster{ Artifact: &Artifact{ - Version: DefaultGreptimeVersion, + Version: artifacts.LatestVersionTag, }, Frontend: &Frontend{ Replicas: 1, @@ -93,7 +92,7 @@ func DefaultConfig() *Config { }, Etcd: &Etcd{ Artifact: &Artifact{ - Version: DefaultEtcdVersion, + Version: artifacts.DefaultEtcdBinVersion, }, }, } diff --git a/pkg/deployer/baremetal/deployer.go b/pkg/deployer/baremetal/deployer.go index 4ffc14a5..b17bfad1 100644 --- a/pkg/deployer/baremetal/deployer.go +++ b/pkg/deployer/baremetal/deployer.go @@ -21,6 +21,7 @@ import ( "os/exec" "os/signal" "path" + "path/filepath" "strings" "sync" "syscall" @@ -28,6 +29,7 @@ import ( "gopkg.in/yaml.v3" + "github.com/GreptimeTeam/gtctl/pkg/artifacts" . "github.com/GreptimeTeam/gtctl/pkg/deployer" "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/component" "github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config" @@ -38,7 +40,7 @@ import ( type Deployer struct { logger logger.Logger config *config.Config - am *ArtifactManager + am artifacts.Manager wg sync.WaitGroup bm *component.BareMetalCluster ctx context.Context @@ -83,11 +85,11 @@ func NewDeployer(l logger.Logger, clusterName string, opts ...Option) (Interface d.baseDir = path.Join(homeDir, config.GtctlDir) } - if err := fileutils.CreateDirIfNotExists(d.baseDir); err != nil { + if err := fileutils.EnsureDir(d.baseDir); err != nil { return nil, err } - am, err := NewArtifactManager(d.baseDir, l, d.alwaysDownload) + am, err := artifacts.NewManager(l) if err != nil { return nil, err } @@ -151,7 +153,7 @@ func (d *Deployer) createClusterDirs() error { } for _, dir := range dirs { - if err := fileutils.CreateDirIfNotExists(dir); err != nil { + if err := fileutils.EnsureDir(dir); err != nil { return err } } @@ -285,24 +287,34 @@ func (d *Deployer) ListGreptimeDBClusters(ctx context.Context, options *ListGrep } func (d *Deployer) CreateGreptimeDBCluster(ctx context.Context, clusterName string, options *CreateGreptimeDBClusterOptions) error { - if err := d.am.PrepareArtifact(ctx, GreptimeArtifactType, d.config.Cluster.Artifact); err != nil { - return err - } + var binPath string + if d.config.Cluster.Artifact != nil { + if d.config.Cluster.Artifact.Local != "" { + binPath = d.config.Cluster.Artifact.Local + } else { + src, err := d.am.NewSource(artifacts.GreptimeBinName, d.config.Cluster.Artifact.Version, artifacts.ArtifactTypeBinary, false) + if err != nil { + return err + } - binary, err := d.am.BinaryPath(GreptimeArtifactType, d.config.Cluster.Artifact) - if err != nil { - return err + destDir := filepath.Join(d.baseDir, "artifacts", "binaries", artifacts.GreptimeBinName, d.config.Cluster.Artifact.Version, "pkg") + artifactFile, err := d.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{UseCache: true}) + if err != nil { + return err + } + binPath = artifactFile + } } - if err := d.bm.MetaSrv.Start(d.ctx, binary); err != nil { + if err := d.bm.MetaSrv.Start(d.ctx, binPath); err != nil { return err } - if err := d.bm.Datanode.Start(d.ctx, binary); err != nil { + if err := d.bm.Datanode.Start(d.ctx, binPath); err != nil { return err } - if err := d.bm.Frontend.Start(d.ctx, binary); err != nil { + if err := d.bm.Frontend.Start(d.ctx, binPath); err != nil { return err } @@ -345,20 +357,30 @@ func (d *Deployer) deleteGreptimeDBClusterForeground(ctx context.Context, option } func (d *Deployer) CreateEtcdCluster(ctx context.Context, clusterName string, options *CreateEtcdClusterOptions) error { - if err := d.am.PrepareArtifact(ctx, EtcdArtifactType, d.config.Etcd.Artifact); err != nil { - return err - } + var binPath string + if d.config.Etcd.Artifact != nil { + if d.config.Etcd.Artifact.Local != "" { + binPath = d.config.Etcd.Artifact.Local + } else { + src, err := d.am.NewSource(artifacts.EtcdBinName, d.config.Etcd.Artifact.Version, artifacts.ArtifactTypeBinary, false) + if err != nil { + return err + } - bin, err := d.am.BinaryPath(EtcdArtifactType, d.config.Etcd.Artifact) - if err != nil { - return err + destDir := filepath.Join(d.baseDir, "artifacts", "binaries", artifacts.EtcdBinName, d.config.Etcd.Artifact.Version, "pkg") + artifactFile, err := d.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{UseCache: true}) + if err != nil { + return err + } + binPath = artifactFile + } } - if err = d.bm.Etcd.Start(d.ctx, bin); err != nil { + if err := d.bm.Etcd.Start(d.ctx, binPath); err != nil { return err } - if err := d.checkEtcdHealth(bin); err != nil { + if err := d.checkEtcdHealth(binPath); err != nil { return err } diff --git a/pkg/deployer/k8s/deployer.go b/pkg/deployer/k8s/deployer.go index 17710646..22431490 100644 --- a/pkg/deployer/k8s/deployer.go +++ b/pkg/deployer/k8s/deployer.go @@ -22,6 +22,7 @@ import ( greptimedbclusterv1alpha1 "github.com/GreptimeTeam/greptimedb-operator/apis/v1alpha1" + "github.com/GreptimeTeam/gtctl/pkg/artifacts" . "github.com/GreptimeTeam/gtctl/pkg/deployer" "github.com/GreptimeTeam/gtctl/pkg/helm" "github.com/GreptimeTeam/gtctl/pkg/kube" @@ -126,7 +127,7 @@ func (d *deployer) CreateGreptimeDBCluster(ctx context.Context, name string, opt options.ConfigValues += fmt.Sprintf("image.registry=%s,initializer.registry=%s,", AliCloudRegistry, AliCloudRegistry) } - manifests, err := d.helmManager.LoadAndRenderChart(ctx, resourceName, resourceNamespace, helm.GreptimeDBChartName, options.GreptimeDBChartVersion, options.UseGreptimeCNArtifacts, *options) + manifests, err := d.helmManager.LoadAndRenderChart(ctx, resourceName, resourceNamespace, artifacts.GreptimeDBChartName, options.GreptimeDBChartVersion, options.UseGreptimeCNArtifacts, *options) if err != nil { return err } @@ -185,7 +186,7 @@ func (d *deployer) CreateEtcdCluster(ctx context.Context, name string, options * options.ConfigValues += fmt.Sprintf("image.registry=%s,", AliCloudRegistry) } - manifests, err := d.helmManager.LoadAndRenderChart(ctx, resourceName, resourceNamespace, helm.EtcdBitnamiOCIRegistry, helm.DefaultEtcdChartVersion, options.UseGreptimeCNArtifacts, *options) + manifests, err := d.helmManager.LoadAndRenderChart(ctx, resourceName, resourceNamespace, artifacts.EtcdChartName, artifacts.DefaultEtcdChartVersion, options.UseGreptimeCNArtifacts, *options) if err != nil { return fmt.Errorf("error while loading helm chart: %v", err) } @@ -221,7 +222,7 @@ func (d *deployer) CreateGreptimeDBOperator(ctx context.Context, name string, op options.ConfigValues += fmt.Sprintf("image.registry=%s,", AliCloudRegistry) } - manifests, err := d.helmManager.LoadAndRenderChart(ctx, resourceName, resourceNamespace, helm.GreptimeDBOperatorChartName, options.GreptimeDBOperatorChartVersion, options.UseGreptimeCNArtifacts, *options) + manifests, err := d.helmManager.LoadAndRenderChart(ctx, resourceName, resourceNamespace, artifacts.GreptimeDBOperatorChartName, options.GreptimeDBOperatorChartVersion, options.UseGreptimeCNArtifacts, *options) if err != nil { return err } diff --git a/pkg/helm/constants.go b/pkg/helm/constants.go deleted file mode 100644 index a4b676b3..00000000 --- a/pkg/helm/constants.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package helm - -const ( - DefaultChartsCache = ".gtctl/charts-cache" - - GreptimeChartIndexURL = "https://raw.githubusercontent.com/GreptimeTeam/helm-charts/gh-pages/index.yaml" - GreptimeChartReleaseDownloadURL = "https://github.com/GreptimeTeam/helm-charts/releases/download" - GreptimeCNCharts = "https://downloads.greptime.cn/releases/charts" - - GreptimeDBChartName = "greptimedb" - GreptimeDBOperatorChartName = "greptimedb-operator" - EtcdBitnamiOCIRegistry = "oci://registry-1.docker.io/bitnamicharts/etcd" - DefaultEtcdChartVersion = "9.2.0" -) diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go index 7d557a2d..f33b244b 100644 --- a/pkg/helm/helm.go +++ b/pkg/helm/helm.go @@ -18,12 +18,7 @@ import ( "bytes" "context" "fmt" - "io" - "log" - "net/http" - "net/url" "os" - "path" "path/filepath" "reflect" "strings" @@ -32,14 +27,10 @@ import ( "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" - "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/registry" - . "helm.sh/helm/v3/pkg/repo" "helm.sh/helm/v3/pkg/strvals" - "sigs.k8s.io/yaml" + "github.com/GreptimeTeam/gtctl/pkg/artifacts" "github.com/GreptimeTeam/gtctl/pkg/logger" - fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file" ) const ( @@ -56,42 +47,44 @@ var ( // 1. Load the chart from remote charts and save them in cache directory. // 2. Generate the manifests from the chart with the values. type Manager struct { - // indexFile is the index file of the remote charts. - indexFile *IndexFile - - // chartCache is the cache directory for the charts. - chartsCacheDir string - // logger is the logger for the Manager. logger logger.Logger + + // artifactsManager is the artifacts manager to manage charts. + am artifacts.Manager + + // metadataDir is the directory to store the metadata of the gtctl. + metadataDir string } type Option func(*Manager) func NewManager(l logger.Logger, opts ...Option) (*Manager, error) { r := &Manager{logger: l} - for _, opt := range opts { - opt(r) - } - if r.chartsCacheDir == "" { - homeDir, err := os.UserHomeDir() - if err != nil { - return nil, err - } - r.chartsCacheDir = filepath.Join(homeDir, DefaultChartsCache) + am, err := artifacts.NewManager(l) + if err != nil { + return nil, err } + r.am = am - if err := fileutils.CreateDirIfNotExists(r.chartsCacheDir); err != nil { + // TODO(zyy17): The metadataDir will be managed by the independent manager in the future. + homeDir, err := os.UserHomeDir() + if err != nil { return nil, err } + r.metadataDir = filepath.Join(homeDir, ".gtctl") + + for _, opt := range opts { + opt(r) + } return r, nil } -func WithChartsCacheDir(chartsCacheDir string) func(*Manager) { +func WithMetadataDir(dir string) Option { return func(r *Manager) { - r.chartsCacheDir = chartsCacheDir + r.metadataDir = dir } } @@ -103,78 +96,37 @@ func (r *Manager) LoadAndRenderChart(ctx context.Context, name, namespace, chart } r.logger.V(3).Infof("create '%s' with values: %v", name, values) - var helmChart *chart.Chart - if isOCIChar(chartName) { - helmChart, err = r.pullFromOCIRegistry(chartName, chartVersion) - if err != nil { - return nil, err - } - } else { - downloadURL, err := r.getChartDownloadURL(ctx, chartName, chartVersion, useGreptimeCNArtifacts) - if err != nil { - return nil, err - } - - alwaysDownload := false - if chartVersion == "" { // always download the latest version. - alwaysDownload = true - } - - helmChart, err = r.loadChartFromRemoteCharts(ctx, downloadURL, alwaysDownload) - if err != nil { - return nil, err - } + if chartVersion == "" { + chartVersion = artifacts.LatestVersionTag } - manifests, err := r.generateManifests(ctx, name, namespace, helmChart, values) + src, err := r.am.NewSource(chartName, chartVersion, artifacts.ArtifactTypeChart, useGreptimeCNArtifacts) if err != nil { return nil, err } - r.logger.V(3).Infof("create '%s' with manifests: %s", name, string(manifests)) - - return manifests, nil -} -func (r *Manager) loadChartFromRemoteCharts(ctx context.Context, downloadURL string, alwaysDownload bool) (*chart.Chart, error) { - parsedURL, err := url.Parse(downloadURL) + destDir := filepath.Join(r.metadataDir, "artifacts", "charts", chartName, chartVersion) + chartFile, err := r.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{UseCache: true}) if err != nil { return nil, err } - var ( - packageName = path.Base(parsedURL.Path) - cachePath = filepath.Join(r.chartsCacheDir, packageName) - ) - - if !alwaysDownload && r.isInChartsCache(packageName) { - data, err := os.ReadFile(cachePath) - if err != nil { - return nil, err - } - return loader.LoadArchive(bytes.NewReader(data)) - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) + data, err := os.ReadFile(chartFile) if err != nil { return nil, err } - - rsp, err := http.DefaultClient.Do(req) + helmChart, err := loader.LoadArchive(bytes.NewReader(data)) if err != nil { return nil, err } - defer rsp.Body.Close() - body, err := io.ReadAll(rsp.Body) + manifests, err := r.generateManifests(ctx, name, namespace, helmChart, values) if err != nil { return nil, err } + r.logger.V(3).Infof("create '%s' with manifests: %s", name, string(manifests)) - if err := os.WriteFile(cachePath, body, 0644); err != nil { - return nil, err - } - - return loader.LoadArchive(bytes.NewReader(body)) + return manifests, nil } func (r *Manager) generateManifests(ctx context.Context, releaseName, namespace string, @@ -230,99 +182,6 @@ func (r *Manager) generateHelmValues(input interface{}) (map[string]interface{}, return nil, nil } -func (r *Manager) getLatestChart(indexFile *IndexFile, chartName string) (*ChartVersion, error) { - if versions, ok := indexFile.Entries[chartName]; ok { - if versions.Len() > 0 { - // The Entries are already sorted by version so the position 0 always point to the latest version. - v := []*ChartVersion(versions) - if len(v[0].URLs) == 0 { - return nil, fmt.Errorf("no download URLs found for %s-%s", chartName, v[0].Version) - } - return v[0], nil - } - return nil, fmt.Errorf("chart %s has empty versions", chartName) - } - - return nil, fmt.Errorf("chart %s not found", chartName) -} - -func (r *Manager) getIndexFile(ctx context.Context, indexURL string) (*IndexFile, error) { - if r.indexFile != nil { - return r.indexFile, nil - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, indexURL, nil) - if err != nil { - return nil, err - } - - rsp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer rsp.Body.Close() - - body, err := io.ReadAll(rsp.Body) - if err != nil { - return nil, err - } - - indexFile, err := loadIndex(body, indexURL) - if err != nil { - return nil, err - } - - // Cache the index file, so we don't request the index every time. - r.indexFile = indexFile - - return indexFile, nil -} - -// pullFromOCIRegistry pulls the chart from the remote OCI registry, for example, oci://registry-1.docker.io/bitnamicharts/etcd. -func (r *Manager) pullFromOCIRegistry(chartsRegistry, version string) (*chart.Chart, error) { - packageName := r.packageName(path.Base(chartsRegistry), version) - if !r.isInChartsCache(packageName) { - registryClient, err := registry.NewClient( - registry.ClientOptDebug(false), - registry.ClientOptEnableCache(false), - registry.ClientOptCredentialsFile(""), - ) - if err != nil { - return nil, err - } - - cfg := new(action.Configuration) - cfg.RegistryClient = registryClient - - // Create a pull action - client := action.NewPullWithOpts(action.WithConfig(cfg)) - client.Settings = cli.New() - client.Version = version - client.DestDir = r.chartsCacheDir - - r.logger.V(3).Infof("pulling chart '%s', version: '%s' from OCI registry", chartsRegistry, version) - // Execute the pull action - if _, err := client.Run(chartsRegistry); err != nil { - return nil, err - } - } - - data, err := os.ReadFile(filepath.Join(r.chartsCacheDir, packageName)) - if err != nil { - return nil, err - } - - return loader.LoadArchive(bytes.NewReader(data)) -} - -func (r *Manager) isInChartsCache(packageName string) bool { - res, _ := fileutils.IsFileExists(filepath.Join(r.chartsCacheDir, packageName)) - if res { - r.logger.V(3).Infof("chart '%s' is already in cache", packageName) - } - return res -} - func (r *Manager) newHelmClient(releaseName, namespace string) (*action.Install, error) { kubeVersion, err := chartutil.ParseKubeVersion(KubeVersion) if err != nil { @@ -340,84 +199,3 @@ func (r *Manager) newHelmClient(releaseName, namespace string) (*action.Install, return helmClient, nil } - -func (r *Manager) getChartDownloadURL(ctx context.Context, chartName, version string, useGreptimeCNArtifacts bool) (string, error) { - // Get the latest version from index file of GitHub repo. - if !useGreptimeCNArtifacts && version == "" { - indexFile, err := r.getIndexFile(ctx, GreptimeChartIndexURL) - if err != nil { - return "", err - } - - chartVersion, err := r.getLatestChart(indexFile, chartName) - if err != nil { - return "", err - } - - downloadURL := chartVersion.URLs[0] - r.logger.V(3).Infof("get latest chart '%s', version '%s', url: '%s'", - chartName, chartVersion.Version, downloadURL) - return downloadURL, nil - } - - if useGreptimeCNArtifacts { - if version == "" { - version = "latest" - } - - // The download URL example: 'https://downloads.greptime.cn/releases/charts/etcd/9.2.0/etcd-9.2.0.tgz'. - downloadURL := fmt.Sprintf("%s/%s/%s/%s.tgz", GreptimeCNCharts, chartName, version, chartName+"-"+version) - r.logger.V(3).Infof("get given version chart '%s', version '%s', url: '%s'", - chartName, version, downloadURL) - return downloadURL, nil - } - - // The download URL example: 'https://github.com/GreptimeTeam/helm-charts/releases/download/greptimedb-0.1.1-alpha.3/greptimedb-0.1.1-alpha.3.tgz'. - downloadURL := fmt.Sprintf("%s/%s/%s.tgz", GreptimeChartReleaseDownloadURL, chartName, chartName+"-"+version) - r.logger.V(3).Infof("get given version chart '%s', version '%s', url: '%s'", - chartName, version, downloadURL) - - return downloadURL, nil -} - -func (r *Manager) packageName(chartName, version string) string { - return fmt.Sprintf("%s-%s.tgz", chartName, version) -} - -func isOCIChar(url string) bool { - return strings.HasPrefix(url, "oci://") -} - -// loadIndex is from 'helm/pkg/index.go'. -func loadIndex(data []byte, source string) (*IndexFile, error) { - i := &IndexFile{} - - if len(data) == 0 { - return i, ErrEmptyIndexYaml - } - - if err := yaml.UnmarshalStrict(data, i); err != nil { - return i, err - } - - for name, cvs := range i.Entries { - for idx := len(cvs) - 1; idx >= 0; idx-- { - if cvs[idx] == nil { - log.Printf("skipping loading invalid entry for chart %q from %s: empty entry", name, source) - continue - } - if cvs[idx].APIVersion == "" { - cvs[idx].APIVersion = chart.APIVersionV1 - } - if err := cvs[idx].Validate(); err != nil { - log.Printf("skipping loading invalid entry for chart %q %q from %s: %s", name, cvs[idx].Version, source, err) - cvs = append(cvs[:idx], cvs[idx+1:]...) - } - } - } - i.SortEntries() - if i.APIVersion == "" { - return i, ErrNoAPIVersion - } - return i, nil -} diff --git a/pkg/helm/helm_test.go b/pkg/helm/helm_test.go index eae50e5d..0cdf77ec 100644 --- a/pkg/helm/helm_test.go +++ b/pkg/helm/helm_test.go @@ -17,31 +17,28 @@ package helm import ( "context" "os" - "sort" "strings" "testing" - "github.com/Masterminds/semver" "github.com/google/go-cmp/cmp" "helm.sh/helm/v3/pkg/strvals" "sigs.k8s.io/kind/pkg/log" + "github.com/GreptimeTeam/gtctl/pkg/artifacts" "github.com/GreptimeTeam/gtctl/pkg/deployer" "github.com/GreptimeTeam/gtctl/pkg/logger" ) const ( - testChartName = "greptimedb" - testChartsCacheDir = "/tmp/gtctl-test" + testMetadataDir = "/tmp/gtctl-test" ) func TestLoadAndRenderChart(t *testing.T) { - r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), - WithChartsCacheDir(testChartsCacheDir)) + r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), WithMetadataDir(testMetadataDir)) if err != nil { t.Errorf("failed to create render: %v", err) } - defer cleanChartsCache() + defer cleanMetadataDir() tests := []struct { name string @@ -53,23 +50,23 @@ func TestLoadAndRenderChart(t *testing.T) { { name: "greptimedb", namespace: "default", - chartName: GreptimeDBChartName, + chartName: artifacts.GreptimeDBChartName, chartVersion: "", // latest values: deployer.CreateGreptimeDBClusterOptions{}, }, { name: "greptimedb-operator", namespace: "default", - chartName: GreptimeDBOperatorChartName, + chartName: artifacts.GreptimeDBOperatorChartName, chartVersion: "", // latest values: deployer.CreateGreptimeDBOperatorOptions{}, }, { name: "etcd", namespace: "default", - chartName: EtcdBitnamiOCIRegistry, - chartVersion: DefaultEtcdChartVersion, - values: deployer.CreateGreptimeDBOperatorOptions{}, + chartName: artifacts.EtcdChartName, + chartVersion: artifacts.DefaultEtcdChartVersion, + values: deployer.CreateEtcdClusterOptions{}, }, } @@ -87,92 +84,12 @@ func TestLoadAndRenderChart(t *testing.T) { } } -func TestRender_GetIndexFile(t *testing.T) { - r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), - WithChartsCacheDir(testChartsCacheDir)) - if err != nil { - t.Errorf("failed to create render: %v", err) - } - defer cleanChartsCache() - - tests := []struct { - url string - }{ - { - url: "https://raw.githubusercontent.com/GreptimeTeam/helm-charts/gh-pages/index.yaml", - }, - { - url: "https://github.com/kubernetes/kube-state-metrics/raw/gh-pages/index.yaml", - }, - } - for _, tt := range tests { - t.Run(tt.url, func(t *testing.T) { - _, err := r.getIndexFile(context.Background(), tt.url) - if err != nil { - t.Errorf("fetch index '%s' failed, err: %v", tt.url, err) - } - }) - } -} - -func TestRender_GetLatestChartLatestChart(t *testing.T) { - r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), - WithChartsCacheDir(testChartsCacheDir)) - if err != nil { - t.Errorf("failed to create render: %v", err) - } - defer cleanChartsCache() - - tests := []struct { - url string - }{ - { - url: "https://raw.githubusercontent.com/GreptimeTeam/helm-charts/gh-pages/index.yaml", - }, - } - for _, tt := range tests { - t.Run(tt.url, func(t *testing.T) { - indexFile, err := r.getIndexFile(context.Background(), tt.url) - if err != nil { - t.Errorf("fetch index '%s' failed, err: %v", tt.url, err) - } - - chart, err := r.getLatestChart(indexFile, testChartName) - if err != nil { - t.Errorf("get latest chart failed, err: %v", err) - } - - var rawVersions []string - for _, v := range indexFile.Entries[testChartName] { - rawVersions = append(rawVersions, v.Version) - } - - vs := make([]*semver.Version, len(rawVersions)) - for i, r := range rawVersions { - v, err := semver.NewVersion(r) - if err != nil { - t.Errorf("Error parsing version: %s", err) - } - - vs[i] = v - } - - sort.Sort(semver.Collection(vs)) - - if chart.Version != vs[len(vs)-1].String() { - t.Errorf("latest chart version not match, expect: %s, got: %s", vs[len(vs)-1].String(), chart.Version) - } - }) - } -} - func TestRender_GenerateGreptimeDBHelmValues(t *testing.T) { - r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), - WithChartsCacheDir(testChartsCacheDir)) + r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), WithMetadataDir(testMetadataDir)) if err != nil { t.Errorf("failed to create render: %v", err) } - defer cleanChartsCache() + defer cleanMetadataDir() options := deployer.CreateGreptimeDBClusterOptions{ GreptimeDBChartVersion: "", @@ -212,12 +129,11 @@ func TestRender_GenerateGreptimeDBHelmValues(t *testing.T) { } func TestRender_GenerateGreptimeDBOperatorHelmValues(t *testing.T) { - r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), - WithChartsCacheDir(testChartsCacheDir)) + r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), WithMetadataDir(testMetadataDir)) if err != nil { t.Errorf("failed to create render: %v", err) } - defer cleanChartsCache() + defer cleanMetadataDir() options := deployer.CreateGreptimeDBOperatorOptions{ GreptimeDBOperatorChartVersion: "", @@ -247,12 +163,11 @@ func TestRender_GenerateGreptimeDBOperatorHelmValues(t *testing.T) { } func TestRender_GenerateEtcdHelmValues(t *testing.T) { - r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), - WithChartsCacheDir(testChartsCacheDir)) + r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), WithMetadataDir(testMetadataDir)) if err != nil { t.Errorf("failed to create render: %v", err) } - defer cleanChartsCache() + defer cleanMetadataDir() options := deployer.CreateEtcdClusterOptions{ EtcdChartVersion: "", @@ -287,6 +202,6 @@ func TestRender_GenerateEtcdHelmValues(t *testing.T) { } } -func cleanChartsCache() { - os.RemoveAll(testChartsCacheDir) +func cleanMetadataDir() { + os.RemoveAll(testMetadataDir) } diff --git a/pkg/utils/file/file.go b/pkg/utils/file/file.go index a020d989..d9eaaf70 100644 --- a/pkg/utils/file/file.go +++ b/pkg/utils/file/file.go @@ -26,10 +26,14 @@ import ( "path/filepath" ) -func CreateDirIfNotExists(dir string) (err error) { - if err := os.MkdirAll(dir, 0755); err != nil && !os.IsExist(err) { - return err +// EnsureDir ensures the directory exists. +func EnsureDir(dir string) error { + // Check if the directory exists + if _, err := os.Stat(dir); os.IsNotExist(err) { + // Create the directory along with any necessary parents. + return os.MkdirAll(dir, 0755) } + return nil } @@ -177,18 +181,24 @@ func untar(file, dst string) error { switch header.Typeflag { case tar.TypeReg: - outFile, err := os.Create(dst + "/" + header.Name) - if err != nil { + filePath := path.Join(dst, header.Name) + outFile, err := os.Create(filePath) + if err != nil && !os.IsExist(err) { return err } if _, err := io.Copy(outFile, tarReader); err != nil { return err } + + if err := os.Chmod(filePath, os.FileMode(header.Mode)); err != nil { + return err + } + if err := outFile.Close(); err != nil { return err } case tar.TypeDir: - if err := os.Mkdir(dst+"/"+header.Name, 0755); err != nil { + if err := os.Mkdir(path.Join(dst, header.Name), 0755); err != nil && !os.IsExist(err) { return err } default: