Merge pull request #497 from gianlucam76/memory-usage

Print memory usage

Makefile

@@ -259,7 +259,7 @@ create-control-cluster: $(KIND) $(CLUSTERCTL) $(KUBECTL)
  sed -e "s/K8S_VERSION/$(K8S_VERSION)/g" test/$(KIND_CONFIG) > test/$(KIND_CONFIG).tmp
  $(KIND) create cluster --name=$(CONTROL_CLUSTER_NAME) --config test/$(KIND_CONFIG).tmp
  @echo "Create control cluster with docker as infrastructure provider"
- CLUSTER_TOPOLOGY=true $(CLUSTERCTL) init --infrastructure docker
+ CLUSTER_TOPOLOGY=true $(CLUSTERCTL) init --core cluster-api --bootstrap kubeadm --control-plane kubeadm --infrastructure docker
 
  @echo wait for capd-system pod
  $(KUBECTL) wait --for=condition=Available deployment/capd-controller-manager -n capd-system --timeout=$(TIMEOUT)

api/v1alpha1/spec.go

@@ -189,6 +189,11 @@ type HelmOptions struct {
  // Labels that would be added to release metadata.
  // +optional
  Labels map[string]string `json:"labels,omitempty"`
+
+ // EnableClientCache is a flag to enable Helm client cache. If it is not specified, it will be set to false.
+ // +kubebuilder:default=false
+ // +optional
+ EnableClientCache bool `json:"enableClientCache,omitempty"`
 }
 
 type HelmChart struct {

cmd/main.go

@@ -23,6 +23,8 @@ import (
  "net/http"
  "net/http/pprof"
  "os"
+ "runtime"
+ "runtime/debug"
  "sync"
  "syscall"
  "time"
@@ -58,6 +60,7 @@ import (
  "github.com/projectsveltos/libsveltos/lib/crd"
  "github.com/projectsveltos/libsveltos/lib/deployer"
  "github.com/projectsveltos/libsveltos/lib/logsettings"
+ logs "github.com/projectsveltos/libsveltos/lib/logsettings"
  libsveltosset "github.com/projectsveltos/libsveltos/lib/set"
  //+kubebuilder:scaffold:imports
 )
@@ -86,6 +89,8 @@ const (
  defaultReconcilers = 10
  defaultWorkers = 20
  defaulReportMode = int(controllers.CollectFromManagementCluster)
+ mebibytes_bytes = 1 << 20
+ gibibytes_per_bytes = 1 << 30
 )
 
 // Add RBAC for the authorized diagnostics endpoint.
@@ -108,7 +113,6 @@ func main() {
  reportMode = controllers.ReportMode(tmpReportMode)
 
  ctrl.SetLogger(klog.Background())
-
  ctrlOptions := ctrl.Options{
  Scheme: scheme,
  Metrics: getDiagnosticsOptions(),
@@ -135,13 +139,15 @@ func main() {
 
  // Setup the context that's going to be used in controllers and for the manager.
  ctx := ctrl.SetupSignalHandler()
-
  controllers.SetManagementClusterAccess(mgr.GetClient(), mgr.GetConfig())
 
- logsettings.RegisterForLogSettings(ctx,
+ logs.RegisterForLogSettings(ctx,
  libsveltosv1alpha1.ComponentAddonManager, ctrl.Log.WithName("log-setter"),
  ctrl.GetConfigOrDie())
 
+ debug.SetMemoryLimit(gibibytes_per_bytes)
+ go printMemUsage(ctrl.Log.WithName("memory-usage"))
+
  startControllersAndWatchers(ctx, mgr)
 
  setupChecks(mgr)
@@ -170,7 +176,7 @@ func initFlags(fs *pflag.FlagSet) {
  fs.StringVar(&diagnosticsAddress, "diagnostics-address", ":8443",
  "The address the diagnostics endpoint binds to. Per default metrics are served via https and with"+
  "authentication/authorization. To serve via http and without authentication/authorization set --insecure-diagnostics."+
- "If --insecure-diagnostics is not set the diagnostics endpoint also serves pprof endpoints and an endpoint to change the log level.")
+ "If --insecure-diagnostics is not set the diagnostics endpoint also serves pprof endpoints")
 
  fs.BoolVar(&insecureDiagnostics, "insecure-diagnostics", false,
  "Enable insecure diagnostics serving. For more details see the description of --diagnostics-address.")
@@ -545,3 +551,23 @@ func startControllersAndWatchers(ctx context.Context, mgr manager.Manager) {
 
  startWatchers(ctx, mgr, watchersForCAPI, watchersForFlux)
 }
+
+// printMemUsage memory stats. Call GC
+func printMemUsage(logger logr.Logger) {
+ for {
+ time.Sleep(time.Minute)
+ var m runtime.MemStats
+ runtime.ReadMemStats(&m)
+ // For info on each, see: /pkg/runtime/#MemStats
+ l := logger.WithValues("Alloc (MiB)", bToMb(m.Alloc)).
+ WithValues("TotalAlloc (MiB)", bToMb(m.TotalAlloc)).
+ WithValues("Sys (MiB)", bToMb(m.Sys)).
+ WithValues("NumGC", m.NumGC)
+ l.V(logs.LogInfo).Info("memory stats")
+ runtime.GC()
+ }
+}
+
+func bToMb(b uint64) uint64 {
+ return b / mebibytes_bytes
+}

config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml

@@ -187,6 +187,12 @@ spec:
  if set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema
  Default to false
  type: boolean
+ enableClientCache:
+ default: false
+ description: EnableClientCache is a flag to enable Helm
+ client cache. If it is not specified, it will be set to
+ false.
+ type: boolean
  labels:
  additionalProperties:
  type: string

config/crd/bases/config.projectsveltos.io_clustersummaries.yaml

@@ -203,6 +203,12 @@ spec:
  if set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema
  Default to false
  type: boolean
+ enableClientCache:
+ default: false
+ description: EnableClientCache is a flag to enable Helm
+ client cache. If it is not specified, it will be set
+ to false.
+ type: boolean
  labels:
  additionalProperties:
  type: string

config/crd/bases/config.projectsveltos.io_profiles.yaml

@@ -187,6 +187,12 @@ spec:
  if set, the installation process will not validate rendered templates against the Kubernetes OpenAPI Schema
  Default to false
  type: boolean
+ enableClientCache:
+ default: false
+ description: EnableClientCache is a flag to enable Helm
+ client cache. If it is not specified, it will be set to
+ false.
+ type: boolean
  labels:
  additionalProperties:
  type: string

config/manager/manager.yaml

@@ -69,7 +69,10 @@ spec:
  periodSeconds: 10
  volumeMounts:
  - mountPath: /tmp
- name: tmp 
+ name: tmp
+ resources:
+ requests:
+ memory: 256Mi
  serviceAccountName: controller
  terminationGracePeriodSeconds: 10
  volumes:

controllers/chartmanager/chartmanager.go

@@ -367,7 +367,7 @@ func (m *instance) getClusterSummaryKey(clusterSummaryName string) string {
  return clusterSummaryName
 }
 
-// getReleaseInfoFromKey returns helm release given a key
+// getReleaseInfoFromKey returns helm release for given a key
 func getReleaseInfoFromKey(releaseKey string) *HelmReleaseInfo {
  info := strings.Split(releaseKey, keySeparator)
  return &HelmReleaseInfo{

controllers/handlers_helm.go

@@ -19,7 +19,6 @@ package controllers
 import (
  "context"
  "crypto/sha256"
- "errors"
  "fmt"
  "os"
  "strconv"
@@ -29,6 +28,7 @@ import (
  "github.com/Masterminds/semver/v3"
  "github.com/gdexlab/go-render/render"
  "github.com/go-logr/logr"
+ "github.com/pkg/errors"
  "helm.sh/helm/v3/pkg/action"
  "helm.sh/helm/v3/pkg/chart"
  "helm.sh/helm/v3/pkg/chart/loader"
@@ -579,7 +579,7 @@ func handleChart(ctx context.Context, clusterSummary *configv1alpha1.ClusterSumm
  kubeconfig string, logger logr.Logger) (*releaseInfo, *configv1alpha1.ReleaseReport, error) {
 
  currentRelease, err := getReleaseInfo(currentChart.ReleaseName,
- currentChart.ReleaseNamespace, kubeconfig)
+ currentChart.ReleaseNamespace, kubeconfig, getEnableClientCacheValue(currentChart.Options))
  if err != nil && !errors.Is(err, driver.ErrReleaseNotFound) {
  return nil, nil, err
  }
@@ -638,7 +638,7 @@ func handleChart(ctx context.Context, clusterSummary *configv1alpha1.ClusterSumm
  report.Message = "Already managing this helm release and specified version already installed"
  }
 
- currentRelease, err = getReleaseInfo(currentChart.ReleaseName, currentChart.ReleaseNamespace, kubeconfig)
+ currentRelease, err = getReleaseInfo(currentChart.ReleaseName, currentChart.ReleaseNamespace, kubeconfig, getEnableClientCacheValue(currentChart.Options))
  if err != nil && !errors.Is(err, driver.ErrReleaseNotFound) {
  return nil, nil, err
  }
@@ -672,6 +672,7 @@ func repoAddOrUpdate(settings *cli.EnvSettings, name, url string, logger logr.Lo
 
  storage.Update(entry)
  const permissions = 0o644
+
  err = storage.WriteFile(settings.RepositoryConfig, permissions)
  if err != nil {
  return err
@@ -748,6 +749,7 @@ func installRelease(ctx context.Context, clusterSummary *configv1alpha1.ClusterS
  }
 
  logger.V(logs.LogDebug).Info("installing release done")
+
  return nil
 }
 
@@ -780,7 +782,7 @@ func checkDependencies(chartRequested *chart.Chart, installClient *action.Instal
 // uninstallRelease removes helm release from a CAPI Cluster.
 // No action in DryRun mode.
 func uninstallRelease(clusterSummary *configv1alpha1.ClusterSummary,
- releaseName, releaseNamespace, kubeconfig string, logger logr.Logger) error {
+ releaseName, releaseNamespace, kubeconfig string, helmChart *configv1alpha1.HelmChart, logger logr.Logger) error {
 
  // No-op in DryRun mode
  if clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1alpha1.SyncModeDryRun {
@@ -790,7 +792,12 @@ func uninstallRelease(clusterSummary *configv1alpha1.ClusterSummary,
  logger = logger.WithValues("release", releaseName, "releaseNamespace", releaseNamespace)
  logger.V(logs.LogDebug).Info("uninstalling release")
 
- actionConfig, err := actionConfigInit(releaseNamespace, kubeconfig)
+ enableClientCache := false
+ if helmChart != nil {
+ enableClientCache = getEnableClientCacheValue(helmChart.Options)
+ }
+
+ actionConfig, err := actionConfigInit(releaseNamespace, kubeconfig, enableClientCache)
  if err != nil {
  return err
  }
@@ -799,13 +806,15 @@ func uninstallRelease(clusterSummary *configv1alpha1.ClusterSummary,
  uninstallClient.DryRun = false
  uninstallClient.Wait = false
  uninstallClient.DisableHooks = false
+ uninstallClient.KeepHistory = false
 
  _, err = uninstallClient.Run(releaseName)
  if err != nil {
  return err
  }
 
  logger.V(logs.LogDebug).Info("uninstalling release done")
+
  return nil
 }
 
@@ -835,7 +844,7 @@ func upgradeRelease(ctx context.Context, clusterSummary *configv1alpha1.ClusterS
  chartName = defaultUploadPath + "/" + chartName
  }
 
- actionConfig, err := actionConfigInit(requestedChart.ReleaseNamespace, kubeconfig)
+ actionConfig, err := actionConfigInit(requestedChart.ReleaseNamespace, kubeconfig, getEnableClientCacheValue(requestedChart.Options))
  if err != nil {
  return err
  }
@@ -882,14 +891,15 @@ func upgradeRelease(ctx context.Context, clusterSummary *configv1alpha1.ClusterS
  }
 
  logger.V(logs.LogDebug).Info("upgrading release done")
+
  return nil
 }
 
 func debugf(format string, v ...interface{}) {
  helmLogger.V(logs.LogDebug).Info(fmt.Sprintf(format, v...))
 }
 
-func actionConfigInit(namespace, kubeconfig string) (*action.Configuration, error) {
+func actionConfigInit(namespace, kubeconfig string, enableClientCache bool) (*action.Configuration, error) {
  settings := getSettings(namespace)
 
  actionConfig := new(action.Configuration)
@@ -902,7 +912,7 @@ func actionConfigInit(namespace, kubeconfig string) (*action.Configuration, erro
 
  registryClient, err := registry.NewClient(
  registry.ClientOptDebug(settings.Debug),
- registry.ClientOptEnableCache(true),
+ registry.ClientOptEnableCache(enableClientCache),
  )
  if err != nil {
  return nil, err
@@ -931,8 +941,8 @@ func isChartInstallable(ch *chart.Chart) bool {
  return false
 }
 
-func getReleaseInfo(releaseName, releaseNamespace, kubeconfig string) (*releaseInfo, error) {
- actionConfig, err := actionConfigInit(releaseNamespace, kubeconfig)
+func getReleaseInfo(releaseName, releaseNamespace, kubeconfig string, enableClientCache bool) (*releaseInfo, error) {
+ actionConfig, err := actionConfigInit(releaseNamespace, kubeconfig, enableClientCache)
 
  if err != nil {
  return nil, err
@@ -1084,7 +1094,7 @@ func doUninstallRelease(clusterSummary *configv1alpha1.ClusterSummary, requested
  requestedChart.RepositoryName))
 
  return uninstallRelease(clusterSummary, requestedChart.ReleaseName, requestedChart.ReleaseNamespace,
- kubeconfig, logger)
+ kubeconfig, requestedChart, logger)
 }
 
 // doUpgradeRelease upgrades helm release in the CAPI Cluster.
@@ -1172,7 +1182,7 @@ func undeployStaleReleases(ctx context.Context, c client.Client, clusterSummary
  managedHelmReleases[i].Name, managedHelmReleases[i].Namespace))
 
  _, err := getReleaseInfo(managedHelmReleases[i].Name,
- managedHelmReleases[i].Namespace, kubeconfig)
+ managedHelmReleases[i].Namespace, kubeconfig, false)
  if err != nil {
  if errors.Is(err, driver.ErrReleaseNotFound) {
  continue
@@ -1181,7 +1191,7 @@ func undeployStaleReleases(ctx context.Context, c client.Client, clusterSummary
  }
 
  if err := uninstallRelease(clusterSummary, managedHelmReleases[i].Name, managedHelmReleases[i].Namespace,
- kubeconfig, logger); err != nil {
+ kubeconfig, nil, logger); err != nil {
  return nil, err
  }
 
@@ -1449,7 +1459,7 @@ func collectResourcesFromManagedHelmCharts(ctx context.Context, c client.Client,
  l := logger.WithValues("chart", currentChart.ChartName, "releaseNamespace", currentChart.ReleaseNamespace)
  l.V(logs.LogDebug).Info("collecting resources for helm chart")
  if chartManager.CanManageChart(clusterSummary, currentChart) {
- actionConfig, err := actionConfigInit(currentChart.ReleaseNamespace, kubeconfig)
+ actionConfig, err := actionConfigInit(currentChart.ReleaseNamespace, kubeconfig, getEnableClientCacheValue(currentChart.Options))
 
  if err != nil {
  return nil, err
@@ -1532,6 +1542,14 @@ func getSettings(namespace string) *cli.EnvSettings {
  return settings
 }
 
+func getEnableClientCacheValue(options *configv1alpha1.HelmOptions) bool {
+ if options != nil {
+ return options.EnableClientCache
+ }
+
+ return false
+}
+
 func getWaitHelmValue(options *configv1alpha1.HelmOptions) bool {
  if options != nil {
  return options.Wait
@@ -1612,7 +1630,7 @@ func getLabelsValue(options *configv1alpha1.HelmOptions) map[string]string {
 }
 
 func getHelmInstallClient(requestedChart *configv1alpha1.HelmChart, kubeconfig string) (*action.Install, error) {
- actionConfig, err := actionConfigInit(requestedChart.ReleaseNamespace, kubeconfig)
+ actionConfig, err := actionConfigInit(requestedChart.ReleaseNamespace, kubeconfig, getEnableClientCacheValue(requestedChart.Options))
  if err != nil {
  return nil, err
  }
@@ -1680,7 +1698,7 @@ func addExtraMetadata(ctx context.Context, requestedChart *configv1alpha1.HelmCh
  // Current hash of current metadata (extraLabels and extraAnnotations)
  metadataHash := getMetadataHash(clusterSummary)
 
- actionConfig, err := actionConfigInit(requestedChart.ReleaseNamespace, kubeconfig)
+ actionConfig, err := actionConfigInit(requestedChart.ReleaseNamespace, kubeconfig, getEnableClientCacheValue(requestedChart.Options))
  if err != nil {
  return err
  }

controllers/profile_transformation_common.go

@@ -48,8 +48,6 @@ func requeueForCluster(cluster client.Object,
 
  profileCurrentlyMatching := getConsumersForEntry(clusterMap, &clusterInfo)
 
- logger.V(logs.LogInfo).Info(fmt.Sprintf("MGIANLUC %d", profileCurrentlyMatching.Len()))
-
  clusterLabels[clusterInfo] = cluster.GetLabels()
 
  // Get all (Cluster)Profiles previously matching this cluster and reconcile those

manifest/deployment-agentless.yaml

@@ -51,6 +51,9 @@ spec:
  scheme: HTTP
  initialDelaySeconds: 5
  periodSeconds: 10
+ resources:
+ requests:
+ memory: 256Mi
  securityContext:
  allowPrivilegeEscalation: false
  capabilities:

manifest/deployment-shard.yaml

@@ -51,6 +51,9 @@ spec:
  scheme: HTTP
  initialDelaySeconds: 5
  periodSeconds: 10
+ resources:
+ requests:
+ memory: 256Mi
  securityContext:
  allowPrivilegeEscalation: false
  capabilities: