Adding a clear_output to remove previous backup version

cli/backup/dashboard.go

@@ -108,7 +108,10 @@
 					"continue (y/n) ", strings.Join(config.Config().GetDefaultGrafanaConfig().GetMonitoredFolders(), ", "),
 				), "", true)
 			}
-			rootCmd.GrafanaSvc().UploadDashboards(filter)
+			err := rootCmd.GrafanaSvc().UploadDashboards(filter)
+			if err != nil {
+				return err
+			}
 
 			rootCmd.TableObj.AppendHeader(table.Row{"Title", "id", "folder", "UID"})
 			boards := rootCmd.GrafanaSvc().ListDashboards(filter)

config/importer-example.yml

@@ -116,6 +116,7 @@ contexts:
 global:
   debug: true
   api_debug: false
+  clear_output: true
   ignore_ssl_errors: false ##when set to true will ignore invalid SSL errors
   retry_count: 3 ## Will retry any failed API request up to 3 times.
   retry_delay: 5s  ## will wait for specified duration before trying again.

internal/config/globals.go

@@ -12,6 +12,7 @@ type AppGlobals struct {
 	IgnoreSSLErrors bool           `mapstructure:"ignore_ssl_errors" yaml:"ignore_ssl_errors"`
 	RetryCount      int            `mapstructure:"retry_count" yaml:"retry_count"`
 	RetryDelay      string         `mapstructure:"retry_delay" yaml:"retry_delay"`
+	ClearOutput     bool           `mapstructure:"clear_output" yaml:"clear_output"`
 	retryTimeout    *time.Duration `mapstructure:"-" yaml:"-"`
 }
 

internal/service/alerting.go

@@ -6,6 +6,8 @@ import (
 	"log"
 	"log/slog"
 
+	"github.com/esnet/gdg/internal/storage"
+
 	"github.com/samber/lo"
 
 	"github.com/esnet/gdg/internal/tools/ptr"
@@ -51,7 +53,7 @@ func (s *DashNGoImpl) DownloadContactPoints() (string, error) {
 		return item.Name != emailReceiver
 	})
 
-	dsPath := buildResourcePath("contacts", config.AlertingResource, s.isLocal())
+	dsPath := buildResourcePath("contacts", config.AlertingResource, s.isLocal(), s.globalConf.ClearOutput)
 	if dsPacked, err = json.MarshalIndent(payload.ContactPoints, "", "	"); err != nil {
 		return "", fmt.Errorf("unable to serialize data to JSON. %w", err)
 	}
@@ -63,7 +65,7 @@ func (s *DashNGoImpl) DownloadContactPoints() (string, error) {
 }
 
 func (s *DashNGoImpl) isLocal() bool {
-	return s.storage.Name() == "LocalStorage"
+	return s.storage.Name() == storage.LocalStorageType.String()
 }
 
 func (s *DashNGoImpl) UploadContactPoints() ([]string, error) {
@@ -82,7 +84,7 @@ func (s *DashNGoImpl) UploadContactPoints() ([]string, error) {
 		m[i.UID] = currentContacts[ndx]
 	}
 
-	fileLocation := buildResourcePath("contacts", config.AlertingResource, s.isLocal())
+	fileLocation := buildResourcePath("contacts", config.AlertingResource, s.isLocal(), false)
 	if rawDS, err = s.storage.ReadFile(fileLocation); err != nil {
 		return nil, fmt.Errorf("failed to read file.  file: %s, err: %w", fileLocation, err)
 	}

internal/service/common.go

@@ -58,22 +58,22 @@ func getFolderFromResourcePath(filePath string, resourceType config.ResourceType
 	return "", errors.New("unable to parse resource to retrieve folder name")
 }
 
-func BuildResourceFolder(folderName string, resourceType config.ResourceType, createDestination bool) string {
+func BuildResourceFolder(folderName string, resourceType config.ResourceType, createDestination bool, clearOutput bool) string {
 	if resourceType == config.DashboardResource && folderName == "" {
 		folderName = DefaultFolderName
 	}
 
 	v := fmt.Sprintf("%s/%s", config.Config().GetDefaultGrafanaConfig().GetPath(resourceType), folderName)
 	if createDestination {
-		tools.CreateDestinationPath(v)
+		tools.CreateDestinationPath(config.Config().GetDefaultGrafanaConfig().GetPath(resourceType), clearOutput, v)
 	}
 	return v
 }
 
-func buildResourcePath(folderName string, resourceType config.ResourceType, createDestination bool) string {
+func buildResourcePath(folderName string, resourceType config.ResourceType, createDestination bool, clearOutput bool) string {
 	v := fmt.Sprintf("%s/%s.json", config.Config().GetDefaultGrafanaConfig().GetPath(resourceType), folderName)
 	if createDestination {
-		tools.CreateDestinationPath(filepath.Dir(v))
+		tools.CreateDestinationPath(config.Config().GetDefaultGrafanaConfig().GetPath(resourceType), clearOutput, filepath.Dir(v))
 	}
 	return v
 }

internal/service/common_test.go

@@ -51,25 +51,25 @@ func TestSlug(t *testing.T) {
 
 func TestUserPath(t *testing.T) {
 	fixEnvironment(t)
-	userPath := BuildResourceFolder("", config.UserResource, false)
+	userPath := BuildResourceFolder("", config.UserResource, false, false)
 	assert.Equal(t, "test/data/users/", userPath)
 }
 
 func TestBuildDashboardPath(t *testing.T) {
 	fixEnvironment(t)
-	result := BuildResourceFolder("General", config.DashboardResource, false)
+	result := BuildResourceFolder("General", config.DashboardResource, false, false)
 	assert.Equal(t, "test/data/org_your-org/dashboards/General", result)
 }
 
 func TestBuildFolderSourcePath(t *testing.T) {
 	fixEnvironment(t)
-	result := buildResourcePath(slug.Make("Some Folder"), config.FolderResource, false)
+	result := buildResourcePath(slug.Make("Some Folder"), config.FolderResource, false, false)
 	assert.Equal(t, "test/data/org_your-org/folders/some-folder.json", result)
 }
 
 func TestBuildDataSourcePath(t *testing.T) {
 	fixEnvironment(t)
 
-	result := buildResourcePath(slug.Make("My DS"), config.ConnectionResource, false)
+	result := buildResourcePath(slug.Make("My DS"), config.ConnectionResource, false, false)
 	assert.Equal(t, "test/data/org_your-org/connections/my-ds.json", result)
 }

internal/service/connection_permissions.go

@@ -77,7 +77,7 @@
 			slog.Error("unable to marshall json ", "err", err.Error(), "connectionName", connection.Connection.Name)
 			continue
 		}
-		dsPath := buildResourcePath(slug.Make(connection.Connection.Name), config.ConnectionPermissionResource, s.isLocal())
+		dsPath := buildResourcePath(slug.Make(connection.Connection.Name), config.ConnectionPermissionResource, s.isLocal(), s.globalConf.ClearOutput)
 		if err = s.storage.WriteFile(dsPath, dsPacked); err != nil {
 			slog.Error("unable to write file. ", "filename", slug.Make(connection.Connection.Name), "error", err.Error())
 		} else {

internal/service/connections.go

@@ -76,7 +76,7 @@ func (s *DashNGoImpl) DownloadConnections(filter filters.Filter) []string {
 			continue
 		}
 
-		dsPath := buildResourcePath(slug.Make(ds.Name), config.ConnectionResource, s.isLocal())
+		dsPath := buildResourcePath(slug.Make(ds.Name), config.ConnectionResource, s.isLocal(), s.globalConf.ClearOutput)
 
 		if err = s.storage.WriteFile(dsPath, dsPacked); err != nil {
 			slog.Error("Unable to write file", "filename", slug.Make(ds.Name), "err", err)

internal/service/contracts.go

@@ -54,7 +54,7 @@ type ConnectionPermissions interface {
 type DashboardsApi interface {
 	ListDashboards(filter filters.Filter) []*models.Hit
 	DownloadDashboards(filter filters.Filter) []string
-	UploadDashboards(filter filters.Filter)
+	UploadDashboards(filter filters.Filter) error
 	DeleteAllDashboards(filter filters.Filter) []string
 	LintDashboards(req types.LintRequest) []string
 }

internal/service/dashboard_permissions.go

@@ -59,7 +59,7 @@
 			continue
 		}
 
-		dsPath := fmt.Sprintf("%s/%s.json", BuildResourceFolder(link.Dashboard.FolderTitle, config.DashboardPermissionsResource, s.isLocal()), slug.Make(link.Dashboard.Title))
+		dsPath := fmt.Sprintf("%s/%s.json", BuildResourceFolder(link.Dashboard.FolderTitle, config.DashboardPermissionsResource, s.isLocal(), s.globalConf.ClearOutput), slug.Make(link.Dashboard.Title))
 		if err = s.storage.WriteFile(dsPath, dsPacked); err != nil {
 			slog.Error("unable to write file. ", "filename", slug.Make(link.Dashboard.Title), "error", err.Error())
 		} else {

internal/service/dashboards.go

@@ -370,7 +370,7 @@
 			continue
 		}
 
-		fileName := buildDashboardFileName(link, metaData.GetPayload().Meta.Slug, folderUidMap, useNestedFolders, s.grafanaConf.Storage == "")
+		fileName := buildDashboardFileName(link, metaData.GetPayload().Meta.Slug, folderUidMap, useNestedFolders, s.isLocal(), s.globalConf.ClearOutput)
 		if err = s.storage.WriteFile(fileName, pretty.Pretty(rawBoard)); err != nil {
 			slog.Error("Unable to save dashboard to file\n", "err", err, "dashboard", metaData.GetPayload().Meta.Slug)
 		} else {
@@ -406,14 +406,14 @@
 }
 
 // buildDashboardFileName for a given dashboard, a full nested folder path is constructed
-func buildDashboardFileName(db *models.Hit, dbSlug string, folderUidMap map[string]*customTypes.FolderDetails, nested, createDestination bool) string {
+func buildDashboardFileName(db *models.Hit, dbSlug string, folderUidMap map[string]*customTypes.FolderDetails, nested, createDestination, clearOutput bool) string {
 	var folderPath string
 	if nested {
 		folderPath = getNestedFolder(db.FolderTitle, db.FolderUID, folderUidMap)
 	} else {
 		folderPath = db.FolderTitle
 	}
-	fileName := fmt.Sprintf("%s/%s.json", BuildResourceFolder(folderPath, config.DashboardResource, createDestination), dbSlug)
+	fileName := fmt.Sprintf("%s/%s.json", BuildResourceFolder(folderPath, config.DashboardResource, createDestination, clearOutput), dbSlug)
 	return fileName
 }
 
@@ -482,7 +482,7 @@
 
 // UploadDashboards finds all the dashboards in the configured location and exports them to grafana.
 // if the folder doesn't exist, it'll be created.
-func (s *DashNGoImpl) UploadDashboards(filterReq filters.Filter) {
+func (s *DashNGoImpl) UploadDashboards(filterReq filters.Filter) error {
 	var (
 		rawBoard   []byte
 		folderName string
@@ -491,13 +491,13 @@
 	dashboardPath := config.Config().GetDefaultGrafanaConfig().GetPath(config.DashboardResource)
 	filesInDir, err := s.storage.FindAllFiles(dashboardPath, true)
 	if err != nil {
-		log.Fatalf("unable to find any files to export from storage engine, err: %v", err)
+		return fmt.Errorf("unable to find any files to export from storage engine, err: %w", err)
 	}
 
 	if !s.grafanaConf.GetDashboardSettings().NestedFolders {
 		p, regexErr := regexp.Compile(nestedDashboardRegexFilter)
 		if regexErr != nil {
-			log.Fatal("unable to compile nested folder validation regex patter")
+			return fmt.Errorf("unable to compile nested folder validation regex patter, err: %w", regexErr)
 		}
 
 		invalidCount := lo.FilterMap(filesInDir, func(file string, index int) (string, bool) {
@@ -521,7 +521,7 @@
 			return file, false
 		})
 		if len(invalidCount) > 0 {
-			log.Fatal("nested folder feature is disabled in GDG but import dashboardPath contains a nested folder.  Please fix the import or configuration.  ", slog.String("files", strings.Join(invalidCount, ", ")))
+			return fmt.Errorf("nested folder feature is disabled in GDG but import dashboardPath contains a nested folder.  Please fix the import or configuration.  Files: %s", strings.Join(invalidCount, ", "))
 		}
 
 	}
@@ -584,7 +584,7 @@
 
 		}
 		if _, ok := alreadyProcessed[board["uid"]]; ok {
-			log.Fatalf("Board with same UID was already processed.  Please check your backup folder. This may occur if you pulled the data multiple times with configuration of: nested folder enabled and disabled, uid: %v, title: %v", board["uid"], slug.Make((board["title"]).(string)))
+			return fmt.Errorf("Board with same UID was already processed.  Please check your backup folder. This may occur if you pulled the data multiple times with configuration of: nested folder enabled and disabled, uid: %v, title: %v", board["uid"], slug.Make((board["title"]).(string)))
 		} else {
 			alreadyProcessed[board["uid"]] = true
 		}
@@ -641,6 +641,7 @@
 		}
 
 	}
+	return nil
 }
 
 // DeleteAllDashboards clears all current dashboards being monitored.  Any folder not white listed

internal/service/folders.go

@@ -86,7 +86,7 @@
 		if fileName == "" {
 			fileName = folder.Title
 		}
-		dsPath := buildResourcePath(slug.Make(fileName), config.FolderPermissionResource, s.isLocal())
+		dsPath := buildResourcePath(slug.Make(fileName), config.FolderPermissionResource, s.isLocal(), s.globalConf.ClearOutput)
 		if err = s.storage.WriteFile(dsPath, dsPacked); err != nil {
 			slog.Error("Unable to write file", "err", err.Error(), "filename", slug.Make(folder.Title))
 		} else {
@@ -240,7 +240,7 @@
 			slog.Error("Unable to serialize data to JSON", "err", err, "folderName", folder.Title)
 			continue
 		}
-		dsPath := buildResourcePath(folder.Title, config.FolderResource, s.isLocal())
+		dsPath := buildResourcePath(folder.Title, config.FolderResource, s.isLocal(), s.globalConf.ClearOutput)
 
 		if !s.checkFolderName(folder.Title) {
 			slog.Warn("Folder has an invalid character and is not supported, skipping folder", "folderName", folder.Title)
@@ -252,8 +252,8 @@
 			if slugFolder != folder.NestedPath {
 				dsPath = strings.Replace(dsPath, slugFolder, folder.NestedPath, 1)
 				baseFolder := filepath.Dir(dsPath)
-				if s.isLocal() {
-					tools.CreateDestinationPath(baseFolder)
+				if s.isLocal() { //&& baseFolder != "" {
+					tools.CreateDestinationPath("", false, baseFolder)
 				}
 			}
 		}

internal/service/gdg_api.go

@@ -11,7 +11,6 @@ import (
 	"github.com/esnet/gdg/internal/api"
 	"github.com/esnet/gdg/internal/config"
 	"github.com/esnet/gdg/internal/storage"
-	"github.com/spf13/viper"
 )
 
 var (
@@ -20,12 +19,9 @@ var (
 )
 
 type DashNGoImpl struct {
-	extended *api.ExtendedApi
-
+	extended    *api.ExtendedApi
 	grafanaConf *config.GrafanaConfig
-	configRef   *viper.Viper
-	debug       bool
-	apiDebug    bool
+	globalConf  *config.AppGlobals
 	storage     storage.Storage
 }
 
@@ -35,16 +31,14 @@ var DefaultConfigProvider config.Provider = func() *config.Configuration {
 
 func setupConfigData(cfg *config.Configuration, obj *DashNGoImpl) {
 	obj.grafanaConf = cfg.GetDefaultGrafanaConfig()
-	obj.configRef = cfg.GetViperConfig()
-	obj.debug = cfg.IsDebug()
-	obj.apiDebug = cfg.IsApiDebug()
+	obj.globalConf = cfg.GetGDGConfig().GetAppGlobals()
 }
 
 func newInstance() *DashNGoImpl {
 	obj := &DashNGoImpl{}
 	setupConfigData(config.Config(), obj)
 
-	if obj.apiDebug {
+	if obj.globalConf.ApiDebug {
 		err := os.Setenv("DEBUG", "1")
 		if err != nil {
 			slog.Debug("unable to set debug env value", slog.Any("err", err))

internal/service/libraryelements.go

@@ -97,7 +97,7 @@ func (s *DashNGoImpl) DownloadLibraryElements(filter filters.Filter) []string {
 			folderName = val
 		}
 
-		libraryPath := fmt.Sprintf("%s/%s.json", BuildResourceFolder(folderName, config.LibraryElementResource, s.isLocal()), slug.Make(item.Name))
+		libraryPath := fmt.Sprintf("%s/%s.json", BuildResourceFolder(folderName, config.LibraryElementResource, s.isLocal(), s.globalConf.ClearOutput), slug.Make(item.Name))
 
 		if err = s.storage.WriteFile(libraryPath, dsPacked); err != nil {
 			slog.Error("Unable to write file", "err", err, "library-element", slug.Make(item.Name))

internal/service/login.go

@@ -73,7 +73,7 @@
 		Schemes:      []string{u.Scheme},
 		NumRetries:   config.Config().GetGDGConfig().GetAppGlobals().RetryCount,
 		RetryTimeout: config.Config().GetGDGConfig().GetAppGlobals().GetRetryTimeout(),
-		Debug:        s.apiDebug,
+		Debug:        s.globalConf.ApiDebug,
 	}
 
 	// If more than one opts is passed, depend on the caller to setup his required configuration
@@ -97,7 +97,7 @@
 	if s.grafanaConf.APIToken != "" {
 		grafanaClient, _ := s.getNewClient(func(clientCfg *client.TransportConfig) {
 			clientCfg.APIKey = s.grafanaConf.APIToken
-			clientCfg.Debug = s.apiDebug
+			clientCfg.Debug = s.globalConf.ApiDebug
 		})
 		return grafanaClient
 	} else {
@@ -123,7 +123,7 @@
 func (s *DashNGoImpl) getDefaultBasicOpts() []NewClientOpts {
 	return []NewClientOpts{func(clientCfg *client.TransportConfig) {
 		clientCfg.BasicAuth = url.UserPassword(s.grafanaConf.UserName, s.grafanaConf.Password)
-		clientCfg.Debug = s.apiDebug
+		clientCfg.Debug = s.globalConf.ApiDebug
 	}}
 }