Adding a clear_output to remove previous backup version

Fixes #325

config/importer-example.yml

@@ -116,6 +116,7 @@ contexts:
 global:
   debug: true
   api_debug: false
+  clear_output: true
   ignore_ssl_errors: false ##when set to true will ignore invalid SSL errors
   retry_count: 3 ## Will retry any failed API request up to 3 times.
   retry_delay: 5s  ## will wait for specified duration before trying again.

internal/config/config.go

@@ -11,3 +11,8 @@ type Configuration struct {
 }
 
 type Provider func() *Configuration
+
+func main() {
+	var c Configuration
+	c.GetViperConfig()
+}

internal/config/globals.go

@@ -12,6 +12,7 @@ type AppGlobals struct {
 	IgnoreSSLErrors bool           `mapstructure:"ignore_ssl_errors" yaml:"ignore_ssl_errors"`
 	RetryCount      int            `mapstructure:"retry_count" yaml:"retry_count"`
 	RetryDelay      string         `mapstructure:"retry_delay" yaml:"retry_delay"`
+	ClearOutput     bool           `mapstructure:"clear_output" yaml:"clear_output"`
 	retryTimeout    *time.Duration `mapstructure:"-" yaml:"-"`
 }
 

internal/service/alerting.go

@@ -6,6 +6,8 @@ import (
 	"log"
 	"log/slog"
 
+	"github.com/esnet/gdg/internal/storage"
+
 	"github.com/samber/lo"
 
 	"github.com/esnet/gdg/internal/tools/ptr"
@@ -51,7 +53,7 @@ func (s *DashNGoImpl) DownloadContactPoints() (string, error) {
 		return item.Name != emailReceiver
 	})
 
-	dsPath := buildResourcePath("contacts", config.AlertingResource, s.isLocal())
+	dsPath := buildResourcePath("contacts", config.AlertingResource, s.isLocal(), s.globalConf.ClearOutput)
 	if dsPacked, err = json.MarshalIndent(payload.ContactPoints, "", "	"); err != nil {
 		return "", fmt.Errorf("unable to serialize data to JSON. %w", err)
 	}
@@ -63,7 +65,7 @@ func (s *DashNGoImpl) DownloadContactPoints() (string, error) {
 }
 
 func (s *DashNGoImpl) isLocal() bool {
-	return s.storage.Name() == "LocalStorage"
+	return s.storage.Name() == storage.LocalStorageType.String()
 }
 
 func (s *DashNGoImpl) UploadContactPoints() ([]string, error) {
@@ -82,7 +84,7 @@ func (s *DashNGoImpl) UploadContactPoints() ([]string, error) {
 		m[i.UID] = currentContacts[ndx]
 	}
 
-	fileLocation := buildResourcePath("contacts", config.AlertingResource, s.isLocal())
+	fileLocation := buildResourcePath("contacts", config.AlertingResource, s.isLocal(), false)
 	if rawDS, err = s.storage.ReadFile(fileLocation); err != nil {
 		return nil, fmt.Errorf("failed to read file.  file: %s, err: %w", fileLocation, err)
 	}

internal/service/common.go

@@ -58,22 +58,22 @@ func getFolderFromResourcePath(filePath string, resourceType config.ResourceType
 	return "", errors.New("unable to parse resource to retrieve folder name")
 }
 
-func BuildResourceFolder(folderName string, resourceType config.ResourceType, createDestination bool) string {
+func BuildResourceFolder(folderName string, resourceType config.ResourceType, createDestination bool, clearOutput bool) string {
 	if resourceType == config.DashboardResource && folderName == "" {
 		folderName = DefaultFolderName
 	}
 
 	v := fmt.Sprintf("%s/%s", config.Config().GetDefaultGrafanaConfig().GetPath(resourceType), folderName)
 	if createDestination {
-		tools.CreateDestinationPath(v)
+		tools.CreateDestinationPath(config.Config().GetDefaultGrafanaConfig().GetPath(resourceType), clearOutput, v)
 	}
 	return v
 }
 
-func buildResourcePath(folderName string, resourceType config.ResourceType, createDestination bool) string {
+func buildResourcePath(folderName string, resourceType config.ResourceType, createDestination bool, clearOutput bool) string {
 	v := fmt.Sprintf("%s/%s.json", config.Config().GetDefaultGrafanaConfig().GetPath(resourceType), folderName)
 	if createDestination {
-		tools.CreateDestinationPath(filepath.Dir(v))
+		tools.CreateDestinationPath(config.Config().GetDefaultGrafanaConfig().GetPath(resourceType), clearOutput, filepath.Dir(v))
 	}
 	return v
 }

internal/service/common_test.go

@@ -51,25 +51,25 @@ func TestSlug(t *testing.T) {
 
 func TestUserPath(t *testing.T) {
 	fixEnvironment(t)
-	userPath := BuildResourceFolder("", config.UserResource, false)
+	userPath := BuildResourceFolder("", config.UserResource, false, false)
 	assert.Equal(t, "test/data/users/", userPath)
 }
 
 func TestBuildDashboardPath(t *testing.T) {
 	fixEnvironment(t)
-	result := BuildResourceFolder("General", config.DashboardResource, false)
+	result := BuildResourceFolder("General", config.DashboardResource, false, false)
 	assert.Equal(t, "test/data/org_your-org/dashboards/General", result)
 }
 
 func TestBuildFolderSourcePath(t *testing.T) {
 	fixEnvironment(t)
-	result := buildResourcePath(slug.Make("Some Folder"), config.FolderResource, false)
+	result := buildResourcePath(slug.Make("Some Folder"), config.FolderResource, false, false)
 	assert.Equal(t, "test/data/org_your-org/folders/some-folder.json", result)
 }
 
 func TestBuildDataSourcePath(t *testing.T) {
 	fixEnvironment(t)
 
-	result := buildResourcePath(slug.Make("My DS"), config.ConnectionResource, false)
+	result := buildResourcePath(slug.Make("My DS"), config.ConnectionResource, false, false)
 	assert.Equal(t, "test/data/org_your-org/connections/my-ds.json", result)
 }

internal/service/connection_permissions.go

@@ -77,7 +77,7 @@ func (s *DashNGoImpl) DownloadConnectionPermissions(filter filters.Filter) []str
 			slog.Error("unable to marshall json ", "err", err.Error(), "connectionName", connection.Connection.Name)
 			continue
 		}
-		dsPath := buildResourcePath(slug.Make(connection.Connection.Name), config.ConnectionPermissionResource, s.isLocal())
+		dsPath := buildResourcePath(slug.Make(connection.Connection.Name), config.ConnectionPermissionResource, s.isLocal(), s.globalConf.ClearOutput)
 		if err = s.storage.WriteFile(dsPath, dsPacked); err != nil {
 			slog.Error("unable to write file. ", "filename", slug.Make(connection.Connection.Name), "error", err.Error())
 		} else {

internal/service/connections.go

@@ -76,7 +76,7 @@ func (s *DashNGoImpl) DownloadConnections(filter filters.Filter) []string {
 			continue
 		}
 
-		dsPath := buildResourcePath(slug.Make(ds.Name), config.ConnectionResource, s.isLocal())
+		dsPath := buildResourcePath(slug.Make(ds.Name), config.ConnectionResource, s.isLocal(), s.globalConf.ClearOutput)
 
 		if err = s.storage.WriteFile(dsPath, dsPacked); err != nil {
 			slog.Error("Unable to write file", "filename", slug.Make(ds.Name), "err", err)

internal/service/dashboard_permissions.go

@@ -59,7 +59,7 @@ func (s *DashNGoImpl) DownloadDashboardPermissions(filterReq filters.Filter) ([]
 			continue
 		}
 
-		dsPath := fmt.Sprintf("%s/%s.json", BuildResourceFolder(link.Dashboard.FolderTitle, config.DashboardPermissionsResource, s.isLocal()), slug.Make(link.Dashboard.Title))
+		dsPath := fmt.Sprintf("%s/%s.json", BuildResourceFolder(link.Dashboard.FolderTitle, config.DashboardPermissionsResource, s.isLocal(), s.globalConf.ClearOutput), slug.Make(link.Dashboard.Title))
 		if err = s.storage.WriteFile(dsPath, dsPacked); err != nil {
 			slog.Error("unable to write file. ", "filename", slug.Make(link.Dashboard.Title), "error", err.Error())
 		} else {

internal/service/dashboards.go

@@ -370,7 +370,7 @@ func (s *DashNGoImpl) DownloadDashboards(filter filters.Filter) []string {
 			continue
 		}
 
-		fileName := buildDashboardFileName(link, metaData.GetPayload().Meta.Slug, folderUidMap, useNestedFolders, s.grafanaConf.Storage == "")
+		fileName := buildDashboardFileName(link, metaData.GetPayload().Meta.Slug, folderUidMap, useNestedFolders, s.isLocal(), s.globalConf.ClearOutput)
 		if err = s.storage.WriteFile(fileName, pretty.Pretty(rawBoard)); err != nil {
 			slog.Error("Unable to save dashboard to file\n", "err", err, "dashboard", metaData.GetPayload().Meta.Slug)
 		} else {
@@ -406,14 +406,14 @@ func getNestedFolder(folderTitle, folderUID string, folderUidMap map[string]*cus
 }
 
 // buildDashboardFileName for a given dashboard, a full nested folder path is constructed
-func buildDashboardFileName(db *models.Hit, dbSlug string, folderUidMap map[string]*customTypes.FolderDetails, nested, createDestination bool) string {
+func buildDashboardFileName(db *models.Hit, dbSlug string, folderUidMap map[string]*customTypes.FolderDetails, nested, createDestination, clearOutput bool) string {
 	var folderPath string
 	if nested {
 		folderPath = getNestedFolder(db.FolderTitle, db.FolderUID, folderUidMap)
 	} else {
 		folderPath = db.FolderTitle
 	}
-	fileName := fmt.Sprintf("%s/%s.json", BuildResourceFolder(folderPath, config.DashboardResource, createDestination), dbSlug)
+	fileName := fmt.Sprintf("%s/%s.json", BuildResourceFolder(folderPath, config.DashboardResource, createDestination, clearOutput), dbSlug)
 	return fileName
 }
 

internal/service/folders.go

@@ -86,7 +86,7 @@ func (s *DashNGoImpl) DownloadFolderPermissions(filter filters.Filter) []string
 		if fileName == "" {
 			fileName = folder.Title
 		}
-		dsPath := buildResourcePath(slug.Make(fileName), config.FolderPermissionResource, s.isLocal())
+		dsPath := buildResourcePath(slug.Make(fileName), config.FolderPermissionResource, s.isLocal(), s.globalConf.ClearOutput)
 		if err = s.storage.WriteFile(dsPath, dsPacked); err != nil {
 			slog.Error("Unable to write file", "err", err.Error(), "filename", slug.Make(folder.Title))
 		} else {
@@ -240,7 +240,7 @@ func (s *DashNGoImpl) DownloadFolders(filter filters.Filter) []string {
 			slog.Error("Unable to serialize data to JSON", "err", err, "folderName", folder.Title)
 			continue
 		}
-		dsPath := buildResourcePath(folder.Title, config.FolderResource, s.isLocal())
+		dsPath := buildResourcePath(folder.Title, config.FolderResource, s.isLocal(), s.globalConf.ClearOutput)
 
 		if !s.checkFolderName(folder.Title) {
 			slog.Warn("Folder has an invalid character and is not supported, skipping folder", "folderName", folder.Title)
@@ -253,7 +253,7 @@ func (s *DashNGoImpl) DownloadFolders(filter filters.Filter) []string {
 				dsPath = strings.Replace(dsPath, slugFolder, folder.NestedPath, 1)
 				baseFolder := filepath.Dir(dsPath)
 				if s.isLocal() {
-					tools.CreateDestinationPath(baseFolder)
+					tools.CreateDestinationPath(baseFolder, false, "")
 				}
 			}
 		}

internal/service/gdg_api.go

@@ -11,7 +11,6 @@ import (
 	"github.com/esnet/gdg/internal/api"
 	"github.com/esnet/gdg/internal/config"
 	"github.com/esnet/gdg/internal/storage"
-	"github.com/spf13/viper"
 )
 
 var (
@@ -20,12 +19,9 @@ var (
 )
 
 type DashNGoImpl struct {
-	extended *api.ExtendedApi
-
+	extended    *api.ExtendedApi
 	grafanaConf *config.GrafanaConfig
-	configRef   *viper.Viper
-	debug       bool
-	apiDebug    bool
+	globalConf  *config.AppGlobals
 	storage     storage.Storage
 }
 
@@ -35,16 +31,14 @@ var DefaultConfigProvider config.Provider = func() *config.Configuration {
 
 func setupConfigData(cfg *config.Configuration, obj *DashNGoImpl) {
 	obj.grafanaConf = cfg.GetDefaultGrafanaConfig()
-	obj.configRef = cfg.GetViperConfig()
-	obj.debug = cfg.IsDebug()
-	obj.apiDebug = cfg.IsApiDebug()
+	obj.globalConf = cfg.GetGDGConfig().GetAppGlobals()
 }
 
 func newInstance() *DashNGoImpl {
 	obj := &DashNGoImpl{}
 	setupConfigData(config.Config(), obj)
 
-	if obj.apiDebug {
+	if obj.globalConf.ApiDebug {
 		err := os.Setenv("DEBUG", "1")
 		if err != nil {
 			slog.Debug("unable to set debug env value", slog.Any("err", err))

internal/service/libraryelements.go

@@ -97,7 +97,7 @@ func (s *DashNGoImpl) DownloadLibraryElements(filter filters.Filter) []string {
 			folderName = val
 		}
 
-		libraryPath := fmt.Sprintf("%s/%s.json", BuildResourceFolder(folderName, config.LibraryElementResource, s.isLocal()), slug.Make(item.Name))
+		libraryPath := fmt.Sprintf("%s/%s.json", BuildResourceFolder(folderName, config.LibraryElementResource, s.isLocal(), s.globalConf.ClearOutput), slug.Make(item.Name))
 
 		if err = s.storage.WriteFile(libraryPath, dsPacked); err != nil {
 			slog.Error("Unable to write file", "err", err, "library-element", slug.Make(item.Name))

internal/service/login.go

@@ -73,7 +73,7 @@ func (s *DashNGoImpl) getNewClient(opts ...NewClientOpts) (*client.GrafanaHTTPAP
 		Schemes:      []string{u.Scheme},
 		NumRetries:   config.Config().GetGDGConfig().GetAppGlobals().RetryCount,
 		RetryTimeout: config.Config().GetGDGConfig().GetAppGlobals().GetRetryTimeout(),
-		Debug:        s.apiDebug,
+		Debug:        s.globalConf.ApiDebug,
 	}
 
 	// If more than one opts is passed, depend on the caller to setup his required configuration
@@ -97,7 +97,7 @@ func (s *DashNGoImpl) GetClient() *client.GrafanaHTTPAPI {
 	if s.grafanaConf.APIToken != "" {
 		grafanaClient, _ := s.getNewClient(func(clientCfg *client.TransportConfig) {
 			clientCfg.APIKey = s.grafanaConf.APIToken
-			clientCfg.Debug = s.apiDebug
+			clientCfg.Debug = s.globalConf.ApiDebug
 		})
 		return grafanaClient
 	} else {
@@ -123,7 +123,7 @@ func (s *DashNGoImpl) GetAdminClient() *client.GrafanaHTTPAPI {
 func (s *DashNGoImpl) getDefaultBasicOpts() []NewClientOpts {
 	return []NewClientOpts{func(clientCfg *client.TransportConfig) {
 		clientCfg.BasicAuth = url.UserPassword(s.grafanaConf.UserName, s.grafanaConf.Password)
-		clientCfg.Debug = s.apiDebug
+		clientCfg.Debug = s.globalConf.ApiDebug
 	}}
 }
 

internal/service/organizations.go

@@ -221,7 +221,7 @@ func (s *DashNGoImpl) DownloadOrganizations(filter filters.Filter) []string {
 			slog.Error("Unable to serialize organization object", "err", err, "organization", organisation.Organization.Name)
 			continue
 		}
-		dsPath := buildResourcePath(slug.Make(organisation.Organization.Name), config.OrganizationResource, s.isLocal())
+		dsPath := buildResourcePath(slug.Make(organisation.Organization.Name), config.OrganizationResource, s.isLocal(), s.globalConf.ClearOutput)
 		if err = s.storage.WriteFile(dsPath, dsPacked); err != nil {
 			slog.Error("Unable to write file", "err", err.Error(), "organization", slug.Make(organisation.Organization.Name))
 		} else {

internal/service/teams.go

@@ -51,7 +51,7 @@ func NewTeamFilter(entries ...string) filters.Filter {
 func (s *DashNGoImpl) DownloadTeams(filter filters.Filter) map[*models.TeamDTO][]*models.TeamMemberDTO {
 	teamListing := maps.Keys(s.ListTeams(filter))
 	importedTeams := make(map[*models.TeamDTO][]*models.TeamMemberDTO)
-	teamPath := BuildResourceFolder("", config.TeamResource, s.isLocal())
+	teamPath := BuildResourceFolder("", config.TeamResource, s.isLocal(), s.globalConf.ClearOutput)
 	for ndx, team := range teamListing {
 		// Teams
 		teamFileName := filepath.Join(teamPath, GetSlug(team.Name), "team.json")

internal/service/user.go

@@ -60,7 +60,7 @@ func (s *DashNGoImpl) DownloadUsers(filter filters.Filter) []string {
 	userListing := s.ListUsers(filter)
 	var importedUsers []string
 
-	userPath := BuildResourceFolder("", config.UserResource, s.isLocal())
+	userPath := BuildResourceFolder("", config.UserResource, s.isLocal(), s.globalConf.ClearOutput)
 	for ndx, user := range userListing {
 		if s.isAdminUser(user.ID, user.Name) {
 			slog.Info("Skipping admin super user")

internal/storage/const.go

@@ -2,16 +2,23 @@ package storage
 
 type ContextStorage string
 
+type Type string
+
+func (t Type) String() string {
+	return string(t)
+}
+
 const (
 	Context = ContextStorage("storage")
 	// Cloud Specific const
-	CloudType  = "cloud_type"
-	BucketName = "bucket_name"
-	Prefix     = "prefix"
-	Custom     = "custom"
-	AccessId   = "access_id"
-	SecretKey  = "secret_key"
-	Endpoint   = "endpoint"
-	Region     = "region"
-	InitBucket = "init_bucket"
+	CloudType             = "cloud_type"
+	BucketName            = "bucket_name"
+	Prefix                = "prefix"
+	Custom                = "custom"
+	AccessId              = "access_id"
+	SecretKey             = "secret_key"
+	Endpoint              = "endpoint"
+	Region                = "region"
+	InitBucket            = "init_bucket"
+	LocalStorageType Type = "LocalStorage"
 )

internal/storage/storage_local.go

@@ -64,7 +64,7 @@ func (s *LocalStorage) WriteFile(filename string, data []byte) error {
 }
 
 func (s *LocalStorage) Name() string {
-	return "LocalStorage"
+	return LocalStorageType.String()
 }
 
 func (s *LocalStorage) FindAllFiles(folder string, fullPath bool) ([]string, error) {

internal/templating/templating.go

@@ -88,7 +88,7 @@ func (t *templateImpl) Generate(templateName string) (map[string][]string, error
 				slog.Any("data", outputEntity.TemplateData),
 			)
 			grafana.OrganizationName = outputEntity.OrganizationName
-			outputPath := service.BuildResourceFolder(outputEntity.Folder, config.DashboardResource, true)
+			outputPath := service.BuildResourceFolder(outputEntity.Folder, config.DashboardResource, true, false)
 			// Merge two maps.
 			tmpl, err := template.New("").Funcs(fns).Parse(string(templateData))
 			if err != nil {
@@ -97,7 +97,7 @@ func (t *templateImpl) Generate(templateName string) (map[string][]string, error
 			}
 
 			// Create new file.
-			tools.CreateDestinationPath(outputPath)
+			tools.CreateDestinationPath(outputPath, false, "")
 			dashboardName := entity.TemplateName
 			if outputEntity.DashboardName != "" {
 				dashboardName = service.GetSlug(outputEntity.DashboardName)

internal/tools/generics_tooling.go

@@ -3,7 +3,9 @@ package tools
 import (
 	"encoding/json"
 	"log"
+	"log/slog"
 	"os"
+	"sync"
 )
 
 func DeepCopy[T any](value T) (*T, error) {
@@ -20,8 +22,22 @@ func DeepCopy[T any](value T) (*T, error) {
 	return clone, nil
 }
 
+var (
+	syncOnce = new(sync.Once)
+	syncMap  = new(sync.Map)
+)
+
 // CreateDestinationPath Handle osMkdir Errors
-func CreateDestinationPath(v string) {
+func CreateDestinationPath(folderName string, output bool, v string) {
+	if output {
+		syncOnce.Do(func() {
+			clearBackup := os.RemoveAll(folderName)
+			if clearBackup != nil {
+				slog.Warn("Unable to remove previous backup at location", "location", v)
+			}
+		})
+	}
+
 	err := os.MkdirAll(v, 0o750)
 	if err != nil {
 		log.Fatalf("unable to create path %s, err: %s", v, err.Error())