Cleanup of pelican config files

Fixes issues with the 3 config files used with Pelican: defaults.yaml, osdf.yaml, and pelican.yaml. Before this fix there was a bit of a mess of where the configs are set up with viper. This fix moves InitConfig to config.go and sets up all the config files in the priority they need to be with pelican.yaml taking higher precedence than defaults.yaml. Also fixed issue in object_copy.go with some weird issues with the -d flag. Only need to check for the -d flag with stashcp since it does not go through root, any other modes are checked within root/initconfig and trigger debug mode.

cmd/config_mgr.go

@@ -28,7 +28,6 @@ import (
 	"github.com/pelicanplatform/pelican/client"
 	"github.com/pelicanplatform/pelican/config"
 	"github.com/pelicanplatform/pelican/namespaces"
-	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 )
 
@@ -304,11 +303,3 @@ func init() {
 	rootConfigCmd.AddCommand(prefixCmd)
 	rootConfigCmd.AddCommand(tokenCmd)
 }
-
-func setLogging(logLevel log.Level) {
-	textFormatter := log.TextFormatter{}
-	textFormatter.DisableLevelTruncation = true
-	textFormatter.FullTimestamp = true
-	log.SetFormatter(&textFormatter)
-	log.SetLevel(logLevel)
-}

cmd/object_copy.go

@@ -32,6 +32,8 @@ import (
 )
 
 var (
+	execName string
+
 	copyCmd = &cobra.Command{
 		Use:   "copy {source ...} {destination}",
 		Short: "Copy a file to/from a Pelican federation",
@@ -40,19 +42,19 @@ var (
 )
 
 func init() {
-	exec_name := filepath.Base(os.Args[0])
+	execName = filepath.Base(os.Args[0])
 	// Take care of our Windows users
-	exec_name = strings.TrimSuffix(exec_name, ".exe")
+	execName = strings.TrimSuffix(execName, ".exe")
 	// Being case-insensitive
-	exec_name = strings.ToLower(exec_name)
+	execName = strings.ToLower(execName)
 	flagSet := copyCmd.Flags()
 	flagSet.StringP("cache", "c", "", "Cache to use")
 	flagSet.StringP("token", "t", "", "Token file to use for transfer")
 	flagSet.BoolP("recursive", "r", false, "Recursively copy a directory.  Forces methods to only be http to get the freshest directory contents")
 	flagSet.StringP("cache-list-name", "n", "xroot", "(Deprecated) Cache list to use, currently either xroot or xroots; may be ignored")
 	flagSet.Lookup("cache-list-name").Hidden = true
 	// All the deprecated or hidden flags that are only relevant if we are in historical "stashcp mode"
-	if exec_name == "stashcp" {
+	if execName == "stashcp" {
 		copyCmd.Use = "stashcp {source ...} {destination}"
 		copyCmd.Short = "Copy a file to/from the OSDF"
 		flagSet.Lookup("cache-list-name").Hidden = false // Expose the help for this option
@@ -78,10 +80,13 @@ func copyMain(cmd *cobra.Command, args []string) {
 
 	client.ObjectClientOptions.Version = version
 
-	if val, err := cmd.Flags().GetBool("debug"); err == nil && val {
-		setLogging(log.DebugLevel)
-	} else {
-		setLogging(log.ErrorLevel)
+	// Need to check just stashcp since it does not go through root, the other modes get checked there
+	if execName == "stashcp" {
+		if val, err := cmd.Flags().GetBool("debug"); err == nil && val {
+			config.SetLogging(log.DebugLevel)
+		} else {
+			config.SetLogging(log.ErrorLevel)
+		}
 	}
 
 	err := config.InitClient()

cmd/plugin.go

@@ -79,7 +79,7 @@ func stashPluginMain(args []string) {
 	client.ObjectClientOptions.ProgressBars = false
 	client.ObjectClientOptions.Version = version
 	client.ObjectClientOptions.Plugin = true
-	setLogging(log.PanicLevel)
+	config.SetLogging(log.PanicLevel)
 	methods := []string{"http"}
 	var infile, outfile, testCachePath string
 	var useOutFile bool = false
@@ -114,7 +114,7 @@ func stashPluginMain(args []string) {
 			useOutFile = true
 			log.Debugln("Outfile:", outfile)
 		} else if args[0] == "-d" {
-			setLogging(log.DebugLevel)
+			config.SetLogging(log.DebugLevel)
 		} else if args[0] == "-get-caches" {
 			if len(args) < 2 {
 				log.Errorln("-get-caches requires an argument")

cmd/root.go

@@ -20,13 +20,10 @@ package main
 
 import (
 	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
 
 	"github.com/pelicanplatform/pelican/config"
-	"github.com/pelicanplatform/pelican/param"
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
@@ -87,7 +84,7 @@ func Execute() {
 
 func init() {
 
-	cobra.OnInitialize(initConfig)
+	cobra.OnInitialize(config.InitConfig)
 	rootCmd.AddCommand(objectCmd)
 	objectCmd.CompletionOptions.DisableDefaultCmd = true
 	rootCmd.AddCommand(directorCmd)
@@ -115,41 +112,14 @@ func init() {
 
 	rootCmd.PersistentFlags().BoolVarP(&outputJSON, "json", "", false, "output results in JSON format")
 	rootCmd.CompletionOptions.DisableDefaultCmd = true
-}
 
-func initConfig() {
-	if cfgFile != "" {
-		viper.SetConfigFile(cfgFile)
-	} else {
-		home, err := os.UserHomeDir()
-		if err != nil {
-			log.Warningln("No home directory found for user -- will check for configuration yaml in /etc/pelican/")
-		}
-
-		viper.AddConfigPath(filepath.Join(home, ".config", "pelican"))
-		viper.AddConfigPath(filepath.Join("/etc", "pelican"))
-		viper.SetConfigType("yaml")
-		viper.SetConfigName("pelican")
+	if err := viper.BindPFlag("config", rootCmd.PersistentFlags().Lookup("config")); err != nil {
+		panic(err)
 	}
 	if err := viper.BindPFlag("Debug", rootCmd.PersistentFlags().Lookup("debug")); err != nil {
 		panic(err)
 	}
 	if err := viper.BindPFlag("Server.Port", portFlag); err != nil {
 		panic(err)
 	}
-
-	viper.SetEnvPrefix(config.GetPreferredPrefix())
-	viper.AutomaticEnv()
-	// This line allows viper to use an env var like ORIGIN_VALUE to override the viper string "Origin.Value"
-	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
-	if err := viper.MergeInConfig(); err != nil {
-		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
-			cobra.CheckErr(err)
-		}
-	}
-
-	setLogging(log.ErrorLevel)
-	if param.Debug.GetBool() {
-		setLogging(log.DebugLevel)
-	}
 }

config/config.go

@@ -41,6 +41,7 @@ import (
 	"github.com/pelicanplatform/pelican/param"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
@@ -293,6 +294,50 @@ func GetTransport() *http.Transport {
 	return transport
 }
 
+func InitConfig() {
+	if viper.GetString("config") != "" {
+		viper.SetConfigFile(viper.GetString("config")) // TODO: bind this to a viper flag/varible in root.go and access here
+	} else {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		log.Warningln("No home directory found for user -- will check for configuration yaml in /etc/pelican/")
+	}
+		// 1) Set up defaults.yaml
+		err = viper.MergeConfig(strings.NewReader(defaultsYaml))
+		if err != nil {
+			cobra.CheckErr(err)
+		}
+		// 2) Set up osdf.yaml (if needed)
+		prefix := GetPreferredPrefix()
+		if prefix == "OSDF" {
+			err := viper.MergeConfig(strings.NewReader(osdfDefaultsYaml))
+			if err != nil {
+				cobra.CheckErr(err)
+			}
+		}
+		// 3) Set up pelican.yaml
+		viper.AddConfigPath(filepath.Join(home, ".config", "pelican"))
+		viper.AddConfigPath(filepath.Join("/etc", "pelican"))
+		viper.SetConfigType("yaml")
+		viper.SetConfigName("pelican")
+	}
+
+	viper.SetEnvPrefix(GetPreferredPrefix())
+	viper.AutomaticEnv()
+	// This line allows viper to use an env var like ORIGIN_VALUE to override the viper string "Origin.Value"
+	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+	if err := viper.MergeInConfig(); err != nil {
+		if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
+			cobra.CheckErr(err)
+		}
+	}
+	if param.Debug.GetBool() {
+		SetLogging(log.DebugLevel)
+	} else {
+		SetLogging(log.ErrorLevel)
+	}
+}
+
 func InitServer() error {
 	configDir := viper.GetString("ConfigDir")
 	viper.SetConfigType("yaml")
@@ -366,26 +411,13 @@ func InitServer() error {
 	viper.SetDefault("Server.Hostname", hostname)
 	viper.SetDefault("Xrootd.Sitename", hostname)
 
-	err = viper.MergeConfig(strings.NewReader(defaultsYaml))
-	if err != nil {
-		return err
-	}
-
 	port := param.Xrootd_Port.GetInt()
 	if port != 443 {
 		viper.SetDefault("Origin.Url", fmt.Sprintf("https://%v:%v", param.Server_Hostname.GetString(), port))
 	} else {
 		viper.SetDefault("Origin.Url", fmt.Sprintf("https://%v", param.Server_Hostname.GetString()))
 	}
 
-	prefix := GetPreferredPrefix()
-	if prefix == "OSDF" {
-		err := viper.MergeConfig(strings.NewReader(osdfDefaultsYaml))
-		if err != nil {
-			return err
-		}
-	}
-
 	setupTransport()
 
 	return nil
@@ -509,3 +541,11 @@ func InitClient() error {
 
 	return DiscoverFederation()
 }
+
+func SetLogging(logLevel log.Level) {
+	textFormatter := log.TextFormatter{}
+	textFormatter.DisableLevelTruncation = true
+	textFormatter.FullTimestamp = true
+	log.SetFormatter(&textFormatter)
+	log.SetLevel(logLevel)
+}

config/config_test.go

@@ -27,6 +27,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/pelicanplatform/pelican/param"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 )
@@ -111,3 +112,23 @@ func TestDialerTimeout(t *testing.T) {
 
 	viper.Set("Transport.Dialer.Timeout", time.Second*10)
 }
+
+func TestDefaultsYaml(t *testing.T) {
+	InitConfig() // should set up pelican.yaml and defaults.yaml
+	// create a temp config file to use
+	tempCfgFile, err := os.CreateTemp("", "pelican-*.yaml")
+	if err != nil {
+		t.Fatalf("Failed to make temp file: %v", err)
+	}
+	// Check if debug is false
+	assert.False(t, param.Debug.GetBool())
+	assert.False(t, viper.GetBool("Debug"))
+	viper.Set("Debug", true) // should write to temp config file (should have higher priority like how pelican.yaml behaves)
+	if err := viper.WriteConfigAs(tempCfgFile.Name()); err != nil {
+		t.Fatalf("Failed to write to config file: %v", err)
+	}
+	// Check if debug was set and is now true
+	assert.True(t, param.Debug.GetBool())
+	assert.True(t, viper.GetBool("Debug"))
+	viper.Reset()
+}