Skip to content

Commit

Permalink
make yaml file source as a common file source (#49)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tianxiaoliang
tianxiaoliang authored Nov 9, 2018
1 parent 467698c commit 4bc93b3
Show file tree
Hide file tree
Showing 8 changed files with 65 additions and 83 deletions.
8 changes: 4 additions & 4 deletions archaius.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,10 @@ type ConfigCenterInfo struct {
func initFileSource(o *Options) (core.ConfigSource, error) {
files := make([]string, 0)
// created file source object
fs = filesource.NewYamlConfigurationSource()
fs = filesource.NewFileSource()
// adding all files with file source
for _, v := range o.RequiredFiles {
if err := fs.AddFileSource(v, filesource.DefaultFilePriority, o.FileHandler); err != nil {
if err := fs.AddFile(v, filesource.DefaultFilePriority, o.FileHandler); err != nil {
openlogging.GetLogger().Errorf("add file source error [%s].", err.Error())
return nil, err
}
Expand All @@ -60,7 +60,7 @@ func initFileSource(o *Options) (core.ConfigSource, error) {
openlogging.GetLogger().Infof("[%s] not exist", v)
continue
}
if err := fs.AddFileSource(v, filesource.DefaultFilePriority, o.FileHandler); err != nil {
if err := fs.AddFile(v, filesource.DefaultFilePriority, o.FileHandler); err != nil {
openlogging.GetLogger().Infof("%v", err)
return nil, err
}
Expand Down Expand Up @@ -295,7 +295,7 @@ func AddFile(file string, opts ...FileOption) error {
for _, f := range opts {
f(o)
}
if err := fs.AddFileSource(file, filesource.DefaultFilePriority, o.Handler); err != nil {
if err := fs.AddFile(file, filesource.DefaultFilePriority, o.Handler); err != nil {
return err
}
return factory.Refresh(fs.GetSourceName())
Expand Down
4 changes: 2 additions & 2 deletions configurationfactory_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,8 @@ func TestConfigFactory(t *testing.T) {
}

fmt.Println("Adding filesource to the configfactroy")
fsource := filesource.NewYamlConfigurationSource()
fsource.AddFileSource(filename1, 0, nil)
fsource := filesource.NewFileSource()
fsource.AddFile(filename1, 0, nil)
factory.AddSource(fsource)

fmt.Println("Generating event through testsource(priority 1)")
Expand Down
6 changes: 3 additions & 3 deletions core/config-manager/configurationmanager_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,8 +307,8 @@ cse:
dispatcher := eventsystem.NewDispatcher()
confmanager := configmanager.NewConfigurationManager(dispatcher)

fsource := filesource.NewYamlConfigurationSource()
fsource.AddFileSource(file1, 0, nil)
fsource := filesource.NewFileSource()
fsource.AddFile(file1, 0, nil)

confmanager.AddSource(fsource, fsource.GetPriority())
time.Sleep(2 * time.Second)
Expand All @@ -326,7 +326,7 @@ cse:

confmanager = configmanager.NewConfigurationManager(dispatcher)

fsource.AddFileSource(lbFileName, 0, nil)
fsource.AddFile(lbFileName, 0, nil)

confmanager.AddSource(fsource, fsource.GetPriority())
time.Sleep(2 * time.Second)
Expand Down
8 changes: 4 additions & 4 deletions examples/hack/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,11 @@ func main() {
"===========================================================\n")

// create file source object
fSource := filesource.NewYamlConfigurationSource()
fSource := filesource.NewFileSource()
// add file in file source.
// file can be regular yaml file or directory like fSource.AddFileSource("./conf", 0)
// file can be regular yaml file or directory like fSource.AddFile("./conf", 0)
// second argument is priority of file
fSource.AddFileSource("./conf/name.yaml", 0, nil)
fSource.AddFile("./conf/name.yaml", 0, nil)
// add file source to go-archaius
ConfigFactory.AddSource(fSource)

Expand Down Expand Up @@ -123,7 +123,7 @@ func main() {
"\n========================\n")

//second argument is priority of file
fSource.AddFileSource("./conf/name.yaml", 0, nil)
fSource.AddFile("./conf/name.yaml", 0, nil)
// add file source to go-`archaius
ConfigFactory.AddSource(fSource)

Expand Down
4 changes: 2 additions & 2 deletions sources/file-source/file_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ import (

//FileHandler decide how to convert a file content into key values
//archaius will manage file content as those key values
type FileHandler func(content []byte) (map[string]interface{}, error)
type FileHandler func(filePath string, content []byte) (map[string]interface{}, error)

//Convert2JavaProps is a FileHandler
//it convert the yaml content into java props
func Convert2JavaProps(content []byte) (map[string]interface{}, error) {
func Convert2JavaProps(p string, content []byte) (map[string]interface{}, error) {
configMap := make(map[string]interface{})

ss := yaml.MapSlice{}
Expand Down
2 changes: 1 addition & 1 deletion sources/file-source/file_handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ b: 2
c:
d: 3
`)
m, err := Convert2JavaProps(b)
m, err := Convert2JavaProps("test.yaml", b)
assert.NoError(t, err)
assert.Equal(t, m["c.d"], 3)
}
88 changes: 35 additions & 53 deletions ...es/file-source/yamlconfigurationsource.go → sources/file-source/filesource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
"github.com/fsnotify/fsnotify"
"github.com/go-chassis/go-archaius/core"
"github.com/go-mesh/openlogging"
"gopkg.in/yaml.v2"
"strings"
"time"
)
Expand Down Expand Up @@ -61,7 +60,7 @@ type ConfigInfo struct {
Value interface{}
}

type yamlConfigurationSource struct {
type fileSource struct {
Configurations map[string]*ConfigInfo
files []file
fileHandlers map[string]FileHandler
Expand All @@ -79,41 +78,41 @@ type watch struct {
//files []string
watcher *fsnotify.Watcher
callback core.DynamicConfigCallback
fileSource *yamlConfigurationSource
fileSource *fileSource
sync.RWMutex
}

var _ core.ConfigSource = &yamlConfigurationSource{}
var _ FileSource = &yamlConfigurationSource{}
var _ core.ConfigSource = &fileSource{}
var _ FileSource = &fileSource{}

var fileConfigSource *yamlConfigurationSource
var fileConfigSource *fileSource

/*
accepts files and directories as file-source
1. Directory: all yaml files considered as file source
2. File: specified yaml file considered as file source
1. Directory: all files considered as file source
2. File: specified file considered as file source
TODO: Currently file sources priority not considered. if key conflicts then latest key will get considered
*/

//FileSource is a interface
type FileSource interface {
core.ConfigSource
AddFileSource(filePath string, priority uint32, handler FileHandler) error
AddFile(filePath string, priority uint32, handler FileHandler) error
}

//NewYamlConfigurationSource creates new yaml configuration
func NewYamlConfigurationSource() FileSource {
//NewFileSource creates a source which can handler local files
func NewFileSource() FileSource {
if fileConfigSource == nil {
fileConfigSource = new(yamlConfigurationSource)
fileConfigSource = new(fileSource)
fileConfigSource.files = make([]file, 0)
fileConfigSource.fileHandlers = make(map[string]FileHandler)
}

return fileConfigSource
}

func (fSource *yamlConfigurationSource) AddFileSource(p string, priority uint32, handle FileHandler) error {
func (fSource *fileSource) AddFile(p string, priority uint32, handle FileHandler) error {
path, err := filepath.Abs(p)
if err != nil {
return err
Expand All @@ -133,7 +132,7 @@ func (fSource *yamlConfigurationSource) AddFileSource(p string, priority uint32,
fileType := fileType(fs)
switch fileType {
case Directory:
// handle Directory input. Include all yaml files as file source.
// handle Directory input. Include all files as file source.
err := fSource.handleDirectory(fs, priority, handle)
if err != nil {
openlogging.GetLogger().Errorf("Failed to handle directory [%s] %s", path, err)
Expand All @@ -158,7 +157,7 @@ func (fSource *yamlConfigurationSource) AddFileSource(p string, priority uint32,
return nil
}

func (fSource *yamlConfigurationSource) isFileSrcExist(filePath string) bool {
func (fSource *fileSource) isFileSrcExist(filePath string) bool {
var exist bool
for _, file := range fSource.files {
if filePath == file.filePath {
Expand Down Expand Up @@ -186,7 +185,7 @@ func fileType(fs *os.File) FileSourceTypes {
return InvalidFileType
}

func (fSource *yamlConfigurationSource) handleDirectory(dir *os.File, priority uint32, handle FileHandler) error {
func (fSource *fileSource) handleDirectory(dir *os.File, priority uint32, handle FileHandler) error {

filesInfo, err := dir.Readdir(-1)
if err != nil {
Expand Down Expand Up @@ -214,16 +213,16 @@ func (fSource *yamlConfigurationSource) handleDirectory(dir *os.File, priority u
return nil
}

func (fSource *yamlConfigurationSource) handleFile(file *os.File, priority uint32, handle FileHandler) error {
yamlContent, err := ioutil.ReadFile(file.Name())
func (fSource *fileSource) handleFile(file *os.File, priority uint32, handle FileHandler) error {
Content, err := ioutil.ReadFile(file.Name())
if err != nil {
return err
}
var config map[string]interface{}
if handle != nil {
config, err = handle(yamlContent)
config, err = handle(file.Name(), Content)
} else {
config, err = Convert2JavaProps(yamlContent)
config, err = Convert2JavaProps(file.Name(), Content)
}
if err != nil {
return fmt.Errorf("failed to pull configurations from [%s] file, %s", file.Name(), err)
Expand All @@ -244,7 +243,7 @@ func (fSource *yamlConfigurationSource) handleFile(file *os.File, priority uint3
return nil
}

func (fSource *yamlConfigurationSource) handlePriority(filePath string, priority uint32) error {
func (fSource *fileSource) handlePriority(filePath string, priority uint32) error {
fSource.Lock()
newFilePriority := make([]file, 0)
var prioritySet bool
Expand Down Expand Up @@ -273,24 +272,7 @@ func (fSource *yamlConfigurationSource) handlePriority(filePath string, priority
return nil
}

func (fSource *yamlConfigurationSource) pullYamlFileConfig(fileName string) (map[string]interface{}, error) {
configMap := make(map[string]interface{})
yamlContent, err := ioutil.ReadFile(fileName)
if err != nil {
return nil, err
}

ss := yaml.MapSlice{}
err = yaml.Unmarshal([]byte(yamlContent), &ss)
if err != nil {
return nil, fmt.Errorf("yaml unmarshal [%s] failed, %s", fileName, err)
}
configMap = retrieveItems("", ss)

return configMap, nil
}

func (fSource *yamlConfigurationSource) GetConfigurations() (map[string]interface{}, error) {
func (fSource *fileSource) GetConfigurations() (map[string]interface{}, error) {
configMap := make(map[string]interface{})

fSource.Lock()
Expand All @@ -307,7 +289,7 @@ func (fSource *yamlConfigurationSource) GetConfigurations() (map[string]interfac
return configMap, nil
}

func (fSource *yamlConfigurationSource) GetConfigurationByKey(key string) (interface{}, error) {
func (fSource *fileSource) GetConfigurationByKey(key string) (interface{}, error) {
fSource.Lock()
defer fSource.Unlock()

Expand All @@ -325,15 +307,15 @@ func (fSource *yamlConfigurationSource) GetConfigurationByKey(key string) (inter
return nil, errors.New("key does not exist")
}

func (*yamlConfigurationSource) GetSourceName() string {
func (*fileSource) GetSourceName() string {
return FileConfigSourceConst
}

func (*yamlConfigurationSource) GetPriority() int {
func (*fileSource) GetPriority() int {
return fileSourcePriority
}

func (fSource *yamlConfigurationSource) DynamicConfigHandler(callback core.DynamicConfigCallback) error {
func (fSource *fileSource) DynamicConfigHandler(callback core.DynamicConfigCallback) error {
if callback == nil {
return errors.New("call back can not be nil")
}
Expand All @@ -350,7 +332,7 @@ func (fSource *yamlConfigurationSource) DynamicConfigHandler(callback core.Dynam
return nil
}

func newWatchPool(callback core.DynamicConfigCallback, cfgSrc *yamlConfigurationSource) (*watch, error) {
func newWatchPool(callback core.DynamicConfigCallback, cfgSrc *fileSource) (*watch, error) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
openlogging.GetLogger().Error("New file watcher failed:" + err.Error())
Expand Down Expand Up @@ -431,13 +413,13 @@ func (wth *watch) watchFile() {
if handle == nil {
handle = Convert2JavaProps
}
yamlContent, err := ioutil.ReadFile(event.Name)
content, err := ioutil.ReadFile(event.Name)
if err != nil {
openlogging.GetLogger().Error("yaml parsing error " + err.Error())
openlogging.GetLogger().Error("read file error " + err.Error())
continue
}

newConf, err := handle(yamlContent)
newConf, err := handle(event.Name, content)
if err != nil {
openlogging.GetLogger().Error("convert error " + err.Error())
continue
Expand All @@ -456,7 +438,7 @@ func (wth *watch) watchFile() {

}

func (fSource *yamlConfigurationSource) compareUpdate(newconf map[string]interface{}, filePath string) []*core.Event {
func (fSource *fileSource) compareUpdate(newconf map[string]interface{}, filePath string) []*core.Event {
events := make([]*core.Event, 0)
fileConfs := make(map[string]*ConfigInfo)
if fSource == nil {
Expand Down Expand Up @@ -542,7 +524,7 @@ func (fSource *yamlConfigurationSource) compareUpdate(newconf map[string]interfa
return events
}

func (fSource *yamlConfigurationSource) addOrCreateConf(fileConfs map[string]*ConfigInfo, newconf map[string]interface{},
func (fSource *fileSource) addOrCreateConf(fileConfs map[string]*ConfigInfo, newconf map[string]interface{},
events []*core.Event, filePath string) (map[string]*ConfigInfo, []*core.Event) {
for key, value := range newconf {
handled := false
Expand Down Expand Up @@ -573,7 +555,7 @@ func (fSource *yamlConfigurationSource) addOrCreateConf(fileConfs map[string]*Co
// return strings.Split(configKey, `#`)
//}

func (fSource *yamlConfigurationSource) Cleanup() error {
func (fSource *fileSource) Cleanup() error {

fSource.filelock.Lock()
defer fSource.filelock.Unlock()
Expand All @@ -596,14 +578,14 @@ func (fSource *yamlConfigurationSource) Cleanup() error {
return nil
}

func (fSource *yamlConfigurationSource) GetConfigurationByKeyAndDimensionInfo(key, di string) (interface{}, error) {
func (fSource *fileSource) GetConfigurationByKeyAndDimensionInfo(key, di string) (interface{}, error) {
return nil, nil
}

func (fSource *yamlConfigurationSource) AddDimensionInfo(dimensionInfo string) (map[string]string, error) {
func (fSource *fileSource) AddDimensionInfo(dimensionInfo string) (map[string]string, error) {
return nil, nil
}

func (fSource *yamlConfigurationSource) GetConfigurationsByDI(dimensionInfo string) (map[string]interface{}, error) {
func (fSource *fileSource) GetConfigurationsByDI(dimensionInfo string) (map[string]interface{}, error) {
return nil, nil
}
Loading

0 comments on commit 4bc93b3

Please sign in to comment.