From a3807834e131c882c1710449fc836b010eb4a183 Mon Sep 17 00:00:00 2001
From: urso <steffen.siering@elastic.co>
Date: Thu, 7 Jul 2016 13:26:12 +0200
Subject: [PATCH] Use go-ucfg based variable expansion support

---
 libbeat/cfgfile/cfgfile.go        |  18 +-
 libbeat/cfgfile/cfgfile_test.go   |  74 +------
 libbeat/cfgfile/env.go            | 352 ------------------------------
 libbeat/common/config.go          |  43 ++--
 libbeat/tests/files/config.yml    |   4 +
 libbeat/tests/system/test_base.py |   3 +-
 6 files changed, 41 insertions(+), 453 deletions(-)
 delete mode 100644 libbeat/cfgfile/env.go

diff --git a/libbeat/cfgfile/cfgfile.go b/libbeat/cfgfile/cfgfile.go
index 3fb893a3f48b..f26822000d29 100644
--- a/libbeat/cfgfile/cfgfile.go
+++ b/libbeat/cfgfile/cfgfile.go
@@ -3,7 +3,6 @@ package cfgfile
 import (
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -60,22 +59,7 @@ func Load(path string) (*common.Config, error) {
 	if path == "" {
 		path = *configfile
 	}
-
-	fileContent, err := ioutil.ReadFile(path)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read %s: %v", path, err)
-	}
-	fileContent, err = expandEnv(filepath.Base(path), fileContent)
-	if err != nil {
-		return nil, err
-	}
-
-	config, err := common.NewConfigWithYAML(fileContent, path)
-	if err != nil {
-		return nil, fmt.Errorf("YAML config parsing failed on %s: %v", path, err)
-	}
-
-	return config, nil
+	return common.LoadFile(path)
 }
 
 // IsTestConfig returns whether or not this is configuration used for testing
diff --git a/libbeat/cfgfile/cfgfile_test.go b/libbeat/cfgfile/cfgfile_test.go
index c010f9725129..a8d1832345b8 100644
--- a/libbeat/cfgfile/cfgfile_test.go
+++ b/libbeat/cfgfile/cfgfile_test.go
@@ -11,7 +11,9 @@ import (
 )
 
 type TestConfig struct {
-	Output ElasticsearchConfig
+	Output     ElasticsearchConfig
+	Env        string `config:"env.test_key"`
+	EnvDefault string `config:"env.default"`
 }
 
 type ElasticsearchConfig struct {
@@ -25,6 +27,7 @@ type Connection struct {
 
 func TestRead(t *testing.T) {
 	absPath, err := filepath.Abs("../tests/files/")
+	os.Setenv("TEST_KEY", "test_value")
 
 	assert.NotNil(t, absPath)
 	assert.Nil(t, err)
@@ -34,72 +37,9 @@ func TestRead(t *testing.T) {
 	err = Read(config, absPath+"/config.yml")
 	assert.Nil(t, err)
 
-	// Access config
+	// validate
 	assert.Equal(t, "localhost", config.Output.Elasticsearch.Host)
-
-	// Chat that it is integer
 	assert.Equal(t, 9200, config.Output.Elasticsearch.Port)
-}
-
-func TestExpandEnv(t *testing.T) {
-	var tests = []struct {
-		in  string
-		out string
-		err string
-	}{
-		// Environment variables can be specified as ${env} only.
-		{"${y}", "y", ""},
-		{"$y", "$y", ""},
-
-		// Environment variables are case-sensitive.
-		{"${Y}", "", ""},
-
-		// Defaults can be specified.
-		{"x${Z:D}", "xD", ""},
-		{"x${Z:A B C D}", "xA B C D", ""}, // Spaces are allowed in the default.
-		{"x${Z:}", "x", ""},
-
-		// Un-matched braces cause an error.
-		{"x${Y ${Z:Z}", "", "unexpected character in variable expression: " +
-			"U+0020 ' ', expected a default value or closing brace"},
-
-		// Special environment variables are not replaced.
-		{"$*", "$*", ""},
-		{"${*}", "", "shell variable cannot start with U+002A '*'"},
-		{"$@", "$@", ""},
-		{"${@}", "", "shell variable cannot start with U+0040 '@'"},
-		{"$1", "$1", ""},
-		{"${1}", "", "shell variable cannot start with U+0031 '1'"},
-
-		{"", "", ""},
-		{"$$", "$$", ""},
-
-		{"${a_b}", "", ""}, // Underscores are allowed in variable names.
-
-		// ${} cannot be split across newlines.
-		{"hello ${name: world\n}", "", "unterminated brace"},
-
-		// To use a literal '${' you write '$${'.
-		{`password: "abc$${!"`, `password: "abc${!"`, ""},
-
-		// The full error contains the line number.
-		{"test:\n  name: ${var", "", "failure while expanding environment " +
-			"variables in config.yml at line=2, unterminated brace"},
-	}
-
-	for _, test := range tests {
-		os.Setenv("y", "y")
-		output, err := expandEnv("config.yml", []byte(test.in))
-
-		switch {
-		case test.err != "" && err == nil:
-			t.Errorf("Expected an error for test case %+v", test)
-		case test.err == "" && err != nil:
-			t.Errorf("Unexpected error for test case %+v, %v", test, err)
-		case err != nil:
-			assert.Contains(t, err.Error(), test.err)
-		default:
-			assert.Equal(t, test.out, string(output), "Input: %s", test.in)
-		}
-	}
+	assert.Equal(t, "test_value", config.Env)
+	assert.Equal(t, "default", config.EnvDefault)
 }
diff --git a/libbeat/cfgfile/env.go b/libbeat/cfgfile/env.go
deleted file mode 100644
index 07af365789a5..000000000000
--- a/libbeat/cfgfile/env.go
+++ /dev/null
@@ -1,352 +0,0 @@
-package cfgfile
-
-import (
-	"bytes"
-	"fmt"
-	"os"
-	"strings"
-	"unicode"
-	"unicode/utf8"
-)
-
-// Inspired by: https://cuddle.googlecode.com/hg/talk/lex.html
-
-const (
-	errUnterminatedBrace = "unterminated brace"
-)
-
-// item represents a token returned from the scanner.
-type item struct {
-	typ itemType // Token type, such as itemVariable.
-	pos int      // The starting position, in bytes, of this item in the input string.
-	val string   // Value, such as "${".
-}
-
-func (i item) String() string {
-	switch {
-	case i.typ == itemEOF:
-		return "EOF"
-	default:
-		return i.val
-	}
-}
-
-// itemType identifies the type of lex items.
-type itemType int
-
-// lex tokens.
-const (
-	itemError itemType = iota + 1
-	itemEscapedLeftDelim
-	itemLeftDelim
-	itemVariable
-	itemDefaultValue
-	itemRightDelim
-	itemText
-	itemEOF
-)
-
-const eof = -1
-
-// stateFn represents the state of the scanner as a function that returns the
-// next state.
-type stateFn func(*lexer) stateFn
-
-// lexer holds the state of the scanner.
-type lexer struct {
-	name    string    // used only for error reports.
-	input   string    // the string being scanned.
-	start   int       // start position of this item.
-	pos     int       // current position in the input.
-	width   int       // width of last rune read from input.
-	lastPos int       // position of most recent item returned by nextItem
-	items   chan item // channel of scanned items.
-}
-
-// next returns the next rune in the input.
-func (l *lexer) next() rune {
-	if int(l.pos) >= len(l.input) {
-		l.width = 0
-		return eof
-	}
-	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
-	l.width = w
-	l.pos += l.width
-	return r
-}
-
-// peek returns but does not consume the next rune in the input.
-func (l *lexer) peek() rune {
-	r := l.next()
-	l.backup()
-	return r
-}
-
-// backup steps back one rune. Can only be called once per call of next.
-func (l *lexer) backup() {
-	l.pos -= l.width
-}
-
-// emit passes an item back to the client.
-func (l *lexer) emit(t itemType) {
-	l.items <- item{t, l.start, l.input[l.start:l.pos]}
-	l.start = l.pos
-}
-
-// ignore skips over the pending input before this point.
-func (l *lexer) ignore() {
-	l.start = l.pos
-}
-
-// lineNumber reports which line we're on, based on the position of
-// the previous item returned by nextItem. Doing it this way
-// means we don't have to worry about peek double counting.
-func (l *lexer) lineNumber() int {
-	return 1 + strings.Count(l.input[:l.lastPos], "\n")
-}
-
-// errorf returns an error token and terminates the scan by passing
-// back a nil pointer that will be the next state, terminating l.nextItem.
-func (l *lexer) errorf(format string, args ...interface{}) stateFn {
-	l.items <- item{itemError, l.start, fmt.Sprintf(format, args...)}
-	return nil
-}
-
-// nextItem returns the next item from the input.
-// Called by the parser, not in the lexing goroutine.
-func (l *lexer) nextItem() item {
-	item := <-l.items
-	l.lastPos = item.pos
-	return item
-}
-
-// run lexes the input by executing state functions until the state is nil.
-func (l *lexer) run() {
-	for state := lexText; state != nil; {
-		state = state(l)
-	}
-	close(l.items) // No more tokens will be delivered.
-}
-
-// state functions
-
-// token values.
-const (
-	leftDelim             = "${"
-	rightDelim            = '}'
-	defaultValueSeperator = ':'
-	escapedLeftDelim      = "$${"
-)
-
-// lexText scans until an opening action delimiter, "${".
-func lexText(l *lexer) stateFn {
-	for {
-		switch {
-		case strings.HasPrefix(l.input[l.pos:], escapedLeftDelim):
-			if l.pos > l.start {
-				l.emit(itemText)
-			}
-			return lexEscapedLeftDelim
-		case strings.HasPrefix(l.input[l.pos:], leftDelim):
-			if l.pos > l.start {
-				l.emit(itemText)
-			}
-			return lexLeftDelim
-		}
-
-		if l.next() == eof {
-			break
-		}
-	}
-	// Correctly reached EOF.
-	if l.pos > l.start {
-		l.emit(itemText)
-	}
-	l.emit(itemEOF)
-	return nil
-}
-
-// lexEscapedLeftDelim scans the escaped left delimiter, which is known to be
-// present.
-func lexEscapedLeftDelim(l *lexer) stateFn {
-	l.pos += len(escapedLeftDelim)
-	l.emit(itemEscapedLeftDelim)
-	return lexText
-}
-
-// lexLeftDelim scans the left delimiter, which is known to be present.
-func lexLeftDelim(l *lexer) stateFn {
-	l.pos += len(leftDelim)
-	l.emit(itemLeftDelim)
-	return lexVariable
-}
-
-// lexVariable scans a shell variable name which is alphanumeric and does not
-// start with a number or other special shell variable character.
-// The ${ has already been scanned.
-func lexVariable(l *lexer) stateFn {
-	var r rune = l.peek()
-	if isShellSpecialVar(r) {
-		return l.errorf("shell variable cannot start with %#U", r)
-	}
-	for {
-		r = l.next()
-		if !isAlphaNumeric(r) {
-			l.backup()
-			break
-		}
-	}
-	l.emit(itemVariable)
-	return lexDefaultValueOrRightDelim
-}
-
-// lexDefaultValueOrRightDelim scans for a default value for the variable
-// expansion or for the '}' to close the variable definition.
-func lexDefaultValueOrRightDelim(l *lexer) stateFn {
-	switch r := l.next(); {
-	case r == eof || isEndOfLine(r):
-		return l.errorf(errUnterminatedBrace)
-	case r == ':':
-		l.ignore()
-		return lexDefaultValue
-	case r == '}':
-		l.backup()
-		return lexRightDelim
-	default:
-		return l.errorf("unexpected character in variable expression: %#U, "+
-			"expected a default value or closing brace", r)
-	}
-}
-
-// lexRightDelim scans the right delimiter, which is known to be present.
-func lexRightDelim(l *lexer) stateFn {
-	l.pos += 1
-	l.emit(itemRightDelim)
-	return lexText
-}
-
-// lexDefaultValue scans the default value for a variable expansion. It scans
-// until a '}' is encountered. If EOF or EOL occur before the '}' then this
-// is an error.
-func lexDefaultValue(l *lexer) stateFn {
-loop:
-	for {
-		r := l.next()
-		switch {
-		case r == eof || isEndOfLine(r):
-			return l.errorf(errUnterminatedBrace)
-		case r == rightDelim:
-			l.backup()
-			break loop
-		}
-	}
-	l.emit(itemDefaultValue)
-	return lexRightDelim
-}
-
-// isSpace reports whether r is a space character.
-func isSpace(r rune) bool {
-	return r == ' ' || r == '\t'
-}
-
-// isEndOfLine reports whether r is an end-of-line character.
-func isEndOfLine(r rune) bool {
-	return r == '\r' || r == '\n'
-}
-
-// isAlphaNumeric reports whether r is an alphabetic, digit, or underscore.
-func isAlphaNumeric(r rune) bool {
-	return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
-}
-
-// isShellSpecialVar reports whether r identifies a special shell variable
-// such as $*.
-func isShellSpecialVar(r rune) bool {
-	switch r {
-	case '*', '#', '$', '@', '!', '?', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
-		return true
-	}
-	return false
-}
-
-// Functions for using the lexer to parse.
-
-// lex creates a new scanner for the input string.
-func lex(name, input string) *lexer {
-	l := &lexer{
-		name:  name,
-		input: input,
-		items: make(chan item),
-	}
-	go l.run() // Concurrently run state machine.
-	return l
-}
-
-// parseLexer parses the tokens from the lexer. It expands the environment
-// variables that it encounters.
-func parseLexer(l *lexer) ([]byte, error) {
-	var peekItem *item
-	next := func() item {
-		if peekItem != nil {
-			rtn := *peekItem
-			peekItem = nil
-			return rtn
-		}
-		return l.nextItem()
-	}
-	peek := func() item {
-		if peekItem != nil {
-			return *peekItem
-		}
-		rtn := l.nextItem()
-		peekItem = &rtn
-		return rtn
-	}
-
-	var buf bytes.Buffer
-loop:
-	for {
-		item := next()
-
-		switch item.typ {
-		case itemText:
-			buf.WriteString(item.val)
-		case itemVariable:
-			variable := item.val
-			value := os.Getenv(variable)
-			if peek().typ == itemDefaultValue {
-				item = next()
-				if value == "" {
-					value = item.val
-				}
-			}
-			buf.WriteString(value)
-		case itemEscapedLeftDelim:
-			buf.WriteString(leftDelim)
-		case itemLeftDelim, itemRightDelim:
-		case itemError:
-			return nil, fmt.Errorf("failure while expanding environment "+
-				"variables in %s at line=%d, %v", l.name, l.lineNumber(),
-				item.val)
-		case itemEOF:
-			break loop
-		default:
-			return nil, fmt.Errorf("unexpected token type %d", item.typ)
-		}
-	}
-	return buf.Bytes(), nil
-}
-
-// expandEnv replaces ${var} in config according to the values of the current
-// environment variables. The replacement is case-sensitive. References to
-// undefined variables are replaced by the empty string. A default value can be
-// given by using the form ${var:default value}.
-//
-// Valid variable names consist of letters, numbers, and underscores and do not
-// begin with numbers. Variable blocks cannot be split across lines. Unmatched
-// braces will causes a parse error. To use a literal '${' in config write
-// '$${'.
-func expandEnv(filename string, contents []byte) ([]byte, error) {
-	l := lex(filename, string(contents))
-	return parseLexer(l)
-}
diff --git a/libbeat/common/config.go b/libbeat/common/config.go
index 54073b8f3583..56dbe7f1303a 100644
--- a/libbeat/common/config.go
+++ b/libbeat/common/config.go
@@ -7,31 +7,43 @@ import (
 
 type Config ucfg.Config
 
+var configOpts = []ucfg.Option{
+	ucfg.PathSep("."),
+	ucfg.ResolveEnv,
+	ucfg.VarExp,
+}
+
 func NewConfig() *Config {
 	return fromConfig(ucfg.New())
 }
 
 func NewConfigFrom(from interface{}) (*Config, error) {
-	c, err := ucfg.NewFrom(from, ucfg.PathSep("."))
+	c, err := ucfg.NewFrom(from, configOpts...)
 	return fromConfig(c), err
 }
 
 func NewConfigWithYAML(in []byte, source string) (*Config, error) {
-	c, err := yaml.NewConfig(in, ucfg.PathSep("."), ucfg.MetaData(ucfg.Meta{source}))
+	opts := append(
+		[]ucfg.Option{
+			ucfg.MetaData(ucfg.Meta{source}),
+		},
+		configOpts...,
+	)
+	c, err := yaml.NewConfig(in, opts...)
 	return fromConfig(c), err
 }
 
 func LoadFile(path string) (*Config, error) {
-	c, err := yaml.NewConfigWithFile(path, ucfg.PathSep("."))
+	c, err := yaml.NewConfigWithFile(path, configOpts...)
 	return fromConfig(c), err
 }
 
 func (c *Config) Merge(from interface{}) error {
-	return c.access().Merge(from, ucfg.PathSep("."))
+	return c.access().Merge(from, configOpts...)
 }
 
 func (c *Config) Unpack(to interface{}) error {
-	return c.access().Unpack(to, ucfg.PathSep("."))
+	return c.access().Unpack(to, configOpts...)
 }
 
 func (c *Config) Path() string {
@@ -51,44 +63,44 @@ func (c *Config) CountField(name string) (int, error) {
 }
 
 func (c *Config) Bool(name string, idx int) (bool, error) {
-	return c.access().Bool(name, idx, ucfg.PathSep("."))
+	return c.access().Bool(name, idx, configOpts...)
 }
 
 func (c *Config) String(name string, idx int) (string, error) {
-	return c.access().String(name, idx, ucfg.PathSep("."))
+	return c.access().String(name, idx, configOpts...)
 }
 
 func (c *Config) Int(name string, idx int) (int64, error) {
-	return c.access().Int(name, idx, ucfg.PathSep("."))
+	return c.access().Int(name, idx, configOpts...)
 }
 
 func (c *Config) Float(name string, idx int) (float64, error) {
-	return c.access().Float(name, idx, ucfg.PathSep("."))
+	return c.access().Float(name, idx, configOpts...)
 }
 
 func (c *Config) Child(name string, idx int) (*Config, error) {
-	sub, err := c.access().Child(name, idx, ucfg.PathSep("."))
+	sub, err := c.access().Child(name, idx, configOpts...)
 	return fromConfig(sub), err
 }
 
 func (c *Config) SetBool(name string, idx int, value bool) error {
-	return c.access().SetBool(name, idx, value, ucfg.PathSep("."))
+	return c.access().SetBool(name, idx, value, configOpts...)
 }
 
 func (c *Config) SetInt(name string, idx int, value int64) error {
-	return c.access().SetInt(name, idx, value, ucfg.PathSep("."))
+	return c.access().SetInt(name, idx, value, configOpts...)
 }
 
 func (c *Config) SetFloat(name string, idx int, value float64) error {
-	return c.access().SetFloat(name, idx, value, ucfg.PathSep("."))
+	return c.access().SetFloat(name, idx, value, configOpts...)
 }
 
 func (c *Config) SetString(name string, idx int, value string) error {
-	return c.access().SetString(name, idx, value, ucfg.PathSep("."))
+	return c.access().SetString(name, idx, value, configOpts...)
 }
 
 func (c *Config) SetChild(name string, idx int, value *Config) error {
-	return c.access().SetChild(name, idx, value.access(), ucfg.PathSep("."))
+	return c.access().SetChild(name, idx, value.access(), configOpts...)
 }
 
 func fromConfig(in *ucfg.Config) *Config {
@@ -98,6 +110,7 @@ func fromConfig(in *ucfg.Config) *Config {
 func (c *Config) access() *ucfg.Config {
 	return (*ucfg.Config)(c)
 }
+
 func (c *Config) GetFields() []string {
 	return c.access().GetFields()
 }
diff --git a/libbeat/tests/files/config.yml b/libbeat/tests/files/config.yml
index b346165f1524..2f60082dcfca 100644
--- a/libbeat/tests/files/config.yml
+++ b/libbeat/tests/files/config.yml
@@ -3,3 +3,7 @@ output:
     enabled: true
     port: 9200
     host: localhost
+
+env:
+  test_key: ${TEST_KEY}
+  default: ${NON_EXISTENT:default}
diff --git a/libbeat/tests/system/test_base.py b/libbeat/tests/system/test_base.py
index fb5f8b19fb2d..8dad618ab315 100644
--- a/libbeat/tests/system/test_base.py
+++ b/libbeat/tests/system/test_base.py
@@ -25,7 +25,7 @@ def test_no_config(self):
 
         assert exit_code == 1
         assert self.log_contains("error loading config file") is True
-        assert self.log_contains("failed to read") is True
+        assert self.log_contains("no such file or directory") is True
 
     def test_invalid_config(self):
         """
@@ -38,7 +38,6 @@ def test_invalid_config(self):
 
         assert exit_code == 1
         assert self.log_contains("error loading config file") is True
-        assert self.log_contains("YAML config parsing failed") is True
 
     def test_config_test(self):
         """