Fix nested escaped dollar signs

go.mod

@@ -1,3 +1,5 @@
 module github.com/buildkite/interpolate
 
 go 1.22
+
+require github.com/google/go-cmp v0.6.0

go.sum

@@ -0,0 +1,2 @@
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=

interpolate.go

@@ -1,8 +1,8 @@
 package interpolate
 
 import (
-	"bytes"
 	"fmt"
+	"strings"
 )
 
 // Interpolate takes a set of environment and interpolates it into the provided string using shell script expansions
@@ -28,12 +28,23 @@ func Identifiers(str string) ([]string, error) {
 
 // An expansion is something that takes in ENV and returns a string or an error
 type Expansion interface {
+	// Expand expands the expansion using variables from env.
 	Expand(env Env) (string, error)
+
+	// Identifiers returns any variable names referenced within the expansion.
+	// Escaped expansions do something special and return identifiers
+	// (starting with $) that *would* become referenced after a round of
+	// unescaping.
 	Identifiers() []string
+
+	// Unescaped returns a string equivalent to the original expression, but
+	// with escaped dollarsigns un-escaped.
+	Unescaped() string
 }
 
 // VariableExpansion represents either $VAR or ${VAR}, our simplest expansion
 type VariableExpansion struct {
+	Braced     bool // in case we need to reconsistute the original string
 	Identifier string
 }
 
@@ -46,6 +57,13 @@ func (e VariableExpansion) Expand(env Env) (string, error) {
 	return val, nil
 }
 
+func (e VariableExpansion) Unescaped() string {
+	if e.Braced {
+		return "${" + e.Identifier + "}"
+	}
+	return "$" + e.Identifier
+}
+
 // EmptyValueExpansion returns either the value of an env, or a default value if it's unset or null
 type EmptyValueExpansion struct {
 	Identifier string
@@ -64,6 +82,10 @@ func (e EmptyValueExpansion) Expand(env Env) (string, error) {
 	return val, nil
 }
 
+func (e EmptyValueExpansion) Unescaped() string {
+	return fmt.Sprintf("${%s:-%s}", e.Identifier, e.Content.Unescaped())
+}
+
 // UnsetValueExpansion returns either the value of an env, or a default value if it's unset
 type UnsetValueExpansion struct {
 	Identifier string
@@ -82,17 +104,45 @@ func (e UnsetValueExpansion) Expand(env Env) (string, error) {
 	return val, nil
 }
 
+func (e UnsetValueExpansion) Unescaped() string {
+	return fmt.Sprintf("${%s-%s}", e.Identifier, e.Content)
+}
+
 // EscapedExpansion is an expansion that is delayed until later on (usually by a later process)
 type EscapedExpansion struct {
+	// Must be either Identifier, Brace, or neither.
 	Identifier string
+	Brace      Expansion
 }
 
 func (e EscapedExpansion) Identifiers() []string {
-	return []string{"$" + e.Identifier}
+	switch {
+	case e.Identifier != "":
+		return []string{"$" + e.Identifier}
+
+	case e.Brace != nil:
+		return e.Brace.Identifiers()
+
+	default:
+		return nil
+	}
 }
 
 func (e EscapedExpansion) Expand(Env) (string, error) {
-	return "$" + e.Identifier, nil
+	return e.Unescaped(), nil
+}
+
+func (e EscapedExpansion) Unescaped() string {
+	switch {
+	case e.Identifier != "":
+		return "$" + e.Identifier
+
+	case e.Brace != nil:
+		return e.Brace.Unescaped() // (brace types).String() includes a $
+
+	default:
+		return "$"
+	}
 }
 
 // SubstringExpansion returns a substring (or slice) of the env
@@ -154,6 +204,25 @@ func (e SubstringExpansion) Expand(env Env) (string, error) {
 	return val[from:to], nil
 }
 
+func (e SubstringExpansion) Unescaped() string {
+	// Ensure negative offsets are not interpreted as an empty value
+	// expansion by adding a space.
+	// Only include length if it was present.
+	switch {
+	case e.Offset < 0 && e.HasLength:
+		return fmt.Sprintf("${%s: %d:%d}", e.Identifier, e.Offset, e.Length)
+
+	case e.Offset < 0:
+		return fmt.Sprintf("${%s: %d}", e.Identifier, e.Offset)
+
+	case e.HasLength:
+		return fmt.Sprintf("${%s:%d:%d}", e.Identifier, e.Offset, e.Length)
+
+	default:
+		return fmt.Sprintf("${%s:%d}", e.Identifier, e.Offset)
+	}
+}
+
 // RequiredExpansion returns an env value, or an error if it is unset
 type RequiredExpansion struct {
 	Identifier string
@@ -179,6 +248,10 @@ func (e RequiredExpansion) Expand(env Env) (string, error) {
 	return val, nil
 }
 
+func (e RequiredExpansion) Unescaped() string {
+	return fmt.Sprintf("${%s:?%s}", e.Identifier, e.Message.Unescaped())
+}
+
 // Expression is a collection of either Text or Expansions
 type Expression []ExpressionItem
 
@@ -193,23 +266,35 @@ func (e Expression) Identifiers() []string {
 }
 
 func (e Expression) Expand(env Env) (string, error) {
-	buf := &bytes.Buffer{}
+	var buf strings.Builder
 
 	for _, item := range e {
 		if item.Expansion != nil {
 			result, err := item.Expansion.Expand(env)
 			if err != nil {
 				return "", err
 			}
-			_, _ = buf.WriteString(result)
+			buf.WriteString(result)
 		} else {
-			_, _ = buf.WriteString(item.Text)
+			buf.WriteString(item.Text)
 		}
 	}
 
 	return buf.String(), nil
 }
 
+func (e Expression) Unescaped() string {
+	var buf strings.Builder
+	for _, item := range e {
+		if item.Expansion != nil {
+			buf.WriteString(item.Expansion.Unescaped())
+		} else {
+			buf.WriteString(item.Text)
+		}
+	}
+	return buf.String()
+}
+
 // ExpressionItem models either an Expansion or Text. Either/Or, never both.
 type ExpressionItem struct {
 	Text string

parser.go

@@ -87,7 +87,7 @@ func (p *Parser) parseExpression(stop ...rune) (Expression, error) {
 				return nil, err
 			}
 
-			expr = append(expr, ee)
+			expr = append(expr, ExpressionItem{Expansion: ee})
 			continue
 		}
 
@@ -122,28 +122,37 @@ func (p *Parser) parseExpression(stop ...rune) (Expression, error) {
 	return expr, nil
 }
 
-func (p *Parser) parseEscapedExpansion() (ExpressionItem, error) {
+func (p *Parser) parseEscapedExpansion() (EscapedExpansion, error) {
 	next := p.peekRune()
+	start := p.pos
 	switch {
 	case next == '{':
-		// if it's an escaped brace expansion, (eg $${MY_COOL_VAR:-5}) consume text until the close brace
-		id := p.scanUntil(func(r rune) bool { return r == '}' })
-		id = id + string(p.nextRune()) // we know that the next rune is a close brace, chuck it on the end
-		return ExpressionItem{Expansion: EscapedExpansion{Identifier: id}}, nil
+		// it *could be* an escaped brace expansion
+		expansion, err := p.parseBraceExpansion()
+		if err != nil {
+			// oh well. reset position to the brace
+			p.pos = start
+			return EscapedExpansion{}, nil
+		}
+		return EscapedExpansion{Brace: expansion}, nil
 
 	case unicode.IsLetter(next):
-		// it's an escaped identifier (eg $$MY_COOL_VAR)
+		// it *could be* an escaped identifier (eg $$MY_COOL_VAR)
 		id, err := p.scanIdentifier()
 		if err != nil {
-			return ExpressionItem{}, err
+			// this should never happen, since scanIdentifier only errors if the
+			// first rune is not a letter, and we just checked that.
+			// oh well. reset to the first letter
+			p.pos = start
+			return EscapedExpansion{}, nil
 		}
 
-		return ExpressionItem{Expansion: EscapedExpansion{Identifier: id}}, nil
+		return EscapedExpansion{Identifier: id}, nil
 
 	default:
-		// there's no identifier or brace afterward, so it's probably a literal escaped dollar sign
-		// just return a text item with the dollar sign
-		return ExpressionItem{Text: "$"}, nil
+		// there's no identifier or brace afterward, so it's probably a literal
+		// escaped dollar sign
+		return EscapedExpansion{Identifier: ""}, nil
 	}
 }
 
@@ -162,7 +171,10 @@ func (p *Parser) parseExpansion() (Expansion, error) {
 		return nil, err
 	}
 
-	return VariableExpansion{Identifier: identifier}, nil
+	return VariableExpansion{
+		Braced:     false,
+		Identifier: identifier,
+	}, nil
 }
 
 func (p *Parser) parseBraceExpansion() (Expansion, error) {
@@ -177,7 +189,10 @@ func (p *Parser) parseBraceExpansion() (Expansion, error) {
 
 	if c := p.peekRune(); c == '}' {
 		_ = p.nextRune()
-		return VariableExpansion{Identifier: identifier}, nil
+		return VariableExpansion{
+			Braced:     true,
+			Identifier: identifier,
+		}, nil
 	}
 
 	var operator string
@@ -298,8 +313,8 @@ func (p *Parser) scanIdentifier() (string, error) {
 	if c := p.peekRune(); !unicode.IsLetter(c) {
 		return "", fmt.Errorf("Expected identifier to start with a letter, got %c", c)
 	}
-	var notIdentifierChar = func(r rune) bool {
-		return (!unicode.IsLetter(r) && !unicode.IsNumber(r) && r != '_')
+	notIdentifierChar := func(r rune) bool {
+		return !(unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_')
 	}
 	return p.scanUntil(notIdentifierChar), nil
 }