Fixing parsing of escaped characters in strenv #2506

Author: mikefarah

pkg/yqlib/lexer_participle.go

@@ -379,15 +379,7 @@ func stringValue() yqAction {
 		log.Debug("rawTokenvalue: %v", rawToken.Value)
 		value := unwrap(rawToken.Value)
 		log.Debug("unwrapped: %v", value)
-		value = strings.ReplaceAll(value, "\\\"", "\"")
-		value = strings.ReplaceAll(value, "\\n", "\n")
-		value = strings.ReplaceAll(value, "\\t", "\t")
-		value = strings.ReplaceAll(value, "\\r", "\r")
-		value = strings.ReplaceAll(value, "\\f", "\f")
-		value = strings.ReplaceAll(value, "\\v", "\v")
-		value = strings.ReplaceAll(value, "\\b", "\b")
-		value = strings.ReplaceAll(value, "\\a", "\a")
-		log.Debug("replaced: %v", value)
+		value = processEscapeCharacters(value)
 		return &token{TokenType: operationToken, Operation: &Operation{
 			OperationType: stringInterpolationOpType,
 			StringValue:   value,

pkg/yqlib/lib.go

@@ -186,6 +186,22 @@ func parseInt(numberString string) (int, error) {
 	return int(parsed), err
 }
 
+func processEscapeCharacters(original string) string {
+	value := original
+	value = strings.ReplaceAll(value, "\\\"", "\"")
+	value = strings.ReplaceAll(value, "\\n", "\n")
+	value = strings.ReplaceAll(value, "\\t", "\t")
+	value = strings.ReplaceAll(value, "\\r", "\r")
+	value = strings.ReplaceAll(value, "\\f", "\f")
+	value = strings.ReplaceAll(value, "\\v", "\v")
+	value = strings.ReplaceAll(value, "\\b", "\b")
+	value = strings.ReplaceAll(value, "\\a", "\a")
+	if value != original {
+		log.Debug("processEscapeCharacters from [%v] to [%v]", original, value)
+	}
+	return value
+}
+
 func headAndLineComment(node *CandidateNode) string {
 	return headComment(node) + lineComment(node)
 }

pkg/yqlib/operator_env.go

@@ -29,7 +29,7 @@ func envOperator(_ *dataTreeNavigator, context Context, expressionNode *Expressi
 		node = &CandidateNode{
 			Kind:  ScalarNode,
 			Tag:   "!!str",
-			Value: rawValue,
+			Value: processEscapeCharacters(rawValue),
 		}
 	} else if rawValue == "" {
 		return Context{}, fmt.Errorf("value for env variable '%v' not provided in env()", envName)

pkg/yqlib/operator_env_test.go

@@ -63,6 +63,78 @@ var envOperatorScenarios = []expressionScenario{
 			"D0, P[], ()::a: \"12\"\n",
 		},
 	},
+	{
+		description:          "strenv with newline escape",
+		skipDoc:              true,
+		environmentVariables: map[string]string{"myenv": "string with a\\n"},
+		expression:           `.a = strenv(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: |\n    string with a\n",
+		},
+	},
+	{
+		description:          "strenv with tab escape",
+		skipDoc:              true,
+		environmentVariables: map[string]string{"myenv": "string with a\\t"},
+		expression:           `.a = strenv(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: \"string with a\\t\"\n",
+		},
+	},
+	{
+		description:          "strenv with carriage return escape",
+		skipDoc:              true,
+		environmentVariables: map[string]string{"myenv": "string with a\\r"},
+		expression:           `.a = strenv(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: \"string with a\\r\"\n",
+		},
+	},
+	{
+		description:          "strenv with form feed escape",
+		skipDoc:              true,
+		environmentVariables: map[string]string{"myenv": "string with a\\f"},
+		expression:           `.a = strenv(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: \"string with a\\f\"\n",
+		},
+	},
+	{
+		description:          "strenv with vertical tab escape",
+		skipDoc:              true,
+		environmentVariables: map[string]string{"myenv": "string with a\\v"},
+		expression:           `.a = strenv(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: \"string with a\\v\"\n",
+		},
+	},
+	{
+		description:          "strenv with backspace escape",
+		skipDoc:              true,
+		environmentVariables: map[string]string{"myenv": "string with a\\b"},
+		expression:           `.a = strenv(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: \"string with a\\b\"\n",
+		},
+	},
+	{
+		description:          "strenv with alert/bell escape",
+		skipDoc:              true,
+		environmentVariables: map[string]string{"myenv": "string with a\\a"},
+		expression:           `.a = strenv(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: \"string with a\\a\"\n",
+		},
+	},
+	{
+		description:          "strenv with double quote escape",
+		skipDoc:              true,
+		environmentVariables: map[string]string{"myenv": "string with a\\\""},
+		expression:           `.a = strenv(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: string with a\"\n",
+		},
+	},
 	{
 		description:          "Dynamically update a path from an environment variable",
 		subdescription:       "The env variable can be any valid yq expression.",