Fix: Handle unescaped quotes in generate_value using safe_eval

Author: milk333445

nemoguardrails/actions/llm/generation.py

@@ -1039,7 +1039,10 @@ async def generate_value(
 
         log.info(f"Generated value for ${var_name}: {value}")
 
-        return literal_eval(value)
+        try:
+            return safe_eval(value)
+        except Exception:
+            raise Exception(f"Invalid LLM response: `{value}`")
 
     @action(is_system_action=True)
     async def generate_intent_steps_message(
@@ -1342,3 +1345,26 @@ def clean_utterance_content(utterance: str) -> str:
         # It should be translated to an actual \n character.
         utterance = utterance.replace("\\n", "\n")
     return utterance
+
+
+def safe_eval(input_value: str) -> str:
+    """
+    Safely evaluate a string to handle unescaped quotes or invalid syntax from the async generate_value action.
+
+    Args:
+        input_value (str): The input string to evaluate.
+
+    Returns:
+        str: The evaluated and properly formatted string.
+
+    Raises:
+        ValueError: If the input cannot be safely evaluated.
+    """
+    if input_value.startswith(("'", '"')) and input_value.endswith(("'", '"')):
+        try:
+            return literal_eval(input_value)
+        except (ValueError, SyntaxError):
+            pass
+    escaped_value = input_value.replace("'", "\\'").replace('"', '\\"')
+    input_value = f"'{escaped_value}'"
+    return literal_eval(input_value)

tests/test_generate_value_safe_eval.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nemoguardrails.actions.llm.generation import safe_eval
+
+
+def test_safe_eval_double_quotes_with_single_quote():
+    """Test safe_eval with a string wrapped in double quotes that contains a single quote."""
+    input_value = '"It\'s a sunny day"'
+    result = safe_eval(input_value)
+    assert result == "It's a sunny day"
+
+
+def test_safe_eval_double_quotes_with_nested_single_quote():
+    """Test safe_eval with a string wrapped in double quotes that contains a nested single quote."""
+    input_value = "\"He said, 'Hello'\""
+    result = safe_eval(input_value)
+    assert result == "He said, 'Hello'"
+
+
+def test_safe_eval_unquoted_string_with_single_quote():
+    """Test safe_eval with a string not wrapped in quotes, containing a single quote."""
+    input_value = "It's a sunny day"
+    result = safe_eval(input_value)
+    assert result == "It's a sunny day"
+
+
+def test_safe_eval_unquoted_plain_string():
+    """Test safe_eval with a plain string not wrapped in quotes."""
+    input_value = "It is a sunny day"
+    result = safe_eval(input_value)
+    assert result == "It is a sunny day"
+
+
+def test_safe_eval_empty_string():
+    """Test safe_eval with an empty string as input."""
+    input_value = ""
+    result = safe_eval(input_value)
+    assert result == ""