Fix: Handle unescaped quotes in generate_value using safe_eval

Author: milk333445

nemoguardrails/actions/llm/generation.py

@@ -64,7 +64,12 @@
 from nemoguardrails.rails.llm.config import EmbeddingSearchProvider, RailsConfig
 from nemoguardrails.rails.llm.options import GenerationOptions
 from nemoguardrails.streaming import StreamingHandler
-from nemoguardrails.utils import get_or_create_event_loop, new_event_dict, new_uuid
+from nemoguardrails.utils import (
+    get_or_create_event_loop,
+    new_event_dict,
+    new_uuid,
+    safe_eval,
+)
 
 log = logging.getLogger(__name__)
 
@@ -1039,7 +1044,11 @@ async def generate_value(
 
         log.info(f"Generated value for ${var_name}: {value}")
 
-        return literal_eval(value)
+        try:
+            return safe_eval(value)
+        except Exception as e:
+            log.error(f"Error evaluating value: {value}. Error: {str(e)}")
+            raise ValueError(f"Invalid LLM response: `{value}`")
 
     @action(is_system_action=True)
     async def generate_intent_steps_message(

nemoguardrails/utils.py

@@ -21,6 +21,7 @@
 import random
 import re
 import uuid
+from ast import literal_eval
 from collections import namedtuple
 from datetime import datetime, timezone
 from enum import Enum
@@ -384,3 +385,26 @@ def is_ignored_by_railsignore(filename: str, ignore_patterns: str) -> bool:
             break
 
     return ignore
+
+
+def safe_eval(input_value: str) -> str:
+    """
+    Safely evaluate a string to handle unescaped quotes or invalid syntax from the async generate_value action.
+
+    Args:
+        input_value (str): The input string to evaluate.
+
+    Returns:
+        str: The evaluated and properly formatted string.
+
+    Raises:
+        ValueError: If the input cannot be safely evaluated.
+    """
+    if input_value.startswith(("'", '"')) and input_value.endswith(("'", '"')):
+        try:
+            return literal_eval(input_value)
+        except (ValueError, SyntaxError):
+            pass
+    escaped_value = input_value.replace("'", "\\'").replace('"', '\\"')
+    input_value = f"'{escaped_value}'"
+    return literal_eval(input_value)

tests/test_generate_value_safe_eval.py

@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import pytest
+
+from nemoguardrails.utils import safe_eval
+
+
+@pytest.mark.parametrize(
+    "input_value, expected_result",
+    [
+        ('"It\'s a sunny day"', "It's a sunny day"),  # double quotes with single quote
+        (
+            "\"He said, 'Hello'\"",
+            "He said, 'Hello'",
+        ),  # double quotes with nested single quote
+        (
+            "It's a sunny day",
+            "It's a sunny day",
+        ),  # unquoted string containing single quote
+        (
+            "It is a sunny day",
+            "It is a sunny day",
+        ),  # plain string not wrapped in quotes
+        ("", ""),  # empty string
+    ],
+)
+def test_safe_eval(input_value, expected_result):
+    """Test safe_eval with various input values."""
+    result = safe_eval(input_value)
+    assert result == expected_result