feat: railsignore added for config loading of LLMRails

Author: ajanitshimanga

.railsignore

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 """Module for the configuration of rails."""
-
+import fnmatch
 import logging
 import os
 import warnings
@@ -28,6 +28,7 @@
 from nemoguardrails.colang.v2_x.lang.colang_ast import Flow
 from nemoguardrails.colang.v2_x.lang.utils import format_colang_parsing_error_message
 from nemoguardrails.colang.v2_x.runtime.errors import ColangParsingError
+from nemoguardrails.utils import get_railsignore_patterns
 
 log = logging.getLogger(__name__)
 
@@ -556,6 +557,12 @@ def _load_path(
             # Followlinks to traverse symlinks instead of ignoring them.
 
             for file in files:
+                # Verify railsignore to skip loading
+                ignored_by_railsignore = _is_file_ignored_by_railsignore(file)
+
+                if ignored_by_railsignore:
+                    continue
+
                 # This is the raw configuration that will be loaded from the file.
                 _raw_config = {}
 
@@ -1203,3 +1210,19 @@ def _generate_rails_flows(flows):
         flow_definitions.insert(1, _LIBRARY_IMPORT + _NEWLINE * 2)
 
     return flow_definitions
+
+
+def _is_file_ignored_by_railsignore(filename: str) -> bool:
+    # Default no skip
+    should_skip_file = False
+
+    # Load candidate patterns from railsignore
+    candidate_patterns = get_railsignore_patterns()
+
+    # Ignore colang, kb, python modules if specified in valid railsignore glob format
+    if filename.endswith(".py") or filename.endswith(".co") or filename.endswith(".kb"):
+        for pattern in candidate_patterns:
+            if fnmatch.fnmatch(filename, pattern):
+                should_skip_file = True
+
+    return should_skip_file

nemoguardrails/rails/llm/config.py

@@ -23,6 +23,7 @@
 from collections import namedtuple
 from datetime import datetime, timezone
 from enum import Enum
+from pathlib import Path
 from typing import Any, Dict, Tuple
 
 import yaml
@@ -312,3 +313,52 @@ def snake_to_camelcase(name: str) -> str:
         str: The converted CamelCase string.
     """
     return "".join(n.capitalize() for n in name.split("_"))
+
+
+def get_railsignore_path() -> Path:
+    """Helper to get railsignore path.
+
+    Returns:
+         Path: The.railsignore file path.
+    """
+    current_path = Path(__file__).resolve()
+
+    # Navigate to the root directory by going up 4 levels
+    root_dir = current_path.parents[1]
+
+    file_path = root_dir / ".railsignore"
+
+    return file_path
+
+
+def get_railsignore_patterns() -> set[str]:
+    """
+    Helper to retrieve all specified patterns in railsignore.
+    Returns:
+        Set[str]: The set of filenames or glob patterns in railsignore
+    """
+    ignored_patterns = set()
+
+    railsignore_path = get_railsignore_path()
+
+    # File doesn't exist or is empty
+    if not railsignore_path.exists() or not os.path.getsize(railsignore_path):
+        return ignored_patterns
+
+    try:
+        with open(railsignore_path, "r") as f:
+            railsignore_entries = f.readlines()
+
+        # Remove comments and empty lines, and strip out any extra spaces/newlines
+        railsignore_entries = [
+            line.strip()
+            for line in railsignore_entries
+            if line.strip() and not line.startswith("#")
+        ]
+
+        ignored_patterns.update(railsignore_entries)
+        return ignored_patterns
+
+    except FileNotFoundError:
+        print(f"No {railsignore_path} found in the current directory.")
+        return ignored_patterns

nemoguardrails/utils.py

@@ -0,0 +1,6 @@
+define user express greeting
+  "hey"
+  "hei"
+
+define flow
+  user express greeting

tests/test_configs/railsignore_config/config_to_load.co

@@ -0,0 +1,7 @@
+define user express greeting
+  "hi"
+  "hello"
+
+define flow
+  user express greeting
+  bot express greeting

tests/test_configs/railsignore_config/ignored_config.co

@@ -0,0 +1,135 @@
+# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+
+import pytest
+
+from nemoguardrails import RailsConfig
+from nemoguardrails.utils import get_railsignore_path, get_railsignore_patterns
+
+CONFIGS_FOLDER = os.path.join(os.path.dirname(__file__), ".", "test_configs")
+
+
+@pytest.fixture(scope="function")
+def cleanup():
+    # Copy current rails ignore and prepare for tests
+    railsignore_path = get_railsignore_path()
+
+    temp_file_path = str(railsignore_path) + "-copy"
+
+    # Copy the original .railsignore to a temporary file
+    shutil.copy(railsignore_path, temp_file_path)
+    print(f"Copied {railsignore_path} to {temp_file_path}")
+
+    # Clean railsignore file before
+    cleanup_railsignore()
+
+    # Yield control to test
+    yield
+
+    # Clean railsignore file before
+    cleanup_railsignore()
+
+    # Restore the original .railsignore from the temporary copy
+    shutil.copy(temp_file_path, railsignore_path)
+    print(f"Restored {railsignore_path} from {temp_file_path}")
+
+    # Delete the temporary file
+    if os.path.exists(temp_file_path):
+        os.remove(temp_file_path)
+        print(f"Deleted temporary file {temp_file_path}")
+
+
+def test_railsignore_config_loading(cleanup):
+    # Setup railsignore
+    append_railsignore("ignored_config.co")
+
+    # Load config
+    config = RailsConfig.from_path(os.path.join(CONFIGS_FOLDER, "railsignore_config"))
+
+    config_string = str(config)
+    # Assert .railsignore successfully ignores
+    assert "ignored_config.co" not in config_string
+
+    # Other files should load successfully
+    assert "config_to_load.co" in config_string
+
+
+def test_get_railsignore_files(cleanup):
+    # Empty railsignore
+    ignored_files = get_railsignore_patterns()
+
+    assert "ignored_module.py" not in ignored_files
+    assert "ignored_colang.co" not in ignored_files
+
+    # Append files to railsignore
+    append_railsignore("ignored_module.py")
+    append_railsignore("ignored_colang.co")
+
+    # Grab ignored files
+    ignored_files = get_railsignore_patterns()
+
+    # Check files exist
+    assert "ignored_module.py" in ignored_files
+    assert "ignored_colang.co" in ignored_files
+
+    # Append comment and whitespace
+    append_railsignore("# This_is_a_comment.py")
+    append_railsignore("  ")
+    append_railsignore("")
+
+    # Grab ignored files
+    ignored_files = get_railsignore_patterns()
+
+    # Comments and whitespace not retrieved
+    assert "# This_is_a_comment.py" not in ignored_files
+    assert "  " not in ignored_files
+    assert "" not in ignored_files
+
+    # Assert files still exist
+    assert "ignored_module.py" in ignored_files
+    assert "ignored_colang.co" in ignored_files
+
+
+def cleanup_railsignore():
+    """
+    Helper for clearing a railsignore file.
+    """
+    railsignore_path = get_railsignore_path()
+
+    try:
+        with open(railsignore_path, "w") as f:
+            pass
+    except OSError as e:
+        print(f"Error: Unable to create {railsignore_path}. {e}")
+    else:
+        print(f"Successfully cleaned up .railsignore: {railsignore_path}")
+
+
+def append_railsignore(file_name: str) -> None:
+    """
+    Helper for appending to a railsignore file.
+    """
+    railsignore_path = get_railsignore_path()
+
+    try:
+        with open(railsignore_path, "a") as f:
+            f.write(file_name + "\n")
+    except FileNotFoundError:
+        print(f"No {railsignore_path} found in the current directory.")
+    except OSError as e:
+        print(f"Error: Failed to write to {railsignore_path}. {e}")