Merge pull request #4584 from opsmill/lgu-restrict-attribute-size

Restrict size of common string attribute kinds

backend/infrahub/core/attribute.py

@@ -28,6 +28,7 @@
 from infrahub.exceptions import ValidationError
 from infrahub.helpers import hash_password
 
+from ..types import ATTRIBUTE_TYPES, LARGE_ATTRIBUTE_TYPES
 from .constants.relationship_label import RELATIONSHIP_TO_NODE_LABEL, RELATIONSHIP_TO_VALUE_LABEL
 
 if TYPE_CHECKING:
@@ -39,6 +40,24 @@
 # pylint: disable=redefined-builtin,c-extension-no-member,too-many-lines,too-many-public-methods
 
 
+# Use a more user-friendly threshold than Neo4j one (8167 bytes).
+MAX_STRING_LENGTH = 4096
+
+
+def validate_string_length(value: Optional[str]) -> None:
+    """
+    Validates input string length does not exceed a given threshold, as Neo4J cannot index string values larger than 8167 bytes,
+    see https://neo4j.com/developer/kb/index-limitations-and-workaround/.
+    Note `value` parameter is optional as this function could be called from an attribute class
+    with optional value such as StringOptional.
+    """
+    if value is None:
+        return
+
+    if 3 + len(value.encode("utf-8")) >= MAX_STRING_LENGTH:
+        raise ValidationError(f"Text attribute length should be less than {MAX_STRING_LENGTH} characters.")
+
+
 class AttributeCreateData(BaseModel):
     uuid: str
     name: str
@@ -250,7 +269,12 @@ def to_db(self) -> dict[str, Any]:
         if self.value is None:
             data["value"] = NULL_VALUE
         else:
-            data["value"] = self.serialize_value()
+            serialized_value = self.serialize_value()
+            if isinstance(serialized_value, str) and ATTRIBUTE_TYPES[self.schema.kind] not in LARGE_ATTRIBUTE_TYPES:
+                # Perform validation here to avoid an extra serialization during validation step.
+                # Standard non-str attributes (integer, boolean) do not exceed limit size related to neo4j indexing.
+                validate_string_length(serialized_value)
+            data["value"] = serialized_value
 
         return data
 
@@ -618,7 +642,7 @@ class HashedPassword(BaseAttribute):
 
     def serialize_value(self) -> str:
         """Serialize the value before storing it in the database."""
-        return hash_password(str(self.value))
+        return hash_password(self.value)
 
 
 class HashedPasswordOptional(HashedPassword):
@@ -838,7 +862,7 @@ def validate_format(cls, value: Any, name: str, schema: AttributeSchema) -> None
     def serialize_value(self) -> str:
         """Serialize the value before storing it in the database."""
 
-        return ipaddress.ip_network(str(self.value)).with_prefixlen
+        return ipaddress.ip_network(self.value).with_prefixlen
 
     def get_db_node_type(self) -> AttributeDBNodeType:
         if self.value is not None:
@@ -964,7 +988,7 @@ def validate_format(cls, value: Any, name: str, schema: AttributeSchema) -> None
     def serialize_value(self) -> str:
         """Serialize the value before storing it in the database."""
 
-        return ipaddress.ip_interface(str(self.value)).with_prefixlen
+        return ipaddress.ip_interface(self.value).with_prefixlen
 
     def get_db_node_type(self) -> AttributeDBNodeType:
         if self.value is not None:
@@ -1085,7 +1109,7 @@ def validate_format(cls, value: Any, name: str, schema: AttributeSchema) -> None
 
     def serialize_value(self) -> str:
         """Serialize the value as standard EUI-48 or EUI-64 before storing it in the database."""
-        return str(netaddr.EUI(addr=str(self.value)))
+        return str(netaddr.EUI(addr=self.value))
 
 
 class MacAddressOptional(MacAddress):

backend/infrahub/types.py

@@ -365,6 +365,9 @@ class Any(InfrahubDataType):
 
 ATTRIBUTE_KIND_LABELS = list(ATTRIBUTE_TYPES.keys())
 
+# Data types supporting large values, which can therefore not be indexed in neo4j.
+LARGE_ATTRIBUTE_TYPES = [TextArea, JSON]
+
 
 def get_attribute_type(kind: str = "Default") -> type[InfrahubDataType]:
     """Return an InfrahubDataType object for a given kind

backend/tests/unit/conftest.py

@@ -1658,6 +1658,7 @@ async def all_attribute_types_schema(
         "attributes": [
             {"name": "name", "kind": "Text", "optional": True},
             {"name": "mystring", "kind": "Text", "optional": True},
+            {"name": "mytextarea", "kind": "TextArea", "optional": True},
             {"name": "mybool", "kind": "Boolean", "optional": True},
             {"name": "myint", "kind": "Number", "optional": True},
             {"name": "mylist", "kind": "List", "optional": True},

backend/tests/unit/core/test_attribute.py

@@ -4,7 +4,17 @@
 from infrahub_sdk import UUIDT
 
 from infrahub import config
-from infrahub.core.attribute import URL, DateTime, Dropdown, Integer, IPHost, IPNetwork, MacAddress, String
+from infrahub.core.attribute import (
+    MAX_STRING_LENGTH,
+    URL,
+    DateTime,
+    Dropdown,
+    Integer,
+    IPHost,
+    IPNetwork,
+    MacAddress,
+    String,
+)
 from infrahub.core.branch import Branch
 from infrahub.core.constants import InfrahubKind
 from infrahub.core.manager import NodeManager
@@ -674,3 +684,21 @@ async def test_to_graphql_no_fields(
         "value": "mystring",
     }
     assert await attr2.to_graphql(db=db) == expected_data
+
+
+async def test_attribute_size(db: InfrahubDatabase, default_branch: Branch, all_attribute_types_schema):
+    obj = await Node.init(db=db, schema="TestAllAttributeTypes")
+
+    large_string = "a" * 5_000
+
+    await obj.new(db=db, name="obj1", mystring=large_string)
+
+    # Text field
+    with pytest.raises(
+        ValidationError, match=f"Text attribute length should be less than {MAX_STRING_LENGTH} characters."
+    ):
+        await obj.save(db=db)
+
+    # TextArea field should have no size limitation
+    await obj.new(db=db, name="obj2", mytextarea=large_string)
+    await obj.save(db=db)

changelog/4432.fixed.md

@@ -0,0 +1 @@
+Add a size restriction on common attribute kinds. Only TextArea and JSON support large values