Merge pull request #772 from neo4j-contrib/rc/5.2.1

Rc/5.2.1

.github/workflows/integration-tests.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.11", "3.10", "3.9", "3.8", "3.7"]
+        python-version: ["3.12", "3.11", "3.10", "3.9", "3.8", "3.7"]
         neo4j-version: ["community", "enterprise", "5.5-enterprise", "4.4-enterprise", "4.4-community"]
 
     steps:

Changelog

@@ -1,3 +1,9 @@
+Version 5.2.1 2023-12
+* Add options to inspection script to skip heavy operations - rel props or cardinality inspection #767
+* Fixes database version parsing issues
+* Fixes bug when combining count with pagination #769
+* Bumps neo4j (driver) to 5.15.0
+
 Version 5.2.0 2023-11
 * Add an option to pass your own driver instead of relying on the automatically created one. See set_connection method. NB : only accepts the synchronous driver for now.
 * Add a close_connection method to explicitly close the driver to match Neo4j deprecation.

doc/source/configuration.rst

@@ -32,7 +32,7 @@ Adjust driver configuration - these options are only available for this connecti
     config.MAX_TRANSACTION_RETRY_TIME = 30.0  # default
     config.RESOLVER = None  # default
     config.TRUST = neo4j.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES  # default
-    config.USER_AGENT = neomodel/v5.2.0  # default
+    config.USER_AGENT = neomodel/v5.2.1  # default
 
 Setting the database name, if different from the default one::
 

doc/source/getting_started.rst

@@ -81,8 +81,18 @@ You can inspect an existing Neo4j database to generate a neomodel definition fil
 This will generate a file called ``models.py`` in the ``yourapp`` directory. This file can be used as a starting point,
 and will contain the necessary module imports, as well as class definition for nodes and, if relevant, relationships.
 
+Ommitting the ``--db`` argument will default to the ``NEO4J_BOLT_URL`` environment variable. This is useful for masking
+your credentials.
+
 Note that you can also print the output to the console instead of writing a file by omitting the ``--write-to`` option.
 
+If you have a database with a large number of nodes and relationships,
+this script can take a long time to run (during our tests, it took 30 seconds for 500k nodes and 1.3M relationships).
+You can speed it up by not scanning for relationship properties and/or relationship cardinality, using these options :
+``--no-rel-props`` and ``--no-rel-cardinality``.
+Note that this will still add relationship definition to your nodes, but without relationship models ;
+and cardinality will be default (ZeroOrMore).
+
 .. note::
 
     This command will only generate the definition for nodes and relationships that are present in the
@@ -108,6 +118,9 @@ script (:ref:`neomodel_install_labels`) to automate this: ::
 
 It is important to execute this after altering the schema and observe the number of classes it reports.
 
+Ommitting the ``--db`` argument will default to the ``NEO4J_BOLT_URL`` environment variable. This is useful for masking
+your credentials.
+
 Remove existing constraints and indexes
 =======================================
 Similarly, ``neomodel`` provides a script (:ref:`neomodel_remove_labels`) to automate the removal of all existing constraints and indexes from
@@ -117,6 +130,9 @@ the database, when this is required: ::
 
 After executing, it will print all indexes and constraints it has removed.
 
+Ommitting the ``--db`` argument will default to the ``NEO4J_BOLT_URL`` environment variable. This is useful for masking
+your credentials.
+
 Create, Update, Delete operations
 =================================
 

neomodel/__init__.py

@@ -1,5 +1,4 @@
 # pep8: noqa
-import pkg_resources
 
 from neomodel.exceptions import *
 from neomodel.match import EITHER, INCOMING, OUTGOING, NodeSet, Traversal

neomodel/_version.py

@@ -1 +1 @@
-__version__ = "5.2.0"
+__version__ = "5.2.1"

neomodel/match.py

@@ -323,6 +323,7 @@ class QueryAST:
     result_class: Optional[type]
     lookup: Optional[str]
     additional_return: Optional[list]
+    is_count: Optional[bool]
 
     def __init__(
         self,
@@ -337,6 +338,7 @@ def __init__(
         result_class: Optional[type] = None,
         lookup: Optional[str] = None,
         additional_return: Optional[list] = None,
+        is_count: Optional[bool] = False,
     ):
         self.match = match if match else []
         self.optional_match = optional_match if optional_match else []
@@ -349,6 +351,7 @@ def __init__(
         self.result_class = result_class
         self.lookup = lookup
         self.additional_return = additional_return if additional_return else []
+        self.is_count = is_count
 
 
 class QueryBuilder:
@@ -649,15 +652,27 @@ def build_query(self):
             query += " ORDER BY "
             query += ", ".join(self._ast.order_by)
 
-        if self._ast.skip:
+        # If we return a count with pagination, pagination has to happen before RETURN
+        # It will then be included in the WITH clause already
+        if self._ast.skip and not self._ast.is_count:
             query += f" SKIP {self._ast.skip}"
 
-        if self._ast.limit:
+        if self._ast.limit and not self._ast.is_count:
             query += f" LIMIT {self._ast.limit}"
 
         return query
 
     def _count(self):
+        self._ast.is_count = True
+        # If we return a count with pagination, pagination has to happen before RETURN
+        # Like : WITH my_var SKIP 10 LIMIT 10 RETURN count(my_var)
+        self._ast.with_clause = f"{self._ast.return_clause}"
+        if self._ast.skip:
+            self._ast.with_clause += f" SKIP {self._ast.skip}"
+
+        if self._ast.limit:
+            self._ast.with_clause += f" LIMIT {self._ast.limit}"
+
         self._ast.return_clause = f"count({self._ast.return_clause})"
         # drop order_by, results in an invalid query
         self._ast.order_by = None

neomodel/scripts/neomodel_inspect_database.py

@@ -19,11 +19,13 @@
     If no file is specified, the tool will print the class definitions to stdout.
     
     options:
-      -h, --help            show this help message and exit
-      --db bolt://neo4j:neo4j@localhost:7687
+        -h, --help            show this help message and exit
+        --db bolt://neo4j:neo4j@localhost:7687
                             Neo4j Server URL
-      -T, --write-to someapp/models.py
+        -T, --write-to someapp/models.py
                             File where to write output.
+        --no-rel-props        Do not inspect relationship properties
+        --no-rel-cardinality  Do not infer relationship cardinality
 """
 
 import argparse
@@ -116,13 +118,20 @@ def get_indexed_properties_for_label(label):
 
 class RelationshipInspector:
     @classmethod
-    def outgoing_relationships(cls, start_label):
-        query = f"""
-            MATCH (n:`{start_label}`)-[r]->(m)
-            WITH DISTINCT type(r) as rel_type, head(labels(m)) AS target_label, keys(r) AS properties, head(collect(r)) AS sampleRel
-            ORDER BY size(properties) DESC
-            RETURN rel_type, target_label, apoc.meta.cypher.types(properties(sampleRel)) AS properties LIMIT 1
-        """
+    def outgoing_relationships(cls, start_label, get_properties: bool = True):
+        if get_properties:
+            query = f"""
+                MATCH (n:`{start_label}`)-[r]->(m)
+                WITH DISTINCT type(r) as rel_type, head(labels(m)) AS target_label, keys(r) AS properties, head(collect(r)) AS sampleRel
+                ORDER BY size(properties) DESC
+                RETURN rel_type, target_label, apoc.meta.cypher.types(properties(sampleRel)) AS properties LIMIT 1
+            """
+        else:
+            query = f"""
+                MATCH (n:`{start_label}`)-[r]->(m)
+                WITH DISTINCT type(r) as rel_type, head(labels(m)) AS target_label
+                RETURN rel_type, target_label, {{}} AS properties LIMIT 1
+            """
         result, _ = db.cypher_query(query)
         return [(record[0], record[1], record[2]) for record in result]
 
@@ -222,7 +231,9 @@ def parse_imports():
     return imports
 
 
-def build_rel_type_definition(label, outgoing_relationships, defined_rel_types):
+def build_rel_type_definition(
+    label, outgoing_relationships, defined_rel_types, infer_cardinality: bool = True
+):
     class_definition_append = ""
     rel_type_definitions = ""
 
@@ -241,9 +252,12 @@ def build_rel_type_definition(label, outgoing_relationships, defined_rel_types):
             rel_type
         )
 
-        cardinality = RelationshipInspector.infer_cardinality(rel_type, label)
+        cardinality_string = ""
+        if infer_cardinality:
+            cardinality = RelationshipInspector.infer_cardinality(rel_type, label)
+            cardinality_string += f", cardinality={cardinality}"
 
-        class_definition_append += f'    {clean_class_member_key(rel_name)} = RelationshipTo("{target_label}", "{rel_type}", cardinality={cardinality}'
+        class_definition_append += f'    {clean_class_member_key(rel_name)} = RelationshipTo("{target_label}", "{rel_type}"{cardinality_string}'
 
         if rel_props and rel_type not in defined_rel_types:
             rel_model_name = generate_rel_class_name(rel_type)
@@ -265,7 +279,11 @@ def build_rel_type_definition(label, outgoing_relationships, defined_rel_types):
     return class_definition_append
 
 
-def inspect_database(bolt_url):
+def inspect_database(
+    bolt_url,
+    get_relationship_properties: bool = True,
+    infer_relationship_cardinality: bool = True,
+):
     # Connect to the database
     print(f"Connecting to {bolt_url}")
     db.set_connection(bolt_url)
@@ -284,23 +302,32 @@ def inspect_database(bolt_url):
         indexed_properties = NodeInspector.get_indexed_properties_for_label(label)
 
         class_definition = f"class {class_name}(StructuredNode):\n"
-        class_definition += "".join(
-            [
-                build_prop_string(
-                    unique_properties, indexed_properties, prop, prop_type
-                )
-                for prop, prop_type in properties.items()
-            ]
-        )
+        if properties:
+            class_definition += "".join(
+                [
+                    build_prop_string(
+                        unique_properties, indexed_properties, prop, prop_type
+                    )
+                    for prop, prop_type in properties.items()
+                ]
+            )
 
-        outgoing_relationships = RelationshipInspector.outgoing_relationships(label)
+        outgoing_relationships = RelationshipInspector.outgoing_relationships(
+            label, get_relationship_properties
+        )
 
         if outgoing_relationships and "StructuredRel" not in IMPORTS:
             IMPORTS.append("RelationshipTo")
-            IMPORTS.append("StructuredRel")
+            # No rel properties = no rel classes
+            # Then StructuredRel import is not needed
+            if get_relationship_properties:
+                IMPORTS.append("StructuredRel")
 
         class_definition += build_rel_type_definition(
-            label, outgoing_relationships, defined_rel_types
+            label,
+            outgoing_relationships,
+            defined_rel_types,
+            infer_relationship_cardinality,
         )
 
         if not properties and not outgoing_relationships:
@@ -353,6 +380,20 @@ def main():
         help="File where to write output.",
     )
 
+    parser.add_argument(
+        "--no-rel-props",
+        dest="get_relationship_properties",
+        action="store_false",
+        help="Do not inspect relationship properties",
+    )
+
+    parser.add_argument(
+        "--no-rel-cardinality",
+        dest="infer_relationship_cardinality",
+        action="store_false",
+        help="Do not infer relationship cardinality",
+    )
+
     args = parser.parse_args()
 
     bolt_url = args.neo4j_bolt_url
@@ -364,12 +405,22 @@ def main():
     # Before connecting to the database
     if args.write_to:
         with open(args.write_to, "w") as file:
-            output = inspect_database(bolt_url=bolt_url)
+            output = inspect_database(
+                bolt_url=bolt_url,
+                get_relationship_properties=args.get_relationship_properties,
+                infer_relationship_cardinality=args.infer_relationship_cardinality,
+            )
             print(f"Writing to {args.write_to}")
             file.write(output)
     # If no file is specified, print to stdout
     else:
-        print(inspect_database(bolt_url=bolt_url))
+        print(
+            inspect_database(
+                bolt_url=bolt_url,
+                get_relationship_properties=args.get_relationship_properties,
+                infer_relationship_cardinality=args.infer_relationship_cardinality,
+            )
+        )
 
 
 if __name__ == "__main__":

neomodel/util.py

@@ -679,15 +679,19 @@ def version_tag_to_integer(version_tag):
     """
     Converts a version string to an integer representation to allow for quick comparisons between versions.
 
-    :param a_version_string: The version string to be converted (e.g. '3.4.0')
+    :param a_version_string: The version string to be converted (e.g. '5.4.0')
     :type a_version_string: str
-    :return: An integer representation of the version string (e.g. '3.4.0' --> 340)
+    :return: An integer representation of the version string (e.g. '5.4.0' --> 50400)
     :rtype: int
     """
     components = version_tag.split(".")
     while len(components) < 3:
         components.append("0")
     num = 0
     for index, component in enumerate(components):
-        num += (10 ** ((len(components) - 1) - index)) * int(component)
+        # Aura started adding a -aura suffix in version numbers, like "5.14-aura"
+        # This will strip the suffix to allow for proper comparison : 14 instead of 14-aura
+        if "-" in component:
+            component = component.split("-")[0]
+        num += (100 ** ((len(components) - 1) - index)) * int(component)
     return num

pyproject.toml

@@ -1,25 +1,15 @@
-[build-system]
-requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
-build-backend = "setuptools.build_meta"
-
-[tool.setuptools_scm]
-
-[tool.setuptools.packages.find]
-where = ["./"]
-
 [project]
 name = "neomodel"
 authors = [
     {name = "Robin Edwards", email = "robin.ge@gmail.com"},
 ]
 maintainers = [
+    {name = "Marius Conjeaud", email = "marius.conjeaud@outlook.com"},
     {name = "Athanasios Anastasiou", email = "athanastasiou@gmail.com"},
     {name = "Cristina Escalante"},
-    {name = "Marius Conjeaud", email = "marius.conjeaud@outlook.com"},
 ]
 description = "An object mapper for the neo4j graph database."
 readme = "README.md"
-requires-python = ">=3.7"
 keywords = ["graph", "neo4j", "ORM", "OGM", "mapper"]
 license = {text = "MIT"}
 classifiers = [
@@ -33,12 +23,10 @@ classifiers = [
     "Topic :: Database",
 ]
 dependencies = [
-    "neo4j==5.12.0",
-    "pytz>=2021.1",
-    "neobolt==1.7.17",
-    "six==1.16.0",
+    "neo4j~=5.15.0",
 ]
-version='5.2.0'
+requires-python = ">=3.7"
+dynamic = ["version"]
 
 [project.urls]
 documentation = "https://neomodel.readthedocs.io/en/latest/"
@@ -57,6 +45,16 @@ dev = [
 pandas = ["pandas"]
 numpy = ["numpy"]
 
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.dynamic]
+version = {attr = "neomodel._version.__version__"}
+
+[tool.setuptools.packages.find]
+where = ["./"]
+
 [tool.pytest.ini_options]
 addopts = "--resetdb"
 testpaths = "test"

requirements-dev.txt

@@ -0,0 +1,9 @@
+# neomodel
+-e .[pandas,numpy]
+
+pytest>=7.1
+pytest-cov>=4.0
+pre-commit
+black
+isort
+Shapely>=2.0.0