[NEAT-374, NEAT-368 Part 2]👻 Refactor utility module part2

cognite/neat/app/api/routers/data_exploration.py

@@ -17,7 +17,7 @@
 from cognite.neat.app.api.utils.data_mapping import rdf_result_to_api_response
 from cognite.neat.app.api.utils.query_templates import query_templates
 from cognite.neat.legacy.graph.transformations import query_generator
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 from cognite.neat.workflows.steps.data_contracts import RulesData, SolutionGraph, SourceGraph
 
 router = APIRouter()

cognite/neat/graph/extractors/_classic_cdf/_assets.py

@@ -12,7 +12,7 @@
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import create_sha256_hash, string_to_ideal_type
+from cognite.neat.utils.auxiliary import create_sha256_hash, string_to_ideal_type
 
 
 class AssetsExtractor(BaseExtractor):

cognite/neat/graph/extractors/_classic_cdf/_events.py

@@ -12,7 +12,7 @@
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.auxiliary import string_to_ideal_type
 
 
 class EventsExtractor(BaseExtractor):

cognite/neat/graph/extractors/_classic_cdf/_files.py

@@ -13,7 +13,7 @@
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.auxiliary import string_to_ideal_type
 
 
 class FilesExtractor(BaseExtractor):

cognite/neat/graph/extractors/_classic_cdf/_labels.py

@@ -10,7 +10,7 @@
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import create_sha256_hash
+from cognite.neat.utils.auxiliary import create_sha256_hash
 
 
 class LabelsExtractor(BaseExtractor):

cognite/neat/graph/extractors/_classic_cdf/_relationships.py

@@ -11,7 +11,7 @@
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import create_sha256_hash
+from cognite.neat.utils.auxiliary import create_sha256_hash
 
 
 class RelationshipsExtractor(BaseExtractor):

cognite/neat/graph/extractors/_classic_cdf/_sequences.py

@@ -12,7 +12,7 @@
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.auxiliary import string_to_ideal_type
 
 
 class SequencesExtractor(BaseExtractor):

cognite/neat/graph/extractors/_classic_cdf/_timeseries.py

@@ -12,7 +12,7 @@
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import string_to_ideal_type
+from cognite.neat.utils.auxiliary import string_to_ideal_type
 
 
 class TimeSeriesExtractor(BaseExtractor):

cognite/neat/graph/extractors/_dexpi.py

@@ -9,7 +9,7 @@
 from cognite.neat.constants import DEFAULT_NAMESPACE
 from cognite.neat.graph.extractors._base import BaseExtractor
 from cognite.neat.graph.models import Triple
-from cognite.neat.utils.utils import as_neat_compliant_uri
+from cognite.neat.utils.rdf_ import as_neat_compliant_uri
 from cognite.neat.utils.xml_ import get_children, iterate_tree
 
 DEXPI = Namespace("http://sandbox.dexpi.org/rdl/")

cognite/neat/graph/extractors/_mock_graph_generator.py

@@ -17,7 +17,7 @@
 from cognite.neat.rules.models.data_types import DataType
 from cognite.neat.rules.models.entities import ClassEntity, EntityTypes
 from cognite.neat.rules.models.information import InformationProperty
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 from ._base import BaseExtractor
 

cognite/neat/graph/loaders/_rdf2dms.py

@@ -20,8 +20,8 @@
 from cognite.neat.issues import NeatIssue, NeatIssueList
 from cognite.neat.rules.models import DMSRules
 from cognite.neat.rules.models.data_types import _DATA_TYPE_BY_DMS_TYPE, Json
+from cognite.neat.utils.auxiliary import create_sha256_hash
 from cognite.neat.utils.upload import UploadResult
-from cognite.neat.utils.utils import create_sha256_hash
 
 from ._base import CDFLoader
 

cognite/neat/graph/queries/_base.py

@@ -8,7 +8,7 @@
 
 from cognite.neat.rules.models.entities import ClassEntity
 from cognite.neat.rules.models.information import InformationRules
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 from ._construct import build_construct_query
 

cognite/neat/graph/queries/_construct.py

@@ -13,7 +13,7 @@
 )
 from cognite.neat.rules.models.entities import ClassEntity
 from cognite.neat.rules.models.information import InformationProperty, InformationRules
-from cognite.neat.utils.utils import most_occurring_element
+from cognite.neat.utils.collection_ import most_occurring_element
 
 from ._shared import Triple, hop2property_path
 

cognite/neat/graph/queries/_shared.py

@@ -11,7 +11,7 @@
     Hop,
     Step,
 )
-from cognite.neat.utils.utils import remove_namespace_from_uri, uri_to_short_form
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri, uri_to_short_form
 
 if sys.version_info >= (3, 11):
     from typing import Self

cognite/neat/legacy/graph/extractors/_mock_graph_generator.py

@@ -20,7 +20,7 @@
 from cognite.neat.legacy.rules.exporters._rules2rules import subset_rules
 from cognite.neat.legacy.rules.models import Rules
 from cognite.neat.legacy.rules.models.value_types import XSD_VALUE_TYPE_MAPPINGS
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 from ._base import BaseExtractor
 

cognite/neat/legacy/graph/loaders/_asset_loader.py

@@ -15,8 +15,8 @@
 from cognite.neat.legacy.graph.stores import NeatGraphStoreBase
 from cognite.neat.legacy.rules.models import Rules
 from cognite.neat.legacy.rules.models.rules import Property
-from cognite.neat.utils import remove_namespace_from_uri
-from cognite.neat.utils.utils import epoch_now_ms
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
+from cognite.neat.utils.time_ import epoch_now_ms
 
 from ._base import CogniteLoader
 

cognite/neat/legacy/graph/loaders/core/rdf_to_assets.py

@@ -18,7 +18,10 @@
 from cognite.neat.legacy.graph.loaders.core.models import AssetTemplate
 from cognite.neat.legacy.graph.stores import NeatGraphStoreBase
 from cognite.neat.legacy.rules.models.rules import Property, Rules
-from cognite.neat.utils.utils import chunker, datetime_utc_now, remove_namespace_from_uri, retry_decorator
+from cognite.neat.utils.auxiliary import retry_decorator
+from cognite.neat.utils.collection_ import chunker
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
+from cognite.neat.utils.time_ import datetime_utc_now
 
 if sys.version_info >= (3, 11):
     from datetime import UTC
@@ -899,7 +902,7 @@ def upsert_assets(batch):
         except CogniteDuplicatedError:
             # this is handling of very rare case when some assets might be lost . Normally this should not happen.
             # Last attempt to recover
-            client.assets.create_hierarchy(batch, upsert=True)
+            client.assets.create_hierarchy(batch, upsert=True)  # type: ignore[arg-type]
 
         delta_time = (datetime_utc_now() - start_time).seconds
 

cognite/neat/legacy/graph/loaders/core/rdf_to_relationships.py

@@ -14,7 +14,10 @@
 from cognite.neat.legacy.graph.loaders.core.rdf_to_assets import _categorize_cdf_assets
 from cognite.neat.legacy.graph.stores import NeatGraphStoreBase
 from cognite.neat.legacy.rules.models.rules import Rules
-from cognite.neat.utils.utils import chunker, datetime_utc_now, epoch_now_ms, remove_namespace_from_uri, retry_decorator
+from cognite.neat.utils.auxiliary import retry_decorator
+from cognite.neat.utils.collection_ import chunker
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
+from cognite.neat.utils.time_ import datetime_utc_now, epoch_now_ms
 
 
 def define_relationships(rules: Rules, data_set_id: int, stop_on_exception: bool = False) -> RelationshipDefinitions:

cognite/neat/legacy/graph/loaders/rdf_to_dms.py

@@ -13,7 +13,9 @@
 from cognite.neat.legacy.rules.exporters._rules2dms import DMSSchemaComponents
 from cognite.neat.legacy.rules.exporters._rules2pydantic_models import add_class_prefix_to_xid, rules_to_pydantic_models
 from cognite.neat.legacy.rules.models.rules import Rules
-from cognite.neat.utils.utils import chunker, datetime_utc_now, retry_decorator
+from cognite.neat.utils.auxiliary import retry_decorator
+from cognite.neat.utils.collection_ import chunker
+from cognite.neat.utils.time_ import datetime_utc_now
 
 from ._base import CogniteLoader
 

cognite/neat/legacy/graph/transformations/query_generator/sparql.py

@@ -21,7 +21,7 @@
     parse_traversal,
 )
 from cognite.neat.legacy.rules.models.rules import Rules
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 
 def _generate_prefix_header(prefixes: dict[str, Namespace] | None = None) -> str:

cognite/neat/legacy/graph/transformations/transformer.py

@@ -24,7 +24,7 @@
     parse_rule,
 )
 from cognite.neat.legacy.rules.models.rules import Rules
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 prom_total_proc_rules_g = Gauge("neat_total_processed_rules", "Number of processed rules", ["state"])
 rules_processing_timing_metric = Gauge(

cognite/neat/legacy/rules/exporters/_rules2dms.py

@@ -45,7 +45,7 @@
 from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant
 from cognite.neat.legacy.rules.models._base import ContainerEntity, EntityTypes, ParentClass
 from cognite.neat.legacy.rules.models.rules import Class, Property, Rules
-from cognite.neat.utils.utils import generate_exception_report
+from cognite.neat.utils.auxiliary import generate_exception_report
 
 if sys.version_info < (3, 11):
     from exceptiongroup import ExceptionGroup

cognite/neat/legacy/rules/exporters/_rules2graphql.py

@@ -14,7 +14,7 @@
 from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant, are_properties_redefined
 from cognite.neat.legacy.rules.models.rules import Rules
 from cognite.neat.legacy.rules.models.value_types import XSD_VALUE_TYPE_MAPPINGS
-from cognite.neat.utils.utils import generate_exception_report
+from cognite.neat.utils.auxiliary import generate_exception_report
 
 from ._base import BaseExporter
 

cognite/neat/legacy/rules/exporters/_rules2ontology.py

@@ -14,7 +14,8 @@
 from cognite.neat.legacy.rules.exporters._validation import are_properties_redefined
 from cognite.neat.legacy.rules.models.rules import Class, Metadata, Property, Rules
 from cognite.neat.legacy.rules.models.value_types import XSD_VALUE_TYPE_MAPPINGS
-from cognite.neat.utils.utils import generate_exception_report, remove_namespace_from_uri
+from cognite.neat.utils.auxiliary import generate_exception_report
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 if sys.version_info >= (3, 11):
     from typing import Self

cognite/neat/legacy/rules/exporters/_rules2pydantic_models.py

@@ -21,7 +21,7 @@
 from cognite.neat.legacy.rules.exporters._validation import are_entity_names_dms_compliant
 from cognite.neat.legacy.rules.models.rules import Property, Rules
 from cognite.neat.legacy.rules.models.value_types import ValueTypeMapping
-from cognite.neat.utils.utils import create_sha256_hash, generate_exception_report
+from cognite.neat.utils.auxiliary import create_sha256_hash, generate_exception_report
 
 if sys.version_info >= (3, 11):
     from datetime import UTC

cognite/neat/legacy/rules/importers/_graph2rules.py

@@ -14,11 +14,7 @@
 from cognite.neat.legacy.rules import exceptions
 from cognite.neat.legacy.rules.exporters._rules2rules import to_dms_name
 from cognite.neat.legacy.rules.models.tables import Tables
-from cognite.neat.utils.utils import (
-    get_namespace,
-    remove_namespace_from_uri,
-    uri_to_short_form,
-)
+from cognite.neat.utils.rdf_ import get_namespace, remove_namespace_from_uri, uri_to_short_form
 
 from ._base import BaseImporter
 
@@ -78,26 +74,11 @@ def _create_default_properties_parsing_config() -> dict[str, tuple[str, ...]]:
 
 def _create_default_classes_parsing_config() -> dict[str, tuple[str, ...]]:
     # TODO: these are to be read from Class pydantic model
-    return {
-        "header": (
-            "Class",
-            "Description",
-            "Parent Class",
-            "Source",
-            "Source Entity Name",
-            "Match Type",
-            "Comment",
-        )
-    }
+    return {"header": ("Class", "Description", "Parent Class", "Source", "Source Entity Name", "Match Type", "Comment")}
 
 
 def _parse_prefixes_df(prefixes: dict[str, Namespace]) -> pd.DataFrame:
-    return pd.DataFrame.from_dict(
-        {
-            "Prefix": list(prefixes.keys()),
-            "URI": [str(uri) for uri in prefixes.values()],
-        }
-    )
+    return pd.DataFrame.from_dict({"Prefix": list(prefixes.keys()), "URI": [str(uri) for uri in prefixes.values()]})
 
 
 def _parse_metadata_df() -> pd.DataFrame:
@@ -202,18 +183,10 @@ def _graph_to_data_model_dict(graph: Graph, max_number_of_instance: int = -1) ->
             )
             class_name = f"{class_name}_{len(data_model)+1}"
 
-        data_model[class_name] = {
-            "properties": {},
-            "uri": uri_to_short_form(class_, prefixes),
-        }
+        data_model[class_name] = {"properties": {}, "uri": uri_to_short_form(class_, prefixes)}
 
         for instance in _get_class_instance_ids(graph, class_, max_number_of_instance):
-            for (
-                property_,
-                occurrence,
-                data_type,
-                object_type,
-            ) in _define_instance_properties(graph, instance):
+            for property_, occurrence, data_type, object_type in _define_instance_properties(graph, instance):
                 property_name = remove_namespace_from_uri(property_)
                 _add_uri_namespace_to_prefixes(property_, prefixes)
 

cognite/neat/legacy/rules/importers/_owl2rules/_owl2classes.py

@@ -4,7 +4,7 @@
 import pandas as pd
 from rdflib import OWL, Graph
 
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 
 def parse_owl_classes(graph: Graph, make_compliant: bool = False, language: str = "en") -> pd.DataFrame:

cognite/neat/legacy/rules/importers/_owl2rules/_owl2metadata.py

@@ -10,7 +10,8 @@
     prefix_compliance_regex,
     version_compliance_regex,
 )
-from cognite.neat.utils.utils import convert_rdflib_content, remove_none_elements_from_set
+from cognite.neat.utils.collection_ import remove_none_elements_from_set
+from cognite.neat.utils.rdf_ import convert_rdflib_content
 
 
 def parse_owl_metadata(graph: Graph, make_compliant: bool = False) -> pd.DataFrame:

cognite/neat/legacy/rules/importers/_owl2rules/_owl2properties.py

@@ -4,7 +4,7 @@
 import pandas as pd
 from rdflib import Graph
 
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 from ._owl2classes import _data_type_property_class, _object_property_class, _thing_class
 

cognite/neat/legacy/rules/models/raw_rules.py

@@ -24,7 +24,7 @@
 
 # importers:
 from cognite.neat.rules import exceptions
-from cognite.neat.utils.utils import generate_exception_report
+from cognite.neat.utils.auxiliary import generate_exception_report
 
 __all__ = ["RawRules"]
 

cognite/neat/rules/analysis/_base.py

@@ -17,7 +17,7 @@
     ReferenceEntity,
 )
 from cognite.neat.rules.models.information import InformationProperty
-from cognite.neat.utils.utils import get_inheritance_path
+from cognite.neat.utils.rdf_ import get_inheritance_path
 
 T_Rules = TypeVar("T_Rules", bound=BaseRules)
 T_Property = TypeVar("T_Property", bound=BaseModel)

cognite/neat/rules/analysis/_information.py

@@ -12,7 +12,7 @@
     InformationProperty,
     InformationRules,
 )
-from cognite.neat.utils.utils import get_inheritance_path
+from cognite.neat.utils.rdf_ import get_inheritance_path
 
 from ._base import BaseAnalysis
 

cognite/neat/rules/exporters/_rules2ontology.py

@@ -20,7 +20,8 @@
     InformationProperty,
     InformationRules,
 )
-from cognite.neat.utils.utils import generate_exception_report, remove_namespace_from_uri
+from cognite.neat.utils.auxiliary import generate_exception_report
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 from ._base import BaseExporter
 from ._validation import are_properties_redefined

cognite/neat/rules/importers/_inference2rules.py

@@ -17,11 +17,7 @@
     InformationMetadata,
     InformationRulesInput,
 )
-from cognite.neat.utils.utils import (
-    get_namespace,
-    remove_namespace_from_uri,
-    uri_to_short_form,
-)
+from cognite.neat.utils.rdf_ import get_namespace, remove_namespace_from_uri, uri_to_short_form
 
 ORDERED_CLASSES_QUERY = """SELECT ?class (count(?s) as ?instances )
                            WHERE { ?s a ?class . }

cognite/neat/rules/importers/_owl2rules/_owl2classes.py

@@ -5,7 +5,7 @@
 from rdflib import OWL, Graph
 
 from cognite.neat.rules.models._base import MatchType
-from cognite.neat.utils.utils import remove_namespace_from_uri
+from cognite.neat.utils.rdf_ import remove_namespace_from_uri
 
 
 def parse_owl_classes(graph: Graph, language: str = "en") -> list[dict]: