[NEAT-325] 🏰 Asset Hierarchy Migration Tutorial (#511)

* refactor: setup notebook

* refactor; testing out functionality

* docs; files extractor

* fix: bug in idp scope method

* fix: bug in idp scope method

* docs; First draft of tutorial

* refactor; added to docs

* refactor: added option for seting .env file name

* fix: interactive prompting

* fix: default namespace for inference rules

* refactor; added helpeq query

* refactor: undesired necessity

* refactor; checkpoint of notebook

* build: bump

* refactor; removed debug code

.gitignore

@@ -171,3 +171,5 @@ tests/data/970_0ca0c919-046a-4940-b191-f5cc4d0e6513.json
 build/*
 scripts/tmp/*
 scripts/config_manual_test.yaml
+real_cases/
+*.env

Makefile

@@ -1,5 +1,5 @@
 .PHONY: run-explorer run-tests run-linters build-ui build-python build-docker run-docker compose-up
-version="0.85.0"
+version="0.85.1"
 run-explorer:
 	@echo "Running explorer API server..."
 	# open "http://localhost:8000/static/index.html" || true

cognite/neat/_version.py

@@ -1 +1 @@
-__version__ = "0.85.0"
+__version__ = "0.85.1"

cognite/neat/graph/extractors/_classic_cdf/_files.py

@@ -16,13 +16,20 @@
 
 
 class FilesExtractor(BaseExtractor):
+    """Extract data from Cognite Data Fusions files metadata into Neat.
+
+    Args:
+        files_metadata (Iterable[FileMetadata]): An iterable of files metadata.
+        namespace (Namespace, optional): The namespace to use. Defaults to DEFAULT_NAMESPACE.
+    """
+
     def __init__(
         self,
-        events: Iterable[FileMetadata],
+        files_metadata: Iterable[FileMetadata],
         namespace: Namespace | None = None,
     ):
         self.namespace = namespace or DEFAULT_NAMESPACE
-        self.events = events
+        self.files_metadata = files_metadata
 
     @classmethod
     def from_dataset(
@@ -39,7 +46,7 @@ def from_file(cls, file_path: str, namespace: Namespace | None = None):
 
     def extract(self) -> Iterable[Triple]:
         """Extract files metadata as triples."""
-        for event in self.events:
+        for event in self.files_metadata:
             yield from self._file2triples(event, self.namespace)
 
     @classmethod

cognite/neat/graph/queries/_base.py

@@ -97,3 +97,27 @@ def construct_instances_of_class(self, class_: str, properties_optional: bool =
         else:
             warnings.warn("No rules found for the graph store, returning empty list.", stacklevel=2)
             return []
+
+    def list_triples(self, limit: int = 25) -> list[ResultRow]:
+        """List triples in the graph store
+
+        Args:
+            limit: Max number of triples to return, by default 25
+
+        Returns:
+            List of triples
+        """
+        query = f"SELECT ?subject ?predicate ?object WHERE {{ ?subject ?predicate ?object }} LIMIT {limit}"
+        return cast(list[ResultRow], list(self.graph.query(query)))
+
+    def list_types(self, limit: int = 25) -> list[ResultRow]:
+        """List types in the graph store
+
+        Args:
+            limit: Max number of types to return, by default 25
+
+        Returns:
+            List of types
+        """
+        query = f"SELECT DISTINCT ?type WHERE {{ ?subject a ?type }} LIMIT {limit}"
+        return cast(list[ResultRow], list(self.graph.query(query)))

cognite/neat/rules/importers/_inference2rules.py

@@ -7,7 +7,7 @@
 from rdflib import Literal as RdfLiteral
 
 import cognite.neat.rules.issues as issues
-from cognite.neat.constants import PREFIXES
+from cognite.neat.constants import DEFAULT_NAMESPACE, PREFIXES
 from cognite.neat.graph.stores import NeatGraphStore
 from cognite.neat.rules.importers._base import BaseImporter, Rules, _handle_issues
 from cognite.neat.rules.issues import IssueList
@@ -246,8 +246,8 @@ def _default_metadata(cls):
             created=datetime.now(),
             updated=datetime.now(),
             description="Inferred model from knowledge graph",
-            prefix="inferred",
-            namespace="http://purl.org/cognite/neat/inferred/",
+            prefix="neat",
+            namespace=DEFAULT_NAMESPACE,
         )
 
     @classmethod

cognite/neat/utils/auth.py

@@ -18,24 +18,26 @@
 _CLIENT_NAME = f"CogniteNeat:{_version.__version__}"
 
 
-def get_cognite_client() -> CogniteClient:
+def get_cognite_client(env_file_name: str = ".env") -> CogniteClient:
+    if not env_file_name.endswith(".env"):
+        raise ValueError("env_file_name must end with '.env'")
     with suppress(KeyError):
         variables = _EnvironmentVariables.create_from_environ()
         return variables.get_client()
 
     repo_root = _repo_root()
     if repo_root:
         with suppress(KeyError, FileNotFoundError, TypeError):
-            variables = _from_dotenv(repo_root / ".env")
+            variables = _from_dotenv(repo_root / env_file_name)
             client = variables.get_client()
             print("Found .env file in repository root. Loaded variables from .env file.")
             return client
     variables = _prompt_user()
-    if repo_root and _env_in_gitignore(repo_root):
+    if repo_root and _env_in_gitignore(repo_root, env_file_name):
         local_import("rich", "jupyter")
         from rich.prompt import Prompt
 
-        env_file = repo_root / ".env"
+        env_file = repo_root / env_file_name
         answer = Prompt.ask(
             "Do you store the variables in an .env file in the repository root for easy reuse?", choices=["y", "n"]
         )
@@ -88,7 +90,7 @@ def idp_audience(self) -> str:
     @property
     def idp_scopes(self) -> list[str]:
         if self.IDP_SCOPES:
-            return self.IDP_SCOPES.split()
+            return self.IDP_SCOPES.split(",")
         return [f"https://{self.CDF_CLUSTER}.cognitedata.com/.default"]
 
     @property
@@ -187,7 +189,7 @@ def create_env_file(self) -> str:
             value = getattr(self, name)
             if value is not None:
                 if isinstance(value, list):
-                    value = " ".join(value)
+                    value = ",".join(value)
                 lines.append(f"{field.name}={value}")
         return "\n".join(lines)
 
@@ -199,7 +201,7 @@ def _from_dotenv(evn_file: Path) -> _EnvironmentVariables:
     valid_variables = {f.name for f in fields(_EnvironmentVariables)}
     variables: dict[str, str] = {}
     for line in content.splitlines():
-        if line.startswith("#") or not line:
+        if line.startswith("#") or "=" not in line:
             continue
         key, value = line.split("=", 1)
         if key in valid_variables:
@@ -241,8 +243,13 @@ def _prompt_user() -> _EnvironmentVariables:
             token_url = Prompt.ask("Enter IDP_TOKEN_URL")
             variables.IDP_TOKEN_URL = token_url
         optional = ["IDP_AUDIENCE", "IDP_SCOPES"]
-    else:
-        optional = ["IDP_TENANT_ID", "IDP_SCOPES"]
+    else:  # login_flow == "interactive"
+        tenant_id = Prompt.ask("Enter IDP_TENANT_ID (leave empty to enter IDP_AUTHORITY_URL instead)")
+        if tenant_id:
+            variables.IDP_TENANT_ID = tenant_id
+        else:
+            variables.IDP_AUTHORITY_URL = Prompt.ask("Enter IDP_TOKEN_URL")
+        optional = ["IDP_SCOPES"]
 
     defaults = "".join(f"\n - {name}: {getattr(variables, name.lower())}" for name in optional)
     use_defaults = Prompt.ask(
@@ -284,15 +291,15 @@ def _repo_root() -> Path | None:
     return None
 
 
-def _env_in_gitignore(repo_root: Path) -> bool:
+def _env_in_gitignore(repo_root: Path, env_file_name: str) -> bool:
     ignore_file = repo_root / ".gitignore"
     if not ignore_file.exists():
         return False
     else:
         ignored = {line.strip() for line in ignore_file.read_text().splitlines()}
-        return ".env" in ignored or "*.env" in ignored
+        return env_file_name in ignored or "*.env" in ignored
 
 
 if __name__ == "__main__":
-    c = _prompt_user().get_client()
+    c = get_cognite_client()
     print(c.iam.token.inspect())

docs/CHANGELOG.md

@@ -15,6 +15,10 @@ Changes are grouped as follows:
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.85.1] - 01-07-24
+### Fixed
+- Bug when using the `get_cognite_client` function with interactive login. This is now fixed.
+
 ## [0.85.0] - 25-06-24
 ### Changed
 - [BREAKING] Interface for `Loaders`. Instead of `.export_to_cdf` now always return `UploadResultList` and