Fixing validation_process for GraphTemplate

.github/workflows/publish-state-mangaer.yml

@@ -15,8 +15,65 @@ env:
   SHA_TAG: ${{ github.sha }}
 
 jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      mongodb:
+        image: mongo:7
+        ports:
+          - 27017:27017
+        options: >-
+          --health-cmd "mongosh --eval 'db.runCommand(\"ping\")'"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        env:
+          MONGO_INITDB_ROOT_USERNAME: admin
+          MONGO_INITDB_ROOT_PASSWORD: password
+          MONGO_INITDB_DATABASE: test_db
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+        with:
+          cache: true
+
+      - name: Install dev dependencies with uv
+        working-directory: state-manager
+        run: |
+          uv sync --group dev
+
+      - name: Run full test suite with coverage
+        working-directory: state-manager
+        env:
+          MONGO_URI: mongodb://admin:password@localhost:27017
+          MONGO_DATABASE_NAME: test_exosphere_state_manager
+          STATE_MANAGER_SECRET: test-secret-key
+          SECRETS_ENCRYPTION_KEY: YTzpUlBGLSwm-3yKJRJTZnb0_aQuQQHyz64s8qAERVU=
+        run: |
+          uv run pytest tests/ --cov=app --cov-report=xml --cov-report=term-missing --cov-report=html -v --junitxml=full-pytest-report.xml
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: exospherehost/exospherehost
+          files: state-manager/coverage.xml
+          flags: unit-tests
+          name: state-manager-coverage-report
+          fail_ci_if_error: true
+
   publish-image:
     runs-on: ubuntu-latest
+    needs: test
 
     permissions:
       contents: read

.github/workflows/release-state-manager.yml

@@ -26,7 +26,11 @@ jobs:
           --health-cmd "mongosh --eval 'db.runCommand(\"ping\")'"
           --health-interval 10s
           --health-timeout 5s
-          --health-retries 5
+          --health-retries 10
+        env:
+          MONGO_INITDB_ROOT_USERNAME: admin
+          MONGO_INITDB_ROOT_PASSWORD: password
+          MONGO_INITDB_DATABASE: test_db
 
     steps:
       - name: Checkout code
@@ -44,30 +48,27 @@ jobs:
         working-directory: state-manager
         run: |
           uv sync --group dev
-
-      - name: Run unit tests with pytest and coverage
+          
+      - name: Run full test suite with coverage
         working-directory: state-manager
+        env:
+          MONGO_URI: mongodb://admin:password@localhost:27017
+          MONGO_DATABASE_NAME: test_exosphere_state_manager
+          STATE_MANAGER_SECRET: test-secret-key
+          SECRETS_ENCRYPTION_KEY: YTzpUlBGLSwm-3yKJRJTZnb0_aQuQQHyz64s8qAERVU=
         run: |
-          uv run pytest tests/unit/ --cov=app --cov-report=xml --cov-report=term-missing -v --junitxml=pytest-report.xml
+          uv run pytest tests/ --cov=app --cov-report=xml --cov-report=term-missing --cov-report=html -v --junitxml=full-pytest-report.xml
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           slug: exospherehost/exospherehost
           files: state-manager/coverage.xml
-          flags: state-manager-unittests
+          flags: unit-tests
           name: state-manager-coverage-report
           fail_ci_if_error: true
 
-      - name: Upload test results
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: state-manager-test-results
-          path: state-manager/pytest-report.xml
-          retention-days: 30
-
   publish-image:
     runs-on: ubuntu-latest
     needs: test

.github/workflows/test-state-manager.yml

@@ -23,6 +23,10 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+        env:
+          MONGO_INITDB_ROOT_USERNAME: admin
+          MONGO_INITDB_ROOT_PASSWORD: password
+          MONGO_INITDB_DATABASE: test_db
 
     steps:
       - name: Checkout code
@@ -42,26 +46,23 @@ jobs:
         working-directory: state-manager
         run: |
           uv sync --group dev
-
-      - name: Run unit tests with pytest and coverage
+          
+      - name: Run full test suite with coverage
         working-directory: state-manager
+        env:
+          MONGO_URI: mongodb://admin:password@localhost:27017
+          MONGO_DATABASE_NAME: test_exosphere_state_manager
+          STATE_MANAGER_SECRET: test-secret-key
+          SECRETS_ENCRYPTION_KEY: YTzpUlBGLSwm-3yKJRJTZnb0_aQuQQHyz64s8qAERVU=
         run: |
-          uv run pytest tests/unit/ --cov=app --cov-report=xml --cov-report=term-missing -v --junitxml=pytest-report.xml
+          uv run pytest tests/ --cov=app --cov-report=xml --cov-report=term-missing --cov-report=html -v --junitxml=full-pytest-report.xml
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           slug: exospherehost/exospherehost
           files: state-manager/coverage.xml
-          flags: state-manager-unittests
+          flags: unit-tests
           name: state-manager-coverage-report
-          fail_ci_if_error: true
-
-      - name: Upload test results
-        uses: actions/upload-artifact@v4
-        if: always()
-        with:
-          name: state-manager-test-results
-          path: state-manager/pytest-report.xml
-          retention-days: 30
+          fail_ci_if_error: true

state-manager/.dockerignore

@@ -23,4 +23,10 @@ __pycache__/
 
 # Other
 .env
-Dockerfile 
+Dockerfile 
+tests/
+pytest.ini
+.pytest_cache/
+.coverage
+.coverage.*
+coverage.xml

state-manager/app/config/settings.py

@@ -0,0 +1,38 @@
+import os
+from pydantic import BaseModel, Field
+from dotenv import load_dotenv
+
+load_dotenv()
+
+class Settings(BaseModel):
+    """Application settings loaded from environment variables."""
+
+    # MongoDB Configuration
+    mongo_uri: str = Field(..., description="MongoDB connection URI" )
+    mongo_database_name: str = Field(default="exosphere-state-manager", description="MongoDB database name")
+    state_manager_secret: str = Field(..., description="Secret key for API authentication")
+    secrets_encryption_key: str = Field(..., description="Key for encrypting secrets")
+
+    @classmethod
+    def from_env(cls) -> "Settings":
+        return cls(
+            mongo_uri=os.getenv("MONGO_URI"), # type: ignore
+            mongo_database_name=os.getenv("MONGO_DATABASE_NAME", "exosphere-state-manager"), # type: ignore
+            state_manager_secret=os.getenv("STATE_MANAGER_SECRET"), # type: ignore
+            secrets_encryption_key=os.getenv("SECRETS_ENCRYPTION_KEY"), # type: ignore
+        )
+
+
+# Global settings instance - will be updated when get_settings() is called
+_settings = None
+
+
+def get_settings() -> Settings:
+    """Get the global settings instance, reloading from environment if needed."""
+    global _settings
+    _settings = Settings.from_env()
+    return _settings
+
+
+# Initialize settings
+settings = get_settings() 

state-manager/app/controller/upsert_graph_template.py

@@ -4,7 +4,7 @@
 from app.models.graph_template_validation_status import GraphTemplateValidationStatus
 from app.tasks.verify_graph import verify_graph
 
-from fastapi import BackgroundTasks
+from fastapi import BackgroundTasks, HTTPException
 from beanie.operators import Set
 
 logger = LogsManager().get_logger()
@@ -15,36 +15,41 @@ async def upsert_graph_template(namespace_name: str, graph_name: str, body: Upse
             GraphTemplate.name == graph_name,
             GraphTemplate.namespace == namespace_name
         )
-        if graph_template:
-            logger.info(
-                "Graph template already exists in namespace", graph_template=graph_template,
-                namespace_name=namespace_name,
-                x_exosphere_request_id=x_exosphere_request_id)
-
-            await graph_template.set_secrets(body.secrets).update(
-                Set({
-                    GraphTemplate.nodes: body.nodes, # type: ignore
-                    GraphTemplate.validation_status: GraphTemplateValidationStatus.PENDING, # type: ignore
-                    GraphTemplate.validation_errors: [] # type: ignore
-                })
-            )
-
-        else:
-            logger.info(
-                "Graph template does not exist in namespace",
-                namespace_name=namespace_name,
-                graph_name=graph_name,
-                x_exosphere_request_id=x_exosphere_request_id)
-
-            graph_template = await GraphTemplate.insert(
-                GraphTemplate(
-                    name=graph_name,
-                    namespace=namespace_name,
-                    nodes=body.nodes,
-                    validation_status=GraphTemplateValidationStatus.PENDING,
-                    validation_errors=[]
-                ).set_secrets(body.secrets)
-            )
+
+        try:
+            if graph_template:
+                logger.info(
+                    "Graph template already exists in namespace", graph_template=graph_template,
+                    namespace_name=namespace_name,
+                    x_exosphere_request_id=x_exosphere_request_id)
+
+                await graph_template.set_secrets(body.secrets).update(
+                    Set({
+                        GraphTemplate.nodes: body.nodes, # type: ignore
+                        GraphTemplate.validation_status: GraphTemplateValidationStatus.PENDING, # type: ignore
+                        GraphTemplate.validation_errors: [] # type: ignore
+                    })
+                )
+
+            else:
+                logger.info(
+                    "Graph template does not exist in namespace",
+                    namespace_name=namespace_name,
+                    graph_name=graph_name,
+                    x_exosphere_request_id=x_exosphere_request_id)
+
+                graph_template = await GraphTemplate.insert(
+                    GraphTemplate(
+                        name=graph_name,
+                        namespace=namespace_name,
+                        nodes=body.nodes,
+                        validation_status=GraphTemplateValidationStatus.PENDING,
+                        validation_errors=[]
+                    ).set_secrets(body.secrets)
+                )
+        except ValueError as e:
+            logger.error("Error validating graph template", error=e, x_exosphere_request_id=x_exosphere_request_id)
+            raise HTTPException(status_code=400, detail=f"Error validating graph template: {str(e)}")
 
         background_tasks.add_task(verify_graph, graph_template)
 

state-manager/app/main.py

@@ -1,12 +1,10 @@
 """
-main file for exosphere apis
+main file for exosphere state manager
 """
-import os
 from beanie import init_beanie
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from contextlib import asynccontextmanager
-from dotenv import load_dotenv
 from pymongo import AsyncMongoClient
 
 # injecting singletons
@@ -29,24 +27,26 @@
 
 # importing CORS config
 from .config.cors import get_cors_config
+from .config.settings import get_settings
 
-load_dotenv()
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     # begaining of the server
     logger = LogsManager().get_logger()
     logger.info("server starting")
 
+    # Get settings
+    settings = get_settings()
+
     # initializing beanie
-    client = AsyncMongoClient(os.getenv("MONGO_URI"))
-    db = client[os.getenv("MONGO_DATABASE_NAME", "exosphere-state-manager")]
+    client = AsyncMongoClient(settings.mongo_uri)
+    db = client[settings.mongo_database_name]
     await init_beanie(db, document_models=[State, Namespace, GraphTemplate, RegisteredNode])
     logger.info("beanie dbs initialized")
 
     # initialize secret
-    secret = os.getenv("STATE_MANAGER_SECRET")
-    if not secret:
+    if not settings.state_manager_secret:
         raise ValueError("STATE_MANAGER_SECRET is not set")
     logger.info("secret initialized")