[Core] Add support to create pages as part of integration setup (#308)

CHANGELOG.md

@@ -7,6 +7,17 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 <!-- towncrier release notes start -->
 
+## 0.4.13 (2023-12-31)
+
+### Features
+
+- Added capability to create pages as part of the integration setup (PORT-5689)
+
+### Improvements
+
+- Added integration and blueprints existence check before creating default resources (#1)
+- Added verbosity to diff deletion process after resync (#2)
+
 ## 0.4.12 (2023-12-22)
 
 

port_ocean/clients/port/mixins/blueprints.py

@@ -14,13 +14,15 @@ def __init__(self, auth: PortAuthentication, client: httpx.AsyncClient):
         self.auth = auth
         self.client = client
 
-    async def get_blueprint(self, identifier: str) -> Blueprint:
+    async def get_blueprint(
+        self, identifier: str, should_log: bool = True
+    ) -> Blueprint:
         logger.info(f"Fetching blueprint with id: {identifier}")
         response = await self.client.get(
             f"{self.auth.api_url}/blueprints/{identifier}",
             headers=await self.auth.headers(),
         )
-        handle_status_code(response)
+        handle_status_code(response, should_log=should_log)
         return Blueprint.parse_obj(response.json()["blueprint"])
 
     async def create_blueprint(
@@ -105,3 +107,30 @@ async def create_scorecard(
         )
 
         handle_status_code(response)
+
+    async def create_page(
+        self,
+        page: dict[str, Any],
+    ) -> dict[str, Any]:
+        logger.info(f"Creating page: {page}")
+        response = await self.client.post(
+            f"{self.auth.api_url}/pages",
+            json=page,
+            headers=await self.auth.headers(),
+        )
+
+        handle_status_code(response)
+        return page
+
+    async def delete_page(
+        self,
+        identifier: str,
+        should_raise: bool = False,
+    ) -> None:
+        logger.info(f"Deleting page: {identifier}")
+        response = await self.client.delete(
+            f"{self.auth.api_url}/pages/{identifier}",
+            headers=await self.auth.headers(),
+        )
+
+        handle_status_code(response, should_raise)

port_ocean/core/defaults/common.py

@@ -26,6 +26,7 @@ class Defaults(BaseModel):
     blueprints: list[dict[str, Any]] = []
     actions: list[Preset] = []
     scorecards: list[Preset] = []
+    pages: list[dict[str, Any]] = []
     port_app_config: Optional[PortAppConfig] = Field(
         default=None, alias="port-app-config"
     )
@@ -108,6 +109,7 @@ def get_port_integration_defaults(
         blueprints=default_jsons.get("blueprints", []),
         actions=default_jsons.get("actions", []),
         scorecards=default_jsons.get("scorecards", []),
+        pages=default_jsons.get("pages", []),
         port_app_config=port_app_config_class(
             **default_jsons.get("port-app-config", {})
         ),

port_ocean/core/defaults/initialize.py

@@ -2,6 +2,7 @@
 from typing import Type, Any
 
 import httpx
+from starlette import status
 from loguru import logger
 
 from port_ocean.clients.port.client import PortClient
@@ -10,6 +11,7 @@
 from port_ocean.context.ocean import ocean
 from port_ocean.core.defaults.common import Defaults, get_port_integration_defaults
 from port_ocean.core.handlers.port_app_config.models import PortAppConfig
+from port_ocean.core.models import Blueprint
 from port_ocean.exceptions.port_defaults import (
     AbortDefaultCreationError,
 )
@@ -52,10 +54,38 @@ async def _create_resources(
     defaults: Defaults,
     integration_config: IntegrationConfiguration,
 ) -> None:
+    response = await port_client._get_current_integration()
+    if response.status_code == status.HTTP_404_NOT_FOUND:
+        logger.info("Integration doesn't exist, creating new integration")
+    else:
+        logger.info("Integration already exists, skipping integration creation...")
+        return
+
     creation_stage, *blueprint_patches = deconstruct_blueprints_to_creation_steps(
         defaults.blueprints
     )
 
+    blueprints_results = await asyncio.gather(
+        *(
+            port_client.get_blueprint(blueprint["identifier"], should_log=False)
+            for blueprint in creation_stage
+        ),
+        return_exceptions=True,
+    )
+
+    existing_blueprints = [
+        result.identifier
+        for result in blueprints_results
+        if not isinstance(result, httpx.HTTPStatusError)
+        and isinstance(result, Blueprint)
+    ]
+
+    if existing_blueprints:
+        logger.info(
+            f"Blueprints already exist: {existing_blueprints}. Skipping integration default creation..."
+        )
+        return
+
     create_results = await asyncio.gather(
         *(
             port_client.create_blueprint(
@@ -81,7 +111,7 @@ async def _create_resources(
                 )
 
         raise AbortDefaultCreationError(created_blueprints, errors)
-
+    created_pages = []
     try:
         for patch_stage in blueprint_patches:
             await asyncio.gather(
@@ -111,16 +141,42 @@ async def _create_resources(
             )
         )
 
+        create_pages_result = await asyncio.gather(
+            *(port_client.create_page(page) for page in defaults.pages),
+            return_exceptions=True,
+        )
+
+        created_pages = [
+            result.get("identifier", "")
+            for result in create_pages_result
+            if not isinstance(result, BaseException)
+        ]
+
+        pages_errors = [
+            result for result in create_pages_result if isinstance(result, Exception)
+        ]
+
+        if pages_errors:
+            for error in pages_errors:
+                if isinstance(error, httpx.HTTPStatusError):
+                    logger.warning(
+                        f"Failed to create resources: {error.response.text}. Rolling back changes..."
+                    )
+
+            raise AbortDefaultCreationError(
+                created_blueprints, pages_errors, created_pages
+            )
+
         await port_client.create_integration(
             integration_config.integration.type,
             integration_config.event_listener.to_request(),
             port_app_config=defaults.port_app_config,
         )
-    except httpx.HTTPStatusError as e:
+    except httpx.HTTPStatusError as err:
         logger.error(
-            f"Failed to create resources: {e.response.text}. Rolling back changes..."
+            f"Failed to create resources: {err.response.text}. Rolling back changes..."
         )
-        raise AbortDefaultCreationError(created_blueprints, [e])
+        raise AbortDefaultCreationError(created_blueprints, [err], created_pages)
 
 
 async def _initialize_defaults(
@@ -133,6 +189,7 @@ async def _initialize_defaults(
         return None
 
     try:
+        logger.info("Found default resources, starting creation process")
         await _create_resources(port_client, defaults, integration_config)
     except AbortDefaultCreationError as e:
         logger.warning(
@@ -148,6 +205,18 @@ async def _initialize_defaults(
                 for identifier in e.blueprints_to_rollback
             )
         )
+        if e.pages_to_rollback:
+            logger.warning(
+                f"Failed to create resources. Rolling back pages : {e.pages_to_rollback}"
+            )
+            await asyncio.gather(
+                *(
+                    port_client.delete_page(
+                        identifier,
+                    )
+                    for identifier in e.pages_to_rollback
+                )
+            )
 
         raise ExceptionGroup(str(e), e.errors)
 

port_ocean/core/handlers/entities_state_applier/port/applier.py

@@ -130,6 +130,7 @@ async def apply_diff(
         logger.info("Upserting modified entities")
         await self.upsert(diff.modified, user_agent_type)
 
+        logger.info("Deleting diff entities")
         await self._delete_diff(
             diff.deleted, diff.created + diff.modified, user_agent_type
         )
@@ -149,6 +150,7 @@ async def delete_diff(
         )
         await self._validate_entity_diff(diff)
 
+        logger.info("Deleting diff entities")
         await self._delete_diff(
             diff.deleted, diff.created + diff.modified, user_agent_type
         )

port_ocean/core/integrations/mixins/sync_raw.py

@@ -352,6 +352,7 @@ async def sync_raw_all(
             except asyncio.CancelledError as e:
                 logger.warning("Resync aborted successfully")
             else:
+                logger.info("Starting resync diff calculation")
                 flat_created_entities, errors = zip_and_sum(creation_results) or [
                     [],
                     [],
@@ -368,6 +369,9 @@ async def sync_raw_all(
 
                     logger.error(message, exc_info=error_group)
                 else:
+                    logger.info(
+                        f"Running resync diff calculation, number of entities at Port before resync: {len(entities_at_port)}, number of entities created during sync: {len(flat_created_entities)}"
+                    )
                     await self.entities_state_applier.delete_diff(
                         {"before": entities_at_port, "after": flat_created_entities},
                         user_agent_type,

port_ocean/exceptions/port_defaults.py

@@ -2,8 +2,14 @@
 
 
 class AbortDefaultCreationError(BaseOceanException):
-    def __init__(self, blueprints_to_rollback: list[str], errors: list[Exception]):
+    def __init__(
+        self,
+        blueprints_to_rollback: list[str],
+        errors: list[Exception],
+        pages_to_rollback: list[str] | None = None,
+    ):
         self.blueprints_to_rollback = blueprints_to_rollback
+        self.pages_to_rollback = pages_to_rollback
         self.errors = errors
         super().__init__("Aborting defaults creation")
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "port-ocean"
-version = "0.4.12"
+version = "0.4.13"
 description = "Port Ocean is a CLI tool for managing your Port projects."
 readme = "README.md"
 homepage = "https://app.getport.io"