Add delete flag to external-resources

reconcile/external_resources/manager.py

@@ -1,4 +1,3 @@
-import json
 import logging
 from collections.abc import Iterable
 from datetime import UTC, datetime
@@ -43,6 +42,7 @@
 from reconcile.utils.secret_reader import SecretReaderBase
 
 FLAG_RESOURCE_MANAGED_BY_ERV2 = "managed_by_erv2"
+FLAG_DELETE_RESOURCE = "delete"
 
 
 def setup_factories(
@@ -157,13 +157,13 @@ def _resource_needs_reconciliation(
     def _get_desired_objects_reconciliations(self) -> set[Reconciliation]:
         r: set[Reconciliation] = set()
         for key, spec in self.er_inventory.items():
+            if spec.marked_to_delete:
+                continue
             module = self.module_inventory.get_from_spec(spec)
-
             try:
                 resource = self._build_external_resource(spec, self.er_inventory)
             except ExternalResourceValidationError as e:
-                k = ExternalResourceKey.from_spec(spec)
-                self.errors[k] = e
+                self.errors[key] = e
                 continue
 
             reconciliation = Reconciliation(
@@ -179,21 +179,23 @@ def _get_desired_objects_reconciliations(self) -> set[Reconciliation]:
         return r
 
     def _get_deleted_objects_reconciliations(self) -> set[Reconciliation]:
-        desired_keys = set(self.er_inventory.keys())
-        state_resource_keys = self.state_mgr.get_all_resource_keys()
-        deleted_keys = state_resource_keys - desired_keys
-        r: set[Reconciliation] = set()
+        to_reconcile: set[Reconciliation] = set()
+        deleted_keys = (k for k, v in self.er_inventory.items() if v.marked_to_delete)
         for key in deleted_keys:
             state = self.state_mgr.get_external_resource_state(key)
-            reconciliation = Reconciliation(
+            if state.resource_status == ResourceStatus.NOT_EXISTS:
+                logging.debug("Resource has already been removed. key: %s", key)
+                continue
+
+            r = Reconciliation(
                 key=key,
                 resource_hash=state.reconciliation.resource_hash,
                 module_configuration=state.reconciliation.module_configuration,
                 input=state.reconciliation.input,
                 action=Action.DESTROY,
             )
-            r.add(reconciliation)
-        return r
+            to_reconcile.add(r)
+        return to_reconcile
 
     def _update_in_progress_state(
         self, r: Reconciliation, state: ExternalResourceState
@@ -310,8 +312,8 @@ def _build_external_resource(
         return resource
 
     def _serialize_resource_input(self, resource: ExternalResource) -> str:
-        return json.dumps(
-            resource.dict(exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2}})
+        return resource.json(
+            exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2, FLAG_DELETE_RESOURCE}}
         )
 
     def handle_resources(self) -> None:

reconcile/external_resources/meta.py

@@ -11,3 +11,6 @@
 SECRET_ANN_IDENTIFIER = SECRET_ANN_PREFIX + "/identifier"
 SECRET_UPDATED_AT = SECRET_ANN_PREFIX + "/updated_at"
 SECRET_UPDATED_AT_TIMEFORMAT = "%Y-%m-%dT%H:%M:%SZ"
+
+FLAG_RESOURCE_MANAGED_BY_ERV2 = "managed_by_erv2"
+FLAG_DELETE_RESOURCE = "delete"

reconcile/external_resources/model.py

@@ -1,4 +1,3 @@
-import base64
 import hashlib
 import json
 from abc import (
@@ -88,7 +87,6 @@ def __init__(self, namespaces: Iterable[NamespaceV1]) -> None:
         ]
 
         for spec in desired_specs:
-            # self.set(ExternalResourceKey.from_spec(spec), spec)
             self._inventory[ExternalResourceKey.from_spec(spec)] = spec
 
     def __getitem__(self, key: ExternalResourceKey) -> ExternalResourceSpec | None:
@@ -239,6 +237,3 @@ def hash(self) -> str:
         return hashlib.md5(
             json.dumps(self.data, sort_keys=True).encode("utf-8")
         ).hexdigest()
-
-    def serialize_input(self) -> str:
-        return base64.b64encode(json.dumps(self.dict()).encode()).decode()

reconcile/gql_definitions/external_resources/external_resources_namespaces.gql

@@ -51,6 +51,7 @@ query ExternalResourcesNamespaces {
                     loss_impact
                 }
                 managed_by_erv2
+                delete
             }
             ... on NamespaceTerraformResourceS3_v1 {
                 region

reconcile/gql_definitions/external_resources/external_resources_namespaces.py

@@ -108,6 +108,7 @@
                     loss_impact
                 }
                 managed_by_erv2
+                delete
             }
             ... on NamespaceTerraformResourceS3_v1 {
                 region
@@ -552,6 +553,7 @@ class NamespaceTerraformResourceRDSV1(NamespaceTerraformResourceAWSV1):
     event_notifications: Optional[list[AWSRDSEventNotificationV1]] = Field(..., alias="event_notifications")
     data_classification: Optional[AWSRDSDataClassificationV1] = Field(..., alias="data_classification")
     managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
+    delete: Optional[bool] = Field(..., alias="delete")
 
 
 class AWSS3EventNotificationV1(ConfiguredBaseModel):

reconcile/gql_definitions/introspection.json

@@ -42173,6 +42173,18 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
+                        },
+                        {
+                            "name": "delete",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "Boolean",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,

reconcile/utils/external_resource_spec.py

@@ -92,6 +92,10 @@ class ExternalResourceSpec:
         init=False, compare=False, repr=False, hash=False, default_factory=lambda: {}
     )
 
+    @property
+    def marked_to_delete(self) -> bool:
+        return self.resource.get("delete") or False
+
     @property
     def provider(self) -> str:
         return self.resource["provider"]