feat: enhance provider and deduplication rule provisioning logic

docs/deployment/provision/overview.mdx

@@ -27,7 +27,8 @@ Provisioning in Keep is controlled through environment variables and configurati
 | Provisioning Type      | Environment Variable           | Purpose                                                                   |
 | ---------------------- | ------------------------------ | ------------------------------------------------------------------------- |
 | **Provider**           | `KEEP_PROVIDERS`               | JSON string containing provider configurations with deduplication rules   |
-| **Workflow**           | `KEEP_WORKFLOW`                | One workflow to provision right from the env variable.                    |
+| **Providers**          | `KEEP_PROVIDERS_DIRECTORY`     | Directory path containing provider configuration files                    |
+| **Workflow**           | `KEEP_WORKFLOW`                | One workflow to provision right from the env variable                     |
 | **Workflows**          | `KEEP_WORKFLOWS_DIRECTORY`     | Directory path containing workflow configuration files                    |
 | **Dashboard**          | `KEEP_DASHBOARDS`              | JSON string containing dashboard configurations                           |
 

docs/deployment/provision/provider.mdx

@@ -99,29 +99,15 @@ deduplication_rules:
 Keep supports a wide range of provider types. Each provider type has its own specific configuration requirements.
 To see the full list of supported providers and their detailed configuration options, please refer to our comprehensive provider documentation.
 
-
 ### Update Provisioned Providers
 
-#### Using KEEP_PROVIDERS
-
-Provider configurations can be updated dynamically by changing the `KEEP_PROVIDERS` environment variable.
-
-On every restart, Keep reads this environment variable and determines which providers need to be added or removed.
-
-This process allows for flexible management of data sources without requiring manual intervention. By simply updating the `KEEP_PROVIDERS` variable and restarting the application, you can efficiently add new providers, remove existing ones, or modify their configurations.
-
-The high-level provisioning mechanism:
-1. Keep reads the `KEEP_PROVIDERS` value.
-2. Keep checks if there are any provisioned providers that are no longer in the `KEEP_PROVIDERS` value, and deletes them.
-3. Keep installs all providers from the `KEEP_PROVIDERS` value.
-
-#### Using KEEP_PROVIDERS_DIRECTORY
+Keep uses a consistent process for updating provider configurations regardless of whether you use `KEEP_PROVIDERS` or `KEEP_PROVIDERS_DIRECTORY`.
 
-Provider configurations can be updated dynamically by changing the YAML files in the `KEEP_PROVIDERS_DIRECTORY` directory.
+#### Provisioning Process
 
-On every restart, Keep reads the YAML files in the `KEEP_PROVIDERS_DIRECTORY` directory and determines which providers need to be added or removed.
+When Keep starts or restarts, it follows these steps to manage provider configurations:
 
-The high-level provisioning mechanism:
-1. Keep reads the YAML files in the `KEEP_PROVIDERS_DIRECTORY` directory.
-2. Keep checks if there are any provisioned providers that are no longer in the YAML files, and deletes them.
-3. Keep installs all providers from the YAML files.
+1. **Read Configurations**: Loads provider definitions from either the `KEEP_PROVIDERS` environment variable or YAML files in the `KEEP_PROVIDERS_DIRECTORY`.
+2. **Create New Providers**: Installs any providers listed in the configuration that are not already present.
+3. **Update When Changed**: If an existing provider's configuration has changed, Keep reapplies the configuration, including deduplication rules. If errors occur during this update, changes are automatically rolled back.
+4. **Delete Providers**: Deletes any currently installed providers that are not found in the loaded configuration.

keep/api/alert_deduplicator/deduplication_rules_provisioning.py

@@ -4,23 +4,28 @@
 
 import keep.api.core.db as db
 from keep.api.core.config import config
-from keep.providers.providers_factory import ProvidersFactory
+from keep.api.models.db.provider import Provider
 
 logger = logging.getLogger(__name__)
 
 
-def provision_deduplication_rules(deduplication_rules: dict[str, any], tenant_id: str):
+def provision_deduplication_rules(
+    deduplication_rules: dict[str, any], tenant_id: str, provider: Provider
+):
     """
     Provisions deduplication rules for a given tenant.
 
     Args:
         deduplication_rules (dict[str, any]): A dictionary where the keys are rule names and the values are
-        DeduplicationRuleRequestDto objects.
+            DeduplicationRuleRequestDto objects.
         tenant_id (str): The ID of the tenant for which deduplication rules are being provisioned.
+        provider (Provider): The provider for which the deduplication rules are being provisioned.
     """
-    enrich_with_providers_info(deduplication_rules, tenant_id)
+    enrich_with_provider_info(deduplication_rules, provider)
 
-    all_deduplication_rules_from_db = db.get_all_deduplication_rules(tenant_id)
+    all_deduplication_rules_from_db = db.get_all_deduplication_rules_by_provider(
+        tenant_id, provider.id, provider.type
+    )
     provisioned_deduplication_rules = [
         rule for rule in all_deduplication_rules_from_db if rule.is_provisioned
     ]
@@ -29,17 +34,6 @@
     }
     actor = "system"
 
-    # delete rules that are not in the env
-    for provisioned_deduplication_rule in provisioned_deduplication_rules:
-        if str(provisioned_deduplication_rule.name) not in deduplication_rules:
-            logger.info(
-                "Deduplication rule with name '%s' is not in the env, deleting from DB",
-                provisioned_deduplication_rule.name,
-            )
-            db.delete_deduplication_rule(
-                rule_id=str(provisioned_deduplication_rule.id), tenant_id=tenant_id
-            )
-
     for (
         deduplication_rule_name,
         deduplication_rule_to_provision,
@@ -97,47 +91,36 @@
             is_provisioned=True,
         )
 
+        logger.info(
+            "Provisioned deduplication rules %s successfully",
+            deduplication_rule_name,
+        )
 
-def provision_deduplication_rules_from_env(tenant_id: str):
-    """
-    Provisions deduplication rules from environment variables for a given tenant.
-    This function reads deduplication rules from environment variables, validates them,
-    and then provisions them into the database. It handles the following:
-    - Deletes deduplication rules from the database that are not present in the environment variables.
-    - Updates existing deduplication rules in the database if they are present in the environment variables.
-    - Creates new deduplication rules in the database if they are not already present.
-    Args:
-        tenant_id (str): The ID of the tenant for which deduplication rules are being provisioned.
-    Raises:
-        ValueError: If the deduplication rules from the environment variables are invalid.
-    """
-
-    deduplication_rules_from_env_dict = get_deduplication_rules_to_provision()
-
-    if not deduplication_rules_from_env_dict:
-        logger.info("No deduplication rules found in env. Nothing to provision.")
-        return
-
-    provision_deduplication_rules(deduplication_rules_from_env_dict, tenant_id)
+    # Delete provisioned deduplication rules that are not provisioned anymore
+    for rule_name, rule in provisioned_deduplication_rules_from_db_dict.items():
+        if rule_name not in deduplication_rules:
+            logger.info(
+                "Deduplication rule with name '%s' no longer in configuration, deleting from DB",
+                rule_name,
+            )
+            db.delete_deduplication_rule(rule_id=str(rule.id), tenant_id=tenant_id)
+            logger.info(
+                "Deleted deduplication rule %s successfully",
+                rule_name,
+            )
 
 
-def enrich_with_providers_info(deduplication_rules: dict[str, any], tenant_id: str):
+def enrich_with_provider_info(deduplication_rules: dict[str, any], provider: Provider):
     """
     Enriches passed deduplication rules with provider ID and type information.
 
     Args:
         deduplication_rules (dict[str, any]): A list of deduplication rules to be enriched.
-        tenant_id (str): The ID of the tenant for which deduplication rules are being provisioned.
+        provider (Provider): The provider for which the deduplication rules are being provisioned.
     """
 
-    installed_providers = ProvidersFactory.get_installed_providers(tenant_id)
-    installed_providers_dict = {
-        provider.details.get("name"): provider for provider in installed_providers
-    }
-
     for rule_name, rule in deduplication_rules.items():
         logger.info(f"Enriching deduplication rule: {rule_name}")
-        provider = installed_providers_dict.get(rule.get("provider_name"))
         rule["provider_id"] = provider.id
         rule["provider_type"] = provider.type
 

keep/api/config.py

@@ -2,9 +2,6 @@
 import os
 
 import keep.api.logging
-from keep.api.alert_deduplicator.deduplication_rules_provisioning import (
-    provision_deduplication_rules_from_env,
-)
 from keep.api.api import AUTH_TYPE
 from keep.api.core.db_on_start import migrate_db, try_create_single_tenant
 from keep.api.core.dependencies import SINGLE_TENANT_UUID
@@ -33,9 +30,6 @@ def provision_resources():
         logger.info("Workflows provisioned successfully")
         provision_dashboards(SINGLE_TENANT_UUID)
         logger.info("Dashboards provisioned successfully")
-        logger.info("Provisioning deduplication rules")
-        provision_deduplication_rules_from_env(SINGLE_TENANT_UUID)
-        logger.info("Deduplication rules provisioned successfully")
     else:
         logger.info("Provisioning resources is disabled")
 

keep/api/core/db.py

@@ -40,7 +40,7 @@
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.dialects.sqlite import insert as sqlite_insert
 from sqlalchemy.exc import IntegrityError, OperationalError
-from sqlalchemy.orm import joinedload, subqueryload, foreign
+from sqlalchemy.orm import foreign, joinedload, subqueryload
 from sqlalchemy.orm.exc import StaleDataError
 from sqlalchemy.sql import exists, expression
 from sqlmodel import Session, SQLModel, col, or_, select, text
@@ -2354,6 +2354,18 @@
     return rules
 
 
+def get_all_deduplication_rules_by_provider(tenant_id, provider_id, provider_type):
+    with Session(engine) as session:
+        rules = session.exec(
+            select(AlertDeduplicationRule).where(
+                AlertDeduplicationRule.tenant_id == tenant_id,
+                AlertDeduplicationRule.provider_id == provider_id,
+                AlertDeduplicationRule.provider_type == provider_type,
+            )
+        ).all()
+    return rules
+
+
 def get_deduplication_rule_by_id(tenant_id, rule_id: str):
     rule_uuid = __convert_to_uuid(rule_id)
     if not rule_uuid: