diff --git a/lib/charms/data_platform_libs/v0/data_interfaces.py b/lib/charms/data_platform_libs/v0/data_interfaces.py index 109603ad49..a3122b144a 100644 --- a/lib/charms/data_platform_libs/v0/data_interfaces.py +++ b/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -16,7 +16,7 @@ This library contains the Requires and Provides classes for handling the relation between an application and multiple managed application supported by the data-team: -MySQL, Postgresql, MongoDB, Redis, and Kafka. +MySQL, Postgresql, MongoDB, Redis, Kafka, and Karapace. ### Database (MySQL, Postgresql, MongoDB, and Redis) @@ -34,6 +34,7 @@ from charms.data_platform_libs.v0.data_interfaces import ( DatabaseCreatedEvent, DatabaseRequires, + DatabaseEntityCreatedEvent, ) class ApplicationCharm(CharmBase): @@ -45,6 +46,7 @@ def __init__(self, *args): # Charm events defined in the database requires charm library. self.database = DatabaseRequires(self, relation_name="database", database_name="database") self.framework.observe(self.database.on.database_created, self._on_database_created) + self.framework.observe(self.database.on.database_entity_created, self._on_database_entity_created) def _on_database_created(self, event: DatabaseCreatedEvent) -> None: # Handle the created database @@ -61,12 +63,17 @@ def _on_database_created(self, event: DatabaseCreatedEvent) -> None: # Set active status self.unit.status = ActiveStatus("received database credentials") + + def _on_database_entity_created(self, event: DatabaseEntityCreatedEvent) -> None: + # Handle the created entity + ... ``` As shown above, the library provides some custom events to handle specific situations, which are listed below: - database_created: event emitted when the requested database is created. +- database_entity_created: event emitted when the requested entity is created. - endpoints_changed: event emitted when the read/write endpoints of the database have changed. - read_only_endpoints_changed: event emitted when the read-only endpoints of the database have changed. Event is not triggered if read/write endpoints changed too. @@ -141,7 +148,6 @@ def _on_cluster2_database_created(self, event: DatabaseCreatedEvent) -> None: event.endpoints, ) ... - ``` When it's needed to check whether a plugin (extension) is enabled on the PostgreSQL @@ -154,7 +160,6 @@ def _on_cluster2_database_created(self, event: DatabaseCreatedEvent) -> None: charm: charm-binary-python-packages: - psycopg[binary] - ``` ### Provider Charm @@ -187,6 +192,7 @@ def _on_database_requested(self, event: DatabaseRequestedEvent) -> None: self.provided_database.set_credentials(event.relation.id, username, password) # set other variables for the relation event.set_tls("False") ``` + As shown above, the library provides a custom event (database_requested) to handle the situation when an application charm requests a new database to be created. It's preferred to subscribe to this event instead of relation changed event to avoid @@ -207,6 +213,7 @@ def _on_database_requested(self, event: DatabaseRequestedEvent) -> None: BootstrapServerChangedEvent, KafkaRequires, TopicCreatedEvent, + TopicEntityCreatedEvent, ) class ApplicationCharm(CharmBase): @@ -220,6 +227,9 @@ def __init__(self, *args): self.framework.observe( self.kafka.on.topic_created, self._on_kafka_topic_created ) + self.framework.observe( + self.kafka.on.topic_entity_created, self._on_kafka_topic_entity_created + ) def _on_kafka_bootstrap_server_changed(self, event: BootstrapServerChangedEvent): # Event triggered when a bootstrap server was changed for this application @@ -238,6 +248,9 @@ def _on_kafka_topic_created(self, event: TopicCreatedEvent): zookeeper_uris = event.zookeeper_uris ... + def _on_kafka_topic_entity_created(self, event: TopicEntityCreatedEvent): + # Event triggered when an entity was created for this application + ... ``` As shown above, the library provides some custom events to handle specific situations, @@ -268,6 +281,7 @@ def __init__(self, *args): # Charm events defined in the Kafka Provides charm library. self.kafka_provider = KafkaProvides(self, relation_name="kafka_client") self.framework.observe(self.kafka_provider.on.topic_requested, self._on_topic_requested) + self.framework.observe(self.kafka_provider.on.topic_entity_requested, self._on_entity_requested) # Kafka generic helper self.kafka = KafkaHelper() @@ -283,12 +297,114 @@ def _on_topic_requested(self, event: TopicRequestedEvent): self.kafka_provider.set_tls(relation_id, "False") self.kafka_provider.set_zookeeper_uris(relation_id, ...) + def _on_entity_requested(self, event: EntityRequestedEvent): + # Handle the on_topic_entity_requested event. + ... ``` As shown above, the library provides a custom event (topic_requested) to handle the situation when an application charm requests a new topic to be created. It is preferred to subscribe to this event instead of relation changed event to avoid creating a new topic when other information other than a topic name is exchanged in the relation databag. + +### Karapace + +This library is the interface to use and interact with the Karapace charm. This library contains +custom events that add convenience to manage Karapace, and provides methods to consume the +application related data. + +#### Requirer Charm + +```python + +from charms.data_platform_libs.v0.data_interfaces import ( + EndpointsChangedEvent, + KarapaceRequires, + SubjectAllowedEvent, +) + +class ApplicationCharm(CharmBase): + + def __init__(self, *args): + super().__init__(*args) + self.karapace = KarapaceRequires(self, relation_name="karapace_client", subject="test-subject") + self.framework.observe( + self.karapace.on.server_changed, self._on_karapace_server_changed + ) + self.framework.observe( + self.karapace.on.subject_allowed, self._on_karapace_subject_allowed + ) + self.framework.observe( + self.karapace.on.subject_entity_created, self._on_subject_entity_created + ) + + + def _on_karapace_server_changed(self, event: EndpointsChangedEvent): + # Event triggered when a server endpoint was changed for this application + new_server = event.endpoints + ... + + def _on_karapace_subject_allowed(self, event: SubjectAllowedEvent): + # Event triggered when a subject was allowed for this application + username = event.username + password = event.password + tls = event.tls + endpoints = event.endpoints + ... + + def _on_subject_entity_created(self, event: SubjectEntityCreatedEvent): + # Event triggered when a subject entity was created this application + entity_name = event.entity_name + entity_password = event.entity_password + ... +``` + +As shown above, the library provides some custom events to handle specific situations, +which are listed below: + +- subject_allowed: event emitted when the requested subject is allowed. +- server_changed: event emitted when the server endpoints have changed. + +#### Provider Charm + +Following the previous example, this is an example of the provider charm. + +```python +class SampleCharm(CharmBase): + +from charms.data_platform_libs.v0.data_interfaces import ( + KarapaceProvides, + SubjectRequestedEvent, +) + + def __init__(self, *args): + super().__init__(*args) + + # Default charm events. + self.framework.observe(self.on.start, self._on_start) + + # Charm events defined in the Karapace Provides charm library. + self.karapace_provider = KarapaceProvides(self, relation_name="karapace_client") + self.framework.observe(self.karapace_provider.on.subject_requested, self._on_subject_requested) + # Karapace generic helper + self.karapace = KarapaceHelper() + + def _on_subject_requested(self, event: SubjectRequestedEvent): + # Handle the on_subject_requested event. + + subject = event.subject + relation_id = event.relation.id + # set connection info in the databag relation + self.karapace_provider.set_endpoint(relation_id, self.karapace.get_endpoint()) + self.karapace_provider.set_credentials(relation_id, username=username, password=password) + self.karapace_provider.set_tls(relation_id, "False") +``` + +As shown above, the library provides a custom event (subject_requested) to handle +the situation when an application charm requests a new subject to be created. +It is preferred to subscribe to this event instead of relation changed event to avoid +creating a new subject when other information other than a subject name is +exchanged in the relation databag. """ import copy @@ -301,6 +417,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): from typing import ( Callable, Dict, + Final, ItemsView, KeysView, List, @@ -331,7 +448,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 49 +LIBPATCH = 54 PYDEPS = ["ops>=2.0.0"] @@ -349,6 +466,8 @@ def _on_topic_requested(self, event: TopicRequestedEvent): changed - keys that still exist but have new values deleted - key that were deleted""" +ENTITY_USER = "USER" +ENTITY_GROUP = "GROUP" PROV_SECRET_PREFIX = "secret-" PROV_SECRET_FIELDS = "provided-secrets" @@ -587,6 +706,7 @@ def __init__(self): self.USER = SecretGroup("user") self.TLS = SecretGroup("tls") self.MTLS = SecretGroup("mtls") + self.ENTITY = SecretGroup("entity") self.EXTRA = SecretGroup("extra") def __setattr__(self, name, value): @@ -953,7 +1073,7 @@ def get(self, key: str, default: Optional[str] = None) -> Optional[str]: class Data(ABC): - """Base relation data mainpulation (abstract) class.""" + """Base relation data manipulation (abstract) class.""" SCOPE = Scope.APP @@ -966,6 +1086,8 @@ class Data(ABC): "tls": SECRET_GROUPS.TLS, "tls-ca": SECRET_GROUPS.TLS, "mtls-cert": SECRET_GROUPS.MTLS, + "entity-name": SECRET_GROUPS.ENTITY, + "entity-password": SECRET_GROUPS.ENTITY, } SECRET_FIELDS = [] @@ -1729,6 +1851,24 @@ def set_credentials(self, relation_id: int, username: str, password: str) -> Non """ self.update_relation_data(relation_id, {"username": username, "password": password}) + def set_entity_credentials( + self, relation_id: int, entity_name: str, entity_password: Optional[str] = None + ) -> None: + """Set entity credentials. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + entity_name: name of the created entity + entity_password: password of the created entity. + """ + self.update_relation_data( + relation_id, + {"entity-name": entity_name, "entity-password": entity_password}, + ) + def set_tls(self, relation_id: int, tls: str) -> None: """Set whether TLS is enabled. @@ -1766,7 +1906,16 @@ def _load_secrets_from_databag(self, relation: Relation) -> None: class RequirerData(Data): """Requirer-side of the relation.""" - SECRET_FIELDS = ["username", "password", "tls", "tls-ca", "uris", "read-only-uris"] + SECRET_FIELDS = [ + "username", + "password", + "tls", + "tls-ca", + "uris", + "read-only-uris", + "entity-name", + "entity-password", + ] def __init__( self, @@ -1774,10 +1923,39 @@ def __init__( relation_name: str, extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, + requested_entity_secret: Optional[str] = None, + requested_entity_name: Optional[str] = None, + requested_entity_password: Optional[str] = None, ): """Manager of base client relations.""" super().__init__(model, relation_name) self.extra_user_roles = extra_user_roles + self.extra_group_roles = extra_group_roles + self.entity_type = entity_type + self.entity_permissions = entity_permissions + self.requested_entity_secret = requested_entity_secret + self.requested_entity_name = requested_entity_name + self.requested_entity_password = requested_entity_password + + if ( + self.requested_entity_secret or self.requested_entity_name + ) and not self.secrets_enabled: + raise SecretsUnavailableError("Secrets unavailable on current Juju version") + + if self.requested_entity_secret and ( + self.requested_entity_name or self.requested_entity_password + ): + raise IllegalOperationError("Unable to use provided and automated entity name secret") + + if self.requested_entity_password and not self.requested_entity_name: + raise IllegalOperationError("Unable to set entity password without an entity name") + + self._validate_entity_type() + self._validate_entity_permissions() + self._remote_secret_fields = list(self.SECRET_FIELDS) self._local_secret_fields = [ field @@ -1788,18 +1966,52 @@ def __init__( self._remote_secret_fields += additional_secret_fields self.data_component = self.local_unit - # Internal helper functions + # Internal functions def _is_resource_created_for_relation(self, relation: Relation) -> bool: if not relation.app: return False - data = self.fetch_relation_data([relation.id], ["username", "password"]).get( - relation.id, {} + data = self.fetch_relation_data( + [relation.id], + ["username", "password", "entity-name", "entity-password"], + ).get(relation.id, {}) + + return any( + [ + all(bool(data.get(field)) for field in ("username", "password")), + all(bool(data.get(field)) for field in ("entity-name",)), + ] ) - return bool(data.get("username")) and bool(data.get("password")) + + def _validate_entity_type(self) -> None: + """Validates the consistency of the provided entity-type and its extra roles.""" + if self.entity_type and self.entity_type not in {ENTITY_USER, ENTITY_GROUP}: + raise ValueError("Invalid entity-type. Possible values are USER and GROUP") + + if self.entity_type == ENTITY_USER and self.extra_group_roles: + raise ValueError("Inconsistent entity information. Use extra_user_roles instead") + + if self.entity_type == ENTITY_GROUP and self.extra_user_roles: + raise ValueError("Inconsistent entity information. Use extra_group_roles instead") + + def _validate_entity_permissions(self) -> None: + """Validates whether the provided entity permissions follow the right JSON format.""" + if not self.entity_permissions: + return + + accepted_keys = {"resource_name", "resource_type", "privileges"} + + try: + permissions = json.loads(self.entity_permissions) + for permission in permissions: + if permission.keys() != accepted_keys: + raise ValueError("Invalid entity permissions format. See accepted keys") + except json.decoder.JSONDecodeError: + raise ValueError("Invalid entity permissions format. It must be JSON format") # Public functions + def is_resource_created(self, relation_id: Optional[int] = None) -> bool: """Check if the resource has been created. @@ -1856,6 +2068,26 @@ def __init__(self, charm: CharmBase, relation_data: RequirerData, unique_key: st """Manager of base client relations.""" super().__init__(charm, relation_data, unique_key) + def _main_credentials_shared(self, diff: Diff) -> bool: + """Whether the relation data-bag contains username / password keys.""" + user_secret = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) + return any( + [ + user_secret in diff.added, + "username" in diff.added and "password" in diff.added, + ] + ) + + def _entity_credentials_shared(self, diff: Diff) -> bool: + """Whether the relation data-bag contains rolename / password keys.""" + entity_secret = self.relation_data._generate_secret_field_name(SECRET_GROUPS.ENTITY) + return any( + [ + entity_secret in diff.added, + "entity-name" in diff.added, + ] + ) + # Event handlers def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: @@ -1902,6 +2134,21 @@ def __init__(self, charm: CharmBase, relation_data: ProviderData, unique_key: st """Manager of base client relations.""" super().__init__(charm, relation_data, unique_key) + @staticmethod + def _validate_entity_consistency(event: RelationEvent, diff: Diff) -> None: + """Validates that entity information is not changed after relation is established. + + - When entity-type changes, backwards compatibility is broken. + - When extra-user-roles changes, role membership checks become incredibly complex. + - When extra-group-roles changes, role membership checks become incredibly complex. + """ + if not isinstance(event, RelationChangedEvent): + return + + for key in ["entity-type", "extra-user-roles", "extra-group-roles"]: + if key in diff.changed: + raise ValueError(f"Cannot change {key} after relation has already been created") + # Event handlers def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: @@ -1931,7 +2178,6 @@ def __init__( self, model, relation_name: str, - extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], additional_secret_group_mapping: Dict[str, str] = {}, secret_field_name: Optional[str] = None, @@ -1939,10 +2185,9 @@ def __init__( ): RequirerData.__init__( self, - model, - relation_name, - extra_user_roles, - additional_secret_fields, + model=model, + relation_name=relation_name, + additional_secret_fields=additional_secret_fields, ) self.secret_field_name = secret_field_name if secret_field_name else self.SECRET_FIELD_NAME self.deleted_label = deleted_label @@ -2006,6 +2251,7 @@ def current_secret_fields(self) -> List[str]: SECRET_GROUPS.get_group("user"), SECRET_GROUPS.get_group("tls"), SECRET_GROUPS.get_group("mtls"), + SECRET_GROUPS.get_group("entity"), ] for group in SECRET_GROUPS.groups(): if group in ignores: @@ -2458,7 +2704,6 @@ def __init__( self, charm, relation_name: str, - extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], additional_secret_group_mapping: Dict[str, str] = {}, secret_field_name: Optional[str] = None, @@ -2469,7 +2714,6 @@ def __init__( self, charm.model, relation_name, - extra_user_roles, additional_secret_fields, additional_secret_group_mapping, secret_field_name, @@ -2494,7 +2738,6 @@ def __init__( self, charm, relation_name: str, - extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], additional_secret_group_mapping: Dict[str, str] = {}, secret_field_name: Optional[str] = None, @@ -2505,7 +2748,6 @@ def __init__( self, charm.model, relation_name, - extra_user_roles, additional_secret_fields, additional_secret_group_mapping, secret_field_name, @@ -2548,7 +2790,6 @@ def __init__( unit: Unit, charm: CharmBase, relation_name: str, - extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], additional_secret_group_mapping: Dict[str, str] = {}, secret_field_name: Optional[str] = None, @@ -2559,7 +2800,6 @@ def __init__( unit, charm.model, relation_name, - extra_user_roles, additional_secret_fields, additional_secret_group_mapping, secret_field_name, @@ -2575,18 +2815,6 @@ def __init__( # Generic events -class ExtraRoleEvent(RelationEvent): - """Base class for data events.""" - - @property - def extra_user_roles(self) -> Optional[str]: - """Returns the extra user roles that were requested.""" - if not self.relation.app: - return None - - return self.relation.data[self.relation.app].get("extra-user-roles") - - class RelationEventWithSecret(RelationEvent): """Base class for Relation Events that need to handle secrets.""" @@ -2618,6 +2846,76 @@ def secrets_enabled(self): return JujuVersion.from_environ().has_secrets +class EntityProvidesEvent(RelationEvent): + """Base class for data events.""" + + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-user-roles") + + @property + def extra_group_roles(self) -> Optional[str]: + """Returns the extra group roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-group-roles") + + @property + def entity_type(self) -> Optional[str]: + """Returns the entity_type that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("entity-type") + + @property + def entity_permissions(self) -> Optional[str]: + """Returns the entity_permissions that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("entity-permissions") + + +class EntityRequiresEvent(RelationEventWithSecret): + """Base class for authentication fields for events. + + The amount of logic added here is not ideal -- but this was the only way to preserve + the interface when moving to Juju Secrets + """ + + @property + def entity_name(self) -> Optional[str]: + """Returns the name for the created entity.""" + if not self.relation.app: + return None + + if self.secrets_enabled: + secret = self._get_secret("entity") + if secret: + return secret.get("entity-name") + + return self.relation.data[self.relation.app].get("entity-name") + + @property + def entity_password(self) -> Optional[str]: + """Returns the password for the created entity.""" + if not self.relation.app: + return None + + if self.secrets_enabled: + secret = self._get_secret("entity") + if secret: + return secret.get("entity-password") + + return self.relation.data[self.relation.app].get("entity-password") + + class AuthenticationEvent(RelationEventWithSecret): """Base class for authentication fields for events. @@ -2693,9 +2991,17 @@ def database(self) -> Optional[str]: return self.relation.data[self.relation.app].get("database") -class DatabaseRequestedEvent(DatabaseProvidesEvent, ExtraRoleEvent): +class DatabaseRequestedEvent(DatabaseProvidesEvent): """Event emitted when a new database is requested for use on this relation.""" + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-user-roles") + @property def external_node_connectivity(self) -> bool: """Returns the requested external_node_connectivity field.""" @@ -2707,6 +3013,29 @@ def external_node_connectivity(self) -> bool: == "true" ) + @property + def requested_entity_secret_content(self) -> Optional[Dict[str, Optional[str]]]: + """Returns the content of the requested entity secret.""" + names = None + if secret_uri := self.relation.data.get(self.relation.app, {}).get( + "requested-entity-secret" + ): + secret = self.framework.model.get_secret(id=secret_uri) + if content := secret.get_content(refresh=True): + if "entity-name" in content: + names = {content["entity-name"]: content.get("password")} + else: + logger.warning("Invalid requested-entity-secret: no entity name") + return names + + +class DatabaseEntityRequestedEvent(DatabaseProvidesEvent, EntityProvidesEvent): + """Event emitted when a new entity is requested for use on this relation.""" + + +class DatabaseEntityPermissionsChangedEvent(DatabaseProvidesEvent, EntityProvidesEvent): + """Event emitted when existing entity permissions are changed on this relation.""" + class DatabaseProvidesEvents(CharmEvents): """Database events. @@ -2715,6 +3044,8 @@ class DatabaseProvidesEvents(CharmEvents): """ database_requested = EventSource(DatabaseRequestedEvent) + database_entity_requested = EventSource(DatabaseEntityRequestedEvent) + database_entity_permissions_changed = EventSource(DatabaseEntityPermissionsChangedEvent) class DatabaseRequiresEvent(RelationEventWithSecret): @@ -2808,6 +3139,10 @@ class DatabaseCreatedEvent(AuthenticationEvent, DatabaseRequiresEvent): """Event emitted when a new database is created for use on this relation.""" +class DatabaseEntityCreatedEvent(EntityRequiresEvent, DatabaseRequiresEvent): + """Event emitted when a new entity is created for use on this relation.""" + + class DatabaseEndpointsChangedEvent(AuthenticationEvent, DatabaseRequiresEvent): """Event emitted when the read/write endpoints are changed.""" @@ -2823,6 +3158,7 @@ class DatabaseRequiresEvents(CharmEvents): """ database_created = EventSource(DatabaseCreatedEvent) + database_entity_created = EventSource(DatabaseEntityCreatedEvent) endpoints_changed = EventSource(DatabaseEndpointsChangedEvent) read_only_endpoints_changed = EventSource(DatabaseReadOnlyEndpointsChangedEvent) @@ -2944,16 +3280,47 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # Leader only if not self.relation_data.local_unit.is_leader(): return + # Check which data has changed to emit customs events. diff = self._diff(event) - # Emit a database requested event if the setup key (database name and optional - # extra user roles) was added to the relation databag by the application. - if "database" in diff.added: + # Validate entity information is not dynamically changed + self._validate_entity_consistency(event, diff) + + # Emit a database requested event if the setup key (database name) + # was added to the relation databag, but the entity-type key was not. + if "database" in diff.added and "entity-type" not in diff.added: getattr(self.on, "database_requested").emit( event.relation, app=event.app, unit=event.unit ) + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit an entity requested event if the setup key (database name) + # was added to the relation databag, in addition to the entity-type key. + if "database" in diff.added and "entity-type" in diff.added: + getattr(self.on, "database_entity_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit a permissions changed event if the setup key (database name) + # was added to the relation databag, and the entity-permissions key changed. + if ( + "database" not in diff.added + and "entity-type" not in diff.added + and ("entity-permissions" in diff.added or "entity-permissions" in diff.changed) + ): + getattr(self.on, "database_entity_permissions_changed").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + def _on_secret_changed_event(self, event: SecretChangedEvent) -> None: """Event emitted when the secret has changed.""" pass @@ -2979,9 +3346,26 @@ def __init__( relations_aliases: Optional[List[str]] = None, additional_secret_fields: Optional[List[str]] = [], external_node_connectivity: bool = False, + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, + requested_entity_secret: Optional[str] = None, + requested_entity_name: Optional[str] = None, + requested_entity_password: Optional[str] = None, ): """Manager of database client relations.""" - super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) + super().__init__( + model, + relation_name, + extra_user_roles, + additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, + requested_entity_secret, + requested_entity_name, + requested_entity_password, + ) self.database = database_name self.relations_aliases = relations_aliases self.external_node_connectivity = external_node_connectivity @@ -3064,9 +3448,17 @@ def __init__( if self.relation_data.relations_aliases: for relation_alias in self.relation_data.relations_aliases: - self.on.define_event(f"{relation_alias}_database_created", DatabaseCreatedEvent) self.on.define_event( - f"{relation_alias}_endpoints_changed", DatabaseEndpointsChangedEvent + f"{relation_alias}_database_created", + DatabaseCreatedEvent, + ) + self.on.define_event( + f"{relation_alias}_database_entity_created", + DatabaseEntityCreatedEvent, + ) + self.on.define_event( + f"{relation_alias}_endpoints_changed", + DatabaseEndpointsChangedEvent, ) self.on.define_event( f"{relation_alias}_read_only_endpoints_changed", @@ -3156,6 +3548,30 @@ def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: if self.relation_data.extra_user_roles: event_data["extra-user-roles"] = self.relation_data.extra_user_roles + if self.relation_data.extra_group_roles: + event_data["extra-group-roles"] = self.relation_data.extra_group_roles + if self.relation_data.entity_type: + event_data["entity-type"] = self.relation_data.entity_type + if self.relation_data.entity_permissions: + event_data["entity-permissions"] = self.relation_data.entity_permissions + if self.relation_data.requested_entity_secret: + event_data["requested-entity-secret"] = self.relation_data.requested_entity_secret + + # Create helper secret if needed + if ( + self.relation_data.requested_entity_name + and not self.relation_data.requested_entity_secret + ): + content = {"entity-name": self.relation_data.requested_entity_name} + if self.relation_data.requested_entity_password: + content["password"] = self.relation_data.requested_entity_password + secret = self.charm.app.add_secret( + content, label=f"{self.model.uuid}-{event.relation.id}-requested-entity" + ) + secret.grant(event.relation) + if not secret.id: + raise SecretError("Secret helper missing Id") + event_data["requested-entity-secret"] = secret.id # set external-node-connectivity field if self.relation_data.external_node_connectivity: @@ -3163,6 +3579,19 @@ def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: self.relation_data.update_relation_data(event.relation.id, event_data) + def _clear_helper_secret(self, event: RelationChangedEvent, app_databag: Dict) -> None: + """Remove helper secret if set.""" + if ( + self.relation_data.local_unit.is_leader() + and self.relation_data.requested_entity_name + and (secret_uri := app_databag.get("requested-entity-secret")) + ): + try: + secret = self.framework.model.get_secret(id=secret_uri) + secret.remove_all_revisions() + except ModelError: + logger.debug("Unable to remove helper secret") + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: """Event emitted when the database relation has changed.""" is_subordinate = False @@ -3174,10 +3603,7 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: is_subordinate = event.relation.data[key].get("subordinated") == "true" if is_subordinate: - if not remote_unit_data: - return - - if remote_unit_data.get("state") != "ready": + if not remote_unit_data or remote_unit_data.get("state") != "ready": return # Check which data has changed to emit customs events. @@ -3187,12 +3613,13 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): self.relation_data._register_secrets_to_relation(event.relation, diff.added) + app_databag = get_encoded_dict(event.relation, event.app, "data") + if app_databag is None: + app_databag = {} + # Check if the database is created # (the database charm shared the credentials). - secret_field_user = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) - if ( - "username" in diff.added and "password" in diff.added - ) or secret_field_user in diff.added: + if self._main_credentials_shared(diff) and "entity-type" not in app_databag: # Emit the default event (the one without an alias). logger.info("database created at %s", datetime.now()) getattr(self.on, "database_created").emit( @@ -3201,9 +3628,23 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # Emit the aliased event (if any). self._emit_aliased_event(event, "database_created") + self._clear_helper_secret(event, app_databag) + + # To avoid unnecessary application restarts do not trigger other events. + return + + if self._entity_credentials_shared(diff) and "entity-type" in app_databag: + # Emit the default event (the one without an alias). + logger.info("entity created at %s", datetime.now()) + getattr(self.on, "database_entity_created").emit( + event.relation, app=event.app, unit=event.unit + ) + + # Emit the aliased event (if any). + self._emit_aliased_event(event, "database_entity_created") + self._clear_helper_secret(event, app_databag) - # To avoid unnecessary application restarts do not trigger - # “endpoints_changed“ event if “database_created“ is triggered. + # To avoid unnecessary application restarts do not trigger other events. return # Emit an endpoints changed event if the database @@ -3218,8 +3659,7 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # Emit the aliased event (if any). self._emit_aliased_event(event, "endpoints_changed") - # To avoid unnecessary application restarts do not trigger - # “read_only_endpoints_changed“ event if “endpoints_changed“ is triggered. + # To avoid unnecessary application restarts do not trigger other events. return # Emit a read only endpoints changed event if the database @@ -3247,6 +3687,12 @@ def __init__( relations_aliases: Optional[List[str]] = None, additional_secret_fields: Optional[List[str]] = [], external_node_connectivity: bool = False, + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, + requested_entity_secret: Optional[str] = None, + requested_entity_name: Optional[str] = None, + requested_entity_password: Optional[str] = None, ): DatabaseRequirerData.__init__( self, @@ -3257,6 +3703,12 @@ def __init__( relations_aliases, additional_secret_fields, external_node_connectivity, + extra_group_roles, + entity_type, + entity_permissions, + requested_entity_secret, + requested_entity_name, + requested_entity_password, ) DatabaseRequirerEventHandlers.__init__(self, charm, self) @@ -3322,17 +3774,35 @@ def restore(self, snapshot): self.old_mtls_cert = snapshot["old_mtls_cert"] -class TopicRequestedEvent(KafkaProvidesEvent, ExtraRoleEvent): +class TopicRequestedEvent(KafkaProvidesEvent): """Event emitted when a new topic is requested for use on this relation.""" + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + if not self.relation.app: + return None -class KafkaProvidesEvents(CharmEvents): - """Kafka events. + return self.relation.data[self.relation.app].get("extra-user-roles") - This class defines the events that the Kafka can emit. + +class TopicEntityRequestedEvent(KafkaProvidesEvent, EntityProvidesEvent): + """Event emitted when a new entity is requested for use on this relation.""" + + +class TopicEntityPermissionsChangedEvent(KafkaProvidesEvent, EntityProvidesEvent): + """Event emitted when existing entity permissions are changed on this relation.""" + + +class KafkaProvidesEvents(CharmEvents): + """Kafka events. + + This class defines the events that the Kafka can emit. """ topic_requested = EventSource(TopicRequestedEvent) + topic_entity_requested = EventSource(TopicEntityRequestedEvent) + topic_entity_permissions_changed = EventSource(TopicEntityPermissionsChangedEvent) mtls_cert_updated = EventSource(KafkaClientMtlsCertUpdatedEvent) @@ -3376,6 +3846,10 @@ class TopicCreatedEvent(AuthenticationEvent, KafkaRequiresEvent): """Event emitted when a new topic is created for use on this relation.""" +class TopicEntityCreatedEvent(EntityRequiresEvent, KafkaRequiresEvent): + """Event emitted when a new entity is created for use on this relation.""" + + class BootstrapServerChangedEvent(AuthenticationEvent, KafkaRequiresEvent): """Event emitted when the bootstrap server is changed.""" @@ -3387,6 +3861,7 @@ class KafkaRequiresEvents(CharmEvents): """ topic_created = EventSource(TopicCreatedEvent) + topic_entity_created = EventSource(TopicEntityCreatedEvent) bootstrap_server_changed = EventSource(BootstrapServerChangedEvent) @@ -3465,13 +3940,43 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # Check which data has changed to emit customs events. diff = self._diff(event) - # Emit a topic requested event if the setup key (topic name and optional - # extra user roles) was added to the relation databag by the application. - if "topic" in diff.added: + # Validate entity information is not dynamically changed + self._validate_entity_consistency(event, diff) + + # Emit a topic requested event if the setup key (topic name) + # was added to the relation databag, but the entity-type key was not. + if "topic" in diff.added and "entity-type" not in diff.added: getattr(self.on, "topic_requested").emit( event.relation, app=event.app, unit=event.unit ) + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit an entity requested event if the setup key (topic name) + # was added to the relation databag, in addition to the entity-type key. + if "topic" in diff.added and "entity-type" in diff.added: + getattr(self.on, "topic_entity_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit a permissions changed event if the setup key (topic name) + # was added to the relation databag, and the entity-permissions key changed. + if ( + "topic" not in diff.added + and "entity-type" not in diff.added + and ("entity-permissions" in diff.added or "entity-permissions" in diff.changed) + ): + getattr(self.on, "topic_entity_permissions_changed").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + def _on_secret_changed_event(self, event: SecretChangedEvent): """Event notifying about a new value of a secret.""" if not event.secret.label: @@ -3520,9 +4025,20 @@ def __init__( consumer_group_prefix: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], mtls_cert: Optional[str] = None, + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, ): """Manager of Kafka client relations.""" - super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) + super().__init__( + model, + relation_name, + extra_user_roles, + additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, + ) self.topic = topic self.consumer_group_prefix = consumer_group_prefix or "" self.mtls_cert = mtls_cert @@ -3576,12 +4092,18 @@ def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: if self.relation_data.mtls_cert: relation_data["mtls-cert"] = self.relation_data.mtls_cert - if self.relation_data.extra_user_roles: - relation_data["extra-user-roles"] = self.relation_data.extra_user_roles - if self.relation_data.consumer_group_prefix: relation_data["consumer-group-prefix"] = self.relation_data.consumer_group_prefix + if self.relation_data.extra_user_roles: + relation_data["extra-user-roles"] = self.relation_data.extra_user_roles + if self.relation_data.extra_group_roles: + relation_data["extra-group-roles"] = self.relation_data.extra_group_roles + if self.relation_data.entity_type: + relation_data["entity-type"] = self.relation_data.entity_type + if self.relation_data.entity_permissions: + relation_data["entity-permissions"] = self.relation_data.entity_permissions + self.relation_data.update_relation_data(event.relation.id, relation_data) def _on_secret_changed_event(self, event: SecretChangedEvent): @@ -3600,16 +4122,26 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): self.relation_data._register_secrets_to_relation(event.relation, diff.added) - secret_field_user = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) - if ( - "username" in diff.added and "password" in diff.added - ) or secret_field_user in diff.added: + app_databag = get_encoded_dict(event.relation, event.app, "data") + if app_databag is None: + app_databag = {} + + if self._main_credentials_shared(diff) and "entity-type" not in app_databag: # Emit the default event (the one without an alias). logger.info("topic created at %s", datetime.now()) getattr(self.on, "topic_created").emit(event.relation, app=event.app, unit=event.unit) - # To avoid unnecessary application restarts do not trigger - # “endpoints_changed“ event if “topic_created“ is triggered. + # To avoid unnecessary application restarts do not trigger other events. + return + + if self._entity_credentials_shared(diff) and "entity-type" in app_databag: + # Emit the default event (the one without an alias). + logger.info("entity created at %s", datetime.now()) + getattr(self.on, "topic_entity_created").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. return # Emit an endpoints (bootstrap-server) changed event if the Kafka endpoints @@ -3620,6 +4152,8 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: getattr(self.on, "bootstrap_server_changed").emit( event.relation, app=event.app, unit=event.unit ) # here check if this is the right design + + # To avoid unnecessary application restarts do not trigger other events. return @@ -3635,6 +4169,9 @@ def __init__( consumer_group_prefix: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], mtls_cert: Optional[str] = None, + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, ) -> None: KafkaRequirerData.__init__( self, @@ -3645,10 +4182,561 @@ def __init__( consumer_group_prefix=consumer_group_prefix, additional_secret_fields=additional_secret_fields, mtls_cert=mtls_cert, + extra_group_roles=extra_group_roles, + entity_type=entity_type, + entity_permissions=entity_permissions, ) KafkaRequirerEventHandlers.__init__(self, charm, self) +# Karapace related events + + +class KarapaceProvidesEvent(RelationEvent): + """Base class for Karapace events.""" + + @property + def subject(self) -> Optional[str]: + """Returns the subject that was requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("subject") + + +class SubjectRequestedEvent(KarapaceProvidesEvent): + """Event emitted when a new subject is requested for use on this relation.""" + + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-user-roles") + + +class SubjectEntityRequestedEvent(KarapaceProvidesEvent, EntityProvidesEvent): + """Event emitted when a new entity is requested for use on this relation.""" + + +class SubjectEntityPermissionsChangedEvent(KarapaceProvidesEvent, EntityProvidesEvent): + """Event emitted when existing entity permissions are changed on this relation.""" + + +class KarapaceProvidesEvents(CharmEvents): + """Karapace events. + + This class defines the events that the Karapace can emit. + """ + + subject_requested = EventSource(SubjectRequestedEvent) + subject_entity_requested = EventSource(SubjectEntityRequestedEvent) + subject_entity_permissions_changed = EventSource(SubjectEntityPermissionsChangedEvent) + + +class KarapaceRequiresEvent(RelationEvent): + """Base class for Karapace events.""" + + @property + def subject(self) -> Optional[str]: + """Returns the subject.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("subject") + + @property + def endpoints(self) -> Optional[str]: + """Returns a comma-separated list of broker uris.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("endpoints") + + +class SubjectAllowedEvent(AuthenticationEvent, KarapaceRequiresEvent): + """Event emitted when a new subject ACL is created for use on this relation.""" + + +class SubjectEntityCreatedEvent(EntityRequiresEvent, KarapaceRequiresEvent): + """Event emitted when a new entity is created for use on this relation.""" + + +class EndpointsChangedEvent(AuthenticationEvent, KarapaceRequiresEvent): + """Event emitted when the endpoints are changed.""" + + +class KarapaceRequiresEvents(CharmEvents): + """Karapace events. + + This class defines the events that Karapace can emit. + """ + + subject_allowed = EventSource(SubjectAllowedEvent) + subject_entity_created = EventSource(SubjectEntityCreatedEvent) + server_changed = EventSource(EndpointsChangedEvent) + + +# Karapace Provides and Requires + + +class KarapaceProviderData(ProviderData): + """Provider-side of the Karapace relation.""" + + RESOURCE_FIELD = "subject" + + def __init__(self, model: Model, relation_name: str) -> None: + super().__init__(model, relation_name) + + def set_subject(self, relation_id: int, subject: str) -> None: + """Set subject name in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + subject: the subject name. + """ + self.update_relation_data(relation_id, {"subject": subject}) + + def set_endpoint(self, relation_id: int, endpoint: str) -> None: + """Set the endpoint in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + endpoint: the server address. + """ + self.update_relation_data(relation_id, {"endpoints": endpoint}) + + +class KarapaceProviderEventHandlers(ProviderEventHandlers): + """Provider-side of the Karapace relation.""" + + on = KarapaceProvidesEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: KarapaceProviderData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + super()._on_relation_changed_event(event) + + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Validate entity information is not dynamically changed + self._validate_entity_consistency(event, diff) + + # Emit a subject requested event if the setup key (subject name) + # was added to the relation databag, but the entity-type key was not. + if "subject" in diff.added and "entity-type" not in diff.added: + getattr(self.on, "subject_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit an entity requested event if the setup key (subject name) + # was added to the relation databag, in addition to the entity-type key. + if "subject" in diff.added and "entity-type" in diff.added: + getattr(self.on, "subject_entity_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit a permissions changed event if the setup key (subject name) + # was added to the relation databag, and the entity-permissions key changed. + if ( + "subject" not in diff.added + and "entity-type" not in diff.added + and ("entity-permissions" in diff.added or "entity-permissions" in diff.changed) + ): + getattr(self.on, "subject_entity_permissions_changed").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + pass + + +class KarapaceProvides(KarapaceProviderData, KarapaceProviderEventHandlers): + """Provider-side of the Karapace relation.""" + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + KarapaceProviderData.__init__(self, charm.model, relation_name) + KarapaceProviderEventHandlers.__init__(self, charm, self) + + +class KarapaceRequirerData(RequirerData): + """Requirer-side of the Karapace relation.""" + + def __init__( + self, + model: Model, + relation_name: str, + subject: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, + ): + """Manager of Karapace client relations.""" + super().__init__( + model, + relation_name, + extra_user_roles, + additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, + ) + self.subject = subject + + @property + def subject(self): + """Topic to use in Karapace.""" + return self._subject + + @subject.setter + def subject(self, value): + # Avoid wildcards + if value == "*": + raise ValueError(f"Error on subject '{value}', cannot be a wildcard.") + self._subject = value + + +class KarapaceRequirerEventHandlers(RequirerEventHandlers): + """Requires-side of the Karapace relation.""" + + on = KarapaceRequiresEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: KarapaceRequirerData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when the Karapace relation is created.""" + super()._on_relation_created_event(event) + + if not self.relation_data.local_unit.is_leader(): + return + + # Sets subject and extra user roles + relation_data = {"subject": self.relation_data.subject} + + if self.relation_data.extra_user_roles: + relation_data["extra-user-roles"] = self.relation_data.extra_user_roles + if self.relation_data.extra_group_roles: + relation_data["extra-group-roles"] = self.relation_data.extra_group_roles + if self.relation_data.entity_type: + relation_data["entity-type"] = self.relation_data.entity_type + if self.relation_data.entity_permissions: + relation_data["entity-permissions"] = self.relation_data.entity_permissions + + self.relation_data.update_relation_data(event.relation.id, relation_data) + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + pass + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the Karapace relation has changed.""" + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Check if the subject ACLs are created + # (the Karapace charm shared the credentials). + + # Register all new secrets with their labels + if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, diff.added) + + app_databag = get_encoded_dict(event.relation, event.app, "data") + if app_databag is None: + app_databag = {} + + if self._main_credentials_shared(diff) and "entity-type" not in app_databag: + # Emit the default event (the one without an alias). + logger.info("subject ACL created at %s", datetime.now()) + getattr(self.on, "subject_allowed").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + if self._entity_credentials_shared(diff) and "entity-type" in app_databag: + # Emit the default event (the one without an alias). + logger.info("entity created at %s", datetime.now()) + getattr(self.on, "subject_entity_created").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit an endpoints changed event if the Karapace endpoints added or changed + # this info in the relation databag. + if "endpoints" in diff.added or "endpoints" in diff.changed: + # Emit the default event (the one without an alias). + logger.info("endpoints changed on %s", datetime.now()) + getattr(self.on, "server_changed").emit( + event.relation, app=event.app, unit=event.unit + ) # here check if this is the right design + + # To avoid unnecessary application restarts do not trigger other events. + return + + +class KarapaceRequires(KarapaceRequirerData, KarapaceRequirerEventHandlers): + """Provider-side of the Karapace relation.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str, + subject: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, + ) -> None: + KarapaceRequirerData.__init__( + self, + charm.model, + relation_name, + subject, + extra_user_roles, + additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, + ) + KarapaceRequirerEventHandlers.__init__(self, charm, self) + + +# Kafka Connect Events + + +class KafkaConnectProvidesEvent(RelationEvent): + """Base class for Kafka Connect Provider events.""" + + @property + def plugin_url(self) -> Optional[str]: + """Returns the REST endpoint URL which serves the connector plugin.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("plugin-url") + + +class IntegrationRequestedEvent(KafkaConnectProvidesEvent): + """Event emitted when a new integrator boots up and is ready to serve the connector plugin.""" + + +class KafkaConnectProvidesEvents(CharmEvents): + """Kafka Connect Provider Events.""" + + integration_requested = EventSource(IntegrationRequestedEvent) + + +class KafkaConnectRequiresEvent(AuthenticationEvent): + """Base class for Kafka Connect Requirer events.""" + + @property + def plugin_url(self) -> Optional[str]: + """Returns the REST endpoint URL which serves the connector plugin.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("plugin-url") + + +class IntegrationCreatedEvent(KafkaConnectRequiresEvent): + """Event emitted when the credentials are created for this integrator.""" + + +class IntegrationEndpointsChangedEvent(KafkaConnectRequiresEvent): + """Event emitted when Kafka Connect REST endpoints change.""" + + +class KafkaConnectRequiresEvents(CharmEvents): + """Kafka Connect Requirer Events.""" + + integration_created = EventSource(IntegrationCreatedEvent) + integration_endpoints_changed = EventSource(IntegrationEndpointsChangedEvent) + + +class KafkaConnectProviderData(ProviderData): + """Provider-side of the Kafka Connect relation.""" + + RESOURCE_FIELD = "plugin-url" + + def __init__(self, model: Model, relation_name: str) -> None: + super().__init__(model, relation_name) + + def set_endpoints(self, relation_id: int, endpoints: str) -> None: + """Sets REST endpoints of the Kafka Connect service.""" + self.update_relation_data(relation_id, {"endpoints": endpoints}) + + +class KafkaConnectProviderEventHandlers(EventHandlers): + """Provider-side implementation of the Kafka Connect event handlers.""" + + on = KafkaConnectProvidesEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: KafkaConnectProviderData) -> None: + super().__init__(charm, relation_data) + self.relation_data = relation_data + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + if "plugin-url" in diff.added: + getattr(self.on, "integration_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + pass + + +class KafkaConnectProvides(KafkaConnectProviderData, KafkaConnectProviderEventHandlers): + """Provider-side implementation of the Kafka Connect relation.""" + + def __init__(self, charm: CharmBase, relation_name: str) -> None: + KafkaConnectProviderData.__init__(self, charm.model, relation_name) + KafkaConnectProviderEventHandlers.__init__(self, charm, self) + + +# Sentinel value passed from Kafka Connect requirer side when it does not need to serve any plugins. +PLUGIN_URL_NOT_REQUIRED: Final[str] = "NOT-REQUIRED" + + +class KafkaConnectRequirerData(RequirerData): + """Requirer-side of the Kafka Connect relation.""" + + def __init__( + self, + model: Model, + relation_name: str, + plugin_url: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + ): + """Manager of Kafka client relations.""" + super().__init__( + model, + relation_name, + extra_user_roles=extra_user_roles, + additional_secret_fields=additional_secret_fields, + ) + self.plugin_url = plugin_url + + @property + def plugin_url(self): + """The REST endpoint URL which serves the connector plugin.""" + return self._plugin_url + + @plugin_url.setter + def plugin_url(self, value): + self._plugin_url = value + + +class KafkaConnectRequirerEventHandlers(RequirerEventHandlers): + """Requirer-side of the Kafka Connect relation.""" + + on = KafkaConnectRequiresEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: KafkaConnectRequirerData) -> None: + super().__init__(charm, relation_data) + self.relation_data = relation_data + + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when the Kafka Connect relation is created.""" + super()._on_relation_created_event(event) + + if not self.relation_data.local_unit.is_leader(): + return + + relation_data = {"plugin-url": self.relation_data.plugin_url} + self.relation_data.update_relation_data(event.relation.id, relation_data) + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + pass + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the Kafka Connect relation has changed.""" + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Register all new secrets with their labels + if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, diff.added) + + if self._main_credentials_shared(diff): + logger.info("integration created at %s", datetime.now()) + getattr(self.on, "integration_created").emit( + event.relation, app=event.app, unit=event.unit + ) + return + + # Emit an endpoints changed event if the provider added or + # changed this info in the relation databag. + if "endpoints" in diff.added or "endpoints" in diff.changed: + # Emit the default event (the one without an alias). + logger.info("endpoints changed on %s", datetime.now()) + getattr(self.on, "integration_endpoints_changed").emit( + event.relation, app=event.app, unit=event.unit + ) + return + + +class KafkaConnectRequires(KafkaConnectRequirerData, KafkaConnectRequirerEventHandlers): + """Requirer-side implementation of the Kafka Connect relation.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str, + plugin_url: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + ) -> None: + KafkaConnectRequirerData.__init__( + self, + charm.model, + relation_name, + plugin_url, + extra_user_roles=extra_user_roles, + additional_secret_fields=additional_secret_fields, + ) + KafkaConnectRequirerEventHandlers.__init__(self, charm, self) + + # Opensearch related events @@ -3664,9 +4752,25 @@ def index(self) -> Optional[str]: return self.relation.data[self.relation.app].get("index") -class IndexRequestedEvent(OpenSearchProvidesEvent, ExtraRoleEvent): +class IndexRequestedEvent(OpenSearchProvidesEvent): """Event emitted when a new index is requested for use on this relation.""" + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-user-roles") + + +class IndexEntityRequestedEvent(OpenSearchProvidesEvent, EntityProvidesEvent): + """Event emitted when a new entity is requested for use on this relation.""" + + +class IndexEntityPermissionsChangedEvent(OpenSearchProvidesEvent, EntityProvidesEvent): + """Event emitted when existing entity permissions are changed on this relation.""" + class OpenSearchProvidesEvents(CharmEvents): """OpenSearch events. @@ -3675,6 +4779,8 @@ class OpenSearchProvidesEvents(CharmEvents): """ index_requested = EventSource(IndexRequestedEvent) + index_entity_requested = EventSource(IndexEntityRequestedEvent) + index_entity_permissions_changed = EventSource(IndexEntityPermissionsChangedEvent) class OpenSearchRequiresEvent(DatabaseRequiresEvent): @@ -3685,6 +4791,10 @@ class IndexCreatedEvent(AuthenticationEvent, OpenSearchRequiresEvent): """Event emitted when a new index is created for use on this relation.""" +class IndexEntityCreatedEvent(EntityRequiresEvent, OpenSearchRequiresEvent): + """Event emitted when a new index is created for use on this relation.""" + + class OpenSearchRequiresEvents(CharmEvents): """OpenSearch events. @@ -3692,6 +4802,7 @@ class OpenSearchRequiresEvents(CharmEvents): """ index_created = EventSource(IndexCreatedEvent) + index_entity_created = EventSource(IndexEntityCreatedEvent) endpoints_changed = EventSource(DatabaseEndpointsChangedEvent) authentication_updated = EventSource(AuthenticationEvent) @@ -3754,16 +4865,47 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # Leader only if not self.relation_data.local_unit.is_leader(): return + # Check which data has changed to emit customs events. diff = self._diff(event) - # Emit an index requested event if the setup key (index name and optional extra user roles) - # have been added to the relation databag by the application. - if "index" in diff.added: + # Validate entity information is not dynamically changed + self._validate_entity_consistency(event, diff) + + # Emit an index requested event if the setup key (index name) + # was added to the relation databag, but the entity-type key was not. + if "index" in diff.added and "entity-type" not in diff.added: getattr(self.on, "index_requested").emit( event.relation, app=event.app, unit=event.unit ) + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit an entity requested event if the setup key (index name) + # was added to the relation databag, in addition to the entity-type key. + if "index" in diff.added and "entity-type" in diff.added: + getattr(self.on, "index_entity_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit a permissions changed event if the setup key (index name) + # was added to the relation databag, and the entity-permissions key changed. + if ( + "index" not in diff.added + and "entity-type" not in diff.added + and ("entity-permissions" in diff.added or "entity-permissions" in diff.changed) + ): + getattr(self.on, "index_entity_permissions_changed").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + def _on_secret_changed_event(self, event: SecretChangedEvent) -> None: """Event emitted when the relation data has changed.""" pass @@ -3787,9 +4929,20 @@ def __init__( index: str, extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, ): """Manager of OpenSearch client relations.""" - super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) + super().__init__( + model, + relation_name, + extra_user_roles, + additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, + ) self.index = index @@ -3813,8 +4966,15 @@ def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: # Sets both index and extra user roles in the relation if the roles are provided. # Otherwise, sets only the index. data = {"index": self.relation_data.index} + if self.relation_data.extra_user_roles: data["extra-user-roles"] = self.relation_data.extra_user_roles + if self.relation_data.extra_group_roles: + data["extra-group-roles"] = self.relation_data.extra_group_roles + if self.relation_data.entity_type: + data["entity-type"] = self.relation_data.entity_type + if self.relation_data.entity_permissions: + data["entity-permissions"] = self.relation_data.entity_permissions self.relation_data.update_relation_data(event.relation.id, data) @@ -3864,27 +5024,40 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: event.relation, app=event.app, unit=event.unit ) + app_databag = get_encoded_dict(event.relation, event.app, "data") + if app_databag is None: + app_databag = {} + # Check if the index is created # (the OpenSearch charm shares the credentials). - if ( - "username" in diff.added and "password" in diff.added - ) or secret_field_user in diff.added: + if self._main_credentials_shared(diff) and "entity-type" not in app_databag: # Emit the default event (the one without an alias). logger.info("index created at: %s", datetime.now()) getattr(self.on, "index_created").emit(event.relation, app=event.app, unit=event.unit) - # To avoid unnecessary application restarts do not trigger - # “endpoints_changed“ event if “index_created“ is triggered. + # To avoid unnecessary application restarts do not trigger other events. return - # Emit a endpoints changed event if the OpenSearch application added or changed this info - # in the relation databag. + if self._entity_credentials_shared(diff) and "entity-type" in app_databag: + # Emit the default event (the one without an alias). + logger.info("entity created at: %s", datetime.now()) + getattr(self.on, "index_entity_created").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit a endpoints changed event if the OpenSearch application + # added or changed this info in the relation databag. if "endpoints" in diff.added or "endpoints" in diff.changed: # Emit the default event (the one without an alias). logger.info("endpoints changed on %s", datetime.now()) getattr(self.on, "endpoints_changed").emit( event.relation, app=event.app, unit=event.unit - ) # here check if this is the right design + ) + + # To avoid unnecessary application restarts do not trigger other events. return @@ -3898,6 +5071,9 @@ def __init__( index: str, extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, ) -> None: OpenSearchRequiresData.__init__( self, @@ -3906,6 +5082,9 @@ def __init__( index, extra_user_roles, additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, ) OpenSearchRequiresEventHandlers.__init__(self, charm, self) @@ -4048,6 +5227,12 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: if any(newval for newval in new_data_keys if self.relation_data._is_secret_field(newval)): self.relation_data._register_secrets_to_relation(event.relation, new_data_keys) + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Validate entity information is not dynamically changed + self._validate_entity_consistency(event, diff) + getattr(self.on, "mtls_cert_updated").emit(event.relation, app=event.app, unit=event.unit) return @@ -4100,9 +5285,20 @@ def __init__( mtls_cert: Optional[str], extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, ): """Manager of Etcd client relations.""" - super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) + super().__init__( + model, + relation_name, + extra_user_roles, + additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, + ) self.prefix = prefix self.mtls_cert = mtls_cert @@ -4212,6 +5408,9 @@ def __init__( mtls_cert: Optional[str], extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, ) -> None: EtcdRequirerData.__init__( self, @@ -4221,6 +5420,9 @@ def __init__( mtls_cert, extra_user_roles, additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, ) EtcdRequirerEventHandlers.__init__(self, charm, self) if not self.secrets_enabled: diff --git a/lib/charms/data_platform_libs/v0/s3.py b/lib/charms/data_platform_libs/v0/s3.py index f5614aaf6b..dbf4d5bb72 100644 --- a/lib/charms/data_platform_libs/v0/s3.py +++ b/lib/charms/data_platform_libs/v0/s3.py @@ -110,6 +110,7 @@ def _on_credential_gone(self, event: CredentialsGoneEvent): ``` """ + import json import logging from collections import namedtuple @@ -137,7 +138,7 @@ def _on_credential_gone(self, event: CredentialsGoneEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 5 +LIBPATCH = 6 logger = logging.getLogger(__name__) @@ -354,7 +355,7 @@ def update_connection_info(self, relation_id: int, connection_data: dict) -> Non updated_connection_data[configuration_option] = configuration_value relation.data[self.local_app].update(updated_connection_data) - logger.debug(f"Updated S3 connection info: {updated_connection_data}") + logger.debug("Updated S3 connection info.") @property def relations(self) -> List[Relation]: @@ -721,7 +722,7 @@ def update_connection_info(self, relation_id: int, connection_data: dict) -> Non updated_connection_data[configuration_option] = configuration_value relation.data[self.local_app].update(updated_connection_data) - logger.debug(f"Updated S3 credentials: {updated_connection_data}") + logger.debug("Updated S3 credentials.") def _load_relation_data(self, raw_relation_data: RelationDataContent) -> Dict[str, str]: """Loads relation data from the relation data bag. diff --git a/lib/charms/grafana_agent/v0/cos_agent.py b/lib/charms/grafana_agent/v0/cos_agent.py index b18c271342..7bf3eb1a5e 100644 --- a/lib/charms/grafana_agent/v0/cos_agent.py +++ b/lib/charms/grafana_agent/v0/cos_agent.py @@ -254,16 +254,12 @@ class _MetricsEndpointDict(TypedDict): LIBID = "dc15fa84cef84ce58155fb84f6c6213a" LIBAPI = 0 -LIBPATCH = 20 +LIBPATCH = 22 PYDEPS = ["cosl >= 0.0.50", "pydantic"] DEFAULT_RELATION_NAME = "cos-agent" DEFAULT_PEER_RELATION_NAME = "peers" -DEFAULT_SCRAPE_CONFIG = { - "static_configs": [{"targets": ["localhost:80"]}], - "metrics_path": "/metrics", -} logger = logging.getLogger(__name__) SnapEndpoint = namedtuple("SnapEndpoint", "owner, name") @@ -472,7 +468,7 @@ def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): return databag -class CosAgentProviderUnitData(DatabagModel): +class CosAgentProviderUnitData(DatabagModel): # type: ignore """Unit databag model for `cos-agent` relation.""" # The following entries are the same for all units of the same principal. @@ -499,7 +495,7 @@ class CosAgentProviderUnitData(DatabagModel): KEY: ClassVar[str] = "config" -class CosAgentPeersUnitData(DatabagModel): +class CosAgentPeersUnitData(DatabagModel): # type: ignore """Unit databag model for `peers` cos-agent machine charm peer relation.""" # We need the principal unit name and relation metadata to be able to render identifiers @@ -598,7 +594,7 @@ class Receiver(pydantic.BaseModel): ) -class CosAgentRequirerUnitData(DatabagModel): # noqa: D101 +class CosAgentRequirerUnitData(DatabagModel): # type: ignore """Application databag model for the COS-agent requirer.""" receivers: List[Receiver] = pydantic.Field( @@ -714,7 +710,7 @@ def _scrape_jobs(self) -> List[Dict]: } ) - scrape_configs = scrape_configs or [DEFAULT_SCRAPE_CONFIG] + scrape_configs = scrape_configs or [] # Augment job name to include the app name and a unique id (index) for idx, scrape_config in enumerate(scrape_configs): @@ -923,6 +919,7 @@ def __init__( relation_name: str = DEFAULT_RELATION_NAME, peer_relation_name: str = DEFAULT_PEER_RELATION_NAME, refresh_events: Optional[List[str]] = None, + is_tracing_ready: Optional[Callable] = None, ): """Create a COSAgentRequirer instance. @@ -931,12 +928,14 @@ def __init__( relation_name: The name of the relation to communicate over. peer_relation_name: The name of the peer relation to communicate over. refresh_events: List of events on which to refresh relation data. + is_tracing_ready: Custom function to evaluate whether the trace receiver url should be sent. """ super().__init__(charm, relation_name) self._charm = charm self._relation_name = relation_name self._peer_relation_name = peer_relation_name self._refresh_events = refresh_events or [self._charm.on.config_changed] + self._is_tracing_ready = is_tracing_ready events = self._charm.on[relation_name] self.framework.observe( @@ -1046,6 +1045,9 @@ def _on_relation_data_changed(self, event: RelationChangedEvent): def update_tracing_receivers(self): """Updates the list of exposed tracing receivers in all relations.""" + tracing_ready = ( + self._is_tracing_ready if self._is_tracing_ready else self._charm.tracing.is_ready # type: ignore + ) try: for relation in self._charm.model.relations[self._relation_name]: CosAgentRequirerUnitData( @@ -1059,7 +1061,7 @@ def update_tracing_receivers(self): # databag contents (as it expects a string in URL) but that won't cause any errors as # tracing endpoints are the only content in the grafana-agent's side of the databag. url=f"{self._get_tracing_receiver_url(protocol)}" - if self._charm.tracing.is_ready() # type: ignore + if tracing_ready() else None, protocol=ProtocolType( name=protocol, diff --git a/lib/charms/operator_libs_linux/v2/snap.py b/lib/charms/operator_libs_linux/v2/snap.py index b0d65017e8..a706c16628 100644 --- a/lib/charms/operator_libs_linux/v2/snap.py +++ b/lib/charms/operator_libs_linux/v2/snap.py @@ -54,6 +54,10 @@ except snap.SnapError as e: logger.error("An exception occurred when installing snaps. Reason: %s" % e.message) ``` + +Dependencies: +Note that this module requires `opentelemetry-api`, which is already included into +your charm's virtual environment via `ops >= 2.21`. """ from __future__ import annotations @@ -85,6 +89,8 @@ TypeVar, ) +import opentelemetry.trace + if typing.TYPE_CHECKING: # avoid typing_extensions import at runtime from typing_extensions import NotRequired, ParamSpec, Required, Self, TypeAlias, Unpack @@ -93,6 +99,7 @@ _T = TypeVar("_T") logger = logging.getLogger(__name__) +tracer = opentelemetry.trace.get_tracer(__name__) # The unique Charmhub library identifier, never change it LIBID = "05394e5893f94f2d90feb7cbe6b633cd" @@ -102,7 +109,9 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 12 +LIBPATCH = 14 + +PYDEPS = ["opentelemetry-api"] # Regex to locate 7-bit C1 ANSI sequences @@ -140,6 +149,7 @@ class _SnapDict(TypedDict, total=True): name: str channel: str revision: str + version: str confinement: str apps: NotRequired[list[dict[str, JSONType]] | None] @@ -277,7 +287,9 @@ def _from_called_process_error(cls, msg: str, error: CalledProcessError) -> Self lines.extend(['Stderr:', error.stderr]) try: cmd = ['journalctl', '--unit', 'snapd', '--lines', '20'] - logs = subprocess.check_output(cmd, text=True) + with tracer.start_as_current_span(cmd[0]) as span: + span.set_attribute("argv", cmd) + logs = subprocess.check_output(cmd, text=True) except Exception as e: lines.extend(['Error fetching logs:', str(e)]) else: @@ -298,6 +310,7 @@ class Snap: - channel: "stable", "candidate", "beta", and "edge" are common - revision: a string representing the snap's revision - confinement: "classic", "strict", or "devmode" + - version: a string representing the snap's version, if set by the snap author """ def __init__( @@ -309,6 +322,8 @@ def __init__( confinement: str, apps: list[dict[str, JSONType]] | None = None, cohort: str | None = None, + *, + version: str | None = None, ) -> None: self._name = name self._state = state @@ -317,6 +332,7 @@ def __init__( self._confinement = confinement self._cohort = cohort or "" self._apps = apps or [] + self._version = version self._snap_client = SnapClient() def __eq__(self, other: object) -> bool: @@ -356,7 +372,9 @@ def _snap(self, command: str, optargs: Iterable[str] | None = None) -> str: optargs = optargs or [] args = ["snap", command, self._name, *optargs] try: - return subprocess.check_output(args, text=True, stderr=subprocess.PIPE) + with tracer.start_as_current_span(args[0]) as span: + span.set_attribute("argv", args) + return subprocess.check_output(args, text=True, stderr=subprocess.PIPE) except CalledProcessError as e: msg = f'Snap: {self._name!r} -- command {args!r} failed!' raise SnapError._from_called_process_error(msg=msg, error=e) from e @@ -384,7 +402,9 @@ def _snap_daemons( args = ["snap", *command, *services] try: - return subprocess.run(args, text=True, check=True, capture_output=True) + with tracer.start_as_current_span(args[0]) as span: + span.set_attribute("argv", args) + return subprocess.run(args, text=True, check=True, capture_output=True) except CalledProcessError as e: msg = f'Snap: {self._name!r} -- command {args!r} failed!' raise SnapError._from_called_process_error(msg=msg, error=e) from e @@ -491,7 +511,9 @@ def connect(self, plug: str, service: str | None = None, slot: str | None = None args = ["snap", *command] try: - subprocess.run(args, text=True, check=True, capture_output=True) + with tracer.start_as_current_span(args[0]) as span: + span.set_attribute("argv", args) + subprocess.run(args, text=True, check=True, capture_output=True) except CalledProcessError as e: msg = f'Snap: {self._name!r} -- command {args!r} failed!' raise SnapError._from_called_process_error(msg=msg, error=e) from e @@ -523,7 +545,9 @@ def alias(self, application: str, alias: str | None = None) -> None: alias = application args = ["snap", "alias", f"{self.name}.{application}", alias] try: - subprocess.run(args, text=True, check=True, capture_output=True) + with tracer.start_as_current_span(args[0]) as span: + span.set_attribute("argv", args) + subprocess.run(args, text=True, check=True, capture_output=True) except CalledProcessError as e: msg = f'Snap: {self._name!r} -- command {args!r} failed!' raise SnapError._from_called_process_error(msg=msg, error=e) from e @@ -764,6 +788,11 @@ def held(self) -> bool: info = self._snap("info") return "hold:" in info + @property + def version(self) -> str | None: + """Returns the version for a snap.""" + return self._version + class _UnixSocketConnection(http.client.HTTPConnection): """Implementation of HTTPConnection that connects to a named Unix socket.""" @@ -932,15 +961,20 @@ def _request_raw( def get_installed_snaps(self) -> list[dict[str, JSONType]]: """Get information about currently installed snaps.""" - return self._request("GET", "snaps") # type: ignore + with tracer.start_as_current_span("get_installed_snaps"): + return self._request("GET", "snaps") # type: ignore def get_snap_information(self, name: str) -> dict[str, JSONType]: """Query the snap server for information about single snap.""" - return self._request("GET", "find", {"name": name})[0] # type: ignore + with tracer.start_as_current_span("get_snap_information") as span: + span.set_attribute("name", name) + return self._request("GET", "find", {"name": name})[0] # type: ignore def get_installed_snap_apps(self, name: str) -> list[dict[str, JSONType]]: """Query the snap server for apps belonging to a named, currently installed snap.""" - return self._request("GET", "apps", {"names": name, "select": "service"}) # type: ignore + with tracer.start_as_current_span("get_installed_snap_apps") as span: + span.set_attribute("name", name) + return self._request("GET", "apps", {"names": name, "select": "service"}) # type: ignore def _put_snap_conf(self, name: str, conf: dict[str, JSONAble]) -> None: """Set the configuration details for an installed snap.""" @@ -1024,6 +1058,7 @@ def _load_installed_snaps(self) -> None: revision=i["revision"], confinement=i["confinement"], apps=i.get("apps"), + version=i.get("version"), ) self._snap_map[snap.name] = snap @@ -1043,6 +1078,7 @@ def _load_info(self, name: str) -> Snap: revision=info["revision"], confinement=info["confinement"], apps=None, + version=info.get("version"), ) @@ -1280,7 +1316,13 @@ def install_local( if dangerous: args.append("--dangerous") try: - result = subprocess.check_output(args, text=True, stderr=subprocess.PIPE).splitlines()[-1] + with tracer.start_as_current_span(args[0]) as span: + span.set_attribute("argv", args) + result = subprocess.check_output( + args, + text=True, + stderr=subprocess.PIPE, + ).splitlines()[-1] snap_name, _ = result.split(" ", 1) snap_name = ansi_filter.sub("", snap_name) @@ -1309,7 +1351,9 @@ def _system_set(config_item: str, value: str) -> None: """ args = ["snap", "set", "system", f"{config_item}={value}"] try: - subprocess.run(args, text=True, check=True, capture_output=True) + with tracer.start_as_current_span(args[0]) as span: + span.set_attribute("argv", args) + subprocess.run(args, text=True, check=True, capture_output=True) except CalledProcessError as e: msg = f"Failed setting system config '{config_item}' to '{value}'" raise SnapError._from_called_process_error(msg=msg, error=e) from e diff --git a/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py b/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py index 61fcf07716..f58ee72e0f 100644 --- a/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py +++ b/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py @@ -273,7 +273,9 @@ def _remove_stale_otel_sdk_packages(): if len(distributions_) <= 1: continue - otel_logger.debug(f"Package {name} has multiple ({len(distributions_)}) distributions.") + otel_logger.debug( + f"Package {name} has multiple ({len(distributions_)}) distributions." + ) for distribution in distributions_: if not distribution.files: # Not None or empty list path = distribution._path # type: ignore @@ -313,8 +315,8 @@ def _remove_stale_otel_sdk_packages(): import opentelemetry import ops -from opentelemetry.exporter.otlp.proto.common._internal.trace_encoder import ( # type: ignore - encode_spans # type: ignore +from opentelemetry.exporter.otlp.proto.common._internal.trace_encoder import ( # type: ignore + encode_spans, # type: ignore ) from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter # type: ignore from opentelemetry.sdk.resources import Resource @@ -348,7 +350,7 @@ def _remove_stale_otel_sdk_packages(): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 9 +LIBPATCH = 10 PYDEPS = ["opentelemetry-exporter-otlp-proto-http==1.21.0"] @@ -394,10 +396,14 @@ class _Buffer: _SPANSEP = b"__CHARM_TRACING_BUFFER_SPAN_SEP__" - def __init__(self, db_file: Path, max_event_history_length: int, max_buffer_size_mib: int): + def __init__( + self, db_file: Path, max_event_history_length: int, max_buffer_size_mib: int + ): self._db_file = db_file self._max_event_history_length = max_event_history_length - self._max_buffer_size_mib = max(max_buffer_size_mib, _BUFFER_CACHE_FILE_SIZE_LIMIT_MiB_MIN) + self._max_buffer_size_mib = max( + max_buffer_size_mib, _BUFFER_CACHE_FILE_SIZE_LIMIT_MiB_MIN + ) # set by caller self.exporter: Optional[OTLPSpanExporter] = None @@ -535,7 +541,9 @@ def flush(self) -> Optional[bool]: ) errors = True except Exception: - logger.exception("unexpected error while flushing span batch from buffer") + logger.exception( + "unexpected error while flushing span batch from buffer" + ) errors = True if not errors: @@ -684,7 +692,9 @@ def _get_tracing_endpoint( f"got {tracing_endpoint} instead." ) - dev_logger.debug(f"Setting up span exporter to endpoint: {tracing_endpoint}/v1/traces") + dev_logger.debug( + f"Setting up span exporter to endpoint: {tracing_endpoint}/v1/traces" + ) return f"{tracing_endpoint}/v1/traces" @@ -765,7 +775,9 @@ def wrap_init(self: CharmBase, framework: Framework, *args, **kwargs): provider = TracerProvider(resource=resource) # if anything goes wrong with retrieving the endpoint, we let the exception bubble up. - tracing_endpoint = _get_tracing_endpoint(tracing_endpoint_attr, self, charm_type) + tracing_endpoint = _get_tracing_endpoint( + tracing_endpoint_attr, self, charm_type + ) buffer_only = False # whether we're only exporting to buffer, or also to the otlp exporter. @@ -776,10 +788,14 @@ def wrap_init(self: CharmBase, framework: Framework, *args, **kwargs): buffer_only = True server_cert: Optional[Union[str, Path]] = ( - _get_server_cert(server_cert_attr, self, charm_type) if server_cert_attr else None + _get_server_cert(server_cert_attr, self, charm_type) + if server_cert_attr + else None ) - if (tracing_endpoint and tracing_endpoint.startswith("https://")) and not server_cert: + if ( + tracing_endpoint and tracing_endpoint.startswith("https://") + ) and not server_cert: logger.error( "Tracing endpoint is https, but no server_cert has been passed." "Please point @trace_charm to a `server_cert` attr. " @@ -810,7 +826,9 @@ def wrap_init(self: CharmBase, framework: Framework, *args, **kwargs): # and retry the next time otlp_exporter = _OTLPSpanExporter( endpoint=tracing_endpoint, - certificate_file=str(Path(server_cert).absolute()) if server_cert else None, + certificate_file=str(Path(server_cert).absolute()) + if server_cert + else None, timeout=_OTLP_SPAN_EXPORTER_TIMEOUT, # give individual requests 1 second to succeed ) exporters.append(otlp_exporter) @@ -825,10 +843,16 @@ def wrap_init(self: CharmBase, framework: Framework, *args, **kwargs): _tracer = get_tracer(_service_name) # type: ignore _tracer_token = tracer.set(_tracer) - dispatch_path = os.getenv("JUJU_DISPATCH_PATH", "") # something like hooks/install - event_name = dispatch_path.split("/")[1] if "/" in dispatch_path else dispatch_path + dispatch_path = os.getenv( + "JUJU_DISPATCH_PATH", "" + ) # something like hooks/install + event_name = ( + dispatch_path.split("/")[1] if "/" in dispatch_path else dispatch_path + ) root_span_name = f"{unit_name}: {event_name} event" - span = _tracer.start_span(root_span_name, attributes={"juju.dispatch_path": dispatch_path}) + span = _tracer.start_span( + root_span_name, attributes={"juju.dispatch_path": dispatch_path} + ) # all these shenanigans are to work around the fact that the opentelemetry tracing API is built # on the assumption that spans will be used as contextmanagers. @@ -863,13 +887,17 @@ def wrap_close(): opentelemetry.context.detach(span_token) # type: ignore tracer.reset(_tracer_token) tp = cast(TracerProvider, get_tracer_provider()) - flush_successful = tp.force_flush(timeout_millis=1000) # don't block for too long + flush_successful = tp.force_flush( + timeout_millis=1000 + ) # don't block for too long if buffer_only: # if we're in buffer_only mode, it means we couldn't even set up the exporter for # tempo as we're missing some data. # so attempting to flush the buffer doesn't make sense - dev_logger.debug("tracing backend unavailable: all spans pushed to buffer") + dev_logger.debug( + "tracing backend unavailable: all spans pushed to buffer" + ) else: dev_logger.debug("tracing backend found: attempting to flush buffer...") @@ -885,7 +913,9 @@ def wrap_close(): if not previous_spans_buffered: # if the buffer was empty to begin with, any spans we collected now can be discarded buffer.drop() - dev_logger.debug("buffer dropped: this trace has been sent already") + dev_logger.debug( + "buffer dropped: this trace has been sent already" + ) else: # if the buffer was nonempty, we can attempt to flush it dev_logger.debug("attempting buffer flush...")