From db6a03e686c6aa3a654c1f3b7b03506f68b72106 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 17:35:13 +0000 Subject: [PATCH 001/159] Remove Foresight https://github.com/canonical/data-platform-workflows/issues/37 --- .github/workflows/ci.yaml | 26 ++------------------------ .github/workflows/release.yaml | 2 -- tox.ini | 1 - 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d8625f2b0..5e7c3624b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,8 +19,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: 'Foresight: Collect workflow telemetry' - uses: runforesight/foresight-workflow-kit-action@v1 - name: Checkout uses: actions/checkout@v3 - name: Install tox @@ -34,24 +32,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: 'Foresight: Collect workflow telemetry' - uses: runforesight/foresight-workflow-kit-action@v1 - name: Checkout uses: actions/checkout@v3 - name: Install tox # TODO: Consider replacing with custom image on self-hosted runner OR pinning version run: python3 -m pip install tox - name: Run tests - run: tox run -e unit -- --junit-xml=pytest_report.xml - - name: 'Foresight: Collect test results' - uses: runforesight/foresight-test-kit-action@v1 - if: ${{ success() || failure() }} - with: - test_framework: PYTEST - test_format: JUNIT - test_path: pytest_report.xml - coverage_format: LCOV/TXT - coverage_path: coverage.lcov + run: tox run -e unit build: name: Build charms @@ -71,8 +58,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 steps: - - name: 'Foresight: Collect workflow telemetry' - uses: runforesight/foresight-workflow-kit-action@v1 - name: Checkout uses: actions/checkout@v3 - name: Setup operator environment @@ -98,16 +83,9 @@ jobs: fi - name: Run integration tests # set a predictable model name so it can be consumed by charm-logdump-action - run: sg microk8s -c "tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --model testing --junit-xml=pytest_report.xml" + run: sg microk8s -c "tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --model testing" env: CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} - - name: 'Foresight: Collect test results' - uses: runforesight/foresight-test-kit-action@v1 - if: ${{ success() || failure() }} - with: - test_framework: PYTEST - test_format: JUNIT - test_path: pytest_report.xml - name: Dump logs uses: canonical/charm-logdump-action@main if: failure() diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 26d8b076b..31d7d2370 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -13,8 +13,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: 'Foresight: Collect workflow telemetry' - uses: runforesight/foresight-workflow-kit-action@v1 - name: Checkout uses: actions/checkout@v3 with: diff --git a/tox.ini b/tox.ini index abbd931b9..dafc8f409 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,6 @@ commands = coverage run --source={[vars]src_path} \ -m pytest -v --tb native -s {posargs} {[vars]tests_path}/unit coverage report - coverage lcov [testenv:integration-database] description = Run integration tests for the database relation From 4bf9636438559eaf46f42d6e9aed05915781bcd0 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 13 Apr 2023 18:40:10 +0000 Subject: [PATCH 002/159] Refactor --- src/charm.py | 306 ++++++++++------------------- src/mysql_router_helpers.py | 135 ------------- src/relations/database_provides.py | 212 +++++++------------- src/relations/database_requires.py | 272 ++++++------------------- src/relations/tls.py | 109 +++++++--- src/utils.py | 20 -- src/workload.py | 88 +++++++++ 7 files changed, 411 insertions(+), 731 deletions(-) delete mode 100644 src/mysql_router_helpers.py delete mode 100644 src/utils.py create mode 100644 src/workload.py diff --git a/src/charm.py b/src/charm.py index 4abd84f7b..37974b117 100755 --- a/src/charm.py +++ b/src/charm.py @@ -4,110 +4,123 @@ # # Learn more at: https://juju.is/docs/sdk -"""MySQL-Router k8s charm.""" +"""MySQL Router k8s charm.""" -import json import logging -from typing import Optional, Set +import charms.data_platform_libs.v0.data_interfaces as data_interfaces +import ops from lightkube import ApiError, Client from lightkube.models.core_v1 import ServicePort, ServiceSpec from lightkube.models.meta_v1 import ObjectMeta from lightkube.resources.core_v1 import Pod, Service -from ops.charm import CharmBase -from ops.main import main -from ops.model import ActiveStatus, BlockedStatus, Relation, WaitingStatus -from ops.pebble import Layer +import relations +import workload from constants import ( DATABASE_PROVIDES_RELATION, DATABASE_REQUIRES_RELATION, - MYSQL_DATABASE_CREATED, MYSQL_ROUTER_CONTAINER_NAME, - MYSQL_ROUTER_REQUIRES_DATA, - MYSQL_ROUTER_SERVICE_NAME, - NUM_UNITS_BOOTSTRAPPED, - PEER, - UNIT_BOOTSTRAPPED, ) -from mysql_router_helpers import MySQLRouter -from relations.database_provides import DatabaseProvidesRelation -from relations.database_requires import DatabaseRequiresRelation -from relations.tls import MySQLRouterTLS logger = logging.getLogger(__name__) -class MySQLRouterOperatorCharm(CharmBase): - """Operator charm for MySQLRouter.""" +class MySQLRouterOperatorCharm(ops.charm.CharmBase): + """Operator charm for MySQL Router.""" - def __init__(self, *args): + def __init__(self, *args) -> None: super().__init__(*args) - self.framework.observe(self.on.install, self._on_install) - self.framework.observe(self.on.leader_elected, self._on_leader_elected) + self.database_requires = relations.database_requires.Relation( + data_interfaces.DatabaseRequires( + self, + relation_name=DATABASE_REQUIRES_RELATION, + # HACK: mysqlrouter needs a user, but not a database + # Use the DatabaseRequires interface to get a user; disregard the database + database_name="_unused_mysqlrouter_database", + extra_user_roles="mysqlrouter", + ) + ) self.framework.observe( - getattr(self.on, "mysql_router_pebble_ready"), self._on_mysql_router_pebble_ready + self.database_requires.interface.on.database_created, + self._reconcile_database_relations, + ) + self.framework.observe( + self.on[DATABASE_REQUIRES_RELATION].relation_broken, + self._reconcile_database_relations, ) - self.framework.observe(self.on[PEER].relation_changed, self._on_peer_relation_changed) - self.framework.observe(self.on.update_status, self._on_update_status) - - self.database_provides = DatabaseProvidesRelation(self) - self.database_requires = DatabaseRequiresRelation(self) - self.tls = MySQLRouterTLS(self) - - # ======================= - # Properties - # ======================= - - @property - def peers(self) -> Optional[Relation]: - """Fetch the peer relation.""" - return self.model.get_relation(PEER) - @property - def app_peer_data(self): - """Application peer data object.""" - if not self.peers: - return {} + self.database_provides = relations.database_provides.Relation( + data_interfaces.DatabaseProvides(self, relation_name=DATABASE_PROVIDES_RELATION) + ) + self.framework.observe( + self.database_provides.interface.on.database_requested, + self._reconcile_database_relations, + ) + self.framework.observe( + self.on[DATABASE_PROVIDES_RELATION].relation_broken, self._reconcile_database_relations + ) - return self.peers.data[self.app] + self.framework.observe( + getattr(self.on, "mysql_router_pebble_ready"), self._on_mysql_router_pebble_ready + ) + self.framework.observe(self.on.start, self._on_start) + self.framework.observe(self.on.leader_elected, self._on_leader_elected) - @property - def unit_peer_data(self): - """Unit peer data object.""" - if not self.peers: - return {} + self.workload = workload.Workload(self.unit.get_container(MYSQL_ROUTER_CONTAINER_NAME)) - return self.peers.data[self.unit] + self.tls = relations.tls.MySQLRouterTLS(self) @property - def endpoint(self): + def _endpoint(self) -> str: """The k8s endpoint for the charm.""" return f"{self.model.app.name}.{self.model.name}.svc.cluster.local" - @property - def unit_hostname(self) -> str: - """Get the hostname.localdomain for a unit. + def _determine_status(self, event) -> ops.model.StatusBase: + inactive_relations = [] + for relation, active in [ + (DATABASE_REQUIRES_RELATION, self.database_requires.is_desired_active(event)), + (DATABASE_PROVIDES_RELATION, self.database_provides.is_desired_active(event)), + ]: + if not active: + inactive_relations.append(relation) + if inactive_relations: + return ops.model.WaitingStatus( + f"Waiting for relation{'s' if len(inactive_relations) > 1 else ''}: {', '.join(inactive_relations)}" + ) + if not self.workload.container_ready: + return ops.model.MaintenanceStatus("Waiting for container") # TODO + return ops.model.ActiveStatus() - Translate juju unit name to hostname.localdomain, necessary - for correct name resolution under k8s. + def _set_status(self, event=None) -> None: + if isinstance(self.unit.status, ops.model.BlockedStatus): + return + self.unit.status = self._determine_status(event) - Returns: - A string representing the hostname.localdomain of the unit. - """ - return f"{self.unit.name.replace('/', '-')}.{self.app.name}-endpoints" + def _create_user(self) -> None: + if self.database_provides.active: + # User already created + return + password = self.database_provides.generate_password() + self.database_requires.create_application_database_and_user( + self.database_provides.username, + password, + self.database_provides.database, + ) + self.database_provides.set_databag(password, self._endpoint) - # ======================= - # Helpers - # ======================= + def _delete_user(self) -> None: + if not self.database_provides.active: + # No user to delete + return + self.database_requires.delete_application_user(self.database_provides.username) + self.database_provides.delete_databag() def _patch_service(self, name: str, ro_port: int, rw_port: int) -> None: """Patch juju created k8s service. - The k8s service will be tied to pod-0 so that the service is auto cleaned by k8s when the last pod is scaled down. - Args: name: The name of the service. ro_port: The read only port. @@ -153,147 +166,46 @@ def _patch_service(self, name: str, ro_port: int, rw_port: int) -> None: field_manager=self.model.app.name, ) - def get_secret(self, scope: str, key: str) -> Optional[str]: - """Get secret from the peer relation databag.""" - if scope == "unit": - return self.unit_peer_data.get(key, None) - elif scope == "app": - return self.app_peer_data.get(key, None) - else: - raise RuntimeError("Unknown secret scope") - - def set_secret(self, scope: str, key: str, value: Optional[str]) -> None: - """Set secret in the peer relation databag.""" - if scope == "unit": - if not value: - del self.unit_peer_data[key] - return - self.unit_peer_data.update({key: value}) - elif scope == "app": - if not value: - del self.app_peer_data[key] - return - self.app_peer_data.update({key: value}) - else: - raise RuntimeError("Unknown secret scope") - - @property - def mysql_router_layer(self) -> Layer: - """Return a layer configuration for the mysql router service.""" - requires_data = json.loads(self.app_peer_data[MYSQL_ROUTER_REQUIRES_DATA]) - host, port = requires_data["endpoints"].split(",")[0].split(":") - return Layer( - { - "summary": "mysql router layer", - "description": "the pebble config layer for mysql router", - "services": { - MYSQL_ROUTER_SERVICE_NAME: { - "override": "replace", - "summary": "mysql router", - "command": "/run.sh mysqlrouter", - "startup": "enabled", - "environment": { - "MYSQL_HOST": host, - "MYSQL_PORT": port, - "MYSQL_USER": requires_data["username"], - "MYSQL_PASSWORD": self.get_secret("app", "database-password") or "", - }, - }, - }, - } - ) - - def _bootstrap_mysqlrouter(self) -> bool: - if not self.app_peer_data.get(MYSQL_DATABASE_CREATED): - return False - - pebble_layer = self.mysql_router_layer - - container = self.unit.get_container(MYSQL_ROUTER_CONTAINER_NAME) - plan = container.get_plan() - - if plan.services != pebble_layer.services: - container.add_layer(MYSQL_ROUTER_SERVICE_NAME, pebble_layer, combine=True) - container.start(MYSQL_ROUTER_SERVICE_NAME) - - MySQLRouter.wait_until_mysql_router_ready() - - self.unit_peer_data[UNIT_BOOTSTRAPPED] = "true" - - return True - - return False - - @property - def missing_relations(self) -> Set[str]: - """Return a set of missing relations.""" - missing_relations = set() - for relation_name in [DATABASE_REQUIRES_RELATION, DATABASE_PROVIDES_RELATION]: - if not self.model.get_relation(relation_name): - missing_relations.add(relation_name) - return missing_relations - # ======================= # Handlers # ======================= - def _on_install(self, _) -> None: - """Handle the install event.""" - self.unit.status = WaitingStatus() - # Try set workload version - container = self.unit.get_container(MYSQL_ROUTER_CONTAINER_NAME) - if container.can_connect(): - if version := MySQLRouter.get_version(container): - self.unit.set_workload_version(version) + def _reconcile_database_relations(self, event=None) -> None: + # TODO: rename method? + if self.database_requires.is_desired_active(event) and self.workload.container_ready: + self.workload.start( + self.database_requires.host, + self.database_requires.port, + self.database_requires.username, + self.database_requires.password, + ) + else: + self.workload.stop() + if self.unit.is_leader(): + if self.database_requires.is_desired_active( + event + ) and self.database_provides.is_desired_active(event): + self._create_user() + else: + self._delete_user() + self._set_status(event) + + def _on_mysql_router_pebble_ready(self, _) -> None: + self.unit.set_workload_version(self.workload.version) + self._reconcile_database_relations() - def _on_leader_elected(self, _) -> None: - """Handle the leader elected event. + def _on_start(self, _) -> None: + # If no relations are active, charm status has not been set + self._set_status() - Patch existing k8s service to include read-write and read-only services. - """ - # Create the read-write and read-only services + def _on_leader_elected(self, _) -> None: + """Patch existing k8s service to include read-write and read-only services.""" try: - self._patch_service(f"{self.app.name}", ro_port=6447, rw_port=6446) + self._patch_service(self.app.name, ro_port=6447, rw_port=6446) except ApiError: logger.exception("Failed to patch k8s service") - self.unit.status = BlockedStatus("Failed to patch k8s service") - return - - def _on_mysql_router_pebble_ready(self, _) -> None: - """Handle the mysql-router pebble ready event.""" - if self._bootstrap_mysqlrouter(): - self.unit.status = ActiveStatus() - - def _on_peer_relation_changed(self, _) -> None: - """Handle the peer relation changed event. - - Bootstraps mysqlrouter if the relations exist, but pebble_ready event - fired before the requires relation was formed. - """ - if ( - isinstance(self.unit.status, WaitingStatus) - and self.app_peer_data.get(MYSQL_DATABASE_CREATED) - and self._bootstrap_mysqlrouter() - ): - self.unit.status = ActiveStatus() - - if self.unit.is_leader(): - num_units_bootstrapped = sum( - 1 - for _ in self.peers.units.union({self.unit}) - if self.unit_peer_data.get(UNIT_BOOTSTRAPPED) - ) - self.app_peer_data[NUM_UNITS_BOOTSTRAPPED] = str(num_units_bootstrapped) - - def _on_update_status(self, _) -> None: - """Handle update-status event.""" - if self.missing_relations: - self.unit.status = WaitingStatus( - f"Waiting for relations: {' '.join(self.missing_relations)}" - ) - return - self.unit.status = ActiveStatus() + self.unit.status = ops.model.BlockedStatus("Failed to patch k8s service") if __name__ == "__main__": - main(MySQLRouterOperatorCharm) + ops.main.main(MySQLRouterOperatorCharm) diff --git a/src/mysql_router_helpers.py b/src/mysql_router_helpers.py deleted file mode 100644 index 82fda2240..000000000 --- a/src/mysql_router_helpers.py +++ /dev/null @@ -1,135 +0,0 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Helper class to manage the MySQL Router lifecycle.""" - -import logging -import socket -from typing import Optional - -import mysql.connector -from ops.model import Container -from tenacity import retry, stop_after_delay, wait_fixed - -logger = logging.getLogger(__name__) - - -class Error(Exception): - """Base class for exceptions in this module.""" - - def __repr__(self): - """String representation of the Error class.""" - return "<{}.{} {}>".format(type(self).__module__, type(self).__name__, self.args) - - @property - def name(self): - """Return a string representation of the model plus class.""" - return "<{}.{}>".format(type(self).__module__, type(self).__name__) - - @property - def message(self): - """Return the message passed as an argument.""" - return self.args[0] - - -class MySQLRouterCreateUserWithDatabasePrivilegesError(Error): - """Exception raised when there is an issue creating a database scoped user.""" - - -class MySQLRouterPortsNotOpenError(Error): - """Exception raised when mysqlrouter is not bootstrapped and started.""" - - -class MySQLRouter: - """Encapsulates all operations related to MySQL and MySQLRouter.""" - - @staticmethod - def create_user_with_database_privileges( - username, password, hostname, database, db_username, db_password, db_host, db_port - ) -> None: - """Create a database scope mysql user. - - Args: - username: Username of the user to create - password: Password of the user to create - hostname: Hostname of the user to create - database: Database that the user should be restricted to - db_username: The user to connect to the database with - db_password: The password to use to connect to the database - db_host: The host name of the database - db_port: The port for the database - - Raises: - MySQLRouterCreateUserWithDatabasePrivilegesError - - when there is an issue creating a database scoped user - """ - try: - connection = mysql.connector.connect( - user=db_username, password=db_password, host=db_host, port=db_port - ) - cursor = connection.cursor() - - cursor.execute(f"CREATE USER `{username}`@`{hostname}` IDENTIFIED BY '{password}'") - cursor.execute(f"GRANT ALL PRIVILEGES ON {database}.* TO `{username}`@`{hostname}`") - - cursor.close() - connection.close() - except mysql.connector.Error as e: - logger.exception("Failed to create user scoped to a database", exc_info=e) - raise MySQLRouterCreateUserWithDatabasePrivilegesError(e.msg) - - @staticmethod - def delete_application_user( - username, hostname, db_username, db_password, db_host, db_port - ) -> None: - """Delete the application user. - - Args: - username: Username of the user to delete - hostname: Hostname of the user to delete - db_username: The user to connect to the database with - db_password: The password to use to connect to the database - db_host: The host name of the database - db_port: The port for the database - """ - try: - connection = mysql.connector.connect( - user=db_username, password=db_password, host=db_host, port=db_port - ) - cursor = connection.cursor() - - cursor.execute(f"DROP USER IF EXISTS `{username}`@`{hostname}`") - - cursor.close() - connection.close() - except mysql.connector.Error as e: - logger.exception("Failed to delete application user", exc_info=e) - - @staticmethod - @retry(reraise=True, stop=stop_after_delay(30), wait=wait_fixed(5)) - def wait_until_mysql_router_ready() -> None: - """Wait until a connection to MySQL router is possible. - - Retry every 5 seconds for 30 seconds if there is an issue obtaining a connection. - """ - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex(("127.0.0.1", 6446)) - if result != 0: - raise MySQLRouterPortsNotOpenError() - sock.close() - - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex(("127.0.0.1", 6447)) - if result != 0: - raise MySQLRouterPortsNotOpenError() - sock.close() - - @staticmethod - def get_version(container: Container) -> Optional[str]: - """Get the MySQL Router version.""" - process = container.exec(["mysqlrouter", "-V"]) - raw_version, _ = process.wait_output() - for version in raw_version.strip().split(): - if version.startswith("8"): - return version - return None diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 6ba920fb8..2d3dd1af5 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -1,142 +1,70 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Library containing the implementation of the database provides relation.""" - -import json -import logging - -from charms.data_platform_libs.v0.data_interfaces import ( - DatabaseProvides, - DatabaseRequestedEvent, -) -from ops.framework import Object -from ops.model import WaitingStatus - -from constants import ( - CREDENTIALS_SHARED, - DATABASE_PROVIDES_RELATION, - DATABASE_REQUIRES_RELATION, - MYSQL_DATABASE_CREATED, - MYSQL_ROUTER_PROVIDES_DATA, - MYSQL_ROUTER_REQUIRES_APPLICATION_DATA, - PEER, - UNIT_BOOTSTRAPPED, -) -from mysql_router_helpers import MySQLRouter - -logger = logging.getLogger(__name__) - - -class DatabaseProvidesRelation(Object): - """Encapsulation of the relation between mysqlrouter and the consumer application.""" - - def __init__(self, charm): - super().__init__(charm, DATABASE_PROVIDES_RELATION) - - self.charm = charm - self.database_provides_relation = DatabaseProvides( - self.charm, relation_name=DATABASE_PROVIDES_RELATION - ) - - self.framework.observe( - self.database_provides_relation.on.database_requested, self._on_database_requested - ) - self.framework.observe( - self.charm.on[PEER].relation_changed, self._on_peer_relation_changed - ) - - self.framework.observe( - self.charm.on[DATABASE_PROVIDES_RELATION].relation_broken, self._on_database_broken - ) - - # ======================= - # Handlers - # ======================= - - def _on_database_requested(self, event: DatabaseRequestedEvent) -> None: - """Handle the database requested event.""" - if not self.charm.unit.is_leader(): - return - - # Store data in databag to trigger DatabaseRequires initialization in database_requires.py - self.charm.app_peer_data[MYSQL_ROUTER_PROVIDES_DATA] = json.dumps( - {"database": event.database, "extra_user_roles": event.extra_user_roles} - ) - - def _on_peer_relation_changed(self, _) -> None: - """Handle the peer relation changed event.""" - if not self.charm.unit.is_leader(): - return - - if self.charm.app_peer_data.get(CREDENTIALS_SHARED): - logger.debug("Credentials already shared") - return - - if not self.charm.app_peer_data.get(MYSQL_DATABASE_CREATED): - logger.debug("Database not created yet") - return - - if not self.charm.unit_peer_data.get(UNIT_BOOTSTRAPPED): - logger.debug("Unit not bootstrapped yet") - return - - if not self.charm.app_peer_data.get(MYSQL_ROUTER_REQUIRES_APPLICATION_DATA): - logger.debug("No requires application data found") - return - - database_provides_relations = self.charm.model.relations.get(DATABASE_PROVIDES_RELATION) - - requires_application_data = json.loads( - self.charm.app_peer_data[MYSQL_ROUTER_REQUIRES_APPLICATION_DATA] - ) - provides_relation_id = database_provides_relations[0].id - - self.database_provides_relation.set_credentials( - provides_relation_id, - requires_application_data["username"], - self.charm.get_secret("app", "application-password"), - ) - - self.database_provides_relation.set_endpoints( - provides_relation_id, f"{self.charm.endpoint}:6446" - ) - - self.database_provides_relation.set_read_only_endpoints( - provides_relation_id, f"{self.charm.endpoint}:6447" - ) - - self.charm.app_peer_data[CREDENTIALS_SHARED] = "true" - - def _on_database_broken(self, _) -> None: - """Handle the database relation broken event.""" - self.charm.unit.status = WaitingStatus( - f"Waiting for relations: {DATABASE_PROVIDES_RELATION}" - ) - if not self.charm.unit.is_leader(): - return - - # application user cleanup when backend relation still in place - if backend_relation := self.charm.model.get_relation(DATABASE_REQUIRES_RELATION): - if app_data := self.charm.app_peer_data.get(MYSQL_ROUTER_REQUIRES_APPLICATION_DATA): - username = json.loads(app_data)["username"] - - db_username = backend_relation.data[backend_relation.app]["username"] - db_password = backend_relation.data[backend_relation.app]["password"] - db_host, db_port = backend_relation.data[backend_relation.app]["endpoints"].split( - ":" - ) - - MySQLRouter.delete_application_user( - username=username, - hostname="%", - db_username=db_username, - db_password=db_password, - db_host=db_host, - db_port=db_port, - ) - # clean up departing app data - self.charm.app_peer_data.pop(MYSQL_ROUTER_REQUIRES_APPLICATION_DATA, None) - self.charm.app_peer_data.pop(MYSQL_ROUTER_PROVIDES_DATA, None) - self.charm.app_peer_data.pop(CREDENTIALS_SHARED, None) - self.charm.set_secret("app", "application-password", None) +import dataclasses +import secrets +import string + +import charms.data_platform_libs.v0.data_interfaces as data_interfaces +import ops + +from constants import PASSWORD_LENGTH + + +@dataclasses.dataclass +class Relation: + interface: data_interfaces.DatabaseProvides + + @property + def active(self) -> bool: + """Whether relation is currently active""" + for key in ["database", "username", "password", "endpoints"]: + if key not in self._local_databag: + return False + return True + + @property + def database(self) -> str: + return self._remote_databag["database"] + + @property + def username(self) -> str: + return f"relation-{self._id}" + + @property + def _relation(self) -> ops.model.Relation: + relations = self.interface.relations + assert len(relations) == 1 + return relations[0] + + @property + def _id(self) -> int: + return self._relation.id + + @property + def _remote_databag(self) -> dict: + return self.interface.fetch_relation_data()[self._id] + + @property + def _local_databag(self) -> ops.model.RelationDataContent: + return self._relation.data[self.interface.local_app] + + def is_desired_active(self, event) -> bool: + """Whether relation should be active once the event is handled""" + if isinstance(event, ops.charm.RelationBrokenEvent) and event.relation.id == self._id: + # Relation is being removed; it is no longer active + return False + if isinstance(event, data_interfaces.DatabaseRequestedEvent): + return True + return self.active + + def set_databag(self, password: str, endpoint: str) -> None: + self.interface.set_database(self._id, self.database) + self.interface.set_credentials(self._id, self.username, password) + self.interface.set_endpoints(self._id, f"{endpoint}:6446") + self.interface.set_read_only_endpoints(self._id, f"{endpoint}:6447") + + def delete_databag(self) -> None: + self._local_databag.clear() + + @staticmethod + def generate_password() -> str: + choices = string.ascii_letters + string.digits + return "".join([secrets.choice(choices) for _ in range(PASSWORD_LENGTH)]) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 19fa3c0b1..b01ddb778 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -1,211 +1,65 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Library containing the implementation of the database requires relation.""" - -import json -import logging -from typing import Dict, Optional - -from charms.data_platform_libs.v0.data_interfaces import ( - DatabaseCreatedEvent, - DatabaseEndpointsChangedEvent, - DatabaseRequires, -) -from ops.framework import Object -from ops.model import BlockedStatus, ModelError, WaitingStatus - -from constants import ( - CREDENTIALS_SHARED, - DATABASE_PROVIDES_RELATION, - DATABASE_REQUIRES_RELATION, - MYSQL_DATABASE_CREATED, - MYSQL_ROUTER_CONTAINER_NAME, - MYSQL_ROUTER_PROVIDES_DATA, - MYSQL_ROUTER_REQUIRES_APPLICATION_DATA, - MYSQL_ROUTER_REQUIRES_DATA, - MYSQL_ROUTER_SERVICE_NAME, - PASSWORD_LENGTH, - PEER, - UNIT_BOOTSTRAPPED, -) -from mysql_router_helpers import ( - MySQLRouter, - MySQLRouterCreateUserWithDatabasePrivilegesError, -) -from utils import generate_random_password - -logger = logging.getLogger(__name__) - - -class DatabaseRequiresRelation(Object): - """Encapsulation of the relation between mysqlrouter and mysql database.""" - - def __init__(self, charm): - super().__init__(charm, DATABASE_REQUIRES_RELATION) - - self.charm = charm - - self.framework.observe( - self.charm.on[DATABASE_REQUIRES_RELATION].relation_joined, - self._on_database_requires_relation_joined, - ) - - provides_data = self._get_provides_data() - if not provides_data: - logger.debug("No provides data found, not handling the relation yet.") - return - - self.database_requires_relation = DatabaseRequires( - self.charm, - relation_name=DATABASE_REQUIRES_RELATION, - database_name=provides_data["database"], - extra_user_roles="mysqlrouter", - ) - - self.framework.observe( - self.database_requires_relation.on.database_created, self._on_backend_database_created - ) - self.framework.observe( - self.database_requires_relation.on.endpoints_changed, self._on_endpoints_changed - ) - - self.framework.observe( - self.charm.on[DATABASE_REQUIRES_RELATION].relation_broken, - self._on_backend_database_broken, - ) - - self.framework.observe( - self.charm.on[PEER].relation_changed, self._on_peer_relation_changed - ) - - # ======================= - # Helpers - # ======================= - - def _get_provides_data(self) -> Optional[Dict]: - """Helper to get the `provides` relation data from the app peer databag.""" - try: - provides_data = self.charm.app_peer_data.get(MYSQL_ROUTER_PROVIDES_DATA) - if not provides_data: - return None - except ModelError: - # Error raised on app removal - return None - - return json.loads(provides_data) - - def _create_application_user( - self, db_username: str, db_password: str, db_endpoint: str +import dataclasses + +import charms.data_platform_libs.v0.data_interfaces as data_interfaces +import ops + + +@dataclasses.dataclass +class Relation: + interface: data_interfaces.DatabaseRequires + + @property + def host(self) -> str: + return self._endpoint.split(":")[0] + + @property + def port(self) -> str: + return self._endpoint.split(":")[1] + + @property + def username(self) -> str: + return self._remote_databag["username"] + + @property + def password(self) -> str: + return self._remote_databag["password"] + + @property + def _id(self) -> int: + relations = self.interface.relations + assert len(relations) == 1 + return relations[0].id + + @property + def _remote_databag(self) -> dict: + return self.interface.fetch_relation_data()[self._id] + + @property + def _endpoint(self) -> str: + endpoints = self._remote_databag["endpoints"].split(",") + assert len(endpoints) == 1 + return endpoints[0] + + @property + def _active(self) -> bool: + """Whether relation is currently active""" + if not self.interface.relations: + return False + return self.interface.is_resource_created() + + def is_desired_active(self, event) -> bool: + """Whether relation should be active once the event is handled""" + if isinstance(event, ops.charm.RelationBrokenEvent) and event.relation.id == self._id: + # Relation is being removed; it is no longer active + return False + return self._active + + def create_application_database_and_user( + self, username: str, password: str, database: str ) -> None: - """Helper to create a database user for the application.""" - provides_data = self._get_provides_data() - provides_relation_id = self.charm.model.relations[DATABASE_PROVIDES_RELATION][0].id - - username = f"application-user-{provides_relation_id}" - password = generate_random_password(PASSWORD_LENGTH) - db_host, db_port = db_endpoint.split(",")[0].split(":") - - try: - MySQLRouter.create_user_with_database_privileges( - username, - password, - "%", - provides_data["database"], - db_username, - db_password, - db_host, - db_port, - ) - except MySQLRouterCreateUserWithDatabasePrivilegesError: - logger.exception("Failed to create a database scoped user") - self.charm.unit.status = BlockedStatus("Failed to create a database scoped user") - return - - self.charm.app_peer_data[MYSQL_ROUTER_REQUIRES_APPLICATION_DATA] = json.dumps( - { - "username": username, - } - ) - self.charm.set_secret("app", "application-password", password) - self.charm.set_secret("app", "database-password", db_password) - self.charm.app_peer_data[MYSQL_DATABASE_CREATED] = "true" - logger.info(f"Created database user {username}.") - - # ======================= - # Handlers - # ======================= - - def _on_database_requires_relation_joined(self, event) -> None: - """Handle the backend-database relation joined event. - - Waits until the database relation with the application is formed before - triggering the database_requires relations joined event (which will request the database). - """ - provides_data = self._get_provides_data() - if not provides_data: - logger.debug("Waiting until a relation with an application is formed") - event.defer() - return - - self.database_requires_relation._on_relation_joined_event(event) - - def _on_backend_database_created(self, event: DatabaseCreatedEvent) -> None: - """Handle the database created event.""" - if not self.charm.unit.is_leader(): - return - - if self.charm.app_peer_data.get(MYSQL_DATABASE_CREATED): - return - - self.charm.app_peer_data[MYSQL_ROUTER_REQUIRES_DATA] = json.dumps( - { - "username": event.username, - "endpoints": event.endpoints, - } - ) - self._create_application_user(event.username, event.password, event.endpoints) - - def _on_endpoints_changed(self, event: DatabaseEndpointsChangedEvent) -> None: - """Handle the endpoints changed event. - - Update the endpoint in the MYSQL_ROUTER_REQUIRES_DATA so that future - bootstrapping units will not fail. - """ - if not self.charm.unit.is_leader(): - return - - if self.charm.app_peer_data.get(MYSQL_ROUTER_REQUIRES_DATA): - requires_data = json.loads(self.charm.app_peer_data[MYSQL_ROUTER_REQUIRES_DATA]) - - requires_data["endpoints"] = event.endpoints - - self.charm.app_peer_data[MYSQL_ROUTER_REQUIRES_DATA] = json.dumps(requires_data) - - def _on_peer_relation_changed(self, _) -> None: - """Handle the peer relation changed event.""" - if not self.charm.unit.is_leader(): - return - if self.charm.unit_peer_data.get(UNIT_BOOTSTRAPPED) and not self.charm.app_peer_data.get( - CREDENTIALS_SHARED - ): - # App related after first bootstrap, add app user - requires_data = json.loads(self.charm.app_peer_data[MYSQL_ROUTER_REQUIRES_DATA]) - - self._create_application_user( - requires_data["username"], - self.charm.get_secret("app", "database-password"), - requires_data["endpoints"], - ) - - def _on_backend_database_broken(self, _) -> None: - """Handle the database relation broken event.""" - self.charm.unit.status = WaitingStatus() - container = self.charm.unit.get_container(MYSQL_ROUTER_CONTAINER_NAME) - container.stop(MYSQL_ROUTER_SERVICE_NAME) + # TODO: port method from mysql_router_helpers or mysql charm lib + pass - self.charm.unit_peer.data.pop(UNIT_BOOTSTRAPPED, None) - if self.charm.unit.is_leader(): - # cleanup control and connection peer data - self.charm.app_peer_data.pop(MYSQL_DATABASE_CREATED, None) - self.charm.app_peer_data.pop(MYSQL_ROUTER_REQUIRES_DATA, None) + def delete_application_user(self, username: str) -> None: + # TODO: port method from mysql_router_helpers or mysql charm lib + pass diff --git a/src/relations/tls.py b/src/relations/tls.py index 86e2ae7d5..73020a300 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -10,6 +10,9 @@ from string import Template from typing import List, Optional +from ops import Relation +from charm import MySQLRouterOperatorCharm + from charms.tls_certificates_interface.v1.tls_certificates import ( CertificateAvailableEvent, CertificateExpiringEvent, @@ -29,8 +32,7 @@ TLS_RELATION, TLS_SSL_CERT_FILE, TLS_SSL_CONFIG_FILE, - TLS_SSL_KEY_FILE, - UNIT_BOOTSTRAPPED, + TLS_SSL_KEY_FILE, PEER, ) SCOPE = "unit" @@ -41,7 +43,7 @@ class MySQLRouterTLS(Object): """TLS Management class for MySQL Router Operator.""" - def __init__(self, charm: CharmBase): + def __init__(self, charm: MySQLRouterOperatorCharm): super().__init__(charm, TLS_RELATION) self.charm = charm self.certs = TLSCertificatesRequiresV1(self.charm, TLS_RELATION) @@ -60,6 +62,37 @@ def __init__(self, charm: CharmBase): self.framework.observe(self.certs.on.certificate_available, self._on_certificate_available) self.framework.observe(self.certs.on.certificate_expiring, self._on_certificate_expiring) + @property + def peers(self) -> Optional[Relation]: + """Fetch the peer relation.""" + return self.model.get_relation(PEER) + + @property + def app_peer_data(self): + """Application peer data object.""" + if not self.peers: + return {} + + return self.peers.data[self.charm.app] + + @property + def unit_peer_data(self): + """Unit peer data object.""" + if not self.peers: + return {} + + return self.peers.data[self.charm.unit] + + @property + def unit_hostname(self) -> str: + """Get the hostname.localdomain for a unit. + Translate juju unit name to hostname.localdomain, necessary + for correct name resolution under k8s. + Returns: + A string representing the hostname.localdomain of the unit. + """ + return f"{self.charm.unit.name.replace('/', '-')}.{self.charm.app.name}-endpoints" + @property def container(self): """Map to the MySQL Router container.""" @@ -70,6 +103,30 @@ def hostname(self): """Return the hostname of the MySQL Router container.""" return socket.gethostname() + def get_secret(self, scope: str, key: str) -> Optional[str]: + """Get secret from the peer relation databag.""" + if scope == "unit": + return self.unit_peer_data.get(key, None) + elif scope == "app": + return self.app_peer_data.get(key, None) + else: + raise RuntimeError("Unknown secret scope") + + def set_secret(self, scope: str, key: str, value: Optional[str]) -> None: + """Set secret in the peer relation databag.""" + if scope == "unit": + if not value: + del self.unit_peer_data[key] + return + self.unit_peer_data.update({key: value}) + elif scope == "app": + if not value: + del self.app_peer_data[key] + return + self.app_peer_data.update({key: value}) + else: + raise RuntimeError("Unknown secret scope") + # Handlers def _on_set_tls_private_key(self, event: ActionEvent) -> None: @@ -90,53 +147,54 @@ def _on_tls_relation_broken(self, _) -> None: """Disable TLS when TLS relation broken.""" for secret in ["cert", "chain", "ca"]: try: - self.charm.set_secret(SCOPE, secret, None) + self.set_secret(SCOPE, secret, None) except KeyError: # ignore key error for unit teardown pass # unset tls flag - self.charm.unit_peer_data.pop("tls") + self.unit_peer_data.pop("tls") self._unset_tls() def _on_certificate_available(self, event: CertificateAvailableEvent) -> None: """Enable TLS when TLS certificate available.""" - if self.charm.unit_peer_data.get(UNIT_BOOTSTRAPPED) != "true": + if not self.charm.workload.active: logger.debug("Unit not bootstrapped, defer TLS setup") event.defer() return if ( event.certificate_signing_request.strip() - != self.charm.get_secret(SCOPE, "csr").strip() + != self.get_secret(SCOPE, "csr").strip() ): logger.warning("Unknown certificate received. Ignoring.") return - if self.charm.unit_peer_data.get("tls") == "enabled": + # https://github.com/canonical/tls-certificates-operator/issues/34 + if self.unit_peer_data.get("tls") == "enabled": logger.debug("TLS is already enabled.") return - self.charm.set_secret( + self.set_secret( SCOPE, "chain", "\n".join(event.chain) if event.chain is not None else None ) - self.charm.set_secret(SCOPE, "cert", event.certificate) - self.charm.set_secret(SCOPE, "ca", event.ca) + self.set_secret(SCOPE, "cert", event.certificate) + self.set_secret(SCOPE, "ca", event.ca) # set member-state to avoid unwanted health-check actions - self.charm.unit_peer_data.update({"tls": "enabled"}) + self.unit_peer_data.update({"tls": "enabled"}) self._set_tls() def _on_certificate_expiring(self, event: CertificateExpiringEvent) -> None: """Request the new certificate when old certificate is expiring.""" - if event.certificate != self.charm.get_secret(SCOPE, "cert"): + if event.certificate != self.get_secret(SCOPE, "cert"): logger.error("An unknown certificate expiring.") return - key = self.charm.get_secret(SCOPE, "key").encode("utf-8") - old_csr = self.charm.get_secret(SCOPE, "csr").encode("utf-8") + key = self.get_secret(SCOPE, "key").encode("utf-8") + old_csr = self.get_secret(SCOPE, "csr").encode("utf-8") new_csr = generate_csr( private_key=key, - subject=self.charm.unit_hostname, + subject=self.unit_hostname, organization=self.charm.app.name, sans=self._get_sans(), ) @@ -161,10 +219,9 @@ def _request_certificate(self, internal_key: Optional[str] = None) -> None: ) # store secrets - self.charm.set_secret(SCOPE, "key", key.decode("utf-8")) - self.charm.set_secret(SCOPE, "csr", csr.decode("utf-8")) - # set control flag - self.charm.unit_peer_data.update({"tls": "requested"}) + self.set_secret(SCOPE, "key", key.decode("utf-8")) + self.set_secret(SCOPE, "csr", csr.decode("utf-8")) + self.unit_peer_data.pop("tls") self.certs.request_certificate_creation(certificate_signing_request=csr) def _get_sans(self) -> List[str]: @@ -176,7 +233,7 @@ def _get_sans(self) -> List[str]: return [ self.hostname, socket.getfqdn(), - str(self.charm.model.get_binding(self.charm.peers).network.bind_address), + str(self.charm.model.get_binding(self.peers).network.bind_address), ] @staticmethod @@ -207,8 +264,7 @@ def _set_tls(self) -> None: self._create_tls_config_file() self._push_tls_files_to_workload() # add tls layer merging with mysql-router layer - self.container.add_layer(MYSQL_ROUTER_SERVICE_NAME, self._tls_layer(), combine=True) - self.container.replan() + self.charm.workload.enable_tls(self._tls_layer()) logger.info("TLS enabled.") def _unset_tls(self) -> None: @@ -216,10 +272,7 @@ def _unset_tls(self) -> None: for file in [TLS_SSL_KEY_FILE, TLS_SSL_CERT_FILE, TLS_SSL_CONFIG_FILE]: self._remove_file(f"{ROUTER_CONFIG_DIRECTORY}/{file}") # remove tls layer overriding with original layer - self.container.add_layer( - MYSQL_ROUTER_SERVICE_NAME, self.charm.mysql_router_layer, combine=True - ) - self.container.replan() + self.charm.workload.disable_tls() logger.info("TLS disabled.") def _write_content_to_file( @@ -267,7 +320,7 @@ def _push_tls_files_to_workload(self) -> None: for key, value in tls_file.items(): self._write_content_to_file( f"{ROUTER_CONFIG_DIRECTORY}/{value}", - self.charm.get_secret(SCOPE, key), + self.get_secret(SCOPE, key), owner=MYSQL_ROUTER_USER_NAME, group=MYSQL_ROUTER_USER_NAME, permission=0o600, diff --git a/src/utils.py b/src/utils.py deleted file mode 100644 index 9428b840f..000000000 --- a/src/utils.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. - -"""A collection of utility functions that are used in the charm.""" - -import secrets -import string - - -def generate_random_password(length: int) -> str: - """Randomly generate a string intended to be used as a password. - - Args: - length: length of the randomly generated string to be returned - - Returns: - a string with random letters and digits of length specified - """ - choices = string.ascii_letters + string.digits - return "".join([secrets.choice(choices) for i in range(length)]) diff --git a/src/workload.py b/src/workload.py new file mode 100644 index 000000000..ab78dbdbc --- /dev/null +++ b/src/workload.py @@ -0,0 +1,88 @@ +import dataclasses + +import ops + +from constants import MYSQL_ROUTER_SERVICE_NAME + + +@dataclasses.dataclass +class Workload: + _container: ops.model.Container + + @property + def container_ready(self) -> bool: + return self._container.can_connect() + + @property + def active(self) -> bool: + return self._container.get_service(MYSQL_ROUTER_SERVICE_NAME).is_running() + + @property + def version(self) -> str: + process = self._container.exec(["mysqlrouter", "-V"]) + raw_version, _ = process.wait_output() + for version in raw_version.strip().split(): + if version.startswith("8"): + return version + return "" + + def start(self, host, port, username, password) -> None: + if self.active: + # If the host or port changes, MySQL Router will receive topology change notifications from MySQL + # Therefore, if the host or port changes, we do not need to restart MySQL Router + # Assumption: username or password will not change while database requires relation is active + # Therefore, MySQL Router does not need to be restarted if it is already running + return + self._container.add_layer( + MYSQL_ROUTER_SERVICE_NAME, + self._get_mysql_router_layer(host, port, username, password), + combine=True, + ) + self._container.start(MYSQL_ROUTER_SERVICE_NAME) + # TODO: wait until mysql router ready? https://github.com/canonical/mysql-router-k8s-operator/blob/45cf3be44f27476a0371c67d50d7a0193c0fadc2/src/charm.py#L219 + + def stop(self) -> None: + self._container.stop(MYSQL_ROUTER_SERVICE_NAME) + + def enable_tls(self, layer: ops.pebble.Layer): + self._container.add_layer(MYSQL_ROUTER_SERVICE_NAME, layer, combine=True) + self._container.replan() + + def disable_tls(self) -> None: + layer = ops.pebble.Layer( + { + "services": { + MYSQL_ROUTER_SERVICE_NAME: { + "override": "merge", + "command": "/run.sh mysqlrouter", + }, + }, + }, + ) + self._container.add_layer(MYSQL_ROUTER_SERVICE_NAME, layer, combine=True) + self._container.replan() + + @staticmethod + def _get_mysql_router_layer( + host: str, port: str, username: str, password: str + ) -> ops.pebble.Layer: + return ops.pebble.Layer( + { + "summary": "mysql router layer", + "description": "the pebble config layer for mysql router", + "services": { + MYSQL_ROUTER_SERVICE_NAME: { + "override": "replace", + "summary": "mysql router", + "command": "/run.sh mysqlrouter", + "startup": "enabled", + "environment": { + "MYSQL_HOST": host, + "MYSQL_PORT": port, + "MYSQL_USER": username, + "MYSQL_PASSWORD": password, + }, + }, + }, + } + ) From 360e2d2a45fbe54b3242c1ac17b254d11bb49547 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 13 Apr 2023 19:22:46 +0000 Subject: [PATCH 003/159] update comment --- src/relations/database_provides.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 2d3dd1af5..f4e369bfd 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -49,7 +49,7 @@ def _local_databag(self) -> ops.model.RelationDataContent: def is_desired_active(self, event) -> bool: """Whether relation should be active once the event is handled""" if isinstance(event, ops.charm.RelationBrokenEvent) and event.relation.id == self._id: - # Relation is being removed; it is no longer active + # Relation is being removed return False if isinstance(event, data_interfaces.DatabaseRequestedEvent): return True From 665d89fb2320f14d23482d897de256f9e448ca1c Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 14 Apr 2023 15:19:53 +0000 Subject: [PATCH 004/159] fix --- src/relations/tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/relations/tls.py b/src/relations/tls.py index 73020a300..957e5eace 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -221,7 +221,7 @@ def _request_certificate(self, internal_key: Optional[str] = None) -> None: # store secrets self.set_secret(SCOPE, "key", key.decode("utf-8")) self.set_secret(SCOPE, "csr", csr.decode("utf-8")) - self.unit_peer_data.pop("tls") + self.unit_peer_data.pop("tls", None) self.certs.request_certificate_creation(certificate_signing_request=csr) def _get_sans(self) -> List[str]: From 3aac704a49aa22aa94410f8f6975c5b31084d3a5 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 14 Apr 2023 15:45:37 +0000 Subject: [PATCH 005/159] Move TLS container methods to workload.py --- src/relations/tls.py | 88 ++++++++------------------------------------ src/workload.py | 42 ++++++++++++++++++++- 2 files changed, 56 insertions(+), 74 deletions(-) diff --git a/src/relations/tls.py b/src/relations/tls.py index 957e5eace..8372a8b1a 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -10,9 +10,6 @@ from string import Template from typing import List, Optional -from ops import Relation -from charm import MySQLRouterOperatorCharm - from charms.tls_certificates_interface.v1.tls_certificates import ( CertificateAvailableEvent, CertificateExpiringEvent, @@ -20,19 +17,22 @@ generate_csr, generate_private_key, ) +from ops import Relation from ops.charm import ActionEvent, CharmBase from ops.framework import Object from ops.pebble import Layer, PathError +from charm import MySQLRouterOperatorCharm from constants import ( MYSQL_ROUTER_CONTAINER_NAME, MYSQL_ROUTER_SERVICE_NAME, MYSQL_ROUTER_USER_NAME, + PEER, ROUTER_CONFIG_DIRECTORY, TLS_RELATION, TLS_SSL_CERT_FILE, TLS_SSL_CONFIG_FILE, - TLS_SSL_KEY_FILE, PEER, + TLS_SSL_KEY_FILE, ) SCOPE = "unit" @@ -93,11 +93,6 @@ def unit_hostname(self) -> str: """ return f"{self.charm.unit.name.replace('/', '-')}.{self.charm.app.name}-endpoints" - @property - def container(self): - """Map to the MySQL Router container.""" - return self.charm.unit.get_container(MYSQL_ROUTER_CONTAINER_NAME) - @property def hostname(self): """Return the hostname of the MySQL Router container.""" @@ -162,10 +157,7 @@ def _on_certificate_available(self, event: CertificateAvailableEvent) -> None: event.defer() return - if ( - event.certificate_signing_request.strip() - != self.get_secret(SCOPE, "csr").strip() - ): + if event.certificate_signing_request.strip() != self.get_secret(SCOPE, "csr").strip(): logger.warning("Unknown certificate received. Ignoring.") return @@ -247,57 +239,28 @@ def _parse_tls_file(raw_content: str) -> bytes: ).encode("utf-8") return base64.b64decode(raw_content) - def _remove_file(self, path: str) -> None: - """Remove a file from container workload. - - Args: - path: Full filesystem path to remove - """ - try: - self.container.remove_path(path) - except PathError: - # ignore file not found - pass - def _set_tls(self) -> None: """Enable TLS.""" - self._create_tls_config_file() - self._push_tls_files_to_workload() # add tls layer merging with mysql-router layer - self.charm.workload.enable_tls(self._tls_layer()) + self.charm.workload.enable_tls( + self._tls_layer(), + self._tls_config_file, + self.get_secret(SCOPE, "key"), + self.get_secret(SCOPE, "cert"), + ) logger.info("TLS enabled.") def _unset_tls(self) -> None: """Disable TLS.""" - for file in [TLS_SSL_KEY_FILE, TLS_SSL_CERT_FILE, TLS_SSL_CONFIG_FILE]: - self._remove_file(f"{ROUTER_CONFIG_DIRECTORY}/{file}") # remove tls layer overriding with original layer self.charm.workload.disable_tls() logger.info("TLS disabled.") - def _write_content_to_file( - self, - path: str, - content: str, - owner: str, - group: str, - permission: int = 0o640, - ) -> None: - """Write content to file. - - Args: - path: filesystem full path (with filename) - content: string content to write - owner: file owner - group: file group - permission: file permission - """ - self.container.push(path, content, permissions=permission, user=owner, group=group) - - def _create_tls_config_file(self) -> None: - """Render TLS template directly to file. + @property + def _tls_config_file(self) -> str: + """Render TLS template to string. - Render and write TLS enabling config file from template. + Config file enables TLS on MySQL Router. """ with open("templates/tls.cnf", "r") as template_file: template = Template(template_file.read()) @@ -305,26 +268,7 @@ def _create_tls_config_file(self) -> None: tls_ssl_key_file=f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_KEY_FILE}", tls_ssl_cert_file=f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CERT_FILE}", ) - - self._write_content_to_file( - f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CONFIG_FILE}", - config_string, - owner=MYSQL_ROUTER_USER_NAME, - group=MYSQL_ROUTER_USER_NAME, - permission=0o600, - ) - - def _push_tls_files_to_workload(self) -> None: - """Push TLS files to unit.""" - tls_file = {"key": TLS_SSL_KEY_FILE, "cert": TLS_SSL_CERT_FILE} - for key, value in tls_file.items(): - self._write_content_to_file( - f"{ROUTER_CONFIG_DIRECTORY}/{value}", - self.get_secret(SCOPE, key), - owner=MYSQL_ROUTER_USER_NAME, - group=MYSQL_ROUTER_USER_NAME, - permission=0o600, - ) + return config_string @staticmethod def _tls_layer() -> Layer: diff --git a/src/workload.py b/src/workload.py index ab78dbdbc..390710999 100644 --- a/src/workload.py +++ b/src/workload.py @@ -2,7 +2,14 @@ import ops -from constants import MYSQL_ROUTER_SERVICE_NAME +from constants import ( + MYSQL_ROUTER_SERVICE_NAME, + MYSQL_ROUTER_USER_NAME, + ROUTER_CONFIG_DIRECTORY, + TLS_SSL_CERT_FILE, + TLS_SSL_CONFIG_FILE, + TLS_SSL_KEY_FILE, +) @dataclasses.dataclass @@ -44,11 +51,18 @@ def start(self, host, port, username, password) -> None: def stop(self) -> None: self._container.stop(MYSQL_ROUTER_SERVICE_NAME) - def enable_tls(self, layer: ops.pebble.Layer): + def enable_tls( + self, layer: ops.pebble.Layer, config_file_content: str, key: str, certificate: str + ): + self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CONFIG_FILE}", config_file_content) + self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_KEY_FILE}", key) + self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CERT_FILE}", certificate) self._container.add_layer(MYSQL_ROUTER_SERVICE_NAME, layer, combine=True) self._container.replan() def disable_tls(self) -> None: + for file in [TLS_SSL_CONFIG_FILE, TLS_SSL_KEY_FILE, TLS_SSL_CERT_FILE]: + self._delete_file(f"{ROUTER_CONFIG_DIRECTORY}/{file}") layer = ops.pebble.Layer( { "services": { @@ -62,6 +76,30 @@ def disable_tls(self) -> None: self._container.add_layer(MYSQL_ROUTER_SERVICE_NAME, layer, combine=True) self._container.replan() + def _write_file(self, path: str, content: str) -> None: + """Write content to file. + + Args: + path: Full filesystem path (with filename) + content: File content + """ + self._container.push( + path, + content, + permissions=0o600, + user=MYSQL_ROUTER_USER_NAME, + group=MYSQL_ROUTER_USER_NAME, + ) + + def _delete_file(self, path: str) -> None: + """Delete file. + + Args: + path: Full filesystem path (with filename) + """ + if self._container.exists(path): + self._container.remove_path(path) + @staticmethod def _get_mysql_router_layer( host: str, port: str, username: str, password: str From 7f96a60208677b727704e8018f0e61c547984579 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 14 Apr 2023 15:47:05 +0000 Subject: [PATCH 006/159] remove unused imports --- src/relations/tls.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/relations/tls.py b/src/relations/tls.py index 8372a8b1a..1186d6460 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -18,15 +18,13 @@ generate_private_key, ) from ops import Relation -from ops.charm import ActionEvent, CharmBase +from ops.charm import ActionEvent from ops.framework import Object -from ops.pebble import Layer, PathError +from ops.pebble import Layer from charm import MySQLRouterOperatorCharm from constants import ( - MYSQL_ROUTER_CONTAINER_NAME, MYSQL_ROUTER_SERVICE_NAME, - MYSQL_ROUTER_USER_NAME, PEER, ROUTER_CONFIG_DIRECTORY, TLS_RELATION, From 761d5b1dbd79c8bb556ab4bd87b94ec66618ef9c Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 17 Apr 2023 15:45:49 +0000 Subject: [PATCH 007/159] Implement mysql connector methods --- src/relations/database_requires.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index b01ddb778..15f751d98 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -1,6 +1,7 @@ import dataclasses import charms.data_platform_libs.v0.data_interfaces as data_interfaces +import mysql.connector import ops @@ -57,9 +58,21 @@ def is_desired_active(self, event) -> bool: def create_application_database_and_user( self, username: str, password: str, database: str ) -> None: - # TODO: port method from mysql_router_helpers or mysql charm lib - pass + self._execute_sql_statements( + [ + f"CREATE DATABASE IF NOT EXISTS `{database}`", + f"CREATE USER `{username}` IDENTIFIED BY '{password}'", + f"GRANT ALL PRIVILEGES ON `{database}`.* TO `{username}`", + ] + ) def delete_application_user(self, username: str) -> None: - # TODO: port method from mysql_router_helpers or mysql charm lib - pass + self._execute_sql_statements([f"DROP USER IF EXISTS `{username}`"]) + + def _execute_sql_statements(self, statements: list[str]) -> None: + # TODO: catch exceptions? + with mysql.connector.connect( + username=self.username, password=self.password, host=self.host, port=self.port + ) as connection, connection.cursor() as cursor: + for statement in statements: + cursor.execute(statement) From 6bbd0cfffa9637586aedb2a16824604f9a60ef42 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 17 Apr 2023 16:19:27 +0000 Subject: [PATCH 008/159] Use --version instead of -V --- src/workload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/workload.py b/src/workload.py index 390710999..8374fc632 100644 --- a/src/workload.py +++ b/src/workload.py @@ -26,9 +26,9 @@ def active(self) -> bool: @property def version(self) -> str: - process = self._container.exec(["mysqlrouter", "-V"]) + process = self._container.exec(["mysqlrouter", "--version"]) raw_version, _ = process.wait_output() - for version in raw_version.strip().split(): + for version in raw_version.split(): if version.startswith("8"): return version return "" From e6ef4726a756b6efb8a19824260c0f558372e09c Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 17 Apr 2023 16:33:33 +0000 Subject: [PATCH 009/159] temporarily skip unit/lint for integration test --- .github/workflows/ci.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5e7c3624b..2f6a3fd5f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,8 +52,6 @@ jobs: - integration-database name: ${{ matrix.tox-environments }} needs: - - lint - - unit-test - build runs-on: ubuntu-latest timeout-minutes: 120 From 48304464ea184434462401070755f648bc978e09 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 17 Apr 2023 16:40:34 +0000 Subject: [PATCH 010/159] Use BlockedStatus for missing relation(s) --- src/charm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/charm.py b/src/charm.py index 37974b117..9959cb58d 100755 --- a/src/charm.py +++ b/src/charm.py @@ -86,7 +86,7 @@ def _determine_status(self, event) -> ops.model.StatusBase: if not active: inactive_relations.append(relation) if inactive_relations: - return ops.model.WaitingStatus( + return ops.model.BlockedStatus( f"Waiting for relation{'s' if len(inactive_relations) > 1 else ''}: {', '.join(inactive_relations)}" ) if not self.workload.container_ready: @@ -94,7 +94,9 @@ def _determine_status(self, event) -> ops.model.StatusBase: return ops.model.ActiveStatus() def _set_status(self, event=None) -> None: - if isinstance(self.unit.status, ops.model.BlockedStatus): + if isinstance( + self.unit.status, ops.model.BlockedStatus + ) and not self.unit.status.message.startswith("Waiting for relation"): return self.unit.status = self._determine_status(event) From 82d28bdf97323b41326938ba01322db79c6f07d4 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 17 Apr 2023 17:10:53 +0000 Subject: [PATCH 011/159] Add src/relations/__init__.py --- src/relations/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/relations/__init__.py diff --git a/src/relations/__init__.py b/src/relations/__init__.py new file mode 100644 index 000000000..e69de29bb From 279228d2f4ba1f2d586d9327341d81ec8d8bb91b Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 17 Apr 2023 17:19:43 +0000 Subject: [PATCH 012/159] Fix relations imports --- src/charm.py | 4 +++- src/relations/__init__.py | 0 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 src/relations/__init__.py diff --git a/src/charm.py b/src/charm.py index 9959cb58d..baf9a984d 100755 --- a/src/charm.py +++ b/src/charm.py @@ -15,7 +15,9 @@ from lightkube.models.meta_v1 import ObjectMeta from lightkube.resources.core_v1 import Pod, Service -import relations +import relations.database_provides +import relations.database_requires +import relations.tls import workload from constants import ( DATABASE_PROVIDES_RELATION, diff --git a/src/relations/__init__.py b/src/relations/__init__.py deleted file mode 100644 index e69de29bb..000000000 From 81aa07fa825e58cb30f0c3d9cb9027a983380fd4 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 17 Apr 2023 18:00:29 +0000 Subject: [PATCH 013/159] Handle non-existent relations --- src/relations/database_provides.py | 12 +++++++++++- src/relations/database_requires.py | 12 ++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index f4e369bfd..fb6805906 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -15,6 +15,8 @@ class Relation: @property def active(self) -> bool: """Whether relation is currently active""" + if not self._exists: + return False for key in ["database", "username", "password", "endpoints"]: if key not in self._local_databag: return False @@ -28,6 +30,14 @@ def database(self) -> str: def username(self) -> str: return f"relation-{self._id}" + @property + def _exists(self) -> bool: + relations = self.interface.relations + if relations: + assert len(relations) == 1 + return True + return False + @property def _relation(self) -> ops.model.Relation: relations = self.interface.relations @@ -48,7 +58,7 @@ def _local_databag(self) -> ops.model.RelationDataContent: def is_desired_active(self, event) -> bool: """Whether relation should be active once the event is handled""" - if isinstance(event, ops.charm.RelationBrokenEvent) and event.relation.id == self._id: + if isinstance(event, ops.charm.RelationBrokenEvent) and self._exists and event.relation.id == self._id: # Relation is being removed return False if isinstance(event, data_interfaces.DatabaseRequestedEvent): diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 15f751d98..b4deee7bd 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -25,6 +25,14 @@ def username(self) -> str: def password(self) -> str: return self._remote_databag["password"] + @property + def _exists(self) -> bool: + relations = self.interface.relations + if relations: + assert len(relations) == 1 + return True + return False + @property def _id(self) -> int: relations = self.interface.relations @@ -44,13 +52,13 @@ def _endpoint(self) -> str: @property def _active(self) -> bool: """Whether relation is currently active""" - if not self.interface.relations: + if not self._exists: return False return self.interface.is_resource_created() def is_desired_active(self, event) -> bool: """Whether relation should be active once the event is handled""" - if isinstance(event, ops.charm.RelationBrokenEvent) and event.relation.id == self._id: + if isinstance(event, ops.charm.RelationBrokenEvent) and self._exists and event.relation.id == self._id: # Relation is being removed; it is no longer active return False return self._active From bee49af91dce88ba754ba1fa9a0da1def0cd175f Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 17 Apr 2023 18:35:36 +0000 Subject: [PATCH 014/159] wait until mysql router ready --- src/workload.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/workload.py b/src/workload.py index 8374fc632..a863237fc 100644 --- a/src/workload.py +++ b/src/workload.py @@ -1,6 +1,8 @@ import dataclasses +import socket import ops +import tenacity from constants import ( MYSQL_ROUTER_SERVICE_NAME, @@ -46,6 +48,7 @@ def start(self, host, port, username, password) -> None: combine=True, ) self._container.start(MYSQL_ROUTER_SERVICE_NAME) + self._wait_until_mysql_router_ready() # TODO: wait until mysql router ready? https://github.com/canonical/mysql-router-k8s-operator/blob/45cf3be44f27476a0371c67d50d7a0193c0fadc2/src/charm.py#L219 def stop(self) -> None: @@ -124,3 +127,21 @@ def _get_mysql_router_layer( }, } ) + + @staticmethod + @tenacity.retry(reraise=True, stop=tenacity.stop_after_delay(30), wait=tenacity.wait_fixed(5)) + def _wait_until_mysql_router_ready() -> None: + """Wait until a connection to MySQL router is possible. + Retry every 5 seconds for 30 seconds if there is an issue obtaining a connection. + """ + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex(("127.0.0.1", 6446)) + if result != 0: + raise BaseException() + sock.close() + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex(("127.0.0.1", 6447)) + if result != 0: + raise BaseException() + sock.close() From 59bacc8e1f86f7cd55b11a51cfbb0f98213b4fab Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 17 Apr 2023 18:36:06 +0000 Subject: [PATCH 015/159] fixup! Handle non-existent relations --- src/relations/database_provides.py | 6 +++++- src/relations/database_requires.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index fb6805906..ec5243343 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -58,7 +58,11 @@ def _local_databag(self) -> ops.model.RelationDataContent: def is_desired_active(self, event) -> bool: """Whether relation should be active once the event is handled""" - if isinstance(event, ops.charm.RelationBrokenEvent) and self._exists and event.relation.id == self._id: + if ( + isinstance(event, ops.charm.RelationBrokenEvent) + and self._exists + and event.relation.id == self._id + ): # Relation is being removed return False if isinstance(event, data_interfaces.DatabaseRequestedEvent): diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index b4deee7bd..e210fc302 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -58,7 +58,11 @@ def _active(self) -> bool: def is_desired_active(self, event) -> bool: """Whether relation should be active once the event is handled""" - if isinstance(event, ops.charm.RelationBrokenEvent) and self._exists and event.relation.id == self._id: + if ( + isinstance(event, ops.charm.RelationBrokenEvent) + and self._exists + and event.relation.id == self._id + ): # Relation is being removed; it is no longer active return False return self._active From 172e541e9162b23a726ee9ab6b76fe6fa1b6a71d Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 17 Apr 2023 18:50:36 +0000 Subject: [PATCH 016/159] fixup! Use BlockedStatus for missing relation(s) --- tests/integration/test_charm.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index be455d312..f889a164d 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -77,8 +77,7 @@ async def test_database_relation(ops_test: OpsTest): ), ops_test.model.wait_for_idle( apps=[MYSQL_ROUTER_APP_NAME, APPLICATION_APP_NAME], - status="waiting", - raise_on_blocked=True, + status="blocked", timeout=SLOW_TIMEOUT, ), ) From 853877e0a33d251f77edee74d0d19cda9fd75be1 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 17 Apr 2023 19:34:31 +0000 Subject: [PATCH 017/159] Handle non-existent service --- src/workload.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index a863237fc..bd3711b77 100644 --- a/src/workload.py +++ b/src/workload.py @@ -24,7 +24,10 @@ def container_ready(self) -> bool: @property def active(self) -> bool: - return self._container.get_service(MYSQL_ROUTER_SERVICE_NAME).is_running() + service = self._container.get_services(MYSQL_ROUTER_SERVICE_NAME).get(MYSQL_ROUTER_SERVICE_NAME) + if service is None: + return False + return service.is_running() @property def version(self) -> str: @@ -52,6 +55,8 @@ def start(self, host, port, username, password) -> None: # TODO: wait until mysql router ready? https://github.com/canonical/mysql-router-k8s-operator/blob/45cf3be44f27476a0371c67d50d7a0193c0fadc2/src/charm.py#L219 def stop(self) -> None: + if not self.active: + return self._container.stop(MYSQL_ROUTER_SERVICE_NAME) def enable_tls( From 79b2cdfc025d3347038fc0bfff682137ae8a4fff Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 18 Apr 2023 11:52:59 +0000 Subject: [PATCH 018/159] fixup! fixup! Use BlockedStatus for missing relation(s) --- tests/integration/test_charm.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index f889a164d..b7eb491e6 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -76,10 +76,16 @@ async def test_database_relation(ops_test: OpsTest): timeout=SLOW_TIMEOUT, ), ops_test.model.wait_for_idle( - apps=[MYSQL_ROUTER_APP_NAME, APPLICATION_APP_NAME], + apps=[MYSQL_ROUTER_APP_NAME], status="blocked", timeout=SLOW_TIMEOUT, ), + ops_test.model.wait_for_idle( + apps=[APPLICATION_APP_NAME], + status="waiting", + raise_on_blocked=True, + timeout=SLOW_TIMEOUT, + ) ) logger.info("Relating mysql, mysqlrouter and application") From bf506d2ee9ef6d827fdff88fcd4657a01767f6c6 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 18 Apr 2023 13:30:35 +0000 Subject: [PATCH 019/159] add sleep to integration test --- tests/integration/test_charm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index b7eb491e6..7cf177a5d 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -98,6 +98,9 @@ async def test_database_relation(ops_test: OpsTest): f"{APPLICATION_APP_NAME}:database", f"{MYSQL_ROUTER_APP_NAME}:database" ) + # Wait for mysqlrouter to exit blockedstatus + await asyncio.sleep(10) + await ops_test.model.wait_for_idle( apps=[MYSQL_APP_NAME, MYSQL_ROUTER_APP_NAME, APPLICATION_APP_NAME], status="active", From b39b0b0a2e6e5405f1a81c565bc0b68a266a22b3 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 18 Apr 2023 14:51:41 +0000 Subject: [PATCH 020/159] increase sleep --- tests/integration/test_charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 7cf177a5d..154117531 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -99,7 +99,7 @@ async def test_database_relation(ops_test: OpsTest): ) # Wait for mysqlrouter to exit blockedstatus - await asyncio.sleep(10) + await asyncio.sleep(60) await ops_test.model.wait_for_idle( apps=[MYSQL_APP_NAME, MYSQL_ROUTER_APP_NAME, APPLICATION_APP_NAME], From 2d27ed85e131be37b1663bda3b5e9a56b27368e2 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 18 Apr 2023 14:56:38 +0000 Subject: [PATCH 021/159] fix integration test --- tests/integration/test_charm.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 154117531..0cd58214d 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -85,7 +85,7 @@ async def test_database_relation(ops_test: OpsTest): status="waiting", raise_on_blocked=True, timeout=SLOW_TIMEOUT, - ) + ), ) logger.info("Relating mysql, mysqlrouter and application") @@ -98,8 +98,9 @@ async def test_database_relation(ops_test: OpsTest): f"{APPLICATION_APP_NAME}:database", f"{MYSQL_ROUTER_APP_NAME}:database" ) - # Wait for mysqlrouter to exit blockedstatus - await asyncio.sleep(60) + await ops_test.model.wait_for_idle( + apps=[MYSQL_ROUTER_APP_NAME], status="active", timeout=SLOW_TIMEOUT + ) await ops_test.model.wait_for_idle( apps=[MYSQL_APP_NAME, MYSQL_ROUTER_APP_NAME, APPLICATION_APP_NAME], From acb3cbb9717079321f4fbf40b7671846611f50f4 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 18 Apr 2023 14:56:50 +0000 Subject: [PATCH 022/159] format --- src/workload.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index bd3711b77..e3da253c8 100644 --- a/src/workload.py +++ b/src/workload.py @@ -24,7 +24,9 @@ def container_ready(self) -> bool: @property def active(self) -> bool: - service = self._container.get_services(MYSQL_ROUTER_SERVICE_NAME).get(MYSQL_ROUTER_SERVICE_NAME) + service = self._container.get_services(MYSQL_ROUTER_SERVICE_NAME).get( + MYSQL_ROUTER_SERVICE_NAME + ) if service is None: return False return service.is_running() From 098a6d0306512c687e999b3d09562e68fda866e0 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 18 Apr 2023 18:07:14 +0000 Subject: [PATCH 023/159] Add debug logging --- src/charm.py | 10 +++++----- src/relations/database_provides.py | 17 +++++++++++++++-- src/relations/database_requires.py | 7 +++++++ src/workload.py | 12 ++++++++++++ 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/src/charm.py b/src/charm.py index baf9a984d..d78446fb3 100755 --- a/src/charm.py +++ b/src/charm.py @@ -89,7 +89,7 @@ def _determine_status(self, event) -> ops.model.StatusBase: inactive_relations.append(relation) if inactive_relations: return ops.model.BlockedStatus( - f"Waiting for relation{'s' if len(inactive_relations) > 1 else ''}: {', '.join(inactive_relations)}" + f"Missing relation{'s' if len(inactive_relations) > 1 else ''}: {', '.join(inactive_relations)}" ) if not self.workload.container_ready: return ops.model.MaintenanceStatus("Waiting for container") # TODO @@ -98,13 +98,13 @@ def _determine_status(self, event) -> ops.model.StatusBase: def _set_status(self, event=None) -> None: if isinstance( self.unit.status, ops.model.BlockedStatus - ) and not self.unit.status.message.startswith("Waiting for relation"): + ) and not self.unit.status.message.startswith("Missing relation"): return self.unit.status = self._determine_status(event) - def _create_user(self) -> None: + def _create_database_and_user(self) -> None: if self.database_provides.active: - # User already created + # Database and user already created return password = self.database_provides.generate_password() self.database_requires.create_application_database_and_user( @@ -189,7 +189,7 @@ def _reconcile_database_relations(self, event=None) -> None: if self.database_requires.is_desired_active( event ) and self.database_provides.is_desired_active(event): - self._create_user() + self._create_database_and_user() else: self._delete_user() self._set_status(event) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index ec5243343..be1947561 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -1,4 +1,5 @@ import dataclasses +import logging import secrets import string @@ -7,6 +8,8 @@ from constants import PASSWORD_LENGTH +logger = logging.getLogger(__name__) + @dataclasses.dataclass class Relation: @@ -70,13 +73,23 @@ def is_desired_active(self, event) -> bool: return self.active def set_databag(self, password: str, endpoint: str) -> None: + endpoint = f"{endpoint}:6446" + read_only_endpoint = f"{endpoint}:6447" + logger.debug( + f"Setting databag {self.database=}, {self.username=}, {endpoint=}, {read_only_endpoint=}" + ) self.interface.set_database(self._id, self.database) self.interface.set_credentials(self._id, self.username, password) - self.interface.set_endpoints(self._id, f"{endpoint}:6446") - self.interface.set_read_only_endpoints(self._id, f"{endpoint}:6447") + self.interface.set_endpoints(self._id, endpoint) + self.interface.set_read_only_endpoints(self._id, read_only_endpoint) + logger.debug( + f"Set databag {self.database=}, {self.username=}, {endpoint=}, {read_only_endpoint=}" + ) def delete_databag(self) -> None: + logger.debug("Deleting databag") self._local_databag.clear() + logger.debug("Deleted databag") @staticmethod def generate_password() -> str: diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index e210fc302..6fc36de30 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -1,9 +1,12 @@ import dataclasses +import logging import charms.data_platform_libs.v0.data_interfaces as data_interfaces import mysql.connector import ops +logger = logging.getLogger(__name__) + @dataclasses.dataclass class Relation: @@ -70,6 +73,7 @@ def is_desired_active(self, event) -> bool: def create_application_database_and_user( self, username: str, password: str, database: str ) -> None: + logger.debug(f"Creating {database=} and {username=}") self._execute_sql_statements( [ f"CREATE DATABASE IF NOT EXISTS `{database}`", @@ -77,9 +81,12 @@ def create_application_database_and_user( f"GRANT ALL PRIVILEGES ON `{database}`.* TO `{username}`", ] ) + logger.debug(f"Created {database=} and {username=}") def delete_application_user(self, username: str) -> None: + logger.debug(f"Deleting {username=}") self._execute_sql_statements([f"DROP USER IF EXISTS `{username}`"]) + logger.debug(f"Deleted {username=}") def _execute_sql_statements(self, statements: list[str]) -> None: # TODO: catch exceptions? diff --git a/src/workload.py b/src/workload.py index e3da253c8..be4324712 100644 --- a/src/workload.py +++ b/src/workload.py @@ -1,4 +1,5 @@ import dataclasses +import logging import socket import ops @@ -13,6 +14,8 @@ TLS_SSL_KEY_FILE, ) +logger = logging.getLogger(__name__) + @dataclasses.dataclass class Workload: @@ -47,30 +50,37 @@ def start(self, host, port, username, password) -> None: # Assumption: username or password will not change while database requires relation is active # Therefore, MySQL Router does not need to be restarted if it is already running return + logger.debug(f"Starting MySQL Router service {host=}, {port=}, {username=}") self._container.add_layer( MYSQL_ROUTER_SERVICE_NAME, self._get_mysql_router_layer(host, port, username, password), combine=True, ) self._container.start(MYSQL_ROUTER_SERVICE_NAME) + logger.debug(f"Started MySQL Router service {host=}, {port=}, {username=}") self._wait_until_mysql_router_ready() # TODO: wait until mysql router ready? https://github.com/canonical/mysql-router-k8s-operator/blob/45cf3be44f27476a0371c67d50d7a0193c0fadc2/src/charm.py#L219 def stop(self) -> None: if not self.active: return + logger.debug("Stopping MySQL Router service") self._container.stop(MYSQL_ROUTER_SERVICE_NAME) + logger.debug("Stopped MySQL Router service") def enable_tls( self, layer: ops.pebble.Layer, config_file_content: str, key: str, certificate: str ): + logger.debug("Enabling TLS") self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CONFIG_FILE}", config_file_content) self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_KEY_FILE}", key) self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CERT_FILE}", certificate) self._container.add_layer(MYSQL_ROUTER_SERVICE_NAME, layer, combine=True) self._container.replan() + logger.debug("Enabled TLS") def disable_tls(self) -> None: + logger.debug("Disabling TLS") for file in [TLS_SSL_CONFIG_FILE, TLS_SSL_KEY_FILE, TLS_SSL_CERT_FILE]: self._delete_file(f"{ROUTER_CONFIG_DIRECTORY}/{file}") layer = ops.pebble.Layer( @@ -85,6 +95,7 @@ def disable_tls(self) -> None: ) self._container.add_layer(MYSQL_ROUTER_SERVICE_NAME, layer, combine=True) self._container.replan() + logger.debug("Disabled TLS") def _write_file(self, path: str, content: str) -> None: """Write content to file. @@ -138,6 +149,7 @@ def _get_mysql_router_layer( @staticmethod @tenacity.retry(reraise=True, stop=tenacity.stop_after_delay(30), wait=tenacity.wait_fixed(5)) def _wait_until_mysql_router_ready() -> None: + # TODO: add debug logging """Wait until a connection to MySQL router is possible. Retry every 5 seconds for 30 seconds if there is an issue obtaining a connection. """ From 74726a29ba09b8e19ebea0618bdce6e952dcf9c7 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 18 Apr 2023 18:09:05 +0000 Subject: [PATCH 024/159] temp collect logs in CI --- .github/workflows/ci.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2f6a3fd5f..4c1bfdf7b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -81,9 +81,15 @@ jobs: fi - name: Run integration tests # set a predictable model name so it can be consumed by charm-logdump-action - run: sg microk8s -c "tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --model testing" + run: sg microk8s -c "tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --model testing --keep-models" env: CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} + - name: Juju status + if: always() + run: juju status --model testing + - name: Juju debug log + if: always() + run: juju debug-log --model testing --replay --no-tail - name: Dump logs uses: canonical/charm-logdump-action@main if: failure() From 9f6437ca5d0cdbcb80711d1ad9e1da4165f126f0 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 18 Apr 2023 19:09:29 +0000 Subject: [PATCH 025/159] deploy mysql-k8s from mysqlrouter-user branch --- tests/integration/test_charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 0cd58214d..1234b81e5 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -45,7 +45,7 @@ async def test_database_relation(ops_test: OpsTest): applications = await asyncio.gather( ops_test.model.deploy( MYSQL_APP_NAME, - channel="8.0/edge", + channel="8.0/edge/mysqlrouter-user", application_name=MYSQL_APP_NAME, num_units=3, trust=True, # Necessary after a6f1f01: Fix/endpoints as k8s services (#142) From 288a2084fc32b19438a3a828b3447e0cc8f54bae Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 18 Apr 2023 20:00:08 +0000 Subject: [PATCH 026/159] add series to test --- tests/integration/test_charm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 1234b81e5..4495a3114 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -47,12 +47,14 @@ async def test_database_relation(ops_test: OpsTest): MYSQL_APP_NAME, channel="8.0/edge/mysqlrouter-user", application_name=MYSQL_APP_NAME, + series="jammy", num_units=3, trust=True, # Necessary after a6f1f01: Fix/endpoints as k8s services (#142) ), ops_test.model.deploy( mysqlrouter_charm, application_name=MYSQL_ROUTER_APP_NAME, + series="jammy", resources=mysqlrouter_resources, num_units=1, ), @@ -60,6 +62,7 @@ async def test_database_relation(ops_test: OpsTest): APPLICATION_APP_NAME, channel="latest/edge", application_name=APPLICATION_APP_NAME, + series="jammy", num_units=1, ), ) From 10be44149f6f7bf3745ad7aa76b6d1823d6568b7 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 18 Apr 2023 20:00:26 +0000 Subject: [PATCH 027/159] add docstring --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index d78446fb3..15d2b01c3 100755 --- a/src/charm.py +++ b/src/charm.py @@ -175,7 +175,7 @@ def _patch_service(self, name: str, ro_port: int, rw_port: int) -> None: # ======================= def _reconcile_database_relations(self, event=None) -> None: - # TODO: rename method? + """Handle database requires/provides events.""" if self.database_requires.is_desired_active(event) and self.workload.container_ready: self.workload.start( self.database_requires.host, From 2cf0d1e4aae8716b73e4df9a6c0f5799840ebc01 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 20 Apr 2023 12:15:32 +0000 Subject: [PATCH 028/159] Remove old unit tests --- tests/unit/test_charm.py | 75 ---------------------------------------- 1 file changed, 75 deletions(-) delete mode 100644 tests/unit/test_charm.py diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py deleted file mode 100644 index 8cdf5a667..000000000 --- a/tests/unit/test_charm.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. - -import unittest -from unittest.mock import MagicMock, patch - -import lightkube -from ops.model import BlockedStatus, MaintenanceStatus -from ops.testing import Harness - -from charm import MySQLRouterOperatorCharm - - -class TestCharm(unittest.TestCase): - def setUp(self): - self.harness = Harness(MySQLRouterOperatorCharm) - self.addCleanup(self.harness.cleanup) - self.harness.begin() - self.peer_relation_id = self.harness.add_relation( - "mysql-router-peers", "mysql-router-peers" - ) - self.harness.add_relation_unit(self.peer_relation_id, "mysql-router-k8s/1") - self.charm = self.harness.charm - - @patch("charm.Client", return_value=MagicMock()) - def test_on_peer_relation_created(self, _lightkube_client): - self.charm.on.leader_elected.emit() - - self.assertEqual(_lightkube_client.return_value.patch.call_count, 1) - - self.assertTrue(isinstance(self.harness.model.unit.status, MaintenanceStatus)) - - @patch("charm.Client", return_value=MagicMock()) - def test_on_peer_relation_created_delete_exception(self, _lightkube_client): - response = MagicMock() - response.json.return_value = {"status": "Bad Request", "code": 400} - api_error = lightkube.ApiError(request=MagicMock(), response=response) - _lightkube_client.return_value.patch.side_effect = api_error - - self.charm.on.leader_elected.emit() - - self.assertTrue(isinstance(self.harness.model.unit.status, BlockedStatus)) - - @patch("charm.Client", return_value=MagicMock()) - def test_on_peer_relation_created_delete_nothing(self, _lightkube_client): - response = MagicMock() - response.json.return_value = {"status": "Not Found", "code": 404} - api_error = lightkube.ApiError(request=MagicMock(), response=response) - _lightkube_client.return_value.delete.side_effect = api_error - - self.charm.on.leader_elected.emit() - - self.assertTrue(isinstance(self.harness.model.unit.status, MaintenanceStatus)) - - @patch("charm.Client", return_value=MagicMock()) - def test_on_leader_elected_create_exception(self, _lightkube_client): - response = MagicMock() - response.json.return_value = {"status": "Bad Request", "code": 400} - api_error = lightkube.ApiError(request=MagicMock(), response=response) - _lightkube_client.return_value.patch.side_effect = api_error - - self.charm.on.leader_elected.emit() - - self.assertTrue(isinstance(self.harness.model.unit.status, BlockedStatus)) - - @patch("charm.Client", return_value=MagicMock()) - def test_on_leader_elected_create_existing_service(self, _lightkube_client): - response = MagicMock() - response.json.return_value = {"status": "Conflict", "code": 409} - api_error = lightkube.ApiError(request=MagicMock(), response=response) - _lightkube_client.return_value.create.side_effect = api_error - - self.charm.on.leader_elected.emit() - - self.assertTrue(isinstance(self.harness.model.unit.status, MaintenanceStatus)) From 6c59feb4a7831953e4cbddf195e6a138e60197ef Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 20 Apr 2023 14:10:54 +0000 Subject: [PATCH 029/159] Fix database requested check --- src/relations/database_provides.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index be1947561..70a83f190 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -41,6 +41,10 @@ def _exists(self) -> bool: return True return False + @property + def _database_requested(self) -> bool: + return self._remote_databag.get("database") is not None + @property def _relation(self) -> ops.model.Relation: relations = self.interface.relations @@ -68,7 +72,7 @@ def is_desired_active(self, event) -> bool: ): # Relation is being removed return False - if isinstance(event, data_interfaces.DatabaseRequestedEvent): + if self._exists and self._database_requested: return True return self.active From 56724d49e759d2ed46da20321f9e031ad42935a6 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 20 Apr 2023 20:18:01 +0000 Subject: [PATCH 030/159] Support multiple provides relations --- src/charm.py | 79 +++++++-------- src/relations/database_provides.py | 158 +++++++++++++++++------------ src/relations/database_requires.py | 96 +++++++++--------- src/relations/tls.py | 2 +- src/workload.py | 7 +- 5 files changed, 178 insertions(+), 164 deletions(-) diff --git a/src/charm.py b/src/charm.py index 15d2b01c3..53dfbf161 100755 --- a/src/charm.py +++ b/src/charm.py @@ -34,7 +34,7 @@ class MySQLRouterOperatorCharm(ops.charm.CharmBase): def __init__(self, *args) -> None: super().__init__(*args) - self.database_requires = relations.database_requires.Relation( + self.database_requires = relations.database_requires.RelationEndpoint( data_interfaces.DatabaseRequires( self, relation_name=DATABASE_REQUIRES_RELATION, @@ -53,7 +53,7 @@ def __init__(self, *args) -> None: self._reconcile_database_relations, ) - self.database_provides = relations.database_provides.Relation( + self.database_provides = relations.database_provides.RelationEndpoint( data_interfaces.DatabaseProvides(self, relation_name=DATABASE_PROVIDES_RELATION) ) self.framework.observe( @@ -79,47 +79,28 @@ def _endpoint(self) -> str: """The k8s endpoint for the charm.""" return f"{self.model.app.name}.{self.model.name}.svc.cluster.local" - def _determine_status(self, event) -> ops.model.StatusBase: - inactive_relations = [] - for relation, active in [ - (DATABASE_REQUIRES_RELATION, self.database_requires.is_desired_active(event)), - (DATABASE_PROVIDES_RELATION, self.database_provides.is_desired_active(event)), + def _determine_status(self) -> ops.model.StatusBase: + missing_relations = [] + for relation, missing in [ + (DATABASE_REQUIRES_RELATION, self.database_requires.relation is None), + (DATABASE_PROVIDES_RELATION, self.database_provides.missing_relation), ]: - if not active: - inactive_relations.append(relation) - if inactive_relations: + if missing: + missing_relations.append(relation) + if missing_relations: return ops.model.BlockedStatus( - f"Missing relation{'s' if len(inactive_relations) > 1 else ''}: {', '.join(inactive_relations)}" + f"Missing relation{'s' if len(missing_relations) > 1 else ''}: {', '.join(missing_relations)}" ) if not self.workload.container_ready: return ops.model.MaintenanceStatus("Waiting for container") # TODO return ops.model.ActiveStatus() - def _set_status(self, event=None) -> None: + def _set_status(self) -> None: if isinstance( self.unit.status, ops.model.BlockedStatus ) and not self.unit.status.message.startswith("Missing relation"): return - self.unit.status = self._determine_status(event) - - def _create_database_and_user(self) -> None: - if self.database_provides.active: - # Database and user already created - return - password = self.database_provides.generate_password() - self.database_requires.create_application_database_and_user( - self.database_provides.username, - password, - self.database_provides.database, - ) - self.database_provides.set_databag(password, self._endpoint) - - def _delete_user(self) -> None: - if not self.database_provides.active: - # No user to delete - return - self.database_requires.delete_application_user(self.database_provides.username) - self.database_provides.delete_databag() + self.unit.status = self._determine_status() def _patch_service(self, name: str, ro_port: int, rw_port: int) -> None: """Patch juju created k8s service. @@ -176,23 +157,31 @@ def _patch_service(self, name: str, ro_port: int, rw_port: int) -> None: def _reconcile_database_relations(self, event=None) -> None: """Handle database requires/provides events.""" - if self.database_requires.is_desired_active(event) and self.workload.container_ready: + if ( + self.unit.is_leader() + and self.database_requires.relation + and self.workload.container_ready + ): + self.database_provides.reconcile_users( + event, + self.database_requires.relation.is_breaking(event), + self._endpoint, + self.database_requires.relation, + ) + if ( + self.database_requires.relation + and not self.database_requires.relation.is_breaking(event) + and self.workload.container_ready + ): self.workload.start( - self.database_requires.host, - self.database_requires.port, - self.database_requires.username, - self.database_requires.password, + self.database_requires.relation.host, + self.database_requires.relation.port, + self.database_requires.relation.username, + self.database_requires.relation.password, ) else: self.workload.stop() - if self.unit.is_leader(): - if self.database_requires.is_desired_active( - event - ) and self.database_provides.is_desired_active(event): - self._create_database_and_user() - else: - self._delete_user() - self._set_status(event) + self._set_status() def _on_mysql_router_pebble_ready(self, _) -> None: self.unit.set_workload_version(self.workload.version) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 70a83f190..0339ec685 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -1,25 +1,32 @@ import dataclasses import logging -import secrets -import string import charms.data_platform_libs.v0.data_interfaces as data_interfaces +import database_requires import ops -from constants import PASSWORD_LENGTH - logger = logging.getLogger(__name__) @dataclasses.dataclass -class Relation: - interface: data_interfaces.DatabaseProvides +class _Relation: + _relation: ops.model.Relation + _interface: data_interfaces.DatabaseProvides + + @property + def _local_databag(self) -> ops.model.RelationDataContent: + return self._relation.data[self._interface.local_app] @property - def active(self) -> bool: - """Whether relation is currently active""" - if not self._exists: - return False + def _remote_databag(self) -> dict: + return self._interface.fetch_relation_data()[self.id] + + @property + def id(self) -> int: + return self._relation.id + + @property + def user_created(self) -> bool: for key in ["database", "username", "password", "endpoints"]: if key not in self._local_databag: return False @@ -31,71 +38,88 @@ def database(self) -> str: @property def username(self) -> str: - return f"relation-{self._id}" - - @property - def _exists(self) -> bool: - relations = self.interface.relations - if relations: - assert len(relations) == 1 - return True - return False - - @property - def _database_requested(self) -> bool: - return self._remote_databag.get("database") is not None + return f"relation-{self.id}" - @property - def _relation(self) -> ops.model.Relation: - relations = self.interface.relations - assert len(relations) == 1 - return relations[0] - - @property - def _id(self) -> int: - return self._relation.id - - @property - def _remote_databag(self) -> dict: - return self.interface.fetch_relation_data()[self._id] - - @property - def _local_databag(self) -> ops.model.RelationDataContent: - return self._relation.data[self.interface.local_app] - - def is_desired_active(self, event) -> bool: - """Whether relation should be active once the event is handled""" - if ( - isinstance(event, ops.charm.RelationBrokenEvent) - and self._exists - and event.relation.id == self._id - ): - # Relation is being removed - return False - if self._exists and self._database_requested: - return True - return self.active - - def set_databag(self, password: str, endpoint: str) -> None: - endpoint = f"{endpoint}:6446" + def _set_databag(self, password: str, endpoint: str) -> None: + read_write_endpoint = f"{endpoint}:6446" read_only_endpoint = f"{endpoint}:6447" logger.debug( - f"Setting databag {self.database=}, {self.username=}, {endpoint=}, {read_only_endpoint=}" + f"Setting databag {self.database=}, {self.username=}, {read_write_endpoint=}, {read_only_endpoint=}" ) - self.interface.set_database(self._id, self.database) - self.interface.set_credentials(self._id, self.username, password) - self.interface.set_endpoints(self._id, endpoint) - self.interface.set_read_only_endpoints(self._id, read_only_endpoint) + self._interface.set_database(self.id, self.database) + self._interface.set_credentials(self.id, self.username, password) + self._interface.set_endpoints(self.id, read_write_endpoint) + self._interface.set_read_only_endpoints(self.id, read_only_endpoint) logger.debug( - f"Set databag {self.database=}, {self.username=}, {endpoint=}, {read_only_endpoint=}" + f"Set databag {self.database=}, {self.username=}, {read_write_endpoint=}, {read_only_endpoint=}" ) - def delete_databag(self) -> None: + def _delete_databag(self) -> None: logger.debug("Deleting databag") self._local_databag.clear() logger.debug("Deleted databag") - @staticmethod - def generate_password() -> str: - choices = string.ascii_letters + string.digits - return "".join([secrets.choice(choices) for _ in range(PASSWORD_LENGTH)]) + def create_database_and_user( + self, + endpoint: str, + database_requires_relation: database_requires._Relation, # TODO: replace with mysqlsh Python module + ) -> None: + password = database_requires_relation.create_application_database_and_user( + self.username, endpoint + ) + self._set_databag(password, endpoint) + + def delete_user( + self, + database_requires_relation: database_requires._Relation, # TODO: replace with mysqlsh Python module + ) -> None: + self._delete_databag() + database_requires_relation.delete_application_user(self.username) + + +@dataclasses.dataclass +class RelationEndpoint: + interface: data_interfaces.DatabaseProvides + + @property + def _relations(self) -> list[_Relation]: + return [_Relation(relation, self.interface) for relation in self.interface.relations] + + def _requested_users(self, event, event_is_database_requires_broken: bool) -> list[_Relation]: + if event_is_database_requires_broken: + # Cluster connection is being removed; delete all users + return [] + requested_users = [] + for relation in self._relations: + if ( + isinstance(event, ops.charm.RelationBrokenEvent) + and event.relation.id == relation.id + ): + # Relation is being removed; delete user + continue + requested_users.append(relation) + return requested_users + + @property + def _created_users(self) -> list[_Relation]: + return [relation for relation in self._relations if relation.user_created] + + @property + def missing_relation(self) -> bool: + return len(self._relations) == 0 + + def reconcile_users( + self, + event, + event_is_database_requires_broken: bool, + endpoint: str, + database_requires_relation: database_requires._Relation, # TODO: replace with mysqlsh Python module + ) -> None: + requested_users = self._requested_users(event, event_is_database_requires_broken) + created_users = self._created_users + for relation in requested_users: + if relation not in created_users: + relation.create_database_and_user(endpoint, database_requires_relation) + for relation in created_users: + if relation not in requested_users: + relation.delete_user(database_requires_relation) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 6fc36de30..86a53e0a5 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -1,16 +1,37 @@ import dataclasses import logging +import secrets +import string +import typing import charms.data_platform_libs.v0.data_interfaces as data_interfaces import mysql.connector import ops +from constants import PASSWORD_LENGTH + logger = logging.getLogger(__name__) -@dataclasses.dataclass -class Relation: - interface: data_interfaces.DatabaseRequires +class _Relation: + def __init__(self, interface: data_interfaces.DatabaseRequires) -> None: + self._interface = interface + + @property + def _id(self) -> int: + relations = self._interface.relations + assert len(relations) == 1 + return relations[0].id + + @property + def _remote_databag(self) -> dict: + return self._interface.fetch_relation_data()[self._id] + + @property + def _endpoint(self) -> str: + endpoints = self._remote_databag["endpoints"].split(",") + assert len(endpoints) == 1 + return endpoints[0] @property def host(self) -> str: @@ -28,52 +49,13 @@ def username(self) -> str: def password(self) -> str: return self._remote_databag["password"] - @property - def _exists(self) -> bool: - relations = self.interface.relations - if relations: - assert len(relations) == 1 - return True - return False - - @property - def _id(self) -> int: - relations = self.interface.relations - assert len(relations) == 1 - return relations[0].id - - @property - def _remote_databag(self) -> dict: - return self.interface.fetch_relation_data()[self._id] + def is_breaking(self, event): + return isinstance(event, ops.charm.RelationBrokenEvent) and event.relation.id == self._id - @property - def _endpoint(self) -> str: - endpoints = self._remote_databag["endpoints"].split(",") - assert len(endpoints) == 1 - return endpoints[0] - - @property - def _active(self) -> bool: - """Whether relation is currently active""" - if not self._exists: - return False - return self.interface.is_resource_created() - - def is_desired_active(self, event) -> bool: - """Whether relation should be active once the event is handled""" - if ( - isinstance(event, ops.charm.RelationBrokenEvent) - and self._exists - and event.relation.id == self._id - ): - # Relation is being removed; it is no longer active - return False - return self._active - - def create_application_database_and_user( - self, username: str, password: str, database: str - ) -> None: + # TODO: move methods below to mysqlsh Python file + def create_application_database_and_user(self, username: str, database: str) -> str: logger.debug(f"Creating {database=} and {username=}") + password = self._generate_password() self._execute_sql_statements( [ f"CREATE DATABASE IF NOT EXISTS `{database}`", @@ -82,10 +64,11 @@ def create_application_database_and_user( ] ) logger.debug(f"Created {database=} and {username=}") + return password def delete_application_user(self, username: str) -> None: logger.debug(f"Deleting {username=}") - self._execute_sql_statements([f"DROP USER IF EXISTS `{username}`"]) + self._execute_sql_statements([f"DROP USER `{username}`"]) logger.debug(f"Deleted {username=}") def _execute_sql_statements(self, statements: list[str]) -> None: @@ -95,3 +78,20 @@ def _execute_sql_statements(self, statements: list[str]) -> None: ) as connection, connection.cursor() as cursor: for statement in statements: cursor.execute(statement) + + @staticmethod + def _generate_password() -> str: + choices = string.ascii_letters + string.digits + return "".join([secrets.choice(choices) for _ in range(PASSWORD_LENGTH)]) + + +@dataclasses.dataclass +class RelationEndpoint: + def __init__(self, interface: data_interfaces.DatabaseRequires) -> None: + self.interface = interface + + @property + def relation(self) -> typing.Optional[_Relation]: + if not self.interface.relations: + return + return _Relation(self.interface) diff --git a/src/relations/tls.py b/src/relations/tls.py index 1186d6460..996021324 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -150,7 +150,7 @@ def _on_tls_relation_broken(self, _) -> None: def _on_certificate_available(self, event: CertificateAvailableEvent) -> None: """Enable TLS when TLS certificate available.""" - if not self.charm.workload.active: + if not self.charm.workload.running: logger.debug("Unit not bootstrapped, defer TLS setup") event.defer() return diff --git a/src/workload.py b/src/workload.py index be4324712..62930f192 100644 --- a/src/workload.py +++ b/src/workload.py @@ -26,7 +26,7 @@ def container_ready(self) -> bool: return self._container.can_connect() @property - def active(self) -> bool: + def running(self) -> bool: service = self._container.get_services(MYSQL_ROUTER_SERVICE_NAME).get( MYSQL_ROUTER_SERVICE_NAME ) @@ -44,9 +44,10 @@ def version(self) -> str: return "" def start(self, host, port, username, password) -> None: - if self.active: + if self.running: # If the host or port changes, MySQL Router will receive topology change notifications from MySQL # Therefore, if the host or port changes, we do not need to restart MySQL Router + # TODO: update comment # Assumption: username or password will not change while database requires relation is active # Therefore, MySQL Router does not need to be restarted if it is already running return @@ -62,7 +63,7 @@ def start(self, host, port, username, password) -> None: # TODO: wait until mysql router ready? https://github.com/canonical/mysql-router-k8s-operator/blob/45cf3be44f27476a0371c67d50d7a0193c0fadc2/src/charm.py#L219 def stop(self) -> None: - if not self.active: + if not self.running: return logger.debug("Stopping MySQL Router service") self._container.stop(MYSQL_ROUTER_SERVICE_NAME) From 5f60dfc7b2ca601bbea6f947fcd88f993ccf1f2b Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 20 Apr 2023 20:19:37 +0000 Subject: [PATCH 031/159] add TODO comment --- src/workload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index 62930f192..034c3c0ee 100644 --- a/src/workload.py +++ b/src/workload.py @@ -139,7 +139,7 @@ def _get_mysql_router_layer( "environment": { "MYSQL_HOST": host, "MYSQL_PORT": port, - "MYSQL_USER": username, + "MYSQL_USER": username, # TODO switch to limited permissions user "MYSQL_PASSWORD": password, }, }, From d4fa0251dae2950b11004ac1d74fe065beba0a4b Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 20 Apr 2023 20:36:53 +0000 Subject: [PATCH 032/159] fix --- src/relations/database_provides.py | 2 +- src/relations/database_requires.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 0339ec685..e15e26fcb 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -2,7 +2,7 @@ import logging import charms.data_platform_libs.v0.data_interfaces as data_interfaces -import database_requires +import relations.database_requires as database_requires import ops logger = logging.getLogger(__name__) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 86a53e0a5..dacbb63ac 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -92,6 +92,7 @@ def __init__(self, interface: data_interfaces.DatabaseRequires) -> None: @property def relation(self) -> typing.Optional[_Relation]: - if not self.interface.relations: + # TODO: rename property? + if not self.interface.is_resource_created(): return return _Relation(self.interface) From 65da976bb69ff78f005df5df6bbe3001bb36b4c7 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 20 Apr 2023 20:53:55 +0000 Subject: [PATCH 033/159] fix --- src/relations/database_provides.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index e15e26fcb..0e28dd9fc 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -65,7 +65,7 @@ def create_database_and_user( database_requires_relation: database_requires._Relation, # TODO: replace with mysqlsh Python module ) -> None: password = database_requires_relation.create_application_database_and_user( - self.username, endpoint + self.username, self.database ) self._set_databag(password, endpoint) From e56423af9fc8ab5db15fa708b6eec73c4c7a9c88 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 21 Apr 2023 11:24:42 +0000 Subject: [PATCH 034/159] Fix integration test --- src/charm.py | 2 +- tests/integration/test_charm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/charm.py b/src/charm.py index 53dfbf161..840ef4985 100755 --- a/src/charm.py +++ b/src/charm.py @@ -40,7 +40,7 @@ def __init__(self, *args) -> None: relation_name=DATABASE_REQUIRES_RELATION, # HACK: mysqlrouter needs a user, but not a database # Use the DatabaseRequires interface to get a user; disregard the database - database_name="_unused_mysqlrouter_database", + database_name="continuous_writes_database", extra_user_roles="mysqlrouter", ) ) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 4495a3114..eaf6ac33c 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -45,7 +45,7 @@ async def test_database_relation(ops_test: OpsTest): applications = await asyncio.gather( ops_test.model.deploy( MYSQL_APP_NAME, - channel="8.0/edge/mysqlrouter-user", + channel="8.0/edge", application_name=MYSQL_APP_NAME, series="jammy", num_units=3, From 24d4fb2a08b7af4c8c001933fabf28d81a19a84b Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 21 Apr 2023 13:49:53 +0000 Subject: [PATCH 035/159] increase timeout --- src/workload.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index 034c3c0ee..973280cea 100644 --- a/src/workload.py +++ b/src/workload.py @@ -148,7 +148,9 @@ def _get_mysql_router_layer( ) @staticmethod - @tenacity.retry(reraise=True, stop=tenacity.stop_after_delay(30), wait=tenacity.wait_fixed(5)) + @tenacity.retry( + reraise=True, stop=tenacity.stop_after_delay(360), wait=tenacity.wait_fixed(5) + ) def _wait_until_mysql_router_ready() -> None: # TODO: add debug logging """Wait until a connection to MySQL router is possible. From 57f42f3ac977f43e9a2648c6327248f09b7d4f99 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 21 Apr 2023 14:04:31 +0000 Subject: [PATCH 036/159] Use shorter ops imports --- src/charm.py | 14 +++++++------- src/relations/database_provides.py | 6 +++--- src/relations/database_requires.py | 2 +- src/workload.py | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/charm.py b/src/charm.py index 840ef4985..7372c5516 100755 --- a/src/charm.py +++ b/src/charm.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) -class MySQLRouterOperatorCharm(ops.charm.CharmBase): +class MySQLRouterOperatorCharm(ops.CharmBase): """Operator charm for MySQL Router.""" def __init__(self, *args) -> None: @@ -79,7 +79,7 @@ def _endpoint(self) -> str: """The k8s endpoint for the charm.""" return f"{self.model.app.name}.{self.model.name}.svc.cluster.local" - def _determine_status(self) -> ops.model.StatusBase: + def _determine_status(self) -> ops.StatusBase: missing_relations = [] for relation, missing in [ (DATABASE_REQUIRES_RELATION, self.database_requires.relation is None), @@ -88,16 +88,16 @@ def _determine_status(self) -> ops.model.StatusBase: if missing: missing_relations.append(relation) if missing_relations: - return ops.model.BlockedStatus( + return ops.BlockedStatus( f"Missing relation{'s' if len(missing_relations) > 1 else ''}: {', '.join(missing_relations)}" ) if not self.workload.container_ready: - return ops.model.MaintenanceStatus("Waiting for container") # TODO - return ops.model.ActiveStatus() + return ops.MaintenanceStatus("Waiting for container") # TODO + return ops.ActiveStatus() def _set_status(self) -> None: if isinstance( - self.unit.status, ops.model.BlockedStatus + self.unit.status, ops.BlockedStatus ) and not self.unit.status.message.startswith("Missing relation"): return self.unit.status = self._determine_status() @@ -197,7 +197,7 @@ def _on_leader_elected(self, _) -> None: self._patch_service(self.app.name, ro_port=6447, rw_port=6446) except ApiError: logger.exception("Failed to patch k8s service") - self.unit.status = ops.model.BlockedStatus("Failed to patch k8s service") + self.unit.status = ops.BlockedStatus("Failed to patch k8s service") if __name__ == "__main__": diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 0e28dd9fc..132291006 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -10,11 +10,11 @@ @dataclasses.dataclass class _Relation: - _relation: ops.model.Relation + _relation: ops.Relation _interface: data_interfaces.DatabaseProvides @property - def _local_databag(self) -> ops.model.RelationDataContent: + def _local_databag(self) -> ops.RelationDataContent: return self._relation.data[self._interface.local_app] @property @@ -92,7 +92,7 @@ def _requested_users(self, event, event_is_database_requires_broken: bool) -> li requested_users = [] for relation in self._relations: if ( - isinstance(event, ops.charm.RelationBrokenEvent) + isinstance(event, ops.RelationBrokenEvent) and event.relation.id == relation.id ): # Relation is being removed; delete user diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index dacbb63ac..6ee4f456c 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -50,7 +50,7 @@ def password(self) -> str: return self._remote_databag["password"] def is_breaking(self, event): - return isinstance(event, ops.charm.RelationBrokenEvent) and event.relation.id == self._id + return isinstance(event, ops.RelationBrokenEvent) and event.relation.id == self._id # TODO: move methods below to mysqlsh Python file def create_application_database_and_user(self, username: str, database: str) -> str: diff --git a/src/workload.py b/src/workload.py index 973280cea..61e9c52f3 100644 --- a/src/workload.py +++ b/src/workload.py @@ -19,7 +19,7 @@ @dataclasses.dataclass class Workload: - _container: ops.model.Container + _container: ops.Container @property def container_ready(self) -> bool: From 5ebf1fe86436aecc79804c28282c7fb5656102cc Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 21 Apr 2023 14:16:27 +0000 Subject: [PATCH 037/159] Add mysql_shell.py --- src/charm.py | 10 +++-- src/constants.py | 2 + src/mysql_shell.py | 63 ++++++++++++++++++++++++++++++ src/relations/database_provides.py | 31 +++++---------- src/relations/database_requires.py | 40 ------------------- src/workload.py | 14 +++++-- 6 files changed, 92 insertions(+), 68 deletions(-) create mode 100644 src/mysql_shell.py diff --git a/src/charm.py b/src/charm.py index 7372c5516..54da665e8 100755 --- a/src/charm.py +++ b/src/charm.py @@ -76,6 +76,7 @@ def __init__(self, *args) -> None: @property def _endpoint(self) -> str: + # TODO rename """The k8s endpoint for the charm.""" return f"{self.model.app.name}.{self.model.name}.svc.cluster.local" @@ -166,7 +167,12 @@ def _reconcile_database_relations(self, event=None) -> None: event, self.database_requires.relation.is_breaking(event), self._endpoint, - self.database_requires.relation, + self.workload.get_shell( + self.database_requires.relation.username, + self.database_requires.relation.password, + self.database_requires.relation.host, + self.database_requires.relation.port, + ), ) if ( self.database_requires.relation @@ -176,8 +182,6 @@ def _reconcile_database_relations(self, event=None) -> None: self.workload.start( self.database_requires.relation.host, self.database_requires.relation.port, - self.database_requires.relation.username, - self.database_requires.relation.password, ) else: self.workload.stop() diff --git a/src/constants.py b/src/constants.py index 508ac8e4c..daf347f1b 100644 --- a/src/constants.py +++ b/src/constants.py @@ -3,6 +3,8 @@ """File containing constants to be used in the charm.""" +# TODO: cleanup + CREDENTIALS_SHARED = "credentials-shared" DATABASE_REQUIRES_RELATION = "backend-database" DATABASE_PROVIDES_RELATION = "database" diff --git a/src/mysql_shell.py b/src/mysql_shell.py new file mode 100644 index 000000000..d2660d99a --- /dev/null +++ b/src/mysql_shell.py @@ -0,0 +1,63 @@ +import dataclasses +import logging +import secrets +import string + +import ops + +from constants import PASSWORD_LENGTH + +logger = logging.getLogger(__name__) + + +@dataclasses.dataclass +class Shell: + _container: ops.Container + _username: str + _password: str + _host: str + _port: str + + def _run_commands(self, commands: list[str]) -> None: + commands.insert( + 0, f"shell.connect('{self._username}:{self._password}@{self._host}:{self._port}" + ) + # TODO + # TODO: catch exceptions + + def _run_sql(self, sql_statements: list[str]) -> None: + commands = [] + for statement in sql_statements: + # Escape double quote (") characters in statement + statement = statement.replace('"', r"\"") + commands.append('session.run_sql("' + statement + '")') + self._run_commands(commands) + + @staticmethod + def _generate_password() -> str: + choices = string.ascii_letters + string.digits + return "".join([secrets.choice(choices) for _ in range(PASSWORD_LENGTH)]) + + def create_application_database_and_user(self, username: str, database: str) -> str: + logger.debug(f"Creating {database=} and {username=}") + password = self._generate_password() + self._run_sql( + [ + f"CREATE DATABASE IF NOT EXISTS `{database}`", + f"CREATE USER `{username}` IDENTIFIED BY '{password}'", + f"GRANT ALL PRIVILEGES ON `{database}`.* TO `{username}`", + ] + ) + logger.debug(f"Created {database=} and {username=}") + return password + + def delete_application_user(self, username: str) -> None: + logger.debug(f"Deleting {username=}") + self._run_sql([f"DROP USER `{username}`"]) + logger.debug(f"Deleted {username=}") + + def create_mysql_router_user(self) -> str: + pass + + def delete_mysql_router_user(self) -> None: + pass diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 132291006..6c32a7d11 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -2,9 +2,10 @@ import logging import charms.data_platform_libs.v0.data_interfaces as data_interfaces -import relations.database_requires as database_requires import ops +import mysql_shell + logger = logging.getLogger(__name__) @@ -59,22 +60,13 @@ def _delete_databag(self) -> None: self._local_databag.clear() logger.debug("Deleted databag") - def create_database_and_user( - self, - endpoint: str, - database_requires_relation: database_requires._Relation, # TODO: replace with mysqlsh Python module - ) -> None: - password = database_requires_relation.create_application_database_and_user( - self.username, self.database - ) + def create_database_and_user(self, endpoint: str, shell: mysql_shell.Shell) -> None: + password = shell.create_application_database_and_user(self.username, self.database) self._set_databag(password, endpoint) - def delete_user( - self, - database_requires_relation: database_requires._Relation, # TODO: replace with mysqlsh Python module - ) -> None: + def delete_user(self, shell: mysql_shell.Shell) -> None: self._delete_databag() - database_requires_relation.delete_application_user(self.username) + shell.delete_application_user(self.username) @dataclasses.dataclass @@ -91,10 +83,7 @@ def _requested_users(self, event, event_is_database_requires_broken: bool) -> li return [] requested_users = [] for relation in self._relations: - if ( - isinstance(event, ops.RelationBrokenEvent) - and event.relation.id == relation.id - ): + if isinstance(event, ops.RelationBrokenEvent) and event.relation.id == relation.id: # Relation is being removed; delete user continue requested_users.append(relation) @@ -113,13 +102,13 @@ def reconcile_users( event, event_is_database_requires_broken: bool, endpoint: str, - database_requires_relation: database_requires._Relation, # TODO: replace with mysqlsh Python module + shell: mysql_shell.Shell, ) -> None: requested_users = self._requested_users(event, event_is_database_requires_broken) created_users = self._created_users for relation in requested_users: if relation not in created_users: - relation.create_database_and_user(endpoint, database_requires_relation) + relation.create_database_and_user(endpoint, shell) for relation in created_users: if relation not in requested_users: - relation.delete_user(database_requires_relation) + relation.delete_user(shell) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 6ee4f456c..026c6f377 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -1,17 +1,9 @@ import dataclasses -import logging -import secrets -import string import typing import charms.data_platform_libs.v0.data_interfaces as data_interfaces -import mysql.connector import ops -from constants import PASSWORD_LENGTH - -logger = logging.getLogger(__name__) - class _Relation: def __init__(self, interface: data_interfaces.DatabaseRequires) -> None: @@ -52,38 +44,6 @@ def password(self) -> str: def is_breaking(self, event): return isinstance(event, ops.RelationBrokenEvent) and event.relation.id == self._id - # TODO: move methods below to mysqlsh Python file - def create_application_database_and_user(self, username: str, database: str) -> str: - logger.debug(f"Creating {database=} and {username=}") - password = self._generate_password() - self._execute_sql_statements( - [ - f"CREATE DATABASE IF NOT EXISTS `{database}`", - f"CREATE USER `{username}` IDENTIFIED BY '{password}'", - f"GRANT ALL PRIVILEGES ON `{database}`.* TO `{username}`", - ] - ) - logger.debug(f"Created {database=} and {username=}") - return password - - def delete_application_user(self, username: str) -> None: - logger.debug(f"Deleting {username=}") - self._execute_sql_statements([f"DROP USER `{username}`"]) - logger.debug(f"Deleted {username=}") - - def _execute_sql_statements(self, statements: list[str]) -> None: - # TODO: catch exceptions? - with mysql.connector.connect( - username=self.username, password=self.password, host=self.host, port=self.port - ) as connection, connection.cursor() as cursor: - for statement in statements: - cursor.execute(statement) - - @staticmethod - def _generate_password() -> str: - choices = string.ascii_letters + string.digits - return "".join([secrets.choice(choices) for _ in range(PASSWORD_LENGTH)]) - @dataclasses.dataclass class RelationEndpoint: diff --git a/src/workload.py b/src/workload.py index 61e9c52f3..e9e92d8c6 100644 --- a/src/workload.py +++ b/src/workload.py @@ -5,6 +5,7 @@ import ops import tenacity +import mysql_shell from constants import ( MYSQL_ROUTER_SERVICE_NAME, MYSQL_ROUTER_USER_NAME, @@ -43,7 +44,13 @@ def version(self) -> str: return version return "" - def start(self, host, port, username, password) -> None: + def get_shell(self, username: str, password: str, host: str, port: str) -> mysql_shell.Shell: + return mysql_shell.Shell(self._container, username, password, host, port) + + def start(self, host, port) -> None: + # TODO use shell to create user + username = "foo" + password = "foo" if self.running: # If the host or port changes, MySQL Router will receive topology change notifications from MySQL # Therefore, if the host or port changes, we do not need to restart MySQL Router @@ -65,6 +72,7 @@ def start(self, host, port, username, password) -> None: def stop(self) -> None: if not self.running: return + # TODO: use shell to delete user logger.debug("Stopping MySQL Router service") self._container.stop(MYSQL_ROUTER_SERVICE_NAME) logger.debug("Stopped MySQL Router service") @@ -148,9 +156,7 @@ def _get_mysql_router_layer( ) @staticmethod - @tenacity.retry( - reraise=True, stop=tenacity.stop_after_delay(360), wait=tenacity.wait_fixed(5) - ) + @tenacity.retry(reraise=True, stop=tenacity.stop_after_delay(360), wait=tenacity.wait_fixed(5)) def _wait_until_mysql_router_ready() -> None: # TODO: add debug logging """Wait until a connection to MySQL router is possible. From 076ec593d8f0bb25288ff5cd9fe15d615d698c40 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 21 Apr 2023 17:53:22 +0000 Subject: [PATCH 038/159] Add AuthenticatedWorkload --- src/charm.py | 37 +++++++++++------- src/mysql_shell.py | 24 ++++++++---- src/relations/database_provides.py | 10 ++--- src/relations/database_requires.py | 1 - src/workload.py | 63 +++++++++++++++--------------- 5 files changed, 76 insertions(+), 59 deletions(-) diff --git a/src/charm.py b/src/charm.py index 54da665e8..48136ffc9 100755 --- a/src/charm.py +++ b/src/charm.py @@ -23,6 +23,7 @@ DATABASE_PROVIDES_RELATION, DATABASE_REQUIRES_RELATION, MYSQL_ROUTER_CONTAINER_NAME, + MYSQL_ROUTER_SERVICE_NAME, ) logger = logging.getLogger(__name__) @@ -40,7 +41,7 @@ def __init__(self, *args) -> None: relation_name=DATABASE_REQUIRES_RELATION, # HACK: mysqlrouter needs a user, but not a database # Use the DatabaseRequires interface to get a user; disregard the database - database_name="continuous_writes_database", + database_name="_unused_mysqlrouter_database", extra_user_roles="mysqlrouter", ) ) @@ -70,10 +71,24 @@ def __init__(self, *args) -> None: self.framework.observe(self.on.start, self._on_start) self.framework.observe(self.on.leader_elected, self._on_leader_elected) - self.workload = workload.Workload(self.unit.get_container(MYSQL_ROUTER_CONTAINER_NAME)) - self.tls = relations.tls.MySQLRouterTLS(self) + @property + def workload(self): + # Defined as a property instead of an attribute in __init__ since this class is + # not re-instantiated between events (if there are deferred events) + container = self.unit.get_container(MYSQL_ROUTER_CONTAINER_NAME) + if self.database_requires.relation: + return workload.AuthenticatedWorkload( + container, + MYSQL_ROUTER_SERVICE_NAME, + self.database_requires.relation.username, + self.database_requires.relation.password, + self.database_requires.relation.host, + self.database_requires.relation.port, + ) + return workload.Workload(container, MYSQL_ROUTER_SERVICE_NAME) + @property def _endpoint(self) -> str: # TODO rename @@ -160,29 +175,21 @@ def _reconcile_database_relations(self, event=None) -> None: """Handle database requires/provides events.""" if ( self.unit.is_leader() - and self.database_requires.relation + and isinstance(self.workload, workload.AuthenticatedWorkload) and self.workload.container_ready ): self.database_provides.reconcile_users( event, self.database_requires.relation.is_breaking(event), self._endpoint, - self.workload.get_shell( - self.database_requires.relation.username, - self.database_requires.relation.password, - self.database_requires.relation.host, - self.database_requires.relation.port, - ), + self.workload.shell, ) if ( - self.database_requires.relation + isinstance(self.workload, workload.AuthenticatedWorkload) and not self.database_requires.relation.is_breaking(event) and self.workload.container_ready ): - self.workload.start( - self.database_requires.relation.host, - self.database_requires.relation.port, - ) + self.workload.start() else: self.workload.stop() self._set_status() diff --git a/src/mysql_shell.py b/src/mysql_shell.py index d2660d99a..bc611a9fb 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -51,13 +51,23 @@ def create_application_database_and_user(self, username: str, database: str) -> logger.debug(f"Created {database=} and {username=}") return password - def delete_application_user(self, username: str) -> None: + def create_mysql_router_user(self, username: str) -> str: + logger.debug(f"Creating router {username=}") + password = self._generate_password() + self._run_commands( + [ + "cluster = dba.get_cluster()", + "cluster.setup_router_account('" + + username + + "', {'password': '" + + password + + "'})", + ] + ) + logger.debug(f"Created router {username=}") + return password + + def delete_user(self, username: str) -> None: logger.debug(f"Deleting {username=}") self._run_sql([f"DROP USER `{username}`"]) logger.debug(f"Deleted {username=}") - - def create_mysql_router_user(self) -> str: - pass - - def delete_mysql_router_user(self) -> None: - pass diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 6c32a7d11..ed809ad66 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -45,20 +45,20 @@ def _set_databag(self, password: str, endpoint: str) -> None: read_write_endpoint = f"{endpoint}:6446" read_only_endpoint = f"{endpoint}:6447" logger.debug( - f"Setting databag {self.database=}, {self.username=}, {read_write_endpoint=}, {read_only_endpoint=}" + f"Setting databag {self.id=} {self.database=}, {self.username=}, {read_write_endpoint=}, {read_only_endpoint=}" ) self._interface.set_database(self.id, self.database) self._interface.set_credentials(self.id, self.username, password) self._interface.set_endpoints(self.id, read_write_endpoint) self._interface.set_read_only_endpoints(self.id, read_only_endpoint) logger.debug( - f"Set databag {self.database=}, {self.username=}, {read_write_endpoint=}, {read_only_endpoint=}" + f"Set databag {self.id=} {self.database=}, {self.username=}, {read_write_endpoint=}, {read_only_endpoint=}" ) def _delete_databag(self) -> None: - logger.debug("Deleting databag") + logger.debug(f"Deleting databag {self.id=}") self._local_databag.clear() - logger.debug("Deleted databag") + logger.debug(f"Deleted databag {self.id=}") def create_database_and_user(self, endpoint: str, shell: mysql_shell.Shell) -> None: password = shell.create_application_database_and_user(self.username, self.database) @@ -66,7 +66,7 @@ def create_database_and_user(self, endpoint: str, shell: mysql_shell.Shell) -> N def delete_user(self, shell: mysql_shell.Shell) -> None: self._delete_databag() - shell.delete_application_user(self.username) + shell.delete_user(self.username) @dataclasses.dataclass diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 026c6f377..885fdada8 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -52,7 +52,6 @@ def __init__(self, interface: data_interfaces.DatabaseRequires) -> None: @property def relation(self) -> typing.Optional[_Relation]: - # TODO: rename property? if not self.interface.is_resource_created(): return return _Relation(self.interface) diff --git a/src/workload.py b/src/workload.py index e9e92d8c6..81620cf6b 100644 --- a/src/workload.py +++ b/src/workload.py @@ -7,7 +7,6 @@ import mysql_shell from constants import ( - MYSQL_ROUTER_SERVICE_NAME, MYSQL_ROUTER_USER_NAME, ROUTER_CONFIG_DIRECTORY, TLS_SSL_CERT_FILE, @@ -21,6 +20,7 @@ @dataclasses.dataclass class Workload: _container: ops.Container + _service_name: str @property def container_ready(self) -> bool: @@ -28,9 +28,7 @@ def container_ready(self) -> bool: @property def running(self) -> bool: - service = self._container.get_services(MYSQL_ROUTER_SERVICE_NAME).get( - MYSQL_ROUTER_SERVICE_NAME - ) + service = self._container.get_services(self._service_name).get(self._service_name) if service is None: return False return service.is_running() @@ -44,37 +42,43 @@ def version(self) -> str: return version return "" - def get_shell(self, username: str, password: str, host: str, port: str) -> mysql_shell.Shell: - return mysql_shell.Shell(self._container, username, password, host, port) - def start(self, host, port) -> None: - # TODO use shell to create user - username = "foo" - password = "foo" +@dataclasses.dataclass +class AuthenticatedWorkload(Workload): + _admin_username: str + _admin_password: str + _host: str + _port: str + + @property + def shell(self) -> mysql_shell.Shell: + return mysql_shell.Shell( + self._container, self._admin_username, self._admin_password, self._host, self._port + ) + + def start(self) -> None: if self.running: # If the host or port changes, MySQL Router will receive topology change notifications from MySQL # Therefore, if the host or port changes, we do not need to restart MySQL Router - # TODO: update comment - # Assumption: username or password will not change while database requires relation is active - # Therefore, MySQL Router does not need to be restarted if it is already running return - logger.debug(f"Starting MySQL Router service {host=}, {port=}, {username=}") + logger.debug(f"Starting MySQL Router service {self._host=}, {self._port=}") + router_password = self.shell.create_mysql_router_user(MYSQL_ROUTER_USER_NAME) self._container.add_layer( - MYSQL_ROUTER_SERVICE_NAME, - self._get_mysql_router_layer(host, port, username, password), + self._service_name, + self._get_mysql_router_layer(MYSQL_ROUTER_USER_NAME, router_password), combine=True, ) - self._container.start(MYSQL_ROUTER_SERVICE_NAME) - logger.debug(f"Started MySQL Router service {host=}, {port=}, {username=}") + self._container.start(self._service_name) + logger.debug(f"Started MySQL Router service {self._host=}, {self._port=}") self._wait_until_mysql_router_ready() # TODO: wait until mysql router ready? https://github.com/canonical/mysql-router-k8s-operator/blob/45cf3be44f27476a0371c67d50d7a0193c0fadc2/src/charm.py#L219 def stop(self) -> None: if not self.running: return - # TODO: use shell to delete user logger.debug("Stopping MySQL Router service") - self._container.stop(MYSQL_ROUTER_SERVICE_NAME) + self.shell.delete_user(MYSQL_ROUTER_USER_NAME) + self._container.stop(self._service_name) logger.debug("Stopped MySQL Router service") def enable_tls( @@ -84,7 +88,7 @@ def enable_tls( self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CONFIG_FILE}", config_file_content) self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_KEY_FILE}", key) self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CERT_FILE}", certificate) - self._container.add_layer(MYSQL_ROUTER_SERVICE_NAME, layer, combine=True) + self._container.add_layer(self._service_name, layer, combine=True) self._container.replan() logger.debug("Enabled TLS") @@ -95,14 +99,14 @@ def disable_tls(self) -> None: layer = ops.pebble.Layer( { "services": { - MYSQL_ROUTER_SERVICE_NAME: { + self._service_name: { "override": "merge", "command": "/run.sh mysqlrouter", }, }, }, ) - self._container.add_layer(MYSQL_ROUTER_SERVICE_NAME, layer, combine=True) + self._container.add_layer(self._service_name, layer, combine=True) self._container.replan() logger.debug("Disabled TLS") @@ -130,24 +134,21 @@ def _delete_file(self, path: str) -> None: if self._container.exists(path): self._container.remove_path(path) - @staticmethod - def _get_mysql_router_layer( - host: str, port: str, username: str, password: str - ) -> ops.pebble.Layer: + def _get_mysql_router_layer(self, username: str, password: str) -> ops.pebble.Layer: return ops.pebble.Layer( { "summary": "mysql router layer", "description": "the pebble config layer for mysql router", "services": { - MYSQL_ROUTER_SERVICE_NAME: { + self._service_name: { "override": "replace", "summary": "mysql router", "command": "/run.sh mysqlrouter", "startup": "enabled", "environment": { - "MYSQL_HOST": host, - "MYSQL_PORT": port, - "MYSQL_USER": username, # TODO switch to limited permissions user + "MYSQL_HOST": self._host, + "MYSQL_PORT": self._port, + "MYSQL_USER": username, "MYSQL_PASSWORD": password, }, }, From ef6276e22867c4f239e01f28a02ea20ef3e04795 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 21 Apr 2023 17:57:20 +0000 Subject: [PATCH 039/159] Remove extraneous inits --- src/relations/database_requires.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 885fdada8..914ea9743 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -5,9 +5,9 @@ import ops +@dataclasses.dataclass class _Relation: - def __init__(self, interface: data_interfaces.DatabaseRequires) -> None: - self._interface = interface + _interface: data_interfaces.DatabaseRequires @property def _id(self) -> int: @@ -47,8 +47,7 @@ def is_breaking(self, event): @dataclasses.dataclass class RelationEndpoint: - def __init__(self, interface: data_interfaces.DatabaseRequires) -> None: - self.interface = interface + interface: data_interfaces.DatabaseRequires @property def relation(self) -> typing.Optional[_Relation]: From e511bfd48ee035d419ec0caeee4caab04a4ed5e7 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 24 Apr 2023 19:09:58 +0000 Subject: [PATCH 040/159] Refactor TLS --- src/charm.py | 17 +- src/relations/tls.py | 408 ++++++++++++++++++------------------------- src/workload.py | 192 +++++++++++--------- 3 files changed, 286 insertions(+), 331 deletions(-) diff --git a/src/charm.py b/src/charm.py index 48136ffc9..172489a8a 100755 --- a/src/charm.py +++ b/src/charm.py @@ -68,10 +68,15 @@ def __init__(self, *args) -> None: self.framework.observe( getattr(self.on, "mysql_router_pebble_ready"), self._on_mysql_router_pebble_ready ) - self.framework.observe(self.on.start, self._on_start) + + # Start workload after pod churn or charm upgrade + # (https://juju.is/docs/sdk/start-event#heading--emission-sequence) + # Also, set status on first start if no relations active + self.framework.observe(self.on.start, self._reconcile_database_relations) + self.framework.observe(self.on.leader_elected, self._on_leader_elected) - self.tls = relations.tls.MySQLRouterTLS(self) + self.tls = relations.tls.RelationEndpoint(self) @property def workload(self): @@ -189,19 +194,15 @@ def _reconcile_database_relations(self, event=None) -> None: and not self.database_requires.relation.is_breaking(event) and self.workload.container_ready ): - self.workload.start() + self.workload.enable(self.tls.certificate_saved) else: - self.workload.stop() + self.workload.disable() self._set_status() def _on_mysql_router_pebble_ready(self, _) -> None: self.unit.set_workload_version(self.workload.version) self._reconcile_database_relations() - def _on_start(self, _) -> None: - # If no relations are active, charm status has not been set - self._set_status() - def _on_leader_elected(self, _) -> None: """Patch existing k8s service to include read-write and read-only services.""" try: diff --git a/src/relations/tls.py b/src/relations/tls.py index 996021324..4eff65c0f 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -1,287 +1,217 @@ -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Library containing the implementation of the database requires relation.""" - import base64 +import inspect +import json import logging import re import socket -from string import Template -from typing import List, Optional - -from charms.tls_certificates_interface.v1.tls_certificates import ( - CertificateAvailableEvent, - CertificateExpiringEvent, - TLSCertificatesRequiresV1, - generate_csr, - generate_private_key, -) -from ops import Relation -from ops.charm import ActionEvent -from ops.framework import Object -from ops.pebble import Layer - -from charm import MySQLRouterOperatorCharm -from constants import ( - MYSQL_ROUTER_SERVICE_NAME, - PEER, - ROUTER_CONFIG_DIRECTORY, - TLS_RELATION, - TLS_SSL_CERT_FILE, - TLS_SSL_CONFIG_FILE, - TLS_SSL_KEY_FILE, -) - -SCOPE = "unit" +import typing -logger = logging.getLogger(__name__) +import charms.tls_certificates_interface.v1.tls_certificates as tls_certificates +import ops +import charm +import constants -class MySQLRouterTLS(Object): - """TLS Management class for MySQL Router Operator.""" +logger = logging.getLogger(__name__) - def __init__(self, charm: MySQLRouterOperatorCharm): - super().__init__(charm, TLS_RELATION) - self.charm = charm - self.certs = TLSCertificatesRequiresV1(self.charm, TLS_RELATION) - self.framework.observe( - self.charm.on.set_tls_private_key_action, - self._on_set_tls_private_key, - ) - self.framework.observe( - self.charm.on[TLS_RELATION].relation_joined, self._on_tls_relation_joined - ) - self.framework.observe( - self.charm.on[TLS_RELATION].relation_broken, self._on_tls_relation_broken - ) +class _PeerUnitDatabag: + key: str + requested_csr: str + active_csr: str + certificate: str + ca: str + chain: str + + def __init__(self, databag: ops.RelationDataContent) -> None: + # Cannot use `self._databag =` since this class overrides `__setattr__()` + super().__setattr__("_databag", databag) - self.framework.observe(self.certs.on.certificate_available, self._on_certificate_available) - self.framework.observe(self.certs.on.certificate_expiring, self._on_certificate_expiring) + @staticmethod + def _get_key(key: str) -> str: + """Create databag key by adding a 'tls_' prefix.""" + return f"tls_{key}" @property - def peers(self) -> Optional[Relation]: - """Fetch the peer relation.""" - return self.model.get_relation(PEER) + def _attribute_names(self) -> list[str]: + return [name for name in inspect.get_annotations(type(self))] + + def __getattr__(self, name: str) -> typing.Optional[str]: + return self._databag.get(self._get_key(name)) + + def __setattr__(self, name: str, value: str) -> None: + self._databag[self._get_key(name)] = value + + def __delattr__(self, name: str) -> None: + self._databag.pop(self._get_key(name), None) + + def clear(self) -> None: + """Delete all items in databag.""" + # Delete all type-annotated class attributes + for attribute_name in inspect.get_annotations(type(self)): + delattr(self, attribute_name) + + +class _Relation: + _charm: charm.MySQLRouterOperatorCharm + _interface: tls_certificates.TLSCertificatesRequiresV1 @property - def app_peer_data(self): - """Application peer data object.""" - if not self.peers: - return {} + def _relation(self) -> ops.Relation: + return self._charm.model.get_relation(constants.PEER) - return self.peers.data[self.charm.app] + @property + def peer_unit_databag(self) -> _PeerUnitDatabag: + return _PeerUnitDatabag(self._relation.data[self._charm.unit]) @property - def unit_peer_data(self): - """Unit peer data object.""" - if not self.peers: - return {} + def certificate_saved(self) -> bool: + for value in [self.peer_unit_databag.certificate, self.peer_unit_databag.ca]: + if not value: + return False + return True + + def save_certificate(self, event: tls_certificates.CertificateAvailableEvent) -> None: + if ( + event.certificate_signing_request.strip() + != self.peer_unit_databag.requested_csr.strip() + ): + logger.warning("Unknown certificate received. Ignoring.") + return + if ( + self.certificate_saved + and event.certificate_signing_request.strip() + == self.peer_unit_databag.active_csr.strip() + ): + # Workaround for https://github.com/canonical/tls-certificates-operator/issues/34 + logger.debug("TLS certificate already saved.") + return + self.peer_unit_databag.certificate = event.certificate + self.peer_unit_databag.ca = event.ca + self.peer_unit_databag.chain = json.dumps(event.chain) + self.peer_unit_databag.active_csr = self.peer_unit_databag.requested_csr - return self.peers.data[self.charm.unit] + @staticmethod + def _parse_tls_key(raw_content: str) -> bytes: + """Parse TLS key from plain text or base64 format.""" + if re.match(r"(-+(BEGIN|END) [A-Z ]+-+)", raw_content): + return re.sub( + r"(-+(BEGIN|END) [A-Z ]+-+)", + "\n\\1\n", + raw_content, + ).encode("utf-8") + return base64.b64decode(raw_content) @property - def unit_hostname(self) -> str: + def _unit_hostname(self) -> str: """Get the hostname.localdomain for a unit. Translate juju unit name to hostname.localdomain, necessary for correct name resolution under k8s. Returns: A string representing the hostname.localdomain of the unit. """ - return f"{self.charm.unit.name.replace('/', '-')}.{self.charm.app.name}-endpoints" + return f"{self._charm.unit.name.replace('/', '-')}.{self._charm.app.name}-endpoints" - @property - def hostname(self): - """Return the hostname of the MySQL Router container.""" - return socket.gethostname() - - def get_secret(self, scope: str, key: str) -> Optional[str]: - """Get secret from the peer relation databag.""" - if scope == "unit": - return self.unit_peer_data.get(key, None) - elif scope == "app": - return self.app_peer_data.get(key, None) - else: - raise RuntimeError("Unknown secret scope") + def _generate_csr(self, key: bytes) -> bytes: + return tls_certificates.generate_csr( + private_key=key, + subject=socket.getfqdn(), + organization=self._charm.app.name, + sans=[ + socket.gethostname(), + self._unit_hostname, + str(self._charm.model.get_binding(self._relation).network.bind_address), + ], + ) - def set_secret(self, scope: str, key: str, value: Optional[str]) -> None: - """Set secret in the peer relation databag.""" - if scope == "unit": - if not value: - del self.unit_peer_data[key] - return - self.unit_peer_data.update({key: value}) - elif scope == "app": - if not value: - del self.app_peer_data[key] - return - self.app_peer_data.update({key: value}) + def request_certificate_creation(self, internal_key: str = None): + if internal_key: + key = self._parse_tls_key(internal_key) else: - raise RuntimeError("Unknown secret scope") + key = tls_certificates.generate_private_key() + csr = self._generate_csr(key) + self._interface.request_certificate_creation(certificate_signing_request=csr) + self.peer_unit_databag.key = key.decode("utf-8") + self.peer_unit_databag.requested_csr = csr.decode("utf-8") + + def request_certificate_renewal(self): + old_csr = self.peer_unit_databag.active_csr.encode("utf-8") + key = self.peer_unit_databag.key.encode("utf-8") + new_csr = self._generate_csr(key) + self._interface.request_certificate_renewal( + old_certificate_signing_request=old_csr, new_certificate_signing_request=new_csr + ) + self.peer_unit_databag.requested_csr = new_csr.decode("utf-8") + + +class RelationEndpoint(ops.Object): + def __init__(self, charm: charm.MySQLRouterOperatorCharm): + super().__init__(charm, constants.TLS_RELATION) + self._charm = charm + self._interface = tls_certificates.TLSCertificatesRequiresV1( + self._charm, constants.TLS_RELATION + ) + + self.framework.observe( + self._charm.on.set_tls_private_key_action, + self._on_set_tls_private_key, + ) + self.framework.observe( + self._charm.on[constants.TLS_RELATION].relation_joined, self._on_tls_relation_joined + ) + self.framework.observe( + self._charm.on[constants.TLS_RELATION].relation_broken, self._on_tls_relation_broken + ) + + self.framework.observe( + self._interface.on.certificate_available, self._on_certificate_available + ) + self.framework.observe( + self._interface.on.certificate_expiring, self._on_certificate_expiring + ) - # Handlers + @property + def _relation(self) -> typing.Optional[_Relation]: + if not self._charm.model.get_relation(constants.TLS_RELATION): + return + return _Relation(self._charm, self._interface) - def _on_set_tls_private_key(self, event: ActionEvent) -> None: + @property + def certificate_saved(self) -> bool: + if self._relation is None: + return False + return self._relation.certificate_saved + + def _on_set_tls_private_key(self, event: ops.ActionEvent) -> None: """Action for setting a TLS private key.""" - if not self.charm.model.get_relation(TLS_RELATION): + if self._relation is None: event.fail("No TLS relation available.") return try: - self._request_certificate(event.params.get("internal-key", None)) + self._relation.request_certificate_creation(event.params.get("internal-key")) except Exception as e: event.fail(f"Failed to request certificate: {e}") def _on_tls_relation_joined(self, _) -> None: """Request certificate when TLS relation joined.""" - self._request_certificate(None) + self._relation.request_certificate_creation() def _on_tls_relation_broken(self, _) -> None: - """Disable TLS when TLS relation broken.""" - for secret in ["cert", "chain", "ca"]: - try: - self.set_secret(SCOPE, secret, None) - except KeyError: - # ignore key error for unit teardown - pass - # unset tls flag - self.unit_peer_data.pop("tls") - self._unset_tls() - - def _on_certificate_available(self, event: CertificateAvailableEvent) -> None: - """Enable TLS when TLS certificate available.""" - if not self.charm.workload.running: - logger.debug("Unit not bootstrapped, defer TLS setup") - event.defer() - return - - if event.certificate_signing_request.strip() != self.get_secret(SCOPE, "csr").strip(): - logger.warning("Unknown certificate received. Ignoring.") - return - - # https://github.com/canonical/tls-certificates-operator/issues/34 - if self.unit_peer_data.get("tls") == "enabled": - logger.debug("TLS is already enabled.") - return - - self.set_secret( - SCOPE, "chain", "\n".join(event.chain) if event.chain is not None else None + """Delete TLS certificate.""" + self._relation.peer_unit_databag.clear() + self._charm.workload.disable_tls() + + def _on_certificate_available(self, event: tls_certificates.CertificateAvailableEvent) -> None: + """Save TLS certificate.""" + self._relation.save_certificate(event) + self._charm.workload.enable_tls( + self._relation.peer_unit_databag.key, self._relation.peer_unit_databag.certificate ) - self.set_secret(SCOPE, "cert", event.certificate) - self.set_secret(SCOPE, "ca", event.ca) - # set member-state to avoid unwanted health-check actions - self.unit_peer_data.update({"tls": "enabled"}) - self._set_tls() - - def _on_certificate_expiring(self, event: CertificateExpiringEvent) -> None: + def _on_certificate_expiring(self, event: tls_certificates.CertificateExpiringEvent) -> None: """Request the new certificate when old certificate is expiring.""" - if event.certificate != self.get_secret(SCOPE, "cert"): + if event.certificate != self._relation.peer_unit_databag.certificate: logger.error("An unknown certificate expiring.") return - key = self.get_secret(SCOPE, "key").encode("utf-8") - old_csr = self.get_secret(SCOPE, "csr").encode("utf-8") - new_csr = generate_csr( - private_key=key, - subject=self.unit_hostname, - organization=self.charm.app.name, - sans=self._get_sans(), - ) - self.certs.request_certificate_renewal( - old_certificate_signing_request=old_csr, - new_certificate_signing_request=new_csr, - ) - - # Helpers - def _request_certificate(self, internal_key: Optional[str] = None) -> None: - """Request a certificate from the TLS relation.""" - if internal_key: - key = self._parse_tls_file(internal_key) - else: - key = generate_private_key() - - csr = generate_csr( - private_key=key, - subject=self.hostname, - organization=self.charm.app.name, - sans=self._get_sans(), - ) - - # store secrets - self.set_secret(SCOPE, "key", key.decode("utf-8")) - self.set_secret(SCOPE, "csr", csr.decode("utf-8")) - self.unit_peer_data.pop("tls", None) - self.certs.request_certificate_creation(certificate_signing_request=csr) - - def _get_sans(self) -> List[str]: - """Create a list of DNS names for a unit. - - Returns: - A list representing the hostnames of the unit. - """ - return [ - self.hostname, - socket.getfqdn(), - str(self.charm.model.get_binding(self.peers).network.bind_address), - ] - - @staticmethod - def _parse_tls_file(raw_content: str) -> bytes: - """Parse TLS files from both plain text or base64 format.""" - if re.match(r"(-+(BEGIN|END) [A-Z ]+-+)", raw_content): - return re.sub( - r"(-+(BEGIN|END) [A-Z ]+-+)", - "\n\\1\n", - raw_content, - ).encode("utf-8") - return base64.b64decode(raw_content) - - def _set_tls(self) -> None: - """Enable TLS.""" - # add tls layer merging with mysql-router layer - self.charm.workload.enable_tls( - self._tls_layer(), - self._tls_config_file, - self.get_secret(SCOPE, "key"), - self.get_secret(SCOPE, "cert"), - ) - logger.info("TLS enabled.") - - def _unset_tls(self) -> None: - """Disable TLS.""" - # remove tls layer overriding with original layer - self.charm.workload.disable_tls() - logger.info("TLS disabled.") - - @property - def _tls_config_file(self) -> str: - """Render TLS template to string. - - Config file enables TLS on MySQL Router. - """ - with open("templates/tls.cnf", "r") as template_file: - template = Template(template_file.read()) - config_string = template.substitute( - tls_ssl_key_file=f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_KEY_FILE}", - tls_ssl_cert_file=f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CERT_FILE}", - ) - return config_string - - @staticmethod - def _tls_layer() -> Layer: - """Create a Pebble layer for TLS. - - Returns: - A Pebble layer object. - """ - return Layer( - { - "services": { - MYSQL_ROUTER_SERVICE_NAME: { - "override": "merge", - "command": f"/run.sh mysqlrouter --extra-config {ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CONFIG_FILE}", - }, - }, - }, - ) + self._relation.request_certificate_renewal() diff --git a/src/workload.py b/src/workload.py index 81620cf6b..570234ea1 100644 --- a/src/workload.py +++ b/src/workload.py @@ -1,6 +1,8 @@ import dataclasses import logging import socket +import string +import typing import ops import tenacity @@ -27,11 +29,11 @@ def container_ready(self) -> bool: return self._container.can_connect() @property - def running(self) -> bool: + def _service(self) -> typing.Optional[ops.pebble.Service]: service = self._container.get_services(self._service_name).get(self._service_name) - if service is None: - return False - return service.is_running() + if service is not None: + assert service.startup == "enabled" + return service @property def version(self) -> str: @@ -50,65 +52,97 @@ class AuthenticatedWorkload(Workload): _host: str _port: str + @staticmethod + def _get_layer(services: dict) -> ops.pebble.Layer: + return ops.pebble.Layer( + { + "summary": "mysql router layer", + "description": "the pebble config layer for mysql router", + "services": services, + } + ) + + def _get_active_layer(self, password: str, tls: bool) -> ops.pebble.Layer: + if tls: + command = f"/run.sh mysqlrouter --extra-config {ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CONFIG_FILE}" + else: + command = "/run.sh mysqlrouter" + return self._get_layer( + { + self._service_name: { + "override": "replace", + "summary": "mysql router", + "command": command, + "startup": "enabled", + "environment": { + "MYSQL_HOST": self._host, + "MYSQL_PORT": self._port, + "MYSQL_USER": MYSQL_ROUTER_USER_NAME, + "MYSQL_PASSWORD": password, + }, + }, + } + ) + + @property + def _inactive_layer(self) -> ops.pebble.Layer: + return self._get_layer({}) + + def _update_layer(self, layer: ops.pebble.Layer) -> None: + self._container.add_layer(self._service_name, layer, combine=True) + self._container.replan() + @property def shell(self) -> mysql_shell.Shell: return mysql_shell.Shell( self._container, self._admin_username, self._admin_password, self._host, self._port ) - def start(self) -> None: - if self.running: + @staticmethod + @tenacity.retry(reraise=True, stop=tenacity.stop_after_delay(360), wait=tenacity.wait_fixed(5)) + def _wait_until_mysql_router_ready() -> None: + # TODO: add debug logging + """Wait until a connection to MySQL router is possible. + Retry every 5 seconds for 30 seconds if there is an issue obtaining a connection. + """ + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex(("127.0.0.1", 6446)) + if result != 0: + raise BaseException() + sock.close() + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + result = sock.connect_ex(("127.0.0.1", 6447)) + if result != 0: + raise BaseException() + sock.close() + + def enable(self, tls: bool) -> None: + if self._service is not None: # If the host or port changes, MySQL Router will receive topology change notifications from MySQL # Therefore, if the host or port changes, we do not need to restart MySQL Router return - logger.debug(f"Starting MySQL Router service {self._host=}, {self._port=}") + logger.debug(f"Enabling MySQL Router service {tls=}, {self._host=}, {self._port=}") router_password = self.shell.create_mysql_router_user(MYSQL_ROUTER_USER_NAME) - self._container.add_layer( - self._service_name, - self._get_mysql_router_layer(MYSQL_ROUTER_USER_NAME, router_password), - combine=True, - ) - self._container.start(self._service_name) - logger.debug(f"Started MySQL Router service {self._host=}, {self._port=}") + self._update_layer(self._get_active_layer(router_password, tls)) + logger.debug(f"Enabled MySQL Router service {tls=}, {self._host=}, {self._port=}") self._wait_until_mysql_router_ready() # TODO: wait until mysql router ready? https://github.com/canonical/mysql-router-k8s-operator/blob/45cf3be44f27476a0371c67d50d7a0193c0fadc2/src/charm.py#L219 - def stop(self) -> None: - if not self.running: + def disable(self) -> None: + if self._service is None: return - logger.debug("Stopping MySQL Router service") + logger.debug("Disabling MySQL Router service") self.shell.delete_user(MYSQL_ROUTER_USER_NAME) - self._container.stop(self._service_name) - logger.debug("Stopped MySQL Router service") + self._update_layer(self._inactive_layer) + logger.debug("Disabled MySQL Router service") - def enable_tls( - self, layer: ops.pebble.Layer, config_file_content: str, key: str, certificate: str - ): - logger.debug("Enabling TLS") - self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CONFIG_FILE}", config_file_content) - self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_KEY_FILE}", key) - self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CERT_FILE}", certificate) - self._container.add_layer(self._service_name, layer, combine=True) - self._container.replan() - logger.debug("Enabled TLS") - - def disable_tls(self) -> None: - logger.debug("Disabling TLS") - for file in [TLS_SSL_CONFIG_FILE, TLS_SSL_KEY_FILE, TLS_SSL_CERT_FILE]: - self._delete_file(f"{ROUTER_CONFIG_DIRECTORY}/{file}") - layer = ops.pebble.Layer( - { - "services": { - self._service_name: { - "override": "merge", - "command": "/run.sh mysqlrouter", - }, - }, - }, - ) - self._container.add_layer(self._service_name, layer, combine=True) - self._container.replan() - logger.debug("Disabled TLS") + def _restart(self, tls: bool) -> None: + """Restart MySQL Router to enable or disable TLS.""" + logger.debug("Restarting MySQL Router service") + password = self._service.environment["MYSQL_PASSWORD"] + self._update_layer(self._get_active_layer(password, tls)) + logger.debug("Restarted MySQL Router service") def _write_file(self, path: str, content: str) -> None: """Write content to file. @@ -134,43 +168,33 @@ def _delete_file(self, path: str) -> None: if self._container.exists(path): self._container.remove_path(path) - def _get_mysql_router_layer(self, username: str, password: str) -> ops.pebble.Layer: - return ops.pebble.Layer( - { - "summary": "mysql router layer", - "description": "the pebble config layer for mysql router", - "services": { - self._service_name: { - "override": "replace", - "summary": "mysql router", - "command": "/run.sh mysqlrouter", - "startup": "enabled", - "environment": { - "MYSQL_HOST": self._host, - "MYSQL_PORT": self._port, - "MYSQL_USER": username, - "MYSQL_PASSWORD": password, - }, - }, - }, - } - ) + @property + def _tls_config_file(self) -> str: + """Render TLS template to string. - @staticmethod - @tenacity.retry(reraise=True, stop=tenacity.stop_after_delay(360), wait=tenacity.wait_fixed(5)) - def _wait_until_mysql_router_ready() -> None: - # TODO: add debug logging - """Wait until a connection to MySQL router is possible. - Retry every 5 seconds for 30 seconds if there is an issue obtaining a connection. + Config file enables TLS on MySQL Router. """ - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex(("127.0.0.1", 6446)) - if result != 0: - raise BaseException() - sock.close() + with open("templates/tls.cnf", "r") as template_file: + template = string.Template(template_file.read()) + config_string = template.substitute( + tls_ssl_key_file=f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_KEY_FILE}", + tls_ssl_cert_file=f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CERT_FILE}", + ) + return config_string - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex(("127.0.0.1", 6447)) - if result != 0: - raise BaseException() - sock.close() + def enable_tls(self, key: str, certificate: str): + logger.debug("Enabling TLS") + self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CONFIG_FILE}", self._tls_config_file) + self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_KEY_FILE}", key) + self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CERT_FILE}", certificate) + if self._service is not None: + self._restart(True) + logger.debug("Enabled TLS") + + def disable_tls(self) -> None: + logger.debug("Disabling TLS") + for file in [TLS_SSL_CONFIG_FILE, TLS_SSL_KEY_FILE, TLS_SSL_CERT_FILE]: + self._delete_file(f"{ROUTER_CONFIG_DIRECTORY}/{file}") + if self._service is not None: + self._restart(False) + logger.debug("Disabled TLS") From 1b9673b604c7d0f52bbe1af35b95b211f389879a Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 24 Apr 2023 19:38:56 +0000 Subject: [PATCH 041/159] fixup tls --- src/charm.py | 1 - src/relations/tls.py | 28 +++++++++++++++++----------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/charm.py b/src/charm.py index 172489a8a..8f7067666 100755 --- a/src/charm.py +++ b/src/charm.py @@ -96,7 +96,6 @@ def workload(self): @property def _endpoint(self) -> str: - # TODO rename """The k8s endpoint for the charm.""" return f"{self.model.app.name}.{self.model.name}.svc.cluster.local" diff --git a/src/relations/tls.py b/src/relations/tls.py index 4eff65c0f..fd2f41286 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -1,4 +1,5 @@ import base64 +import dataclasses import inspect import json import logging @@ -13,6 +14,7 @@ import constants logger = logging.getLogger(__name__) +# TODO: fix logging levels class _PeerUnitDatabag: @@ -37,32 +39,36 @@ def _attribute_names(self) -> list[str]: return [name for name in inspect.get_annotations(type(self))] def __getattr__(self, name: str) -> typing.Optional[str]: + assert name in self._attribute_names, f"Invalid attribute {name=}" return self._databag.get(self._get_key(name)) def __setattr__(self, name: str, value: str) -> None: + assert name in self._attribute_names, f"Invalid attribute {name=}" self._databag[self._get_key(name)] = value def __delattr__(self, name: str) -> None: + assert name in self._attribute_names, f"Invalid attribute {name=}" self._databag.pop(self._get_key(name), None) def clear(self) -> None: """Delete all items in databag.""" # Delete all type-annotated class attributes - for attribute_name in inspect.get_annotations(type(self)): - delattr(self, attribute_name) + for name in self._attribute_names: + delattr(self, name) +@dataclasses.dataclass class _Relation: _charm: charm.MySQLRouterOperatorCharm _interface: tls_certificates.TLSCertificatesRequiresV1 @property - def _relation(self) -> ops.Relation: + def _peer_relation(self) -> ops.Relation: return self._charm.model.get_relation(constants.PEER) @property def peer_unit_databag(self) -> _PeerUnitDatabag: - return _PeerUnitDatabag(self._relation.data[self._charm.unit]) + return _PeerUnitDatabag(self._peer_relation.data[self._charm.unit]) @property def certificate_saved(self) -> bool: @@ -90,6 +96,9 @@ def save_certificate(self, event: tls_certificates.CertificateAvailableEvent) -> self.peer_unit_databag.ca = event.ca self.peer_unit_databag.chain = json.dumps(event.chain) self.peer_unit_databag.active_csr = self.peer_unit_databag.requested_csr + self._charm.workload.enable_tls( + self.peer_unit_databag.key, self.peer_unit_databag.certificate + ) @staticmethod def _parse_tls_key(raw_content: str) -> bytes: @@ -120,7 +129,7 @@ def _generate_csr(self, key: bytes) -> bytes: sans=[ socket.gethostname(), self._unit_hostname, - str(self._charm.model.get_binding(self._relation).network.bind_address), + str(self._charm.model.get_binding(self._peer_relation).network.bind_address), ], ) @@ -145,9 +154,9 @@ def request_certificate_renewal(self): class RelationEndpoint(ops.Object): - def __init__(self, charm: charm.MySQLRouterOperatorCharm): - super().__init__(charm, constants.TLS_RELATION) - self._charm = charm + def __init__(self, charm_: charm.MySQLRouterOperatorCharm): + super().__init__(charm_, constants.TLS_RELATION) + self._charm = charm_ self._interface = tls_certificates.TLSCertificatesRequiresV1( self._charm, constants.TLS_RELATION ) @@ -204,9 +213,6 @@ def _on_tls_relation_broken(self, _) -> None: def _on_certificate_available(self, event: tls_certificates.CertificateAvailableEvent) -> None: """Save TLS certificate.""" self._relation.save_certificate(event) - self._charm.workload.enable_tls( - self._relation.peer_unit_databag.key, self._relation.peer_unit_databag.certificate - ) def _on_certificate_expiring(self, event: tls_certificates.CertificateExpiringEvent) -> None: """Request the new certificate when old certificate is expiring.""" From 55ac7e73f47666958c9c1f31508bdd5958ba7ab8 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 24 Apr 2023 19:39:48 +0000 Subject: [PATCH 042/159] format --- src/mysql_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index bc611a9fb..1b03f3968 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -22,7 +22,7 @@ def _run_commands(self, commands: list[str]) -> None: commands.insert( 0, f"shell.connect('{self._username}:{self._password}@{self._host}:{self._port}" ) - # TODO + self._container.exec([]) # TODO # TODO: catch exceptions def _run_sql(self, sql_statements: list[str]) -> None: From 6ece1f9d351fc79c0d4dd2e9b9cebbbf0f3f8a16 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 24 Apr 2023 19:51:16 +0000 Subject: [PATCH 043/159] fix inactive layer --- src/workload.py | 55 ++++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/src/workload.py b/src/workload.py index 570234ea1..dd217803c 100644 --- a/src/workload.py +++ b/src/workload.py @@ -30,10 +30,13 @@ def container_ready(self) -> bool: @property def _service(self) -> typing.Optional[ops.pebble.Service]: - service = self._container.get_services(self._service_name).get(self._service_name) - if service is not None: - assert service.startup == "enabled" - return service + return self._container.get_services(self._service_name).get(self._service_name) + + @property + def _enabled(self) -> bool: + if self._service is None: + return False + return self._service.startup == "enabled" @property def version(self) -> str: @@ -52,13 +55,14 @@ class AuthenticatedWorkload(Workload): _host: str _port: str - @staticmethod - def _get_layer(services: dict) -> ops.pebble.Layer: + def _get_layer(self, service_info: dict) -> ops.pebble.Layer: return ops.pebble.Layer( { "summary": "mysql router layer", "description": "the pebble config layer for mysql router", - "services": services, + "services": { + self._service_name: service_info, + }, } ) @@ -69,24 +73,29 @@ def _get_active_layer(self, password: str, tls: bool) -> ops.pebble.Layer: command = "/run.sh mysqlrouter" return self._get_layer( { - self._service_name: { - "override": "replace", - "summary": "mysql router", - "command": command, - "startup": "enabled", - "environment": { - "MYSQL_HOST": self._host, - "MYSQL_PORT": self._port, - "MYSQL_USER": MYSQL_ROUTER_USER_NAME, - "MYSQL_PASSWORD": password, - }, + "override": "replace", + "summary": "mysql router", + "command": command, + "startup": "enabled", + "environment": { + "MYSQL_HOST": self._host, + "MYSQL_PORT": self._port, + "MYSQL_USER": MYSQL_ROUTER_USER_NAME, + "MYSQL_PASSWORD": password, }, } ) @property def _inactive_layer(self) -> ops.pebble.Layer: - return self._get_layer({}) + return self._get_layer( + { + "override": "replace", + "summary": "mysql router", + "command": "", + "startup": "disabled", + } + ) def _update_layer(self, layer: ops.pebble.Layer) -> None: self._container.add_layer(self._service_name, layer, combine=True) @@ -118,7 +127,7 @@ def _wait_until_mysql_router_ready() -> None: sock.close() def enable(self, tls: bool) -> None: - if self._service is not None: + if self._enabled: # If the host or port changes, MySQL Router will receive topology change notifications from MySQL # Therefore, if the host or port changes, we do not need to restart MySQL Router return @@ -130,7 +139,7 @@ def enable(self, tls: bool) -> None: # TODO: wait until mysql router ready? https://github.com/canonical/mysql-router-k8s-operator/blob/45cf3be44f27476a0371c67d50d7a0193c0fadc2/src/charm.py#L219 def disable(self) -> None: - if self._service is None: + if not self._enabled: return logger.debug("Disabling MySQL Router service") self.shell.delete_user(MYSQL_ROUTER_USER_NAME) @@ -187,7 +196,7 @@ def enable_tls(self, key: str, certificate: str): self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CONFIG_FILE}", self._tls_config_file) self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_KEY_FILE}", key) self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CERT_FILE}", certificate) - if self._service is not None: + if self._enabled: self._restart(True) logger.debug("Enabled TLS") @@ -195,6 +204,6 @@ def disable_tls(self) -> None: logger.debug("Disabling TLS") for file in [TLS_SSL_CONFIG_FILE, TLS_SSL_KEY_FILE, TLS_SSL_CERT_FILE]: self._delete_file(f"{ROUTER_CONFIG_DIRECTORY}/{file}") - if self._service is not None: + if self._enabled: self._restart(False) logger.debug("Disabled TLS") From 096a176bfed1c9ff69fefb961c2c2518220bbccf Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 24 Apr 2023 19:59:38 +0000 Subject: [PATCH 044/159] Remove unused imports --- requirements.txt | 1 - src/constants.py | 11 +---------- src/relations/tls.py | 2 +- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index bc8be31f5..85f874289 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,5 @@ cryptography==39.0.1 jsonschema==4.17.3 lightkube-models==1.24.1.4 lightkube==0.11.0 -mysql-connector-python ops >= 2.0.0 tenacity==8.0.1 diff --git a/src/constants.py b/src/constants.py index daf347f1b..bdd7e6f4a 100644 --- a/src/constants.py +++ b/src/constants.py @@ -3,23 +3,14 @@ """File containing constants to be used in the charm.""" -# TODO: cleanup - -CREDENTIALS_SHARED = "credentials-shared" DATABASE_REQUIRES_RELATION = "backend-database" DATABASE_PROVIDES_RELATION = "database" -NUM_UNITS_BOOTSTRAPPED = "num-units-bootstrapped" MYSQL_ROUTER_CONTAINER_NAME = "mysql-router" -MYSQL_DATABASE_CREATED = "database-created" -MYSQL_ROUTER_PROVIDES_DATA = "provides-data" -MYSQL_ROUTER_REQUIRES_DATA = "requires-data" -MYSQL_ROUTER_REQUIRES_APPLICATION_DATA = "requires-application-data" MYSQL_ROUTER_SERVICE_NAME = "mysql_router" MYSQL_ROUTER_USER_NAME = "mysqlrouter" PASSWORD_LENGTH = 24 -PEER = "mysql-router-peers" +PEER_RELATION = "mysql-router-peers" ROUTER_CONFIG_DIRECTORY = "/tmp/mysqlrouter" -UNIT_BOOTSTRAPPED = "unit-bootstrapped" TLS_RELATION = "certificates" TLS_SSL_CONFIG_FILE = "tls.conf" TLS_SSL_CERT_FILE = "custom-cert.pem" diff --git a/src/relations/tls.py b/src/relations/tls.py index fd2f41286..0ced5b03f 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -64,7 +64,7 @@ class _Relation: @property def _peer_relation(self) -> ops.Relation: - return self._charm.model.get_relation(constants.PEER) + return self._charm.model.get_relation(constants.PEER_RELATION) @property def peer_unit_databag(self) -> _PeerUnitDatabag: From f8b4b89f26b49e428b7af478bd2be907de1e560d Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 11:58:20 +0000 Subject: [PATCH 045/159] Move constants to modules --- src/charm.py | 31 +++++++--------- src/constants.py | 17 --------- src/mysql_shell.py | 5 ++- src/relations/database_provides.py | 1 + src/relations/database_requires.py | 5 +++ src/relations/tls.py | 18 +++++----- src/workload.py | 57 ++++++++++++++++-------------- 7 files changed, 59 insertions(+), 75 deletions(-) delete mode 100644 src/constants.py diff --git a/src/charm.py b/src/charm.py index 8f7067666..616ff6c7e 100755 --- a/src/charm.py +++ b/src/charm.py @@ -19,12 +19,6 @@ import relations.database_requires import relations.tls import workload -from constants import ( - DATABASE_PROVIDES_RELATION, - DATABASE_REQUIRES_RELATION, - MYSQL_ROUTER_CONTAINER_NAME, - MYSQL_ROUTER_SERVICE_NAME, -) logger = logging.getLogger(__name__) @@ -38,7 +32,7 @@ def __init__(self, *args) -> None: self.database_requires = relations.database_requires.RelationEndpoint( data_interfaces.DatabaseRequires( self, - relation_name=DATABASE_REQUIRES_RELATION, + relation_name=relations.database_requires.RelationEndpoint.NAME, # HACK: mysqlrouter needs a user, but not a database # Use the DatabaseRequires interface to get a user; disregard the database database_name="_unused_mysqlrouter_database", @@ -50,19 +44,22 @@ def __init__(self, *args) -> None: self._reconcile_database_relations, ) self.framework.observe( - self.on[DATABASE_REQUIRES_RELATION].relation_broken, + self.on[relations.database_requires.RelationEndpoint.NAME].relation_broken, self._reconcile_database_relations, ) self.database_provides = relations.database_provides.RelationEndpoint( - data_interfaces.DatabaseProvides(self, relation_name=DATABASE_PROVIDES_RELATION) + data_interfaces.DatabaseProvides( + self, relation_name=relations.database_provides.RelationEndpoint.NAME + ) ) self.framework.observe( self.database_provides.interface.on.database_requested, self._reconcile_database_relations, ) self.framework.observe( - self.on[DATABASE_PROVIDES_RELATION].relation_broken, self._reconcile_database_relations + self.on[relations.database_provides.RelationEndpoint.NAME].relation_broken, + self._reconcile_database_relations, ) self.framework.observe( @@ -82,17 +79,16 @@ def __init__(self, *args) -> None: def workload(self): # Defined as a property instead of an attribute in __init__ since this class is # not re-instantiated between events (if there are deferred events) - container = self.unit.get_container(MYSQL_ROUTER_CONTAINER_NAME) + container = self.unit.get_container(workload.Workload.CONTAINER_NAME) if self.database_requires.relation: return workload.AuthenticatedWorkload( container, - MYSQL_ROUTER_SERVICE_NAME, self.database_requires.relation.username, self.database_requires.relation.password, self.database_requires.relation.host, self.database_requires.relation.port, ) - return workload.Workload(container, MYSQL_ROUTER_SERVICE_NAME) + return workload.Workload(container) @property def _endpoint(self) -> str: @@ -101,12 +97,9 @@ def _endpoint(self) -> str: def _determine_status(self) -> ops.StatusBase: missing_relations = [] - for relation, missing in [ - (DATABASE_REQUIRES_RELATION, self.database_requires.relation is None), - (DATABASE_PROVIDES_RELATION, self.database_provides.missing_relation), - ]: - if missing: - missing_relations.append(relation) + for relation in [self.database_requires, self.database_provides]: + if relation.missing_relation: + missing_relations.append(relation.NAME) if missing_relations: return ops.BlockedStatus( f"Missing relation{'s' if len(missing_relations) > 1 else ''}: {', '.join(missing_relations)}" diff --git a/src/constants.py b/src/constants.py deleted file mode 100644 index bdd7e6f4a..000000000 --- a/src/constants.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. - -"""File containing constants to be used in the charm.""" - -DATABASE_REQUIRES_RELATION = "backend-database" -DATABASE_PROVIDES_RELATION = "database" -MYSQL_ROUTER_CONTAINER_NAME = "mysql-router" -MYSQL_ROUTER_SERVICE_NAME = "mysql_router" -MYSQL_ROUTER_USER_NAME = "mysqlrouter" -PASSWORD_LENGTH = 24 -PEER_RELATION = "mysql-router-peers" -ROUTER_CONFIG_DIRECTORY = "/tmp/mysqlrouter" -TLS_RELATION = "certificates" -TLS_SSL_CONFIG_FILE = "tls.conf" -TLS_SSL_CERT_FILE = "custom-cert.pem" -TLS_SSL_KEY_FILE = "custom-key.pem" diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 1b03f3968..d683358c7 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -5,8 +5,7 @@ import ops -from constants import PASSWORD_LENGTH - +_PASSWORD_LENGTH = 24 logger = logging.getLogger(__name__) @@ -36,7 +35,7 @@ def _run_sql(self, sql_statements: list[str]) -> None: @staticmethod def _generate_password() -> str: choices = string.ascii_letters + string.digits - return "".join([secrets.choice(choices) for _ in range(PASSWORD_LENGTH)]) + return "".join([secrets.choice(choices) for _ in range(_PASSWORD_LENGTH)]) def create_application_database_and_user(self, username: str, database: str) -> str: logger.debug(f"Creating {database=} and {username=}") diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index ed809ad66..46c70631a 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -72,6 +72,7 @@ def delete_user(self, shell: mysql_shell.Shell) -> None: @dataclasses.dataclass class RelationEndpoint: interface: data_interfaces.DatabaseProvides + NAME = "database" @property def _relations(self) -> list[_Relation]: diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 914ea9743..3451787a5 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -48,9 +48,14 @@ def is_breaking(self, event): @dataclasses.dataclass class RelationEndpoint: interface: data_interfaces.DatabaseRequires + NAME = "backend-database" @property def relation(self) -> typing.Optional[_Relation]: if not self.interface.is_resource_created(): return return _Relation(self.interface) + + @property + def missing_relation(self) -> bool: + return self.relation is None diff --git a/src/relations/tls.py b/src/relations/tls.py index 0ced5b03f..4e8aae3f0 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -11,8 +11,8 @@ import ops import charm -import constants +_PEER_RELATION_ENDPOINT_NAME = "mysql-router-peers" logger = logging.getLogger(__name__) # TODO: fix logging levels @@ -64,7 +64,7 @@ class _Relation: @property def _peer_relation(self) -> ops.Relation: - return self._charm.model.get_relation(constants.PEER_RELATION) + return self._charm.model.get_relation(_PEER_RELATION_ENDPOINT_NAME) @property def peer_unit_databag(self) -> _PeerUnitDatabag: @@ -154,22 +154,22 @@ def request_certificate_renewal(self): class RelationEndpoint(ops.Object): + NAME = "certificates" + def __init__(self, charm_: charm.MySQLRouterOperatorCharm): - super().__init__(charm_, constants.TLS_RELATION) + super().__init__(charm_, self.NAME) self._charm = charm_ - self._interface = tls_certificates.TLSCertificatesRequiresV1( - self._charm, constants.TLS_RELATION - ) + self._interface = tls_certificates.TLSCertificatesRequiresV1(self._charm, self.NAME) self.framework.observe( self._charm.on.set_tls_private_key_action, self._on_set_tls_private_key, ) self.framework.observe( - self._charm.on[constants.TLS_RELATION].relation_joined, self._on_tls_relation_joined + self._charm.on[self.NAME].relation_joined, self._on_tls_relation_joined ) self.framework.observe( - self._charm.on[constants.TLS_RELATION].relation_broken, self._on_tls_relation_broken + self._charm.on[self.NAME].relation_broken, self._on_tls_relation_broken ) self.framework.observe( @@ -181,7 +181,7 @@ def __init__(self, charm_: charm.MySQLRouterOperatorCharm): @property def _relation(self) -> typing.Optional[_Relation]: - if not self._charm.model.get_relation(constants.TLS_RELATION): + if not self._charm.model.get_relation(self.NAME): return return _Relation(self._charm, self._interface) diff --git a/src/workload.py b/src/workload.py index dd217803c..963c0657b 100644 --- a/src/workload.py +++ b/src/workload.py @@ -1,5 +1,6 @@ import dataclasses import logging +import pathlib import socket import string import typing @@ -8,13 +9,6 @@ import tenacity import mysql_shell -from constants import ( - MYSQL_ROUTER_USER_NAME, - ROUTER_CONFIG_DIRECTORY, - TLS_SSL_CERT_FILE, - TLS_SSL_CONFIG_FILE, - TLS_SSL_KEY_FILE, -) logger = logging.getLogger(__name__) @@ -22,7 +16,8 @@ @dataclasses.dataclass class Workload: _container: ops.Container - _service_name: str + CONTAINER_NAME = "mysql-router" + _SERVICE_NAME = "mysql_router" @property def container_ready(self) -> bool: @@ -30,7 +25,7 @@ def container_ready(self) -> bool: @property def _service(self) -> typing.Optional[ops.pebble.Service]: - return self._container.get_services(self._service_name).get(self._service_name) + return self._container.get_services(self._SERVICE_NAME).get(self._SERVICE_NAME) @property def _enabled(self) -> bool: @@ -54,6 +49,11 @@ class AuthenticatedWorkload(Workload): _admin_password: str _host: str _port: str + _ROUTER_USERNAME = "mysqlrouter" + _ROUTER_CONFIG_DIRECTORY = pathlib.Path("/tmp/mysqlrouter") + _TLS_CONFIG_FILE = "tls.conf" + _TLS_KEY_FILE = "custom-key.pem" + _TLS_CERTIFICATE_FILE = "custom-certificate.pem" def _get_layer(self, service_info: dict) -> ops.pebble.Layer: return ops.pebble.Layer( @@ -61,14 +61,14 @@ def _get_layer(self, service_info: dict) -> ops.pebble.Layer: "summary": "mysql router layer", "description": "the pebble config layer for mysql router", "services": { - self._service_name: service_info, + self._SERVICE_NAME: service_info, }, } ) def _get_active_layer(self, password: str, tls: bool) -> ops.pebble.Layer: if tls: - command = f"/run.sh mysqlrouter --extra-config {ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CONFIG_FILE}" + command = f"/run.sh mysqlrouter --extra-config {self._ROUTER_CONFIG_DIRECTORY / self._TLS_CONFIG_FILE}" else: command = "/run.sh mysqlrouter" return self._get_layer( @@ -80,7 +80,7 @@ def _get_active_layer(self, password: str, tls: bool) -> ops.pebble.Layer: "environment": { "MYSQL_HOST": self._host, "MYSQL_PORT": self._port, - "MYSQL_USER": MYSQL_ROUTER_USER_NAME, + "MYSQL_USER": self._ROUTER_USERNAME, "MYSQL_PASSWORD": password, }, } @@ -98,7 +98,7 @@ def _inactive_layer(self) -> ops.pebble.Layer: ) def _update_layer(self, layer: ops.pebble.Layer) -> None: - self._container.add_layer(self._service_name, layer, combine=True) + self._container.add_layer(self._SERVICE_NAME, layer, combine=True) self._container.replan() @property @@ -132,7 +132,7 @@ def enable(self, tls: bool) -> None: # Therefore, if the host or port changes, we do not need to restart MySQL Router return logger.debug(f"Enabling MySQL Router service {tls=}, {self._host=}, {self._port=}") - router_password = self.shell.create_mysql_router_user(MYSQL_ROUTER_USER_NAME) + router_password = self.shell.create_mysql_router_user(self._ROUTER_USERNAME) self._update_layer(self._get_active_layer(router_password, tls)) logger.debug(f"Enabled MySQL Router service {tls=}, {self._host=}, {self._port=}") self._wait_until_mysql_router_ready() @@ -142,7 +142,7 @@ def disable(self) -> None: if not self._enabled: return logger.debug("Disabling MySQL Router service") - self.shell.delete_user(MYSQL_ROUTER_USER_NAME) + self.shell.delete_user(self._ROUTER_USERNAME) self._update_layer(self._inactive_layer) logger.debug("Disabled MySQL Router service") @@ -153,7 +153,7 @@ def _restart(self, tls: bool) -> None: self._update_layer(self._get_active_layer(password, tls)) logger.debug("Restarted MySQL Router service") - def _write_file(self, path: str, content: str) -> None: + def _write_file(self, path: pathlib.Path, content: str) -> None: """Write content to file. Args: @@ -161,19 +161,20 @@ def _write_file(self, path: str, content: str) -> None: content: File content """ self._container.push( - path, + str(path), content, permissions=0o600, - user=MYSQL_ROUTER_USER_NAME, - group=MYSQL_ROUTER_USER_NAME, + user=self._ROUTER_USERNAME, + group=self._ROUTER_USERNAME, ) - def _delete_file(self, path: str) -> None: + def _delete_file(self, path: pathlib.Path) -> None: """Delete file. Args: path: Full filesystem path (with filename) """ + path = str(path) if self._container.exists(path): self._container.remove_path(path) @@ -186,24 +187,26 @@ def _tls_config_file(self) -> str: with open("templates/tls.cnf", "r") as template_file: template = string.Template(template_file.read()) config_string = template.substitute( - tls_ssl_key_file=f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_KEY_FILE}", - tls_ssl_cert_file=f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CERT_FILE}", + tls_ssl_key_file=self._ROUTER_CONFIG_DIRECTORY / self._TLS_KEY_FILE, + tls_ssl_cert_file=self._ROUTER_CONFIG_DIRECTORY / self._TLS_CERTIFICATE_FILE, ) return config_string def enable_tls(self, key: str, certificate: str): logger.debug("Enabling TLS") - self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CONFIG_FILE}", self._tls_config_file) - self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_KEY_FILE}", key) - self._write_file(f"{ROUTER_CONFIG_DIRECTORY}/{TLS_SSL_CERT_FILE}", certificate) + self._write_file( + self._ROUTER_CONFIG_DIRECTORY / self._TLS_CONFIG_FILE, self._tls_config_file + ) + self._write_file(self._ROUTER_CONFIG_DIRECTORY / self._TLS_KEY_FILE, key) + self._write_file(self._ROUTER_CONFIG_DIRECTORY / self._TLS_CERTIFICATE_FILE, certificate) if self._enabled: self._restart(True) logger.debug("Enabled TLS") def disable_tls(self) -> None: logger.debug("Disabling TLS") - for file in [TLS_SSL_CONFIG_FILE, TLS_SSL_KEY_FILE, TLS_SSL_CERT_FILE]: - self._delete_file(f"{ROUTER_CONFIG_DIRECTORY}/{file}") + for file in [self._TLS_CONFIG_FILE, self._TLS_KEY_FILE, self._TLS_CERTIFICATE_FILE]: + self._delete_file(self._ROUTER_CONFIG_DIRECTORY / file) if self._enabled: self._restart(False) logger.debug("Disabled TLS") From c88df2e4e92c83ec92a4a5a373493b6f2a8439ea Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 12:11:18 +0000 Subject: [PATCH 046/159] waitingstatus --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 616ff6c7e..18fa845d2 100755 --- a/src/charm.py +++ b/src/charm.py @@ -105,7 +105,7 @@ def _determine_status(self) -> ops.StatusBase: f"Missing relation{'s' if len(missing_relations) > 1 else ''}: {', '.join(missing_relations)}" ) if not self.workload.container_ready: - return ops.MaintenanceStatus("Waiting for container") # TODO + return ops.WaitingStatus("Waiting for container") return ops.ActiveStatus() def _set_status(self) -> None: From c6f53f397747e4557f0d7ce00d66e05636325e3b Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 12:12:38 +0000 Subject: [PATCH 047/159] Add newline before constants --- src/relations/database_provides.py | 1 + src/relations/database_requires.py | 1 + src/workload.py | 2 ++ 3 files changed, 4 insertions(+) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 46c70631a..5b8e60e19 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -72,6 +72,7 @@ def delete_user(self, shell: mysql_shell.Shell) -> None: @dataclasses.dataclass class RelationEndpoint: interface: data_interfaces.DatabaseProvides + NAME = "database" @property diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 3451787a5..900adaf79 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -48,6 +48,7 @@ def is_breaking(self, event): @dataclasses.dataclass class RelationEndpoint: interface: data_interfaces.DatabaseRequires + NAME = "backend-database" @property diff --git a/src/workload.py b/src/workload.py index 963c0657b..940543160 100644 --- a/src/workload.py +++ b/src/workload.py @@ -16,6 +16,7 @@ @dataclasses.dataclass class Workload: _container: ops.Container + CONTAINER_NAME = "mysql-router" _SERVICE_NAME = "mysql_router" @@ -49,6 +50,7 @@ class AuthenticatedWorkload(Workload): _admin_password: str _host: str _port: str + _ROUTER_USERNAME = "mysqlrouter" _ROUTER_CONFIG_DIRECTORY = pathlib.Path("/tmp/mysqlrouter") _TLS_CONFIG_FILE = "tls.conf" From 0e6e46d8edfda6138af9a6a2693761313f241cb3 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 12:16:11 +0000 Subject: [PATCH 048/159] fixup! waitingstatus --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 18fa845d2..b3d297e72 100755 --- a/src/charm.py +++ b/src/charm.py @@ -105,7 +105,7 @@ def _determine_status(self) -> ops.StatusBase: f"Missing relation{'s' if len(missing_relations) > 1 else ''}: {', '.join(missing_relations)}" ) if not self.workload.container_ready: - return ops.WaitingStatus("Waiting for container") + return ops.MaintenanceStatus("Waiting for container") return ops.ActiveStatus() def _set_status(self) -> None: From a8024d93bdd52195acce71d1bdd647f520c82d66 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 12:37:19 +0000 Subject: [PATCH 049/159] Error handling --- src/charm.py | 2 +- src/mysql_shell.py | 8 ++++++-- src/relations/tls.py | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/charm.py b/src/charm.py index b3d297e72..f15bcdfc2 100755 --- a/src/charm.py +++ b/src/charm.py @@ -201,7 +201,7 @@ def _on_leader_elected(self, _) -> None: self._patch_service(self.app.name, ro_port=6447, rw_port=6446) except ApiError: logger.exception("Failed to patch k8s service") - self.unit.status = ops.BlockedStatus("Failed to patch k8s service") + raise if __name__ == "__main__": diff --git a/src/mysql_shell.py b/src/mysql_shell.py index d683358c7..086df1138 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -21,8 +21,12 @@ def _run_commands(self, commands: list[str]) -> None: commands.insert( 0, f"shell.connect('{self._username}:{self._password}@{self._host}:{self._port}" ) - self._container.exec([]) # TODO - # TODO: catch exceptions + try: + process = self._container.exec([]) # TODO + process.wait_output() + except ops.pebble.ExecError as e: + logger.exception(f"Failed to run {commands=}\nstderr:\n{e.stderr}\n") + raise def _run_sql(self, sql_statements: list[str]) -> None: commands = [] diff --git a/src/relations/tls.py b/src/relations/tls.py index 4e8aae3f0..fd48bf2ba 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -200,6 +200,8 @@ def _on_set_tls_private_key(self, event: ops.ActionEvent) -> None: self._relation.request_certificate_creation(event.params.get("internal-key")) except Exception as e: event.fail(f"Failed to request certificate: {e}") + logger.exception("Failed to set TLS private key via action") + raise def _on_tls_relation_joined(self, _) -> None: """Request certificate when TLS relation joined.""" From a1ba3d6f362c5a0e545c14130b824cfc1b012597 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 12:41:42 +0000 Subject: [PATCH 050/159] copyright --- src/mysql_shell.py | 2 ++ src/relations/database_provides.py | 2 ++ src/relations/database_requires.py | 2 ++ src/relations/tls.py | 2 ++ src/workload.py | 2 ++ 5 files changed, 10 insertions(+) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 086df1138..e9c9bde2a 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -1,3 +1,5 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. import dataclasses import logging import secrets diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 5b8e60e19..e2b8cb0c9 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -1,3 +1,5 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. import dataclasses import logging diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 900adaf79..3bc5729fb 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -1,3 +1,5 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. import dataclasses import typing diff --git a/src/relations/tls.py b/src/relations/tls.py index fd48bf2ba..6d3aebc2d 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -1,3 +1,5 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. import base64 import dataclasses import inspect diff --git a/src/workload.py b/src/workload.py index 940543160..58f34f2d4 100644 --- a/src/workload.py +++ b/src/workload.py @@ -1,3 +1,5 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. import dataclasses import logging import pathlib From ed686483c8a730ea95bd17225f895210ed1a5679 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 12:47:18 +0000 Subject: [PATCH 051/159] lightkube imports --- src/charm.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/charm.py b/src/charm.py index f15bcdfc2..9c067b2e6 100755 --- a/src/charm.py +++ b/src/charm.py @@ -9,11 +9,11 @@ import logging import charms.data_platform_libs.v0.data_interfaces as data_interfaces +import lightkube +import lightkube.models.core_v1 +import lightkube.models.meta_v1 +import lightkube.resources.core_v1 import ops -from lightkube import ApiError, Client -from lightkube.models.core_v1 import ServicePort, ServiceSpec -from lightkube.models.meta_v1 import ObjectMeta -from lightkube.resources.core_v1 import Pod, Service import relations.database_provides import relations.database_requires @@ -116,7 +116,7 @@ def _set_status(self) -> None: self.unit.status = self._determine_status() def _patch_service(self, name: str, ro_port: int, rw_port: int) -> None: - """Patch juju created k8s service. + """Patch Juju-created k8s service. The k8s service will be tied to pod-0 so that the service is auto cleaned by k8s when the last pod is scaled down. Args: @@ -124,14 +124,14 @@ def _patch_service(self, name: str, ro_port: int, rw_port: int) -> None: ro_port: The read only port. rw_port: The read write port. """ - client = Client() + client = lightkube.Client() pod0 = client.get( - res=Pod, + res=lightkube.resources.core_v1.Pod, name=self.app.name + "-0", namespace=self.model.name, ) - service = Service( - metadata=ObjectMeta( + service = lightkube.resources.core_v1.Service( + metadata=lightkube.models.meta_v1.ObjectMeta( name=name, namespace=self.model.name, ownerReferences=pod0.metadata.ownerReferences, @@ -139,14 +139,14 @@ def _patch_service(self, name: str, ro_port: int, rw_port: int) -> None: "app.kubernetes.io/name": self.app.name, }, ), - spec=ServiceSpec( + spec=lightkube.models.core_v1.ServiceSpec( ports=[ - ServicePort( + lightkube.models.core_v1.ServicePort( name="mysql-ro", port=ro_port, targetPort=ro_port, ), - ServicePort( + lightkube.models.core_v1.ServicePort( name="mysql-rw", port=rw_port, targetPort=rw_port, @@ -156,7 +156,7 @@ def _patch_service(self, name: str, ro_port: int, rw_port: int) -> None: ), ) client.patch( - res=Service, + res=lightkube.resources.core_v1.Service, obj=service, name=service.metadata.name, namespace=service.metadata.namespace, @@ -199,7 +199,7 @@ def _on_leader_elected(self, _) -> None: """Patch existing k8s service to include read-write and read-only services.""" try: self._patch_service(self.app.name, ro_port=6447, rw_port=6446) - except ApiError: + except lightkube.ApiError: logger.exception("Failed to patch k8s service") raise From 48f05ed6581546069e5694add4a823089e7fdd56 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 13:53:50 +0000 Subject: [PATCH 052/159] add docstrings --- src/charm.py | 13 ++++++++++--- src/mysql_shell.py | 11 +++++++++++ src/relations/database_provides.py | 20 +++++++++++++++++++- src/relations/database_requires.py | 14 ++++++++++++++ src/relations/tls.py | 18 +++++++++++++++++- src/workload.py | 22 +++++++++++++++++++++- 6 files changed, 92 insertions(+), 6 deletions(-) diff --git a/src/charm.py b/src/charm.py index 9c067b2e6..f6b6fe8c0 100755 --- a/src/charm.py +++ b/src/charm.py @@ -4,7 +4,7 @@ # # Learn more at: https://juju.is/docs/sdk -"""MySQL Router k8s charm.""" +"""MySQL Router kubernetes (k8s) charm""" import logging @@ -24,7 +24,7 @@ class MySQLRouterOperatorCharm(ops.CharmBase): - """Operator charm for MySQL Router.""" + """Operator charm for MySQL Router""" def __init__(self, *args) -> None: super().__init__(*args) @@ -92,10 +92,11 @@ def workload(self): @property def _endpoint(self) -> str: - """The k8s endpoint for the charm.""" + """K8s endpoint for the charm""" return f"{self.model.app.name}.{self.model.name}.svc.cluster.local" def _determine_status(self) -> ops.StatusBase: + """Report charm status.""" missing_relations = [] for relation in [self.database_requires, self.database_provides]: if relation.missing_relation: @@ -109,6 +110,10 @@ def _determine_status(self) -> ops.StatusBase: return ops.ActiveStatus() def _set_status(self) -> None: + """Set charm status. + + Except if charm is in unrecognized state + """ if isinstance( self.unit.status, ops.BlockedStatus ) and not self.unit.status.message.startswith("Missing relation"): @@ -117,8 +122,10 @@ def _set_status(self) -> None: def _patch_service(self, name: str, ro_port: int, rw_port: int) -> None: """Patch Juju-created k8s service. + The k8s service will be tied to pod-0 so that the service is auto cleaned by k8s when the last pod is scaled down. + Args: name: The name of the service. ro_port: The read only port. diff --git a/src/mysql_shell.py b/src/mysql_shell.py index e9c9bde2a..1a0c690da 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -1,5 +1,11 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. + +"""MySQL Shell in Python execution mode + +https://dev.mysql.com/doc/mysql-shell/8.0/en/ +""" + import dataclasses import logging import secrets @@ -13,6 +19,7 @@ @dataclasses.dataclass class Shell: + """MySQL Shell connected to MySQL cluster""" _container: ops.Container _username: str _password: str @@ -20,6 +27,7 @@ class Shell: _port: str def _run_commands(self, commands: list[str]) -> None: + """Connect to MySQL cluster and run commands.""" commands.insert( 0, f"shell.connect('{self._username}:{self._password}@{self._host}:{self._port}" ) @@ -31,6 +39,7 @@ def _run_commands(self, commands: list[str]) -> None: raise def _run_sql(self, sql_statements: list[str]) -> None: + """Connect to MySQL cluster and execute SQL.""" commands = [] for statement in sql_statements: # Escape double quote (") characters in statement @@ -44,6 +53,7 @@ def _generate_password() -> str: return "".join([secrets.choice(choices) for _ in range(_PASSWORD_LENGTH)]) def create_application_database_and_user(self, username: str, database: str) -> str: + """Create database and user for related database_provides application.""" logger.debug(f"Creating {database=} and {username=}") password = self._generate_password() self._run_sql( @@ -57,6 +67,7 @@ def create_application_database_and_user(self, username: str, database: str) -> return password def create_mysql_router_user(self, username: str) -> str: + """Create user to run MySQL Router service.""" logger.debug(f"Creating router {username=}") password = self._generate_password() self._run_commands( diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index e2b8cb0c9..4fa561ac1 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -1,5 +1,8 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. + +"""Relation(s) to one or more application charms""" + import dataclasses import logging @@ -13,15 +16,18 @@ @dataclasses.dataclass class _Relation: + """Relation to one application charm""" _relation: ops.Relation _interface: data_interfaces.DatabaseProvides @property def _local_databag(self) -> ops.RelationDataContent: + """MySQL Router charm databag.""" return self._relation.data[self._interface.local_app] @property def _remote_databag(self) -> dict: + """MySQL charm databag.""" return self._interface.fetch_relation_data()[self.id] @property @@ -30,6 +36,7 @@ def id(self) -> int: @property def user_created(self) -> bool: + """Whether database user has been shared with application charm""" for key in ["database", "username", "password", "endpoints"]: if key not in self._local_databag: return False @@ -37,13 +44,16 @@ def user_created(self) -> bool: @property def database(self) -> str: + """Requested database name""" return self._remote_databag["database"] @property def username(self) -> str: + """Database username""" return f"relation-{self.id}" def _set_databag(self, password: str, endpoint: str) -> None: + """Share connection information with application charm.""" read_write_endpoint = f"{endpoint}:6446" read_only_endpoint = f"{endpoint}:6447" logger.debug( @@ -58,21 +68,25 @@ def _set_databag(self, password: str, endpoint: str) -> None: ) def _delete_databag(self) -> None: + """Remove connection information from databag.""" logger.debug(f"Deleting databag {self.id=}") self._local_databag.clear() logger.debug(f"Deleted databag {self.id=}") def create_database_and_user(self, endpoint: str, shell: mysql_shell.Shell) -> None: + """Create database & user and update databag.""" password = shell.create_application_database_and_user(self.username, self.database) self._set_databag(password, endpoint) def delete_user(self, shell: mysql_shell.Shell) -> None: + """Delete user and update databag.""" self._delete_databag() shell.delete_user(self.username) @dataclasses.dataclass class RelationEndpoint: + """Relation endpoint for application charm(s)""" interface: data_interfaces.DatabaseProvides NAME = "database" @@ -82,8 +96,9 @@ def _relations(self) -> list[_Relation]: return [_Relation(relation, self.interface) for relation in self.interface.relations] def _requested_users(self, event, event_is_database_requires_broken: bool) -> list[_Relation]: + """Related application charms that have requested a database & user""" if event_is_database_requires_broken: - # Cluster connection is being removed; delete all users + # MySQL cluster connection is being removed; delete all users return [] requested_users = [] for relation in self._relations: @@ -95,10 +110,12 @@ def _requested_users(self, event, event_is_database_requires_broken: bool) -> li @property def _created_users(self) -> list[_Relation]: + """Users that have been created and shared with an application charm""" return [relation for relation in self._relations if relation.user_created] @property def missing_relation(self) -> bool: + """Whether zero relations to application charms exist""" return len(self._relations) == 0 def reconcile_users( @@ -108,6 +125,7 @@ def reconcile_users( endpoint: str, shell: mysql_shell.Shell, ) -> None: + """Create requested users and delete inactive users.""" requested_users = self._requested_users(event, event_is_database_requires_broken) created_users = self._created_users for relation in requested_users: diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 3bc5729fb..4d96ec38e 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -1,5 +1,8 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. + +"""Relation to MySQL charm""" + import dataclasses import typing @@ -9,6 +12,7 @@ @dataclasses.dataclass class _Relation: + """Relation to MySQL charm.""" _interface: data_interfaces.DatabaseRequires @property @@ -19,46 +23,56 @@ def _id(self) -> int: @property def _remote_databag(self) -> dict: + """MySQL charm databag.""" return self._interface.fetch_relation_data()[self._id] @property def _endpoint(self) -> str: + """MySQL cluster primary endpoint.""" endpoints = self._remote_databag["endpoints"].split(",") assert len(endpoints) == 1 return endpoints[0] @property def host(self) -> str: + """MySQL cluster primary host.""" return self._endpoint.split(":")[0] @property def port(self) -> str: + """MySQL cluster primary port.""" return self._endpoint.split(":")[1] @property def username(self) -> str: + """Admin username""" return self._remote_databag["username"] @property def password(self) -> str: + """Admin password""" return self._remote_databag["password"] def is_breaking(self, event): + """Whether relation will be broken after the current event is handled""" return isinstance(event, ops.RelationBrokenEvent) and event.relation.id == self._id @dataclasses.dataclass class RelationEndpoint: + """Relation endpoint for MySQL charm""" interface: data_interfaces.DatabaseRequires NAME = "backend-database" @property def relation(self) -> typing.Optional[_Relation]: + """Relation to MySQL charm""" if not self.interface.is_resource_created(): return return _Relation(self.interface) @property def missing_relation(self) -> bool: + """Whether relation to MySQL charm does not exist""" return self.relation is None diff --git a/src/relations/tls.py b/src/relations/tls.py index 6d3aebc2d..7908a91cc 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -1,5 +1,8 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. + +"""Relation to TLS certificate provider""" + import base64 import dataclasses import inspect @@ -20,6 +23,7 @@ class _PeerUnitDatabag: + """Peer relation unit databag""" key: str requested_csr: str active_csr: str @@ -38,6 +42,7 @@ def _get_key(key: str) -> str: @property def _attribute_names(self) -> list[str]: + """Class attributes with type annotation""" return [name for name in inspect.get_annotations(type(self))] def __getattr__(self, name: str) -> typing.Optional[str]: @@ -54,32 +59,36 @@ def __delattr__(self, name: str) -> None: def clear(self) -> None: """Delete all items in databag.""" - # Delete all type-annotated class attributes for name in self._attribute_names: delattr(self, name) @dataclasses.dataclass class _Relation: + """Relation to TLS certificate provider""" _charm: charm.MySQLRouterOperatorCharm _interface: tls_certificates.TLSCertificatesRequiresV1 @property def _peer_relation(self) -> ops.Relation: + """MySQL Router charm peer relation""" return self._charm.model.get_relation(_PEER_RELATION_ENDPOINT_NAME) @property def peer_unit_databag(self) -> _PeerUnitDatabag: + """MySQL Router charm peer relation unit databag""" return _PeerUnitDatabag(self._peer_relation.data[self._charm.unit]) @property def certificate_saved(self) -> bool: + """Whether a TLS certificate is available to use""" for value in [self.peer_unit_databag.certificate, self.peer_unit_databag.ca]: if not value: return False return True def save_certificate(self, event: tls_certificates.CertificateAvailableEvent) -> None: + """Save TLS certificate in peer relation unit databag.""" if ( event.certificate_signing_request.strip() != self.peer_unit_databag.requested_csr.strip() @@ -116,14 +125,17 @@ def _parse_tls_key(raw_content: str) -> bytes: @property def _unit_hostname(self) -> str: """Get the hostname.localdomain for a unit. + Translate juju unit name to hostname.localdomain, necessary for correct name resolution under k8s. + Returns: A string representing the hostname.localdomain of the unit. """ return f"{self._charm.unit.name.replace('/', '-')}.{self._charm.app.name}-endpoints" def _generate_csr(self, key: bytes) -> bytes: + """Generate certificate signing request (CSR).""" return tls_certificates.generate_csr( private_key=key, subject=socket.getfqdn(), @@ -136,6 +148,7 @@ def _generate_csr(self, key: bytes) -> bytes: ) def request_certificate_creation(self, internal_key: str = None): + """Request new TLS certificate from related provider charm.""" if internal_key: key = self._parse_tls_key(internal_key) else: @@ -146,6 +159,7 @@ def request_certificate_creation(self, internal_key: str = None): self.peer_unit_databag.requested_csr = csr.decode("utf-8") def request_certificate_renewal(self): + """Request TLS certificate renewal from related provider charm.""" old_csr = self.peer_unit_databag.active_csr.encode("utf-8") key = self.peer_unit_databag.key.encode("utf-8") new_csr = self._generate_csr(key) @@ -156,6 +170,7 @@ def request_certificate_renewal(self): class RelationEndpoint(ops.Object): + """Relation endpoint and handlers for TLS certificate provider""" NAME = "certificates" def __init__(self, charm_: charm.MySQLRouterOperatorCharm): @@ -189,6 +204,7 @@ def _relation(self) -> typing.Optional[_Relation]: @property def certificate_saved(self) -> bool: + """Whether a TLS certificate is available to use""" if self._relation is None: return False return self._relation.certificate_saved diff --git a/src/workload.py b/src/workload.py index 58f34f2d4..fe6370bbe 100644 --- a/src/workload.py +++ b/src/workload.py @@ -1,5 +1,8 @@ # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. + +"""MySQL Router workload""" + import dataclasses import logging import pathlib @@ -17,6 +20,7 @@ @dataclasses.dataclass class Workload: + """MySQL Router workload""" _container: ops.Container CONTAINER_NAME = "mysql-router" @@ -28,10 +32,12 @@ def container_ready(self) -> bool: @property def _service(self) -> typing.Optional[ops.pebble.Service]: + """MySQL Router service""" return self._container.get_services(self._SERVICE_NAME).get(self._SERVICE_NAME) @property def _enabled(self) -> bool: + """Service status""" if self._service is None: return False return self._service.startup == "enabled" @@ -48,6 +54,7 @@ def version(self) -> str: @dataclasses.dataclass class AuthenticatedWorkload(Workload): + """Workload with connection to MySQL cluster""" _admin_username: str _admin_password: str _host: str @@ -60,6 +67,7 @@ class AuthenticatedWorkload(Workload): _TLS_CERTIFICATE_FILE = "custom-certificate.pem" def _get_layer(self, service_info: dict) -> ops.pebble.Layer: + """Create layer.""" return ops.pebble.Layer( { "summary": "mysql router layer", @@ -71,6 +79,12 @@ def _get_layer(self, service_info: dict) -> ops.pebble.Layer: ) def _get_active_layer(self, password: str, tls: bool) -> ops.pebble.Layer: + """Create layer with startup enabled. + + Args: + password: MySQL Router user password + tls: Whether TLS is enabled + """ if tls: command = f"/run.sh mysqlrouter --extra-config {self._ROUTER_CONFIG_DIRECTORY / self._TLS_CONFIG_FILE}" else: @@ -92,6 +106,7 @@ def _get_active_layer(self, password: str, tls: bool) -> ops.pebble.Layer: @property def _inactive_layer(self) -> ops.pebble.Layer: + """Create layer with startup disabled.""" return self._get_layer( { "override": "replace", @@ -102,6 +117,7 @@ def _inactive_layer(self) -> ops.pebble.Layer: ) def _update_layer(self, layer: ops.pebble.Layer) -> None: + """Update and restart services.""" self._container.add_layer(self._SERVICE_NAME, layer, combine=True) self._container.replan() @@ -131,6 +147,7 @@ def _wait_until_mysql_router_ready() -> None: sock.close() def enable(self, tls: bool) -> None: + """Start and enable MySQL Router service.""" if self._enabled: # If the host or port changes, MySQL Router will receive topology change notifications from MySQL # Therefore, if the host or port changes, we do not need to restart MySQL Router @@ -143,6 +160,7 @@ def enable(self, tls: bool) -> None: # TODO: wait until mysql router ready? https://github.com/canonical/mysql-router-k8s-operator/blob/45cf3be44f27476a0371c67d50d7a0193c0fadc2/src/charm.py#L219 def disable(self) -> None: + """Stop and disable MySQL Router service.""" if not self._enabled: return logger.debug("Disabling MySQL Router service") @@ -184,7 +202,7 @@ def _delete_file(self, path: pathlib.Path) -> None: @property def _tls_config_file(self) -> str: - """Render TLS template to string. + """Render config file template to string. Config file enables TLS on MySQL Router. """ @@ -197,6 +215,7 @@ def _tls_config_file(self) -> str: return config_string def enable_tls(self, key: str, certificate: str): + """Enable TLS and restart MySQL Router service.""" logger.debug("Enabling TLS") self._write_file( self._ROUTER_CONFIG_DIRECTORY / self._TLS_CONFIG_FILE, self._tls_config_file @@ -208,6 +227,7 @@ def enable_tls(self, key: str, certificate: str): logger.debug("Enabled TLS") def disable_tls(self) -> None: + """Disable TLS and restart MySQL Router service.""" logger.debug("Disabling TLS") for file in [self._TLS_CONFIG_FILE, self._TLS_KEY_FILE, self._TLS_CERTIFICATE_FILE]: self._delete_file(self._ROUTER_CONFIG_DIRECTORY / file) From a2a5d151f6859bf16fad26ca1c46e713d82cd82c Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 14:06:17 +0000 Subject: [PATCH 053/159] kw only parameters --- src/charm.py | 16 +++++++++------- src/mysql_shell.py | 3 ++- src/relations/database_provides.py | 27 ++++++++++++++++++--------- src/relations/database_requires.py | 2 ++ src/relations/tls.py | 3 +++ src/workload.py | 18 ++++++++++-------- 6 files changed, 44 insertions(+), 25 deletions(-) diff --git a/src/charm.py b/src/charm.py index f6b6fe8c0..eb54c565f 100755 --- a/src/charm.py +++ b/src/charm.py @@ -120,7 +120,7 @@ def _set_status(self) -> None: return self.unit.status = self._determine_status() - def _patch_service(self, name: str, ro_port: int, rw_port: int) -> None: + def _patch_service(self, *, name: str, ro_port: int, rw_port: int) -> None: """Patch Juju-created k8s service. The k8s service will be tied to pod-0 so that the service is auto cleaned by @@ -183,17 +183,19 @@ def _reconcile_database_relations(self, event=None) -> None: and self.workload.container_ready ): self.database_provides.reconcile_users( - event, - self.database_requires.relation.is_breaking(event), - self._endpoint, - self.workload.shell, + event=event, + event_is_database_requires_broken=self.database_requires.relation.is_breaking( + event + ), + endpoint=self._endpoint, + shell=self.workload.shell, ) if ( isinstance(self.workload, workload.AuthenticatedWorkload) and not self.database_requires.relation.is_breaking(event) and self.workload.container_ready ): - self.workload.enable(self.tls.certificate_saved) + self.workload.enable(tls=self.tls.certificate_saved) else: self.workload.disable() self._set_status() @@ -205,7 +207,7 @@ def _on_mysql_router_pebble_ready(self, _) -> None: def _on_leader_elected(self, _) -> None: """Patch existing k8s service to include read-write and read-only services.""" try: - self._patch_service(self.app.name, ro_port=6447, rw_port=6446) + self._patch_service(name=self.app.name, ro_port=6447, rw_port=6446) except lightkube.ApiError: logger.exception("Failed to patch k8s service") raise diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 1a0c690da..ab8002122 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -20,6 +20,7 @@ @dataclasses.dataclass class Shell: """MySQL Shell connected to MySQL cluster""" + _container: ops.Container _username: str _password: str @@ -52,7 +53,7 @@ def _generate_password() -> str: choices = string.ascii_letters + string.digits return "".join([secrets.choice(choices) for _ in range(_PASSWORD_LENGTH)]) - def create_application_database_and_user(self, username: str, database: str) -> str: + def create_application_database_and_user(self, *, username: str, database: str) -> str: """Create database and user for related database_provides application.""" logger.debug(f"Creating {database=} and {username=}") password = self._generate_password() diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 4fa561ac1..3f6e60318 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -17,6 +17,7 @@ @dataclasses.dataclass class _Relation: """Relation to one application charm""" + _relation: ops.Relation _interface: data_interfaces.DatabaseProvides @@ -52,7 +53,7 @@ def username(self) -> str: """Database username""" return f"relation-{self.id}" - def _set_databag(self, password: str, endpoint: str) -> None: + def _set_databag(self, *, password: str, endpoint: str) -> None: """Share connection information with application charm.""" read_write_endpoint = f"{endpoint}:6446" read_only_endpoint = f"{endpoint}:6447" @@ -73,12 +74,14 @@ def _delete_databag(self) -> None: self._local_databag.clear() logger.debug(f"Deleted databag {self.id=}") - def create_database_and_user(self, endpoint: str, shell: mysql_shell.Shell) -> None: + def create_database_and_user(self, *, endpoint: str, shell: mysql_shell.Shell) -> None: """Create database & user and update databag.""" - password = shell.create_application_database_and_user(self.username, self.database) - self._set_databag(password, endpoint) + password = shell.create_application_database_and_user( + username=self.username, database=self.database + ) + self._set_databag(password=password, endpoint=endpoint) - def delete_user(self, shell: mysql_shell.Shell) -> None: + def delete_user(self, *, shell: mysql_shell.Shell) -> None: """Delete user and update databag.""" self._delete_databag() shell.delete_user(self.username) @@ -87,6 +90,7 @@ def delete_user(self, shell: mysql_shell.Shell) -> None: @dataclasses.dataclass class RelationEndpoint: """Relation endpoint for application charm(s)""" + interface: data_interfaces.DatabaseProvides NAME = "database" @@ -95,7 +99,9 @@ class RelationEndpoint: def _relations(self) -> list[_Relation]: return [_Relation(relation, self.interface) for relation in self.interface.relations] - def _requested_users(self, event, event_is_database_requires_broken: bool) -> list[_Relation]: + def _requested_users( + self, *, event, event_is_database_requires_broken: bool + ) -> list[_Relation]: """Related application charms that have requested a database & user""" if event_is_database_requires_broken: # MySQL cluster connection is being removed; delete all users @@ -120,17 +126,20 @@ def missing_relation(self) -> bool: def reconcile_users( self, + *, event, event_is_database_requires_broken: bool, endpoint: str, shell: mysql_shell.Shell, ) -> None: """Create requested users and delete inactive users.""" - requested_users = self._requested_users(event, event_is_database_requires_broken) + requested_users = self._requested_users( + event=event, event_is_database_requires_broken=event_is_database_requires_broken + ) created_users = self._created_users for relation in requested_users: if relation not in created_users: - relation.create_database_and_user(endpoint, shell) + relation.create_database_and_user(endpoint=endpoint, shell=shell) for relation in created_users: if relation not in requested_users: - relation.delete_user(shell) + relation.delete_user(shell=shell) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 4d96ec38e..6f7f5bcc8 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -13,6 +13,7 @@ @dataclasses.dataclass class _Relation: """Relation to MySQL charm.""" + _interface: data_interfaces.DatabaseRequires @property @@ -61,6 +62,7 @@ def is_breaking(self, event): @dataclasses.dataclass class RelationEndpoint: """Relation endpoint for MySQL charm""" + interface: data_interfaces.DatabaseRequires NAME = "backend-database" diff --git a/src/relations/tls.py b/src/relations/tls.py index 7908a91cc..e13f0d16f 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -24,6 +24,7 @@ class _PeerUnitDatabag: """Peer relation unit databag""" + key: str requested_csr: str active_csr: str @@ -66,6 +67,7 @@ def clear(self) -> None: @dataclasses.dataclass class _Relation: """Relation to TLS certificate provider""" + _charm: charm.MySQLRouterOperatorCharm _interface: tls_certificates.TLSCertificatesRequiresV1 @@ -171,6 +173,7 @@ def request_certificate_renewal(self): class RelationEndpoint(ops.Object): """Relation endpoint and handlers for TLS certificate provider""" + NAME = "certificates" def __init__(self, charm_: charm.MySQLRouterOperatorCharm): diff --git a/src/workload.py b/src/workload.py index fe6370bbe..a37d9e6b5 100644 --- a/src/workload.py +++ b/src/workload.py @@ -21,6 +21,7 @@ @dataclasses.dataclass class Workload: """MySQL Router workload""" + _container: ops.Container CONTAINER_NAME = "mysql-router" @@ -55,6 +56,7 @@ def version(self) -> str: @dataclasses.dataclass class AuthenticatedWorkload(Workload): """Workload with connection to MySQL cluster""" + _admin_username: str _admin_password: str _host: str @@ -78,7 +80,7 @@ def _get_layer(self, service_info: dict) -> ops.pebble.Layer: } ) - def _get_active_layer(self, password: str, tls: bool) -> ops.pebble.Layer: + def _get_active_layer(self, *, password: str, tls: bool) -> ops.pebble.Layer: """Create layer with startup enabled. Args: @@ -146,7 +148,7 @@ def _wait_until_mysql_router_ready() -> None: raise BaseException() sock.close() - def enable(self, tls: bool) -> None: + def enable(self, *, tls: bool) -> None: """Start and enable MySQL Router service.""" if self._enabled: # If the host or port changes, MySQL Router will receive topology change notifications from MySQL @@ -154,7 +156,7 @@ def enable(self, tls: bool) -> None: return logger.debug(f"Enabling MySQL Router service {tls=}, {self._host=}, {self._port=}") router_password = self.shell.create_mysql_router_user(self._ROUTER_USERNAME) - self._update_layer(self._get_active_layer(router_password, tls)) + self._update_layer(self._get_active_layer(password=router_password, tls=tls)) logger.debug(f"Enabled MySQL Router service {tls=}, {self._host=}, {self._port=}") self._wait_until_mysql_router_ready() # TODO: wait until mysql router ready? https://github.com/canonical/mysql-router-k8s-operator/blob/45cf3be44f27476a0371c67d50d7a0193c0fadc2/src/charm.py#L219 @@ -168,11 +170,11 @@ def disable(self) -> None: self._update_layer(self._inactive_layer) logger.debug("Disabled MySQL Router service") - def _restart(self, tls: bool) -> None: + def _restart(self, *, tls: bool) -> None: """Restart MySQL Router to enable or disable TLS.""" logger.debug("Restarting MySQL Router service") password = self._service.environment["MYSQL_PASSWORD"] - self._update_layer(self._get_active_layer(password, tls)) + self._update_layer(self._get_active_layer(password=password, tls=tls)) logger.debug("Restarted MySQL Router service") def _write_file(self, path: pathlib.Path, content: str) -> None: @@ -214,7 +216,7 @@ def _tls_config_file(self) -> str: ) return config_string - def enable_tls(self, key: str, certificate: str): + def enable_tls(self, *, key: str, certificate: str): """Enable TLS and restart MySQL Router service.""" logger.debug("Enabling TLS") self._write_file( @@ -223,7 +225,7 @@ def enable_tls(self, key: str, certificate: str): self._write_file(self._ROUTER_CONFIG_DIRECTORY / self._TLS_KEY_FILE, key) self._write_file(self._ROUTER_CONFIG_DIRECTORY / self._TLS_CERTIFICATE_FILE, certificate) if self._enabled: - self._restart(True) + self._restart(tls=True) logger.debug("Enabled TLS") def disable_tls(self) -> None: @@ -232,5 +234,5 @@ def disable_tls(self) -> None: for file in [self._TLS_CONFIG_FILE, self._TLS_KEY_FILE, self._TLS_CERTIFICATE_FILE]: self._delete_file(self._ROUTER_CONFIG_DIRECTORY / file) if self._enabled: - self._restart(False) + self._restart(tls=False) logger.debug("Disabled TLS") From 048d5a548ec0e1b658b305dab06d55dbc3f790a8 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 14:28:34 +0000 Subject: [PATCH 054/159] kw only dataclasses --- src/charm.py | 14 +++++++------- src/mysql_shell.py | 2 +- src/relations/database_provides.py | 7 +++++-- src/relations/tls.py | 4 ++-- src/workload.py | 10 +++++++--- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/charm.py b/src/charm.py index eb54c565f..a5100c315 100755 --- a/src/charm.py +++ b/src/charm.py @@ -33,7 +33,7 @@ def __init__(self, *args) -> None: data_interfaces.DatabaseRequires( self, relation_name=relations.database_requires.RelationEndpoint.NAME, - # HACK: mysqlrouter needs a user, but not a database + # HACK: MySQL Router needs a user, but not a database # Use the DatabaseRequires interface to get a user; disregard the database database_name="_unused_mysqlrouter_database", extra_user_roles="mysqlrouter", @@ -82,13 +82,13 @@ def workload(self): container = self.unit.get_container(workload.Workload.CONTAINER_NAME) if self.database_requires.relation: return workload.AuthenticatedWorkload( - container, - self.database_requires.relation.username, - self.database_requires.relation.password, - self.database_requires.relation.host, - self.database_requires.relation.port, + _container=container, + _admin_username=self.database_requires.relation.username, + _admin_password=self.database_requires.relation.password, + _host=self.database_requires.relation.host, + _port=self.database_requires.relation.port, ) - return workload.Workload(container) + return workload.Workload(_container=container) @property def _endpoint(self) -> str: diff --git a/src/mysql_shell.py b/src/mysql_shell.py index ab8002122..22e889dfb 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) -@dataclasses.dataclass +@dataclasses.dataclass(kw_only=True) class Shell: """MySQL Shell connected to MySQL cluster""" diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 3f6e60318..0758491ef 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) -@dataclasses.dataclass +@dataclasses.dataclass(kw_only=True) class _Relation: """Relation to one application charm""" @@ -97,7 +97,10 @@ class RelationEndpoint: @property def _relations(self) -> list[_Relation]: - return [_Relation(relation, self.interface) for relation in self.interface.relations] + return [ + _Relation(_relation=relation, _interface=self.interface) + for relation in self.interface.relations + ] def _requested_users( self, *, event, event_is_database_requires_broken: bool diff --git a/src/relations/tls.py b/src/relations/tls.py index e13f0d16f..2694c9219 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -64,7 +64,7 @@ def clear(self) -> None: delattr(self, name) -@dataclasses.dataclass +@dataclasses.dataclass(kw_only=True) class _Relation: """Relation to TLS certificate provider""" @@ -203,7 +203,7 @@ def __init__(self, charm_: charm.MySQLRouterOperatorCharm): def _relation(self) -> typing.Optional[_Relation]: if not self._charm.model.get_relation(self.NAME): return - return _Relation(self._charm, self._interface) + return _Relation(_charm=self._charm, _interface=self._interface) @property def certificate_saved(self) -> bool: diff --git a/src/workload.py b/src/workload.py index a37d9e6b5..2aa17ff95 100644 --- a/src/workload.py +++ b/src/workload.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) -@dataclasses.dataclass +@dataclasses.dataclass(kw_only=True) class Workload: """MySQL Router workload""" @@ -53,7 +53,7 @@ def version(self) -> str: return "" -@dataclasses.dataclass +@dataclasses.dataclass(kw_only=True) class AuthenticatedWorkload(Workload): """Workload with connection to MySQL cluster""" @@ -126,7 +126,11 @@ def _update_layer(self, layer: ops.pebble.Layer) -> None: @property def shell(self) -> mysql_shell.Shell: return mysql_shell.Shell( - self._container, self._admin_username, self._admin_password, self._host, self._port + _container=self._container, + _username=self._admin_username, + _password=self._admin_password, + _host=self._host, + _port=self._port, ) @staticmethod From a11142796e5324cb745b270c4c8499ca66bc6bd4 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 14:42:27 +0000 Subject: [PATCH 055/159] fix docstrings --- src/relations/database_provides.py | 4 ++-- src/relations/database_requires.py | 10 +++++----- src/relations/tls.py | 2 +- src/workload.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 0758491ef..b0cbd267f 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -23,12 +23,12 @@ class _Relation: @property def _local_databag(self) -> ops.RelationDataContent: - """MySQL Router charm databag.""" + """MySQL Router charm databag""" return self._relation.data[self._interface.local_app] @property def _remote_databag(self) -> dict: - """MySQL charm databag.""" + """MySQL charm databag""" return self._interface.fetch_relation_data()[self.id] @property diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 6f7f5bcc8..233feaa2c 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -12,7 +12,7 @@ @dataclasses.dataclass class _Relation: - """Relation to MySQL charm.""" + """Relation to MySQL charm""" _interface: data_interfaces.DatabaseRequires @@ -24,24 +24,24 @@ def _id(self) -> int: @property def _remote_databag(self) -> dict: - """MySQL charm databag.""" + """MySQL charm databag""" return self._interface.fetch_relation_data()[self._id] @property def _endpoint(self) -> str: - """MySQL cluster primary endpoint.""" + """MySQL cluster primary endpoint""" endpoints = self._remote_databag["endpoints"].split(",") assert len(endpoints) == 1 return endpoints[0] @property def host(self) -> str: - """MySQL cluster primary host.""" + """MySQL cluster primary host""" return self._endpoint.split(":")[0] @property def port(self) -> str: - """MySQL cluster primary port.""" + """MySQL cluster primary port""" return self._endpoint.split(":")[1] @property diff --git a/src/relations/tls.py b/src/relations/tls.py index 2694c9219..08ebf0ece 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -213,7 +213,7 @@ def certificate_saved(self) -> bool: return self._relation.certificate_saved def _on_set_tls_private_key(self, event: ops.ActionEvent) -> None: - """Action for setting a TLS private key.""" + """Handle action to set unit TLS private key.""" if self._relation is None: event.fail("No TLS relation available.") return diff --git a/src/workload.py b/src/workload.py index 2aa17ff95..8c8a15ae2 100644 --- a/src/workload.py +++ b/src/workload.py @@ -108,7 +108,7 @@ def _get_active_layer(self, *, password: str, tls: bool) -> ops.pebble.Layer: @property def _inactive_layer(self) -> ops.pebble.Layer: - """Create layer with startup disabled.""" + """Layer with startup disabled""" return self._get_layer( { "override": "replace", From 779b45ee4e04b67c020e5d65b3777285fd8a46b4 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 14:59:48 +0000 Subject: [PATCH 056/159] cleanup --- src/charm.py | 4 ++-- src/relations/database_provides.py | 22 +++++++++++----------- src/relations/tls.py | 3 ++- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/charm.py b/src/charm.py index a5100c315..595185272 100755 --- a/src/charm.py +++ b/src/charm.py @@ -92,7 +92,7 @@ def workload(self): @property def _endpoint(self) -> str: - """K8s endpoint for the charm""" + """K8s endpoint for MySQL Router""" return f"{self.model.app.name}.{self.model.name}.svc.cluster.local" def _determine_status(self) -> ops.StatusBase: @@ -187,7 +187,7 @@ def _reconcile_database_relations(self, event=None) -> None: event_is_database_requires_broken=self.database_requires.relation.is_breaking( event ), - endpoint=self._endpoint, + router_endpoint=self._endpoint, shell=self.workload.shell, ) if ( diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index b0cbd267f..e6c65b513 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -21,6 +21,10 @@ class _Relation: _relation: ops.Relation _interface: data_interfaces.DatabaseProvides + @property + def id(self) -> int: + return self._relation.id + @property def _local_databag(self) -> ops.RelationDataContent: """MySQL Router charm databag""" @@ -31,10 +35,6 @@ def _remote_databag(self) -> dict: """MySQL charm databag""" return self._interface.fetch_relation_data()[self.id] - @property - def id(self) -> int: - return self._relation.id - @property def user_created(self) -> bool: """Whether database user has been shared with application charm""" @@ -53,10 +53,10 @@ def username(self) -> str: """Database username""" return f"relation-{self.id}" - def _set_databag(self, *, password: str, endpoint: str) -> None: + def _set_databag(self, *, password: str, router_endpoint: str) -> None: """Share connection information with application charm.""" - read_write_endpoint = f"{endpoint}:6446" - read_only_endpoint = f"{endpoint}:6447" + read_write_endpoint = f"{router_endpoint}:6446" + read_only_endpoint = f"{router_endpoint}:6447" logger.debug( f"Setting databag {self.id=} {self.database=}, {self.username=}, {read_write_endpoint=}, {read_only_endpoint=}" ) @@ -74,12 +74,12 @@ def _delete_databag(self) -> None: self._local_databag.clear() logger.debug(f"Deleted databag {self.id=}") - def create_database_and_user(self, *, endpoint: str, shell: mysql_shell.Shell) -> None: + def create_database_and_user(self, *, router_endpoint: str, shell: mysql_shell.Shell) -> None: """Create database & user and update databag.""" password = shell.create_application_database_and_user( username=self.username, database=self.database ) - self._set_databag(password=password, endpoint=endpoint) + self._set_databag(password=password, router_endpoint=router_endpoint) def delete_user(self, *, shell: mysql_shell.Shell) -> None: """Delete user and update databag.""" @@ -132,7 +132,7 @@ def reconcile_users( *, event, event_is_database_requires_broken: bool, - endpoint: str, + router_endpoint: str, shell: mysql_shell.Shell, ) -> None: """Create requested users and delete inactive users.""" @@ -142,7 +142,7 @@ def reconcile_users( created_users = self._created_users for relation in requested_users: if relation not in created_users: - relation.create_database_and_user(endpoint=endpoint, shell=shell) + relation.create_database_and_user(router_endpoint=router_endpoint, shell=shell) for relation in created_users: if relation not in requested_users: relation.delete_user(shell=shell) diff --git a/src/relations/tls.py b/src/relations/tls.py index 08ebf0ece..90dc7d63d 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -26,10 +26,11 @@ class _PeerUnitDatabag: """Peer relation unit databag""" key: str + # CSR stands for certificate signing request requested_csr: str active_csr: str certificate: str - ca: str + ca: str # Certificate authority chain: str def __init__(self, databag: ops.RelationDataContent) -> None: From 8bc905973af2f836ce46f77a2dc9da753b7857dc Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 15:40:23 +0000 Subject: [PATCH 057/159] add debug logging --- src/charm.py | 10 ++++++++++ src/relations/database_provides.py | 7 +++++++ src/relations/tls.py | 12 ++++++++++++ src/workload.py | 6 ++++-- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/charm.py b/src/charm.py index 595185272..a2419f198 100755 --- a/src/charm.py +++ b/src/charm.py @@ -119,6 +119,7 @@ def _set_status(self) -> None: ) and not self.unit.status.message.startswith("Missing relation"): return self.unit.status = self._determine_status() + logger.debug(f"Set status to {self.unit.status}") def _patch_service(self, *, name: str, ro_port: int, rw_port: int) -> None: """Patch Juju-created k8s service. @@ -131,6 +132,7 @@ def _patch_service(self, *, name: str, ro_port: int, rw_port: int) -> None: ro_port: The read only port. rw_port: The read write port. """ + logger.debug(f"Patching k8s service {name=}, {ro_port=}, {rw_port=}") client = lightkube.Client() pod0 = client.get( res=lightkube.resources.core_v1.Pod, @@ -170,6 +172,7 @@ def _patch_service(self, *, name: str, ro_port: int, rw_port: int) -> None: force=True, field_manager=self.model.app.name, ) + logger.debug(f"Patched k8s service {name=}, {ro_port=}, {rw_port=}") # ======================= # Handlers @@ -177,6 +180,13 @@ def _patch_service(self, *, name: str, ro_port: int, rw_port: int) -> None: def _reconcile_database_relations(self, event=None) -> None: """Handle database requires/provides events.""" + logger.debug( + "State of reconcile " + f"{self.unit.is_leader()=}, " + f"{isinstance(self.workload, workload.AuthenticatedWorkload)=}, " + f"{self.database_requires.relation.is_breaking(event)=}, " + f"{self.workload.container_ready=}" + ) if ( self.unit.is_leader() and isinstance(self.workload, workload.AuthenticatedWorkload) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index e6c65b513..26e461c49 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -136,13 +136,20 @@ def reconcile_users( shell: mysql_shell.Shell, ) -> None: """Create requested users and delete inactive users.""" + logger.debug( + f"Reconciling users {event=}, {event_is_database_requires_broken=}, {router_endpoint=}" + ) requested_users = self._requested_users( event=event, event_is_database_requires_broken=event_is_database_requires_broken ) created_users = self._created_users + logger.debug(f"State of reconcile users {requested_users=}, {created_users=}") for relation in requested_users: if relation not in created_users: relation.create_database_and_user(router_endpoint=router_endpoint, shell=shell) for relation in created_users: if relation not in requested_users: relation.delete_user(shell=shell) + logger.debug( + f"Reconciled users {event=}, {event_is_database_requires_broken=}, {router_endpoint=}" + ) diff --git a/src/relations/tls.py b/src/relations/tls.py index 90dc7d63d..11813c162 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -106,10 +106,12 @@ def save_certificate(self, event: tls_certificates.CertificateAvailableEvent) -> # Workaround for https://github.com/canonical/tls-certificates-operator/issues/34 logger.debug("TLS certificate already saved.") return + logger.debug(f"Saving TLS certificate {event=}") self.peer_unit_databag.certificate = event.certificate self.peer_unit_databag.ca = event.ca self.peer_unit_databag.chain = json.dumps(event.chain) self.peer_unit_databag.active_csr = self.peer_unit_databag.requested_csr + logger.debug(f"Saved TLS certificate {event=}") self._charm.workload.enable_tls( self.peer_unit_databag.key, self.peer_unit_databag.certificate ) @@ -152,6 +154,7 @@ def _generate_csr(self, key: bytes) -> bytes: def request_certificate_creation(self, internal_key: str = None): """Request new TLS certificate from related provider charm.""" + logger.debug("Requesting TLS certificate creation") if internal_key: key = self._parse_tls_key(internal_key) else: @@ -160,9 +163,11 @@ def request_certificate_creation(self, internal_key: str = None): self._interface.request_certificate_creation(certificate_signing_request=csr) self.peer_unit_databag.key = key.decode("utf-8") self.peer_unit_databag.requested_csr = csr.decode("utf-8") + logger.debug(f"Requested TLS certificate creation {self.peer_unit_databag.requested_csr=}") def request_certificate_renewal(self): """Request TLS certificate renewal from related provider charm.""" + logger.debug(f"Requesting TLS certificate renewal {self.peer_unit_databag.active_csr=}") old_csr = self.peer_unit_databag.active_csr.encode("utf-8") key = self.peer_unit_databag.key.encode("utf-8") new_csr = self._generate_csr(key) @@ -170,6 +175,7 @@ def request_certificate_renewal(self): old_certificate_signing_request=old_csr, new_certificate_signing_request=new_csr ) self.peer_unit_databag.requested_csr = new_csr.decode("utf-8") + logger.debug(f"Requested TLS certificate renewal {self.peer_unit_databag.requested_csr=}") class RelationEndpoint(ops.Object): @@ -215,8 +221,10 @@ def certificate_saved(self) -> bool: def _on_set_tls_private_key(self, event: ops.ActionEvent) -> None: """Handle action to set unit TLS private key.""" + logger.debug("Handling set TLS private key action") if self._relation is None: event.fail("No TLS relation available.") + logger.debug("Unable to set TLS private key: no TLS relation available") return try: self._relation.request_certificate_creation(event.params.get("internal-key")) @@ -224,6 +232,8 @@ def _on_set_tls_private_key(self, event: ops.ActionEvent) -> None: event.fail(f"Failed to request certificate: {e}") logger.exception("Failed to set TLS private key via action") raise + else: + logger.debug("Handled set TLS private key action") def _on_tls_relation_joined(self, _) -> None: """Request certificate when TLS relation joined.""" @@ -231,8 +241,10 @@ def _on_tls_relation_joined(self, _) -> None: def _on_tls_relation_broken(self, _) -> None: """Delete TLS certificate.""" + logger.debug("Deleting TLS certificate") self._relation.peer_unit_databag.clear() self._charm.workload.disable_tls() + logger.debug("Deleted TLS certificate") def _on_certificate_available(self, event: tls_certificates.CertificateAvailableEvent) -> None: """Save TLS certificate.""" diff --git a/src/workload.py b/src/workload.py index 8c8a15ae2..41ba4f10a 100644 --- a/src/workload.py +++ b/src/workload.py @@ -176,10 +176,10 @@ def disable(self) -> None: def _restart(self, *, tls: bool) -> None: """Restart MySQL Router to enable or disable TLS.""" - logger.debug("Restarting MySQL Router service") + logger.debug(f"Restarting MySQL Router service {tls=}") password = self._service.environment["MYSQL_PASSWORD"] self._update_layer(self._get_active_layer(password=password, tls=tls)) - logger.debug("Restarted MySQL Router service") + logger.debug(f"Restarted MySQL Router service {tls=}") def _write_file(self, path: pathlib.Path, content: str) -> None: """Write content to file. @@ -195,6 +195,7 @@ def _write_file(self, path: pathlib.Path, content: str) -> None: user=self._ROUTER_USERNAME, group=self._ROUTER_USERNAME, ) + logger.debug(f"Wrote file {path=}") def _delete_file(self, path: pathlib.Path) -> None: """Delete file. @@ -205,6 +206,7 @@ def _delete_file(self, path: pathlib.Path) -> None: path = str(path) if self._container.exists(path): self._container.remove_path(path) + logger.debug(f"Deleted file {path=}") @property def _tls_config_file(self) -> str: From 4a8e759c3c1d97127a1448de5ebb1f9454f888ef Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 15:58:46 +0000 Subject: [PATCH 058/159] logs --- src/relations/tls.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/relations/tls.py b/src/relations/tls.py index 11813c162..1cd32b3fe 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -19,7 +19,6 @@ _PEER_RELATION_ENDPOINT_NAME = "mysql-router-peers" logger = logging.getLogger(__name__) -# TODO: fix logging levels class _PeerUnitDatabag: @@ -253,7 +252,7 @@ def _on_certificate_available(self, event: tls_certificates.CertificateAvailable def _on_certificate_expiring(self, event: tls_certificates.CertificateExpiringEvent) -> None: """Request the new certificate when old certificate is expiring.""" if event.certificate != self._relation.peer_unit_databag.certificate: - logger.error("An unknown certificate expiring.") + logger.error("Unknown certificate expiring") return self._relation.request_certificate_renewal() From 83fcfacfbad90f20817739920f6e4f29d143bb8d Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 16:00:33 +0000 Subject: [PATCH 059/159] update maintainers in metadata.yaml --- metadata.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/metadata.yaml b/metadata.yaml index 146c5f617..abb713fbe 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -5,6 +5,7 @@ display-name: MySQL Router maintainers: - Paulo Machado - Shayan Patel + - Carl Csaposs description: | K8S charmed operator for mysql-router. summary: | From 4d462ba222dbb1c773ec8bcf33eac278f2af921c Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 16:14:56 +0000 Subject: [PATCH 060/159] change password on rebootstrap instead of grabbing it from service --- src/mysql_shell.py | 8 ++++++++ src/workload.py | 14 +++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 22e889dfb..53452a3f4 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -84,6 +84,14 @@ def create_mysql_router_user(self, username: str) -> str: logger.debug(f"Created router {username=}") return password + def change_mysql_router_user_password(self, username: str) -> str: + """Change MySQL Router service user password.""" + logger.debug(f"Changing router password {username=}") + password = self._generate_password() + self._run_sql([f"ALTER USER `{username}` IDENTIFIED BY '{password}'"]) + logger.debug(f"Changed router password {username=}") + return password + def delete_user(self, username: str) -> None: logger.debug(f"Deleting {username=}") self._run_sql([f"DROP USER `{username}`"]) diff --git a/src/workload.py b/src/workload.py index 41ba4f10a..339f4daf1 100644 --- a/src/workload.py +++ b/src/workload.py @@ -31,17 +31,13 @@ class Workload: def container_ready(self) -> bool: return self._container.can_connect() - @property - def _service(self) -> typing.Optional[ops.pebble.Service]: - """MySQL Router service""" - return self._container.get_services(self._SERVICE_NAME).get(self._SERVICE_NAME) - @property def _enabled(self) -> bool: """Service status""" - if self._service is None: + service = self._container.get_services(self._SERVICE_NAME).get(self._SERVICE_NAME) + if service is None: return False - return self._service.startup == "enabled" + return service.startup == "enabled" @property def version(self) -> str: @@ -177,8 +173,8 @@ def disable(self) -> None: def _restart(self, *, tls: bool) -> None: """Restart MySQL Router to enable or disable TLS.""" logger.debug(f"Restarting MySQL Router service {tls=}") - password = self._service.environment["MYSQL_PASSWORD"] - self._update_layer(self._get_active_layer(password=password, tls=tls)) + router_password = self.shell.change_mysql_router_user_password(self._ROUTER_USERNAME) + self._update_layer(self._get_active_layer(password=router_password, tls=tls)) logger.debug(f"Restarted MySQL Router service {tls=}") def _write_file(self, path: pathlib.Path, content: str) -> None: From 0d2fc02072e7b1c98b82f5ad223410fea9f171fe Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 16:19:39 +0000 Subject: [PATCH 061/159] Switch to mysql user --- src/workload.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/workload.py b/src/workload.py index 339f4daf1..8b2322d4f 100644 --- a/src/workload.py +++ b/src/workload.py @@ -58,7 +58,7 @@ class AuthenticatedWorkload(Workload): _host: str _port: str - _ROUTER_USERNAME = "mysqlrouter" + _UNIX_USERNAME = "mysql" _ROUTER_CONFIG_DIRECTORY = pathlib.Path("/tmp/mysqlrouter") _TLS_CONFIG_FILE = "tls.conf" _TLS_KEY_FILE = "custom-key.pem" @@ -96,7 +96,7 @@ def _get_active_layer(self, *, password: str, tls: bool) -> ops.pebble.Layer: "environment": { "MYSQL_HOST": self._host, "MYSQL_PORT": self._port, - "MYSQL_USER": self._ROUTER_USERNAME, + "MYSQL_USER": self._UNIX_USERNAME, "MYSQL_PASSWORD": password, }, } @@ -155,7 +155,7 @@ def enable(self, *, tls: bool) -> None: # Therefore, if the host or port changes, we do not need to restart MySQL Router return logger.debug(f"Enabling MySQL Router service {tls=}, {self._host=}, {self._port=}") - router_password = self.shell.create_mysql_router_user(self._ROUTER_USERNAME) + router_password = self.shell.create_mysql_router_user(self._UNIX_USERNAME) self._update_layer(self._get_active_layer(password=router_password, tls=tls)) logger.debug(f"Enabled MySQL Router service {tls=}, {self._host=}, {self._port=}") self._wait_until_mysql_router_ready() @@ -166,14 +166,14 @@ def disable(self) -> None: if not self._enabled: return logger.debug("Disabling MySQL Router service") - self.shell.delete_user(self._ROUTER_USERNAME) + self.shell.delete_user(self._UNIX_USERNAME) self._update_layer(self._inactive_layer) logger.debug("Disabled MySQL Router service") def _restart(self, *, tls: bool) -> None: """Restart MySQL Router to enable or disable TLS.""" logger.debug(f"Restarting MySQL Router service {tls=}") - router_password = self.shell.change_mysql_router_user_password(self._ROUTER_USERNAME) + router_password = self.shell.change_mysql_router_user_password(self._UNIX_USERNAME) self._update_layer(self._get_active_layer(password=router_password, tls=tls)) logger.debug(f"Restarted MySQL Router service {tls=}") @@ -188,8 +188,8 @@ def _write_file(self, path: pathlib.Path, content: str) -> None: str(path), content, permissions=0o600, - user=self._ROUTER_USERNAME, - group=self._ROUTER_USERNAME, + user=self._UNIX_USERNAME, + group=self._UNIX_USERNAME, ) logger.debug(f"Wrote file {path=}") From ff031c19396708b8072ec2abee1c73857572e7ef Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 16:21:59 +0000 Subject: [PATCH 062/159] fixup restart --- src/workload.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/workload.py b/src/workload.py index 8b2322d4f..836526d35 100644 --- a/src/workload.py +++ b/src/workload.py @@ -172,10 +172,11 @@ def disable(self) -> None: def _restart(self, *, tls: bool) -> None: """Restart MySQL Router to enable or disable TLS.""" - logger.debug(f"Restarting MySQL Router service {tls=}") + logger.debug(f"Restarting MySQL Router service {tls=}, {self._host=}, {self._port=}") router_password = self.shell.change_mysql_router_user_password(self._UNIX_USERNAME) self._update_layer(self._get_active_layer(password=router_password, tls=tls)) - logger.debug(f"Restarted MySQL Router service {tls=}") + logger.debug(f"Restarted MySQL Router service {tls=}, {self._host=}, {self._port=}") + self._wait_until_mysql_router_ready() def _write_file(self, path: pathlib.Path, content: str) -> None: """Write content to file. From 4aaa7ddb1707fdbf55374f124cbe20cc8aba04af Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 16:25:24 +0000 Subject: [PATCH 063/159] add _bootstrap_router() --- src/workload.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/workload.py b/src/workload.py index 836526d35..8a9933393 100644 --- a/src/workload.py +++ b/src/workload.py @@ -148,16 +148,21 @@ def _wait_until_mysql_router_ready() -> None: raise BaseException() sock.close() + def _bootstrap_router(self, *, password: str, tls: bool) -> None: + logger.debug(f"Bootstrapping router {tls=}, {self._host=}, {self._port=}") + self._update_layer(self._get_active_layer(password=password, tls=tls)) + logger.debug(f"Bootstrapped router {tls=}, {self._host=}, {self._port=}") + def enable(self, *, tls: bool) -> None: """Start and enable MySQL Router service.""" if self._enabled: # If the host or port changes, MySQL Router will receive topology change notifications from MySQL # Therefore, if the host or port changes, we do not need to restart MySQL Router return - logger.debug(f"Enabling MySQL Router service {tls=}, {self._host=}, {self._port=}") + logger.debug("Enabling MySQL Router service") router_password = self.shell.create_mysql_router_user(self._UNIX_USERNAME) - self._update_layer(self._get_active_layer(password=router_password, tls=tls)) - logger.debug(f"Enabled MySQL Router service {tls=}, {self._host=}, {self._port=}") + self._bootstrap_router(password=router_password, tls=tls) + logger.debug("Enabled MySQL Router service") self._wait_until_mysql_router_ready() # TODO: wait until mysql router ready? https://github.com/canonical/mysql-router-k8s-operator/blob/45cf3be44f27476a0371c67d50d7a0193c0fadc2/src/charm.py#L219 @@ -172,10 +177,10 @@ def disable(self) -> None: def _restart(self, *, tls: bool) -> None: """Restart MySQL Router to enable or disable TLS.""" - logger.debug(f"Restarting MySQL Router service {tls=}, {self._host=}, {self._port=}") + logger.debug("Restarting MySQL Router service") router_password = self.shell.change_mysql_router_user_password(self._UNIX_USERNAME) - self._update_layer(self._get_active_layer(password=router_password, tls=tls)) - logger.debug(f"Restarted MySQL Router service {tls=}, {self._host=}, {self._port=}") + self._bootstrap_router(password=router_password, tls=tls) + logger.debug("Restarted MySQL Router service") self._wait_until_mysql_router_ready() def _write_file(self, path: pathlib.Path, content: str) -> None: From 80f1051408db8cd30a84325edadb95d0fd4405da Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 17:09:57 +0000 Subject: [PATCH 064/159] Port changes from https://github.com/canonical/mysql-router-k8s-operator/pull/55 on a991939 --- metadata.yaml | 3 +- src/workload.py | 103 +++++++++++++++++++++++++----------------------- 2 files changed, 54 insertions(+), 52 deletions(-) diff --git a/metadata.yaml b/metadata.yaml index abb713fbe..48bf2cf27 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -31,7 +31,6 @@ resources: mysql-router-image: type: oci-image description: OCI image for mysql-router - # TODO: replace with canonical maintained image - upstream-source: dataplatformoci/mysql-router:8.0-22.04_edge + upstream-source: ghcr.io/canonical/charmed-mysql@sha256:924cb913733002e0936a36b6edab0bb8a4d72f1b92be0e0b7a933e797b7ef215 assumes: - k8s-api diff --git a/src/workload.py b/src/workload.py index 8a9933393..4a418133f 100644 --- a/src/workload.py +++ b/src/workload.py @@ -59,63 +59,45 @@ class AuthenticatedWorkload(Workload): _port: str _UNIX_USERNAME = "mysql" - _ROUTER_CONFIG_DIRECTORY = pathlib.Path("/tmp/mysqlrouter") + _ROUTER_USERNAME = "mysqlrouter" + _ROUTER_CONFIG_DIRECTORY = pathlib.Path("/etc/mysqlrouter") + _ROUTER_CONFIG_FILE = "mysqlrouter.conf" _TLS_CONFIG_FILE = "tls.conf" _TLS_KEY_FILE = "custom-key.pem" _TLS_CERTIFICATE_FILE = "custom-certificate.pem" - def _get_layer(self, service_info: dict) -> ops.pebble.Layer: - """Create layer.""" - return ops.pebble.Layer( - { - "summary": "mysql router layer", - "description": "the pebble config layer for mysql router", - "services": { - self._SERVICE_NAME: service_info, - }, - } - ) - - def _get_active_layer(self, *, password: str, tls: bool) -> ops.pebble.Layer: - """Create layer with startup enabled. + def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: + """Update and restart services. Args: - password: MySQL Router user password - tls: Whether TLS is enabled + enabled: Whether MySQL Router service is enabled + tls: Whether TLS is enabled. Required if enabled=True """ - if tls: - command = f"/run.sh mysqlrouter --extra-config {self._ROUTER_CONFIG_DIRECTORY / self._TLS_CONFIG_FILE}" + if enabled: + command = ( + f"mysqlrouter --config {self._ROUTER_CONFIG_DIRECTORY / self._ROUTER_CONFIG_FILE}" + ) + assert tls is not None, "`tls` argument required when enabled=True" + if tls: + command = f"{command} --extra-config {self._ROUTER_CONFIG_DIRECTORY / self._TLS_CONFIG_FILE}" else: - command = "/run.sh mysqlrouter" - return self._get_layer( + command = "" + layer = ops.pebble.Layer( { - "override": "replace", - "summary": "mysql router", - "command": command, - "startup": "enabled", - "environment": { - "MYSQL_HOST": self._host, - "MYSQL_PORT": self._port, - "MYSQL_USER": self._UNIX_USERNAME, - "MYSQL_PASSWORD": password, + "summary": "mysql router layer", + "description": "the pebble config layer for mysql router", + "services": { + self._SERVICE_NAME: { + "override": "replace", + "summary": "mysql router", + "command": command, + "startup": "enabled" if enabled else "disabled", + "user": self._UNIX_USERNAME, + "group": self._UNIX_USERNAME, + }, }, } ) - - @property - def _inactive_layer(self) -> ops.pebble.Layer: - """Layer with startup disabled""" - return self._get_layer( - { - "override": "replace", - "summary": "mysql router", - "command": "", - "startup": "disabled", - } - ) - - def _update_layer(self, layer: ops.pebble.Layer) -> None: - """Update and restart services.""" self._container.add_layer(self._SERVICE_NAME, layer, combine=True) self._container.replan() @@ -149,8 +131,29 @@ def _wait_until_mysql_router_ready() -> None: sock.close() def _bootstrap_router(self, *, password: str, tls: bool) -> None: + """Bootstrap MySQL Router and enable service.""" logger.debug(f"Bootstrapping router {tls=}, {self._host=}, {self._port=}") - self._update_layer(self._get_active_layer(password=password, tls=tls)) + try: + # Bootstrap MySQL Router + process = self._container.exec( + [ + "mysqlrouter", + "--bootstrap", + f"{self._ROUTER_USERNAME}:{password}@{self._host}:{self._port}", + "--user", + self._UNIX_USERNAME, + "--conf-set-option", + "http_server.bind_address=127.0.0.1", + "--force", # TODO: needed? + ] + ) + process.wait_output() + except ops.pebble.ExecError as e: + logger.exception(f"Failed to bootstrap router\nstderr:\n{e.stderr}\n") + raise + # Enable service + self._update_layer(enabled=True, tls=tls) + logger.debug(f"Bootstrapped router {tls=}, {self._host=}, {self._port=}") def enable(self, *, tls: bool) -> None: @@ -160,7 +163,7 @@ def enable(self, *, tls: bool) -> None: # Therefore, if the host or port changes, we do not need to restart MySQL Router return logger.debug("Enabling MySQL Router service") - router_password = self.shell.create_mysql_router_user(self._UNIX_USERNAME) + router_password = self.shell.create_mysql_router_user(self._ROUTER_USERNAME) self._bootstrap_router(password=router_password, tls=tls) logger.debug("Enabled MySQL Router service") self._wait_until_mysql_router_ready() @@ -171,14 +174,14 @@ def disable(self) -> None: if not self._enabled: return logger.debug("Disabling MySQL Router service") - self.shell.delete_user(self._UNIX_USERNAME) - self._update_layer(self._inactive_layer) + self.shell.delete_user(self._ROUTER_USERNAME) + self._update_layer(enabled=False) logger.debug("Disabled MySQL Router service") def _restart(self, *, tls: bool) -> None: """Restart MySQL Router to enable or disable TLS.""" logger.debug("Restarting MySQL Router service") - router_password = self.shell.change_mysql_router_user_password(self._UNIX_USERNAME) + router_password = self.shell.change_mysql_router_user_password(self._ROUTER_USERNAME) self._bootstrap_router(password=router_password, tls=tls) logger.debug("Restarted MySQL Router service") self._wait_until_mysql_router_ready() From 384e77a7968e3a0634c78de50e2e0dcab06041d3 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 17:23:26 +0000 Subject: [PATCH 065/159] Implement mysql_shell _run_commands() --- src/mysql_shell.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 53452a3f4..ba6e4e0cd 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -27,17 +27,24 @@ class Shell: _host: str _port: str + _TEMPORARY_SCRIPT_FILE = "/tmp/script.py" + def _run_commands(self, commands: list[str]) -> None: """Connect to MySQL cluster and run commands.""" commands.insert( 0, f"shell.connect('{self._username}:{self._password}@{self._host}:{self._port}" ) + self._container.push(self._TEMPORARY_SCRIPT_FILE, "\n".join(commands)) try: - process = self._container.exec([]) # TODO + process = self._container.exec( + ["mysqlsh", "--no-wizard", "--python", "--file", self._TEMPORARY_SCRIPT_FILE] + ) process.wait_output() except ops.pebble.ExecError as e: logger.exception(f"Failed to run {commands=}\nstderr:\n{e.stderr}\n") raise + finally: + self._container.remove_path(self._TEMPORARY_SCRIPT_FILE) def _run_sql(self, sql_statements: list[str]) -> None: """Connect to MySQL cluster and execute SQL.""" From 79eaa18f1b4d81bd8d7cb75a90305a5a059e1c1e Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 17:37:09 +0000 Subject: [PATCH 066/159] temp fix ci branch remove foresight --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4c1bfdf7b..9eb3d24cb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,7 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@remove-foresight integration-test: strategy: From 07b1a3c4761905fd0e20df4d97c384be7e3c1881 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 17:54:25 +0000 Subject: [PATCH 067/159] fix: add mysql-connector-python to integration test deps --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index dafc8f409..29dd2b3f8 100644 --- a/tox.ini +++ b/tox.ini @@ -77,6 +77,7 @@ deps = juju==2.9.38.1 pytest pytest-operator + mysql-connector-python -r {tox_root}/requirements.txt commands = pytest -v --tb native --log-cli-level=INFO -s {posargs} {[vars]tests_path}/integration/test_charm.py From faf440bb8d04967c0bd9806cc42206c70d4f3d0f Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 17:55:07 +0000 Subject: [PATCH 068/159] Revert "temp fix ci branch remove foresight" This reverts commit 79eaa18f1b4d81bd8d7cb75a90305a5a059e1c1e. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9eb3d24cb..4c1bfdf7b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,7 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@remove-foresight + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 integration-test: strategy: From 1bcaacca01ae59477ecffe2fdbb0e3b8d4b3b1d0 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 18:11:14 +0000 Subject: [PATCH 069/159] Fix log --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index a2419f198..5fdf02f12 100755 --- a/src/charm.py +++ b/src/charm.py @@ -184,7 +184,7 @@ def _reconcile_database_relations(self, event=None) -> None: "State of reconcile " f"{self.unit.is_leader()=}, " f"{isinstance(self.workload, workload.AuthenticatedWorkload)=}, " - f"{self.database_requires.relation.is_breaking(event)=}, " + f"{self.database_requires.relation and self.database_requires.relation.is_breaking(event)=}, " f"{self.workload.container_ready=}" ) if ( From 468778d31a8c1700b1e58e5ec23ec92e87ee691f Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 18:39:17 +0000 Subject: [PATCH 070/159] Check if workload is enabled before trying to disable it disable() is only available on AuthenticatedWorkload, not Workload --- src/charm.py | 16 +++++++++++++--- src/workload.py | 12 +++--------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/charm.py b/src/charm.py index 5fdf02f12..ff83d6339 100755 --- a/src/charm.py +++ b/src/charm.py @@ -185,7 +185,8 @@ def _reconcile_database_relations(self, event=None) -> None: f"{self.unit.is_leader()=}, " f"{isinstance(self.workload, workload.AuthenticatedWorkload)=}, " f"{self.database_requires.relation and self.database_requires.relation.is_breaking(event)=}, " - f"{self.workload.container_ready=}" + f"{self.workload.container_ready=}, " + f"{self.workload.enabled=}" ) if ( self.unit.is_leader() @@ -205,9 +206,18 @@ def _reconcile_database_relations(self, event=None) -> None: and not self.database_requires.relation.is_breaking(event) and self.workload.container_ready ): - self.workload.enable(tls=self.tls.certificate_saved) + # Enable workload + if self.workload.enabled: + # If the host or port changes, MySQL Router will receive topology change + # notifications from MySQL. + # Therefore, if the host or port changes, we do not need to restart MySQL Router. + pass + else: + self.workload.enable(tls=self.tls.certificate_saved) else: - self.workload.disable() + # Disable workload + if self.workload.enabled: + self.workload.disable() self._set_status() def _on_mysql_router_pebble_ready(self, _) -> None: diff --git a/src/workload.py b/src/workload.py index 4a418133f..45fd4f767 100644 --- a/src/workload.py +++ b/src/workload.py @@ -32,7 +32,7 @@ def container_ready(self) -> bool: return self._container.can_connect() @property - def _enabled(self) -> bool: + def enabled(self) -> bool: """Service status""" service = self._container.get_services(self._SERVICE_NAME).get(self._SERVICE_NAME) if service is None: @@ -158,10 +158,6 @@ def _bootstrap_router(self, *, password: str, tls: bool) -> None: def enable(self, *, tls: bool) -> None: """Start and enable MySQL Router service.""" - if self._enabled: - # If the host or port changes, MySQL Router will receive topology change notifications from MySQL - # Therefore, if the host or port changes, we do not need to restart MySQL Router - return logger.debug("Enabling MySQL Router service") router_password = self.shell.create_mysql_router_user(self._ROUTER_USERNAME) self._bootstrap_router(password=router_password, tls=tls) @@ -171,8 +167,6 @@ def enable(self, *, tls: bool) -> None: def disable(self) -> None: """Stop and disable MySQL Router service.""" - if not self._enabled: - return logger.debug("Disabling MySQL Router service") self.shell.delete_user(self._ROUTER_USERNAME) self._update_layer(enabled=False) @@ -235,7 +229,7 @@ def enable_tls(self, *, key: str, certificate: str): ) self._write_file(self._ROUTER_CONFIG_DIRECTORY / self._TLS_KEY_FILE, key) self._write_file(self._ROUTER_CONFIG_DIRECTORY / self._TLS_CERTIFICATE_FILE, certificate) - if self._enabled: + if self.enabled: self._restart(tls=True) logger.debug("Enabled TLS") @@ -244,6 +238,6 @@ def disable_tls(self) -> None: logger.debug("Disabling TLS") for file in [self._TLS_CONFIG_FILE, self._TLS_KEY_FILE, self._TLS_CERTIFICATE_FILE]: self._delete_file(self._ROUTER_CONFIG_DIRECTORY / file) - if self._enabled: + if self.enabled: self._restart(tls=False) logger.debug("Disabled TLS") From 6618a6c822f0f0cab6a08aa99155291f654425bc Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 18:40:04 +0000 Subject: [PATCH 071/159] update todo comment --- src/workload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index 45fd4f767..a1cd36d35 100644 --- a/src/workload.py +++ b/src/workload.py @@ -144,7 +144,7 @@ def _bootstrap_router(self, *, password: str, tls: bool) -> None: self._UNIX_USERNAME, "--conf-set-option", "http_server.bind_address=127.0.0.1", - "--force", # TODO: needed? + "--force", # TODO: Remove after https://github.com/canonical/charmed-mysql-snap/pull/23 is merged ] ) process.wait_output() From 8311125ad3712d112449f9faee296080448fd139 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 18:51:15 +0000 Subject: [PATCH 072/159] fix log --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index ff83d6339..c1e3065d1 100755 --- a/src/charm.py +++ b/src/charm.py @@ -186,7 +186,7 @@ def _reconcile_database_relations(self, event=None) -> None: f"{isinstance(self.workload, workload.AuthenticatedWorkload)=}, " f"{self.database_requires.relation and self.database_requires.relation.is_breaking(event)=}, " f"{self.workload.container_ready=}, " - f"{self.workload.enabled=}" + f"{self.workload.container_ready and self.workload.enabled=}" ) if ( self.unit.is_leader() From f5ab8362cbd1b3e5d8f899fbc8fd7b8a0a526990 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 19:08:57 +0000 Subject: [PATCH 073/159] fix mysqlsh connect --- src/mysql_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index ba6e4e0cd..ac9d0f44e 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -32,7 +32,7 @@ class Shell: def _run_commands(self, commands: list[str]) -> None: """Connect to MySQL cluster and run commands.""" commands.insert( - 0, f"shell.connect('{self._username}:{self._password}@{self._host}:{self._port}" + 0, f"shell.connect('{self._username}:{self._password}@{self._host}:{self._port}')" ) self._container.push(self._TEMPORARY_SCRIPT_FILE, "\n".join(commands)) try: From ac1b50c602dca2a359d76e0b8d3830ff0e987328 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 19:35:15 +0000 Subject: [PATCH 074/159] clean up wait until mysql router ready --- src/charm.py | 27 +++++++++++++++++++++++++++ src/workload.py | 29 ++++------------------------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/src/charm.py b/src/charm.py index c1e3065d1..1c2809f88 100755 --- a/src/charm.py +++ b/src/charm.py @@ -7,6 +7,7 @@ """MySQL Router kubernetes (k8s) charm""" import logging +import socket import charms.data_platform_libs.v0.data_interfaces as data_interfaces import lightkube @@ -14,6 +15,7 @@ import lightkube.models.meta_v1 import lightkube.resources.core_v1 import ops +import tenacity import relations.database_provides import relations.database_requires @@ -87,6 +89,7 @@ def workload(self): _admin_password=self.database_requires.relation.password, _host=self.database_requires.relation.host, _port=self.database_requires.relation.port, + _charm=self, ) return workload.Workload(_container=container) @@ -121,6 +124,30 @@ def _set_status(self) -> None: self.unit.status = self._determine_status() logger.debug(f"Set status to {self.unit.status}") + def wait_until_mysql_router_ready(self) -> None: + """Wait until a connection to MySQL Router is possible. + + Retry every 5 seconds for up to 360 seconds. + """ + logger.debug("Waiting until MySQL Router is ready") + self.unit.status = ops.WaitingStatus("MySQL Router starting") + try: + for attempt in tenacity.Retrying( + reraise=True, + stop=tenacity.stop_after_delay(360), # TODO: adjust timeout + wait=tenacity.wait_fixed(5), + ): + with attempt: + for port in [6446, 6447]: + with socket.socket() as s: + assert s.connect_ex(("localhost", port)) != 0 + except AssertionError: + logger.exception("Unable to connect to MySQL Router") + raise + else: + logger.debug("MySQL Router is ready") + self._set_status() + def _patch_service(self, *, name: str, ro_port: int, rw_port: int) -> None: """Patch Juju-created k8s service. diff --git a/src/workload.py b/src/workload.py index a1cd36d35..05bdabf65 100644 --- a/src/workload.py +++ b/src/workload.py @@ -6,13 +6,11 @@ import dataclasses import logging import pathlib -import socket import string -import typing import ops -import tenacity +import charm import mysql_shell logger = logging.getLogger(__name__) @@ -57,6 +55,7 @@ class AuthenticatedWorkload(Workload): _admin_password: str _host: str _port: str + _charm: charm.MySQLRouterOperatorCharm _UNIX_USERNAME = "mysql" _ROUTER_USERNAME = "mysqlrouter" @@ -111,25 +110,6 @@ def shell(self) -> mysql_shell.Shell: _port=self._port, ) - @staticmethod - @tenacity.retry(reraise=True, stop=tenacity.stop_after_delay(360), wait=tenacity.wait_fixed(5)) - def _wait_until_mysql_router_ready() -> None: - # TODO: add debug logging - """Wait until a connection to MySQL router is possible. - Retry every 5 seconds for 30 seconds if there is an issue obtaining a connection. - """ - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex(("127.0.0.1", 6446)) - if result != 0: - raise BaseException() - sock.close() - - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = sock.connect_ex(("127.0.0.1", 6447)) - if result != 0: - raise BaseException() - sock.close() - def _bootstrap_router(self, *, password: str, tls: bool) -> None: """Bootstrap MySQL Router and enable service.""" logger.debug(f"Bootstrapping router {tls=}, {self._host=}, {self._port=}") @@ -162,8 +142,7 @@ def enable(self, *, tls: bool) -> None: router_password = self.shell.create_mysql_router_user(self._ROUTER_USERNAME) self._bootstrap_router(password=router_password, tls=tls) logger.debug("Enabled MySQL Router service") - self._wait_until_mysql_router_ready() - # TODO: wait until mysql router ready? https://github.com/canonical/mysql-router-k8s-operator/blob/45cf3be44f27476a0371c67d50d7a0193c0fadc2/src/charm.py#L219 + self._charm.wait_until_mysql_router_ready() def disable(self) -> None: """Stop and disable MySQL Router service.""" @@ -178,7 +157,7 @@ def _restart(self, *, tls: bool) -> None: router_password = self.shell.change_mysql_router_user_password(self._ROUTER_USERNAME) self._bootstrap_router(password=router_password, tls=tls) logger.debug("Restarted MySQL Router service") - self._wait_until_mysql_router_ready() + self._charm.wait_until_mysql_router_ready() def _write_file(self, path: pathlib.Path, content: str) -> None: """Write content to file. From 252ab942c89394caf8d695784327ef132c9e800a Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 19:44:29 +0000 Subject: [PATCH 075/159] fix disable --- src/charm.py | 16 +++------------- src/workload.py | 13 ++++++++++--- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/charm.py b/src/charm.py index 1c2809f88..28c973c42 100755 --- a/src/charm.py +++ b/src/charm.py @@ -212,8 +212,7 @@ def _reconcile_database_relations(self, event=None) -> None: f"{self.unit.is_leader()=}, " f"{isinstance(self.workload, workload.AuthenticatedWorkload)=}, " f"{self.database_requires.relation and self.database_requires.relation.is_breaking(event)=}, " - f"{self.workload.container_ready=}, " - f"{self.workload.container_ready and self.workload.enabled=}" + f"{self.workload.container_ready=}" ) if ( self.unit.is_leader() @@ -230,21 +229,12 @@ def _reconcile_database_relations(self, event=None) -> None: ) if ( isinstance(self.workload, workload.AuthenticatedWorkload) - and not self.database_requires.relation.is_breaking(event) and self.workload.container_ready ): - # Enable workload - if self.workload.enabled: - # If the host or port changes, MySQL Router will receive topology change - # notifications from MySQL. - # Therefore, if the host or port changes, we do not need to restart MySQL Router. - pass + if self.database_requires.relation.is_breaking(event): + self.workload.disable() else: self.workload.enable(tls=self.tls.certificate_saved) - else: - # Disable workload - if self.workload.enabled: - self.workload.disable() self._set_status() def _on_mysql_router_pebble_ready(self, _) -> None: diff --git a/src/workload.py b/src/workload.py index 05bdabf65..1b34882b3 100644 --- a/src/workload.py +++ b/src/workload.py @@ -30,7 +30,7 @@ def container_ready(self) -> bool: return self._container.can_connect() @property - def enabled(self) -> bool: + def _enabled(self) -> bool: """Service status""" service = self._container.get_services(self._SERVICE_NAME).get(self._SERVICE_NAME) if service is None: @@ -138,6 +138,11 @@ def _bootstrap_router(self, *, password: str, tls: bool) -> None: def enable(self, *, tls: bool) -> None: """Start and enable MySQL Router service.""" + if self._enabled: + # If the host or port changes, MySQL Router will receive topology change + # notifications from MySQL. + # Therefore, if the host or port changes, we do not need to restart MySQL Router. + return logger.debug("Enabling MySQL Router service") router_password = self.shell.create_mysql_router_user(self._ROUTER_USERNAME) self._bootstrap_router(password=router_password, tls=tls) @@ -146,6 +151,8 @@ def enable(self, *, tls: bool) -> None: def disable(self) -> None: """Stop and disable MySQL Router service.""" + if not self._enabled: + return logger.debug("Disabling MySQL Router service") self.shell.delete_user(self._ROUTER_USERNAME) self._update_layer(enabled=False) @@ -208,7 +215,7 @@ def enable_tls(self, *, key: str, certificate: str): ) self._write_file(self._ROUTER_CONFIG_DIRECTORY / self._TLS_KEY_FILE, key) self._write_file(self._ROUTER_CONFIG_DIRECTORY / self._TLS_CERTIFICATE_FILE, certificate) - if self.enabled: + if self._enabled: self._restart(tls=True) logger.debug("Enabled TLS") @@ -217,6 +224,6 @@ def disable_tls(self) -> None: logger.debug("Disabling TLS") for file in [self._TLS_CONFIG_FILE, self._TLS_KEY_FILE, self._TLS_CERTIFICATE_FILE]: self._delete_file(self._ROUTER_CONFIG_DIRECTORY / file) - if self.enabled: + if self._enabled: self._restart(tls=False) logger.debug("Disabled TLS") From d38cbc221f611174e4960c448dfc65450ac1d935 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 19:51:45 +0000 Subject: [PATCH 076/159] Create different router users for each unit --- src/workload.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/workload.py b/src/workload.py index 1b34882b3..70bf3d94f 100644 --- a/src/workload.py +++ b/src/workload.py @@ -58,13 +58,17 @@ class AuthenticatedWorkload(Workload): _charm: charm.MySQLRouterOperatorCharm _UNIX_USERNAME = "mysql" - _ROUTER_USERNAME = "mysqlrouter" _ROUTER_CONFIG_DIRECTORY = pathlib.Path("/etc/mysqlrouter") _ROUTER_CONFIG_FILE = "mysqlrouter.conf" _TLS_CONFIG_FILE = "tls.conf" _TLS_KEY_FILE = "custom-key.pem" _TLS_CERTIFICATE_FILE = "custom-certificate.pem" + @property + def _router_username(self) -> str: + unit_id = self._charm.unit.name.split("/")[1] + return f"mysqlrouter_{unit_id}" + def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: """Update and restart services. @@ -119,7 +123,7 @@ def _bootstrap_router(self, *, password: str, tls: bool) -> None: [ "mysqlrouter", "--bootstrap", - f"{self._ROUTER_USERNAME}:{password}@{self._host}:{self._port}", + f"{self._router_username}:{password}@{self._host}:{self._port}", "--user", self._UNIX_USERNAME, "--conf-set-option", @@ -144,7 +148,7 @@ def enable(self, *, tls: bool) -> None: # Therefore, if the host or port changes, we do not need to restart MySQL Router. return logger.debug("Enabling MySQL Router service") - router_password = self.shell.create_mysql_router_user(self._ROUTER_USERNAME) + router_password = self.shell.create_mysql_router_user(self._router_username) self._bootstrap_router(password=router_password, tls=tls) logger.debug("Enabled MySQL Router service") self._charm.wait_until_mysql_router_ready() @@ -154,14 +158,14 @@ def disable(self) -> None: if not self._enabled: return logger.debug("Disabling MySQL Router service") - self.shell.delete_user(self._ROUTER_USERNAME) + self.shell.delete_user(self._router_username) self._update_layer(enabled=False) logger.debug("Disabled MySQL Router service") def _restart(self, *, tls: bool) -> None: """Restart MySQL Router to enable or disable TLS.""" logger.debug("Restarting MySQL Router service") - router_password = self.shell.change_mysql_router_user_password(self._ROUTER_USERNAME) + router_password = self.shell.change_mysql_router_user_password(self._router_username) self._bootstrap_router(password=router_password, tls=tls) logger.debug("Restarted MySQL Router service") self._charm.wait_until_mysql_router_ready() From d5f6812cfc8f3d26ce7c9afe21461c517c0b0dc0 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 19:55:38 +0000 Subject: [PATCH 077/159] Fix lint --- pyproject.toml | 2 +- src/charm.py | 1 + src/mysql_shell.py | 1 + src/workload.py | 3 +++ 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 070ea6e60..46ab23b3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ exclude = [".git", "__pycache__", ".tox", "build", "dist", "*.egg_info", "venv"] select = ["E", "W", "F", "C", "N", "R", "D", "H"] # Ignore W503, E501 because using black creates errors with this # Ignore D107 Missing docstring in __init__ -ignore = ["W503", "E501", "D107"] +ignore = ["W503", "E501", "D107", "D415", "D403"] # D100, D101, D102, D103: Ignore missing docstrings in tests per-file-ignores = ["tests/*:D100,D101,D102,D103,D104"] docstring-convention = "google" diff --git a/src/charm.py b/src/charm.py index 28c973c42..d80c2c7d2 100755 --- a/src/charm.py +++ b/src/charm.py @@ -79,6 +79,7 @@ def __init__(self, *args) -> None: @property def workload(self): + """MySQL Router workload""" # Defined as a property instead of an attribute in __init__ since this class is # not re-instantiated between events (if there are deferred events) container = self.unit.get_container(workload.Workload.CONTAINER_NAME) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index ac9d0f44e..9701a55c5 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -100,6 +100,7 @@ def change_mysql_router_user_password(self, username: str) -> str: return password def delete_user(self, username: str) -> None: + """Delete user.""" logger.debug(f"Deleting {username=}") self._run_sql([f"DROP USER `{username}`"]) logger.debug(f"Deleted {username=}") diff --git a/src/workload.py b/src/workload.py index 70bf3d94f..0313afe66 100644 --- a/src/workload.py +++ b/src/workload.py @@ -27,6 +27,7 @@ class Workload: @property def container_ready(self) -> bool: + """Whether container is ready""" return self._container.can_connect() @property @@ -39,6 +40,7 @@ def _enabled(self) -> bool: @property def version(self) -> str: + """MySQL Router version""" process = self._container.exec(["mysqlrouter", "--version"]) raw_version, _ = process.wait_output() for version in raw_version.split(): @@ -106,6 +108,7 @@ def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: @property def shell(self) -> mysql_shell.Shell: + """MySQL Shell""" return mysql_shell.Shell( _container=self._container, _username=self._admin_username, From 8ce42e976cd22316a9e0eda357823306baad8b4f Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 19:58:00 +0000 Subject: [PATCH 078/159] Fix circular import --- src/workload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/workload.py b/src/workload.py index 0313afe66..15cec7652 100644 --- a/src/workload.py +++ b/src/workload.py @@ -10,7 +10,7 @@ import ops -import charm +from charm import MySQLRouterOperatorCharm import mysql_shell logger = logging.getLogger(__name__) @@ -57,7 +57,7 @@ class AuthenticatedWorkload(Workload): _admin_password: str _host: str _port: str - _charm: charm.MySQLRouterOperatorCharm + _charm: MySQLRouterOperatorCharm _UNIX_USERNAME = "mysql" _ROUTER_CONFIG_DIRECTORY = pathlib.Path("/etc/mysqlrouter") From bc5f8cda3660a0fe05347a84eb46355bb795a20c Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 25 Apr 2023 20:02:19 +0000 Subject: [PATCH 079/159] Fix circular import --- src/workload.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/workload.py b/src/workload.py index 15cec7652..800d8b17c 100644 --- a/src/workload.py +++ b/src/workload.py @@ -7,12 +7,14 @@ import logging import pathlib import string +import typing import ops -from charm import MySQLRouterOperatorCharm import mysql_shell +if typing.TYPE_CHECKING: + import charm logger = logging.getLogger(__name__) @@ -57,7 +59,7 @@ class AuthenticatedWorkload(Workload): _admin_password: str _host: str _port: str - _charm: MySQLRouterOperatorCharm + _charm: "charm.MySQLRouterOperatorCharm" _UNIX_USERNAME = "mysql" _ROUTER_CONFIG_DIRECTORY = pathlib.Path("/etc/mysqlrouter") From b7c3953da3982051cb90f3acd99f6eeb34bca9ae Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 26 Apr 2023 11:51:41 +0000 Subject: [PATCH 080/159] include app name in mysqlrouter username --- src/workload.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/workload.py b/src/workload.py index 800d8b17c..a4c6765f9 100644 --- a/src/workload.py +++ b/src/workload.py @@ -70,8 +70,7 @@ class AuthenticatedWorkload(Workload): @property def _router_username(self) -> str: - unit_id = self._charm.unit.name.split("/")[1] - return f"mysqlrouter_{unit_id}" + return f"mysqlrouter_{self._charm.unit.name}" def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: """Update and restart services. From cc772527d5b8a21d637e6b366cb70cbe152a1c65 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 26 Apr 2023 12:06:12 +0000 Subject: [PATCH 081/159] don't create mysqlrouter account on bootstrap --- src/workload.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/workload.py b/src/workload.py index a4c6765f9..979695fd0 100644 --- a/src/workload.py +++ b/src/workload.py @@ -128,6 +128,11 @@ def _bootstrap_router(self, *, password: str, tls: bool) -> None: "mysqlrouter", "--bootstrap", f"{self._router_username}:{password}@{self._host}:{self._port}", + "--strict", + "--account", + self._router_username, + "--account-create", + "never", "--user", self._UNIX_USERNAME, "--conf-set-option", From 82cfa05b0c03036fe85fb8251896f2ed7864bbec Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 26 Apr 2023 12:25:36 +0000 Subject: [PATCH 082/159] remove `/` from username --- src/workload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index 979695fd0..9edf2fca8 100644 --- a/src/workload.py +++ b/src/workload.py @@ -70,7 +70,7 @@ class AuthenticatedWorkload(Workload): @property def _router_username(self) -> str: - return f"mysqlrouter_{self._charm.unit.name}" + return f"mysqlrouter_{self._charm.unit.name.replace('/', '_')}" def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: """Update and restart services. From 694d05180181de4564fd6935ce7662428916d217 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 26 Apr 2023 14:19:28 +0000 Subject: [PATCH 083/159] add timeout --- src/workload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index 9edf2fca8..1142b4f57 100644 --- a/src/workload.py +++ b/src/workload.py @@ -138,7 +138,8 @@ def _bootstrap_router(self, *, password: str, tls: bool) -> None: "--conf-set-option", "http_server.bind_address=127.0.0.1", "--force", # TODO: Remove after https://github.com/canonical/charmed-mysql-snap/pull/23 is merged - ] + ], + timeout=5 * 60, ) process.wait_output() except ops.pebble.ExecError as e: From 7bb78b4a106e8c488101bb563d194e405d4ae84d Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 26 Apr 2023 14:21:22 +0000 Subject: [PATCH 084/159] add stdout debug --- src/workload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index 1142b4f57..c6b27114f 100644 --- a/src/workload.py +++ b/src/workload.py @@ -139,11 +139,12 @@ def _bootstrap_router(self, *, password: str, tls: bool) -> None: "http_server.bind_address=127.0.0.1", "--force", # TODO: Remove after https://github.com/canonical/charmed-mysql-snap/pull/23 is merged ], - timeout=5 * 60, + timeout=5 * 60, # todo: adjust? ) process.wait_output() except ops.pebble.ExecError as e: logger.exception(f"Failed to bootstrap router\nstderr:\n{e.stderr}\n") + logger.debug(f"\nstdout:\n{e.stdout}\n") # TODO: remove raise # Enable service self._update_layer(enabled=True, tls=tls) From 4d34553fd2226be9c5638c58f67aa54d486df0f0 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 26 Apr 2023 14:21:48 +0000 Subject: [PATCH 085/159] 2 min timeout --- src/workload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index c6b27114f..22f8013ed 100644 --- a/src/workload.py +++ b/src/workload.py @@ -139,7 +139,7 @@ def _bootstrap_router(self, *, password: str, tls: bool) -> None: "http_server.bind_address=127.0.0.1", "--force", # TODO: Remove after https://github.com/canonical/charmed-mysql-snap/pull/23 is merged ], - timeout=5 * 60, # todo: adjust? + timeout=2 * 60, # todo: adjust? ) process.wait_output() except ops.pebble.ExecError as e: From a4cdc55600b6e9e42fc50abd9a244ac69f873c93 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 26 Apr 2023 14:34:46 +0000 Subject: [PATCH 086/159] catch change error --- src/workload.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/workload.py b/src/workload.py index 22f8013ed..98386416a 100644 --- a/src/workload.py +++ b/src/workload.py @@ -142,9 +142,9 @@ def _bootstrap_router(self, *, password: str, tls: bool) -> None: timeout=2 * 60, # todo: adjust? ) process.wait_output() - except ops.pebble.ExecError as e: + except (ops.pebble.ExecError, ops.pebble.ChangeError) as e: logger.exception(f"Failed to bootstrap router\nstderr:\n{e.stderr}\n") - logger.debug(f"\nstdout:\n{e.stdout}\n") # TODO: remove + logger.debug(f"\nstdout:\n{e.stdout}\n") # TODO: remove raise # Enable service self._update_layer(enabled=True, tls=tls) From d8c1904fae23abe4fa864464dac91e984ccf554c Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 2 May 2023 10:07:37 +0000 Subject: [PATCH 087/159] fix wait for mysqlrouter ready --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index d80c2c7d2..27ba35901 100755 --- a/src/charm.py +++ b/src/charm.py @@ -141,7 +141,7 @@ def wait_until_mysql_router_ready(self) -> None: with attempt: for port in [6446, 6447]: with socket.socket() as s: - assert s.connect_ex(("localhost", port)) != 0 + assert s.connect_ex(("localhost", port)) == 0 except AssertionError: logger.exception("Unable to connect to MySQL Router") raise From dd6f28d6762ba0f342f27d43e96a20380bc633b4 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 2 May 2023 12:30:35 +0000 Subject: [PATCH 088/159] use auto-created user from bootstrap --- src/mysql_shell.py | 25 --------------------- src/workload.py | 55 ++++++++++++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 49 deletions(-) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 9701a55c5..16e649f3b 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -74,31 +74,6 @@ def create_application_database_and_user(self, *, username: str, database: str) logger.debug(f"Created {database=} and {username=}") return password - def create_mysql_router_user(self, username: str) -> str: - """Create user to run MySQL Router service.""" - logger.debug(f"Creating router {username=}") - password = self._generate_password() - self._run_commands( - [ - "cluster = dba.get_cluster()", - "cluster.setup_router_account('" - + username - + "', {'password': '" - + password - + "'})", - ] - ) - logger.debug(f"Created router {username=}") - return password - - def change_mysql_router_user_password(self, username: str) -> str: - """Change MySQL Router service user password.""" - logger.debug(f"Changing router password {username=}") - password = self._generate_password() - self._run_sql([f"ALTER USER `{username}` IDENTIFIED BY '{password}'"]) - logger.debug(f"Changed router password {username=}") - return password - def delete_user(self, username: str) -> None: """Delete user.""" logger.debug(f"Deleting {username=}") diff --git a/src/workload.py b/src/workload.py index 98386416a..d642387cc 100644 --- a/src/workload.py +++ b/src/workload.py @@ -3,6 +3,7 @@ """MySQL Router workload""" +import configparser import dataclasses import logging import pathlib @@ -55,6 +56,10 @@ def version(self) -> str: class AuthenticatedWorkload(Workload): """Workload with connection to MySQL cluster""" + # Admin user with permission to: + # - Create databases & users + # - Grant all privileges on a database to a user + # (Different from user that MySQL Router runs with after bootstrap.) _admin_username: str _admin_password: str _host: str @@ -68,10 +73,6 @@ class AuthenticatedWorkload(Workload): _TLS_KEY_FILE = "custom-key.pem" _TLS_CERTIFICATE_FILE = "custom-certificate.pem" - @property - def _router_username(self) -> str: - return f"mysqlrouter_{self._charm.unit.name.replace('/', '_')}" - def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: """Update and restart services. @@ -118,7 +119,7 @@ def shell(self) -> mysql_shell.Shell: _port=self._port, ) - def _bootstrap_router(self, *, password: str, tls: bool) -> None: + def _bootstrap_router(self, *, tls: bool) -> None: """Bootstrap MySQL Router and enable service.""" logger.debug(f"Bootstrapping router {tls=}, {self._host=}, {self._port=}") try: @@ -127,24 +128,19 @@ def _bootstrap_router(self, *, password: str, tls: bool) -> None: [ "mysqlrouter", "--bootstrap", - f"{self._router_username}:{password}@{self._host}:{self._port}", - "--strict", - "--account", - self._router_username, - "--account-create", - "never", + f"{self._admin_username}:{self._admin_password}@{self._host}:{self._port}", + "--strict", # todo: does this do anything without `--account`? "--user", self._UNIX_USERNAME, "--conf-set-option", "http_server.bind_address=127.0.0.1", "--force", # TODO: Remove after https://github.com/canonical/charmed-mysql-snap/pull/23 is merged ], - timeout=2 * 60, # todo: adjust? + timeout=30, ) process.wait_output() - except (ops.pebble.ExecError, ops.pebble.ChangeError) as e: + except ops.pebble.ExecError as e: logger.exception(f"Failed to bootstrap router\nstderr:\n{e.stderr}\n") - logger.debug(f"\nstdout:\n{e.stdout}\n") # TODO: remove raise # Enable service self._update_layer(enabled=True, tls=tls) @@ -159,27 +155,38 @@ def enable(self, *, tls: bool) -> None: # Therefore, if the host or port changes, we do not need to restart MySQL Router. return logger.debug("Enabling MySQL Router service") - router_password = self.shell.create_mysql_router_user(self._router_username) - self._bootstrap_router(password=router_password, tls=tls) + self._bootstrap_router(tls=tls) logger.debug("Enabled MySQL Router service") self._charm.wait_until_mysql_router_ready() + @property + def _router_username(self) -> str: + """Read MySQL Router username from config file. + + During bootstrap, MySQL Router creates a config file at + `/etc/mysqlrouter/mysqlrouter.conf`. This file contains the username that was created + during bootstrap. + """ + config = configparser.ConfigParser() + config.read_file(self._container.pull("/etc/mysqlrouter/mysqlrouter.conf")) + return config["metadata_cache:bootstrap"]["user"] + def disable(self) -> None: """Stop and disable MySQL Router service.""" if not self._enabled: return logger.debug("Disabling MySQL Router service") - self.shell.delete_user(self._router_username) self._update_layer(enabled=False) + self.shell.delete_user(self._router_username) logger.debug("Disabled MySQL Router service") def _restart(self, *, tls: bool) -> None: """Restart MySQL Router to enable or disable TLS.""" - logger.debug("Restarting MySQL Router service") - router_password = self.shell.change_mysql_router_user_password(self._router_username) - self._bootstrap_router(password=router_password, tls=tls) - logger.debug("Restarted MySQL Router service") - self._charm.wait_until_mysql_router_ready() + logger.debug("Restarting MySQL Router") + assert self._enabled is True + self.disable() + self.enable(tls=tls) + logger.debug("Restarted MySQL Router") def _write_file(self, path: pathlib.Path, content: str) -> None: """Write content to file. @@ -223,7 +230,7 @@ def _tls_config_file(self) -> str: return config_string def enable_tls(self, *, key: str, certificate: str): - """Enable TLS and restart MySQL Router service.""" + """Enable TLS and restart MySQL Router.""" logger.debug("Enabling TLS") self._write_file( self._ROUTER_CONFIG_DIRECTORY / self._TLS_CONFIG_FILE, self._tls_config_file @@ -235,7 +242,7 @@ def enable_tls(self, *, key: str, certificate: str): logger.debug("Enabled TLS") def disable_tls(self) -> None: - """Disable TLS and restart MySQL Router service.""" + """Disable TLS and restart MySQL Router.""" logger.debug("Disabling TLS") for file in [self._TLS_CONFIG_FILE, self._TLS_KEY_FILE, self._TLS_CERTIFICATE_FILE]: self._delete_file(self._ROUTER_CONFIG_DIRECTORY / file) From 8bea6afafd4f54d229d2a62e42ba0ddda4099f52 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 2 May 2023 12:31:26 +0000 Subject: [PATCH 089/159] add unmerged fix --- src/workload.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index d642387cc..620bba5e9 100644 --- a/src/workload.py +++ b/src/workload.py @@ -121,6 +121,10 @@ def shell(self) -> mysql_shell.Shell: def _bootstrap_router(self, *, tls: bool) -> None: """Bootstrap MySQL Router and enable service.""" + # TODO: remove when https://github.com/canonical/charmed-mysql-rock/pull/23 merged + self._container.exec(["mkdir", "-p", "/var/log/mysqlrouter"]) + self._container.exec(["chown", "-R", "584788", "/var/log/mysqlrouter"]) + logger.debug(f"Bootstrapping router {tls=}, {self._host=}, {self._port=}") try: # Bootstrap MySQL Router @@ -129,7 +133,7 @@ def _bootstrap_router(self, *, tls: bool) -> None: "mysqlrouter", "--bootstrap", f"{self._admin_username}:{self._admin_password}@{self._host}:{self._port}", - "--strict", # todo: does this do anything without `--account`? + "--strict", # todo: does this do anything without `--account`? "--user", self._UNIX_USERNAME, "--conf-set-option", From 4d739f6d4f7a3871f0c4c9c6fc7dacb97a1fff81 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 2 May 2023 12:44:05 +0000 Subject: [PATCH 090/159] update timeout --- src/charm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/charm.py b/src/charm.py index 27ba35901..da79881e2 100755 --- a/src/charm.py +++ b/src/charm.py @@ -128,14 +128,14 @@ def _set_status(self) -> None: def wait_until_mysql_router_ready(self) -> None: """Wait until a connection to MySQL Router is possible. - Retry every 5 seconds for up to 360 seconds. + Retry every 5 seconds for up to 30 seconds. """ logger.debug("Waiting until MySQL Router is ready") self.unit.status = ops.WaitingStatus("MySQL Router starting") try: for attempt in tenacity.Retrying( reraise=True, - stop=tenacity.stop_after_delay(360), # TODO: adjust timeout + stop=tenacity.stop_after_delay(30), wait=tenacity.wait_fixed(5), ): with attempt: From 5585339a0092a5f95e0c7c2930b0c0ff8f12725a Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 2 May 2023 12:50:31 +0000 Subject: [PATCH 091/159] update rock --- metadata.yaml | 2 +- src/workload.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/metadata.yaml b/metadata.yaml index 48bf2cf27..2cd613b57 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -31,6 +31,6 @@ resources: mysql-router-image: type: oci-image description: OCI image for mysql-router - upstream-source: ghcr.io/canonical/charmed-mysql@sha256:924cb913733002e0936a36b6edab0bb8a4d72f1b92be0e0b7a933e797b7ef215 + upstream-source: ghcr.io/canonical/charmed-mysql@sha256:5fdd3e847925e7564567604957630ea7fc6e84839fe492187652d56ebc6ae621 assumes: - k8s-api diff --git a/src/workload.py b/src/workload.py index 620bba5e9..4c47575b5 100644 --- a/src/workload.py +++ b/src/workload.py @@ -121,10 +121,6 @@ def shell(self) -> mysql_shell.Shell: def _bootstrap_router(self, *, tls: bool) -> None: """Bootstrap MySQL Router and enable service.""" - # TODO: remove when https://github.com/canonical/charmed-mysql-rock/pull/23 merged - self._container.exec(["mkdir", "-p", "/var/log/mysqlrouter"]) - self._container.exec(["chown", "-R", "584788", "/var/log/mysqlrouter"]) - logger.debug(f"Bootstrapping router {tls=}, {self._host=}, {self._port=}") try: # Bootstrap MySQL Router From b8dbc9560b33d105f789a87d11097c0865d0bc9d Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 2 May 2023 12:50:44 +0000 Subject: [PATCH 092/159] remove --force --- src/workload.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index 4c47575b5..15eece105 100644 --- a/src/workload.py +++ b/src/workload.py @@ -134,7 +134,6 @@ def _bootstrap_router(self, *, tls: bool) -> None: self._UNIX_USERNAME, "--conf-set-option", "http_server.bind_address=127.0.0.1", - "--force", # TODO: Remove after https://github.com/canonical/charmed-mysql-snap/pull/23 is merged ], timeout=30, ) From 66bb1118bda2972523c2d1eae5cfe02fe5946f85 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 2 May 2023 13:40:41 +0000 Subject: [PATCH 093/159] Revert "update rock" This reverts commit 5585339a0092a5f95e0c7c2930b0c0ff8f12725a. --- metadata.yaml | 2 +- src/workload.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/metadata.yaml b/metadata.yaml index 2cd613b57..48bf2cf27 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -31,6 +31,6 @@ resources: mysql-router-image: type: oci-image description: OCI image for mysql-router - upstream-source: ghcr.io/canonical/charmed-mysql@sha256:5fdd3e847925e7564567604957630ea7fc6e84839fe492187652d56ebc6ae621 + upstream-source: ghcr.io/canonical/charmed-mysql@sha256:924cb913733002e0936a36b6edab0bb8a4d72f1b92be0e0b7a933e797b7ef215 assumes: - k8s-api diff --git a/src/workload.py b/src/workload.py index 15eece105..00b34ba0a 100644 --- a/src/workload.py +++ b/src/workload.py @@ -121,6 +121,10 @@ def shell(self) -> mysql_shell.Shell: def _bootstrap_router(self, *, tls: bool) -> None: """Bootstrap MySQL Router and enable service.""" + # TODO: remove when https://github.com/canonical/charmed-mysql-rock/pull/23 merged + self._container.exec(["mkdir", "-p", "/var/log/mysqlrouter"]) + self._container.exec(["chown", "-R", "584788", "/var/log/mysqlrouter"]) + logger.debug(f"Bootstrapping router {tls=}, {self._host=}, {self._port=}") try: # Bootstrap MySQL Router From 2b1d7cd32dd137ef9a41128b2fa38fc4e11917d4 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 2 May 2023 13:54:54 +0000 Subject: [PATCH 094/159] remove mkdir --- src/workload.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/workload.py b/src/workload.py index 00b34ba0a..15eece105 100644 --- a/src/workload.py +++ b/src/workload.py @@ -121,10 +121,6 @@ def shell(self) -> mysql_shell.Shell: def _bootstrap_router(self, *, tls: bool) -> None: """Bootstrap MySQL Router and enable service.""" - # TODO: remove when https://github.com/canonical/charmed-mysql-rock/pull/23 merged - self._container.exec(["mkdir", "-p", "/var/log/mysqlrouter"]) - self._container.exec(["chown", "-R", "584788", "/var/log/mysqlrouter"]) - logger.debug(f"Bootstrapping router {tls=}, {self._host=}, {self._port=}") try: # Bootstrap MySQL Router From d5e1cc709da3719bbad19ac62023a7e9a990087c Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 2 May 2023 13:55:00 +0000 Subject: [PATCH 095/159] fix tls --- src/relations/tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/relations/tls.py b/src/relations/tls.py index 1cd32b3fe..268c5c1b0 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -112,7 +112,7 @@ def save_certificate(self, event: tls_certificates.CertificateAvailableEvent) -> self.peer_unit_databag.active_csr = self.peer_unit_databag.requested_csr logger.debug(f"Saved TLS certificate {event=}") self._charm.workload.enable_tls( - self.peer_unit_databag.key, self.peer_unit_databag.certificate + key=self.peer_unit_databag.key, certificate=self.peer_unit_databag.certificate ) @staticmethod From 000fd264d222731dc8e9eb4f6a459db57ff08a00 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 2 May 2023 15:16:02 +0000 Subject: [PATCH 096/159] fix enabled check --- src/workload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index 15eece105..1d4b65f46 100644 --- a/src/workload.py +++ b/src/workload.py @@ -39,7 +39,7 @@ def _enabled(self) -> bool: service = self._container.get_services(self._SERVICE_NAME).get(self._SERVICE_NAME) if service is None: return False - return service.startup == "enabled" + return service.startup == ops.pebble.ServiceStartup.ENABLED @property def version(self) -> str: From 4a836949a6d679bb6b7dafbb1104ef0cb3479538 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 2 May 2023 15:16:25 +0000 Subject: [PATCH 097/159] test with personal image --- metadata.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metadata.yaml b/metadata.yaml index 48bf2cf27..ec39fc771 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -31,6 +31,7 @@ resources: mysql-router-image: type: oci-image description: OCI image for mysql-router - upstream-source: ghcr.io/canonical/charmed-mysql@sha256:924cb913733002e0936a36b6edab0bb8a4d72f1b92be0e0b7a933e797b7ef215 + # TODO: switch back to canonical image + upstream-source: ghcr.io/carlcsaposs-canonical/charmed-mysql@sha256:d848dc94c604758a32a5ddccfa2226f0d40524e24e32f1b69f994b3ebdb65561 assumes: - k8s-api From 1e6741c683ff8e19cfafe5024537705a4fdd3b01 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 2 May 2023 15:20:25 +0000 Subject: [PATCH 098/159] remove todo comment --- src/workload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index 1d4b65f46..9f5757af8 100644 --- a/src/workload.py +++ b/src/workload.py @@ -129,7 +129,7 @@ def _bootstrap_router(self, *, tls: bool) -> None: "mysqlrouter", "--bootstrap", f"{self._admin_username}:{self._admin_password}@{self._host}:{self._port}", - "--strict", # todo: does this do anything without `--account`? + "--strict", "--user", self._UNIX_USERNAME, "--conf-set-option", From 875b668fb431fbf4064bdf30c756cf17aa2538fc Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 3 May 2023 08:28:13 +0000 Subject: [PATCH 099/159] update rock --- metadata.yaml | 3 +-- src/workload.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/metadata.yaml b/metadata.yaml index ec39fc771..300ec2d8b 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -31,7 +31,6 @@ resources: mysql-router-image: type: oci-image description: OCI image for mysql-router - # TODO: switch back to canonical image - upstream-source: ghcr.io/carlcsaposs-canonical/charmed-mysql@sha256:d848dc94c604758a32a5ddccfa2226f0d40524e24e32f1b69f994b3ebdb65561 + upstream-source: ghcr.io/canonical/charmed-mysql@sha256:017605f168fcc569d10372bb74b29ef9041256bd066013dec39e9ceee8c88539 assumes: - k8s-api diff --git a/src/workload.py b/src/workload.py index 9f5757af8..86af9c274 100644 --- a/src/workload.py +++ b/src/workload.py @@ -88,7 +88,7 @@ def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: if tls: command = f"{command} --extra-config {self._ROUTER_CONFIG_DIRECTORY / self._TLS_CONFIG_FILE}" else: - command = "" + command = "exit 1" # command should not run TODO layer = ops.pebble.Layer( { "summary": "mysql router layer", @@ -106,7 +106,7 @@ def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: } ) self._container.add_layer(self._SERVICE_NAME, layer, combine=True) - self._container.replan() + self._container.replan() # don't replan? TODO @property def shell(self) -> mysql_shell.Shell: From e940743a1f0b860b273485316220fcdfc754acc3 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 3 May 2023 09:54:40 +0000 Subject: [PATCH 100/159] don't delete user during restart --- src/workload.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/workload.py b/src/workload.py index 86af9c274..e907d7a8b 100644 --- a/src/workload.py +++ b/src/workload.py @@ -81,14 +81,13 @@ def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: tls: Whether TLS is enabled. Required if enabled=True """ if enabled: - command = ( - f"mysqlrouter --config {self._ROUTER_CONFIG_DIRECTORY / self._ROUTER_CONFIG_FILE}" - ) assert tls is not None, "`tls` argument required when enabled=True" - if tls: - command = f"{command} --extra-config {self._ROUTER_CONFIG_DIRECTORY / self._TLS_CONFIG_FILE}" - else: - command = "exit 1" # command should not run TODO + command = ( + f"mysqlrouter --config {self._ROUTER_CONFIG_DIRECTORY / self._ROUTER_CONFIG_FILE}" + ) + if tls: + command = f"{command} --extra-config {self._ROUTER_CONFIG_DIRECTORY / self._TLS_CONFIG_FILE}" + startup = ops.pebble.ServiceStartup.ENABLED.value if enabled else ops.pebble.ServiceStartup.DISABLED.value layer = ops.pebble.Layer( { "summary": "mysql router layer", @@ -98,7 +97,7 @@ def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: "override": "replace", "summary": "mysql router", "command": command, - "startup": "enabled" if enabled else "disabled", + "startup": startup, "user": self._UNIX_USERNAME, "group": self._UNIX_USERNAME, }, @@ -158,6 +157,14 @@ def enable(self, *, tls: bool) -> None: logger.debug("Enabled MySQL Router service") self._charm.wait_until_mysql_router_ready() + def _restart(self, *, tls: bool) -> None: + """Restart MySQL Router to enable or disable TLS.""" + logger.debug("Restarting MySQL Router") + assert self._enabled is True + self._bootstrap_router(tls=tls) + logger.debug("Restarted MySQL Router") + self._charm.wait_until_mysql_router_ready() + @property def _router_username(self) -> str: """Read MySQL Router username from config file. @@ -179,14 +186,6 @@ def disable(self) -> None: self.shell.delete_user(self._router_username) logger.debug("Disabled MySQL Router service") - def _restart(self, *, tls: bool) -> None: - """Restart MySQL Router to enable or disable TLS.""" - logger.debug("Restarting MySQL Router") - assert self._enabled is True - self.disable() - self.enable(tls=tls) - logger.debug("Restarted MySQL Router") - def _write_file(self, path: pathlib.Path, content: str) -> None: """Write content to file. From 41c8cb91e724fea5d770e9c0ff41ad86d1eff5db Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 3 May 2023 09:55:10 +0000 Subject: [PATCH 101/159] remove todo comment --- src/workload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index e907d7a8b..19c4b765d 100644 --- a/src/workload.py +++ b/src/workload.py @@ -105,7 +105,7 @@ def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: } ) self._container.add_layer(self._SERVICE_NAME, layer, combine=True) - self._container.replan() # don't replan? TODO + self._container.replan() @property def shell(self) -> mysql_shell.Shell: From c9770b9271d9f518ad39b1745f81c512e3be99ec Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 3 May 2023 09:57:24 +0000 Subject: [PATCH 102/159] format --- src/workload.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/workload.py b/src/workload.py index 19c4b765d..f78613cc5 100644 --- a/src/workload.py +++ b/src/workload.py @@ -86,8 +86,13 @@ def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: f"mysqlrouter --config {self._ROUTER_CONFIG_DIRECTORY / self._ROUTER_CONFIG_FILE}" ) if tls: - command = f"{command} --extra-config {self._ROUTER_CONFIG_DIRECTORY / self._TLS_CONFIG_FILE}" - startup = ops.pebble.ServiceStartup.ENABLED.value if enabled else ops.pebble.ServiceStartup.DISABLED.value + command = ( + f"{command} --extra-config {self._ROUTER_CONFIG_DIRECTORY / self._TLS_CONFIG_FILE}" + ) + if enabled: + startup = ops.pebble.ServiceStartup.ENABLED.value + else: + startup = ops.pebble.ServiceStartup.DISABLED.value layer = ops.pebble.Layer( { "summary": "mysql router layer", From 662c7c4fab584e73cbb3d56a20b3015e82f512e0 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 3 May 2023 09:59:46 +0000 Subject: [PATCH 103/159] revert ci debugging --- .github/workflows/ci.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4c1bfdf7b..4a2c5477a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -52,6 +52,9 @@ jobs: - integration-database name: ${{ matrix.tox-environments }} needs: + - lint + # TODO: re-enable after adding unit tests + #- unit-test - build runs-on: ubuntu-latest timeout-minutes: 120 @@ -81,15 +84,9 @@ jobs: fi - name: Run integration tests # set a predictable model name so it can be consumed by charm-logdump-action - run: sg microk8s -c "tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --model testing --keep-models" + run: sg microk8s -c "tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --model testing" env: CI_PACKED_CHARMS: ${{ needs.build.outputs.charms }} - - name: Juju status - if: always() - run: juju status --model testing - - name: Juju debug log - if: always() - run: juju debug-log --model testing --replay --no-tail - name: Dump logs uses: canonical/charm-logdump-action@main if: failure() From 992c39ce59a7610d3fb6421e23da08e7bc3ae892 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 4 May 2023 09:19:16 +0000 Subject: [PATCH 104/159] remove delete user add attribute so mysql charm cleans up user --- src/charm.py | 8 ++++---- src/mysql_shell.py | 16 +++++++++++++++- src/relations/database_requires.py | 6 +++--- src/workload.py | 28 +++++++++++++++------------- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/src/charm.py b/src/charm.py index da79881e2..c6c70b3a5 100755 --- a/src/charm.py +++ b/src/charm.py @@ -231,11 +231,11 @@ def _reconcile_database_relations(self, event=None) -> None: if ( isinstance(self.workload, workload.AuthenticatedWorkload) and self.workload.container_ready + and not self.database_requires.relation.is_breaking(event) ): - if self.database_requires.relation.is_breaking(event): - self.workload.disable() - else: - self.workload.enable(tls=self.tls.certificate_saved) + self.workload.enable(tls=self.tls.certificate_saved) + else: + self.workload.disable() self._set_status() def _on_mysql_router_pebble_ready(self, _) -> None: diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 16e649f3b..2071df41b 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -7,6 +7,7 @@ """ import dataclasses +import json import logging import secrets import string @@ -26,6 +27,7 @@ class Shell: _password: str _host: str _port: str + _mysql_relation_id: int _TEMPORARY_SCRIPT_FILE = "/tmp/script.py" @@ -60,6 +62,14 @@ def _generate_password() -> str: choices = string.ascii_letters + string.digits return "".join([secrets.choice(choices) for _ in range(_PASSWORD_LENGTH)]) + @property + def _user_attributes(self) -> str: + # TODO: docstring + # TODO: explain that mysql will clean up users + # TODO update foo + # TODO: check int values accepted + return json.dumps({"mysql_relation_id": self._mysql_relation_id}) + def create_application_database_and_user(self, *, username: str, database: str) -> str: """Create database and user for related database_provides application.""" logger.debug(f"Creating {database=} and {username=}") @@ -67,13 +77,17 @@ def create_application_database_and_user(self, *, username: str, database: str) self._run_sql( [ f"CREATE DATABASE IF NOT EXISTS `{database}`", - f"CREATE USER `{username}` IDENTIFIED BY '{password}'", + f"CREATE USER `{username}` IDENTIFIED BY '{password}' ATTRIBUTE '{self._user_attributes}'" f"GRANT ALL PRIVILEGES ON `{database}`.* TO `{username}`", ] ) logger.debug(f"Created {database=} and {username=}") return password + def add_attributes_to_mysql_router_user(self, username: str) -> None: + # TODO: docstring + self._run_sql([f"ALTER USER `{username}` ATTRIBUTE '{self._user_attributes}'"]) + def delete_user(self, username: str) -> None: """Delete user.""" logger.debug(f"Deleting {username=}") diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 233feaa2c..470bd15cb 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -17,7 +17,7 @@ class _Relation: _interface: data_interfaces.DatabaseRequires @property - def _id(self) -> int: + def id(self) -> int: relations = self._interface.relations assert len(relations) == 1 return relations[0].id @@ -25,7 +25,7 @@ def _id(self) -> int: @property def _remote_databag(self) -> dict: """MySQL charm databag""" - return self._interface.fetch_relation_data()[self._id] + return self._interface.fetch_relation_data()[self.id] @property def _endpoint(self) -> str: @@ -56,7 +56,7 @@ def password(self) -> str: def is_breaking(self, event): """Whether relation will be broken after the current event is handled""" - return isinstance(event, ops.RelationBrokenEvent) and event.relation.id == self._id + return isinstance(event, ops.RelationBrokenEvent) and event.relation.id == self.id @dataclasses.dataclass diff --git a/src/workload.py b/src/workload.py index f78613cc5..8f38f48db 100644 --- a/src/workload.py +++ b/src/workload.py @@ -121,8 +121,21 @@ def shell(self) -> mysql_shell.Shell: _password=self._admin_password, _host=self._host, _port=self._port, + _mysql_relation_id=self._charm.database_requires.relation.id, ) + @property + def _router_username(self) -> str: + """Read MySQL Router username from config file. + + During bootstrap, MySQL Router creates a config file at + `/etc/mysqlrouter/mysqlrouter.conf`. This file contains the username that was created + during bootstrap. + """ + config = configparser.ConfigParser() + config.read_file(self._container.pull("/etc/mysqlrouter/mysqlrouter.conf")) + return config["metadata_cache:bootstrap"]["user"] + def _bootstrap_router(self, *, tls: bool) -> None: """Bootstrap MySQL Router and enable service.""" logger.debug(f"Bootstrapping router {tls=}, {self._host=}, {self._port=}") @@ -145,6 +158,8 @@ def _bootstrap_router(self, *, tls: bool) -> None: except ops.pebble.ExecError as e: logger.exception(f"Failed to bootstrap router\nstderr:\n{e.stderr}\n") raise + # TODO: test that this can safely run more than once + self.shell.add_attributes_to_mysql_router_user(self._router_username) # Enable service self._update_layer(enabled=True, tls=tls) @@ -170,25 +185,12 @@ def _restart(self, *, tls: bool) -> None: logger.debug("Restarted MySQL Router") self._charm.wait_until_mysql_router_ready() - @property - def _router_username(self) -> str: - """Read MySQL Router username from config file. - - During bootstrap, MySQL Router creates a config file at - `/etc/mysqlrouter/mysqlrouter.conf`. This file contains the username that was created - during bootstrap. - """ - config = configparser.ConfigParser() - config.read_file(self._container.pull("/etc/mysqlrouter/mysqlrouter.conf")) - return config["metadata_cache:bootstrap"]["user"] - def disable(self) -> None: """Stop and disable MySQL Router service.""" if not self._enabled: return logger.debug("Disabling MySQL Router service") self._update_layer(enabled=False) - self.shell.delete_user(self._router_username) logger.debug("Disabled MySQL Router service") def _write_file(self, path: pathlib.Path, content: str) -> None: From f5713feeafd04e4ed9ad10a8dc6d963d144ddd6e Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 4 May 2023 11:41:51 +0000 Subject: [PATCH 105/159] try workflow branch --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a2c5477a..bcd8f0379 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,8 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 + # TODO: revert + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@charmcraft-2.3 integration-test: strategy: From e0681bb5781afe45aa7fd936a8f9b5767abdef42 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 07:07:24 +0000 Subject: [PATCH 106/159] debug workflow --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bcd8f0379..927e0cf59 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -43,7 +43,7 @@ jobs: build: name: Build charms # TODO: revert - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@charmcraft-2.3 + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@charmcraft-2.3-debug integration-test: strategy: From fb0b724ac3b41b0fbc947d38e6620de4c957989a Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 07:25:17 +0000 Subject: [PATCH 107/159] Revert "debug workflow" This reverts commit e0681bb5781afe45aa7fd936a8f9b5767abdef42. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 927e0cf59..bcd8f0379 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -43,7 +43,7 @@ jobs: build: name: Build charms # TODO: revert - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@charmcraft-2.3-debug + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@charmcraft-2.3 integration-test: strategy: From b4ab5b448e61f737cc19e9bd194d1ae6c5c9de12 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 07:34:19 +0000 Subject: [PATCH 108/159] move handler registration to __init__ in relations modules --- src/charm.py | 39 ++++-------------------------- src/relations/database_provides.py | 22 +++++++++++++---- src/relations/database_requires.py | 28 +++++++++++++++++---- src/relations/tls.py | 7 +++--- 4 files changed, 49 insertions(+), 47 deletions(-) diff --git a/src/charm.py b/src/charm.py index c6c70b3a5..549fb0ec0 100755 --- a/src/charm.py +++ b/src/charm.py @@ -31,38 +31,9 @@ class MySQLRouterOperatorCharm(ops.CharmBase): def __init__(self, *args) -> None: super().__init__(*args) - self.database_requires = relations.database_requires.RelationEndpoint( - data_interfaces.DatabaseRequires( - self, - relation_name=relations.database_requires.RelationEndpoint.NAME, - # HACK: MySQL Router needs a user, but not a database - # Use the DatabaseRequires interface to get a user; disregard the database - database_name="_unused_mysqlrouter_database", - extra_user_roles="mysqlrouter", - ) - ) - self.framework.observe( - self.database_requires.interface.on.database_created, - self._reconcile_database_relations, - ) - self.framework.observe( - self.on[relations.database_requires.RelationEndpoint.NAME].relation_broken, - self._reconcile_database_relations, - ) + self.database_requires = relations.database_requires.RelationEndpoint(self) - self.database_provides = relations.database_provides.RelationEndpoint( - data_interfaces.DatabaseProvides( - self, relation_name=relations.database_provides.RelationEndpoint.NAME - ) - ) - self.framework.observe( - self.database_provides.interface.on.database_requested, - self._reconcile_database_relations, - ) - self.framework.observe( - self.on[relations.database_provides.RelationEndpoint.NAME].relation_broken, - self._reconcile_database_relations, - ) + self.database_provides = relations.database_provides.RelationEndpoint(self) self.framework.observe( getattr(self.on, "mysql_router_pebble_ready"), self._on_mysql_router_pebble_ready @@ -71,7 +42,7 @@ def __init__(self, *args) -> None: # Start workload after pod churn or charm upgrade # (https://juju.is/docs/sdk/start-event#heading--emission-sequence) # Also, set status on first start if no relations active - self.framework.observe(self.on.start, self._reconcile_database_relations) + self.framework.observe(self.on.start, self.reconcile_database_relations) self.framework.observe(self.on.leader_elected, self._on_leader_elected) @@ -206,7 +177,7 @@ def _patch_service(self, *, name: str, ro_port: int, rw_port: int) -> None: # Handlers # ======================= - def _reconcile_database_relations(self, event=None) -> None: + def reconcile_database_relations(self, event=None) -> None: """Handle database requires/provides events.""" logger.debug( "State of reconcile " @@ -240,7 +211,7 @@ def _reconcile_database_relations(self, event=None) -> None: def _on_mysql_router_pebble_ready(self, _) -> None: self.unit.set_workload_version(self.workload.version) - self._reconcile_database_relations() + self.reconcile_database_relations() def _on_leader_elected(self, _) -> None: """Patch existing k8s service to include read-write and read-only services.""" diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 26e461c49..9f07d36b6 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -5,12 +5,16 @@ import dataclasses import logging +import typing import charms.data_platform_libs.v0.data_interfaces as data_interfaces import ops import mysql_shell +if typing.TYPE_CHECKING: + import charm + logger = logging.getLogger(__name__) @@ -87,19 +91,27 @@ def delete_user(self, *, shell: mysql_shell.Shell) -> None: shell.delete_user(self.username) -@dataclasses.dataclass class RelationEndpoint: """Relation endpoint for application charm(s)""" - interface: data_interfaces.DatabaseProvides - NAME = "database" + def __init__(self, charm_: "charm.MySQLRouterOperatorCharm") -> None: + self._interface = data_interfaces.DatabaseProvides(charm_, relation_name=self.NAME) + charm_.framework.observe( + self._interface.on.database_requested, + charm_.reconcile_database_relations, + ) + charm_.framework.observe( + charm_.on[self.NAME].relation_broken, + charm_.reconcile_database_relations, + ) + @property def _relations(self) -> list[_Relation]: return [ - _Relation(_relation=relation, _interface=self.interface) - for relation in self.interface.relations + _Relation(_relation=relation, _interface=self._interface) + for relation in self._interface.relations ] def _requested_users( diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 470bd15cb..7dcc93f56 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -9,6 +9,9 @@ import charms.data_platform_libs.v0.data_interfaces as data_interfaces import ops +if typing.TYPE_CHECKING: + import charm + @dataclasses.dataclass class _Relation: @@ -59,20 +62,35 @@ def is_breaking(self, event): return isinstance(event, ops.RelationBrokenEvent) and event.relation.id == self.id -@dataclasses.dataclass class RelationEndpoint: """Relation endpoint for MySQL charm""" - interface: data_interfaces.DatabaseRequires - NAME = "backend-database" + def __init__(self, charm_: "charm.MySQLRouterOperatorCharm") -> None: + self._interface = data_interfaces.DatabaseRequires( + charm_, + relation_name=self.NAME, + # HACK: MySQL Router needs a user, but not a database + # Use the DatabaseRequires interface to get a user; disregard the database + database_name="_unused_mysqlrouter_database", + extra_user_roles="mysqlrouter", + ) + charm_.framework.observe( + self._interface.on.database_created, + charm_.reconcile_database_relations, + ) + charm_.framework.observe( + charm_.on[self.NAME].relation_broken, + charm_.reconcile_database_relations, + ) + @property def relation(self) -> typing.Optional[_Relation]: """Relation to MySQL charm""" - if not self.interface.is_resource_created(): + if not self._interface.is_resource_created(): return - return _Relation(self.interface) + return _Relation(self._interface) @property def missing_relation(self) -> bool: diff --git a/src/relations/tls.py b/src/relations/tls.py index 268c5c1b0..6c1cb580c 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -15,7 +15,8 @@ import charms.tls_certificates_interface.v1.tls_certificates as tls_certificates import ops -import charm +if typing.TYPE_CHECKING: + import charm _PEER_RELATION_ENDPOINT_NAME = "mysql-router-peers" logger = logging.getLogger(__name__) @@ -68,7 +69,7 @@ def clear(self) -> None: class _Relation: """Relation to TLS certificate provider""" - _charm: charm.MySQLRouterOperatorCharm + _charm: "charm.MySQLRouterOperatorCharm" _interface: tls_certificates.TLSCertificatesRequiresV1 @property @@ -182,7 +183,7 @@ class RelationEndpoint(ops.Object): NAME = "certificates" - def __init__(self, charm_: charm.MySQLRouterOperatorCharm): + def __init__(self, charm_: "charm.MySQLRouterOperatorCharm"): super().__init__(charm_, self.NAME) self._charm = charm_ self._interface = tls_certificates.TLSCertificatesRequiresV1(self._charm, self.NAME) From 7d2cee530950b11b86adeb6e15f2708b5f1cf0c6 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 07:47:20 +0000 Subject: [PATCH 109/159] move disable method from AuthenticatedWorload to Workload --- src/workload.py | 60 ++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/workload.py b/src/workload.py index 8f38f48db..cb2d75922 100644 --- a/src/workload.py +++ b/src/workload.py @@ -27,6 +27,10 @@ class Workload: CONTAINER_NAME = "mysql-router" _SERVICE_NAME = "mysql_router" + _UNIX_USERNAME = "mysql" + _ROUTER_CONFIG_DIRECTORY = pathlib.Path("/etc/mysqlrouter") + _ROUTER_CONFIG_FILE = "mysqlrouter.conf" + _TLS_CONFIG_FILE = "tls.conf" @property def container_ready(self) -> bool: @@ -51,28 +55,6 @@ def version(self) -> str: return version return "" - -@dataclasses.dataclass(kw_only=True) -class AuthenticatedWorkload(Workload): - """Workload with connection to MySQL cluster""" - - # Admin user with permission to: - # - Create databases & users - # - Grant all privileges on a database to a user - # (Different from user that MySQL Router runs with after bootstrap.) - _admin_username: str - _admin_password: str - _host: str - _port: str - _charm: "charm.MySQLRouterOperatorCharm" - - _UNIX_USERNAME = "mysql" - _ROUTER_CONFIG_DIRECTORY = pathlib.Path("/etc/mysqlrouter") - _ROUTER_CONFIG_FILE = "mysqlrouter.conf" - _TLS_CONFIG_FILE = "tls.conf" - _TLS_KEY_FILE = "custom-key.pem" - _TLS_CERTIFICATE_FILE = "custom-certificate.pem" - def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: """Update and restart services. @@ -112,6 +94,32 @@ def _update_layer(self, *, enabled: bool, tls: bool = None) -> None: self._container.add_layer(self._SERVICE_NAME, layer, combine=True) self._container.replan() + def disable(self) -> None: + """Stop and disable MySQL Router service.""" + if not self._enabled: + return + logger.debug("Disabling MySQL Router service") + self._update_layer(enabled=False) + logger.debug("Disabled MySQL Router service") + + +@dataclasses.dataclass(kw_only=True) +class AuthenticatedWorkload(Workload): + """Workload with connection to MySQL cluster""" + + # Admin user with permission to: + # - Create databases & users + # - Grant all privileges on a database to a user + # (Different from user that MySQL Router runs with after bootstrap.) + _admin_username: str + _admin_password: str + _host: str + _port: str + _charm: "charm.MySQLRouterOperatorCharm" + + _TLS_KEY_FILE = "custom-key.pem" + _TLS_CERTIFICATE_FILE = "custom-certificate.pem" + @property def shell(self) -> mysql_shell.Shell: """MySQL Shell""" @@ -185,14 +193,6 @@ def _restart(self, *, tls: bool) -> None: logger.debug("Restarted MySQL Router") self._charm.wait_until_mysql_router_ready() - def disable(self) -> None: - """Stop and disable MySQL Router service.""" - if not self._enabled: - return - logger.debug("Disabling MySQL Router service") - self._update_layer(enabled=False) - logger.debug("Disabled MySQL Router service") - def _write_file(self, path: pathlib.Path, content: str) -> None: """Write content to file. From df827f6e9d6e7b1e525bb4692ef523a04f05c276 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 08:49:44 +0000 Subject: [PATCH 110/159] add missing comma --- src/mysql_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 2071df41b..6009193b5 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -77,7 +77,7 @@ def create_application_database_and_user(self, *, username: str, database: str) self._run_sql( [ f"CREATE DATABASE IF NOT EXISTS `{database}`", - f"CREATE USER `{username}` IDENTIFIED BY '{password}' ATTRIBUTE '{self._user_attributes}'" + f"CREATE USER `{username}` IDENTIFIED BY '{password}' ATTRIBUTE '{self._user_attributes}'", f"GRANT ALL PRIVILEGES ON `{database}`.* TO `{username}`", ] ) From 3d8c60ae67a0fd19bdf13a16edc7a2ff49ba754b Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 08:57:44 +0000 Subject: [PATCH 111/159] Revert "try workflow branch" This reverts commit f5713feeafd04e4ed9ad10a8dc6d963d144ddd6e. --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bcd8f0379..4a2c5477a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,8 +42,7 @@ jobs: build: name: Build charms - # TODO: revert - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@charmcraft-2.3 + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 integration-test: strategy: From 1c099014d3996945c3a5566b8cda78aa2d5eb9f7 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 09:36:54 +0000 Subject: [PATCH 112/159] debug ci --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a2c5477a..7b60ab03c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,7 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@charmcraft-2.3-debug integration-test: strategy: From 547f2f70f10d13f834226df01ce20fc3fdf1f511 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 09:42:49 +0000 Subject: [PATCH 113/159] Don't wait for all apps to be ready before relating in integration test --- tests/integration/test_charm.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index eaf6ac33c..7a3443dcd 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -69,26 +69,12 @@ async def test_database_relation(ops_test: OpsTest): mysql_app, application_app = applications[0], applications[2] - logger.info("Waiting for mysql, mysqlrouter and application to be ready") async with ops_test.fast_forward(): - await asyncio.gather( - ops_test.model.wait_for_idle( - apps=[MYSQL_APP_NAME], - status="active", - raise_on_blocked=True, - timeout=SLOW_TIMEOUT, - ), - ops_test.model.wait_for_idle( - apps=[MYSQL_ROUTER_APP_NAME], - status="blocked", - timeout=SLOW_TIMEOUT, - ), - ops_test.model.wait_for_idle( - apps=[APPLICATION_APP_NAME], - status="waiting", - raise_on_blocked=True, - timeout=SLOW_TIMEOUT, - ), + logger.info("Waiting for mysqlrouter to be in BlockedStatus") + await ops_test.model.wait_for_idle( + apps=[MYSQL_ROUTER_APP_NAME], + status="blocked", + timeout=SLOW_TIMEOUT, ) logger.info("Relating mysql, mysqlrouter and application") From 1441be257be2dbb2dca2b17198c54677fb5fb003 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 09:46:27 +0000 Subject: [PATCH 114/159] remove unused import --- src/charm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 549fb0ec0..d9fd434ff 100755 --- a/src/charm.py +++ b/src/charm.py @@ -9,7 +9,6 @@ import logging import socket -import charms.data_platform_libs.v0.data_interfaces as data_interfaces import lightkube import lightkube.models.core_v1 import lightkube.models.meta_v1 From 84aba2cd6e4cbd02227504ca21e437eda7f92a81 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 09:46:43 +0000 Subject: [PATCH 115/159] remove todo comment --- src/workload.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index cb2d75922..56f2a5c82 100644 --- a/src/workload.py +++ b/src/workload.py @@ -166,7 +166,6 @@ def _bootstrap_router(self, *, tls: bool) -> None: except ops.pebble.ExecError as e: logger.exception(f"Failed to bootstrap router\nstderr:\n{e.stderr}\n") raise - # TODO: test that this can safely run more than once self.shell.add_attributes_to_mysql_router_user(self._router_username) # Enable service self._update_layer(enabled=True, tls=tls) From d1b79d96d442ac99ea4a74463ad4d74055db7bd8 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 09:50:09 +0000 Subject: [PATCH 116/159] Revert "debug ci" This reverts commit 1c099014d3996945c3a5566b8cda78aa2d5eb9f7. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7b60ab03c..4a2c5477a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,7 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@charmcraft-2.3-debug + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 integration-test: strategy: From f597fe723bdcc21865d7dd63bda792b7d65f1d9d Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 09:54:56 +0000 Subject: [PATCH 117/159] add docstrings --- src/mysql_shell.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 6009193b5..bc17937a1 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -64,10 +64,11 @@ def _generate_password() -> str: @property def _user_attributes(self) -> str: - # TODO: docstring - # TODO: explain that mysql will clean up users - # TODO update foo - # TODO: check int values accepted + """Attributes for (MySQL) users created by this charm. + + If the relation with the MySQL charm is broken, the MySQL charm will use this attribute + to delete all users created by this charm. + """ return json.dumps({"mysql_relation_id": self._mysql_relation_id}) def create_application_database_and_user(self, *, username: str, database: str) -> str: @@ -85,7 +86,7 @@ def create_application_database_and_user(self, *, username: str, database: str) return password def add_attributes_to_mysql_router_user(self, username: str) -> None: - # TODO: docstring + """Add attributes to user created during MySQL Router bootstrap.""" self._run_sql([f"ALTER USER `{username}` ATTRIBUTE '{self._user_attributes}'"]) def delete_user(self, username: str) -> None: From a3555dad53f4ce664022e1bfb80c5a161805bf9d Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 09:57:36 +0000 Subject: [PATCH 118/159] Revert "Revert "debug ci"" This reverts commit d1b79d96d442ac99ea4a74463ad4d74055db7bd8. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a2c5477a..7b60ab03c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,7 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@charmcraft-2.3-debug integration-test: strategy: From 9b0c38797521c7a8c7de1c99358d6590ad8068cd Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 10:26:48 +0000 Subject: [PATCH 119/159] Revert "Revert "Revert "debug ci""" This reverts commit a3555dad53f4ce664022e1bfb80c5a161805bf9d. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7b60ab03c..4a2c5477a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,7 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@charmcraft-2.3-debug + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 integration-test: strategy: From 4477ed3caa8111f61995f00a420612c20db0ed89 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 5 May 2023 09:36:54 +0000 Subject: [PATCH 120/159] debug ci --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a2c5477a..7b60ab03c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,7 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@charmcraft-2.3-debug integration-test: strategy: From 0e8690681295052f15b44b331d729cee756b1b02 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 8 May 2023 11:05:30 +0000 Subject: [PATCH 121/159] only disable workload if container ready --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index d9fd434ff..df48e1e12 100755 --- a/src/charm.py +++ b/src/charm.py @@ -204,7 +204,7 @@ def reconcile_database_relations(self, event=None) -> None: and not self.database_requires.relation.is_breaking(event) ): self.workload.enable(tls=self.tls.certificate_saved) - else: + elif self.workload.container_ready: self.workload.disable() self._set_status() From 9237e7d39bcc5f3f4f802465f886b16a4783e2d6 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 8 May 2023 11:37:57 +0000 Subject: [PATCH 122/159] Remove deletion of users when mysql charm relation broken --- src/charm.py | 3 --- src/relations/database_provides.py | 27 ++++++++++----------------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/charm.py b/src/charm.py index df48e1e12..0ee02250c 100755 --- a/src/charm.py +++ b/src/charm.py @@ -192,9 +192,6 @@ def reconcile_database_relations(self, event=None) -> None: ): self.database_provides.reconcile_users( event=event, - event_is_database_requires_broken=self.database_requires.relation.is_breaking( - event - ), router_endpoint=self._endpoint, shell=self.workload.shell, ) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 9f07d36b6..d56ca7978 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -114,13 +114,8 @@ def _relations(self) -> list[_Relation]: for relation in self._interface.relations ] - def _requested_users( - self, *, event, event_is_database_requires_broken: bool - ) -> list[_Relation]: + def _requested_users(self, *, event) -> list[_Relation]: """Related application charms that have requested a database & user""" - if event_is_database_requires_broken: - # MySQL cluster connection is being removed; delete all users - return [] requested_users = [] for relation in self._relations: if isinstance(event, ops.RelationBrokenEvent) and event.relation.id == relation.id: @@ -143,17 +138,17 @@ def reconcile_users( self, *, event, - event_is_database_requires_broken: bool, router_endpoint: str, shell: mysql_shell.Shell, ) -> None: - """Create requested users and delete inactive users.""" - logger.debug( - f"Reconciling users {event=}, {event_is_database_requires_broken=}, {router_endpoint=}" - ) - requested_users = self._requested_users( - event=event, event_is_database_requires_broken=event_is_database_requires_broken - ) + """Create requested users and delete inactive users. + + When the relation to the MySQL charm is broken, the MySQL charm will delete all users + created by this charm. Therefore, this charm does not need to delete users when that + relation is broken. + """ + logger.debug(f"Reconciling users {event=}, {router_endpoint=}") + requested_users = self._requested_users(event=event) created_users = self._created_users logger.debug(f"State of reconcile users {requested_users=}, {created_users=}") for relation in requested_users: @@ -162,6 +157,4 @@ def reconcile_users( for relation in created_users: if relation not in requested_users: relation.delete_user(shell=shell) - logger.debug( - f"Reconciled users {event=}, {event_is_database_requires_broken=}, {router_endpoint=}" - ) + logger.debug(f"Reconciled users {event=}, {router_endpoint=}") From a9b7272684fbe9e4e92a59c7d18a839eebbaaa94 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 8 May 2023 12:00:53 +0000 Subject: [PATCH 123/159] Use WaitingStatus while mysql relation is active but resource unavailable --- src/charm.py | 2 ++ src/relations/database_requires.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/charm.py b/src/charm.py index 0ee02250c..841360a8a 100755 --- a/src/charm.py +++ b/src/charm.py @@ -79,6 +79,8 @@ def _determine_status(self) -> ops.StatusBase: return ops.BlockedStatus( f"Missing relation{'s' if len(missing_relations) > 1 else ''}: {', '.join(missing_relations)}" ) + if self.database_requires.waiting_for_resource: + return ops.WaitingStatus(f"Waiting for related application: {self.database_requires.NAME}") if not self.workload.container_ready: return ops.MaintenanceStatus("Waiting for container") return ops.ActiveStatus() diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 7dcc93f56..d44e28eb1 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -95,4 +95,9 @@ def relation(self) -> typing.Optional[_Relation]: @property def missing_relation(self) -> bool: """Whether relation to MySQL charm does not exist""" + return len(self._interface.relations) == 0 + + @property + def waiting_for_resource(self) -> bool: + """Whether resource (database & user) has not been created by the MySQL charm""" return self.relation is None From afd0e3522b425c12e8474a19ebc103566bbc74fe Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 8 May 2023 12:09:25 +0000 Subject: [PATCH 124/159] format --- src/charm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 841360a8a..69978a5b2 100755 --- a/src/charm.py +++ b/src/charm.py @@ -80,7 +80,9 @@ def _determine_status(self) -> ops.StatusBase: f"Missing relation{'s' if len(missing_relations) > 1 else ''}: {', '.join(missing_relations)}" ) if self.database_requires.waiting_for_resource: - return ops.WaitingStatus(f"Waiting for related application: {self.database_requires.NAME}") + return ops.WaitingStatus( + f"Waiting for related application: {self.database_requires.NAME}" + ) if not self.workload.container_ready: return ops.MaintenanceStatus("Waiting for container") return ops.ActiveStatus() From a677943bdeac0adf08b0a06cd2f8abc1d443add0 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 8 May 2023 16:15:00 +0000 Subject: [PATCH 125/159] fix status handling on relation broken --- src/charm.py | 15 +++++++-------- src/relations/database_provides.py | 14 ++++++++++---- src/relations/database_requires.py | 7 ++++--- src/workload.py | 2 ++ 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/charm.py b/src/charm.py index 69978a5b2..86b1de34b 100755 --- a/src/charm.py +++ b/src/charm.py @@ -69,12 +69,12 @@ def _endpoint(self) -> str: """K8s endpoint for MySQL Router""" return f"{self.model.app.name}.{self.model.name}.svc.cluster.local" - def _determine_status(self) -> ops.StatusBase: + def _determine_status(self, event) -> ops.StatusBase: """Report charm status.""" missing_relations = [] - for relation in [self.database_requires, self.database_provides]: - if relation.missing_relation: - missing_relations.append(relation.NAME) + for endpoint in [self.database_requires, self.database_provides]: + if endpoint.is_missing_relation(event): + missing_relations.append(endpoint.NAME) if missing_relations: return ops.BlockedStatus( f"Missing relation{'s' if len(missing_relations) > 1 else ''}: {', '.join(missing_relations)}" @@ -87,7 +87,7 @@ def _determine_status(self) -> ops.StatusBase: return ops.MaintenanceStatus("Waiting for container") return ops.ActiveStatus() - def _set_status(self) -> None: + def set_status(self, event) -> None: """Set charm status. Except if charm is in unrecognized state @@ -96,7 +96,7 @@ def _set_status(self) -> None: self.unit.status, ops.BlockedStatus ) and not self.unit.status.message.startswith("Missing relation"): return - self.unit.status = self._determine_status() + self.unit.status = self._determine_status(event) logger.debug(f"Set status to {self.unit.status}") def wait_until_mysql_router_ready(self) -> None: @@ -121,7 +121,6 @@ def wait_until_mysql_router_ready(self) -> None: raise else: logger.debug("MySQL Router is ready") - self._set_status() def _patch_service(self, *, name: str, ro_port: int, rw_port: int) -> None: """Patch Juju-created k8s service. @@ -207,7 +206,7 @@ def reconcile_database_relations(self, event=None) -> None: self.workload.enable(tls=self.tls.certificate_saved) elif self.workload.container_ready: self.workload.disable() - self._set_status() + self.set_status(event) def _on_mysql_router_pebble_ready(self, _) -> None: self.unit.set_workload_version(self.workload.version) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index d56ca7978..e78ace979 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -90,6 +90,10 @@ def delete_user(self, *, shell: mysql_shell.Shell) -> None: self._delete_databag() shell.delete_user(self.username) + def is_breaking(self, event): + """Whether relation will be broken after the current event is handled""" + return isinstance(event, ops.RelationBrokenEvent) and event.relation.id == self.id + class RelationEndpoint: """Relation endpoint for application charm(s)""" @@ -129,10 +133,12 @@ def _created_users(self) -> list[_Relation]: """Users that have been created and shared with an application charm""" return [relation for relation in self._relations if relation.user_created] - @property - def missing_relation(self) -> bool: - """Whether zero relations to application charms exist""" - return len(self._relations) == 0 + def is_missing_relation(self, event) -> bool: + """Whether zero relations to application charms (will) exist""" + for relation in self._relations: + if not relation.is_breaking(event): + return False + return True def reconcile_users( self, diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index d44e28eb1..2dc512a50 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -92,9 +92,10 @@ def relation(self) -> typing.Optional[_Relation]: return return _Relation(self._interface) - @property - def missing_relation(self) -> bool: - """Whether relation to MySQL charm does not exist""" + def is_missing_relation(self, event) -> bool: + """Whether relation to MySQL charm does (or will) not exist""" + if self.relation.is_breaking(event): + return True return len(self._interface.relations) == 0 @property diff --git a/src/workload.py b/src/workload.py index 56f2a5c82..ff32bc05d 100644 --- a/src/workload.py +++ b/src/workload.py @@ -191,6 +191,8 @@ def _restart(self, *, tls: bool) -> None: self._bootstrap_router(tls=tls) logger.debug("Restarted MySQL Router") self._charm.wait_until_mysql_router_ready() + # wait_until_mysql_router_ready will set WaitingStatus—override it with current charm status + self._charm.set_status(event=None) def _write_file(self, path: pathlib.Path, content: str) -> None: """Write content to file. From 2c901191390e18da006515f04d418da456185939 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 8 May 2023 16:18:00 +0000 Subject: [PATCH 126/159] fix --- src/relations/database_requires.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 2dc512a50..594386c23 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -94,7 +94,7 @@ def relation(self) -> typing.Optional[_Relation]: def is_missing_relation(self, event) -> bool: """Whether relation to MySQL charm does (or will) not exist""" - if self.relation.is_breaking(event): + if self.relation is not None and self.relation.is_breaking(event): return True return len(self._interface.relations) == 0 From 71993e234baa3fedee608ed66308b1ce2f6e856f Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 8 May 2023 16:26:27 +0000 Subject: [PATCH 127/159] fix --- src/relations/database_requires.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 594386c23..98fc98169 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -94,7 +94,8 @@ def relation(self) -> typing.Optional[_Relation]: def is_missing_relation(self, event) -> bool: """Whether relation to MySQL charm does (or will) not exist""" - if self.relation is not None and self.relation.is_breaking(event): + # Cannot use `self.relation.is_breaking()` in case relation exists but resource not created + if self._interface.relations and _Relation(self._interface).is_breaking(event): return True return len(self._interface.relations) == 0 From 73d3a7313f3ea69d1801b2569a138f0b9da69db2 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 8 May 2023 19:28:21 +0000 Subject: [PATCH 128/159] alphabetize --- metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata.yaml b/metadata.yaml index 300ec2d8b..94c291e97 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -3,9 +3,9 @@ name: mysql-router-k8s display-name: MySQL Router maintainers: + - Carl Csaposs - Paulo Machado - Shayan Patel - - Carl Csaposs description: | K8S charmed operator for mysql-router. summary: | From a9ff4a1213bbf052e9f4585f915c5023221ba457 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 9 May 2023 10:07:11 +0000 Subject: [PATCH 129/159] lint --- src/workload.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index ff32bc05d..b36e833d3 100644 --- a/src/workload.py +++ b/src/workload.py @@ -191,7 +191,8 @@ def _restart(self, *, tls: bool) -> None: self._bootstrap_router(tls=tls) logger.debug("Restarted MySQL Router") self._charm.wait_until_mysql_router_ready() - # wait_until_mysql_router_ready will set WaitingStatus—override it with current charm status + # wait_until_mysql_router_ready will set WaitingStatus—override it with current charm + # status self._charm.set_status(event=None) def _write_file(self, path: pathlib.Path, content: str) -> None: From c4357fead700978890504985e4dc02fc9cb09ba4 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 9 May 2023 11:12:40 +0000 Subject: [PATCH 130/159] Add model name to database username --- src/relations/database_provides.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index e78ace979..66d4bc39b 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -24,6 +24,7 @@ class _Relation: _relation: ops.Relation _interface: data_interfaces.DatabaseProvides + _model_name: str @property def id(self) -> int: @@ -55,7 +56,10 @@ def database(self) -> str: @property def username(self) -> str: """Database username""" - return f"relation-{self.id}" + # Model name is necessary to create a unique username if MySQL Router is deployed in a + # different Juju model from MySQL. + # (Relation IDs are only unique within a Juju model.) + return f"relation-{self._model_name}-{self.id}" def _set_databag(self, *, password: str, router_endpoint: str) -> None: """Share connection information with application charm.""" @@ -102,6 +106,7 @@ class RelationEndpoint: def __init__(self, charm_: "charm.MySQLRouterOperatorCharm") -> None: self._interface = data_interfaces.DatabaseProvides(charm_, relation_name=self.NAME) + self._model_name = charm_.model.name charm_.framework.observe( self._interface.on.database_requested, charm_.reconcile_database_relations, @@ -114,7 +119,7 @@ def __init__(self, charm_: "charm.MySQLRouterOperatorCharm") -> None: @property def _relations(self) -> list[_Relation]: return [ - _Relation(_relation=relation, _interface=self._interface) + _Relation(_relation=relation, _interface=self._interface, _model_name=self._model_name) for relation in self._interface.relations ] From 80ef5ad0abb1c8c1ffd176e29480a4e2dcfe150f Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 9 May 2023 13:27:28 +0000 Subject: [PATCH 131/159] fix private --- src/relations/database_provides.py | 16 ++++++++-------- src/relations/database_requires.py | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index 66d4bc39b..a9a6da3e7 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -49,12 +49,12 @@ def user_created(self) -> bool: return True @property - def database(self) -> str: + def _database(self) -> str: """Requested database name""" return self._remote_databag["database"] @property - def username(self) -> str: + def _username(self) -> str: """Database username""" # Model name is necessary to create a unique username if MySQL Router is deployed in a # different Juju model from MySQL. @@ -66,14 +66,14 @@ def _set_databag(self, *, password: str, router_endpoint: str) -> None: read_write_endpoint = f"{router_endpoint}:6446" read_only_endpoint = f"{router_endpoint}:6447" logger.debug( - f"Setting databag {self.id=} {self.database=}, {self.username=}, {read_write_endpoint=}, {read_only_endpoint=}" + f"Setting databag {self.id=} {self._database=}, {self._username=}, {read_write_endpoint=}, {read_only_endpoint=}" ) - self._interface.set_database(self.id, self.database) - self._interface.set_credentials(self.id, self.username, password) + self._interface.set_database(self.id, self._database) + self._interface.set_credentials(self.id, self._username, password) self._interface.set_endpoints(self.id, read_write_endpoint) self._interface.set_read_only_endpoints(self.id, read_only_endpoint) logger.debug( - f"Set databag {self.id=} {self.database=}, {self.username=}, {read_write_endpoint=}, {read_only_endpoint=}" + f"Set databag {self.id=} {self._database=}, {self._username=}, {read_write_endpoint=}, {read_only_endpoint=}" ) def _delete_databag(self) -> None: @@ -85,14 +85,14 @@ def _delete_databag(self) -> None: def create_database_and_user(self, *, router_endpoint: str, shell: mysql_shell.Shell) -> None: """Create database & user and update databag.""" password = shell.create_application_database_and_user( - username=self.username, database=self.database + username=self._username, database=self._database ) self._set_databag(password=password, router_endpoint=router_endpoint) def delete_user(self, *, shell: mysql_shell.Shell) -> None: """Delete user and update databag.""" self._delete_databag() - shell.delete_user(self.username) + shell.delete_user(self._username) def is_breaking(self, event): """Whether relation will be broken after the current event is handled""" diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 98fc98169..ffe46f685 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -20,7 +20,7 @@ class _Relation: _interface: data_interfaces.DatabaseRequires @property - def id(self) -> int: + def _id(self) -> int: relations = self._interface.relations assert len(relations) == 1 return relations[0].id @@ -28,7 +28,7 @@ def id(self) -> int: @property def _remote_databag(self) -> dict: """MySQL charm databag""" - return self._interface.fetch_relation_data()[self.id] + return self._interface.fetch_relation_data()[self._id] @property def _endpoint(self) -> str: @@ -59,7 +59,7 @@ def password(self) -> str: def is_breaking(self, event): """Whether relation will be broken after the current event is handled""" - return isinstance(event, ops.RelationBrokenEvent) and event.relation.id == self.id + return isinstance(event, ops.RelationBrokenEvent) and event.relation.id == self._id class RelationEndpoint: From 5845c7e9e8d3fc7dda59d52fd8cfd35dc8036e90 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 9 May 2023 13:57:56 +0000 Subject: [PATCH 132/159] Use username from mysql relation as prefix --- src/mysql_shell.py | 7 +++---- src/relations/database_provides.py | 25 +++++++++++++------------ src/workload.py | 3 +-- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index bc17937a1..39db8ff53 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -23,18 +23,17 @@ class Shell: """MySQL Shell connected to MySQL cluster""" _container: ops.Container - _username: str + username: str _password: str _host: str _port: str - _mysql_relation_id: int _TEMPORARY_SCRIPT_FILE = "/tmp/script.py" def _run_commands(self, commands: list[str]) -> None: """Connect to MySQL cluster and run commands.""" commands.insert( - 0, f"shell.connect('{self._username}:{self._password}@{self._host}:{self._port}')" + 0, f"shell.connect('{self.username}:{self._password}@{self._host}:{self._port}')" ) self._container.push(self._TEMPORARY_SCRIPT_FILE, "\n".join(commands)) try: @@ -69,7 +68,7 @@ def _user_attributes(self) -> str: If the relation with the MySQL charm is broken, the MySQL charm will use this attribute to delete all users created by this charm. """ - return json.dumps({"mysql_relation_id": self._mysql_relation_id}) + return json.dumps({"mysql_relation_user": self.username}) def create_application_database_and_user(self, *, username: str, database: str) -> str: """Create database and user for related database_provides application.""" diff --git a/src/relations/database_provides.py b/src/relations/database_provides.py index a9a6da3e7..71d2414f2 100644 --- a/src/relations/database_provides.py +++ b/src/relations/database_provides.py @@ -53,27 +53,27 @@ def _database(self) -> str: """Requested database name""" return self._remote_databag["database"] - @property - def _username(self) -> str: + def _get_username(self, database_requires_username: str) -> str: """Database username""" - # Model name is necessary to create a unique username if MySQL Router is deployed in a - # different Juju model from MySQL. + # Prefix username with username from database requires relation. + # This ensures a unique username if MySQL Router is deployed in a different Juju model + # from MySQL. # (Relation IDs are only unique within a Juju model.) - return f"relation-{self._model_name}-{self.id}" + return f"{database_requires_username}-{self.id}" - def _set_databag(self, *, password: str, router_endpoint: str) -> None: + def _set_databag(self, *, username: str, password: str, router_endpoint: str) -> None: """Share connection information with application charm.""" read_write_endpoint = f"{router_endpoint}:6446" read_only_endpoint = f"{router_endpoint}:6447" logger.debug( - f"Setting databag {self.id=} {self._database=}, {self._username=}, {read_write_endpoint=}, {read_only_endpoint=}" + f"Setting databag {self.id=} {self._database=}, {username=}, {read_write_endpoint=}, {read_only_endpoint=}" ) self._interface.set_database(self.id, self._database) - self._interface.set_credentials(self.id, self._username, password) + self._interface.set_credentials(self.id, username, password) self._interface.set_endpoints(self.id, read_write_endpoint) self._interface.set_read_only_endpoints(self.id, read_only_endpoint) logger.debug( - f"Set databag {self.id=} {self._database=}, {self._username=}, {read_write_endpoint=}, {read_only_endpoint=}" + f"Set databag {self.id=} {self._database=}, {username=}, {read_write_endpoint=}, {read_only_endpoint=}" ) def _delete_databag(self) -> None: @@ -84,15 +84,16 @@ def _delete_databag(self) -> None: def create_database_and_user(self, *, router_endpoint: str, shell: mysql_shell.Shell) -> None: """Create database & user and update databag.""" + username = self._get_username(shell.username) password = shell.create_application_database_and_user( - username=self._username, database=self._database + username=username, database=self._database ) - self._set_databag(password=password, router_endpoint=router_endpoint) + self._set_databag(username=username, password=password, router_endpoint=router_endpoint) def delete_user(self, *, shell: mysql_shell.Shell) -> None: """Delete user and update databag.""" self._delete_databag() - shell.delete_user(self._username) + shell.delete_user(self._get_username(shell.username)) def is_breaking(self, event): """Whether relation will be broken after the current event is handled""" diff --git a/src/workload.py b/src/workload.py index b36e833d3..0e185c2c3 100644 --- a/src/workload.py +++ b/src/workload.py @@ -125,11 +125,10 @@ def shell(self) -> mysql_shell.Shell: """MySQL Shell""" return mysql_shell.Shell( _container=self._container, - _username=self._admin_username, + username=self._admin_username, _password=self._admin_password, _host=self._host, _port=self._port, - _mysql_relation_id=self._charm.database_requires.relation.id, ) @property From 259245652af2a9e98daa5e345f0cd88a914374de Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 9 May 2023 14:37:02 +0000 Subject: [PATCH 133/159] change attribute name --- src/mysql_shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 39db8ff53..68170c0de 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -68,7 +68,7 @@ def _user_attributes(self) -> str: If the relation with the MySQL charm is broken, the MySQL charm will use this attribute to delete all users created by this charm. """ - return json.dumps({"mysql_relation_user": self.username}) + return json.dumps({"created_by_user": self.username}) def create_application_database_and_user(self, *, username: str, database: str) -> str: """Create database and user for related database_provides application.""" From a410463cde6bd758b8614e0c071bd315415e6f5e Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 10 May 2023 18:08:01 +0000 Subject: [PATCH 134/159] Clean up router metadata --- src/charm.py | 23 +++++----- src/mysql_shell.py | 18 +++++++- src/relations/database_requires.py | 29 ++++++++++--- src/workload.py | 70 +++++++++++++++++++++--------- 4 files changed, 100 insertions(+), 40 deletions(-) diff --git a/src/charm.py b/src/charm.py index 86b1de34b..410d65be6 100755 --- a/src/charm.py +++ b/src/charm.py @@ -34,17 +34,17 @@ def __init__(self, *args) -> None: self.database_provides = relations.database_provides.RelationEndpoint(self) + # Set status on first start if no relations active + self.framework.observe(self.on.start, self.reconcile_database_relations) + self.framework.observe( getattr(self.on, "mysql_router_pebble_ready"), self._on_mysql_router_pebble_ready ) - - # Start workload after pod churn or charm upgrade - # (https://juju.is/docs/sdk/start-event#heading--emission-sequence) - # Also, set status on first start if no relations active - self.framework.observe(self.on.start, self.reconcile_database_relations) - self.framework.observe(self.on.leader_elected, self._on_leader_elected) + # Start workload after pod restart + self.framework.observe(self.on.upgrade_charm, self.reconcile_database_relations) + self.tls = relations.tls.RelationEndpoint(self) @property @@ -56,10 +56,7 @@ def workload(self): if self.database_requires.relation: return workload.AuthenticatedWorkload( _container=container, - _admin_username=self.database_requires.relation.username, - _admin_password=self.database_requires.relation.password, - _host=self.database_requires.relation.host, - _port=self.database_requires.relation.port, + _database_requires_relation=self.database_requires, _charm=self, ) return workload.Workload(_container=container) @@ -186,7 +183,8 @@ def reconcile_database_relations(self, event=None) -> None: f"{self.unit.is_leader()=}, " f"{isinstance(self.workload, workload.AuthenticatedWorkload)=}, " f"{self.database_requires.relation and self.database_requires.relation.is_breaking(event)=}, " - f"{self.workload.container_ready=}" + f"{self.workload.container_ready=}, " + f"{isinstance(event, ops.UpgradeCharmEvent)=}" ) if ( self.unit.is_leader() @@ -203,6 +201,9 @@ def reconcile_database_relations(self, event=None) -> None: and self.workload.container_ready and not self.database_requires.relation.is_breaking(event) ): + if isinstance(event, ops.UpgradeCharmEvent): + # Pod restart (https://juju.is/docs/sdk/start-event#heading--emission-sequence) + self.workload.remove_router_from_cluster_metadata() self.workload.enable(tls=self.tls.certificate_saved) elif self.workload.container_ready: self.workload.disable() diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 68170c0de..59476cb1f 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -72,7 +72,7 @@ def _user_attributes(self) -> str: def create_application_database_and_user(self, *, username: str, database: str) -> str: """Create database and user for related database_provides application.""" - logger.debug(f"Creating {database=} and {username=}") + logger.debug(f"Creating {database=} and {username=} with {self._user_attributes=}") password = self._generate_password() self._run_sql( [ @@ -81,15 +81,29 @@ def create_application_database_and_user(self, *, username: str, database: str) f"GRANT ALL PRIVILEGES ON `{database}`.* TO `{username}`", ] ) - logger.debug(f"Created {database=} and {username=}") + logger.debug(f"Created {database=} and {username=} with {self._user_attributes=}") return password def add_attributes_to_mysql_router_user(self, username: str) -> None: """Add attributes to user created during MySQL Router bootstrap.""" + logger.debug(f"Adding {self._user_attributes=} to {username=}") self._run_sql([f"ALTER USER `{username}` ATTRIBUTE '{self._user_attributes}'"]) + logger.debug(f"Added {self._user_attributes=} to {username=}") def delete_user(self, username: str) -> None: """Delete user.""" logger.debug(f"Deleting {username=}") self._run_sql([f"DROP USER `{username}`"]) logger.debug(f"Deleted {username=}") + + def remove_router_from_cluster_metadata(self, router_id: str) -> None: + """Remove MySQL Router from InnoDB Cluster metadata. + + On pod restart, MySQL Router bootstrap will fail without `--force` if cluster metadata + already exists for the router ID. + """ + logger.debug(f"Removing {router_id=} from cluster metadata") + self._run_commands( + ["cluster = dba.get_cluster()", f'cluster.remove_router_metadata("{router_id}")'] + ) + logger.debug(f"Removed {router_id=} from cluster metadata") diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index ffe46f685..0711568eb 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -14,16 +14,20 @@ @dataclasses.dataclass -class _Relation: +class Relation: """Relation to MySQL charm""" _interface: data_interfaces.DatabaseRequires @property - def _id(self) -> int: + def _relation(self) -> ops.Relation: relations = self._interface.relations assert len(relations) == 1 - return relations[0].id + return relations[0] + + @property + def _id(self) -> int: + return self._relation.id @property def _remote_databag(self) -> dict: @@ -61,6 +65,19 @@ def is_breaking(self, event): """Whether relation will be broken after the current event is handled""" return isinstance(event, ops.RelationBrokenEvent) and event.relation.id == self._id + @property + def _local_unit_databag(self) -> ops.RelationDataContent: + """Unit databag""" + return self._relation.data[self._interface.local_unit] + + def set_router_id_in_unit_databag(self, router_id: str) -> None: + """Set router ID in unit databag. + + Used by MySQL charm to remove router metadata from InnoDB cluster when a MySQL Router unit + departs the relation + """ + self._local_unit_databag["router_id"] = router_id + class RelationEndpoint: """Relation endpoint for MySQL charm""" @@ -86,16 +103,16 @@ def __init__(self, charm_: "charm.MySQLRouterOperatorCharm") -> None: ) @property - def relation(self) -> typing.Optional[_Relation]: + def relation(self) -> typing.Optional[Relation]: """Relation to MySQL charm""" if not self._interface.is_resource_created(): return - return _Relation(self._interface) + return Relation(self._interface) def is_missing_relation(self, event) -> bool: """Whether relation to MySQL charm does (or will) not exist""" # Cannot use `self.relation.is_breaking()` in case relation exists but resource not created - if self._interface.relations and _Relation(self._interface).is_breaking(event): + if self._interface.relations and Relation(self._interface).is_breaking(event): return True return len(self._interface.relations) == 0 diff --git a/src/workload.py b/src/workload.py index 0e185c2c3..a0cfba71c 100644 --- a/src/workload.py +++ b/src/workload.py @@ -7,6 +7,7 @@ import dataclasses import logging import pathlib +import socket import string import typing @@ -16,6 +17,8 @@ if typing.TYPE_CHECKING: import charm + import relations.database_requires + logger = logging.getLogger(__name__) @@ -107,14 +110,11 @@ def disable(self) -> None: class AuthenticatedWorkload(Workload): """Workload with connection to MySQL cluster""" - # Admin user with permission to: + # Database requires relation provides an admin user with permission to: # - Create databases & users # - Grant all privileges on a database to a user # (Different from user that MySQL Router runs with after bootstrap.) - _admin_username: str - _admin_password: str - _host: str - _port: str + _database_requires_relation: relations.database_requires.Relation _charm: "charm.MySQLRouterOperatorCharm" _TLS_KEY_FILE = "custom-key.pem" @@ -125,34 +125,47 @@ def shell(self) -> mysql_shell.Shell: """MySQL Shell""" return mysql_shell.Shell( _container=self._container, - username=self._admin_username, - _password=self._admin_password, - _host=self._host, - _port=self._port, + username=self._database_requires_relation.username, + _password=self._database_requires_relation.password, + _host=self._database_requires_relation.host, + _port=self._database_requires_relation.port, ) @property - def _router_username(self) -> str: - """Read MySQL Router username from config file. + def _router_id(self) -> str: + """MySQL Router ID in InnoDB Cluster metadata - During bootstrap, MySQL Router creates a config file at - `/etc/mysqlrouter/mysqlrouter.conf`. This file contains the username that was created - during bootstrap. + Used to remove MySQL Router metadata from InnoDB cluster """ - config = configparser.ConfigParser() - config.read_file(self._container.pull("/etc/mysqlrouter/mysqlrouter.conf")) - return config["metadata_cache:bootstrap"]["user"] + # MySQL Router is bootstrapped without `--directory`—there is one system-wide instance. + return f"{socket.getfqdn()}::system" + + def remove_router_from_cluster_metadata(self) -> None: + """Remove MySQL Router from InnoDB Cluster metadata. + + On pod restart, MySQL Router bootstrap will fail without `--force` if cluster metadata + already exists for the router ID. + """ + self.shell.remove_router_from_cluster_metadata(self._router_id) def _bootstrap_router(self, *, tls: bool) -> None: """Bootstrap MySQL Router and enable service.""" - logger.debug(f"Bootstrapping router {tls=}, {self._host=}, {self._port=}") + logger.debug( + f"Bootstrapping router {tls=}, {self._database_requires_relation.host=}, {self._database_requires_relation.port=}" + ) try: # Bootstrap MySQL Router process = self._container.exec( [ "mysqlrouter", "--bootstrap", - f"{self._admin_username}:{self._admin_password}@{self._host}:{self._port}", + self._database_requires_relation.username + + ":" + + self._database_requires_relation.password + + "@" + + self._database_requires_relation.host + + ":" + + self._database_requires_relation.port, "--strict", "--user", self._UNIX_USERNAME, @@ -165,11 +178,24 @@ def _bootstrap_router(self, *, tls: bool) -> None: except ops.pebble.ExecError as e: logger.exception(f"Failed to bootstrap router\nstderr:\n{e.stderr}\n") raise - self.shell.add_attributes_to_mysql_router_user(self._router_username) # Enable service self._update_layer(enabled=True, tls=tls) - logger.debug(f"Bootstrapped router {tls=}, {self._host=}, {self._port=}") + logger.debug( + f"Bootstrapped router {tls=}, {self._database_requires_relation.host=}, {self._database_requires_relation.port=}" + ) + + @property + def _router_username(self) -> str: + """Read MySQL Router username from config file. + + During bootstrap, MySQL Router creates a config file at + `/etc/mysqlrouter/mysqlrouter.conf`. This file contains the username that was created + during bootstrap. + """ + config = configparser.ConfigParser() + config.read_file(self._container.pull("/etc/mysqlrouter/mysqlrouter.conf")) + return config["metadata_cache:bootstrap"]["user"] def enable(self, *, tls: bool) -> None: """Start and enable MySQL Router service.""" @@ -180,6 +206,8 @@ def enable(self, *, tls: bool) -> None: return logger.debug("Enabling MySQL Router service") self._bootstrap_router(tls=tls) + self.shell.add_attributes_to_mysql_router_user(self._router_username) + self._database_requires_relation.set_router_id_in_unit_databag(self._router_id) logger.debug("Enabled MySQL Router service") self._charm.wait_until_mysql_router_ready() From 25f1cfd62899d44e838e5c7fdc7faec79f4a431b Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 10 May 2023 18:16:50 +0000 Subject: [PATCH 135/159] Fix type annotation --- src/workload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index a0cfba71c..a7b94b9c9 100644 --- a/src/workload.py +++ b/src/workload.py @@ -114,7 +114,7 @@ class AuthenticatedWorkload(Workload): # - Create databases & users # - Grant all privileges on a database to a user # (Different from user that MySQL Router runs with after bootstrap.) - _database_requires_relation: relations.database_requires.Relation + _database_requires_relation: "relations.database_requires.Relation" _charm: "charm.MySQLRouterOperatorCharm" _TLS_KEY_FILE = "custom-key.pem" From b53cbd69bb1a3d3370487b8d754d86963fa28011 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 10 May 2023 18:23:44 +0000 Subject: [PATCH 136/159] Fix argument --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 410d65be6..de1f9e94f 100755 --- a/src/charm.py +++ b/src/charm.py @@ -56,7 +56,7 @@ def workload(self): if self.database_requires.relation: return workload.AuthenticatedWorkload( _container=container, - _database_requires_relation=self.database_requires, + _database_requires_relation=self.database_requires.relation, _charm=self, ) return workload.Workload(_container=container) From ef8889f96fc43c7ee315682957338fa05b6a1f9b Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Wed, 10 May 2023 18:26:49 +0000 Subject: [PATCH 137/159] Add comments for flake8 ignores --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 46ab23b3a..d90dbd047 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,8 @@ exclude = [".git", "__pycache__", ".tox", "build", "dist", "*.egg_info", "venv"] select = ["E", "W", "F", "C", "N", "R", "D", "H"] # Ignore W503, E501 because using black creates errors with this # Ignore D107 Missing docstring in __init__ +# Ignore D415 Docstring first line punctuation (doesn't make sense for properties) +# Ignore D403 First word of the first line should be properly capitalized (false positive on "MySQL") ignore = ["W503", "E501", "D107", "D415", "D403"] # D100, D101, D102, D103: Ignore missing docstrings in tests per-file-ignores = ["tests/*:D100,D101,D102,D103,D104"] From 04ad8b39df1d36a755ec4d62397e1174f0b4a743 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 11 May 2023 11:53:50 +0000 Subject: [PATCH 138/159] switch log to warning --- src/relations/tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/relations/tls.py b/src/relations/tls.py index 6c1cb580c..c83d1e60d 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -253,7 +253,7 @@ def _on_certificate_available(self, event: tls_certificates.CertificateAvailable def _on_certificate_expiring(self, event: tls_certificates.CertificateExpiringEvent) -> None: """Request the new certificate when old certificate is expiring.""" if event.certificate != self._relation.peer_unit_databag.certificate: - logger.error("Unknown certificate expiring") + logger.warning("Unknown certificate expiring") return self._relation.request_certificate_renewal() From 89dfe92a1c33edf63a24d555b8443caade6355e1 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 11 May 2023 17:55:02 +0000 Subject: [PATCH 139/159] Delete router user on pod restart --- src/charm.py | 2 +- src/mysql_shell.py | 42 +++++++++++++++++++++++++++++++----------- src/workload.py | 13 ++++++------- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/src/charm.py b/src/charm.py index de1f9e94f..2f36415ef 100755 --- a/src/charm.py +++ b/src/charm.py @@ -203,7 +203,7 @@ def reconcile_database_relations(self, event=None) -> None: ): if isinstance(event, ops.UpgradeCharmEvent): # Pod restart (https://juju.is/docs/sdk/start-event#heading--emission-sequence) - self.workload.remove_router_from_cluster_metadata() + self.workload.cleanup_after_pod_restart() self.workload.enable(tls=self.tls.certificate_saved) elif self.workload.container_ready: self.workload.disable() diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 59476cb1f..7e28c5b88 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -61,34 +61,38 @@ def _generate_password() -> str: choices = string.ascii_letters + string.digits return "".join([secrets.choice(choices) for _ in range(_PASSWORD_LENGTH)]) - @property - def _user_attributes(self) -> str: - """Attributes for (MySQL) users created by this charm. + def _get_attributes(self, additional_attributes: dict = None) -> str: + """Attributes for (MySQL) users created by this charm If the relation with the MySQL charm is broken, the MySQL charm will use this attribute to delete all users created by this charm. """ - return json.dumps({"created_by_user": self.username}) + attributes = {"created_by_user": self.username} + if additional_attributes: + attributes.update(additional_attributes) + return json.dumps(attributes) def create_application_database_and_user(self, *, username: str, database: str) -> str: """Create database and user for related database_provides application.""" - logger.debug(f"Creating {database=} and {username=} with {self._user_attributes=}") + attributes = self._get_attributes() + logger.debug(f"Creating {database=} and {username=} with {attributes=}") password = self._generate_password() self._run_sql( [ f"CREATE DATABASE IF NOT EXISTS `{database}`", - f"CREATE USER `{username}` IDENTIFIED BY '{password}' ATTRIBUTE '{self._user_attributes}'", + f"CREATE USER `{username}` IDENTIFIED BY '{password}' ATTRIBUTE '{attributes}'", f"GRANT ALL PRIVILEGES ON `{database}`.* TO `{username}`", ] ) - logger.debug(f"Created {database=} and {username=} with {self._user_attributes=}") + logger.debug(f"Created {database=} and {username=} with {attributes=}") return password - def add_attributes_to_mysql_router_user(self, username: str) -> None: + def add_attributes_to_mysql_router_user(self, *, username: str, router_id: str) -> None: """Add attributes to user created during MySQL Router bootstrap.""" - logger.debug(f"Adding {self._user_attributes=} to {username=}") - self._run_sql([f"ALTER USER `{username}` ATTRIBUTE '{self._user_attributes}'"]) - logger.debug(f"Added {self._user_attributes=} to {username=}") + attributes = self._get_attributes({"router_id": router_id}) + logger.debug(f"Adding {attributes=} to {username=}") + self._run_sql([f"ALTER USER `{username}` ATTRIBUTE '{attributes}'"]) + logger.debug(f"Added {attributes=} to {username=}") def delete_user(self, username: str) -> None: """Delete user.""" @@ -96,6 +100,22 @@ def delete_user(self, username: str) -> None: self._run_sql([f"DROP USER `{username}`"]) logger.debug(f"Deleted {username=}") + def delete_router_user_after_pod_restart(self, router_id: str) -> None: + """Delete MySQL Router user created by a previous instance of this unit. + + Before pod restart, the charm does not have an opportunity to delete the MySQL Router user. + During MySQL Router bootstrap, a new user is created. Before bootstrap, the old user + should be deleted. + """ + self._run_sql( + [ + f"session.run_sql(\"SELECT CONCAT('DROP USER ', GROUP_CONCAT(QUOTE(USER), '@', QUOTE(HOST))) INTO @sql FROM INFORMATION_SCHEMA.USER_ATTRIBUTES WHERE ATTRIBUTE->'$.created_by_user'='{self.username}' AND ATTRIBUTE->'$.router_id'='{router_id}'\")", + 'session.run_sql("PREPARE stmt FROM @sql")', + 'session.run_sql("EXECUTE stmt")', + 'session.run_sql("DEALLOCATE PREPARE stmt")', + ] + ) + def remove_router_from_cluster_metadata(self, router_id: str) -> None: """Remove MySQL Router from InnoDB Cluster metadata. diff --git a/src/workload.py b/src/workload.py index a7b94b9c9..1727cedcb 100644 --- a/src/workload.py +++ b/src/workload.py @@ -140,13 +140,10 @@ def _router_id(self) -> str: # MySQL Router is bootstrapped without `--directory`—there is one system-wide instance. return f"{socket.getfqdn()}::system" - def remove_router_from_cluster_metadata(self) -> None: - """Remove MySQL Router from InnoDB Cluster metadata. - - On pod restart, MySQL Router bootstrap will fail without `--force` if cluster metadata - already exists for the router ID. - """ + def cleanup_after_pod_restart(self) -> None: + """Remove MySQL Router cluster metadata & user after pod restart.""" self.shell.remove_router_from_cluster_metadata(self._router_id) + self.shell.delete_router_user_after_pod_restart(self._router_id) def _bootstrap_router(self, *, tls: bool) -> None: """Bootstrap MySQL Router and enable service.""" @@ -206,7 +203,9 @@ def enable(self, *, tls: bool) -> None: return logger.debug("Enabling MySQL Router service") self._bootstrap_router(tls=tls) - self.shell.add_attributes_to_mysql_router_user(self._router_username) + self.shell.add_attributes_to_mysql_router_user( + username=self._router_username, router_id=self._router_id + ) self._database_requires_relation.set_router_id_in_unit_databag(self._router_id) logger.debug("Enabled MySQL Router service") self._charm.wait_until_mysql_router_ready() From 60d21fc65338788697e54fdb3f60c8492ab5ebe7 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 11 May 2023 18:04:06 +0000 Subject: [PATCH 140/159] debug log --- src/mysql_shell.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index 7e28c5b88..dc3ddd768 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -107,6 +107,7 @@ def delete_router_user_after_pod_restart(self, router_id: str) -> None: During MySQL Router bootstrap, a new user is created. Before bootstrap, the old user should be deleted. """ + logger.debug(f"Deleting MySQL Router user {router_id=} created by {self.username=}") self._run_sql( [ f"session.run_sql(\"SELECT CONCAT('DROP USER ', GROUP_CONCAT(QUOTE(USER), '@', QUOTE(HOST))) INTO @sql FROM INFORMATION_SCHEMA.USER_ATTRIBUTES WHERE ATTRIBUTE->'$.created_by_user'='{self.username}' AND ATTRIBUTE->'$.router_id'='{router_id}'\")", @@ -115,6 +116,7 @@ def delete_router_user_after_pod_restart(self, router_id: str) -> None: 'session.run_sql("DEALLOCATE PREPARE stmt")', ] ) + logger.debug(f"Deleted MySQL Router user {router_id=} created by {self.username=}") def remove_router_from_cluster_metadata(self, router_id: str) -> None: """Remove MySQL Router from InnoDB Cluster metadata. From e68d0e13fd77ed19e79680c0ecba67ff2d1a61da Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 11 May 2023 18:07:39 +0000 Subject: [PATCH 141/159] fix sql arguments --- src/mysql_shell.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index dc3ddd768..f2a0020ea 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -110,10 +110,10 @@ def delete_router_user_after_pod_restart(self, router_id: str) -> None: logger.debug(f"Deleting MySQL Router user {router_id=} created by {self.username=}") self._run_sql( [ - f"session.run_sql(\"SELECT CONCAT('DROP USER ', GROUP_CONCAT(QUOTE(USER), '@', QUOTE(HOST))) INTO @sql FROM INFORMATION_SCHEMA.USER_ATTRIBUTES WHERE ATTRIBUTE->'$.created_by_user'='{self.username}' AND ATTRIBUTE->'$.router_id'='{router_id}'\")", - 'session.run_sql("PREPARE stmt FROM @sql")', - 'session.run_sql("EXECUTE stmt")', - 'session.run_sql("DEALLOCATE PREPARE stmt")', + f"SELECT CONCAT('DROP USER ', GROUP_CONCAT(QUOTE(USER), '@', QUOTE(HOST))) INTO @sql FROM INFORMATION_SCHEMA.USER_ATTRIBUTES WHERE ATTRIBUTE->'$.created_by_user'='{self.username}' AND ATTRIBUTE->'$.router_id'='{router_id}'", + "PREPARE stmt FROM @sql", + "EXECUTE stmt", + "DEALLOCATE PREPARE stmt", ] ) logger.debug(f"Deleted MySQL Router user {router_id=} created by {self.username=}") From 17740a8aac5658b6953312f96a84de158feff6f9 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 11 May 2023 18:39:49 +0000 Subject: [PATCH 142/159] only report relations status on leader unit --- src/charm.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/src/charm.py b/src/charm.py index 2f36415ef..fcd138c08 100755 --- a/src/charm.py +++ b/src/charm.py @@ -41,6 +41,8 @@ def __init__(self, *args) -> None: getattr(self.on, "mysql_router_pebble_ready"), self._on_mysql_router_pebble_ready ) self.framework.observe(self.on.leader_elected, self._on_leader_elected) + self.framework.observe(self.on.leader_elected, self._on_leadership_change) + self.framework.observe(self.on.leader_settings_changed, self._on_leadership_change) # Start workload after pod restart self.framework.observe(self.on.upgrade_charm, self.reconcile_database_relations) @@ -68,18 +70,23 @@ def _endpoint(self) -> str: def _determine_status(self, event) -> ops.StatusBase: """Report charm status.""" - missing_relations = [] - for endpoint in [self.database_requires, self.database_provides]: - if endpoint.is_missing_relation(event): - missing_relations.append(endpoint.NAME) - if missing_relations: - return ops.BlockedStatus( - f"Missing relation{'s' if len(missing_relations) > 1 else ''}: {', '.join(missing_relations)}" - ) - if self.database_requires.waiting_for_resource: - return ops.WaitingStatus( - f"Waiting for related application: {self.database_requires.NAME}" - ) + if self.unit.is_leader(): + # Only report status about related applications on leader unit + # (The `data_interfaces.DatabaseProvides` `on.database_requested` event is only + # emitted on the leader unit—non-leader units may not have a chance to update status + # when the status about related applications changes.) + missing_relations = [] + for endpoint in [self.database_requires, self.database_provides]: + if endpoint.is_missing_relation(event): + missing_relations.append(endpoint.NAME) + if missing_relations: + return ops.BlockedStatus( + f"Missing relation{'s' if len(missing_relations) > 1 else ''}: {', '.join(missing_relations)}" + ) + if self.database_requires.waiting_for_resource: + return ops.WaitingStatus( + f"Waiting for related application: {self.database_requires.NAME}" + ) if not self.workload.container_ready: return ops.MaintenanceStatus("Waiting for container") return ops.ActiveStatus() @@ -213,6 +220,11 @@ def _on_mysql_router_pebble_ready(self, _) -> None: self.unit.set_workload_version(self.workload.version) self.reconcile_database_relations() + def _on_leadership_change(self, _) -> None: + # The leader unit is responsible for reporting status about related applications. + # If leadership changes, all units should update status. + self.set_status(event=None) + def _on_leader_elected(self, _) -> None: """Patch existing k8s service to include read-write and read-only services.""" try: From 14ee1106f885f3c17e751ab535877346bbae6c6d Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 12 May 2023 13:47:24 +0000 Subject: [PATCH 143/159] shorten application to app in status message --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index fcd138c08..1b1cf751d 100755 --- a/src/charm.py +++ b/src/charm.py @@ -85,7 +85,7 @@ def _determine_status(self, event) -> ops.StatusBase: ) if self.database_requires.waiting_for_resource: return ops.WaitingStatus( - f"Waiting for related application: {self.database_requires.NAME}" + f"Waiting for related app: {self.database_requires.NAME}" ) if not self.workload.container_ready: return ops.MaintenanceStatus("Waiting for container") From 43dcc548103beb2550d61385a0210cd23e6e34e4 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 12 May 2023 13:59:53 +0000 Subject: [PATCH 144/159] format --- src/charm.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/charm.py b/src/charm.py index 1b1cf751d..e94ee85b7 100755 --- a/src/charm.py +++ b/src/charm.py @@ -84,9 +84,7 @@ def _determine_status(self, event) -> ops.StatusBase: f"Missing relation{'s' if len(missing_relations) > 1 else ''}: {', '.join(missing_relations)}" ) if self.database_requires.waiting_for_resource: - return ops.WaitingStatus( - f"Waiting for related app: {self.database_requires.NAME}" - ) + return ops.WaitingStatus(f"Waiting for related app: {self.database_requires.NAME}") if not self.workload.container_ready: return ops.MaintenanceStatus("Waiting for container") return ops.ActiveStatus() From f016e20917e0ece2ca7a1eb3283bf5cea2857f83 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 12 May 2023 15:18:11 +0000 Subject: [PATCH 145/159] add unit name attribute to mysql router user --- src/charm.py | 2 +- src/mysql_shell.py | 4 ++-- src/workload.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/charm.py b/src/charm.py index e94ee85b7..464022cb9 100755 --- a/src/charm.py +++ b/src/charm.py @@ -209,7 +209,7 @@ def reconcile_database_relations(self, event=None) -> None: if isinstance(event, ops.UpgradeCharmEvent): # Pod restart (https://juju.is/docs/sdk/start-event#heading--emission-sequence) self.workload.cleanup_after_pod_restart() - self.workload.enable(tls=self.tls.certificate_saved) + self.workload.enable(tls=self.tls.certificate_saved, unit_name=self.unit.name) elif self.workload.container_ready: self.workload.disable() self.set_status(event) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index f2a0020ea..bc1b35f3c 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -87,9 +87,9 @@ def create_application_database_and_user(self, *, username: str, database: str) logger.debug(f"Created {database=} and {username=} with {attributes=}") return password - def add_attributes_to_mysql_router_user(self, *, username: str, router_id: str) -> None: + def add_attributes_to_mysql_router_user(self, *, username: str, router_id: str, unit_name: str) -> None: """Add attributes to user created during MySQL Router bootstrap.""" - attributes = self._get_attributes({"router_id": router_id}) + attributes = self._get_attributes({"router_id": router_id, "created_by_juju_unit": unit_name}) logger.debug(f"Adding {attributes=} to {username=}") self._run_sql([f"ALTER USER `{username}` ATTRIBUTE '{attributes}'"]) logger.debug(f"Added {attributes=} to {username=}") diff --git a/src/workload.py b/src/workload.py index 1727cedcb..4ddaf4342 100644 --- a/src/workload.py +++ b/src/workload.py @@ -194,7 +194,7 @@ def _router_username(self) -> str: config.read_file(self._container.pull("/etc/mysqlrouter/mysqlrouter.conf")) return config["metadata_cache:bootstrap"]["user"] - def enable(self, *, tls: bool) -> None: + def enable(self, *, tls: bool, unit_name: str) -> None: """Start and enable MySQL Router service.""" if self._enabled: # If the host or port changes, MySQL Router will receive topology change @@ -204,7 +204,7 @@ def enable(self, *, tls: bool) -> None: logger.debug("Enabling MySQL Router service") self._bootstrap_router(tls=tls) self.shell.add_attributes_to_mysql_router_user( - username=self._router_username, router_id=self._router_id + username=self._router_username, router_id=self._router_id, unit_name=unit_name ) self._database_requires_relation.set_router_id_in_unit_databag(self._router_id) logger.debug("Enabled MySQL Router service") From 7c54c3d86d5586a4c593247d7ef616913dbc7841 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 12 May 2023 15:29:12 +0000 Subject: [PATCH 146/159] format --- src/mysql_shell.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/mysql_shell.py b/src/mysql_shell.py index bc1b35f3c..b4610e106 100644 --- a/src/mysql_shell.py +++ b/src/mysql_shell.py @@ -87,9 +87,13 @@ def create_application_database_and_user(self, *, username: str, database: str) logger.debug(f"Created {database=} and {username=} with {attributes=}") return password - def add_attributes_to_mysql_router_user(self, *, username: str, router_id: str, unit_name: str) -> None: + def add_attributes_to_mysql_router_user( + self, *, username: str, router_id: str, unit_name: str + ) -> None: """Add attributes to user created during MySQL Router bootstrap.""" - attributes = self._get_attributes({"router_id": router_id, "created_by_juju_unit": unit_name}) + attributes = self._get_attributes( + {"router_id": router_id, "created_by_juju_unit": unit_name} + ) logger.debug(f"Adding {attributes=} to {username=}") self._run_sql([f"ALTER USER `{username}` ATTRIBUTE '{attributes}'"]) logger.debug(f"Added {attributes=} to {username=}") From 44dead8c4cb856a1473e7e2dbea70a0ed03ee457 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 12 May 2023 18:34:43 +0000 Subject: [PATCH 147/159] enable GR notifications --- src/workload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/workload.py b/src/workload.py index 4ddaf4342..d348b75a7 100644 --- a/src/workload.py +++ b/src/workload.py @@ -168,6 +168,7 @@ def _bootstrap_router(self, *, tls: bool) -> None: self._UNIX_USERNAME, "--conf-set-option", "http_server.bind_address=127.0.0.1", + "--conf-use-gr-notifications", ], timeout=30, ) From 12889d50d02125d465275b096b7f5dc7ab6ea6be Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 12 May 2023 19:27:49 +0000 Subject: [PATCH 148/159] use constant --- src/workload.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index d348b75a7..f8e1db690 100644 --- a/src/workload.py +++ b/src/workload.py @@ -192,7 +192,9 @@ def _router_username(self) -> str: during bootstrap. """ config = configparser.ConfigParser() - config.read_file(self._container.pull("/etc/mysqlrouter/mysqlrouter.conf")) + config.read_file( + self._container.pull(self._ROUTER_CONFIG_DIRECTORY / self._ROUTER_CONFIG_FILE) + ) return config["metadata_cache:bootstrap"]["user"] def enable(self, *, tls: bool, unit_name: str) -> None: From 118f92137110848de6204f62c4610952cf77b8f4 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 12 May 2023 20:18:19 +0000 Subject: [PATCH 149/159] remove svc.cluster.local hard-coding --- src/charm.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 464022cb9..4ca2561af 100755 --- a/src/charm.py +++ b/src/charm.py @@ -66,7 +66,13 @@ def workload(self): @property def _endpoint(self) -> str: """K8s endpoint for MySQL Router""" - return f"{self.model.app.name}.{self.model.name}.svc.cluster.local" + # Example: "mysql-router-k8s-0.mysql-router-k8s-endpoints.my-model.svc.cluster.local" + fqdn = socket.getfqdn() + # Example: "mysql-router-k8s-0.mysql-router-k8s-endpoints." + prefix = f"{self.unit.name.replace('/', '-')}.{self.app.name}-endpoints." + assert fqdn.startswith(f"{prefix}{self.model.name}.") + # Example: mysql-router-k8s.my-model.svc.cluster.local + return f"{self.app.name}.{fqdn.removeprefix(prefix)}" def _determine_status(self, event) -> ops.StatusBase: """Report charm status.""" From 0f465620190a775b8c167393087f612b840c1d5a Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 15 May 2023 11:23:13 +0000 Subject: [PATCH 150/159] Remove `--conf-use-gr-notifications` --- src/workload.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/workload.py b/src/workload.py index f8e1db690..7b27a92ec 100644 --- a/src/workload.py +++ b/src/workload.py @@ -168,7 +168,6 @@ def _bootstrap_router(self, *, tls: bool) -> None: self._UNIX_USERNAME, "--conf-set-option", "http_server.bind_address=127.0.0.1", - "--conf-use-gr-notifications", ], timeout=30, ) From 1b2d2065624a39e503acf5048bc8b3f74e044aae Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 15 May 2023 11:32:19 +0000 Subject: [PATCH 151/159] update comment --- src/relations/database_requires.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index 0711568eb..f1561d246 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -88,8 +88,8 @@ def __init__(self, charm_: "charm.MySQLRouterOperatorCharm") -> None: self._interface = data_interfaces.DatabaseRequires( charm_, relation_name=self.NAME, - # HACK: MySQL Router needs a user, but not a database - # Use the DatabaseRequires interface to get a user; disregard the database + # HACK: The MySQL Router charm needs a new user, but not a new database + # Use the DatabaseRequires interface to get a user; disregard the created database database_name="_unused_mysqlrouter_database", extra_user_roles="mysqlrouter", ) From 2befc4c473a5b96104d1e6dc9988c534add63a02 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 15 May 2023 12:08:41 +0000 Subject: [PATCH 152/159] patch k8s service on install --- src/charm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/charm.py b/src/charm.py index 4ca2561af..fe9c643c5 100755 --- a/src/charm.py +++ b/src/charm.py @@ -37,10 +37,10 @@ def __init__(self, *args) -> None: # Set status on first start if no relations active self.framework.observe(self.on.start, self.reconcile_database_relations) + self.framework.observe(self.on.install, self._on_install) self.framework.observe( getattr(self.on, "mysql_router_pebble_ready"), self._on_mysql_router_pebble_ready ) - self.framework.observe(self.on.leader_elected, self._on_leader_elected) self.framework.observe(self.on.leader_elected, self._on_leadership_change) self.framework.observe(self.on.leader_settings_changed, self._on_leadership_change) @@ -229,8 +229,10 @@ def _on_leadership_change(self, _) -> None: # If leadership changes, all units should update status. self.set_status(event=None) - def _on_leader_elected(self, _) -> None: + def _on_install(self, _) -> None: """Patch existing k8s service to include read-write and read-only services.""" + if not self.unit.is_leader(): + return try: self._patch_service(name=self.app.name, ro_port=6447, rw_port=6446) except lightkube.ApiError: From f9e1ea895880be3d0dfbc4093b0dfbd01f49b60e Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 15 May 2023 14:01:35 +0000 Subject: [PATCH 153/159] Revert "Remove `--conf-use-gr-notifications`" This reverts commit 0f465620190a775b8c167393087f612b840c1d5a. --- src/workload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/workload.py b/src/workload.py index 7b27a92ec..f8e1db690 100644 --- a/src/workload.py +++ b/src/workload.py @@ -168,6 +168,7 @@ def _bootstrap_router(self, *, tls: bool) -> None: self._UNIX_USERNAME, "--conf-set-option", "http_server.bind_address=127.0.0.1", + "--conf-use-gr-notifications", ], timeout=30, ) From ba1da6025b803d26bcd7d2cea877cfb64f8e4b9a Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 15 May 2023 17:54:14 +0000 Subject: [PATCH 154/159] Update sans --- src/relations/tls.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/relations/tls.py b/src/relations/tls.py index c83d1e60d..dcf884a21 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -127,27 +127,18 @@ def _parse_tls_key(raw_content: str) -> bytes: ).encode("utf-8") return base64.b64decode(raw_content) - @property - def _unit_hostname(self) -> str: - """Get the hostname.localdomain for a unit. - - Translate juju unit name to hostname.localdomain, necessary - for correct name resolution under k8s. - - Returns: - A string representing the hostname.localdomain of the unit. - """ - return f"{self._charm.unit.name.replace('/', '-')}.{self._charm.app.name}-endpoints" - def _generate_csr(self, key: bytes) -> bytes: """Generate certificate signing request (CSR).""" return tls_certificates.generate_csr( private_key=key, subject=socket.getfqdn(), organization=self._charm.app.name, - sans=[ + sans_dns=[ socket.gethostname(), - self._unit_hostname, + f"{socket.gethostname()}.{self._charm.app.name}-endpoints", + socket.getfqdn(), + ], + sans_ip=[ str(self._charm.model.get_binding(self._peer_relation).network.bind_address), ], ) From 2270eaed1063dc003b8e754e1face9668708de10 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 15 May 2023 18:16:21 +0000 Subject: [PATCH 155/159] Update comment --- src/relations/database_requires.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index f1561d246..baf740f94 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -88,8 +88,7 @@ def __init__(self, charm_: "charm.MySQLRouterOperatorCharm") -> None: self._interface = data_interfaces.DatabaseRequires( charm_, relation_name=self.NAME, - # HACK: The MySQL Router charm needs a new user, but not a new database - # Use the DatabaseRequires interface to get a user; disregard the created database + # Database name disregarded by MySQL charm if "mysqlrouter" extra user role requested database_name="_unused_mysqlrouter_database", extra_user_roles="mysqlrouter", ) From 3a0da55aacb7203ce701fadaa683e35c478d8624 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 15 May 2023 18:26:01 +0000 Subject: [PATCH 156/159] revert workflow debug --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7b60ab03c..4a2c5477a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,7 @@ jobs: build: name: Build charms - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@charmcraft-2.3-debug + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v2 integration-test: strategy: From a443d54f3867eaac1a519c1c2244c9a1ed39719f Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 15 May 2023 18:49:46 +0000 Subject: [PATCH 157/159] change database name --- src/relations/database_requires.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index baf740f94..fe4608693 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -89,7 +89,7 @@ def __init__(self, charm_: "charm.MySQLRouterOperatorCharm") -> None: charm_, relation_name=self.NAME, # Database name disregarded by MySQL charm if "mysqlrouter" extra user role requested - database_name="_unused_mysqlrouter_database", + database_name="mysql_innodb_cluster_metadata", extra_user_roles="mysqlrouter", ) charm_.framework.observe( From 78e99221ab8826a5c426800e114925b13e9a28c9 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Mon, 15 May 2023 19:06:38 +0000 Subject: [PATCH 158/159] update sans --- src/charm.py | 12 +++++++++--- src/relations/tls.py | 13 ++++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/charm.py b/src/charm.py index fe9c643c5..a48387cb8 100755 --- a/src/charm.py +++ b/src/charm.py @@ -64,15 +64,21 @@ def workload(self): return workload.Workload(_container=container) @property - def _endpoint(self) -> str: - """K8s endpoint for MySQL Router""" + def model_service_domain(self): + """K8s service domain for Juju model""" # Example: "mysql-router-k8s-0.mysql-router-k8s-endpoints.my-model.svc.cluster.local" fqdn = socket.getfqdn() # Example: "mysql-router-k8s-0.mysql-router-k8s-endpoints." prefix = f"{self.unit.name.replace('/', '-')}.{self.app.name}-endpoints." assert fqdn.startswith(f"{prefix}{self.model.name}.") + # Example: my-model.svc.cluster.local + return fqdn.removeprefix(prefix) + + @property + def _endpoint(self) -> str: + """K8s endpoint for MySQL Router""" # Example: mysql-router-k8s.my-model.svc.cluster.local - return f"{self.app.name}.{fqdn.removeprefix(prefix)}" + return f"{self.app.name}.{self.model_service_domain}" def _determine_status(self, event) -> ops.StatusBase: """Report charm status.""" diff --git a/src/relations/tls.py b/src/relations/tls.py index dcf884a21..99f4a42cd 100644 --- a/src/relations/tls.py +++ b/src/relations/tls.py @@ -129,14 +129,21 @@ def _parse_tls_key(raw_content: str) -> bytes: def _generate_csr(self, key: bytes) -> bytes: """Generate certificate signing request (CSR).""" + unit_name = self._charm.unit.name.replace("/", "-") return tls_certificates.generate_csr( private_key=key, subject=socket.getfqdn(), organization=self._charm.app.name, sans_dns=[ - socket.gethostname(), - f"{socket.gethostname()}.{self._charm.app.name}-endpoints", - socket.getfqdn(), + unit_name, + f"{unit_name}.{self._charm.app.name}-endpoints", + f"{unit_name}.{self._charm.app.name}-endpoints.{self._charm.model_service_domain}", + f"{self._charm.app.name}-endpoints", + f"{self._charm.app.name}-endpoints.{self._charm.model_service_domain}", + f"{unit_name}.{self._charm.app.name}", + f"{unit_name}.{self._charm.app.name}.{self._charm.model_service_domain}", + self._charm.app.name, + f"{self._charm.app.name}.{self._charm.model_service_domain}", ], sans_ip=[ str(self._charm.model.get_binding(self._peer_relation).network.bind_address), From 177b580197037e0eb3fc6df03189c91033a6c339 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 16 May 2023 13:36:53 +0000 Subject: [PATCH 159/159] remove router id from databag --- src/relations/database_requires.py | 13 ------------- src/workload.py | 1 - 2 files changed, 14 deletions(-) diff --git a/src/relations/database_requires.py b/src/relations/database_requires.py index fe4608693..eaf96af28 100644 --- a/src/relations/database_requires.py +++ b/src/relations/database_requires.py @@ -65,19 +65,6 @@ def is_breaking(self, event): """Whether relation will be broken after the current event is handled""" return isinstance(event, ops.RelationBrokenEvent) and event.relation.id == self._id - @property - def _local_unit_databag(self) -> ops.RelationDataContent: - """Unit databag""" - return self._relation.data[self._interface.local_unit] - - def set_router_id_in_unit_databag(self, router_id: str) -> None: - """Set router ID in unit databag. - - Used by MySQL charm to remove router metadata from InnoDB cluster when a MySQL Router unit - departs the relation - """ - self._local_unit_databag["router_id"] = router_id - class RelationEndpoint: """Relation endpoint for MySQL charm""" diff --git a/src/workload.py b/src/workload.py index f8e1db690..f438bdb44 100644 --- a/src/workload.py +++ b/src/workload.py @@ -209,7 +209,6 @@ def enable(self, *, tls: bool, unit_name: str) -> None: self.shell.add_attributes_to_mysql_router_user( username=self._router_username, router_id=self._router_id, unit_name=unit_name ) - self._database_requires_relation.set_router_id_in_unit_databag(self._router_id) logger.debug("Enabled MySQL Router service") self._charm.wait_until_mysql_router_ready()