Skip to content

Commit f882a87

Browse files
[DPE-5116] Add pgAudit (#688)
* Add pgAudit Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com> * Fix extensions enablement on database request Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com> * Make pgAudit work and add integration test Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com> * Split integration tests Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com> * Fix integration test Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com> * Add unit tests Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com> * Enable pgAudit by default Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com> * Fix test_no_password_exposed_on_logs Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com> * Enable/disable the plugin at the unit test Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com> --------- Signed-off-by: Marcelo Henrique Neppel <marcelo.neppel@canonical.com>
1 parent e70eb0e commit f882a87

File tree

14 files changed

+223
-22
lines changed

14 files changed

+223
-22
lines changed

config.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -303,6 +303,10 @@ options:
303303
default: false
304304
type: boolean
305305
description: Enable timescaledb extension
306+
plugin_audit_enable:
307+
default: true
308+
type: boolean
309+
description: Enable pgAudit extension
306310
profile:
307311
description: |
308312
Profile representing the scope of deployment, and used to tune resource allocation.

lib/charms/postgresql_k8s/v0/postgresql.py

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@
3636

3737
# Increment this PATCH version before using `charmcraft publish-lib` or reset
3838
# to 0 if you are raising the major API version
39-
LIBPATCH = 34
39+
LIBPATCH = 35
4040

4141
INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles"
4242

@@ -114,6 +114,25 @@ def __init__(
114114
self.database = database
115115
self.system_users = system_users
116116

117+
def _configure_pgaudit(self, enable: bool) -> None:
118+
connection = None
119+
try:
120+
connection = self._connect_to_database()
121+
connection.autocommit = True
122+
with connection.cursor() as cursor:
123+
if enable:
124+
cursor.execute("ALTER SYSTEM SET pgaudit.log = 'ROLE,DDL,MISC,MISC_SET';")
125+
cursor.execute("ALTER SYSTEM SET pgaudit.log_client TO off;")
126+
cursor.execute("ALTER SYSTEM SET pgaudit.log_parameter TO off")
127+
else:
128+
cursor.execute("ALTER SYSTEM RESET pgaudit.log;")
129+
cursor.execute("ALTER SYSTEM RESET pgaudit.log_client;")
130+
cursor.execute("ALTER SYSTEM RESET pgaudit.log_parameter;")
131+
cursor.execute("SELECT pg_reload_conf();")
132+
finally:
133+
if connection is not None:
134+
connection.close()
135+
117136
def _connect_to_database(
118137
self, database: str = None, database_host: str = None
119138
) -> psycopg2.extensions.connection:
@@ -325,6 +344,7 @@ def enable_disable_extensions(self, extensions: Dict[str, bool], database: str =
325344
if enable
326345
else f"DROP EXTENSION IF EXISTS {extension};"
327346
)
347+
self._configure_pgaudit(ordered_extensions.get("pgaudit", False))
328348
except psycopg2.errors.UniqueViolation:
329349
pass
330350
except psycopg2.errors.DependentObjectsStillExist:

src/charm.py

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,13 +91,15 @@
9191
MONITORING_USER,
9292
PATRONI_PASSWORD_KEY,
9393
PEER,
94+
PLUGIN_OVERRIDES,
9495
POSTGRES_LOG_FILES,
9596
REPLICATION_PASSWORD_KEY,
9697
REPLICATION_USER,
9798
REWIND_PASSWORD_KEY,
9899
SECRET_DELETED_LABEL,
99100
SECRET_INTERNAL_LABEL,
100101
SECRET_KEY_OVERRIDES,
102+
SPI_MODULE,
101103
SYSTEM_USERS,
102104
TLS_CA_FILE,
103105
TLS_CERT_FILE,
@@ -663,8 +665,6 @@ def enable_disable_extensions(self, database: str = None) -> None:
663665
if self._patroni.get_primary() is None:
664666
logger.debug("Early exit enable_disable_extensions: standby cluster")
665667
return
666-
spi_module = ["refint", "autoinc", "insert_username", "moddatetime"]
667-
plugins_exception = {"uuid_ossp": '"uuid-ossp"'}
668668
original_status = self.unit.status
669669
extensions = {}
670670
# collect extensions
@@ -674,10 +674,10 @@ def enable_disable_extensions(self, database: str = None) -> None:
674674
# Enable or disable the plugin/extension.
675675
extension = "_".join(plugin.split("_")[1:-1])
676676
if extension == "spi":
677-
for ext in spi_module:
677+
for ext in SPI_MODULE:
678678
extensions[ext] = enable
679679
continue
680-
extension = plugins_exception.get(extension, extension)
680+
extension = PLUGIN_OVERRIDES.get(extension, extension)
681681
if self._check_extension_dependencies(extension, enable):
682682
self.unit.status = BlockedStatus(EXTENSIONS_DEPENDENCY_MESSAGE)
683683
return
@@ -2132,6 +2132,20 @@ def log_pitr_last_transaction_time(self) -> None:
21322132
else:
21332133
logger.error("Can't tell last completed transaction time")
21342134

2135+
def get_plugins(self) -> List[str]:
2136+
"""Return a list of installed plugins."""
2137+
plugins = [
2138+
"_".join(plugin.split("_")[1:-1])
2139+
for plugin in self.config.plugin_keys()
2140+
if self.config[plugin]
2141+
]
2142+
plugins = [PLUGIN_OVERRIDES.get(plugin, plugin) for plugin in plugins]
2143+
if "spi" in plugins:
2144+
plugins.remove("spi")
2145+
for ext in SPI_MODULE:
2146+
plugins.append(ext)
2147+
return plugins
2148+
21352149

21362150
if __name__ == "__main__":
21372151
main(PostgresqlOperatorCharm, use_juju_for_storage=True)

src/config.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ class CharmConfig(BaseConfigModel):
3434
optimizer_join_collapse_limit: Optional[int]
3535
profile: str
3636
profile_limit_memory: Optional[int]
37+
plugin_audit_enable: bool
3738
plugin_citext_enable: bool
3839
plugin_debversion_enable: bool
3940
plugin_hstore_enable: bool

src/constants.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@
4242

4343
SECRET_KEY_OVERRIDES = {"ca": "cauth"}
4444
BACKUP_TYPE_OVERRIDES = {"full": "full", "differential": "diff", "incremental": "incr"}
45+
PLUGIN_OVERRIDES = {"audit": "pgaudit", "uuid_ossp": '"uuid-ossp"'}
46+
47+
SPI_MODULE = ["refint", "autoinc", "insert_username", "moddatetime"]
4548

4649
TRACING_RELATION_NAME = "tracing"
4750
TRACING_PROTOCOL = "otlp_http"

src/relations/db.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,11 @@
2222
from ops.model import ActiveStatus, BlockedStatus, Relation, Unit
2323
from pgconnstr import ConnectionString
2424

25-
from constants import ALL_LEGACY_RELATIONS, DATABASE_PORT, ENDPOINT_SIMULTANEOUSLY_BLOCKING_MESSAGE
25+
from constants import (
26+
ALL_LEGACY_RELATIONS,
27+
DATABASE_PORT,
28+
ENDPOINT_SIMULTANEOUSLY_BLOCKING_MESSAGE,
29+
)
2630
from utils import new_password
2731

2832
logger = logging.getLogger(__name__)
@@ -176,11 +180,7 @@ def set_up_relation(self, relation: Relation) -> bool:
176180
user = f"relation_id_{relation.id}"
177181
password = unit_relation_databag.get("password", new_password())
178182
self.charm.postgresql.create_user(user, password, self.admin)
179-
plugins = [
180-
"_".join(plugin.split("_")[1:-1])
181-
for plugin in self.charm.config.plugin_keys()
182-
if self.charm.config[plugin]
183-
]
183+
plugins = self.charm.get_plugins()
184184

185185
self.charm.postgresql.create_database(
186186
database, user, plugins=plugins, client_relations=self.charm.client_relations

src/relations/postgresql_provider.py

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@
2020
from ops.framework import Object
2121
from ops.model import ActiveStatus, BlockedStatus, Relation
2222

23-
from constants import DATABASE_PORT, ENDPOINT_SIMULTANEOUSLY_BLOCKING_MESSAGE
23+
from constants import (
24+
DATABASE_PORT,
25+
ENDPOINT_SIMULTANEOUSLY_BLOCKING_MESSAGE,
26+
)
2427
from utils import new_password
2528

2629
logger = logging.getLogger(__name__)
@@ -87,11 +90,7 @@ def _on_database_requested(self, event: DatabaseRequestedEvent) -> None:
8790
user = f"relation_id_{event.relation.id}"
8891
password = new_password()
8992
self.charm.postgresql.create_user(user, password, extra_user_roles=extra_user_roles)
90-
plugins = [
91-
"_".join(plugin.split("_")[1:-1])
92-
for plugin in self.charm.config.plugin_keys()
93-
if self.charm.config[plugin]
94-
]
93+
plugins = self.charm.get_plugins()
9594

9695
self.charm.postgresql.create_database(
9796
database, user, plugins=plugins, client_relations=self.charm.client_relations

templates/patroni.yml.j2

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ bootstrap:
4040
log_truncate_on_rotation: 'on'
4141
logging_collector: 'on'
4242
wal_level: logical
43-
shared_preload_libraries: 'timescaledb'
43+
shared_preload_libraries: 'timescaledb,pgaudit'
4444
{%- if pg_parameters %}
4545
{%- for key, value in pg_parameters.items() %}
4646
{{key}}: {{value}}

tests/integration/test_audit.py

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2024 Canonical Ltd.
3+
# See LICENSE file for licensing details.
4+
import asyncio
5+
import logging
6+
7+
import psycopg2 as psycopg2
8+
import pytest as pytest
9+
from pytest_operator.plugin import OpsTest
10+
from tenacity import Retrying, stop_after_delay, wait_fixed
11+
12+
from .helpers import (
13+
APPLICATION_NAME,
14+
DATABASE_APP_NAME,
15+
build_and_deploy,
16+
run_command_on_unit,
17+
)
18+
from .new_relations.helpers import build_connection_string
19+
20+
logger = logging.getLogger(__name__)
21+
22+
RELATION_ENDPOINT = "database"
23+
24+
25+
@pytest.mark.group(1)
26+
@pytest.mark.abort_on_fail
27+
async def test_audit_plugin(ops_test: OpsTest) -> None:
28+
"""Test the audit plugin."""
29+
await asyncio.gather(build_and_deploy(ops_test, 1), ops_test.model.deploy(APPLICATION_NAME))
30+
await ops_test.model.relate(f"{APPLICATION_NAME}:{RELATION_ENDPOINT}", DATABASE_APP_NAME)
31+
async with ops_test.fast_forward():
32+
await ops_test.model.wait_for_idle(
33+
apps=[APPLICATION_NAME, DATABASE_APP_NAME], status="active"
34+
)
35+
36+
logger.info("Checking that the audit plugin is enabled")
37+
connection_string = await build_connection_string(
38+
ops_test, APPLICATION_NAME, RELATION_ENDPOINT
39+
)
40+
connection = None
41+
try:
42+
connection = psycopg2.connect(connection_string)
43+
with connection.cursor() as cursor:
44+
cursor.execute("CREATE TABLE test2(value TEXT);")
45+
cursor.execute("GRANT SELECT ON test2 TO PUBLIC;")
46+
cursor.execute("SET TIME ZONE 'Europe/Rome';")
47+
finally:
48+
if connection is not None:
49+
connection.close()
50+
unit_name = f"{DATABASE_APP_NAME}/0"
51+
for attempt in Retrying(stop=stop_after_delay(90), wait=wait_fixed(10), reraise=True):
52+
with attempt:
53+
try:
54+
logs = await run_command_on_unit(
55+
ops_test,
56+
unit_name,
57+
"grep AUDIT /var/log/postgresql/postgresql-*.log",
58+
)
59+
assert "MISC,BEGIN,,,BEGIN" in logs
60+
assert (
61+
"DDL,CREATE TABLE,TABLE,public.test2,CREATE TABLE test2(value TEXT);" in logs
62+
)
63+
assert "ROLE,GRANT,TABLE,,GRANT SELECT ON test2 TO PUBLIC;" in logs
64+
assert "MISC,SET,,,SET TIME ZONE 'Europe/Rome';" in logs
65+
except Exception:
66+
assert False, "Audit logs were not found when the plugin is enabled."
67+
68+
logger.info("Disabling the audit plugin")
69+
await ops_test.model.applications[DATABASE_APP_NAME].set_config({
70+
"plugin_audit_enable": "False"
71+
})
72+
await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active")
73+
74+
logger.info("Removing the previous logs")
75+
await run_command_on_unit(ops_test, unit_name, "rm /var/log/postgresql/postgresql-*.log")
76+
77+
logger.info("Checking that the audit plugin is disabled")
78+
try:
79+
connection = psycopg2.connect(connection_string)
80+
with connection.cursor() as cursor:
81+
cursor.execute("CREATE TABLE test1(value TEXT);")
82+
cursor.execute("GRANT SELECT ON test1 TO PUBLIC;")
83+
cursor.execute("SET TIME ZONE 'Europe/Rome';")
84+
finally:
85+
if connection is not None:
86+
connection.close()
87+
try:
88+
logs = await run_command_on_unit(
89+
ops_test,
90+
unit_name,
91+
"grep AUDIT /var/log/postgresql/postgresql-*.log",
92+
)
93+
except Exception:
94+
pass
95+
else:
96+
logger.info(f"Logs: {logs}")
97+
assert False, "Audit logs were found when the plugin is disabled."

tests/integration/test_password_rotation.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# Copyright 2022 Canonical Ltd.
33
# See LICENSE file for licensing details.
44
import json
5+
import re
56
import time
67

78
import psycopg2
@@ -180,4 +181,8 @@ async def test_no_password_exposed_on_logs(ops_test: OpsTest) -> None:
180181
)
181182
except Exception:
182183
continue
183-
assert len(logs) == 0, f"Sensitive information detected on {unit.name} logs"
184+
regex = re.compile("(PASSWORD )(?!<REDACTED>)")
185+
logs_without_false_positives = regex.findall(logs)
186+
assert (
187+
len(logs_without_false_positives) == 0
188+
), f"Sensitive information detected on {unit.name} logs"

0 commit comments

Comments
 (0)