diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 05dc4e23ef..9a15830dbd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,7 @@ ## Issue - ## Solution + +## Checklist +- [ ] I have added or updated any relevant documentation. +- [ ] I have cleaned any remaining cloud resources from my accounts. diff --git a/.github/workflows/tiobe_scan.yaml b/.github/workflows/tiobe_scan.yaml new file mode 100644 index 0000000000..53d27b6d81 --- /dev/null +++ b/.github/workflows/tiobe_scan.yaml @@ -0,0 +1,44 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +name: Weekly TICS scan + +on: + schedule: + - cron: "0 2 * * 6" # Every Saturday 2:00 AM UTC + workflow_dispatch: + +jobs: + TICS: + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create and activate virtual environment + run: | + python3 -m venv .venv + . .venv/bin/activate + pip install flake8 poetry pylint pytest tox + poetry install --all-groups + echo PATH="$PATH" >> "$GITHUB_ENV" + + - name: Run coverage tests + run: | + tox -e unit + + - name: Move results to the necessary folder for TICS + run: | + mkdir -p .cover + mv coverage.xml .cover/cobertura.xml + + - name: TICS GitHub Action + uses: tiobe/tics-github-action@v3 + with: + mode: qserver + project: postgresql-k8s-operator + viewerUrl: https://canonical.tiobe.com/tiobeweb/TICS/api/cfg?name=default + branchdir: ${{ env.GITHUB_WORKSPACE }} + ticsAuthToken: ${{ secrets.TICSAUTHTOKEN }} + installTics: true + calc: ALL diff --git a/README.md b/README.md index 3a5583dcf4..6185fcdfea 100644 --- a/README.md +++ b/README.md @@ -95,18 +95,24 @@ juju relate postgresql-k8s:db finos-waltz-k8s **Note:** The endpoint `db-admin` provides the same legacy interface `pgsql` with PostgreSQL admin-level privileges. It is NOT recommended to use it from security point of view. ## OCI Images + This charm uses pinned and tested version of the [charmed-postgresql](https://github.com/canonical/charmed-postgresql-rock/pkgs/container/charmed-postgresql) rock. ## Security -Security issues in the Charmed PostgreSQL K8s Operator can be reported through [LaunchPad](https://wiki.ubuntu.com/DebuggingSecurity#How%20to%20File). Please do not file GitHub issues about security issues. + +Security issues in the Charmed PostgreSQL K8s Operator can be reported through [private security reports](https://github.com/canonical/postgresql-k8s-operator/security/advisories/new) on GitHub. +For more information, see the [Security policy](SECURITY.md). ## Contributing + Please see the [Juju SDK docs](https://juju.is/docs/sdk) for guidelines on enhancements to this charm following best practice guidelines, and [CONTRIBUTING.md](https://github.com/canonical/postgresql-k8s-operator/blob/main/CONTRIBUTING.md) for developer guidance. ## License + The Charmed PostgreSQL K8s Operator [is distributed](https://github.com/canonical/postgresql-k8s-operator/blob/main/LICENSE) under the Apache Software License, version 2.0. It installs/operates/depends on [PostgreSQL](https://www.postgresql.org/ftp/source/), which [is licensed](https://www.postgresql.org/about/licence/) under PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses. ## Trademark Notice + PostgreSQL is a trademark or registered trademark of PostgreSQL Global Development Group. Other trademarks are property of their respective owners. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..1881a21566 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,25 @@ +# Security policy + +## What qualifies as a security issue + +Credentials leakage, outdated dependencies with known vulnerabilities, and +other issues that could lead to unprivileged or unauthorized access to the +database or the system. + +## Reporting a vulnerability + +The easiest way to report a security issue is through +[GitHub](https://github.com/canonical/postgresql-k8s-operator/security/advisories/new). See +[Privately reporting a security +vulnerability](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing/privately-reporting-a-security-vulnerability) +for instructions. + +The repository admins will be notified of the issue and will work with you +to determine whether the issue qualifies as a security issue and, if so, in +which component. We will then handle figuring out a fix, getting a CVE +assigned and coordinating the release of the fix. + +The [Ubuntu Security disclosure and embargo +policy](https://ubuntu.com/security/disclosure-policy) contains more +information about what you can expect when you contact us, and what we +expect from you. diff --git a/lib/charms/postgresql_k8s/v0/postgresql.py b/lib/charms/postgresql_k8s/v0/postgresql.py index 5975197f1b..b7eb90908b 100644 --- a/lib/charms/postgresql_k8s/v0/postgresql.py +++ b/lib/charms/postgresql_k8s/v0/postgresql.py @@ -35,7 +35,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 48 +LIBPATCH = 49 # Groups to distinguish HBA access ACCESS_GROUP_IDENTITY = "identity_access" @@ -626,6 +626,7 @@ def list_access_groups(self) -> Set[str]: Returns: List of PostgreSQL database access groups. """ + connection = None try: with self._connect_to_database() as connection, connection.cursor() as cursor: cursor.execute( @@ -646,6 +647,7 @@ def list_users(self) -> Set[str]: Returns: List of PostgreSQL database users. """ + connection = None try: with self._connect_to_database() as connection, connection.cursor() as cursor: cursor.execute("SELECT usename FROM pg_catalog.pg_user;") @@ -664,6 +666,7 @@ def list_users_from_relation(self) -> Set[str]: Returns: List of PostgreSQL database users. """ + connection = None try: with self._connect_to_database() as connection, connection.cursor() as cursor: cursor.execute( diff --git a/poetry.lock b/poetry.lock index c624c0fbfc..00b6630685 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "allure-pytest" @@ -68,7 +68,7 @@ typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} [package.extras] doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1) ; python_version >= \"3.10\"", "uvloop (>=0.21) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\" and python_version < \"3.14\""] trio = ["trio (>=0.26.1)"] [[package]] @@ -100,12 +100,12 @@ files = [ ] [package.extras] -benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] -tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] -tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] +tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""] [[package]] name = "backoff" @@ -173,34 +173,34 @@ typecheck = ["mypy"] [[package]] name = "boto3" -version = "1.35.99" +version = "1.37.22" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" groups = ["main", "integration"] files = [ - {file = "boto3-1.35.99-py3-none-any.whl", hash = "sha256:83e560faaec38a956dfb3d62e05e1703ee50432b45b788c09e25107c5058bd71"}, - {file = "boto3-1.35.99.tar.gz", hash = "sha256:e0abd794a7a591d90558e92e29a9f8837d25ece8e3c120e530526fe27eba5fca"}, + {file = "boto3-1.37.22-py3-none-any.whl", hash = "sha256:a14324d5fa5f4fea00c0e3c69754cbd28100f7fe194693eeecf2dc07446cf4ef"}, + {file = "boto3-1.37.22.tar.gz", hash = "sha256:78a0ec0aafbf6044104c98ad80b69e6d1c83d8233fda2c2d241029e6c705c510"}, ] [package.dependencies] -botocore = ">=1.35.99,<1.36.0" +botocore = ">=1.37.22,<1.38.0" jmespath = ">=0.7.1,<2.0.0" -s3transfer = ">=0.10.0,<0.11.0" +s3transfer = ">=0.11.0,<0.12.0" [package.extras] crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.99" +version = "1.37.22" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" groups = ["main", "integration"] files = [ - {file = "botocore-1.35.99-py3-none-any.whl", hash = "sha256:b22d27b6b617fc2d7342090d6129000af2efd20174215948c0d7ae2da0fab445"}, - {file = "botocore-1.35.99.tar.gz", hash = "sha256:1eab44e969c39c5f3d9a3104a0836c24715579a455f12b3979a31d7cde51b3c3"}, + {file = "botocore-1.37.22-py3-none-any.whl", hash = "sha256:184db7c9314d13002bc827f511a5140574b5da1acda342d51e093dad6317de98"}, + {file = "botocore-1.37.22.tar.gz", hash = "sha256:b3b26f1a90236bcd17d4092f8c85a256b44e9955a16b633319a2f5678d605e9f"}, ] [package.dependencies] @@ -209,7 +209,7 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} [package.extras] -crt = ["awscrt (==0.22.0)"] +crt = ["awscrt (==0.23.8)"] [[package]] name = "cachetools" @@ -433,7 +433,7 @@ files = [ [package.extras] dev = ["Pygments", "build", "chardet", "pre-commit", "pytest", "pytest-cov", "pytest-dependency", "ruff", "tomli", "twine"] hard-encoding-detection = ["chardet"] -toml = ["tomli"] +toml = ["tomli ; python_version < \"3.11\""] types = ["chardet (>=5.1.0)", "mypy", "pytest", "pytest-cov", "pytest-dependency"] [[package]] @@ -546,7 +546,7 @@ files = [ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} [package.extras] -toml = ["tomli"] +toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "cryptography" @@ -593,10 +593,10 @@ files = [ cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} [package.extras] -docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0)"] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0) ; python_version >= \"3.8\""] docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"] -nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2)"] -pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] +nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2) ; python_version >= \"3.8\""] +pep8test = ["check-sdist ; python_version >= \"3.8\"", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] test = ["certifi (>=2024)", "cryptography-vectors (==44.0.1)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] @@ -630,7 +630,7 @@ files = [ wrapt = ">=1.10,<2" [package.extras] -dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools", "tox"] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools ; python_version >= \"3.12\"", "tox"] [[package]] name = "exceptiongroup" @@ -661,7 +661,7 @@ files = [ ] [package.extras] -tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] +tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich ; python_version >= \"3.11\""] [[package]] name = "google-auth" @@ -759,7 +759,7 @@ httpcore = "==1.*" idna = "*" [package.extras] -brotli = ["brotli", "brotlicffi"] +brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] @@ -816,7 +816,7 @@ zipp = ">=0.5" [package.extras] docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] +testing = ["flufl.flake8", "importlib-resources (>=1.3) ; python_version < \"3.9\"", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7) ; platform_python_implementation != \"PyPy\"", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1) ; platform_python_implementation != \"PyPy\"", "pytest-perf (>=0.9.2)", "pytest-ruff"] [[package]] name = "iniconfig" @@ -875,7 +875,7 @@ typing_extensions = {version = ">=4.6", markers = "python_version < \"3.12\""} [package.extras] all = ["ipython[black,doc,kernel,matplotlib,nbconvert,nbformat,notebook,parallel,qtconsole]", "ipython[test,test-extra]"] black = ["black"] -doc = ["docrepr", "exceptiongroup", "intersphinx_registry", "ipykernel", "ipython[test]", "matplotlib", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "sphinxcontrib-jquery", "tomli", "typing_extensions"] +doc = ["docrepr", "exceptiongroup", "intersphinx_registry", "ipykernel", "ipython[test]", "matplotlib", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "sphinxcontrib-jquery", "tomli ; python_version < \"3.11\"", "typing_extensions"] kernel = ["ipykernel"] matplotlib = ["matplotlib"] nbconvert = ["nbconvert"] @@ -1349,8 +1349,8 @@ cryptography = ">=3.3" pynacl = ">=1.5" [package.extras] -all = ["gssapi (>=1.4.1)", "invoke (>=2.0)", "pyasn1 (>=0.1.7)", "pywin32 (>=2.1.8)"] -gssapi = ["gssapi (>=1.4.1)", "pyasn1 (>=0.1.7)", "pywin32 (>=2.1.8)"] +all = ["gssapi (>=1.4.1) ; platform_system != \"Windows\"", "invoke (>=2.0)", "pyasn1 (>=0.1.7)", "pywin32 (>=2.1.8) ; platform_system == \"Windows\""] +gssapi = ["gssapi (>=1.4.1) ; platform_system != \"Windows\"", "pyasn1 (>=0.1.7)", "pywin32 (>=2.1.8) ; platform_system == \"Windows\""] invoke = ["invoke (>=2.0)"] [[package]] @@ -2135,21 +2135,21 @@ files = [ [[package]] name = "s3transfer" -version = "0.10.4" +version = "0.11.4" description = "An Amazon S3 Transfer Manager" optional = false python-versions = ">=3.8" groups = ["main", "integration"] files = [ - {file = "s3transfer-0.10.4-py3-none-any.whl", hash = "sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e"}, - {file = "s3transfer-0.10.4.tar.gz", hash = "sha256:29edc09801743c21eb5ecbc617a152df41d3c287f67b615f73e5f750583666a7"}, + {file = "s3transfer-0.11.4-py3-none-any.whl", hash = "sha256:ac265fa68318763a03bf2dc4f39d5cbd6a9e178d81cc9483ad27da33637e320d"}, + {file = "s3transfer-0.11.4.tar.gz", hash = "sha256:559f161658e1cf0a911f45940552c696735f5c74e64362e515f333ebed87d679"}, ] [package.dependencies] -botocore = ">=1.33.2,<2.0a.0" +botocore = ">=1.37.4,<2.0a.0" [package.extras] -crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] [[package]] name = "six" @@ -2323,7 +2323,7 @@ files = [ ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -2538,14 +2538,14 @@ files = [ ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "38d461f9c341e81b1034d0b3d789f39a5b6cb7c5fe83dbf3845e334e8c93d9a2" +content-hash = "824e1bff0e19325e59732dac0846c5232574bd375d65c0e966025e06ca88bad6" diff --git a/pyproject.toml b/pyproject.toml index a9328d1126..ce57af3173 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires-poetry = ">=2.0.0" [tool.poetry.dependencies] python = "^3.10" ops = "^2.18.1" -boto3 = "^1.35.99" +boto3 = "^1.37.22" pgconnstr = "^1.0.1" requests = "^2.32.3" tenacity = "^9.0.0" diff --git a/src/backups.py b/src/backups.py index d9a1fee86e..ce85cb21ff 100644 --- a/src/backups.py +++ b/src/backups.py @@ -12,7 +12,7 @@ from datetime import datetime, timezone from io import BytesIO -import boto3 as boto3 +import boto3 import botocore from botocore.exceptions import ClientError from charms.data_platform_libs.v0.s3 import CredentialsChangedEvent, S3Requirer @@ -88,6 +88,23 @@ def _tls_ca_chain_filename(self) -> str: return f"{self.charm._storage_path}/pgbackrest-tls-ca-chain.crt" return "" + def _get_s3_session_resource(self, s3_parameters: dict): + session = boto3.session.Session( + aws_access_key_id=s3_parameters["access-key"], + aws_secret_access_key=s3_parameters["secret-key"], + region_name=s3_parameters["region"], + ) + return session.resource( + "s3", + endpoint_url=self._construct_endpoint(s3_parameters), + verify=(self._tls_ca_chain_filename or None), + config=botocore.client.Config( + # https://github.com/boto/boto3/issues/4400#issuecomment-2600742103 + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ), + ) + def _are_backup_settings_ok(self) -> tuple[bool, str | None]: """Validates whether backup settings are OK.""" if self.model.get_relation(self.relation_name) is None: @@ -227,18 +244,9 @@ def _create_bucket_if_not_exists(self) -> None: bucket_name = s3_parameters["bucket"] region = s3_parameters.get("region") - session = boto3.session.Session( - aws_access_key_id=s3_parameters["access-key"], - aws_secret_access_key=s3_parameters["secret-key"], - region_name=s3_parameters["region"], - ) try: - s3 = session.resource( - "s3", - endpoint_url=self._construct_endpoint(s3_parameters), - verify=(self._tls_ca_chain_filename or None), - ) + s3 = self._get_s3_session_resource(s3_parameters) except ValueError as e: logger.exception("Failed to create a session '%s' in region=%s.", bucket_name, region) raise e @@ -1316,17 +1324,8 @@ def _upload_content_to_s3( processed_s3_path = os.path.join(s3_parameters["path"], s3_path).lstrip("/") try: logger.info(f"Uploading content to bucket={bucket_name}, path={processed_s3_path}") - session = boto3.session.Session( - aws_access_key_id=s3_parameters["access-key"], - aws_secret_access_key=s3_parameters["secret-key"], - region_name=s3_parameters["region"], - ) - s3 = session.resource( - "s3", - endpoint_url=self._construct_endpoint(s3_parameters), - verify=(self._tls_ca_chain_filename or None), - ) + s3 = self._get_s3_session_resource(s3_parameters) bucket = s3.Bucket(bucket_name) with tempfile.NamedTemporaryFile() as temp_file: @@ -1359,16 +1358,7 @@ def _read_content_from_s3(self, s3_path: str, s3_parameters: dict) -> str | None processed_s3_path = os.path.join(s3_parameters["path"], s3_path).lstrip("/") try: logger.info(f"Reading content from bucket={bucket_name}, path={processed_s3_path}") - session = boto3.session.Session( - aws_access_key_id=s3_parameters["access-key"], - aws_secret_access_key=s3_parameters["secret-key"], - region_name=s3_parameters["region"], - ) - s3 = session.resource( - "s3", - endpoint_url=self._construct_endpoint(s3_parameters), - verify=(self._tls_ca_chain_filename or None), - ) + s3 = self._get_s3_session_resource(s3_parameters) bucket = s3.Bucket(bucket_name) with BytesIO() as buf: bucket.download_fileobj(processed_s3_path, buf) diff --git a/templates/patroni.yml.j2 b/templates/patroni.yml.j2 index 570865700e..ea8cc6d436 100644 --- a/templates/patroni.yml.j2 +++ b/templates/patroni.yml.j2 @@ -47,11 +47,6 @@ bootstrap: logging_collector: 'on' wal_level: logical shared_preload_libraries: 'timescaledb,pgaudit' - {%- if pg_parameters %} - {%- for key, value in pg_parameters.items() %} - {{key}}: {{value}} - {%- endfor -%} - {% endif %} {%- if restoring_backup %} method: pgbackrest pgbackrest: diff --git a/tests/integration/test_backups_gcp.py b/tests/integration/test_backups_gcp.py index 07e452be93..cf34bc4ff1 100644 --- a/tests/integration/test_backups_gcp.py +++ b/tests/integration/test_backups_gcp.py @@ -66,6 +66,7 @@ async def test_backup_gcp(ops_test: OpsTest, charm, gcp_cloud_configs: tuple[dic ) +@pytest.mark.abort_on_fail async def test_restore_on_new_cluster( ops_test: OpsTest, charm, gcp_cloud_configs: tuple[dict, dict] ) -> None: diff --git a/tests/integration/test_config.py b/tests/integration/test_config.py index 229450b715..e1fc86485b 100644 --- a/tests/integration/test_config.py +++ b/tests/integration/test_config.py @@ -106,9 +106,7 @@ async def test_config_parameters(ops_test: OpsTest, charm) -> None: "optimizer_parallel_tuple_cost": ["-1", "0.1"] }, # config option is between 0 and 1.80E+308 {"profile": [test_string, "testing"]}, # config option is one of `testing` or `production` - # { - # "profile_limit_memory": {"127", "128"} - # }, # config option is between 128 and 9999999 + {"profile_limit_memory": ["127", "128"]}, # config option is between 128 and 9999999 { "request_backslash_quote": [test_string, "safe_encoding"] }, # config option is one of `safe_encoding` and `on` and `off` diff --git a/tests/unit/test_backups.py b/tests/unit/test_backups.py index 03cbfb7773..e3bfa0fb4d 100644 --- a/tests/unit/test_backups.py +++ b/tests/unit/test_backups.py @@ -331,6 +331,7 @@ def test_create_bucket_if_not_exists(harness, tls_ca_chain_filename): new_callable=PropertyMock(return_value=tls_ca_chain_filename), ) as _tls_ca_chain_filename, patch("charm.PostgreSQLBackups._retrieve_s3_parameters") as _retrieve_s3_parameters, + patch("backups.botocore.client.Config") as _config, ): # Test when there are missing S3 parameters. _retrieve_s3_parameters.return_value = ([], ["bucket", "access-key", "secret-key"]) @@ -357,13 +358,22 @@ def test_create_bucket_if_not_exists(harness, tls_ca_chain_filename): # Test when the bucket already exists. _resource.reset_mock() + _config.reset_mock() _resource.side_effect = None head_bucket = _resource.return_value.Bucket.return_value.meta.client.head_bucket create = _resource.return_value.Bucket.return_value.create wait_until_exists = _resource.return_value.Bucket.return_value.wait_until_exists harness.charm.backup._create_bucket_if_not_exists() _resource.assert_called_once_with( - "s3", endpoint_url="test-endpoint", verify=(tls_ca_chain_filename or None) + "s3", + endpoint_url="test-endpoint", + verify=(tls_ca_chain_filename or None), + config=_config.return_value, + ) + _config.assert_called_once_with( + # https://github.com/boto/boto3/issues/4400#issuecomment-2600742103 + request_checksum_calculation="when_required", + response_checksum_validation="when_required", ) head_bucket.assert_called_once() create.assert_not_called() @@ -2003,6 +2013,7 @@ def test_upload_content_to_s3(harness, tls_ca_chain_filename): patch("tempfile.NamedTemporaryFile") as _named_temporary_file, patch("charm.PostgreSQLBackups._construct_endpoint") as _construct_endpoint, patch("boto3.session.Session.resource") as _resource, + patch("backups.botocore.client.Config") as _config, patch( "charm.PostgreSQLBackups._tls_ca_chain_filename", new_callable=PropertyMock(return_value=tls_ca_chain_filename), @@ -2030,11 +2041,18 @@ def test_upload_content_to_s3(harness, tls_ca_chain_filename): "s3", endpoint_url="https://s3.us-east-1.amazonaws.com", verify=(tls_ca_chain_filename or None), + config=_config.return_value, + ) + _config.assert_called_once_with( + # https://github.com/boto/boto3/issues/4400#issuecomment-2600742103 + request_checksum_calculation="when_required", + response_checksum_validation="when_required", ) _named_temporary_file.assert_not_called() upload_file.assert_not_called() _resource.reset_mock() + _config.reset_mock() _resource.side_effect = None upload_file.side_effect = S3UploadFailedError assert harness.charm.backup._upload_content_to_s3(content, s3_path, s3_parameters) is False @@ -2042,12 +2060,19 @@ def test_upload_content_to_s3(harness, tls_ca_chain_filename): "s3", endpoint_url="https://s3.us-east-1.amazonaws.com", verify=(tls_ca_chain_filename or None), + config=_config.return_value, + ) + _config.assert_called_once_with( + # https://github.com/boto/boto3/issues/4400#issuecomment-2600742103 + request_checksum_calculation="when_required", + response_checksum_validation="when_required", ) _named_temporary_file.assert_called_once() upload_file.assert_called_once_with("/tmp/test-file", "test-path/test-file.") # Test when the upload succeeds _resource.reset_mock() + _config.reset_mock() _named_temporary_file.reset_mock() upload_file.reset_mock() upload_file.side_effect = None @@ -2056,6 +2081,12 @@ def test_upload_content_to_s3(harness, tls_ca_chain_filename): "s3", endpoint_url="https://s3.us-east-1.amazonaws.com", verify=(tls_ca_chain_filename or None), + config=_config.return_value, + ) + _config.assert_called_once_with( + # https://github.com/boto/boto3/issues/4400#issuecomment-2600742103 + request_checksum_calculation="when_required", + response_checksum_validation="when_required", ) _named_temporary_file.assert_called_once() upload_file.assert_called_once_with("/tmp/test-file", "test-path/test-file.")