diff --git a/.github/release.yaml b/.github/release.yaml new file mode 100644 index 0000000000..9ef36aca6d --- /dev/null +++ b/.github/release.yaml @@ -0,0 +1,8 @@ +changelog: + categories: + - title: Features + labels: + - enhancement + - title: Bug fixes + labels: + - bug diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 34085c9225..cd60ef68a5 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -6,6 +6,7 @@ reviewers: [ 'team:data-platform-postgresql', ], + "baseBranches": ["main", "/^*\\/edge$/"], packageRules: [ { matchPackageNames: [ @@ -13,32 +14,7 @@ ], allowedVersions: '<2.0.0', }, - { - matchManagers: [ - 'custom.regex', - ], - matchDepNames: [ - 'juju', - ], - matchDatasources: [ - 'pypi', - ], - allowedVersions: '<3', - groupName: 'Juju agents', - }, ], customManagers: [ - { - customType: 'regex', - fileMatch: [ - '^\\.github/workflows/[^/]+\\.ya?ml$', - ], - matchStrings: [ - '(libjuju: )==(?.*?) +# renovate: latest libjuju 2', - ], - depNameTemplate: 'juju', - datasourceTemplate: 'pypi', - versioningTemplate: 'loose', - }, ], } diff --git a/.github/workflows/approve_renovate_pr.yaml b/.github/workflows/approve_renovate_pr.yaml new file mode 100644 index 0000000000..4449576ea3 --- /dev/null +++ b/.github/workflows/approve_renovate_pr.yaml @@ -0,0 +1,15 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. +name: Approve Renovate pull request + +on: + pull_request: + types: + - opened + +jobs: + approve-pr: + name: Approve Renovate pull request + uses: canonical/data-platform-workflows/.github/workflows/approve_renovate_pr.yaml@v30.2.0 + permissions: + pull-requests: write # Needed to approve PR diff --git a/.github/workflows/check_pr.yaml b/.github/workflows/check_pr.yaml new file mode 100644 index 0000000000..6eb3823585 --- /dev/null +++ b/.github/workflows/check_pr.yaml @@ -0,0 +1,18 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. +name: Check pull request + +on: + pull_request: + types: + - opened + - labeled + - unlabeled + - edited + branches: + - main + +jobs: + check-pr: + name: Check pull request + uses: canonical/data-platform-workflows/.github/workflows/check_charm_pr.yaml@v30.2.0 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b9dd27ad30..a90828304a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,7 +27,7 @@ on: jobs: lint: name: Lint - uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v29.1.0 + uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v30.2.0 unit-test: name: Unit test charm @@ -49,48 +49,19 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v29.1.0 + uses: canonical/data-platform-workflows/.github/workflows/build_charm.yaml@v30.2.0 + with: + cache: false # TODO remove when 16/edge branch is set up integration-test: - strategy: - fail-fast: false - matrix: - juju: - - agent: 2.9.51 # renovate: juju-agent-pin-minor - libjuju: ==2.9.49.1 # renovate: latest libjuju 2 - allure_on_amd64: false - - agent: 3.6.2 # renovate: juju-agent-pin-minor - allure_on_amd64: true - architecture: - - amd64 - include: - - juju: - agent: 3.6.2 # renovate: juju-agent-pin-minor - allure_on_amd64: true - architecture: arm64 - name: Integration | ${{ matrix.juju.agent }} | ${{ matrix.architecture }} + name: Integration test charm needs: - lint - unit-test - build - uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@v29.1.0 + uses: ./.github/workflows/integration_test.yaml with: artifact-prefix: ${{ needs.build.outputs.artifact-prefix }} - architecture: ${{ matrix.architecture }} - cloud: lxd - juju-agent-version: ${{ matrix.juju.agent }} - libjuju-version-constraint: ${{ matrix.juju.libjuju }} - _beta_allure_report: ${{ matrix.juju.allure_on_amd64 && matrix.architecture == 'amd64' }} - secrets: - integration-test: | - { - "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}", - "AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}", - "GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}", - "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}", - "UBUNTU_PRO_TOKEN" : "${{ secrets.UBUNTU_PRO_TOKEN }}", - "LANDSCAPE_ACCOUNT_NAME": "${{ secrets.LANDSCAPE_ACCOUNT_NAME }}", - "LANDSCAPE_REGISTRATION_KEY": "${{ secrets.LANDSCAPE_REGISTRATION_KEY }}", - } + secrets: inherit permissions: - contents: write # Needed for Allure Report beta + contents: write # Needed for Allure Report diff --git a/.github/workflows/cla-check.yml b/.github/workflows/cla-check.yml index f0590d5b65..2567517472 100644 --- a/.github/workflows/cla-check.yml +++ b/.github/workflows/cla-check.yml @@ -9,4 +9,4 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Check if Canonical's Contributor License Agreement has been signed - uses: canonical/has-signed-canonical-cla@v1 + uses: canonical/has-signed-canonical-cla@v2 diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml new file mode 100644 index 0000000000..a89194c525 --- /dev/null +++ b/.github/workflows/integration_test.yaml @@ -0,0 +1,316 @@ +on: + workflow_call: + inputs: + artifact-prefix: + description: | + Prefix for charm package GitHub artifact(s) + + Use canonical/data-platform-workflows build_charm.yaml to build the charm(s) + required: true + type: string + +jobs: + collect-integration-tests: + name: Collect integration test spread jobs + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up environment + run: | + sudo snap install charmcraft --classic + pipx install tox poetry + - name: Collect spread jobs + id: collect-jobs + shell: python + run: | + import json + import os + import subprocess + + spread_jobs = ( + subprocess.run( + ["charmcraft", "test", "--list", "github-ci"], capture_output=True, check=True, text=True + ) + .stdout.strip() + .split("\n") + ) + jobs = [] + for job in spread_jobs: + # Example `job`: "github-ci:ubuntu-24.04:tests/spread/test_charm.py:juju36" + _, runner, task, variant = job.split(":") + # Example: "test_charm.py" + task = task.removeprefix("tests/spread/") + if runner.endswith("-arm"): + architecture = "arm64" + else: + architecture = "amd64" + # Example: "test_charm.py:juju36 | amd64" + name = f"{task}:{variant} | {architecture}" + # ":" character not valid in GitHub Actions artifact + name_in_artifact = f"{task}-{variant}-{architecture}" + jobs.append({ + "spread_job": job, + "name": name, + "name_in_artifact": name_in_artifact, + "runner": runner, + }) + output = f"jobs={json.dumps(jobs)}" + print(output) + with open(os.environ["GITHUB_OUTPUT"], "a") as file: + file.write(output) + - name: Generate Allure default test results + if: ${{ github.event_name == 'schedule' && github.run_attempt == '1' }} + run: tox run -e integration -- tests/integration --allure-default-dir=allure-default-results + - name: Upload Allure default results + # Default test results in case the integration tests time out or runner set up fails + # (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test) + if: ${{ github.event_name == 'schedule' && github.run_attempt == '1' }} + uses: actions/upload-artifact@v4 + with: + name: allure-default-results-integration-test + path: allure-default-results/ + if-no-files-found: error + outputs: + jobs: ${{ steps.collect-jobs.outputs.jobs }} + + integration-test: + strategy: + fail-fast: false + matrix: + job: ${{ fromJSON(needs.collect-integration-tests.outputs.jobs) }} + name: ${{ matrix.job.name }} + needs: + - collect-integration-tests + runs-on: ${{ matrix.job.runner }} + timeout-minutes: 217 # Sum of steps `timeout-minutes` + 5 + steps: + - name: Free up disk space + timeout-minutes: 1 + run: | + printf '\nDisk usage before cleanup\n' + df --human-readable + # Based on https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + rm -r /opt/hostedtoolcache/ + printf '\nDisk usage after cleanup\n' + df --human-readable + - name: Checkout + timeout-minutes: 3 + uses: actions/checkout@v4 + - name: Set up environment + timeout-minutes: 5 + run: sudo snap install charmcraft --classic + # TODO: remove when https://github.com/canonical/charmcraft/issues/2105 and + # https://github.com/canonical/charmcraft/issues/2130 fixed + - run: | + sudo snap install go --classic + go install github.com/snapcore/spread/cmd/spread@latest + - name: Download packed charm(s) + timeout-minutes: 5 + uses: actions/download-artifact@v4 + with: + pattern: ${{ inputs.artifact-prefix }}-* + merge-multiple: true + - name: Run spread job + timeout-minutes: 180 + id: spread + # TODO: replace with `charmcraft test` when + # https://github.com/canonical/charmcraft/issues/2105 and + # https://github.com/canonical/charmcraft/issues/2130 fixed + run: ~/go/bin/spread -vv -artifacts=artifacts '${{ matrix.job.spread_job }}' + env: + AWS_ACCESS_KEY: ${{ secrets.AWS_ACCESS_KEY }} + AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} + GCP_ACCESS_KEY: ${{ secrets.GCP_ACCESS_KEY }} + GCP_SECRET_KEY: ${{ secrets.GCP_SECRET_KEY }} + UBUNTU_PRO_TOKEN: ${{ secrets.UBUNTU_PRO_TOKEN }} + LANDSCAPE_ACCOUNT_NAME: ${{ secrets.LANDSCAPE_ACCOUNT_NAME }} + LANDSCAPE_REGISTRATION_KEY: ${{ secrets.LANDSCAPE_REGISTRATION_KEY }} + - name: Upload Allure results + timeout-minutes: 3 + # Only upload results from one spread system & one spread variant + # Allure can only process one result per pytest test ID. If parameterization is done via + # spread instead of pytest, there will be overlapping pytest test IDs. + if: ${{ (success() || (failure() && steps.spread.outcome == 'failure')) && startsWith(matrix.job.spread_job, 'github-ci:ubuntu-24.04:') && endsWith(matrix.job.spread_job, ':juju36') && github.event_name == 'schedule' && github.run_attempt == '1' }} + uses: actions/upload-artifact@v4 + with: + name: allure-results-integration-test-${{ matrix.job.name_in_artifact }} + path: artifacts/${{ matrix.job.spread_job }}/allure-results/ + if-no-files-found: error + - timeout-minutes: 1 + if: ${{ success() || (failure() && steps.spread.outcome == 'failure') }} + run: snap list + - name: Select model + timeout-minutes: 1 + # `!contains(matrix.job.spread_job, 'juju29')` workaround for juju 2 error: + # "ERROR cannot acquire lock file to read controller concierge-microk8s: unable to open + # /tmp/juju-store-lock-3635383939333230: permission denied" + # Unable to workaround error with `sudo rm /tmp/juju-*` + if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }} + id: juju-switch + run: | + # sudo needed since spread runs scripts as root + # "testing" is default model created by concierge + sudo juju switch testing + mkdir ~/logs/ + - name: juju status + timeout-minutes: 1 + if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }} + run: sudo juju status --color --relations | tee ~/logs/juju-status.txt + - name: juju debug-log + timeout-minutes: 3 + if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }} + run: sudo juju debug-log --color --replay --no-tail | tee ~/logs/juju-debug-log.txt + - name: jhack tail + timeout-minutes: 3 + if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }} + run: sudo jhack tail --printer raw --replay --no-watch | tee ~/logs/jhack-tail.txt + - name: Upload logs + timeout-minutes: 5 + if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }} + uses: actions/upload-artifact@v4 + with: + name: logs-integration-test-${{ matrix.job.name_in_artifact }} + path: ~/logs/ + if-no-files-found: error + - name: Disk usage + timeout-minutes: 1 + if: ${{ success() || (failure() && steps.spread.outcome == 'failure') }} + run: df --human-readable + + allure-report: + # TODO future improvement: use concurrency group for job + name: Publish Allure report + if: ${{ !cancelled() && github.event_name == 'schedule' && github.run_attempt == '1' }} + needs: + - integration-test + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Download Allure + # Following instructions from https://allurereport.org/docs/install-for-linux/#install-from-a-deb-package + run: gh release download --repo allure-framework/allure2 --pattern 'allure_*.deb' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install Allure + run: | + sudo apt-get update + sudo apt-get install ./allure_*.deb -y + # For first run, manually create branch with no history + # (e.g. + # git checkout --orphan gh-pages-beta + # git rm -rf . + # touch .nojekyll + # git add .nojekyll + # git commit -m "Initial commit" + # git push origin gh-pages-beta + # ) + - name: Checkout GitHub pages branch + uses: actions/checkout@v4 + with: + ref: gh-pages-beta + path: repo/ + - name: Download default test results + # Default test results in case the integration tests time out or runner set up fails + # (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test) + uses: actions/download-artifact@v4 + with: + path: allure-default-results/ + name: allure-default-results-integration-test + - name: Download test results + uses: actions/download-artifact@v4 + with: + path: allure-results/ + pattern: allure-results-integration-test-* + merge-multiple: true + - name: Combine Allure default results & actual results + # For every test: if actual result available, use that. Otherwise, use default result + # So that, if actual result not available, Allure report will show "unknown"/"failed" test result + # instead of omitting the test + shell: python + run: | + import dataclasses + import json + import pathlib + + + @dataclasses.dataclass(frozen=True) + class Result: + test_case_id: str + path: pathlib.Path + + def __eq__(self, other): + if not isinstance(other, type(self)): + return False + return self.test_case_id == other.test_case_id + + + actual_results = pathlib.Path("allure-results") + default_results = pathlib.Path("allure-default-results") + + results: dict[pathlib.Path, set[Result]] = { + actual_results: set(), + default_results: set(), + } + for directory, results_ in results.items(): + for path in directory.glob("*-result.json"): + with path.open("r") as file: + id_ = json.load(file)["testCaseId"] + results_.add(Result(id_, path)) + + actual_results.mkdir(exist_ok=True) + + missing_results = results[default_results] - results[actual_results] + for default_result in missing_results: + # Move to `actual_results` directory + default_result.path.rename(actual_results / default_result.path.name) + - name: Load test report history + run: | + if [[ -d repo/_latest/history/ ]] + then + echo 'Loading history' + cp -r repo/_latest/history/ allure-results/ + fi + - name: Create executor.json + shell: python + run: | + # Reverse engineered from https://github.com/simple-elf/allure-report-action/blob/eca283b643d577c69b8e4f048dd6cd8eb8457cfd/entrypoint.sh + import json + + DATA = { + "name": "GitHub Actions", + "type": "github", + "buildOrder": ${{ github.run_number }}, # TODO future improvement: use run ID + "buildName": "Run ${{ github.run_id }}", + "buildUrl": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}", + "reportUrl": "../${{ github.run_number }}/", + } + with open("allure-results/executor.json", "w") as file: + json.dump(DATA, file) + - name: Generate Allure report + run: allure generate + - name: Create index.html + shell: python + run: | + DATA = f""" + + + + """ + with open("repo/index.html", "w") as file: + file.write(DATA) + - name: Update GitHub pages branch + working-directory: repo/ + # TODO future improvement: commit message + run: | + mkdir '${{ github.run_number }}' + rm -f _latest + ln -s '${{ github.run_number }}' _latest + cp -r ../allure-report/. _latest/ + git add . + git config user.name "GitHub Actions" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git commit -m "Allure report ${{ github.run_number }}" + # Uses token set in checkout step + git push origin gh-pages-beta diff --git a/.github/workflows/promote.yaml b/.github/workflows/promote.yaml new file mode 100644 index 0000000000..6b2832b4ec --- /dev/null +++ b/.github/workflows/promote.yaml @@ -0,0 +1,36 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. +name: Promote charm + +on: + workflow_dispatch: + inputs: + from-risk: + description: Promote from this Charmhub risk + required: true + type: choice + options: + - edge + - beta + - candidate + to-risk: + description: Promote to this Charmhub risk + required: true + type: choice + options: + - beta + - candidate + - stable + +jobs: + promote: + name: Promote charm + uses: canonical/data-platform-workflows/.github/workflows/_promote_charm.yaml@v30.2.0 + with: + track: '16' + from-risk: ${{ inputs.from-risk }} + to-risk: ${{ inputs.to-risk }} + secrets: + charmhub-token: ${{ secrets.CHARMHUB_TOKEN }} + permissions: + contents: write # Needed to edit GitHub releases diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index deefea45b2..c194536590 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -5,7 +5,7 @@ name: Release to Charmhub on: push: branches: - - main + - '*/edge' paths-ignore: - 'tests/**' - 'docs/**' @@ -21,15 +21,15 @@ jobs: uses: ./.github/workflows/ci.yaml secrets: inherit permissions: - contents: write # Needed for Allure Report beta + contents: write # Needed for Allure Report release: name: Release charm needs: - ci-tests - uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v29.1.0 + uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v30.2.0 with: - channel: 14/edge + channel: ${{ github.ref_name }} artifact-prefix: ${{ needs.ci-tests.outputs.artifact-prefix }} secrets: charmhub-token: ${{ secrets.CHARMHUB_TOKEN }} diff --git a/.github/workflows/sync_docs.yaml b/.github/workflows/sync_docs.yaml index 179e10d527..99dce73a21 100644 --- a/.github/workflows/sync_docs.yaml +++ b/.github/workflows/sync_docs.yaml @@ -10,7 +10,7 @@ on: jobs: sync-docs: name: Sync docs from Discourse - uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v29.1.0 + uses: canonical/data-platform-workflows/.github/workflows/sync_docs.yaml@v30.2.0 with: reviewers: a-velasco,izmalk permissions: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e22c8c702..388378cdff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,7 @@ source venv/bin/activate tox run -e format # update your code according to linting rules tox run -e lint # code style tox run -e unit # unit tests -tox run -e integration # integration tests +charmcraft test lxd-vm: # integration tests tox # runs 'lint' and 'unit' environments ``` diff --git a/README.md b/README.md index 7c3e83c644..db0c8bb6b1 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ juju add-model sample-model To deploy a single unit of PostgreSQL using its [default configuration](config.yaml), run the following command: ```shell -juju deploy postgresql --channel 14/stable +juju deploy postgresql --channel 16/stable ``` It is customary to use PostgreSQL with replication to ensure high availability. A replica is equivalent to a juju unit. @@ -37,7 +37,7 @@ It is customary to use PostgreSQL with replication to ensure high availability. To deploy PostgreSQL with multiple replicas, specify the number of desired units with the `-n` option: ```shell -juju deploy postgresql --channel 14/stable -n +juju deploy postgresql --channel 16/stable -n ``` To add replicas to an existing deployment, see the [Add replicas](#add-replicas) section. diff --git a/charmcraft.yaml b/charmcraft.yaml index 87a3f72d53..e997e8bc3d 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -3,8 +3,8 @@ type: charm platforms: - ubuntu@22.04:amd64: - ubuntu@22.04:arm64: + ubuntu@24.04:amd64: + ubuntu@24.04:arm64: # Files implicitly created by charmcraft without a part: # - dispatch (https://github.com/canonical/charmcraft/pull/1898) # - manifest.yaml @@ -24,10 +24,10 @@ parts: # Use environment variable instead of `--break-system-packages` to avoid failing on older # versions of pip that do not recognize `--break-system-packages` # `--user` needed (in addition to `--break-system-packages`) for Ubuntu >=24.04 - PIP_BREAK_SYSTEM_PACKAGES=true python3 -m pip install --user --upgrade pip==25.0 # renovate: charmcraft-pip-latest + PIP_BREAK_SYSTEM_PACKAGES=true python3 -m pip install --user --upgrade pip==25.0.1 # renovate: charmcraft-pip-latest # Use uv to install poetry so that a newer version of Python can be installed if needed by poetry - curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.29/uv-installer.sh | sh # renovate: charmcraft-uv-latest + curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.6.5/uv-installer.sh | sh # renovate: charmcraft-uv-latest # poetry 2.0.0 requires Python >=3.9 if ! "$HOME/.local/bin/uv" python find '>=3.9' then @@ -35,7 +35,7 @@ parts: # (to reduce the number of Python versions we use) "$HOME/.local/bin/uv" python install 3.10.12 # renovate: charmcraft-python-ubuntu-22.04 fi - "$HOME/.local/bin/uv" tool install --no-python-downloads --python '>=3.9' poetry==2.0.1 --with poetry-plugin-export==1.9.0 # renovate: charmcraft-poetry-latest + "$HOME/.local/bin/uv" tool install --no-python-downloads --python '>=3.9' poetry==2.1.1 --with poetry-plugin-export==1.9.0 # renovate: charmcraft-poetry-latest ln -sf "$HOME/.local/bin/poetry" /usr/local/bin/poetry # "charm-poetry" part name is arbitrary; use for consistency @@ -75,7 +75,7 @@ parts: # rpds-py (Python package) >=0.19.0 requires rustc >=1.76, which is not available in the # Ubuntu 22.04 archive. Install rustc and cargo using rustup instead of the Ubuntu archive rustup set profile minimal - rustup default 1.84.1 # renovate: charmcraft-rust-latest + rustup default 1.85.0 # renovate: charmcraft-rust-latest craftctl default # Include requirements.txt in *.charm artifact for easier debugging diff --git a/concierge.yaml b/concierge.yaml new file mode 100644 index 0000000000..15a78cc947 --- /dev/null +++ b/concierge.yaml @@ -0,0 +1,13 @@ +juju: + model-defaults: + logging-config: =INFO; unit=DEBUG +providers: + lxd: + enable: true + bootstrap: true +host: + snaps: + jhack: + channel: latest/edge + connections: + - jhack:dot-local-share-juju snapd diff --git a/config.yaml b/config.yaml index 8b474ec962..46c3c9f3a7 100644 --- a/config.yaml +++ b/config.yaml @@ -2,6 +2,12 @@ # See LICENSE file for licensing details. options: + synchronous_node_count: + description: | + Sets the number of synchronous nodes to be maintained in the cluster. Should be + either "all", "majority" or a positive integer value. + type: string + default: "all" durability_synchronous_commit: description: | Sets the current transactions synchronization level. This charm allows only the diff --git a/docs/explanation.md b/docs/explanation.md new file mode 100644 index 0000000000..8fc131b9e6 --- /dev/null +++ b/docs/explanation.md @@ -0,0 +1,21 @@ +# Explanation + +This section contains pages with more detailed explanations that provide additional context about some of the key concepts behind the PostgreSQL charm: + +* [Architecture] +* [Interfaces and endpoints] +* [Connection pooling] +* [Users] +* [Logs] +* [Juju] +* [Legacy charm] + + + +[Architecture]: /t/11857 +[Interfaces and endpoints]: /t/10251 +[Users]: /t/10798 +[Logs]: /t/12099 +[Juju]: /t/11985 +[Legacy charm]: /t/10690 +[Connection pooling]: /t/15777 \ No newline at end of file diff --git a/docs/explanation/e-cryptography.md b/docs/explanation/e-cryptography.md new file mode 100644 index 0000000000..51e3c6bed6 --- /dev/null +++ b/docs/explanation/e-cryptography.md @@ -0,0 +1,61 @@ +# Cryptography + +This document describes the cryptography used by Charmed PostgreSQL. + +## Resource checksums + +Charmed PostgreSQL and Charmed PgBouncer operators use pinned versions of the respective snaps to provide reproducible and secure environments. + +The snaps package their workloads along with the necessary dependencies and utilities required for the operators’ lifecycle. For more details, see the snaps content in the `snapcraft.yaml` file for [PostgreSQL](https://github.com/canonical/charmed-postgresql-snap/blob/14/edge/snap/snapcraft.yaml) and [PgBouncer](https://github.com/canonical/charmed-pgbouncer-snap/blob/1/edge/snap/snapcraft.yaml). + +Every artifact bundled into a snap is verified against its MD5, SHA256, or SHA512 checksum after download. The installation of certified snap into the rock is ensured by snap primitives that verify their squashfs filesystems images GPG signature. For more information on the snap verification process, refer to the [snapcraft.io documentation](https://snapcraft.io/docs/assertions). + +## Sources verification + +PostgreSQL and its extra components are built by Canonical from upstream source codes on [Launchpad](https://launchpad.net/ubuntu/+source/postgresql-common). PostgreSQL and PgBouncer are built as deb packages, other components - as PPAs. + +Charmed PostgreSQL and Charmed PgBouncer charms and snaps are published and released programmatically using release pipelines implemented via GitHub Actions in their respective repositories. + +All repositories in GitHub are set up with branch protection rules, requiring: + +* new commits to be merged to main branches via pull request with at least 2 approvals from repository maintainers +* new commits to be signed (e.g. using GPG keys) +* developers to sign the [Canonical Contributor License Agreement (CLA)](https://ubuntu.com/legal/contributors) + +## Encryption + +Charmed PostgreSQL can be used to deploy a secure PostgreSQL cluster that provides encryption-in-transit capabilities out of the box for: + +* Cluster internal communications +* PgBouncer connections +* External clients connections + +To set up a secure connection Charmed PostgreSQL and Charmed PgBouncer need to be integrated with TLS Certificate Provider charms, e.g. self-signed-certificates operator. Certificate Signing Requests (CSRs) are generated for every unit using the tls_certificates_interface library that uses the cryptography Python library to create X.509 compatible certificates. The CSR is signed by the TLS Certificate Provider, returned to the units, and stored in Juju secret. The relation also provides the CA certificate, which is loaded into Juju secret. + +Encryption at rest is currently not supported, although it can be provided by the substrate (cloud or on-premises). + +## Authentication + +In Charmed PostgreSQL, authentication layers can be enabled for: + +1. PgBouncer authentication to PostgreSQL +2. PostgreSQL cluster authentication +3. Clients authentication to PostgreSQL + +### PgBouncer authentication to PostgreSQL + +Authentication of PgBouncer to PostgreSQL is based on the password-based `scram-sha-256` authentication method. See the [PostgreSQL official documentation](https://www.postgresql.org/docs/14/auth-password.html) for more details. + +Credentials are exchanged via [Juju secrets](https://canonical-juju.readthedocs-hosted.com/en/latest/user/howto/manage-secrets/). + +### PostgreSQL cluster authentication + +Authentication among members of a PostgreSQL cluster is based on the password-based `scram-sha-256` authentication method. + +An internal user is used for this authentication with its hashed password stored in a system metadata database. These credentials are also stored as a plain text file on the disk of each unit for the Patroni HA service. + +### Clients authentication to PostgreSQL + +Authentication of clients to PostgreSQL is based on the password-based `scram-sha-256` authentication method. See the [PostgreSQL official documentation](https://www.postgresql.org/docs/14/auth-password.html) for more details. + +Credentials are exchanged via [Juju secrets](https://canonical-juju.readthedocs-hosted.com/en/latest/user/howto/manage-secrets/). \ No newline at end of file diff --git a/docs/explanation/e-juju-details.md b/docs/explanation/e-juju-details.md index 65f3b702cd..a8554674eb 100644 --- a/docs/explanation/e-juju-details.md +++ b/docs/explanation/e-juju-details.md @@ -1,13 +1,15 @@ -# Juju tech details +# Juju [Juju](https://juju.is/) is an open source orchestration engine for software operators that enables the deployment, integration and lifecycle management of applications at any scale, on any infrastructure using charms. -This [charm](https://charmhub.io/postgresql) is an operator - business logic encapsulated in reusable software packages that automate every aspect of an application's life. Charms are shared via [CharmHub](https://charmhub.io/). +> See also: [Juju client documentation](https://juju.is/docs/juju), [Juju blog](https://ubuntu.com/blog/tag/juju) -See also: +## Compatibility with PostgreSQL -* [Juju Documentation](https://juju.is/docs/juju) and [Blog](https://ubuntu.com/blog/tag/juju) -* [Charm SDK](https://juju.is/docs/sdk) +Current stable releases of this charm can still be deployed on Juju 2.9. However, newer features are not supported. +> See the [Releases page](/t/11875) for more information about the minimum Juju version required to operate the features of each revision. + +Additionally, there are limitations regarding integrations with other charms. For example, integration with [modern TLS charms](https://charmhub.io/topics/security-with-x-509-certificates) requires Juju 3.x. ## Breaking changes between Juju 2.9.x and 3.x @@ -15,18 +17,18 @@ As this charm documentation is written for Juju 3.x, users of 2.9.x will encount Breaking changes have been introduced in the Juju client between versions 2.9.x and 3.x. These are caused by the renaming and re-purposing of several commands - functionality and command options remain unchanged. -In the context of this guide, the pertinent changes are shown here: +In the context of this guide, the pertinent changes are as follows: -|2.9.x|3.x| +| v2.9.x | v3.x | | --- | --- | -|**add-relation**|**integrate**| -|**relate**|**integrate**| -|**run**|**exec**| -|**run-action --wait**|**run**| +|`add-relation`|`integrate`| +|`relate`|`integrate`| +|`run`|`exec`| +|`run-action --wait`|`run`| See the [Juju 3.0 release notes](https://juju.is/docs/juju/roadmap#heading--juju-3-0-0---22-oct-2022) for the comprehensive list of changes. -The response is to therefore substitute the documented command with the equivalent 2.9.x command. For example: +Example substitutions: ### Juju 3.x: ```shell diff --git a/docs/explanation/e-security.md b/docs/explanation/e-security.md new file mode 100644 index 0000000000..24c97804cc --- /dev/null +++ b/docs/explanation/e-security.md @@ -0,0 +1,92 @@ +# Security hardening guide + +This document provides an overview of security features and guidance for hardening the security of [Charmed PostgreSQL](https://charmhub.io/postgresql) deployments, including setting up and managing a secure environment. + +## Environment + +The environment where Charmed PostgreSQL operates can be divided into two components: + +1. Cloud +2. Juju + +### Cloud + +Charmed PostgreSQL can be deployed on top of several clouds and virtualization layers: + +|Cloud|Security guides| +| --- | --- | +|OpenStack|[OpenStack Security Guide](https://docs.openstack.org/security-guide/)| +|AWS|[Best Practices for Security, Identity and Compliance](https://aws.amazon.com/architecture/security-identity-compliance), [AWS security credentials](https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html#access-keys-and-secret-access-keys)| +|Azure|[Azure security best practices and patterns](https://learn.microsoft.com/en-us/azure/security/fundamentals/best-practices-and-patterns), [Managed identities for Azure resource](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/)| +|GCP|[Google security overview](https://cloud.google.com/docs/security)| + +### Juju + +Juju is the component responsible for orchestrating the entire lifecycle, from deployment to Day 2 operations. For more information on Juju security hardening, see the [Juju security page](https://canonical-juju.readthedocs-hosted.com/en/latest/user/explanation/juju-security/) and the [How to harden your deployment](https://juju.is/docs/juju/harden-your-deployment) guide. + +#### Cloud credentials + +When configuring cloud credentials to be used with Juju, ensure that users have correct permissions to operate at the required level. Juju superusers responsible for bootstrapping and managing controllers require elevated permissions to manage several kinds of resources, such as virtual machines, networks, storages, etc. Please refer to the links below for more information on the policies required to be used depending on the cloud. + +|Cloud|Cloud user policies| +| --- | --- | +|OpenStack|N/A| +|AWS|[Juju AWS Permission](/t/juju-aws-permissions/5307), [AWS Instance Profiles](/t/using-aws-instance-profiles-with-juju-2-9/5185), [Juju on AWS](https://juju.is/docs/juju/amazon-ec2)| +|Azure|[Juju Azure Permission](https://juju.is/docs/juju/microsoft-azure), [How to use Juju with Microsoft Azure](/t/how-to-use-juju-with-microsoft-azure/15219)| +|GCP|[Google Cloud's Identity and Access Management](https://cloud.google.com/iam/docs/overview), [GCE role recommendations](https://cloud.google.com/policy-intelligence/docs/role-recommendations-overview), [Google GCE cloud and Juju](https://canonical-juju.readthedocs-hosted.com/en/latest/user/reference/cloud/list-of-supported-clouds/the-google-gce-cloud-and-juju/)| + +#### Juju users + +It is very important that Juju users are set up with minimal permissions depending on the scope of their operations. Please refer to the [User access levels](https://juju.is/docs/juju/user-permissions) documentation for more information on the access levels and corresponding abilities. + +Juju user credentials must be stored securely and rotated regularly to limit the chances of unauthorized access due to credentials leakage. + +## Applications + +In the following sections, we provide guidance on how to harden your deployment using: + +1. Operating system +2. Security upgrades +3. Encryption +4. Authentication +5. Monitoring and auditing + +### Operating system + +Charmed PostgreSQL and Charmed PgBouncer run on top of Ubuntu 22.04. Deploy a [Landscape Client Charm](https://charmhub.io/landscape-client?) to connect the underlying VM to a Landscape User Account to manage security upgrades and integrate [Ubuntu Pro](https://ubuntu.com/pro) subscriptions. + +### Security upgrades + +[Charmed PostgreSQL](https://charmhub.io/postgresql) and [Charmed PgBouncer](https://charmhub.io/pgbouncer) operators install pinned versions of their respective snaps to provide reproducible and secure environments. + +New versions (revisions) of the charmed operators can be released to update the operator's code, workloads, or both. It is important to refresh the charms regularly to make sure the workloads are as secure as possible. + +For more information on upgrading Charmed PostgreSQL, see the [How to upgrade PostgreSQL](https://canonical.com/data/docs/postgresql/iaas/h-upgrade) and [How to upgrade PgBouncer](https://charmhub.io/pgbouncer/docs/h-upgrade) guides, as well as the respective Release notes for [PostgreSQL](https://canonical.com/data/docs/postgresql/iaas/r-releases) and [PgBouncer](https://charmhub.io/pgbouncer/docs/r-releases). + +### Encryption + +To utilise encryption at transit for all internal and external cluster connections, integrate Charmed PostgreSQL with a TLS certificate provider. Please refer to the [Charming Security page](https://charmhub.io/topics/security-with-x-509-certificates) for more information on how to select the right certificate provider for your use case. + +Encryption in transit for backups is provided by the storage service (Charmed PostgreSQL is a client for an S3-compatible storage). + +For more information on encryption, see the [Cryptography](/t/charmed-postgresql-explanations-encryption/16853) explanation page and [How to enable encryption](https://canonical.com/data/docs/postgresql/iaas/h-enable-tls) guide. + +### Authentication + +Charmed PostgreSQL supports the password-based `scram-sha-256` authentication method for authentication between: + +* External connections to clients +* Internal connections between members of cluster +* PgBouncer connections + +For more implementation details, see the [PostgreSQL documentation](https://www.postgresql.org/docs/14/auth-password.html). + +### Monitoring and auditing + +Charmed PostgreSQL provides native integration with the [Canonical Observability Stack (COS)](https://charmhub.io/topics/canonical-observability-stack). To reduce the blast radius of infrastructure disruptions, the general recommendation is to deploy COS and the observed application into separate environments, isolated from one another. Refer to the [COS production deployments best practices](https://charmhub.io/topics/canonical-observability-stack/reference/best-practices) for more information or see the How to guides for PostgreSQL [monitoring](https://canonical.com/data/docs/postgresql/iaas/h-enable-monitoring), [alert rules](https://canonical.com/data/docs/postgresql/iaas/h-enable-alert-rules), and [tracing](https://canonical.com/data/docs/postgresql/iaas/h-enable-tracing) for practical instructions. + +PostgreSQL logs are stored in `/var/snap/charmed-postgresql/common/var/log/postgresql` within the postgresql container of each unit. It’s recommended to integrate the charm with [COS](/t/10600), from where the logs can be easily persisted and queried using [Loki](https://charmhub.io/loki-k8s)/[Grafana](https://charmhub.io/grafana). + +## Additional Resources + +For details on the cryptography used by Charmed PostgreSQL, see the [Cryptography](/t/charmed-postgresql-explanations-encryption/16853) explanation page. \ No newline at end of file diff --git a/docs/how-to.md b/docs/how-to.md new file mode 100644 index 0000000000..497d33ce18 --- /dev/null +++ b/docs/how-to.md @@ -0,0 +1,102 @@ +# How-to guides + +The following guides cover key processes and common tasks for managing and using Charmed PostgreSQL on machines. + +## Deployment and setup + +The following guides walk you through the details of how to install different cloud services and bootstrap them to Juju: +* [Sunbeam] +* [LXD] +* [MAAS] +* [AWS EC2] +* [GCE] +* [Azure] +* [Multi-availability zones (AZ)][Multi-AZ] + +The following guides cover some specific deployment scenarios and architectures: +* [Terraform] +* [Air-gapped] +* [TLS VIP access] + +## Usage and maintenance + +* [Integrate with another application] +* [External access] +* [Scale replicas] +* [Enable TLS] +* [Enable plugins/extensions] + +## Backup and restore +* [Configure S3 AWS] +* [Configure S3 RadosGW] +* [Create a backup] +* [Restore a backup] +* [Manage backup retention] +* [Migrate a cluster] + +## Monitoring (COS) + +* [Enable monitoring] +* [Enable alert rules] +* [Enable tracing] + +## Minor upgrades +* [Perform a minor upgrade] +* [Perform a minor rollback] + +## Cross-regional (cluster-cluster) async replication + +* [Cross-regional async replication] + * [Set up clusters] + * [Integrate with a client app] + * [Remove or recover a cluster] + * [Enable plugins/extensions] + +## Development + +This section is aimed at charm developers looking to support PostgreSQL integrations with their charm. + +* [Integrate with your charm] +* [Migrate data via pg_dump] +* [Migrate data via backup/restore] + + + +[Sunbeam]: /t/15972 +[LXD]: /t/11861 +[MAAS]: /t/14293 +[AWS EC2]: /t/15703 +[GCE]: /t/15722 +[Azure]: /t/15733 +[Multi-AZ]: /t/15749 +[Terraform]: /t/14916 +[Air-gapped]: /t/15746 +[TLS VIP access]: /t/16576 +[Integrate with another application]: /t/9687 +[External access]: /t/15802 +[Scale replicas]: /t/9689 +[Enable TLS]: /t/9685 + +[Configure S3 AWS]: /t/9681 +[Configure S3 RadosGW]: /t/10313 +[Create a backup]: /t/9683 +[Restore a backup]: /t/9693 +[Manage backup retention]: /t/14249 +[Migrate a cluster]: /t/9691 + +[Enable monitoring]: /t/10600 +[Enable alert rules]: /t/13084 +[Enable tracing]: /t/14521 + +[Perform a minor upgrade]: /t/12089 +[Perform a minor rollback]: /t/12090 + +[Cross-regional async replication]: /t/15412 +[Set up clusters]: /t/13991 +[Integrate with a client app]: /t/13992 +[Remove or recover a cluster]: /t/13994 +[Enable plugins/extensions]: /t/10906 + +[Integrate with your charm]: /t/11865 +[Migrate data via pg_dump]: /t/12163 +[Migrate data via backup/restore]: /t/12164 \ No newline at end of file diff --git a/docs/how-to/h-deploy-lxd.md b/docs/how-to/h-deploy-lxd.md deleted file mode 100644 index 02a1b4323c..0000000000 --- a/docs/how-to/h-deploy-lxd.md +++ /dev/null @@ -1,44 +0,0 @@ -# How to deploy on LXD - -This guide assumes you have a running Juju and LXD environment. - -For a detailed walkthrough of setting up an environment and deploying the charm on LXD, refer to the [Tutorial](/t/9707). - -## Prerequisites -* Canonical LXD 5.12+ -* Fulfil the general [system requirements](/t/11743) - ---- - -[Bootstrap](https://juju.is/docs/juju/juju-bootstrap) a juju controller and create a [model](https://juju.is/docs/juju/juju-add-model) if you haven't already: -```shell -juju bootstrap localhost -juju add-model -``` - -Deploy PostgreSQL: -```shell -juju deploy postgresql -``` -> See the [`juju deploy` documentation](https://juju.is/docs/juju/juju-deploy) for all available options at deploy time. -> -> See the [Configurations tab](https://charmhub.io/postgresql/configurations) for specific PostgreSQL parameters. - -Sample output of `juju status --watch 1s`: -```shell -Model Controller Cloud/Region Version SLA Timestamp -postgresql overlord localhost/localhost 2.9.42 unsupported 09:41:53+01:00 - -App Version Status Scale Charm Channel Rev Exposed Message -postgresql active 1 postgresql 14/stable 281 no - -Unit Workload Agent Machine Public address Ports Message -postgresql/0* active idle 0 10.89.49.129 - -Machine State Address Inst id Series AZ Message -0 started 10.89.49.129 juju-a8a31d-0 jammy Running -``` - -[note] -If you expect having several concurrent connections frequently, it is highly recommended to deploy [PgBouncer](https://charmhub.io/pgbouncer?channel=1/stable) alongside PostgreSQL. For more information, read our explanation about [Connection pooling](/t/15777). -[/note] \ No newline at end of file diff --git a/docs/how-to/h-deploy.md b/docs/how-to/h-deploy.md new file mode 100644 index 0000000000..4769d863c6 --- /dev/null +++ b/docs/how-to/h-deploy.md @@ -0,0 +1,83 @@ +# How to deploy + +This page aims to provide an introduction to the PostgreSQL deployment process and lists all the related guides. It contains the following sections: +* [General deployment instructions](#general-deployment-instructions) +* [Clouds](#clouds) +* [Special deployments](#special-deployments) + +--- + +## General deployment instructions + +The basic requirements for deploying a charm are the [**Juju client**](https://juju.is/docs/juju) and a machine [**cloud**](https://juju.is/docs/juju/cloud). + +First, [bootstrap](https://juju.is/docs/juju/juju-bootstrap) the cloud controller and create a [model](https://canonical-juju.readthedocs-hosted.com/en/latest/user/reference/model/): +```shell +juju bootstrap +juju add-model +``` + +Then, either continue with the `juju` client **or** use the `terraform juju` client to deploy the PostgreSQL charm. + +To deploy with the `juju` client: +```shell +juju deploy postgresql +``` +> See also: [`juju deploy` command](https://canonical-juju.readthedocs-hosted.com/en/latest/user/reference/juju-cli/list-of-juju-cli-commands/deploy/) + +To deploy with `terraform juju`, follow the guide [How to deploy using Terraform]. +> See also: [Terraform Provider for Juju documentation](https://canonical-terraform-provider-juju.readthedocs-hosted.com/en/latest/) + +If you are not sure where to start or would like a more guided walkthrough for setting up your environment, see the [Charmed PostgreSQL tutorial][Tutorial]. + +## Clouds + +The guides below go through the steps to install different cloud services and bootstrap them to Juju: +* [Sunbeam] +* [Canonical MAAS] +* [Amazon Web Services EC2] +* [Google Cloud Engine] +* [Azure] + +[How to deploy on multiple availability zones (AZ)] demonstrates how to deploy a cluster on a cloud using different AZs for high availability. + +## Special deployments + +These guides cover some specific deployment scenarios and architectures. + +### External TLS access +[How to deploy for external TLS VIP access] goes over an example deployment of PostgreSQL, PgBouncer and HAcluster that require external TLS/SSL access via [Virtual IP (VIP)](https://en.wikipedia.org/wiki/Virtual_IP_address). + +See also: +* [How to enable TLS] +* [How to connect from outside the local network] + +### Airgapped +[How to deploy in an offline or air-gapped environment] goes over the special configuration steps for installing PostgreSQL in an airgapped environment via CharmHub and the Snap Store Proxy. + +### Cluster-cluster replication +Cluster-cluster, cross-regional, or multi-server asynchronous replication focuses on disaster recovery by distributing data across different servers. + +The [Cross-regional async replication] guide goes through the steps to set up clusters for cluster-cluster replication, integrate with a client, and remove or recover a failed cluster. + +--- + + + +[Tutorial]: /t/9707 + +[How to deploy using Terraform]: /t/14916 + +[Sunbeam]: /t/15972 +[Canonical MAAS]: /t/14293 +[Amazon Web Services EC2]: /t/15703 +[Google Cloud Engine]: /t/15722 +[Azure]: /t/15733 +[How to deploy on multiple availability zones (AZ)]: /t/15749 + +[How to deploy for external TLS VIP access]: /t/16576 +[How to enable TLS]: /t/9685 +[How to connect from outside the local network]: /t/15802 + +[How to deploy in an offline or air-gapped environment]: /t/15746 +[Cross-regional async replication]: /t/15412 \ No newline at end of file diff --git a/docs/how-to/h-upgrade.md b/docs/how-to/h-upgrade.md new file mode 100644 index 0000000000..9a95915cac --- /dev/null +++ b/docs/how-to/h-upgrade.md @@ -0,0 +1,12 @@ +# Upgrade + +Currently, the charm supports PostgreSQL major version 14 only. Therefore, in-place upgrades/rollbacks are not possible for major versions. + +> **Note**: Canonical is not planning to support in-place upgrades for major version change. The new PostgreSQL charm will have to be installed nearby, and the data will be copied from the old to the new installation. After announcing the next PostgreSQL major version support, the appropriate documentation for data migration will be published. + +For instructions on carrying out **minor version upgrades**, see the following guides: + +* [Minor upgrade](/t/12089), e.g. PostgreSQL 14.8 -> PostgreSQL 14.9
+(including charm revision bump 42 -> 43). +* [Minor rollback](/t/12090), e.g. PostgreSQL 14.9 -> PostgreSQL 14.8
+(including charm revision return 43 -> 42). \ No newline at end of file diff --git a/docs/overview.md b/docs/overview.md index 949bf3c098..273a9b0f53 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -41,8 +41,7 @@ PostgreSQL is a trademark or registered trademark of PostgreSQL Global Developme | Level | Path | Navlink | |--------|--------|-------------| -| 1 | tutorial | [Tutorial]() | -| 2 | t-overview | [Overview](/t/9707) | +| 1 | tutorial | [Tutorial](/t/9707) | | 2 | t-set-up | [1. Set up environment](/t/9709) | | 2 | t-deploy | [2. Deploy PostgreSQL](/t/9697) | | 2 | t-access| [3. Access PostgreSQL](/t/15798) | @@ -51,10 +50,9 @@ PostgreSQL is a trademark or registered trademark of PostgreSQL Global Developme | 2 | t-integrate | [6. Integrate with other applications](/t/9701) | | 2 | t-enable-tls | [7. Enable TLS encryption](/t/9699) | | 2 | t-clean-up | [8. Clean up environment](/t/9695) | -| 1 | how-to | [How to]() | -| 2 | h-deploy | [Deploy]() | +| 1 | how-to | [How-to guides](/t/16766) | +| 2 | h-deploy | [Deploy](/t/16811) | | 3 | h-deploy-sunbeam | [Sunbeam](/t/15972) | -| 3 | h-deploy-lxd | [LXD](/t/11861) | | 3 | h-deploy-maas | [MAAS](/t/14293) | | 3 | h-deploy-ec2 | [AWS EC2](/t/15703) | | 3 | h-deploy-gce | [GCE](/t/15722) | @@ -64,9 +62,10 @@ PostgreSQL is a trademark or registered trademark of PostgreSQL Global Developme | 3 | h-deploy-airgapped | [Air-gapped](/t/15746) | | 3 | h-deploy-tls-vip-access | [TLS VIP access](/t/16576) | | 2 | h-integrate | [Integrate with another application](/t/9687) | -| 2 | h-external-access | [External access](/t/15802) | +| 2 | h-external-access | [External network access](/t/15802) | | 2 | h-scale | [Scale replicas](/t/9689) | | 2 | h-enable-tls | [Enable TLS](/t/9685) | +| 2 | h-enable-plugins-extensions | [Enable plugins/extensions](/t/10906) | | 2 | h-backup | [Back up and restore]() | | 3 | h-configure-s3-aws | [Configure S3 AWS](/t/9681) | | 3 | h-configure-s3-radosgw | [Configure S3 RadosGW](/t/10313) | @@ -78,28 +77,19 @@ PostgreSQL is a trademark or registered trademark of PostgreSQL Global Developme | 3 | h-enable-monitoring | [Enable monitoring](/t/10600) | | 3 | h-enable-alert-rules | [Enable alert rules](/t/13084) | | 3 | h-enable-tracing | [Enable tracing](/t/14521) | -| 2 | h-upgrade | [Minor upgrades]() | +| 2 | h-upgrade | [Upgrade](/t/12086) | | 3 | h-upgrade-minor | [Perform a minor upgrade](/t/12089) | | 3 | h-rollback-minor | [Perform a minor rollback](/t/12090) | | 2 | h-async | [Cross-regional async replication](/t/15412) | | 3 | h-async-set-up | [Set up clusters](/t/13991) | | 3 | h-async-integrate | [Integrate with a client app](/t/13992) | | 3 | h-async-remove-recover | [Remove or recover a cluster](/t/13994) | -| 2 | h-enable-plugins-extensions | [Enable plugins/extensions](/t/10906) | | 2 | h-development| [Development]() | | 3 | h-development-integrate | [Integrate with your charm](/t/11865) | | 3 | h-migrate-pgdump | [Migrate data via pg_dump](/t/12163) | | 3 | h-migrate-backup-restore | [Migrate data via backup/restore](/t/12164) | -| 1 | reference | [Reference]() | -| 2 | r-overview | [Overview](/t/13976) | -| 2 | r-releases | [Release Notes](/t/11875) | -| 3 | r-revision-544-545| [Revision 544/545](/t/16007) | -| 3 | r-revision-467-468 | [Revision 467/468](/t/15378) | -| 3 | r-revision-429-430 | [Revision 429/430](/t/14067) | -| 3 | r-revision-363 | [Revision 363](/t/13124) | -| 3 | r-revision-351 | [Revision 351](/t/12823) | -| 3 | r-revision-336 | [Revision 336](/t/11877) | -| 3 | r-revision-288 | [Revision 288](/t/11876) | +| 1 | reference | [Reference](/t/13976) | +| 2 | r-releases | [Releases](/t/11875) | | 2 | r-system-requirements | [System requirements](/t/11743) | | 2 | r-software-testing | [Software testing](/t/11773) | | 2 | r-performance | [Performance and resources](/t/11974) | @@ -108,20 +98,31 @@ PostgreSQL is a trademark or registered trademark of PostgreSQL Global Developme | 2 | r-alert-rules | [Alert rules](/t/15841) | | 2 | r-statuses | [Statuses](/t/10844) | | 2 | r-contacts | [Contacts](/t/11863) | -| 1 | explanation | [Explanation]() | +| 1 | explanation | [Explanation](/t/16768) | | 2 | e-architecture | [Architecture](/t/11857) | +| 2 | e-security | [Security](/t/16852) | +| 2 | e-cryptography | [Cryptography](/t/16853) | | 2 | e-interfaces-endpoints | [Interfaces and endpoints](/t/10251) | +| 2 | e-connection-pooling| [Connection pooling](/t/15777) | | 2 | e-users | [Users](/t/10798) | | 2 | e-logs | [Logs](/t/12099) | | 2 | e-juju-details | [Juju](/t/11985) | | 2 | e-legacy-charm | [Legacy charm](/t/10690) | -| 2 | e-connection-pooling| [Connection pooling](/t/15777) | | 1 | search | [Search](https://canonical.com/data/docs/postgresql/iaas) | [/details] - \ No newline at end of file diff --git a/docs/reference/r-overview.md b/docs/reference.md similarity index 82% rename from docs/reference/r-overview.md rename to docs/reference.md index 8392dce767..f686a8f86b 100644 --- a/docs/reference/r-overview.md +++ b/docs/reference.md @@ -1,4 +1,4 @@ -# Overview +# Reference The Reference section of our documentation contains pages for technical specifications, APIs, release notes, and other reference material for fast lookup. @@ -7,14 +7,16 @@ The Reference section of our documentation contains pages for technical specific |---------------------------|---------------------------------------------------| | [Release Notes](/t/11875) | Release notes for major revisions of this charm | | [System requirements](/t/11743) | Software and hardware requirements | -| [Testing](/t/11773) | Software tests (e.g. smoke, unit, performance...) | +| [Software testing](/t/11773) | Software tests (e.g. smoke, unit, performance...) | +| [Performance and resources](/t/11974) | Config profiles related to performance | | [Troubleshooting](/t/11864) | Troubleshooting tips and tricks | -| [Profiles](/t/11974) | Config profiles related to performance | | [Plugins/extensions](/t/10946) | Plugins/extensions supported by each charm revision | +| [Alert rules](/t/15841) | Pre-configured Prometheus alert rules | +| [Charm statuses](/t/10844) | Juju application statuses | | [Contacts](/t/11863) | Contact information | -**In the tabs at the top of the page**, you can find the following automatically generated API references: +**In the tabs at the top of the page**, you will find the following automatically generated API references: | Page | Description | |----------------------------------------------------------------------------|---------------------------------------------------------| diff --git a/docs/reference/r-releases.md b/docs/reference/r-releases.md index 5c4b104362..f67d6fc623 100644 --- a/docs/reference/r-releases.md +++ b/docs/reference/r-releases.md @@ -1,4 +1,4 @@ -# Release Notes +# Releases This page provides high-level overviews of the dependencies and features that are supported by each revision in every stable release. @@ -9,14 +9,14 @@ To see all releases and commits, check the [Charmed PostgreSQL Releases page on ## Dependencies and supported features For a given release, this table shows: -* The PostgreSQL version packaged inside +* The PostgreSQL version packaged inside. * The minimum Juju 3 version required to reliably operate **all** features of the release > This charm still supports older versions of Juju down to 2.9. See the [Juju section of the system requirements](/t/11743) for more details. * Support for specific features | Release | PostgreSQL version | Juju 3 version | [TLS encryption](/t/9685)* | [COS monitoring](/t/10600) | [Minor version upgrades](/t/12089) | [Cross-regional async replication](/t/15412) | [Point-in-time recovery](/t/9693) | |:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [544], [545] (14/candidate) | 14.15 | `3.6.1+` | ![check] | ![check] | ![check] | ![check] | ![check] | +| [552], [553] | 14.15 | `3.6.1+` | ![check] | ![check] | ![check] | ![check] | ![check] | | [467], [468] | 14.12 | `3.4.3+` | ![check] | ![check] | ![check] | ![check] | ![check] | | [429], [430] | 14.11 | `3.4.2+` | ![check] | ![check] | ![check] | ![check] | | | [363] | 14.10 | `3.4.2+` | ![check] | ![check] | ![check] | ![check] | | @@ -36,23 +36,22 @@ Several [revisions](https://juju.is/docs/sdk/revision) are released simultaneous > If you deploy a specific revision, **you must make sure it matches your base and architecture** via the tables below or with [`juju info`](https://juju.is/docs/juju/juju-info) - +|[553] | ![check] | | ![check] | +|[552] | | ![check] | ![check] | -### Release 467-468 (`14/stable`) +[details=Older releases] + +### Release 467-468 | Revision | amd64 | arm64 | Ubuntu 22.04 LTS |:--------:|:-----:|:-----:|:-----:| |[468] |![check] | | ![check] | |[467] | | ![check]| ![check] | -[details=Older releases] ### Release 429-430 | Revision | amd64 | arm64 | Ubuntu 22.04 LTS @@ -93,21 +92,23 @@ Several [revisions](https://juju.is/docs/sdk/revision) are released simultaneous For a list of all plugins supported for each revision, see the reference page [Plugins/extensions](/t/10946). -[note] - Our release notes are an ongoing work in progress. If there is any additional information about releases that you would like to see or suggestions for other improvements, don't hesitate to contact us on [Matrix ](https://matrix.to/#/#charmhub-data-platform:ubuntu.com) or [leave a comment](https://discourse.charmhub.io/t/charmed-postgresql-reference-release-notes/11875). -[/note] +> **Note**: Our release notes are an ongoing work in progress. If there is any additional information about releases that you would like to see or suggestions for other improvements, don't hesitate to contact us on [Matrix ](https://matrix.to/#/#charmhub-data-platform:ubuntu.com) or [leave a comment](https://discourse.charmhub.io/t/charmed-postgresql-reference-release-notes/11875). -[545]: /t/16007 -[544]: /t/16007 -[468]: /t/15378 -[467]: /t/15378 -[430]: /t/14067 -[429]: /t/14067 -[363]: /t/13124 -[351]: /t/12823 -[336]: /t/11877 -[288]: /t/11876 +[553]: https://github.com/canonical/postgresql-operator/releases/tag/rev552 +[552]: https://github.com/canonical/postgresql-operator/releases/tag/rev552 + +[468]: https://github.com/canonical/postgresql-operator/releases/tag/rev467 +[467]: https://github.com/canonical/postgresql-operator/releases/tag/rev467 + +[430]: https://github.com/canonical/postgresql-operator/releases/tag/rev429 +[429]: https://github.com/canonical/postgresql-operator/releases/tag/rev429 + +[363]: https://github.com/canonical/postgresql-operator/releases/tag/rev363 +[351]: https://github.com/canonical/postgresql-operator/releases/tag/rev351 +[336]: https://github.com/canonical/postgresql-operator/releases/tag/rev336 +[288]: https://github.com/canonical/postgresql-operator/releases/tag/rev288 + [check]: https://img.icons8.com/color/20/checkmark--v1.png \ No newline at end of file diff --git a/docs/reference/r-revision-288.md b/docs/reference/r-revision-288.md deleted file mode 100644 index 3297b52ac6..0000000000 --- a/docs/reference/r-revision-288.md +++ /dev/null @@ -1,50 +0,0 @@ ->Reference > Release Notes > [All revisions](/t/11875) > [Revision 288](/t/11876) -# Revision 288 -Thursday, April 20, 2023 - -Dear community, - -We'd like to announce that Canonical's newest Charmed PostgreSQL operator for IAAS/VM has been published in the `14/stable` [channel](https://charmhub.io/postgresql?channel=14/stable). :tada: - -## Features you can start using today -* Deploying on VM (tested with LXD, MAAS) -* Scaling up/down in one simple juju command -* HA using [Patroni](https://github.com/zalando/patroni) -* Full backups and restores are supported when using any S3-compatible storage -* TLS support (using “[tls-certificates](https://charmhub.io/tls-certificates-operator)” operator) -* DB access outside of Juju using “[data-integrator](https://charmhub.io/data-integrator)” -* Data import using standard tools e.g. “psql”. -* Documentation: - - -## Inside the charms: - -* Charmed PostgreSQL charm ships the latest PostgreSQL “14.7-0ubuntu0.22.04.1” -* VM charms [based on our](https://snapcraft.io/publisher/dataplatformbot) SNAP (Ubuntu LTS “22.04” - core22-based) -* Principal charms supports the latest LTS series “22.04” only. -* Subordinate charms support LTS “22.04” and “20.04” only. - -## Technical notes - - * The new PostgreSQL charm is also a juju interface-compatible replacement for legacy PostgreSQL charms (using legacy interface `pgsql`, via endpoints `db` and `db-admin`). -However, **it is highly recommended to migrate to the modern interface [`postgresql_client`](https://github.com/canonical/charm-relation-interfaces)** (endpoint `database`). - * Please [contact us](#heading--contact) if you are considering migrating from other “legacy” charms not mentioned above. -* Charmed PostgreSQL follows SNAP track “14”. -* No “latest” track in use (no surprises in tracking “latest/stable”)! - * PostgreSQL charm provide [legacy charm](/t/10690) through “latest/stable”. -* You can find charm lifecycle flowchart diagrams [here](https://github.com/canonical/postgresql-k8s-operator/tree/main/docs/reference). -* Modern interfaces are well described in the [Interfaces catalogue](https://github.com/canonical/charm-relation-interfaces) and implemented by [`data-platform-libs`](https://github.com/canonical/data-platform-libs/). -* Known limitation: PostgreSQL extensions are not yet supported. - -

Contact us

-Charmed PostgreSQL is an open source project that warmly welcomes community contributions, suggestions, fixes, and constructive feedback. - -* Raise software issues or feature requests on [**GitHub**](https://github.com/canonical/postgresql-operator/issues/new/choose) -* Report security issues through [**Launchpad**](https://wiki.ubuntu.com/DebuggingSecurity#How%20to%20File) -* Contact the Canonical Data Platform team through our [Matrix](https://matrix.to/#/#charmhub-data-platform:ubuntu.com) channel! - - \ No newline at end of file diff --git a/docs/reference/r-revision-336.md b/docs/reference/r-revision-336.md deleted file mode 100644 index 4c3d98fa3a..0000000000 --- a/docs/reference/r-revision-336.md +++ /dev/null @@ -1,81 +0,0 @@ ->Reference > Release Notes > [All revisions](/t/11875) > [Revision 336](/t/11877) -# Revision 336 -Wednesday, October 18, 2023 - -Dear community, - -We'd like to announce that Canonical's newest Charmed PostgreSQL operator for IAAS/VM has been published in the `14/stable` [channel](https://charmhub.io/postgresql?channel=14/stable). :tada: - -If you are jumping over several stable revisions, make sure to check [previous release notes](/t/11875) before upgrading to this revision. - -## Features you can start using today -* [Add Juju 3 support](/t/11743) (Juju 2 is still supported) [[DPE-1758](https://warthogs.atlassian.net/browse/DPE-1758)] -* All secrets are now stored in [Juju secrets](https://juju.is/docs/juju/manage-secrets) [[DPE-1758](https://warthogs.atlassian.net/browse/DPE-1758)] -* Charm [minor upgrades](/t/12089) and [minor rollbacks](/t/12090) [[DPE-1767](https://warthogs.atlassian.net/browse/DPE-1767)] -* [Canonical Observability Stack (COS)](https://charmhub.io/topics/canonical-observability-stack) support [[DPE-1775](https://warthogs.atlassian.net/browse/DPE-1775)] -* [PostgreSQL plugins support](/t/10906) [[DPE-1373](https://warthogs.atlassian.net/browse/DPE-1373)] -* [Profiles configuration](/t/11974) support [[DPE-2655](https://warthogs.atlassian.net/browse/DPE-2655)] -* [Logs rotation](/t/12099) [[DPE-1754](https://warthogs.atlassian.net/browse/DPE-1754)] -* Workload updated to [PostgreSQL 14.9](https://www.postgresql.org/docs/14/release-14-9.html) [[PR#18](https://github.com/canonical/charmed-postgresql-snap/pull/18)] -* Add '`admin`' [extra user role](https://github.com/canonical/postgresql-operator/pull/199) [[DPE-2167](https://warthogs.atlassian.net/browse/DPE-2167)] -* New charm '[PostgreSQL Test App](https://charmhub.io/postgresql-test-app)' -* New documentation: - * [Architecture (HLD/LLD)](/t/11857) - * [Upgrade section](/t/12086) - * [Release Notes](/t/11875) - * [Requirements](/t/11743) - * [Profiles](/t/11974) - * [Users](/t/10798) - * [Logs](/t/12099) - * [Statuses](/t/10844) - * [Development](/t/11862) - * [Testing reference](/t/11773) - * [Legacy charm](/t/10690) - * [Plugins/extensions](/t/10906), [supported](/t/10946) - * [Juju 2.x vs 3.x hints](/t/11985) - * [Contacts](/t/11863) -* All the functionality from [the previous revisions](/t/11875) - -## Bugfixes -* [DPE-1624](https://warthogs.atlassian.net/browse/DPE-1624), [DPE-1625](https://warthogs.atlassian.net/browse/DPE-1625) Backup/restore fixes -* [DPE-1926](https://warthogs.atlassian.net/browse/DPE-1926) Remove fallback_application_name field from relation data -* [DPE-1712](https://warthogs.atlassian.net/browse/DPE-1712) Enabled the user to fix network issues and rerun stanza related hooks -* [DPE-2173](https://warthogs.atlassian.net/browse/DPE-2173) Fix allowed units relation data field -* [DPE-2127](https://warthogs.atlassian.net/browse/DPE-2127) Fixed databases access -* [DPE-2341](https://warthogs.atlassian.net/browse/DPE-2341) Populate extensions in unit databag -* [DPE-2218](https://warthogs.atlassian.net/browse/DPE-2218) Update charm libs to get s3 relation fix -* [DPE-1210](https://warthogs.atlassian.net/browse/DPE-1210), [DPE-2330](https://warthogs.atlassian.net/browse/DPE-2330), [DPE-2212](https://warthogs.atlassian.net/browse/DPE-2212) Open port (ability to expose charm) -* [DPE-2614](https://warthogs.atlassian.net/browse/DPE-2614) Split stanza create and stanza check -* [DPE-2721](https://warthogs.atlassian.net/browse/DPE-2721) Allow network access for pg_dump, pg_dumpall and pg_restore -* [DPE-2717](https://warthogs.atlassian.net/browse/DPE-2717) Copy dashboard changes from K8s and use the correct topology dispatcher -* [MISC] Copy fixes of DPE-2626 and DPE-2627 from k8s -* [MISC] Don't fail if the unit is already missing -* [MISC] More resilient topology observer - -Canonical Data issues are now public on both [Jira](https://warthogs.atlassian.net/jira/software/c/projects/DPE/issues/) and [GitHub](https://github.com/canonical/postgresql-operator/issues) platforms. - -[GitHub Releases](https://github.com/canonical/postgresql-operator/releases) provide a detailed list of bugfixes, PRs, and commits for each revision. - -## Inside the charms - -* Charmed PostgreSQL ships the latest PostgreSQL “14.9-0ubuntu0.22.04.1” -* PostgreSQL cluster manager Patroni updated to "3.0.2" -* Backup tools pgBackRest updated to "2.47" -* The Prometheus postgres-exporter is "0.12.1-0ubuntu0.22.04.1~ppa1" -* VM charms based on [Charmed PostgreSQL](https://snapcraft.io/charmed-postgresql) SNAP (Ubuntu LTS “22.04” - ubuntu:22.04-based) -* Principal charms supports the latest LTS series “22.04” only. -* Subordinate charms support LTS “22.04” and “20.04” only. - -## Technical notes - -* `juju refresh` from the old-stable revision 288 to the current-revision 324 is **NOT** supported!!!
The [upgrade](/t/12086) functionality is new and supported for revision 324+ only! -* Please check additionally [the previously posted restrictions](/t/11876). -* Ensure [the charm requirements](/t/11743) met - -## Contact us - -Charmed PostgreSQL is an open source project that warmly welcomes community contributions, suggestions, fixes, and constructive feedback. - -* Raise software issues or feature requests on [**GitHub**](https://github.com/canonical/postgresql-operator/issues/new/choose) -* Report security issues through [**Launchpad**](https://wiki.ubuntu.com/DebuggingSecurity#How%20to%20File) -* Contact the Canonical Data Platform team through our [Matrix](https://matrix.to/#/#charmhub-data-platform:ubuntu.com) channel. \ No newline at end of file diff --git a/docs/reference/r-revision-351.md b/docs/reference/r-revision-351.md deleted file mode 100644 index 1e3648b1ba..0000000000 --- a/docs/reference/r-revision-351.md +++ /dev/null @@ -1,57 +0,0 @@ ->Reference > Release Notes > [All revisions](/t/11875) > [Revision 351](/t/12823) -# Revision 351 -January 3, 2024 - -Dear community, - -We'd like to announce that Canonical's newest Charmed PostgreSQL operator for IAAS/VM has been published in the `14/stable` [channel](https://charmhub.io/postgresql?channel=14/stable). - -If you are jumping over several stable revisions, make sure to check [previous release notes](/t/11875) before upgrading to this revision. - -## Features you can start using today - -* [Core] Updated `Charmed PostgreSQL` SNAP image ([PR#291](https://github.com/canonical/postgresql-operator/pull/291))([DPE-3039](https://warthogs.atlassian.net/browse/DPE-3039)): - * `Patroni` updated from 3.0.2 to 3.1.2 - * `Pgbackrest` updated from 2.47 to 2.48 -* [Plugins] [Add 24 new plugins/extension](https://charmhub.io/postgresql/docs/r-plugins-extensions) in ([PR#251](https://github.com/canonical/postgresql-operator/pull/251)) -* [Plugins] **NOTE**: extension `plpython3u` is deprecated and will be removed from [list of supported plugins](/t/10946) soon! -* [Config] [Add 29 new configuration options](https://charmhub.io/postgresql/configure) in ([PR#239](https://github.com/canonical/postgresql-operator/pull/239))([DPE-1781](https://warthogs.atlassian.net/browse/DPE-1781)) -* [Config] **NOTE:** the config option `profile-limit-memory` is deprecated. Use `profile_limit_memory` (to follow the [naming conventions](https://juju.is/docs/sdk/naming))! ([PR#306](https://github.com/canonical/postgresql-operator/pull/306))([DPE-3096](https://warthogs.atlassian.net/browse/DPE-3096)) -* [Charm] Add Juju Secret labels in ([PR#270](https://github.com/canonical/postgresql-operator/pull/270))([DPE-2838](https://warthogs.atlassian.net/browse/DPE-2838)) -* [Charm] Update Python dependencies in ([PR#293](https://github.com/canonical/postgresql-operator/pull/293)) -* [DB] Add handling of tables ownership in ([PR#298](https://github.com/canonical/postgresql-operator/pull/298))([DPE-2740](https://warthogs.atlassian.net/browse/DPE-2740)) -* ([COS](https://charmhub.io/topics/canonical-observability-stack)) Moved Grafana dashboard legends to the bottom of the graph in ([PR#295](https://github.com/canonical/postgresql-operator/pull/295))([DPE-2622](https://warthogs.atlassian.net/browse/DPE-2622)) -* ([COS](https://charmhub.io/topics/canonical-observability-stack)) Add Patroni COS support ([#261](https://github.com/canonical/postgresql-operator/pull/261))([DPE-1993](https://warthogs.atlassian.net/browse/DPE-1993)) -* [CI/CD] Charm migrated to GitHub Data reusable workflow in ([PR#263](https://github.com/canonical/postgresql-operator/pull/263))([DPE-2789](https://warthogs.atlassian.net/browse/DPE-2789)) -* All the functionality from [the previous revisions](/t/11875) - -## Bugfixes - -* Fixed enabling extensions when new database is created ([PR#252](https://github.com/canonical/postgresql-operator/pull/252))([DPE-2569](https://warthogs.atlassian.net/browse/DPE-2569)) -* Block the charm if the legacy interface requests [roles](https://discourse.charmhub.io/t/charmed-postgresql-explanations-interfaces-endpoints/10251) ([DPE-3077](https://warthogs.atlassian.net/browse/DPE-3077)) - -Canonical Data issues are now public on both [Jira](https://warthogs.atlassian.net/jira/software/c/projects/DPE/issues/) and [GitHub](https://github.com/canonical/postgresql-operator/issues) platforms. -[GitHub Releases](https://github.com/canonical/postgresql-operator/releases) provide a detailed list of bugfixes, PRs, and commits for each revision. -## Inside the charms - -* Charmed PostgreSQL ships the latest PostgreSQL “14.9-0ubuntu0.22.04.1” -* PostgreSQL cluster manager Patroni updated to "3.2.1" -* Backup tools pgBackRest updated to "2.48" -* The Prometheus postgres-exporter is "0.12.1-0ubuntu0.22.04.1~ppa1" -* VM charms based on [Charmed PostgreSQL](https://snapcraft.io/charmed-postgresql) SNAP (Ubuntu LTS “22.04” - ubuntu:22.04-based) revision 89 -* Principal charms supports the latest LTS series “22.04” only -* Subordinate charms support LTS “22.04” and “20.04” only - -## Technical notes - -* Upgrade (`juju refresh`) is possible from this revision 336+ -* Use this operator together with a modern operator "[pgBouncer](https://charmhub.io/pgbouncer?channel=1/stable)" -* Please check additionally [the previously posted restrictions](/t/11875) -* Ensure [the charm requirements](/t/11743) met - -## Contact us - -Charmed PostgreSQL is an open source project that warmly welcomes community contributions, suggestions, fixes, and constructive feedback. -* Raise software issues or feature requests on [**GitHub**](https://github.com/canonical/postgresql-operator/issues/new/choose) -* Report security issues through [**Launchpad**](https://wiki.ubuntu.com/DebuggingSecurity#How%20to%20File) -* Contact the Canonical Data Platform team through our [Matrix](https://matrix.to/#/#charmhub-data-platform:ubuntu.com) channel. \ No newline at end of file diff --git a/docs/reference/r-revision-363.md b/docs/reference/r-revision-363.md deleted file mode 100644 index e3614ea343..0000000000 --- a/docs/reference/r-revision-363.md +++ /dev/null @@ -1,55 +0,0 @@ ->Reference > Release Notes > [All revisions](/t/11875) > [Revision 363](/t/13124) -# Revision 363 -February 21, 2024 - -Dear community, - -We'd like to announce that Canonical's newest Charmed PostgreSQL operator for IAAS/VM has been published in the `14/stable` [channel](https://charmhub.io/postgresql?channel=14/stable) :tada: - -If you are jumping over several stable revisions, make sure to check [previous release notes](/t/11875) before upgrading to this revision. - -## Features you can start using today -* [CORE] PostgreSQL upgrade 14.9 -> 14.10. ([DPE-3217](https://warthogs.atlassian.net/browse/DPE-3217)) - * **Note**: It is advisable to REINDEX potentially-affected indexes after installing this update! (See [PostgreSQL changelog](https://changelogs.ubuntu.com/changelogs/pool/main/p/postgresql-14/postgresql-14_14.10-0ubuntu0.22.04.1/changelog)) -* [CORE] Juju 3.1.7+ support ([#2037120](https://bugs.launchpad.net/juju/+bug/2037120)) -* [PLUGINS] pgVector extension/plugin ([DPE-3159](https://warthogs.atlassian.net/browse/DPE-3159)) -* [PLUGINS] New PostGIS plugin ([#312](https://github.com/canonical/postgresql-operator/pull/312)) -* [PLUGINS] More new plugins - [50 in total](/t/10946) -* [MONITORING] COS Awesome Alert rules ([DPE-3160](https://warthogs.atlassian.net/browse/DPE-3160)) -* [SECURITY] Updated TLS libraries for compatibility with new charms - * [manual-tls-certificates](https://charmhub.io/manual-tls-certificates) - * [self-signed-certificates](https://charmhub.io/self-signed-certificates) - * Any charms compatible with [ tls_certificates_interface.v2.tls_certificates](https://charmhub.io/tls-certificates-interface/libraries/tls_certificates) -* All functionality from [previous revisions](/t/11875) - -## Bugfixes - -* [DPE-3199](https://warthogs.atlassian.net/browse/DPE-3199) Stabilized internal Juju secrets management -* [DPE-3258](https://warthogs.atlassian.net/browse/DPE-3258) Check system identifier in stanza (backups setup stabilization) - -Canonical Data issues are now public on both [Jira](https://warthogs.atlassian.net/jira/software/c/projects/DPE/issues/) and [GitHub](https://github.com/canonical/postgresql-operator/issues) platforms. -[GitHub Releases](https://github.com/canonical/postgresql-operator/releases) provide a detailed list of bugfixes, PRs, and commits for each revision. - -## What is inside the charms - -* Charmed PostgreSQL ships the latest PostgreSQL `14.10-0ubuntu0.22.04.1` -* PostgreSQL cluster manager Patroni updated to `v.3.1.2` -* Backup tools pgBackRest updated to `v.2.48` -* The Prometheus postgres-exporter is `0.12.1-0ubuntu0.22.04.1~ppa1` -* VM charms based on [Charmed PostgreSQL](https://snapcraft.io/charmed-postgresql) SNAP (Ubuntu LTS 22.04 - `ubuntu:22.04-based`) revision 96 -* Principal charms supports the latest LTS series 22.04 only -* Subordinate charms support LTS 22.04 and 20.04 only - -## Technical notes - -* Starting with this revision (336+), you can use `juju refresh` to upgrade Charmed PostgreSQL -* Use this operator together with modern [Charmed PgBouncer operator](https://charmhub.io/pgbouncer?channel=1/stable) -* Please check [the previously posted restrictions](/t/11875) -* Ensure [the charm requirements](/t/11743) met - -## Contact us - -Charmed PostgreSQL is an open source project that warmly welcomes community contributions, suggestions, fixes, and constructive feedback. -* Raise software issues or feature requests on [**GitHub**](https://github.com/canonical/postgresql-operator/issues/new/choose) -* Report security issues through [**Launchpad**](https://wiki.ubuntu.com/DebuggingSecurity#How%20to%20File) -* Contact the Canonical Data Platform team through our [Matrix](https://matrix.to/#/#charmhub-data-platform:ubuntu.com) channel. \ No newline at end of file diff --git a/docs/reference/r-revision-429-430.md b/docs/reference/r-revision-429-430.md deleted file mode 100644 index a1efd30106..0000000000 --- a/docs/reference/r-revision-429-430.md +++ /dev/null @@ -1,98 +0,0 @@ ->Reference > Release Notes > [All revisions](t/11875) > Revision 429/430 - -# Revision 429/430 - -June 28, 2024 - -Dear community, - -Canonical's newest Charmed PostgreSQL operator has been published in the 14/stable [channel](https://charmhub.io/postgresql?channel=14/stable) :tada: - -Due to the newly added support for `arm64` architecture, the PostgreSQL charm now releases two revisions simultaneously: -* Revision 429 is built for `amd64` -* Revision 430 is built for for `arm64` - -To make sure you deploy for the right architecture, we recommend setting an [architecture constraint](https://juju.is/docs/juju/constraint#heading--arch) for your entire juju model. - -Otherwise, it can be done at deploy time with the `--constraints` flag: -```shell -juju deploy postgresql --constraints arch= -``` -where `` can be `amd64` or `arm64`. - ---- - -## Highlights -Below are the major highlights of this release. To see all changes since the previous stable release, check the [release notes on GitHub](https://github.com/canonical/postgresql-operator/releases/tag/rev430). - -* Upgraded PostgreSQL from v.14.10 → v.14.11 ([PR #432](https://github.com/canonical/postgresql-operator/pull/432)) - * Check the official [PostgreSQL release notes](https://www.postgresql.org/docs/release/14.11/) -* Added support for ARM64 architecture ([PR #381](https://github.com/canonical/postgresql-operator/pull/381)) -* Added support for cross-regional asynchronous replication ([PR #452](https://github.com/canonical/postgresql-operator/pull/452)) ([DPE-2953](https://warthogs.atlassian.net/browse/DPE-2953)) - * This feature focuses on disaster recovery by distributing data across different servers. Check our [new how-to guides](https://charmhub.io/postgresql/docs/h-async-set-up) for a walkthrough of the cross-model setup, promotion, switchover, and other details. -* Added support for tracing with Tempo K8s ([PR #485](https://github.com/canonical/postgresql-operator/pull/485)) ([DPE-4616](https://warthogs.atlassian.net/browse/DPE-4616)) - * Check our new guide: [How to enable tracing](https://charmhub.io/postgresql/docs/h-enable-tracing) -* Released new [Charmed Sysbench operator](https://charmhub.io/sysbench) for easy performance testing - -### Enhancements -* Added timescaledb plugin/extension ([PR#470](https://github.com/canonical/postgresql-operator/pull/470)) - * See the [Configuration tab]((https://charmhub.io/postgresql/configuration?channel=14/candidate#plugin_timescaledb_enable)) for a full list of supported plugins/extensions -* Added incremental and differential backup support ([PR #479](https://github.com/canonical/postgresql-operator/pull/479)) ([DPE-4462](https://warthogs.atlassian.net/browse/DPE-4462)) - * Check our guide: [How to create and list backups](https://charmhub.io/postgresql/docs/h-create-backup) -* Added support for disabling the operator ([PR#412](https://github.com/canonical/postgresql-operator/pull/412)) ([DPE-2469](https://warthogs.atlassian.net/browse/DPE-2469)) -* Added support for subordination with: - * `ubuntu-advantage` ([PR#397](https://github.com/canonical/postgresql-operator/pull/397)) ([DPE-3644](https://warthogs.atlassian.net/browse/DPE-3644)) - * `landscape-client` ([PR#388](https://github.com/canonical/postgresql-operator/pull/388)) ([DPE-3644](https://warthogs.atlassian.net/browse/DPE-3644)) -* Added configuration option for backup retention time ([PR#474](https://github.com/canonical/postgresql-operator/pull/474))([DPE-4401](https://warthogs.atlassian.net/browse/DPE-4401)) -* Added `experimental_max_connections` config option ([PR#472](https://github.com/canonical/postgresql-operator/pull/472)) -* Added check for replicas encrypted connection ([PR#437](https://github.com/canonical/postgresql-operator/pull/437)) - -### Bugfixes -* Fixed slow charm bootstrap time ([PR#413](https://github.com/canonical/postgresql-operator/pull/413)) -* Fixed large objects ownership ([PR#349](https://github.com/canonical/postgresql-operator/pull/349)) -* Fixed secrets crash for "certificates-relation-changed" after the refresh ([PR#475](https://github.com/canonical/postgresql-operator/pull/475)) -* Fixed network cut tests ([PR#346](https://github.com/canonical/postgresql-operator/pull/346)) ([DPE-3257](https://warthogs.atlassian.net/browse/DPE-3257)) - -Canonical Data issues are now public on both [Jira](https://warthogs.atlassian.net/jira/software/c/projects/DPE/issues/) and [GitHub](https://github.com/canonical/postgresql-operator/issues). - -For a full list of all changes in this revision, see the [GitHub Release](https://github.com/canonical/postgresql-operator/releases/tag/rev430). - -## Technical details -This section contains some technical details about the charm's contents and dependencies. Make sure to also check the [system requirements](/t/11743). - -If you are jumping over several stable revisions, check [previous release notes](/t/11875) before upgrading. - -### Packaging -This charm is based on the [`charmed-postgresql` snap](https://snapcraft.io/charmed-postgresql) (pinned revision 113). It packages: -* postgresql `v.14.11` - * [`14.11-0ubuntu0.22.04.1`](https://launchpad.net/ubuntu/+source/postgresql-14/14.11-0ubuntu0.22.04.1) -* pgbouncer `v.1.21` - * [`1.21.0-0ubuntu0.22.04.1~ppa1`](https://launchpad.net/~data-platform/+archive/ubuntu/pgbouncer) -* patroni `v.3.1.2 ` - * [`3.1.2-0ubuntu0.22.04.1~ppa2`](https://launchpad.net/~data-platform/+archive/ubuntu/patroni) -* pgBackRest `v.2.48` - * [`2.48-0ubuntu0.22.04.1~ppa1`](https://launchpad.net/~data-platform/+archive/ubuntu/pgbackrest) -* prometheus-postgres-exporter `v.0.12.1` - -### Libraries and interfaces -This charm revision imports the following libraries: - -* **grafana_agent `v0`** for integration with Grafana - * Implements `cos_agent` interface -* **rolling_ops `v0`** for rolling operations across units - * Implements `rolling_op` interface -* **tempo_k8s `v1`, `v2`** for integration with Tempo charm - * Implements `tracing` interface -* **tls_certificates_interface `v2`** for integration with TLS charms - * Implements `tls-certificates` interface - -See the [`/lib/charms` directory on GitHub](https://github.com/canonical/postgresql-operator/tree/main/lib/charms) for more details about all supported libraries. - -See the [`metadata.yaml` file on GitHub](https://github.com/canonical/postgresql-operator/blob/main/metadata.yaml) for a full list of supported interfaces - -## Contact us - -Charmed PostgreSQL is an open source project that warmly welcomes community contributions, suggestions, fixes, and constructive feedback. -* Raise software issues or feature requests on [**GitHub**](https://github.com/canonical/postgresql-operator/issues) -* Report security issues through [**Launchpad**](https://wiki.ubuntu.com/DebuggingSecurity#How%20to%20File) -* Contact the Canonical Data Platform team through our [Matrix](https://matrix.to/#/#charmhub-data-platform:ubuntu.com) channel. \ No newline at end of file diff --git a/docs/reference/r-revision-467-468.md b/docs/reference/r-revision-467-468.md deleted file mode 100644 index 5f5e73d693..0000000000 --- a/docs/reference/r-revision-467-468.md +++ /dev/null @@ -1,155 +0,0 @@ ->Reference > Release Notes > [All revisions] > Revision 467/468 - -# Revision 467/468 -September 11, 2024 - -Canonical's newest Charmed PostgreSQL operator has been published in the [14/stable channel]. - -Due to the newly added support for `arm64` architecture, the PostgreSQL charm now releases multiple revisions simultaneously: -* Revision 468 is built for `amd64` on Ubuntu 22.04 LTS -* Revision 467 is built for `arm64` on Ubuntu 22.04 LTS - -To make sure you deploy for the right architecture, we recommend setting an [architecture constraint](https://juju.is/docs/juju/constraint#heading--arch) for your entire juju model. - -Otherwise, it can be done at deploy time with the `--constraints` flag: -```shell -juju deploy postgresql --constraints arch= -``` -where `` can be `amd64` or `arm64`. - ---- - -## Highlights -* Upgraded PostgreSQL from v.14.11 → v.14.12 ([PR #530](https://github.com/canonical/postgresql-operator/pull/530)) - * Check the official [PostgreSQL release notes](https://www.postgresql.org/docs/release/14.12/) -* Added support for Point In Time Recovery ([PR #391](https://github.com/canonical/postgresql-operator/pull/391)) ([DPE-2582](https://warthogs.atlassian.net/browse/DPE-2582)) -* Secure Syncobj and Patroni with passwords ([PR #596](https://github.com/canonical/postgresql-operator/pull/596)) ([DPE-5269](https://warthogs.atlassian.net/browse/DPE-5269)) -* Removed deprecated config option `profile-limit-memory` ([PR #564](https://github.com/canonical/postgresql-operator/pull/564)) ([DPE-4889](https://warthogs.atlassian.net/browse/DPE-4889)) - -## Features - -* Added URI connection string to relations ([PR #527](https://github.com/canonical/postgresql-operator/pull/527)) ([DPE-2278](https://warthogs.atlassian.net/browse/DPE-2278)) -* Improve `list-backups` action output ([PR #522](https://github.com/canonical/postgresql-operator/pull/522)) ([DPE-4479](https://warthogs.atlassian.net/browse/DPE-4479)) -* Show start/end time in UTC time in list-backups output ([PR #551](https://github.com/canonical/postgresql-operator/pull/551)) -* Switched to constant snap locales ([PR #559](https://github.com/canonical/postgresql-operator/pull/559)) ([DPE-4198](https://warthogs.atlassian.net/browse/DPE-4198)) -* Moved URI generation to update endpoints ([PR #584](https://github.com/canonical/postgresql-operator/pull/584)) - -## Bugfixes - -* Wait for exact number of units after scale down ([PR #565](https://github.com/canonical/postgresql-operator/pull/565)) ([DPE-5029](https://warthogs.atlassian.net/browse/DPE-5029)) -* Improved test stability by pausing Patroni in the TLS test ([PR #534](https://github.com/canonical/postgresql-operator/pull/534)) ([DPE-4533](https://warthogs.atlassian.net/browse/DPE-4533)) -* Block charm if it detects objects dependent on disabled plugins ([PR #560](https://github.com/canonical/postgresql-operator/pull/560)) ([DPE-4967](https://warthogs.atlassian.net/browse/DPE-4967)) -* Disabled pgBackRest service initialization ([PR #530](https://github.com/canonical/postgresql-operator/pull/530)) ([DPE-4345](https://warthogs.atlassian.net/browse/DPE-4345)) -* Increased timeout and terminate processes that are still up ([PR #514](https://github.com/canonical/postgresql-operator/pull/514)) ([DPE-4532](https://warthogs.atlassian.net/browse/DPE-4532)) -* Fixed GCP backup test ([PR #521](https://github.com/canonical/postgresql-operator/pull/521)) ([DPE-4820](https://warthogs.atlassian.net/browse/DPE-4820)) -* Handled on start secret exception and remove stale test ([PR #550](https://github.com/canonical/postgresql-operator/pull/550)) -* Removed block on failure to get the db version ([PR #578](https://github.com/canonical/postgresql-operator/pull/578)) ([DPE-3562](https://warthogs.atlassian.net/browse/DPE-3562)) -* Updated unit tests after fixing GCP backup test ([PR #528](https://github.com/canonical/postgresql-operator/pull/528)) ([DPE-4820](https://warthogs.atlassian.net/browse/DPE-4820)) -* Ported some `test_self_healing` CI fixes + update check for invalid extra user credentials ([PR #546](https://github.com/canonical/postgresql-operator/pull/546)) ([DPE-4856](https://warthogs.atlassian.net/browse/DPE-4856)) -* Fixed slow bootstrap of replicas ([PR #510](https://github.com/canonical/postgresql-operator/pull/510)) ([DPE-4759](https://warthogs.atlassian.net/browse/DPE-4759)) -* Fixed conditional password ([PR #604](https://github.com/canonical/postgresql-operator/pull/604)) -* Added enforcement of Juju versions ([PR #518](https://github.com/canonical/postgresql-operator/pull/518)) ([DPE-4809](https://warthogs.atlassian.net/browse/DPE-4809)) -* Fixed a missing case for peer to secrets translation. ([PR #533](https://github.com/canonical/postgresql-operator/pull/533)) -* Updated README.md ([PR #538](https://github.com/canonical/postgresql-operator/pull/538)) ([DPE-4901](https://warthogs.atlassian.net/browse/DPE-4901)) -* Increased test coverage ([PR #505](https://github.com/canonical/postgresql-operator/pull/505)) - -## Known limitations - - * The unit action `resume-upgrade` randomly raises a [harmless error message](https://warthogs.atlassian.net/browse/DPE-5420): `terminated`. - * The [charm sysbench](https://charmhub.io/sysbench) may [crash](https://warthogs.atlassian.net/browse/DPE-5436) during a PostgreSQL charm refresh. - * Make sure that [cluster-cluster replication](/t/13991) is requested for the same charm/workload revisions. An automated check is [planned](https://warthogs.atlassian.net/browse/DPE-5418). - * [Contact us](/t/11863) to schedule [the cluster-cluster replication](/t/13991) upgrade with you. - -If you are jumping over several stable revisions, check [previous release notes][All revisions] before upgrading. - -## Requirements and compatibility - -This charm revision features the following changes in dependencies: -* (increased) The minimum Juju version required to reliably operate **all** features of the release is `v3.4.5` - > You can upgrade to this revision on Juju `v2.9.50+`, but it will not support newer features like cross-regional asynchronous replication, point-in-time recovery, and modern TLS certificate charm integrations. -* (increased) PostgreSQL version 14.12 - -Check the [system requirements] page for more details, such as supported minor versions of Juju and hardware requirements. - -### Integration tests -Below are the charm integrations tested with this revision on different Juju environments and architectures: -* Juju `v.2.9.50` on `amd64` -* Juju `v.3.4.5` on `amd64` and `arm64` - -| Software | Version | Notes | -|-----|-----|-----| -| [lxd] | `5.12/stable` | | -| [nextcloud] | `v29.0.5.1`, `rev 26` | | -| [mailman3-core] | `rev 18` | | -| [data-integrator] | `rev 41` | | -| [s3-integrator] | `rev 31` | | -| [postgresql-test-app] | `rev 237` | | - -See the [`/lib/charms` directory on GitHub] for details about all supported libraries. - -See the [`metadata.yaml` file on GitHub] for a full list of supported interfaces. - -### Packaging - -This charm is based on the Charmed PostgreSQL [snap Revision 120/121]. It packages: -* [postgresql `v.14.12`] -* [pgbouncer `v.1.21`] -* [patroni `v.3.1.2 `] -* [pgBackRest `v.2.48`] -* [prometheus-postgres-exporter `v.0.12.1`] - -### Dependencies and automations - -[details=This section contains a list of updates to libs, dependencies, actions, and workflows.] - -* Added jinja2 as a dependency ([PR #520](https://github.com/canonical/postgresql-operator/pull/520)) ([DPE-4816](https://warthogs.atlassian.net/browse/DPE-4816)) -* Switched Jira issue sync from workflow to bot ([PR #586](https://github.com/canonical/postgresql-operator/pull/586)) -* Updated canonical/charming-actions action to v2.6.2 ([PR #523](https://github.com/canonical/postgresql-operator/pull/523)) -* Updated data-platform-workflows to v21.0.1 ([PR #599](https://github.com/canonical/postgresql-operator/pull/599)) -* Updated dependency cryptography to v43 ([PR #539](https://github.com/canonical/postgresql-operator/pull/539)) -* Updated dependency tenacity to v9 ([PR #558](https://github.com/canonical/postgresql-operator/pull/558)) -* Updated Juju agents (patch) ([PR #553](https://github.com/canonical/postgresql-operator/pull/553)) -* Switch to resusable presets ([PR #513](https://github.com/canonical/postgresql-operator/pull/513)) -* Use poetry package-mode=false ([PR #556](https://github.com/canonical/postgresql-operator/pull/556)) -* Switched test-app interface ([PR #557](https://github.com/canonical/postgresql-operator/pull/557)) -[/details] - - -[All revisions]: /t/11875 -[system requirements]: /t/11743 - - -[`/lib/charms` directory on GitHub]: https://github.com/canonical/postgresql-operator/tree/rev468/lib/charms -[`metadata.yaml` file on GitHub]: https://github.com/canonical/postgresql-operator/blob/rev468/metadata.yaml - - -[14/stable channel]: https://charmhub.io/postgresql?channel=14/stable - - -[`charmed-postgresql` packaging]: https://github.com/canonical/charmed-postgresql-snap -[snap Revision 120/121]: https://github.com/canonical/charmed-postgresql-snap/releases/tag/rev121 -[rock image]: ghcr.io/canonical/charmed-postgresql@sha256:7ef86a352c94e2a664f621a1cc683d7a983fd86e923d98c32b863f717cb1c173 - -[postgresql `v.14.12`]: https://launchpad.net/ubuntu/+source/postgresql-14/14.12-0ubuntu0.22.04.1 -[pgbouncer `v.1.21`]: https://launchpad.net/~data-platform/+archive/ubuntu/pgbouncer -[patroni `v.3.1.2 `]: https://launchpad.net/~data-platform/+archive/ubuntu/patroni -[pgBackRest `v.2.48`]: https://launchpad.net/~data-platform/+archive/ubuntu/pgbackrest -[prometheus-postgres-exporter `v.0.12.1`]: https://launchpad.net/~data-platform/+archive/ubuntu/postgres-exporter - - -[juju]: https://juju.is/docs/juju/ -[lxd]: https://documentation.ubuntu.com/lxd/en/latest/ -[nextcloud]: https://charmhub.io/nextcloud -[mailman3-core]: https://charmhub.io/mailman3-core -[data-integrator]: https://charmhub.io/data-integrator -[s3-integrator]: https://charmhub.io/s3-integrator -[postgresql-test-app]: https://charmhub.io/postgresql-test-app -[discourse-k8s]: https://charmhub.io/discourse-k8s -[indico]: https://charmhub.io/indico -[microk8s]: https://charmhub.io/microk8s -[tls-certificates-operator]: https://charmhub.io/tls-certificates-operator -[self-signed-certificates]: https://charmhub.io/self-signed-certificates - - -[amd64]: https://img.shields.io/badge/amd64-darkgreen -[arm64]: https://img.shields.io/badge/arm64-blue \ No newline at end of file diff --git a/docs/reference/r-revision-544-545.md b/docs/reference/r-revision-544-545.md deleted file mode 100644 index 39732165fc..0000000000 --- a/docs/reference/r-revision-544-545.md +++ /dev/null @@ -1,162 +0,0 @@ ->Reference > Release Notes > [All revisions] > Revision 544/545 - -[note type=caution] -This page is a work in progress for a **future release**. Please revisit at a later date! -[/note] - -# Revision 544/545 - - -Canonical's newest Charmed PostgreSQL operator has been published in the [14/stable channel]. - -Due to the newly added support for `arm64` architecture, the PostgreSQL charm now releases multiple revisions simultaneously: -* Revision is built for `amd64` on Ubuntu 22.04 LTS -* Revision is built for `arm64` on Ubuntu 22.04 LTS - -> See also: [How to perform a minor upgrade] - -### Contents -* [Highlights](#highlights) -* [Features and improvements](#features-and-improvements) -* [Bugfixes and maintenance](#bugfixes-and-maintenance) -* [Known limitations](#known-limitations) -* [Requirements and compatibility](#requirements-and-compatibility) - * [Integration tests](#integration-tests) - * [Packaging](#packaging) ---- - -## Highlights - -* Upgraded PostgreSQL from v.14.12 → v.14.15 ([PR #730](https://github.com/canonical/postgresql-operator/pull/730)) - * Check the official [PostgreSQL 14.13 release notes](https://www.postgresql.org/docs/release/14.13/) - * Check the official [PostgreSQL 14.14 release notes](https://www.postgresql.org/docs/release/14.14/) - * Check the official [PostgreSQL 14.15 release notes](https://www.postgresql.org/docs/release/14.15/) -* Added timeline management to point-in-time recovery (PITR) ([PR #629](https://github.com/canonical/postgresql-operator/pull/629)) ([DPE-5561](https://warthogs.atlassian.net/browse/DPE-5561)) -* Added pgAudit plugin/extension ([PR #612](https://github.com/canonical/postgresql-operator/pull/612)) ([DPE-5248](https://warthogs.atlassian.net/browse/DPE-5248)) -* Observability stack (COS) improvements - * Polished built-in Grafana dashboard ([PR #646](https://github.com/canonical/postgresql-operator/pull/646)) - * Improved COS alert rule descriptions ([PR #651](https://github.com/canonical/postgresql-operator/pull/651)) ([DPE-5658](https://warthogs.atlassian.net/browse/DPE-5658)) -* Added fully-featured terraform module ([PR #643](https://github.com/canonical/postgresql-operator/pull/643)) -* Several S3 stability improvements ([PR #642](https://github.com/canonical/postgresql-operator/pull/642)) - -## Features and improvements -* Removed patching of private ops class. ([PR #617](https://github.com/canonical/postgresql-operator/pull/617)) -* Switched charm libs from `tempo_k8s` to `tempo_coordinator_k8s` and relay tracing traffic through `grafana-agent` ([PR #640](https://github.com/canonical/postgresql-operator/pull/640)) -* Implemented more meaningful group naming for multi-group tests ([PR #625](https://github.com/canonical/postgresql-operator/pull/625)) -* Ignoring alias error in case alias is already existing ([PR #637](https://github.com/canonical/postgresql-operator/pull/637)) -* Stopped tracking channel for held snaps ([PR #638](https://github.com/canonical/postgresql-operator/pull/638)) -* Added pgBackRest logrotate configuration ([PR #645](https://github.com/canonical/postgresql-operator/pull/645)) ([DPE-5601](https://warthogs.atlassian.net/browse/DPE-5601)) -* Grant priviledges to non-public schemas ([PR #647](https://github.com/canonical/postgresql-operator/pull/647)) ([DPE-5387](https://warthogs.atlassian.net/browse/DPE-5387)) -* Added `tls` and `tls-ca` fields to databag ([PR #666](https://github.com/canonical/postgresql-operator/pull/666)) -* Merged `update_tls_flag` into `update_endpoints` ([PR #669](https://github.com/canonical/postgresql-operator/pull/669)) -* Made tox commands resilient to white-space paths ([PR #678](https://github.com/canonical/postgresql-operator/pull/678)) ([DPE-6042](https://warthogs.atlassian.net/browse/DPE-6042)) -* Added microceph (local backup) integration test + bump snap version ([PR #633](https://github.com/canonical/postgresql-operator/pull/633)) ([DPE-5386](https://warthogs.atlassian.net/browse/DPE-5386)) -* Add `max_locks_per_transaction` config option in ([PR#718](https://github.com/canonical/postgresql-operator/pull/718)) ([DPE-5533](https://warthogs.atlassian.net/browse/DPE-5533)) -* Split PITR backup test in AWS and GCP ([PR #605](https://github.com/canonical/postgresql-operator/pull/605)) ([DPE-5181](https://warthogs.atlassian.net/browse/DPE-5181)) - - -## Bugfixes and maintenance -* Juju secrets resetting fix on Juju 3.6 in ([PR#726](https://github.com/canonical/postgresql-operator/pull/726)) ([DPE-6320](https://warthogs.atlassian.net/browse/DPE-6320)) ([DPE-6325](https://warthogs.atlassian.net/browse/DPE-6325)) -* Fallback to trying to create bucket without LocationConstraint in ([PR#690](https://github.com/canonical/postgresql-operator/pull/690)) -* Added warning logs to Patroni reinitialisation ([PR #660](https://github.com/canonical/postgresql-operator/pull/660)) -* Fixed some `postgresql.conf` parameters for hardening ([PR #621](https://github.com/canonical/postgresql-operator/pull/621)) ([DPE-5512](https://warthogs.atlassian.net/browse/DPE-5512)) -* Fixed lib check ([PR #627](https://github.com/canonical/postgresql-operator/pull/627)) -* Allow `--restore-to-time=latest` without a `backup-id` in ([PR#683](https://github.com/canonical/postgresql-operator/pull/683)) -* Clean-up duplicated Patroni config reloads in ([PR#682](https://github.com/canonical/postgresql-operator/pull/682)) -* Filter out degraded read only endpoints in ([PR#679](https://github.com/canonical/postgresql-operator/pull/679)) ([DPE-5714](https://warthogs.atlassian.net/browse/DPE-5714)) - -[details=Libraries, testing, and CI] -* Data Interafces v40 ([PR #615](https://github.com/canonical/postgresql-operator/pull/615)) ([DPE-5306](https://warthogs.atlassian.net/browse/DPE-5306)) -* Bump libs and remove TestCase ([PR #622](https://github.com/canonical/postgresql-operator/pull/622)) -* Run tests against juju 3.6 on a nightly schedule ([PR #601](https://github.com/canonical/postgresql-operator/pull/601)) ([DPE-4977](https://warthogs.atlassian.net/browse/DPE-4977)) -* Test against juju 3.6/candidate + upgrade dpw to v23.0.5 ([PR #675](https://github.com/canonical/postgresql-operator/pull/675)) -* Lock file maintenance Python dependencies ([PR #644](https://github.com/canonical/postgresql-operator/pull/644)) -* Migrate config .github/renovate.json5 ([PR #673](https://github.com/canonical/postgresql-operator/pull/673)) -* Switch from tox build wrapper to charmcraft.yaml overrides ([PR #626](https://github.com/canonical/postgresql-operator/pull/626)) -* Update canonical/charming-actions action to v2.6.3 ([PR #608](https://github.com/canonical/postgresql-operator/pull/608)) -* Update codecov/codecov-action action to v5 ([PR #674](https://github.com/canonical/postgresql-operator/pull/674)) -* Update data-platform-workflows to v23.0.5 ([PR #676](https://github.com/canonical/postgresql-operator/pull/676)) -* Update dependency cryptography to v43.0.1 [SECURITY] ([PR #614](https://github.com/canonical/postgresql-operator/pull/614)) -* Update dependency ubuntu to v24 ([PR #631](https://github.com/canonical/postgresql-operator/pull/631)) -* Update Juju agents ([PR #634](https://github.com/canonical/postgresql-operator/pull/634)) -* Bump libs ([PR #677](https://github.com/canonical/postgresql-operator/pull/677)) -* Increase linting rules ([PR #649](https://github.com/canonical/postgresql-operator/pull/649)) ([DPE-5324](https://warthogs.atlassian.net/browse/DPE-5324)) -[/details] - -## Requirements and compatibility -* (no change) Minimum Juju 2 version: `v.2.9.49` -* (no change) Minimum Juju 3 version: `v.3.4.3` -* (recommended) Juju LTS 3.6.1+ - -See the [system requirements] for more details about Juju versions and other software and hardware prerequisites. - -### Integration tests -Below are some of the charm integrations tested with this revision on different Juju environments and architectures: -* Juju `v.2.9.51` on `amd64` -* Juju `v.3.4.6` on `amd64` and `arm64` - -| Software | Revision | Tested on | | -|-----|-----|----|---| -| [postgresql-test-app] | `rev 281` | ![juju-2_amd64] ![juju-3_amd64] | -| | `rev 279` | ![juju-2_amd64] ![juju-3_amd64] | -| | `rev 280` | ![juju-3_arm64] | -| | `rev 278` | ![juju-3_arm64] | -| [data-integrator] | `rev 41` | ![juju-2_amd64] ![juju-3_amd64] | -| | `rev 40` | ![juju-3_arm64] | -| [nextcloud] | `rev 26` | ![juju-2_amd64] ![juju-3_amd64] | | -| [s3-integrator] | `rev 77` | ![juju-2_amd64] ![juju-3_amd64] | -| | `rev 78` | ![juju-3_arm64] | -| [tls-certificates-operator] | `rev 22` | ![juju-2_amd64] | -| [self-signed-certificates] | `rev 155` | ![juju-3_amd64] | -| | `rev 205` | ![juju-3_arm64] | -| [mailman3-core] | `rev 18` | ![juju-2_amd64] ![juju-3_amd64] ![juju-3_arm64] | -| [landscape-client] | `rev 70` | ![juju-2_amd64] ![juju-3_amd64] ![juju-3_arm64] | -| [ubuntu-advantage] | `rev 137` | ![juju-2_amd64] ![juju-3_amd64] | -| | `rev 139` | ![juju-3_arm64]| - -See the [`/lib/charms` directory on GitHub] for details about all supported libraries. - -See the [`metadata.yaml` file on GitHub] for a full list of supported interfaces. - -### Packaging -This charm is based on the Charmed PostgreSQL [snap revision 132/133](https://github.com/canonical/charmed-postgresql-snap/tree/rev121). It packages: -* [postgresql] `v.14.12` -* [pgbouncer] `v.1.21` -* [patroni] `v.3.1.2 ` -* [pgBackRest] `v.2.53` -* [prometheus-postgres-exporter] `v.0.12.1` - - -[14/stable channel]: https://charmhub.io/postgresql?channel=14/stable - -[All revisions]: /t/11875 -[system requirements]: /t/11743 -[How to perform a minor upgrade]: /t/12089 - -[juju]: https://juju.is/docs/juju/ -[lxd]: https://documentation.ubuntu.com/lxd/en/latest/ -[nextcloud]: https://charmhub.io/nextcloud -[mailman3-core]: https://charmhub.io/mailman3-core -[data-integrator]: https://charmhub.io/data-integrator -[s3-integrator]: https://charmhub.io/s3-integrator -[postgresql-test-app]: https://charmhub.io/postgresql-test-app -[discourse-k8s]: https://charmhub.io/discourse-k8s -[indico]: https://charmhub.io/indico -[microk8s]: https://charmhub.io/microk8s -[tls-certificates-operator]: https://charmhub.io/tls-certificates-operator -[self-signed-certificates]: https://charmhub.io/self-signed-certificates -[landscape-client]: https://charmhub.io/landscape-client -[ubuntu-advantage]: https://charmhub.io/ubuntu-advantage - -[`/lib/charms` directory on GitHub]: https://github.com/canonical/postgresql-operator/tree/rev518/lib/charms -[`metadata.yaml` file on GitHub]: https://github.com/canonical/postgresql-operator/blob/rev518/metadata.yaml - -[postgresql]: https://launchpad.net/ubuntu/+source/postgresql-14/ -[pgbouncer]: https://launchpad.net/~data-platform/+archive/ubuntu/pgbouncer -[patroni]: https://launchpad.net/~data-platform/+archive/ubuntu/patroni -[pgBackRest]: https://launchpad.net/~data-platform/+archive/ubuntu/pgbackrest -[prometheus-postgres-exporter]: https://launchpad.net/~data-platform/+archive/ubuntu/postgres-exporter - -[juju-2_amd64]: https://img.shields.io/badge/Juju_2.9.51-amd64-darkgreen?labelColor=ea7d56 -[juju-3_amd64]: https://img.shields.io/badge/Juju_3.4.6-amd64-darkgreen?labelColor=E95420 -[juju-3_arm64]: https://img.shields.io/badge/Juju_3.4.6-arm64-blue?labelColor=E95420 \ No newline at end of file diff --git a/docs/reference/r-system-requirements.md b/docs/reference/r-system-requirements.md index 0c659fee12..5d4c4a16b2 100644 --- a/docs/reference/r-system-requirements.md +++ b/docs/reference/r-system-requirements.md @@ -11,7 +11,7 @@ The charm supports several Juju releases from [2.9 LTS](https://juju.is/docs/juj | Juju major release | Supported minor versions | Compatible charm revisions |Comment | |:--------|:-----|:-----|:-----| -| ![3.6 LTS] | `3.6.0-beta2` | [363]+ | No known issues, but still in beta. Not recommended for production. | +| ![3.6 LTS] | `3.6.1+` | [552]+ | `3.6.0` is not recommended, while `3.6.1+` works excellent. Recommended for production! | | [![3.5]](https://juju.is/docs/juju/roadmap#juju-juju-35) | `3.5.1+` | [363]+ | [Known Juju issue](https://bugs.launchpad.net/juju/+bug/2066517) in `3.5.0` | | [![3.4]](https://juju.is/docs/juju/roadmap#juju-juju-34) | `3.4.3+` | [363]+ | Know Juju issues with previous minor versions | | [![3.3]](https://juju.is/docs/juju/roadmap#juju-juju-33) | `3.3.0+` | from [363] to [430] | No known issues | @@ -52,6 +52,7 @@ The charm is based on the [charmed-postgresql snap](https://snapcraft.io/charmed [3.6 LTS]: https://img.shields.io/badge/3.6_LTS-%23E95420?label=Juju +[552]: /t/16007 [288]: /t/11876 [336]: /t/11877 [363]: /t/13124 diff --git a/docs/tutorial/t-overview.md b/docs/tutorial.md similarity index 98% rename from docs/tutorial/t-overview.md rename to docs/tutorial.md index d8a7a11511..f532210bb6 100644 --- a/docs/tutorial/t-overview.md +++ b/docs/tutorial.md @@ -1,4 +1,4 @@ -# Charmed PostgreSQL Tutorial +# Tutorial This section of our documentation contains comprehensive, hands-on tutorials to help you learn how to deploy Charmed PostgreSQL on machines and become familiar with its available operations. diff --git a/lib/charms/data_platform_libs/v0/data_interfaces.py b/lib/charms/data_platform_libs/v0/data_interfaces.py index 3bc2dd8503..9717119030 100644 --- a/lib/charms/data_platform_libs/v0/data_interfaces.py +++ b/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -331,7 +331,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 40 +LIBPATCH = 41 PYDEPS = ["ops>=2.0.0"] @@ -609,7 +609,7 @@ def get_group(self, group: str) -> Optional[SecretGroup]: class CachedSecret: """Locally cache a secret. - The data structure is precisely re-using/simulating as in the actual Secret Storage + The data structure is precisely reusing/simulating as in the actual Secret Storage """ KNOWN_MODEL_ERRORS = [MODEL_ERRORS["no_label_and_uri"], MODEL_ERRORS["owner_no_refresh"]] @@ -2363,7 +2363,6 @@ def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> Non def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: """Delete data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" if self.secret_fields and self.deleted_label: - _, normal_fields = self._process_secret_fields( relation, self.secret_fields, diff --git a/lib/charms/data_platform_libs/v0/data_models.py b/lib/charms/data_platform_libs/v0/data_models.py index a1dbb8299a..087f6f3c58 100644 --- a/lib/charms/data_platform_libs/v0/data_models.py +++ b/lib/charms/data_platform_libs/v0/data_models.py @@ -168,7 +168,7 @@ class MergedDataBag(ProviderDataBag, RequirerDataBag): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 4 +LIBPATCH = 5 PYDEPS = ["ops>=2.0.0", "pydantic>=1.10,<2"] @@ -209,7 +209,7 @@ def validate_params(cls: Type[T]): """ def decorator( - f: Callable[[CharmBase, ActionEvent, Union[T, ValidationError]], G] + f: Callable[[CharmBase, ActionEvent, Union[T, ValidationError]], G], ) -> Callable[[CharmBase, ActionEvent], G]: @wraps(f) def event_wrapper(self: CharmBase, event: ActionEvent): @@ -287,7 +287,7 @@ def decorator( Optional[Union[UnitModel, ValidationError]], ], G, - ] + ], ) -> Callable[[CharmBase, RelationEvent], G]: @wraps(f) def event_wrapper(self: CharmBase, event: RelationEvent): diff --git a/lib/charms/grafana_agent/v0/cos_agent.py b/lib/charms/grafana_agent/v0/cos_agent.py index 1ea79a625b..b18c271342 100644 --- a/lib/charms/grafana_agent/v0/cos_agent.py +++ b/lib/charms/grafana_agent/v0/cos_agent.py @@ -8,6 +8,8 @@ - `COSAgentProvider`: Use in machine charms that need to have a workload's metrics or logs scraped, or forward rule files or dashboards to Prometheus, Loki or Grafana through the Grafana Agent machine charm. + NOTE: Be sure to add `limit: 1` in your charm for the cos-agent relation. That is the only + way we currently have to prevent two different grafana agent apps deployed on the same VM. - `COSAgentConsumer`: Used in the Grafana Agent machine charm to manage the requirer side of the `cos_agent` interface. @@ -232,8 +234,8 @@ def __init__(self, *args): ) import pydantic -from cosl import GrafanaDashboard, JujuTopology -from cosl.rules import AlertRules +from cosl import DashboardPath40UID, JujuTopology, LZMABase64 +from cosl.rules import AlertRules, generic_alert_groups from ops.charm import RelationChangedEvent from ops.framework import EventBase, EventSource, Object, ObjectEvents from ops.model import ModelError, Relation @@ -252,9 +254,9 @@ class _MetricsEndpointDict(TypedDict): LIBID = "dc15fa84cef84ce58155fb84f6c6213a" LIBAPI = 0 -LIBPATCH = 12 +LIBPATCH = 20 -PYDEPS = ["cosl", "pydantic"] +PYDEPS = ["cosl >= 0.0.50", "pydantic"] DEFAULT_RELATION_NAME = "cos-agent" DEFAULT_PEER_RELATION_NAME = "peers" @@ -266,7 +268,6 @@ class _MetricsEndpointDict(TypedDict): logger = logging.getLogger(__name__) SnapEndpoint = namedtuple("SnapEndpoint", "owner, name") - # Note: MutableMapping is imported from the typing module and not collections.abc # because subscripting collections.abc.MutableMapping was added in python 3.9, but # most of our charms are based on 20.04, which has python 3.8. @@ -316,7 +317,11 @@ class NotReadyError(TracingError): """Raised by the provider wrapper if a requirer hasn't published the required data (yet).""" -class ProtocolNotRequestedError(TracingError): +class ProtocolNotFoundError(TracingError): + """Raised if the user doesn't receive an endpoint for a protocol it requested.""" + + +class ProtocolNotRequestedError(ProtocolNotFoundError): """Raised if the user attempts to obtain an endpoint for a protocol it did not request.""" @@ -475,7 +480,7 @@ class CosAgentProviderUnitData(DatabagModel): # this needs to make its way to the gagent leader metrics_alert_rules: dict log_alert_rules: dict - dashboards: List[GrafanaDashboard] + dashboards: List[str] # subordinate is no longer used but we should keep it until we bump the library to ensure # we don't break compatibility. subordinate: Optional[bool] = None @@ -508,7 +513,7 @@ class CosAgentPeersUnitData(DatabagModel): # of the outgoing o11y relations. metrics_alert_rules: Optional[dict] log_alert_rules: Optional[dict] - dashboards: Optional[List[GrafanaDashboard]] + dashboards: Optional[List[str]] # when this whole datastructure is dumped into a databag, it will be nested under this key. # while not strictly necessary (we could have it 'flattened out' into the databag), @@ -578,7 +583,7 @@ class Receiver(pydantic.BaseModel): """Specification of an active receiver.""" protocol: ProtocolType = pydantic.Field(..., description="Receiver protocol name and type.") - url: str = pydantic.Field( + url: Optional[str] = pydantic.Field( ..., description="""URL at which the receiver is reachable. If there's an ingress, it would be the external URL. Otherwise, it would be the service's fqdn or internal IP. @@ -726,6 +731,10 @@ def _metrics_alert_rules(self) -> Dict: query_type="promql", topology=JujuTopology.from_charm(self._charm) ) alert_rules.add_path(self._metrics_rules, recursive=self._recursive) + alert_rules.add( + generic_alert_groups.application_rules, + group_name_prefix=JujuTopology.from_charm(self._charm).identifier, + ) return alert_rules.as_dict() @property @@ -736,12 +745,27 @@ def _log_alert_rules(self) -> Dict: return alert_rules.as_dict() @property - def _dashboards(self) -> List[GrafanaDashboard]: - dashboards: List[GrafanaDashboard] = [] + def _dashboards(self) -> List[str]: + dashboards: List[str] = [] for d in self._dashboard_dirs: for path in Path(d).glob("*"): - dashboard = GrafanaDashboard._serialize(path.read_bytes()) - dashboards.append(dashboard) + with open(path, "rt") as fp: + dashboard = json.load(fp) + rel_path = str( + path.relative_to(self._charm.charm_dir) if path.is_absolute() else path + ) + # COSAgentProvider is somewhat analogous to GrafanaDashboardProvider. We need to overwrite the uid here + # because there is currently no other way to communicate the dashboard path separately. + # https://github.com/canonical/grafana-k8s-operator/pull/363 + dashboard["uid"] = DashboardPath40UID.generate(self._charm.meta.name, rel_path) + + # Add tags + tags: List[str] = dashboard.get("tags", []) + if not any(tag.startswith("charm: ") for tag in tags): + tags.append(f"charm: {self._charm.meta.name}") + dashboard["tags"] = tags + + dashboards.append(LZMABase64.compress(json.dumps(dashboard))) return dashboards @property @@ -767,7 +791,7 @@ def is_ready(self, relation: Optional[Relation] = None): """Is this endpoint ready?""" relation = relation or self._relation if not relation: - logger.debug(f"no relation on {self._relation_name !r}: tracing not ready") + logger.debug(f"no relation on {self._relation_name!r}: tracing not ready") return False if relation.data is None: logger.error(f"relation data is None for {relation}") @@ -801,29 +825,48 @@ def get_all_endpoints( def _get_tracing_endpoint( self, relation: Optional[Relation], protocol: ReceiverProtocol - ) -> Optional[str]: + ) -> str: + """Return a tracing endpoint URL if it is available or raise a ProtocolNotFoundError.""" unit_data = self.get_all_endpoints(relation) if not unit_data: - return None + # we didn't find the protocol because the remote end didn't publish any data yet + # it might also mean that grafana-agent doesn't have a relation to the tracing backend + raise ProtocolNotFoundError(protocol) receivers: List[Receiver] = [i for i in unit_data.receivers if i.protocol.name == protocol] if not receivers: - logger.error(f"no receiver found with protocol={protocol!r}") - return None + # we didn't find the protocol because grafana-agent didn't return us the protocol that we requested + # the caller might want to verify that we did indeed request this protocol + raise ProtocolNotFoundError(protocol) if len(receivers) > 1: - logger.error( + logger.warning( f"too many receivers with protocol={protocol!r}; using first one. Found: {receivers}" ) - return None receiver = receivers[0] + if not receiver.url: + # grafana-agent isn't connected to the tracing backend yet + raise ProtocolNotFoundError(protocol) return receiver.url def get_tracing_endpoint( self, protocol: ReceiverProtocol, relation: Optional[Relation] = None - ) -> Optional[str]: - """Receiver endpoint for the given protocol.""" - endpoint = self._get_tracing_endpoint(relation or self._relation, protocol=protocol) - if not endpoint: + ) -> str: + """Receiver endpoint for the given protocol. + + It could happen that this function gets called before the provider publishes the endpoints. + In such a scenario, if a non-leader unit calls this function, a permission denied exception will be raised due to + restricted access. To prevent this, this function needs to be guarded by the `is_ready` check. + + Raises: + ProtocolNotRequestedError: + If the charm unit is the leader unit and attempts to obtain an endpoint for a protocol it did not request. + ProtocolNotFoundError: + If the charm attempts to obtain an endpoint when grafana-agent isn't related to a tracing backend. + """ + try: + return self._get_tracing_endpoint(relation or self._relation, protocol=protocol) + except ProtocolNotFoundError: + # let's see if we didn't find it because we didn't request the endpoint requested_protocols = set() relations = [relation] if relation else self.relations for relation in relations: @@ -838,8 +881,7 @@ def get_tracing_endpoint( if protocol not in requested_protocols: raise ProtocolNotRequestedError(protocol, relation) - return None - return endpoint + raise class COSAgentDataChanged(EventBase): @@ -901,6 +943,8 @@ def __init__( events.relation_joined, self._on_relation_data_changed ) # TODO: do we need this? self.framework.observe(events.relation_changed, self._on_relation_data_changed) + self.framework.observe(events.relation_departed, self._on_relation_departed) + for event in self._refresh_events: self.framework.observe(event, self.trigger_refresh) # pyright: ignore @@ -928,6 +972,26 @@ def _on_peer_relation_changed(self, _): if self._charm.unit.is_leader(): self.on.data_changed.emit() # pyright: ignore + def _on_relation_departed(self, event): + """Remove provider's (principal's) alert rules and dashboards from peer data when the cos-agent relation to the principal is removed.""" + if not self.peer_relation: + event.defer() + return + # empty the departing unit's alert rules and dashboards from peer data + data = CosAgentPeersUnitData( + unit_name=event.unit.name, + relation_id=str(event.relation.id), + relation_name=event.relation.name, + metrics_alert_rules={}, + log_alert_rules={}, + dashboards=[], + ) + self.peer_relation.data[self._charm.unit][ + f"{CosAgentPeersUnitData.KEY}-{event.unit.name}" + ] = data.json() + + self.on.data_changed.emit() # pyright: ignore + def _on_relation_data_changed(self, event: RelationChangedEvent): # Peer data is the only means of communication between subordinate units. if not self.peer_relation: @@ -987,7 +1051,16 @@ def update_tracing_receivers(self): CosAgentRequirerUnitData( receivers=[ Receiver( - url=f"{self._get_tracing_receiver_url(protocol)}", + # if tracing isn't ready, we don't want the wrong receiver URLs present in the databag. + # however, because of the backwards compatibility requirements, we need to still provide + # the protocols list so that the charm with older cos_agent version doesn't error its hooks. + # before this change was added, the charm with old cos_agent version threw exceptions with + # connections to grafana-agent timing out. After the change, the charm will fail validating + # databag contents (as it expects a string in URL) but that won't cause any errors as + # tracing endpoints are the only content in the grafana-agent's side of the databag. + url=f"{self._get_tracing_receiver_url(protocol)}" + if self._charm.tracing.is_ready() # type: ignore + else None, protocol=ProtocolType( name=protocol, type=receiver_protocol_to_transport_protocol[protocol], @@ -1029,8 +1102,7 @@ def _get_requested_protocols(self, relation: Relation): if len(units) > 1: # should never happen raise ValueError( - f"unexpected error: subordinate relation {relation} " - f"should have exactly one unit" + f"unexpected error: subordinate relation {relation} should have exactly one unit" ) unit = next(iter(units), None) @@ -1286,7 +1358,7 @@ def dashboards(self) -> List[Dict[str, str]]: seen_apps.append(app_name) for encoded_dashboard in data.dashboards or (): - content = GrafanaDashboard(encoded_dashboard)._deserialize() + content = json.loads(LZMABase64.decompress(encoded_dashboard)) title = content.get("title", "no_title") @@ -1313,44 +1385,32 @@ def charm_tracing_config( If https endpoint is provided but cert_path is not found on disk: disable charm tracing. If https endpoint is provided and cert_path is None: - ERROR + raise TracingError Else: proceed with charm tracing (with or without tls, as appropriate) Usage: - If you are using charm_tracing >= v1.9: - >>> from lib.charms.tempo_k8s.v1.charm_tracing import trace_charm - >>> from lib.charms.tempo_k8s.v0.cos_agent import charm_tracing_config + >>> from lib.charms.tempo_coordinator_k8s.v0.charm_tracing import trace_charm + >>> from lib.charms.tempo_coordinator_k8s.v0.tracing import charm_tracing_config >>> @trace_charm(tracing_endpoint="my_endpoint", cert_path="cert_path") >>> class MyCharm(...): >>> _cert_path = "/path/to/cert/on/charm/container.crt" >>> def __init__(self, ...): - >>> self.cos_agent = COSAgentProvider(...) + >>> self.tracing = TracingEndpointRequirer(...) >>> self.my_endpoint, self.cert_path = charm_tracing_config( - ... self.cos_agent, self._cert_path) - - If you are using charm_tracing < v1.9: - >>> from lib.charms.tempo_k8s.v1.charm_tracing import trace_charm - >>> from lib.charms.tempo_k8s.v2.tracing import charm_tracing_config - >>> @trace_charm(tracing_endpoint="my_endpoint", cert_path="cert_path") - >>> class MyCharm(...): - >>> _cert_path = "/path/to/cert/on/charm/container.crt" - >>> def __init__(self, ...): - >>> self.cos_agent = COSAgentProvider(...) - >>> self.my_endpoint, self.cert_path = charm_tracing_config( - ... self.cos_agent, self._cert_path) - >>> @property - >>> def my_endpoint(self): - >>> return self._my_endpoint - >>> @property - >>> def cert_path(self): - >>> return self._cert_path - + ... self.tracing, self._cert_path) """ if not endpoint_requirer.is_ready(): return None, None - endpoint = endpoint_requirer.get_tracing_endpoint("otlp_http") + try: + endpoint = endpoint_requirer.get_tracing_endpoint("otlp_http") + except ProtocolNotFoundError: + logger.warn( + "Endpoint for tracing wasn't provided as tracing backend isn't ready yet. If grafana-agent isn't connected to a tracing backend, integrate it. Otherwise this issue should resolve itself in a few events." + ) + return None, None + if not endpoint: return None, None diff --git a/lib/charms/operator_libs_linux/v2/snap.py b/lib/charms/operator_libs_linux/v2/snap.py index d14f864fd9..5cd0ffd4b2 100644 --- a/lib/charms/operator_libs_linux/v2/snap.py +++ b/lib/charms/operator_libs_linux/v2/snap.py @@ -56,6 +56,8 @@ ``` """ +from __future__ import annotations + import http.client import json import logging @@ -65,14 +67,30 @@ import subprocess import sys import time +import typing import urllib.error import urllib.parse import urllib.request -from collections.abc import Mapping from datetime import datetime, timedelta, timezone from enum import Enum from subprocess import CalledProcessError, CompletedProcess -from typing import Any, Dict, Iterable, List, Optional, Union +from typing import ( + Callable, + Iterable, + Literal, + Mapping, + NoReturn, + Sequence, + TypedDict, + TypeVar, +) + +if typing.TYPE_CHECKING: + # avoid typing_extensions import at runtime + from typing_extensions import NotRequired, ParamSpec, Required, TypeAlias, Unpack + + _P = ParamSpec("_P") + _T = TypeVar("_T") logger = logging.getLogger(__name__) @@ -84,15 +102,15 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 9 +LIBPATCH = 10 # Regex to locate 7-bit C1 ANSI sequences ansi_filter = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") -def _cache_init(func): - def inner(*args, **kwargs): +def _cache_init(func: Callable[_P, _T]) -> Callable[_P, _T]: + def inner(*args: _P.args, **kwargs: _P.kwargs) -> _T: if _Cache.cache is None: _Cache.cache = SnapCache() return func(*args, **kwargs) @@ -100,8 +118,59 @@ def inner(*args, **kwargs): return inner -# recursive hints seems to error out pytest -JSONType = Union[Dict[str, Any], List[Any], str, int, float] +# this is used for return types, so it (a) uses concrete types and (b) does not contain None +# because setting snap config values to null removes the key so a null value can't be returned +_JSONLeaf: TypeAlias = 'str | int | float | bool' +JSONType: TypeAlias = "dict[str, JSONType] | list[JSONType] | _JSONLeaf" +# we also need a jsonable type for arguments, +# which (a) uses abstract types and (b) may contain None +JSONAble: TypeAlias = "Mapping[str, JSONAble] | Sequence[JSONAble] | _JSONLeaf | None" + + +class _AsyncChangeDict(TypedDict, total=True): + """The subset of the json returned by GET changes that we care about internally.""" + + status: str + data: JSONType + + +class _SnapDict(TypedDict, total=True): + """The subset of the json returned by GET snap/find that we care about internally.""" + + name: str + channel: str + revision: str + confinement: str + apps: NotRequired[list[dict[str, JSONType]] | None] + + +class SnapServiceDict(TypedDict, total=True): + """Dictionary representation returned by SnapService.as_dict.""" + + daemon: str | None + daemon_scope: str | None + enabled: bool + active: bool + activators: list[str] + + +# TypedDicts with hyphenated keys +_SnapServiceKwargsDict = TypedDict("_SnapServiceKwargsDict", {"daemon-scope": str}, total=False) +# the kwargs accepted by SnapService +_SnapServiceAppDict = TypedDict( + # the data we expect a Snap._apps entry to contain for a daemon + "_SnapServiceAppDict", + { + "name": "Required[str]", + "daemon": str, + "daemon_scope": str, + "daemon-scope": str, + "enabled": bool, + "active": bool, + "activators": "list[str]", + }, + total=False, +) class SnapService: @@ -109,20 +178,20 @@ class SnapService: def __init__( self, - daemon: Optional[str] = None, - daemon_scope: Optional[str] = None, + daemon: str | None = None, + daemon_scope: str | None = None, enabled: bool = False, active: bool = False, - activators: List[str] = [], - **kwargs, + activators: list[str] | None = None, + **kwargs: Unpack[_SnapServiceKwargsDict], ): self.daemon = daemon - self.daemon_scope = kwargs.get("daemon-scope", None) or daemon_scope + self.daemon_scope = kwargs.get("daemon-scope") or daemon_scope self.enabled = enabled self.active = active - self.activators = activators + self.activators = activators if activators is not None else [] - def as_dict(self) -> Dict: + def as_dict(self) -> SnapServiceDict: """Return instance representation as dict.""" return { "daemon": self.daemon, @@ -137,57 +206,54 @@ class MetaCache(type): """MetaCache class used for initialising the snap cache.""" @property - def cache(cls) -> "SnapCache": + def cache(cls) -> SnapCache: """Property for returning the snap cache.""" return cls._cache @cache.setter - def cache(cls, cache: "SnapCache") -> None: + def cache(cls, cache: SnapCache) -> None: """Setter for the snap cache.""" cls._cache = cache - def __getitem__(cls, name) -> "Snap": + def __getitem__(cls, name: str) -> Snap: """Snap cache getter.""" return cls._cache[name] -class _Cache(object, metaclass=MetaCache): +class _Cache(metaclass=MetaCache): _cache = None class Error(Exception): """Base class of most errors raised by this library.""" - def __repr__(self): + def __init__(self, message: str = "", *args: object): + super().__init__(message, *args) + self.message = message + + def __repr__(self) -> str: """Represent the Error class.""" - return "<{}.{} {}>".format(type(self).__module__, type(self).__name__, self.args) + return f"<{type(self).__module__}.{type(self).__name__} {self.args}>" @property - def name(self): + def name(self) -> str: """Return a string representation of the model plus class.""" - return "<{}.{}>".format(type(self).__module__, type(self).__name__) - - @property - def message(self): - """Return the message passed as an argument.""" - return self.args[0] + return f"<{type(self).__module__}.{type(self).__name__}>" class SnapAPIError(Error): """Raised when an HTTP API error occurs talking to the Snapd server.""" - def __init__(self, body: Dict, code: int, status: str, message: str): + def __init__(self, body: Mapping[str, JSONAble], code: int, status: str, message: str): super().__init__(message) # Makes str(e) return message self.body = body self.code = code self.status = status self._message = message - def __repr__(self): + def __repr__(self) -> str: """Represent the SnapAPIError class.""" - return "APIError({!r}, {!r}, {!r}, {!r})".format( - self.body, self.code, self.status, self._message - ) + return f"APIError({self.body!r}, {self.code!r}, {self.status!r}, {self._message!r})" class SnapState(Enum): @@ -207,7 +273,7 @@ class SnapNotFoundError(Error): """Raised when a requested snap is not known to the system.""" -class Snap(object): +class Snap: """Represents a snap package and its properties. `Snap` exposes the following properties about a snap: @@ -220,49 +286,47 @@ class Snap(object): def __init__( self, - name, + name: str, state: SnapState, channel: str, revision: str, confinement: str, - apps: Optional[List[Dict[str, str]]] = None, - cohort: Optional[str] = "", + apps: list[dict[str, JSONType]] | None = None, + cohort: str | None = None, ) -> None: self._name = name self._state = state self._channel = channel self._revision = revision self._confinement = confinement - self._cohort = cohort + self._cohort = cohort or "" self._apps = apps or [] self._snap_client = SnapClient() - def __eq__(self, other) -> bool: + def __eq__(self, other: object) -> bool: """Equality for comparison.""" return isinstance(other, self.__class__) and ( self._name, self._revision, ) == (other._name, other._revision) - def __hash__(self): + def __hash__(self) -> int: """Calculate a hash for this snap.""" return hash((self._name, self._revision)) - def __repr__(self): + def __repr__(self) -> str: """Represent the object such that it can be reconstructed.""" - return "<{}.{}: {}>".format(self.__module__, self.__class__.__name__, self.__dict__) + return f"<{self.__module__}.{type(self).__name__}: {self.__dict__}>" - def __str__(self): + def __str__(self) -> str: """Represent the snap object as a string.""" - return "<{}: {}-{}.{} -- {}>".format( - self.__class__.__name__, - self._name, - self._revision, - self._channel, - str(self._state), + return ( + f"<{type(self).__name__}: " + f"{self._name}-{self._revision}.{self._channel}" + f" -- {self._state}>" ) - def _snap(self, command: str, optargs: Optional[Iterable[str]] = None) -> str: + def _snap(self, command: str, optargs: Iterable[str] | None = None) -> str: """Perform a snap operation. Args: @@ -276,19 +340,17 @@ def _snap(self, command: str, optargs: Optional[Iterable[str]] = None) -> str: optargs = optargs or [] args = ["snap", command, self._name, *optargs] try: - return subprocess.check_output(args, universal_newlines=True) + return subprocess.check_output(args, text=True) except CalledProcessError as e: raise SnapError( - "Snap: {!r}; command {!r} failed with output = {!r}".format( - self._name, args, e.output - ) - ) + f"Snap: {self._name!r}; command {args!r} failed with output = {e.output!r}" + ) from e def _snap_daemons( self, - command: List[str], - services: Optional[List[str]] = None, - ) -> CompletedProcess: + command: list[str], + services: list[str] | None = None, + ) -> CompletedProcess[str]: """Perform snap app commands. Args: @@ -300,18 +362,26 @@ def _snap_daemons( """ if services: # an attempt to keep the command constrained to the snap instance's services - services = ["{}.{}".format(self._name, service) for service in services] + services = [f"{self._name}.{service}" for service in services] else: services = [self._name] args = ["snap", *command, *services] try: - return subprocess.run(args, universal_newlines=True, check=True, capture_output=True) + return subprocess.run(args, text=True, check=True, capture_output=True) except CalledProcessError as e: - raise SnapError("Could not {} for snap [{}]: {}".format(args, self._name, e.stderr)) - - def get(self, key: Optional[str], *, typed: bool = False) -> Any: + raise SnapError(f"Could not {args} for snap [{self._name}]: {e.stderr}") from e + + @typing.overload + def get(self, key: None | Literal[""], *, typed: Literal[False] = False) -> NoReturn: ... + @typing.overload + def get(self, key: str, *, typed: Literal[False] = False) -> str: ... + @typing.overload + def get(self, key: None | Literal[""], *, typed: Literal[True]) -> dict[str, JSONType]: ... + @typing.overload + def get(self, key: str, *, typed: Literal[True]) -> JSONType: ... + def get(self, key: str | None, *, typed: bool = False) -> JSONType | str: """Fetch snap configuration values. Args: @@ -323,7 +393,7 @@ def get(self, key: Optional[str], *, typed: bool = False) -> Any: args = ["-d"] if key: args.append(key) - config = json.loads(self._snap("get", args)) + config = json.loads(self._snap("get", args)) # json.loads -> Any if key: return config.get(key) return config @@ -331,9 +401,10 @@ def get(self, key: Optional[str], *, typed: bool = False) -> Any: if not key: raise TypeError("Key must be provided when typed=False") + # return a string return self._snap("get", [key]).strip() - def set(self, config: Dict[str, Any], *, typed: bool = False) -> None: + def set(self, config: dict[str, JSONAble], *, typed: bool = False) -> None: """Set a snap configuration value. Args: @@ -345,7 +416,7 @@ def set(self, config: Dict[str, Any], *, typed: bool = False) -> None: config = {k: str(v) for k, v in config.items()} self._snap_client._put_snap_conf(self._name, config) - def unset(self, key) -> str: + def unset(self, key: str) -> str: """Unset a snap configuration value. Args: @@ -353,7 +424,7 @@ def unset(self, key) -> str: """ return self._snap("unset", [key]) - def start(self, services: Optional[List[str]] = None, enable: Optional[bool] = False) -> None: + def start(self, services: list[str] | None = None, enable: bool = False) -> None: """Start a snap's services. Args: @@ -363,7 +434,7 @@ def start(self, services: Optional[List[str]] = None, enable: Optional[bool] = F args = ["start", "--enable"] if enable else ["start"] self._snap_daemons(args, services) - def stop(self, services: Optional[List[str]] = None, disable: Optional[bool] = False) -> None: + def stop(self, services: list[str] | None = None, disable: bool = False) -> None: """Stop a snap's services. Args: @@ -373,7 +444,7 @@ def stop(self, services: Optional[List[str]] = None, disable: Optional[bool] = F args = ["stop", "--disable"] if disable else ["stop"] self._snap_daemons(args, services) - def logs(self, services: Optional[List[str]] = None, num_lines: Optional[int] = 10) -> str: + def logs(self, services: list[str] | None = None, num_lines: int = 10) -> str: """Fetch a snap services' logs. Args: @@ -381,12 +452,10 @@ def logs(self, services: Optional[List[str]] = None, num_lines: Optional[int] = (otherwise all) num_lines (int): (optional) integer number of log lines to return. Default `10` """ - args = ["logs", "-n={}".format(num_lines)] if num_lines else ["logs"] + args = ["logs", f"-n={num_lines}"] if num_lines else ["logs"] return self._snap_daemons(args, services).stdout - def connect( - self, plug: str, service: Optional[str] = None, slot: Optional[str] = None - ) -> None: + def connect(self, plug: str, service: str | None = None, slot: str | None = None) -> None: """Connect a plug to a slot. Args: @@ -397,20 +466,20 @@ def connect( Raises: SnapError if there is a problem encountered """ - command = ["connect", "{}:{}".format(self._name, plug)] + command = ["connect", f"{self._name}:{plug}"] if service and slot: - command = command + ["{}:{}".format(service, slot)] + command.append(f"{service}:{slot}") elif slot: - command = command + [slot] + command.append(slot) args = ["snap", *command] try: - subprocess.run(args, universal_newlines=True, check=True, capture_output=True) + subprocess.run(args, text=True, check=True, capture_output=True) except CalledProcessError as e: - raise SnapError("Could not {} for snap [{}]: {}".format(args, self._name, e.stderr)) + raise SnapError(f"Could not {args} for snap [{self._name}]: {e.stderr}") from e - def hold(self, duration: Optional[timedelta] = None) -> None: + def hold(self, duration: timedelta | None = None) -> None: """Add a refresh hold to a snap. Args: @@ -426,7 +495,7 @@ def unhold(self) -> None: """Remove the refresh hold of a snap.""" self._snap("refresh", ["--unhold"]) - def alias(self, application: str, alias: Optional[str] = None) -> None: + def alias(self, application: str, alias: str | None = None) -> None: """Create an alias for a given application. Args: @@ -437,17 +506,13 @@ def alias(self, application: str, alias: Optional[str] = None) -> None: alias = application args = ["snap", "alias", f"{self.name}.{application}", alias] try: - subprocess.check_output(args, universal_newlines=True) + subprocess.check_output(args, text=True) except CalledProcessError as e: raise SnapError( - "Snap: {!r}; command {!r} failed with output = {!r}".format( - self._name, args, e.output - ) - ) + f"Snap: {self._name!r}; command {args!r} failed with output = {e.output!r}" + ) from e - def restart( - self, services: Optional[List[str]] = None, reload: Optional[bool] = False - ) -> None: + def restart(self, services: list[str] | None = None, reload: bool = False) -> None: """Restarts a snap's services. Args: @@ -461,9 +526,9 @@ def restart( def _install( self, - channel: Optional[str] = "", - cohort: Optional[str] = "", - revision: Optional[str] = None, + channel: str = "", + cohort: str = "", + revision: str = "", ) -> None: """Add a snap to the system. @@ -474,27 +539,27 @@ def _install( """ cohort = cohort or self._cohort - args = [] + args: list[str] = [] if self.confinement == "classic": args.append("--classic") if self.confinement == "devmode": args.append("--devmode") if channel: - args.append('--channel="{}"'.format(channel)) + args.append(f'--channel="{channel}"') if revision: - args.append('--revision="{}"'.format(revision)) + args.append(f'--revision="{revision}"') if cohort: - args.append('--cohort="{}"'.format(cohort)) + args.append(f'--cohort="{cohort}"') self._snap("install", args) def _refresh( self, - channel: Optional[str] = "", - cohort: Optional[str] = "", - revision: Optional[str] = None, + channel: str = "", + cohort: str = "", + revision: str = "", devmode: bool = False, - leave_cohort: Optional[bool] = False, + leave_cohort: bool = False, ) -> None: """Refresh a snap. @@ -505,12 +570,12 @@ def _refresh( devmode: optionally, specify devmode confinement leave_cohort: leave the current cohort. """ - args = [] + args: list[str] = [] if channel: - args.append('--channel="{}"'.format(channel)) + args.append(f'--channel="{channel}"') if revision: - args.append('--revision="{}"'.format(revision)) + args.append(f'--revision="{revision}"') if devmode: args.append("--devmode") @@ -522,7 +587,7 @@ def _refresh( self._cohort = "" args.append("--leave-cohort") elif cohort: - args.append('--cohort="{}"'.format(cohort)) + args.append(f'--cohort="{cohort}"') self._snap("refresh", args) @@ -538,11 +603,11 @@ def name(self) -> str: def ensure( self, state: SnapState, - classic: Optional[bool] = False, + classic: bool = False, devmode: bool = False, - channel: Optional[str] = "", - cohort: Optional[str] = "", - revision: Optional[str] = None, + channel: str | None = None, + cohort: str | None = None, + revision: str | None = None, ): """Ensure that a snap is in a given state. @@ -560,6 +625,10 @@ def ensure( Raises: SnapError if an error is encountered """ + channel = channel or "" + cohort = cohort or "" + revision = revision or "" + if classic and devmode: raise ValueError("Cannot set both classic and devmode confinement") @@ -605,7 +674,7 @@ def _update_snap_apps(self) -> None: try: self._apps = self._snap_client.get_installed_snap_apps(self._name) except SnapAPIError: - logger.debug("Unable to retrieve snap apps for {}".format(self._name)) + logger.debug("Unable to retrieve snap apps for %s", self._name) self._apps = [] @property @@ -653,18 +722,19 @@ def confinement(self) -> str: return self._confinement @property - def apps(self) -> List: + def apps(self) -> list[dict[str, JSONType]]: """Returns (if any) the installed apps of the snap.""" self._update_snap_apps() return self._apps @property - def services(self) -> Dict: + def services(self) -> dict[str, SnapServiceDict]: """Returns (if any) the installed services of the snap.""" self._update_snap_apps() - services = {} + services: dict[str, SnapServiceDict] = {} for app in self._apps: if "daemon" in app: + app = typing.cast("_SnapServiceAppDict", app) services[app["name"]] = SnapService(**app).as_dict() return services @@ -679,7 +749,7 @@ def held(self) -> bool: class _UnixSocketConnection(http.client.HTTPConnection): """Implementation of HTTPConnection that connects to a named Unix socket.""" - def __init__(self, host, timeout=None, socket_path=None): + def __init__(self, host: str, timeout: float | None = None, socket_path: str | None = None): if timeout is None: super().__init__(host) else: @@ -689,7 +759,8 @@ def __init__(self, host, timeout=None, socket_path=None): def connect(self): """Override connect to use Unix socket (instead of TCP socket).""" if not hasattr(socket, "AF_UNIX"): - raise NotImplementedError("Unix sockets not supported on {}".format(sys.platform)) + raise NotImplementedError(f"Unix sockets not supported on {sys.platform}") + assert self.socket_path is not None # else TypeError on self.socket.connect self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) self.sock.connect(self.socket_path) if self.timeout is not None: @@ -703,9 +774,13 @@ def __init__(self, socket_path: str): super().__init__() self.socket_path = socket_path - def http_open(self, req) -> http.client.HTTPResponse: + def http_open(self, req: urllib.request.Request) -> http.client.HTTPResponse: """Override http_open to use a Unix socket connection (instead of TCP).""" - return self.do_open(_UnixSocketConnection, req, socket_path=self.socket_path) + return self.do_open( + typing.cast("urllib.request._HTTPConnectionProtocol", _UnixSocketConnection), + req, + socket_path=self.socket_path, + ) class SnapClient: @@ -719,7 +794,7 @@ class SnapClient: def __init__( self, socket_path: str = "/run/snapd.socket", - opener: Optional[urllib.request.OpenerDirector] = None, + opener: urllib.request.OpenerDirector | None = None, base_url: str = "http://localhost/v2/", timeout: float = 30.0, ): @@ -728,18 +803,21 @@ def __init__( Args: socket_path: a path to the socket on the filesystem. Defaults to /run/snap/snapd.socket opener: specifies an opener for unix socket, if unspecified a default is used - base_url: base url for making requests to the snap client. Defaults to - http://localhost/v2/ + base_url: base URL for making requests to the snap client. Must be an HTTP(S) URL. + Defaults to http://localhost/v2/ timeout: timeout in seconds to use when making requests to the API. Default is 30.0s. """ if opener is None: opener = self._get_default_opener(socket_path) self.opener = opener + # Address ruff's suspicious-url-open-usage (S310) + if not base_url.startswith(("http:", "https:")): + raise ValueError("base_url must start with 'http:' or 'https:'") self.base_url = base_url self.timeout = timeout @classmethod - def _get_default_opener(cls, socket_path): + def _get_default_opener(cls, socket_path: str) -> urllib.request.OpenerDirector: """Build the default opener to use for requests (HTTP over Unix socket).""" opener = urllib.request.OpenerDirector() opener.add_handler(_UnixSocketHandler(socket_path)) @@ -752,9 +830,9 @@ def _request( self, method: str, path: str, - query: Dict = None, - body: Dict = None, - ) -> JSONType: + query: dict[str, str] | None = None, + body: dict[str, JSONAble] | None = None, + ) -> JSONType | None: """Make a JSON request to the Snapd server with the given HTTP method and path. If query dict is provided, it is encoded and appended as a query string @@ -769,12 +847,12 @@ def _request( headers["Content-Type"] = "application/json" response = self._request_raw(method, path, query, headers, data) - response = json.loads(response.read().decode()) + response = json.loads(response.read().decode()) # json.loads -> Any if response["type"] == "async": - return self._wait(response["change"]) + return self._wait(response["change"]) # may be `None` due to `get` return response["result"] - def _wait(self, change_id: str, timeout=300) -> JSONType: + def _wait(self, change_id: str, timeout: float = 300) -> JSONType | None: """Wait for an async change to complete. The poll time is 100 milliseconds, the same as in snap clients. @@ -784,6 +862,7 @@ def _wait(self, change_id: str, timeout=300) -> JSONType: if time.time() > deadline: raise TimeoutError(f"timeout waiting for snap change {change_id}") response = self._request("GET", f"changes/{change_id}") + response = typing.cast("_AsyncChangeDict", response) status = response["status"] if status == "Done": return response.get("data") @@ -801,9 +880,9 @@ def _request_raw( self, method: str, path: str, - query: Dict = None, - headers: Dict = None, - data: bytes = None, + query: dict[str, str] | None = None, + headers: dict[str, str] | None = None, + data: bytes | None = None, ) -> http.client.HTTPResponse: """Make a request to the Snapd server; return the raw HTTPResponse object.""" url = self.base_url + path @@ -812,7 +891,7 @@ def _request_raw( if headers is None: headers = {} - request = urllib.request.Request(url, method=method, data=data, headers=headers) + request = urllib.request.Request(url, method=method, data=data, headers=headers) # noqa: S310 try: response = self.opener.open(request, timeout=self.timeout) @@ -820,35 +899,36 @@ def _request_raw( code = e.code status = e.reason message = "" + body: dict[str, JSONType] try: - body = json.loads(e.read().decode())["result"] - except (IOError, ValueError, KeyError) as e2: + body = json.loads(e.read().decode())["result"] # json.loads -> Any + except (OSError, ValueError, KeyError) as e2: # Will only happen on read error or if Pebble sends invalid JSON. body = {} - message = "{} - {}".format(type(e2).__name__, e2) - raise SnapAPIError(body, code, status, message) + message = f"{type(e2).__name__} - {e2}" + raise SnapAPIError(body, code, status, message) from e except urllib.error.URLError as e: - raise SnapAPIError({}, 500, "Not found", e.reason) + raise SnapAPIError({}, 500, "Not found", str(e.reason)) from e return response - def get_installed_snaps(self) -> Dict: + def get_installed_snaps(self) -> list[dict[str, JSONType]]: """Get information about currently installed snaps.""" - return self._request("GET", "snaps") + return self._request("GET", "snaps") # type: ignore - def get_snap_information(self, name: str) -> Dict: + def get_snap_information(self, name: str) -> dict[str, JSONType]: """Query the snap server for information about single snap.""" - return self._request("GET", "find", {"name": name})[0] + return self._request("GET", "find", {"name": name})[0] # type: ignore - def get_installed_snap_apps(self, name: str) -> List: + def get_installed_snap_apps(self, name: str) -> list[dict[str, JSONType]]: """Query the snap server for apps belonging to a named, currently installed snap.""" - return self._request("GET", "apps", {"names": name, "select": "service"}) + return self._request("GET", "apps", {"names": name, "select": "service"}) # type: ignore - def _put_snap_conf(self, name: str, conf: Dict[str, Any]): + def _put_snap_conf(self, name: str, conf: dict[str, JSONAble]) -> None: """Set the configuration details for an installed snap.""" - return self._request("PUT", f"snaps/{name}/conf", body=conf) + self._request("PUT", f"snaps/{name}/conf", body=conf) -class SnapCache(Mapping): +class SnapCache(Mapping[str, Snap]): """An abstraction to represent installed/available packages. When instantiated, `SnapCache` iterates through the list of installed @@ -861,12 +941,12 @@ def __init__(self): if not self.snapd_installed: raise SnapError("snapd is not installed or not in /usr/bin") from None self._snap_client = SnapClient() - self._snap_map = {} + self._snap_map: dict[str, Snap | None] = {} if self.snapd_installed: self._load_available_snaps() self._load_installed_snaps() - def __contains__(self, key: str) -> bool: + def __contains__(self, key: object) -> bool: """Check if a given snap is in the cache.""" return key in self._snap_map @@ -874,26 +954,26 @@ def __len__(self) -> int: """Report number of items in the snap cache.""" return len(self._snap_map) - def __iter__(self) -> Iterable["Snap"]: + def __iter__(self) -> Iterable[Snap | None]: # pyright: ignore[reportIncompatibleMethodOverride] """Provide iterator for the snap cache.""" return iter(self._snap_map.values()) def __getitem__(self, snap_name: str) -> Snap: """Return either the installed version or latest version for a given snap.""" - snap = self._snap_map.get(snap_name, None) - if snap is None: - # The snapd cache file may not have existed when _snap_map was - # populated. This is normal. - try: - self._snap_map[snap_name] = self._load_info(snap_name) - except SnapAPIError: - raise SnapNotFoundError("Snap '{}' not found!".format(snap_name)) - - return self._snap_map[snap_name] + snap = self._snap_map.get(snap_name) + if snap is not None: + return snap + # The snapd cache file may not have existed when _snap_map was + # populated. This is normal. + try: + snap = self._snap_map[snap_name] = self._load_info(snap_name) + except SnapAPIError as e: + raise SnapNotFoundError(f"Snap '{snap_name}' not found!") from e + return snap @property def snapd_installed(self) -> bool: - """Check whether snapd has been installled on the system.""" + """Check whether snapd has been installed on the system.""" return os.path.isfile("/usr/bin/snap") def _load_available_snaps(self) -> None: @@ -907,7 +987,7 @@ def _load_available_snaps(self) -> None: # currently exist. return - with open("/var/cache/snapd/names", "r") as f: + with open("/var/cache/snapd/names") as f: for line in f: if line.strip(): self._snap_map[line.strip()] = None @@ -917,23 +997,25 @@ def _load_installed_snaps(self) -> None: installed = self._snap_client.get_installed_snaps() for i in installed: + i = typing.cast("_SnapDict", i) snap = Snap( name=i["name"], state=SnapState.Latest, channel=i["channel"], revision=i["revision"], confinement=i["confinement"], - apps=i.get("apps", None), + apps=i.get("apps"), ) self._snap_map[snap.name] = snap - def _load_info(self, name) -> Snap: + def _load_info(self, name: str) -> Snap: """Load info for snaps which are not installed if requested. Args: name: a string representing the name of the snap """ info = self._snap_client.get_snap_information(name) + info = typing.cast("_SnapDict", info) return Snap( name=info["name"], @@ -945,16 +1027,36 @@ def _load_info(self, name) -> Snap: ) +@typing.overload +def add( # return a single Snap if snap name is given as a string + snap_names: str, + state: str | SnapState = SnapState.Latest, + channel: str | None = None, + classic: bool = False, + devmode: bool = False, + cohort: str | None = None, + revision: str | None = None, +) -> Snap: ... +@typing.overload +def add( # may return a single Snap or a list depending if one or more snap names were given + snap_names: list[str], + state: str | SnapState = SnapState.Latest, + channel: str | None = None, + classic: bool = False, + devmode: bool = False, + cohort: str | None = None, + revision: str | None = None, +) -> Snap | list[Snap]: ... @_cache_init def add( - snap_names: Union[str, List[str]], - state: Union[str, SnapState] = SnapState.Latest, - channel: Optional[str] = "", - classic: Optional[bool] = False, + snap_names: str | list[str], + state: str | SnapState = SnapState.Latest, + channel: str | None = None, + classic: bool = False, devmode: bool = False, - cohort: Optional[str] = "", - revision: Optional[str] = None, -) -> Union[Snap, List[Snap]]: + cohort: str | None = None, + revision: str | None = None, +) -> Snap | list[Snap]: """Add a snap to the system. Args: @@ -982,11 +1084,25 @@ def add( if isinstance(state, str): state = SnapState(state) - return _wrap_snap_operations(snap_names, state, channel, classic, devmode, cohort, revision) + return _wrap_snap_operations( + snap_names=snap_names, + state=state, + channel=channel or "", + classic=classic, + devmode=devmode, + cohort=cohort or "", + revision=revision or "", + ) +@typing.overload +def remove(snap_names: str) -> Snap: ... +# return a single Snap if snap name is given as a string +@typing.overload +def remove(snap_names: list[str]) -> Snap | list[Snap]: ... +# may return a single Snap or a list depending if one or more snap names were given @_cache_init -def remove(snap_names: Union[str, List[str]]) -> Union[Snap, List[Snap]]: +def remove(snap_names: str | list[str]) -> Snap | list[Snap]: """Remove specified snap(s) from the system. Args: @@ -1007,16 +1123,36 @@ def remove(snap_names: Union[str, List[str]]) -> Union[Snap, List[Snap]]: ) +@typing.overload +def ensure( # return a single Snap if snap name is given as a string + snap_names: str, + state: str, + channel: str | None = None, + classic: bool = False, + devmode: bool = False, + cohort: str | None = None, + revision: int | None = None, +) -> Snap: ... +@typing.overload +def ensure( # may return a single Snap or a list depending if one or more snap names were given + snap_names: list[str], + state: str, + channel: str | None = None, + classic: bool = False, + devmode: bool = False, + cohort: str | None = None, + revision: int | None = None, +) -> Snap | list[Snap]: ... @_cache_init def ensure( - snap_names: Union[str, List[str]], + snap_names: str | list[str], state: str, - channel: Optional[str] = "", - classic: Optional[bool] = False, + channel: str | None = None, + classic: bool = False, devmode: bool = False, - cohort: Optional[str] = "", - revision: Optional[int] = None, -) -> Union[Snap, List[Snap]]: + cohort: str | None = None, + revision: int | None = None, +) -> Snap | list[Snap]: """Ensure specified snaps are in a given state on the system. Args: @@ -1047,23 +1183,24 @@ def ensure( classic=classic, devmode=devmode, cohort=cohort, - revision=revision, + revision=str(revision) if revision is not None else None, ) else: return remove(snap_names) def _wrap_snap_operations( - snap_names: List[str], + snap_names: list[str], state: SnapState, channel: str, classic: bool, devmode: bool, - cohort: Optional[str] = "", - revision: Optional[str] = None, -) -> Union[Snap, List[Snap]]: + cohort: str = "", + revision: str = "", +) -> Snap | list[Snap]: """Wrap common operations for bare commands.""" - snaps = {"success": [], "failed": []} + snaps: list[Snap] = [] + errors: list[str] = [] op = "remove" if state is SnapState.Absent else "install or refresh" @@ -1081,27 +1218,25 @@ def _wrap_snap_operations( cohort=cohort, revision=revision, ) - snaps["success"].append(snap) - except SnapError as e: - logger.warning("Failed to {} snap {}: {}!".format(op, s, e.message)) - snaps["failed"].append(s) + snaps.append(snap) + except SnapError as e: # noqa: PERF203 + logger.warning("Failed to %s snap %s: %s!", op, s, e.message) + errors.append(s) except SnapNotFoundError: - logger.warning("Snap '{}' not found in cache!".format(s)) - snaps["failed"].append(s) + logger.warning("Snap '%s' not found in cache!", s) + errors.append(s) - if len(snaps["failed"]): - raise SnapError( - "Failed to install or refresh snap(s): {}".format(", ".join(list(snaps["failed"]))) - ) + if errors: + raise SnapError(f"Failed to install or refresh snap(s): {', '.join(errors)}") - return snaps["success"] if len(snaps["success"]) > 1 else snaps["success"][0] + return snaps if len(snaps) > 1 else snaps[0] def install_local( filename: str, - classic: Optional[bool] = False, - devmode: Optional[bool] = False, - dangerous: Optional[bool] = False, + classic: bool = False, + devmode: bool = False, + dangerous: bool = False, ) -> Snap: """Perform a snap operation. @@ -1126,7 +1261,7 @@ def install_local( if dangerous: args.append("--dangerous") try: - result = subprocess.check_output(args, universal_newlines=True).splitlines()[-1] + result = subprocess.check_output(args, text=True).splitlines()[-1] snap_name, _ = result.split(" ", 1) snap_name = ansi_filter.sub("", snap_name) @@ -1136,11 +1271,13 @@ def install_local( return c[snap_name] except SnapAPIError as e: logger.error( - "Could not find snap {} when querying Snapd socket: {}".format(snap_name, e.body) + "Could not find snap %s when querying Snapd socket: %s", + snap_name, + e.body, ) - raise SnapError("Failed to find snap {} in Snap cache".format(snap_name)) + raise SnapError(f"Failed to find snap {snap_name} in Snap cache") from e except CalledProcessError as e: - raise SnapError("Could not install snap {}: {}".format(filename, e.output)) + raise SnapError(f"Could not install snap {filename}: {e.output}") from e def _system_set(config_item: str, value: str) -> None: @@ -1150,14 +1287,14 @@ def _system_set(config_item: str, value: str) -> None: config_item: name of snap system setting. E.g. 'refresh.hold' value: value to assign """ - args = ["snap", "set", "system", "{}={}".format(config_item, value)] + args = ["snap", "set", "system", f"{config_item}={value}"] try: - subprocess.check_call(args, universal_newlines=True) - except CalledProcessError: - raise SnapError("Failed setting system config '{}' to '{}'".format(config_item, value)) + subprocess.check_call(args, text=True) + except CalledProcessError as e: + raise SnapError(f"Failed setting system config '{config_item}' to '{value}'") from e -def hold_refresh(days: int = 90, forever: bool = False) -> bool: +def hold_refresh(days: int = 90, forever: bool = False) -> None: """Set the system-wide snap refresh hold. Args: @@ -1183,7 +1320,7 @@ def hold_refresh(days: int = 90, forever: bool = False) -> bool: # Format for the correct datetime format hold_date = target_date.strftime("%Y-%m-%dT%H:%M:%S%z") # Python dumps the offset in format '+0100', we need '+01:00' - hold_date = "{0}:{1}".format(hold_date[:-2], hold_date[-2:]) + hold_date = f"{hold_date[:-2]}:{hold_date[-2:]}" # Actually set the hold date _system_set("refresh.hold", hold_date) logger.info("Set system-wide snap refresh hold to: %s", hold_date) diff --git a/lib/charms/postgresql_k8s/v0/postgresql.py b/lib/charms/postgresql_k8s/v0/postgresql.py index 986ae71f0d..8e2b7072ad 100644 --- a/lib/charms/postgresql_k8s/v0/postgresql.py +++ b/lib/charms/postgresql_k8s/v0/postgresql.py @@ -21,7 +21,7 @@ import logging from collections import OrderedDict -from typing import List, Optional, Set, Tuple +from typing import Dict, List, Optional, Set, Tuple import psycopg2 from ops.model import Relation @@ -35,7 +35,10 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 41 +LIBPATCH = 45 + +# Groups to distinguish database permissions +PERMISSIONS_GROUP_ADMIN = "admin" INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles" @@ -187,7 +190,7 @@ def create_database( Identifier(database) ) ) - for user_to_grant_access in [user, "admin", *self.system_users]: + for user_to_grant_access in [user, PERMISSIONS_GROUP_ADMIN, *self.system_users]: cursor.execute( SQL("GRANT ALL PRIVILEGES ON DATABASE {} TO {};").format( Identifier(database), Identifier(user_to_grant_access) @@ -220,7 +223,7 @@ def create_user( user: str, password: Optional[str] = None, admin: bool = False, - extra_user_roles: Optional[str] = None, + extra_user_roles: Optional[List[str]] = None, ) -> None: """Creates a database user. @@ -235,16 +238,17 @@ def create_user( admin_role = False roles = privileges = None if extra_user_roles: - extra_user_roles = tuple(extra_user_roles.lower().split(",")) - admin_role = "admin" in extra_user_roles + admin_role = PERMISSIONS_GROUP_ADMIN in extra_user_roles valid_privileges, valid_roles = self.list_valid_privileges_and_roles() roles = [ - role for role in extra_user_roles if role in valid_roles and role != "admin" + role + for role in extra_user_roles + if role in valid_roles and role != PERMISSIONS_GROUP_ADMIN ] privileges = { extra_user_role for extra_user_role in extra_user_roles - if extra_user_role not in roles and extra_user_role != "admin" + if extra_user_role not in roles and extra_user_role != PERMISSIONS_GROUP_ADMIN } invalid_privileges = [ privilege for privilege in privileges if privilege not in valid_privileges @@ -318,7 +322,7 @@ def delete_user(self, user: str) -> None: raise PostgreSQLDeleteUserError() from e def enable_disable_extensions( - self, extensions: dict[str, bool], database: Optional[str] = None + self, extensions: Dict[str, bool], database: Optional[str] = None ) -> None: """Enables or disables a PostgreSQL extension. @@ -566,8 +570,8 @@ def set_up_database(self) -> None: ) ) self.create_user( - "admin", - extra_user_roles="pg_read_all_data,pg_write_all_data", + PERMISSIONS_GROUP_ADMIN, + extra_user_roles=["pg_read_all_data", "pg_write_all_data"], ) cursor.execute("GRANT CONNECT ON DATABASE postgres TO admin;") except psycopg2.Error as e: diff --git a/lib/charms/rolling_ops/v0/rollingops.py b/lib/charms/rolling_ops/v0/rollingops.py index 57aa9bf352..13b51a3051 100644 --- a/lib/charms/rolling_ops/v0/rollingops.py +++ b/lib/charms/rolling_ops/v0/rollingops.py @@ -63,13 +63,14 @@ def _on_trigger_restart(self, event): juju run-action some-charm/0 some-charm/1 <... some-charm/n> restart ``` -Note that all units that plan to restart must receive the action and emit the aquire +Note that all units that plan to restart must receive the action and emit the acquire event. Any units that do not run their acquire handler will be left out of the rolling restart. (An operator might take advantage of this fact to recover from a failed rolling operation without restarting workloads that were able to successfully restart -- simply omit the successful units from a subsequent run-action call.) """ + import logging from enum import Enum from typing import AnyStr, Callable, Optional @@ -88,7 +89,7 @@ def _on_trigger_restart(self, event): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 7 +LIBPATCH = 8 class LockNoRelationError(Exception): @@ -149,7 +150,6 @@ class Lock: """ def __init__(self, manager, unit=None): - self.relation = manager.model.relations[manager.name][0] if not self.relation: # TODO: defer caller in this case (probably just fired too soon). @@ -246,7 +246,7 @@ def __init__(self, manager): # Gather all the units. relation = manager.model.relations[manager.name][0] - units = [unit for unit in relation.units] + units = list(relation.units) # Plus our unit ... units.append(manager.model.unit) diff --git a/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py b/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py index cf8def11ac..e2208f756f 100644 --- a/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py +++ b/lib/charms/tempo_coordinator_k8s/v0/charm_tracing.py @@ -10,24 +10,28 @@ in real time from the Grafana dashboard the execution flow of your charm. # Quickstart -Fetch the following charm libs (and ensure the minimum version/revision numbers are satisfied): +Fetch the following charm libs: - charmcraft fetch-lib charms.tempo_coordinator_k8s.v0.tracing # >= 1.10 - charmcraft fetch-lib charms.tempo_coordinator_k8s.v0.charm_tracing # >= 2.7 + charmcraft fetch-lib charms.tempo_coordinator_k8s.v0.tracing + charmcraft fetch-lib charms.tempo_coordinator_k8s.v0.charm_tracing Then edit your charm code to include: ```python # import the necessary charm libs -from charms.tempo_coordinator_k8s.v0.tracing import TracingEndpointRequirer, charm_tracing_config +from charms.tempo_coordinator_k8s.v0.tracing import ( + TracingEndpointRequirer, + charm_tracing_config, +) from charms.tempo_coordinator_k8s.v0.charm_tracing import charm_tracing + # decorate your charm class with charm_tracing: @charm_tracing( # forward-declare the instance attributes that the instrumentor will look up to obtain the # tempo endpoint and server certificate tracing_endpoint="tracing_endpoint", - server_cert="server_cert" + server_cert="server_cert", ) class MyCharm(CharmBase): _path_to_cert = "/path/to/cert.crt" @@ -37,10 +41,12 @@ class MyCharm(CharmBase): # If you do support TLS, you'll need to make sure that the server cert is copied to this location # and kept up to date so the instrumentor can use it. - def __init__(self, ...): - ... - self.tracing = TracingEndpointRequirer(self, ...) - self.tracing_endpoint, self.server_cert = charm_tracing_config(self.tracing, self._path_to_cert) + def __init__(self, framework): + # ... + self.tracing = TracingEndpointRequirer(self) + self.tracing_endpoint, self.server_cert = charm_tracing_config( + self.tracing, self._path_to_cert + ) ``` # Detailed usage @@ -168,9 +174,10 @@ class MyCharm(CharmBase): ... ``` -## Upgrading from `v0` +## Upgrading from `tempo_k8s.v0` -If you are upgrading from `charm_tracing` v0, you need to take the following steps (assuming you already +If you are upgrading from `tempo_k8s.v0.charm_tracing` (note that since then, the charm library moved to +`tempo_coordinator_k8s.v0.charm_tracing`), you need to take the following steps (assuming you already have the newest version of the library in your charm): 1) If you need the dependency for your tests, add the following dependency to your charm project (or, if your project had a dependency on `opentelemetry-exporter-otlp-proto-grpc` only because @@ -183,7 +190,7 @@ class MyCharm(CharmBase): For example: ``` - from charms.tempo_coordinator_k8s.v0.charm_tracing import trace_charm + from charms.tempo_k8s.v0.charm_tracing import trace_charm @trace_charm( tracing_endpoint="my_tracing_endpoint", @@ -225,12 +232,6 @@ def my_tracing_endpoint(self) -> Optional[str]: 3) If you were passing a certificate (str) using `server_cert`, you need to change it to provide an *absolute* path to the certificate file instead. """ -import typing - -from opentelemetry.exporter.otlp.proto.common._internal.trace_encoder import ( - encode_spans, -) -from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter def _remove_stale_otel_sdk_packages(): @@ -285,12 +286,15 @@ def _remove_stale_otel_sdk_packages(): # apply hacky patch to remove stale opentelemetry sdk packages on upgrade-charm. # it could be trouble if someone ever decides to implement their own tracer parallel to # ours and before the charm has inited. We assume they won't. +# !!IMPORTANT!! keep all otlp imports UNDER this call. _remove_stale_otel_sdk_packages() import functools import inspect import logging import os +import typing +from collections import deque from contextlib import contextmanager from contextvars import Context, ContextVar, copy_context from pathlib import Path @@ -309,6 +313,9 @@ def _remove_stale_otel_sdk_packages(): import opentelemetry import ops +from opentelemetry.exporter.otlp.proto.common._internal.trace_encoder import ( + encode_spans, +) from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, Span, TracerProvider @@ -317,7 +324,11 @@ def _remove_stale_otel_sdk_packages(): SpanExporter, SpanExportResult, ) -from opentelemetry.trace import INVALID_SPAN, Tracer +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import ( + INVALID_SPAN, + Tracer, +) from opentelemetry.trace import get_current_span as otlp_get_current_span from opentelemetry.trace import ( get_tracer, @@ -337,7 +348,7 @@ def _remove_stale_otel_sdk_packages(): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 4 +LIBPATCH = 7 PYDEPS = ["opentelemetry-exporter-otlp-proto-http==1.21.0"] @@ -365,7 +376,9 @@ def _remove_stale_otel_sdk_packages(): BUFFER_DEFAULT_MAX_EVENT_HISTORY_LENGTH = 100 _MiB_TO_B = 2**20 # megabyte to byte conversion rate _OTLP_SPAN_EXPORTER_TIMEOUT = 1 -"""Timeout in seconds that the OTLP span exporter has to push traces to the backend.""" + + +# Timeout in seconds that the OTLP span exporter has to push traces to the backend. class _Buffer: @@ -397,45 +410,75 @@ def save(self, spans: typing.Sequence[ReadableSpan]): if self._max_event_history_length < 1: dev_logger.debug("buffer disabled: max history length < 1") return - - current_history_length = len(self.load()) - new_history_length = current_history_length + len(spans) - if (diff := self._max_event_history_length - new_history_length) < 0: - self.drop(diff) self._save(spans) def _serialize(self, spans: Sequence[ReadableSpan]) -> bytes: # encode because otherwise we can't json-dump them return encode_spans(spans).SerializeToString() + def _prune(self, queue: Sequence[bytes]) -> Sequence[bytes]: + """Prune the queue until it fits in our constraints.""" + n_dropped_spans = 0 + # drop older events if we are past the max history length + overflow = len(queue) - self._max_event_history_length + if overflow > 0: + n_dropped_spans += overflow + logger.warning( + f"charm tracing buffer exceeds max history length ({self._max_event_history_length} events)" + ) + + new_spans = deque(queue[-self._max_event_history_length :]) + + # drop older events if the buffer is too big; all units are bytes + logged_drop = False + target_size = self._max_buffer_size_mib * _MiB_TO_B + current_size = sum(len(span) for span in new_spans) + while current_size > target_size: + current_size -= len(new_spans.popleft()) + n_dropped_spans += 1 + + # only do this once + if not logged_drop: + logger.warning( + f"charm tracing buffer exceeds size limit ({self._max_buffer_size_mib}MiB)." + ) + logged_drop = True + + if n_dropped_spans > 0: + dev_logger.debug( + f"charm tracing buffer overflow: dropped {n_dropped_spans} older spans. " + f"Please increase the buffer limits, or ensure the spans can be flushed." + ) + return new_spans + def _save(self, spans: Sequence[ReadableSpan], replace: bool = False): dev_logger.debug(f"saving {len(spans)} new spans to buffer") old = [] if replace else self.load() - new = self._serialize(spans) + queue = old + [self._serialize(spans)] + new_buffer = self._prune(queue) - try: - # if the buffer exceeds the size limit, we start dropping old spans until it does - - while len((new + self._SPANSEP.join(old))) > (self._max_buffer_size_mib * _MiB_TO_B): - if not old: - # if we've already dropped all spans and still we can't get under the - # size limit, we can't save this span - logger.error( - f"span exceeds total buffer size limit ({self._max_buffer_size_mib}MiB); " - f"buffering FAILED" - ) - return - - old = old[1:] - logger.warning( - f"buffer size exceeds {self._max_buffer_size_mib}MiB; dropping older spans... " - f"Please increase the buffer size, disable buffering, or ensure the spans can be flushed." - ) + if queue and not new_buffer: + # this means that, given our constraints, we are pruning so much that there are no events left. + logger.error( + "No charm events could be buffered into charm traces buffer. Please increase the memory or history size limits." + ) + return - self._db_file.write_bytes(new + self._SPANSEP.join(old)) + try: + self._write(new_buffer) except Exception: logger.exception("error buffering spans") + def _write(self, spans: Sequence[bytes]): + """Write the spans to the db file.""" + # ensure the destination folder exists + db_file_dir = self._db_file.parent + if not db_file_dir.exists(): + dev_logger.info(f"creating buffer dir: {db_file_dir}") + db_file_dir.mkdir(parents=True) + + self._db_file.write_bytes(self._SPANSEP.join(spans)) + def load(self) -> List[bytes]: """Load currently buffered spans from the cache file. @@ -460,8 +503,10 @@ def drop(self, n_spans: Optional[int] = None): else: dev_logger.debug("emptying buffer") new = [] - - self._db_file.write_bytes(self._SPANSEP.join(new)) + try: + self._write(new) + except Exception: + logger.exception("error writing charm traces buffer") def flush(self) -> Optional[bool]: """Export all buffered spans to the given exporter, then clear the buffer. diff --git a/metadata.yaml b/metadata.yaml index 94cb47ec89..4ce46fe9c0 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -32,10 +32,6 @@ provides: optional: true database: interface: postgresql_client - db: - interface: pgsql - db-admin: - interface: pgsql cos-agent: interface: cos_agent limit: 1 @@ -66,9 +62,6 @@ storage: assumes: - juju - any-of: - - all-of: - - juju >= 2.9.49 - - juju < 3 - all-of: - juju >= 3.4.3 - juju < 3.5 diff --git a/poetry.lock b/poetry.lock index c3e2e6d8ba..c776be907a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -17,26 +17,21 @@ allure-python-commons = "2.13.5" pytest = ">=4.5.0" [[package]] -name = "allure-pytest-collection-report" -version = "0.1.0" -description = "" +name = "allure-pytest-default-results" +version = "0.1.2" +description = "Generate default \"unknown\" results to show in Allure Report if test case does not run" optional = false python-versions = ">=3.8" groups = ["integration"] -files = [] -develop = false +files = [ + {file = "allure_pytest_default_results-0.1.2-py3-none-any.whl", hash = "sha256:8dc6c5a5d548661c38111a2890509e794204586fa81cefbe61315fb63996e50c"}, + {file = "allure_pytest_default_results-0.1.2.tar.gz", hash = "sha256:eb6c16aa1c2ede69e653a0ee38094791685eaacb0ac6b2cae5c6da1379dbdbfd"}, +] [package.dependencies] allure-pytest = ">=2.13.5" pytest = "*" -[package.source] -type = "git" -url = "https://github.com/canonical/data-platform-workflows" -reference = "v29.1.0" -resolved_reference = "cf3e292107a8d420c452e35cf7552c225add7fbd" -subdirectory = "python/pytest_plugins/allure_pytest_collection_report" - [[package]] name = "allure-python-commons" version = "2.13.5" @@ -178,18 +173,18 @@ typecheck = ["mypy"] [[package]] name = "boto3" -version = "1.35.87" +version = "1.35.99" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" groups = ["main", "integration"] files = [ - {file = "boto3-1.35.87-py3-none-any.whl", hash = "sha256:588ab05e2771c50fca5c242be14e7a25200ffd3dd95c45950ce40993473864c7"}, - {file = "boto3-1.35.87.tar.gz", hash = "sha256:341c58602889078a4a25dc4331b832b5b600a33acd73471d2532c6f01b16fbb4"}, + {file = "boto3-1.35.99-py3-none-any.whl", hash = "sha256:83e560faaec38a956dfb3d62e05e1703ee50432b45b788c09e25107c5058bd71"}, + {file = "boto3-1.35.99.tar.gz", hash = "sha256:e0abd794a7a591d90558e92e29a9f8837d25ece8e3c120e530526fe27eba5fca"}, ] [package.dependencies] -botocore = ">=1.35.87,<1.36.0" +botocore = ">=1.35.99,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -230,14 +225,14 @@ files = [ [[package]] name = "certifi" -version = "2024.12.14" +version = "2025.1.31" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" groups = ["main", "charm-libs", "integration"] files = [ - {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"}, - {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"}, + {file = "certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"}, + {file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"}, ] [[package]] @@ -425,14 +420,14 @@ files = [ [[package]] name = "codespell" -version = "2.4.0" +version = "2.4.1" description = "Fix common misspellings in text files" optional = false python-versions = ">=3.8" groups = ["lint"] files = [ - {file = "codespell-2.4.0-py3-none-any.whl", hash = "sha256:b4c5b779f747dd481587aeecb5773301183f52b94b96ed51a28126d0482eec1d"}, - {file = "codespell-2.4.0.tar.gz", hash = "sha256:587d45b14707fb8ce51339ba4cce50ae0e98ce228ef61f3c5e160e34f681be58"}, + {file = "codespell-2.4.1-py3-none-any.whl", hash = "sha256:3dadafa67df7e4a3dbf51e0d7315061b80d265f9552ebd699b3dd6834b47e425"}, + {file = "codespell-2.4.1.tar.gz", hash = "sha256:299fcdcb09d23e81e35a671bbe746d5ad7e8385972e65dbb833a2eaac33c01e5"}, ] [package.extras] @@ -456,14 +451,14 @@ files = [ [[package]] name = "cosl" -version = "0.0.51" +version = "0.0.55" description = "Utils for COS Lite charms" optional = false python-versions = ">=3.8" groups = ["charm-libs"] files = [ - {file = "cosl-0.0.51-py3-none-any.whl", hash = "sha256:2ef43a94f0ca130fb4f2af924b75329f3c5e74b5c40ad4036af16713ad7d47d4"}, - {file = "cosl-0.0.51.tar.gz", hash = "sha256:32af380475bba32df7334d53ff16fb93466a169c7433e79a9fef8dbbecfdd43c"}, + {file = "cosl-0.0.55-py3-none-any.whl", hash = "sha256:bf641d611f982c8f494f3cf72ac4181b24e30c69504cfbd55aa8f54964797f90"}, + {file = "cosl-0.0.55.tar.gz", hash = "sha256:d3b8ee6f78302ac111d3a15d36c42a38c298a806161d762869513d348d778316"}, ] [package.dependencies] @@ -476,74 +471,75 @@ typing-extensions = "*" [[package]] name = "coverage" -version = "7.6.10" +version = "7.6.12" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.9" groups = ["unit"] files = [ - {file = "coverage-7.6.10-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5c912978f7fbf47ef99cec50c4401340436d200d41d714c7a4766f377c5b7b78"}, - {file = "coverage-7.6.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a01ec4af7dfeb96ff0078ad9a48810bb0cc8abcb0115180c6013a6b26237626c"}, - {file = "coverage-7.6.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3b204c11e2b2d883946fe1d97f89403aa1811df28ce0447439178cc7463448a"}, - {file = "coverage-7.6.10-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32ee6d8491fcfc82652a37109f69dee9a830e9379166cb73c16d8dc5c2915165"}, - {file = "coverage-7.6.10-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675cefc4c06e3b4c876b85bfb7c59c5e2218167bbd4da5075cbe3b5790a28988"}, - {file = "coverage-7.6.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f4f620668dbc6f5e909a0946a877310fb3d57aea8198bde792aae369ee1c23b5"}, - {file = "coverage-7.6.10-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:4eea95ef275de7abaef630c9b2c002ffbc01918b726a39f5a4353916ec72d2f3"}, - {file = "coverage-7.6.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e2f0280519e42b0a17550072861e0bc8a80a0870de260f9796157d3fca2733c5"}, - {file = "coverage-7.6.10-cp310-cp310-win32.whl", hash = "sha256:bc67deb76bc3717f22e765ab3e07ee9c7a5e26b9019ca19a3b063d9f4b874244"}, - {file = "coverage-7.6.10-cp310-cp310-win_amd64.whl", hash = "sha256:0f460286cb94036455e703c66988851d970fdfd8acc2a1122ab7f4f904e4029e"}, - {file = "coverage-7.6.10-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ea3c8f04b3e4af80e17bab607c386a830ffc2fb88a5484e1df756478cf70d1d3"}, - {file = "coverage-7.6.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:507a20fc863cae1d5720797761b42d2d87a04b3e5aeb682ef3b7332e90598f43"}, - {file = "coverage-7.6.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d37a84878285b903c0fe21ac8794c6dab58150e9359f1aaebbeddd6412d53132"}, - {file = "coverage-7.6.10-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a534738b47b0de1995f85f582d983d94031dffb48ab86c95bdf88dc62212142f"}, - {file = "coverage-7.6.10-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d7a2bf79378d8fb8afaa994f91bfd8215134f8631d27eba3e0e2c13546ce994"}, - {file = "coverage-7.6.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6713ba4b4ebc330f3def51df1d5d38fad60b66720948112f114968feb52d3f99"}, - {file = "coverage-7.6.10-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ab32947f481f7e8c763fa2c92fd9f44eeb143e7610c4ca9ecd6a36adab4081bd"}, - {file = "coverage-7.6.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7bbd8c8f1b115b892e34ba66a097b915d3871db7ce0e6b9901f462ff3a975377"}, - {file = "coverage-7.6.10-cp311-cp311-win32.whl", hash = "sha256:299e91b274c5c9cdb64cbdf1b3e4a8fe538a7a86acdd08fae52301b28ba297f8"}, - {file = "coverage-7.6.10-cp311-cp311-win_amd64.whl", hash = "sha256:489a01f94aa581dbd961f306e37d75d4ba16104bbfa2b0edb21d29b73be83609"}, - {file = "coverage-7.6.10-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:27c6e64726b307782fa5cbe531e7647aee385a29b2107cd87ba7c0105a5d3853"}, - {file = "coverage-7.6.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c56e097019e72c373bae32d946ecf9858fda841e48d82df7e81c63ac25554078"}, - {file = "coverage-7.6.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7827a5bc7bdb197b9e066cdf650b2887597ad124dd99777332776f7b7c7d0d0"}, - {file = "coverage-7.6.10-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:204a8238afe787323a8b47d8be4df89772d5c1e4651b9ffa808552bdf20e1d50"}, - {file = "coverage-7.6.10-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e67926f51821b8e9deb6426ff3164870976fe414d033ad90ea75e7ed0c2e5022"}, - {file = "coverage-7.6.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e78b270eadb5702938c3dbe9367f878249b5ef9a2fcc5360ac7bff694310d17b"}, - {file = "coverage-7.6.10-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:714f942b9c15c3a7a5fe6876ce30af831c2ad4ce902410b7466b662358c852c0"}, - {file = "coverage-7.6.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:abb02e2f5a3187b2ac4cd46b8ced85a0858230b577ccb2c62c81482ca7d18852"}, - {file = "coverage-7.6.10-cp312-cp312-win32.whl", hash = "sha256:55b201b97286cf61f5e76063f9e2a1d8d2972fc2fcfd2c1272530172fd28c359"}, - {file = "coverage-7.6.10-cp312-cp312-win_amd64.whl", hash = "sha256:e4ae5ac5e0d1e4edfc9b4b57b4cbecd5bc266a6915c500f358817a8496739247"}, - {file = "coverage-7.6.10-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:05fca8ba6a87aabdd2d30d0b6c838b50510b56cdcfc604d40760dae7153b73d9"}, - {file = "coverage-7.6.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:9e80eba8801c386f72e0712a0453431259c45c3249f0009aff537a517b52942b"}, - {file = "coverage-7.6.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a372c89c939d57abe09e08c0578c1d212e7a678135d53aa16eec4430adc5e690"}, - {file = "coverage-7.6.10-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec22b5e7fe7a0fa8509181c4aac1db48f3dd4d3a566131b313d1efc102892c18"}, - {file = "coverage-7.6.10-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26bcf5c4df41cad1b19c84af71c22cbc9ea9a547fc973f1f2cc9a290002c8b3c"}, - {file = "coverage-7.6.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e4630c26b6084c9b3cb53b15bd488f30ceb50b73c35c5ad7871b869cb7365fd"}, - {file = "coverage-7.6.10-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2396e8116db77789f819d2bc8a7e200232b7a282c66e0ae2d2cd84581a89757e"}, - {file = "coverage-7.6.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:79109c70cc0882e4d2d002fe69a24aa504dec0cc17169b3c7f41a1d341a73694"}, - {file = "coverage-7.6.10-cp313-cp313-win32.whl", hash = "sha256:9e1747bab246d6ff2c4f28b4d186b205adced9f7bd9dc362051cc37c4a0c7bd6"}, - {file = "coverage-7.6.10-cp313-cp313-win_amd64.whl", hash = "sha256:254f1a3b1eef5f7ed23ef265eaa89c65c8c5b6b257327c149db1ca9d4a35f25e"}, - {file = "coverage-7.6.10-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2ccf240eb719789cedbb9fd1338055de2761088202a9a0b73032857e53f612fe"}, - {file = "coverage-7.6.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:0c807ca74d5a5e64427c8805de15b9ca140bba13572d6d74e262f46f50b13273"}, - {file = "coverage-7.6.10-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bcfa46d7709b5a7ffe089075799b902020b62e7ee56ebaed2f4bdac04c508d8"}, - {file = "coverage-7.6.10-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e0de1e902669dccbf80b0415fb6b43d27edca2fbd48c74da378923b05316098"}, - {file = "coverage-7.6.10-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f7b444c42bbc533aaae6b5a2166fd1a797cdb5eb58ee51a92bee1eb94a1e1cb"}, - {file = "coverage-7.6.10-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b330368cb99ef72fcd2dc3ed260adf67b31499584dc8a20225e85bfe6f6cfed0"}, - {file = "coverage-7.6.10-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:9a7cfb50515f87f7ed30bc882f68812fd98bc2852957df69f3003d22a2aa0abf"}, - {file = "coverage-7.6.10-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6f93531882a5f68c28090f901b1d135de61b56331bba82028489bc51bdd818d2"}, - {file = "coverage-7.6.10-cp313-cp313t-win32.whl", hash = "sha256:89d76815a26197c858f53c7f6a656686ec392b25991f9e409bcef020cd532312"}, - {file = "coverage-7.6.10-cp313-cp313t-win_amd64.whl", hash = "sha256:54a5f0f43950a36312155dae55c505a76cd7f2b12d26abeebbe7a0b36dbc868d"}, - {file = "coverage-7.6.10-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:656c82b8a0ead8bba147de9a89bda95064874c91a3ed43a00e687f23cc19d53a"}, - {file = "coverage-7.6.10-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ccc2b70a7ed475c68ceb548bf69cec1e27305c1c2606a5eb7c3afff56a1b3b27"}, - {file = "coverage-7.6.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5e37dc41d57ceba70956fa2fc5b63c26dba863c946ace9705f8eca99daecdc4"}, - {file = "coverage-7.6.10-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0aa9692b4fdd83a4647eeb7db46410ea1322b5ed94cd1715ef09d1d5922ba87f"}, - {file = "coverage-7.6.10-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa744da1820678b475e4ba3dfd994c321c5b13381d1041fe9c608620e6676e25"}, - {file = "coverage-7.6.10-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c0b1818063dc9e9d838c09e3a473c1422f517889436dd980f5d721899e66f315"}, - {file = "coverage-7.6.10-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:59af35558ba08b758aec4d56182b222976330ef8d2feacbb93964f576a7e7a90"}, - {file = "coverage-7.6.10-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7ed2f37cfce1ce101e6dffdfd1c99e729dd2ffc291d02d3e2d0af8b53d13840d"}, - {file = "coverage-7.6.10-cp39-cp39-win32.whl", hash = "sha256:4bcc276261505d82f0ad426870c3b12cb177752834a633e737ec5ee79bbdff18"}, - {file = "coverage-7.6.10-cp39-cp39-win_amd64.whl", hash = "sha256:457574f4599d2b00f7f637a0700a6422243b3565509457b2dbd3f50703e11f59"}, - {file = "coverage-7.6.10-pp39.pp310-none-any.whl", hash = "sha256:fd34e7b3405f0cc7ab03d54a334c17a9e802897580d964bd8c2001f4b9fd488f"}, - {file = "coverage-7.6.10.tar.gz", hash = "sha256:7fb105327c8f8f0682e29843e2ff96af9dcbe5bab8eeb4b398c6a33a16d80a23"}, + {file = "coverage-7.6.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:704c8c8c6ce6569286ae9622e534b4f5b9759b6f2cd643f1c1a61f666d534fe8"}, + {file = "coverage-7.6.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ad7525bf0241e5502168ae9c643a2f6c219fa0a283001cee4cf23a9b7da75879"}, + {file = "coverage-7.6.12-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06097c7abfa611c91edb9e6920264e5be1d6ceb374efb4986f38b09eed4cb2fe"}, + {file = "coverage-7.6.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:220fa6c0ad7d9caef57f2c8771918324563ef0d8272c94974717c3909664e674"}, + {file = "coverage-7.6.12-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3688b99604a24492bcfe1c106278c45586eb819bf66a654d8a9a1433022fb2eb"}, + {file = "coverage-7.6.12-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d1a987778b9c71da2fc8948e6f2656da6ef68f59298b7e9786849634c35d2c3c"}, + {file = "coverage-7.6.12-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:cec6b9ce3bd2b7853d4a4563801292bfee40b030c05a3d29555fd2a8ee9bd68c"}, + {file = "coverage-7.6.12-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ace9048de91293e467b44bce0f0381345078389814ff6e18dbac8fdbf896360e"}, + {file = "coverage-7.6.12-cp310-cp310-win32.whl", hash = "sha256:ea31689f05043d520113e0552f039603c4dd71fa4c287b64cb3606140c66f425"}, + {file = "coverage-7.6.12-cp310-cp310-win_amd64.whl", hash = "sha256:676f92141e3c5492d2a1596d52287d0d963df21bf5e55c8b03075a60e1ddf8aa"}, + {file = "coverage-7.6.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e18aafdfb3e9ec0d261c942d35bd7c28d031c5855dadb491d2723ba54f4c3015"}, + {file = "coverage-7.6.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66fe626fd7aa5982cdebad23e49e78ef7dbb3e3c2a5960a2b53632f1f703ea45"}, + {file = "coverage-7.6.12-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ef01d70198431719af0b1f5dcbefc557d44a190e749004042927b2a3fed0702"}, + {file = "coverage-7.6.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07e92ae5a289a4bc4c0aae710c0948d3c7892e20fd3588224ebe242039573bf0"}, + {file = "coverage-7.6.12-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e695df2c58ce526eeab11a2e915448d3eb76f75dffe338ea613c1201b33bab2f"}, + {file = "coverage-7.6.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d74c08e9aaef995f8c4ef6d202dbd219c318450fe2a76da624f2ebb9c8ec5d9f"}, + {file = "coverage-7.6.12-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e995b3b76ccedc27fe4f477b349b7d64597e53a43fc2961db9d3fbace085d69d"}, + {file = "coverage-7.6.12-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b1f097878d74fe51e1ddd1be62d8e3682748875b461232cf4b52ddc6e6db0bba"}, + {file = "coverage-7.6.12-cp311-cp311-win32.whl", hash = "sha256:1f7ffa05da41754e20512202c866d0ebfc440bba3b0ed15133070e20bf5aeb5f"}, + {file = "coverage-7.6.12-cp311-cp311-win_amd64.whl", hash = "sha256:e216c5c45f89ef8971373fd1c5d8d1164b81f7f5f06bbf23c37e7908d19e8558"}, + {file = "coverage-7.6.12-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b172f8e030e8ef247b3104902cc671e20df80163b60a203653150d2fc204d1ad"}, + {file = "coverage-7.6.12-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:641dfe0ab73deb7069fb972d4d9725bf11c239c309ce694dd50b1473c0f641c3"}, + {file = "coverage-7.6.12-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e549f54ac5f301e8e04c569dfdb907f7be71b06b88b5063ce9d6953d2d58574"}, + {file = "coverage-7.6.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:959244a17184515f8c52dcb65fb662808767c0bd233c1d8a166e7cf74c9ea985"}, + {file = "coverage-7.6.12-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bda1c5f347550c359f841d6614fb8ca42ae5cb0b74d39f8a1e204815ebe25750"}, + {file = "coverage-7.6.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1ceeb90c3eda1f2d8c4c578c14167dbd8c674ecd7d38e45647543f19839dd6ea"}, + {file = "coverage-7.6.12-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f16f44025c06792e0fb09571ae454bcc7a3ec75eeb3c36b025eccf501b1a4c3"}, + {file = "coverage-7.6.12-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b076e625396e787448d27a411aefff867db2bffac8ed04e8f7056b07024eed5a"}, + {file = "coverage-7.6.12-cp312-cp312-win32.whl", hash = "sha256:00b2086892cf06c7c2d74983c9595dc511acca00665480b3ddff749ec4fb2a95"}, + {file = "coverage-7.6.12-cp312-cp312-win_amd64.whl", hash = "sha256:7ae6eabf519bc7871ce117fb18bf14e0e343eeb96c377667e3e5dd12095e0288"}, + {file = "coverage-7.6.12-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:488c27b3db0ebee97a830e6b5a3ea930c4a6e2c07f27a5e67e1b3532e76b9ef1"}, + {file = "coverage-7.6.12-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d1095bbee1851269f79fd8e0c9b5544e4c00c0c24965e66d8cba2eb5bb535fd"}, + {file = "coverage-7.6.12-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0533adc29adf6a69c1baa88c3d7dbcaadcffa21afbed3ca7a225a440e4744bf9"}, + {file = "coverage-7.6.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53c56358d470fa507a2b6e67a68fd002364d23c83741dbc4c2e0680d80ca227e"}, + {file = "coverage-7.6.12-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64cbb1a3027c79ca6310bf101014614f6e6e18c226474606cf725238cf5bc2d4"}, + {file = "coverage-7.6.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:79cac3390bfa9836bb795be377395f28410811c9066bc4eefd8015258a7578c6"}, + {file = "coverage-7.6.12-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:9b148068e881faa26d878ff63e79650e208e95cf1c22bd3f77c3ca7b1d9821a3"}, + {file = "coverage-7.6.12-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8bec2ac5da793c2685ce5319ca9bcf4eee683b8a1679051f8e6ec04c4f2fd7dc"}, + {file = "coverage-7.6.12-cp313-cp313-win32.whl", hash = "sha256:200e10beb6ddd7c3ded322a4186313d5ca9e63e33d8fab4faa67ef46d3460af3"}, + {file = "coverage-7.6.12-cp313-cp313-win_amd64.whl", hash = "sha256:2b996819ced9f7dbb812c701485d58f261bef08f9b85304d41219b1496b591ef"}, + {file = "coverage-7.6.12-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:299cf973a7abff87a30609879c10df0b3bfc33d021e1adabc29138a48888841e"}, + {file = "coverage-7.6.12-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4b467a8c56974bf06e543e69ad803c6865249d7a5ccf6980457ed2bc50312703"}, + {file = "coverage-7.6.12-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2458f275944db8129f95d91aee32c828a408481ecde3b30af31d552c2ce284a0"}, + {file = "coverage-7.6.12-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a9d8be07fb0832636a0f72b80d2a652fe665e80e720301fb22b191c3434d924"}, + {file = "coverage-7.6.12-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14d47376a4f445e9743f6c83291e60adb1b127607a3618e3185bbc8091f0467b"}, + {file = "coverage-7.6.12-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b95574d06aa9d2bd6e5cc35a5bbe35696342c96760b69dc4287dbd5abd4ad51d"}, + {file = "coverage-7.6.12-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:ecea0c38c9079570163d663c0433a9af4094a60aafdca491c6a3d248c7432827"}, + {file = "coverage-7.6.12-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2251fabcfee0a55a8578a9d29cecfee5f2de02f11530e7d5c5a05859aa85aee9"}, + {file = "coverage-7.6.12-cp313-cp313t-win32.whl", hash = "sha256:eb5507795caabd9b2ae3f1adc95f67b1104971c22c624bb354232d65c4fc90b3"}, + {file = "coverage-7.6.12-cp313-cp313t-win_amd64.whl", hash = "sha256:f60a297c3987c6c02ffb29effc70eadcbb412fe76947d394a1091a3615948e2f"}, + {file = "coverage-7.6.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e7575ab65ca8399c8c4f9a7d61bbd2d204c8b8e447aab9d355682205c9dd948d"}, + {file = "coverage-7.6.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8161d9fbc7e9fe2326de89cd0abb9f3599bccc1287db0aba285cb68d204ce929"}, + {file = "coverage-7.6.12-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a1e465f398c713f1b212400b4e79a09829cd42aebd360362cd89c5bdc44eb87"}, + {file = "coverage-7.6.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f25d8b92a4e31ff1bd873654ec367ae811b3a943583e05432ea29264782dc32c"}, + {file = "coverage-7.6.12-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a936309a65cc5ca80fa9f20a442ff9e2d06927ec9a4f54bcba9c14c066323f2"}, + {file = "coverage-7.6.12-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:aa6f302a3a0b5f240ee201297fff0bbfe2fa0d415a94aeb257d8b461032389bd"}, + {file = "coverage-7.6.12-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f973643ef532d4f9be71dd88cf7588936685fdb576d93a79fe9f65bc337d9d73"}, + {file = "coverage-7.6.12-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:78f5243bb6b1060aed6213d5107744c19f9571ec76d54c99cc15938eb69e0e86"}, + {file = "coverage-7.6.12-cp39-cp39-win32.whl", hash = "sha256:69e62c5034291c845fc4df7f8155e8544178b6c774f97a99e2734b05eb5bed31"}, + {file = "coverage-7.6.12-cp39-cp39-win_amd64.whl", hash = "sha256:b01a840ecc25dce235ae4c1b6a0daefb2a203dba0e6e980637ee9c2f6ee0df57"}, + {file = "coverage-7.6.12-pp39.pp310-none-any.whl", hash = "sha256:7e39e845c4d764208e7b8f6a21c541ade741e2c41afabdfa1caa28687a3c98cf"}, + {file = "coverage-7.6.12-py3-none-any.whl", hash = "sha256:eb8668cfbc279a536c633137deeb9435d2962caec279c3f8cf8b91fff6ff8953"}, + {file = "coverage-7.6.12.tar.gz", hash = "sha256:48cfc4641d95d34766ad41d9573cc0f22a48aa88d22657a1fe01dca0dbae4de2"}, ] [package.dependencies] @@ -554,39 +550,43 @@ toml = ["tomli"] [[package]] name = "cryptography" -version = "44.0.0" +version = "44.0.1" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = "!=3.9.0,!=3.9.1,>=3.7" groups = ["charm-libs", "integration"] files = [ - {file = "cryptography-44.0.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:84111ad4ff3f6253820e6d3e58be2cc2a00adb29335d4cacb5ab4d4d34f2a123"}, - {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b15492a11f9e1b62ba9d73c210e2416724633167de94607ec6069ef724fad092"}, - {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:831c3c4d0774e488fdc83a1923b49b9957d33287de923d58ebd3cec47a0ae43f"}, - {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb"}, - {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b"}, - {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543"}, - {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e"}, - {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e"}, - {file = "cryptography-44.0.0-cp37-abi3-win32.whl", hash = "sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053"}, - {file = "cryptography-44.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:abc998e0c0eee3c8a1904221d3f67dcfa76422b23620173e28c11d3e626c21bd"}, - {file = "cryptography-44.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:660cb7312a08bc38be15b696462fa7cc7cd85c3ed9c576e81f4dc4d8b2b31591"}, - {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1923cb251c04be85eec9fda837661c67c1049063305d6be5721643c22dd4e2b7"}, - {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:404fdc66ee5f83a1388be54300ae978b2efd538018de18556dde92575e05defc"}, - {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289"}, - {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7"}, - {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c"}, - {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64"}, - {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285"}, - {file = "cryptography-44.0.0-cp39-abi3-win32.whl", hash = "sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417"}, - {file = "cryptography-44.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:708ee5f1bafe76d041b53a4f95eb28cdeb8d18da17e597d46d7833ee59b97ede"}, - {file = "cryptography-44.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37d76e6863da3774cd9db5b409a9ecfd2c71c981c38788d3fcfaf177f447b731"}, - {file = "cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:f677e1268c4e23420c3acade68fac427fffcb8d19d7df95ed7ad17cdef8404f4"}, - {file = "cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f5e7cb1e5e56ca0933b4873c0220a78b773b24d40d186b6738080b73d3d0a756"}, - {file = "cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:8b3e6eae66cf54701ee7d9c83c30ac0a1e3fa17be486033000f2a73a12ab507c"}, - {file = "cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:be4ce505894d15d5c5037167ffb7f0ae90b7be6f2a98f9a5c3442395501c32fa"}, - {file = "cryptography-44.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:62901fb618f74d7d81bf408c8719e9ec14d863086efe4185afd07c352aee1d2c"}, - {file = "cryptography-44.0.0.tar.gz", hash = "sha256:cd4e834f340b4293430701e772ec543b0fbe6c2dea510a5286fe0acabe153a02"}, + {file = "cryptography-44.0.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf688f615c29bfe9dfc44312ca470989279f0e94bb9f631f85e3459af8efc009"}, + {file = "cryptography-44.0.1-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd7c7e2d71d908dc0f8d2027e1604102140d84b155e658c20e8ad1304317691f"}, + {file = "cryptography-44.0.1-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:887143b9ff6bad2b7570da75a7fe8bbf5f65276365ac259a5d2d5147a73775f2"}, + {file = "cryptography-44.0.1-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:322eb03ecc62784536bc173f1483e76747aafeb69c8728df48537eb431cd1911"}, + {file = "cryptography-44.0.1-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:21377472ca4ada2906bc313168c9dc7b1d7ca417b63c1c3011d0c74b7de9ae69"}, + {file = "cryptography-44.0.1-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:df978682c1504fc93b3209de21aeabf2375cb1571d4e61907b3e7a2540e83026"}, + {file = "cryptography-44.0.1-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:eb3889330f2a4a148abead555399ec9a32b13b7c8ba969b72d8e500eb7ef84cd"}, + {file = "cryptography-44.0.1-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:8e6a85a93d0642bd774460a86513c5d9d80b5c002ca9693e63f6e540f1815ed0"}, + {file = "cryptography-44.0.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6f76fdd6fd048576a04c5210d53aa04ca34d2ed63336d4abd306d0cbe298fddf"}, + {file = "cryptography-44.0.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6c8acf6f3d1f47acb2248ec3ea261171a671f3d9428e34ad0357148d492c7864"}, + {file = "cryptography-44.0.1-cp37-abi3-win32.whl", hash = "sha256:24979e9f2040c953a94bf3c6782e67795a4c260734e5264dceea65c8f4bae64a"}, + {file = "cryptography-44.0.1-cp37-abi3-win_amd64.whl", hash = "sha256:fd0ee90072861e276b0ff08bd627abec29e32a53b2be44e41dbcdf87cbee2b00"}, + {file = "cryptography-44.0.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:a2d8a7045e1ab9b9f803f0d9531ead85f90c5f2859e653b61497228b18452008"}, + {file = "cryptography-44.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8272f257cf1cbd3f2e120f14c68bff2b6bdfcc157fafdee84a1b795efd72862"}, + {file = "cryptography-44.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e8d181e90a777b63f3f0caa836844a1182f1f265687fac2115fcf245f5fbec3"}, + {file = "cryptography-44.0.1-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:436df4f203482f41aad60ed1813811ac4ab102765ecae7a2bbb1dbb66dcff5a7"}, + {file = "cryptography-44.0.1-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4f422e8c6a28cf8b7f883eb790695d6d45b0c385a2583073f3cec434cc705e1a"}, + {file = "cryptography-44.0.1-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:72198e2b5925155497a5a3e8c216c7fb3e64c16ccee11f0e7da272fa93b35c4c"}, + {file = "cryptography-44.0.1-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:2a46a89ad3e6176223b632056f321bc7de36b9f9b93b2cc1cccf935a3849dc62"}, + {file = "cryptography-44.0.1-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:53f23339864b617a3dfc2b0ac8d5c432625c80014c25caac9082314e9de56f41"}, + {file = "cryptography-44.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:888fcc3fce0c888785a4876ca55f9f43787f4c5c1cc1e2e0da71ad481ff82c5b"}, + {file = "cryptography-44.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:00918d859aa4e57db8299607086f793fa7813ae2ff5a4637e318a25ef82730f7"}, + {file = "cryptography-44.0.1-cp39-abi3-win32.whl", hash = "sha256:9b336599e2cb77b1008cb2ac264b290803ec5e8e89d618a5e978ff5eb6f715d9"}, + {file = "cryptography-44.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:e403f7f766ded778ecdb790da786b418a9f2394f36e8cc8b796cc056ab05f44f"}, + {file = "cryptography-44.0.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1f9a92144fa0c877117e9748c74501bea842f93d21ee00b0cf922846d9d0b183"}, + {file = "cryptography-44.0.1-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:610a83540765a8d8ce0f351ce42e26e53e1f774a6efb71eb1b41eb01d01c3d12"}, + {file = "cryptography-44.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:5fed5cd6102bb4eb843e3315d2bf25fede494509bddadb81e03a859c1bc17b83"}, + {file = "cryptography-44.0.1-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:f4daefc971c2d1f82f03097dc6f216744a6cd2ac0f04c68fb935ea2ba2a0d420"}, + {file = "cryptography-44.0.1-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:94f99f2b943b354a5b6307d7e8d19f5c423a794462bde2bf310c770ba052b1c4"}, + {file = "cryptography-44.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d9c5b9f698a83c8bd71e0f4d3f9f839ef244798e5ffe96febfa9714717db7af7"}, + {file = "cryptography-44.0.1.tar.gz", hash = "sha256:f51f5705ab27898afda1aaa430f34ad90dc117421057782022edf0600bec5f14"}, ] [package.dependencies] @@ -599,7 +599,7 @@ nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2)"] pep8test = ["check-sdist", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"] sdist = ["build (>=1.0.0)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["certifi (>=2024)", "cryptography-vectors (==44.0.0)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] +test = ["certifi (>=2024)", "cryptography-vectors (==44.0.1)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"] test-randomorder = ["pytest-randomly"] [[package]] @@ -616,14 +616,14 @@ files = [ [[package]] name = "deprecated" -version = "1.2.17" +version = "1.2.18" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" groups = ["charm-libs"] files = [ - {file = "Deprecated-1.2.17-py2.py3-none-any.whl", hash = "sha256:69cdc0a751671183f569495e2efb14baee4344b0236342eec29f1fde25d61818"}, - {file = "deprecated-1.2.17.tar.gz", hash = "sha256:0114a10f0bbb750b90b2c2296c90cf7e9eaeb0abb5cf06c80de2c60138de0a82"}, + {file = "Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec"}, + {file = "deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d"}, ] [package.dependencies] @@ -690,14 +690,14 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "googleapis-common-protos" -version = "1.66.0" +version = "1.67.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" groups = ["charm-libs"] files = [ - {file = "googleapis_common_protos-1.66.0-py2.py3-none-any.whl", hash = "sha256:d7abcd75fabb2e0ec9f74466401f6c119a0b498e27370e9be4c94cb7e382b8ed"}, - {file = "googleapis_common_protos-1.66.0.tar.gz", hash = "sha256:c3e7b33d15fdca5374cc0a7346dd92ffa847425cc4ea941d970f13680052ec8c"}, + {file = "googleapis_common_protos-1.67.0-py2.py3-none-any.whl", hash = "sha256:579de760800d13616f51cf8be00c876f00a9f146d3e6510e19d1f4111758b741"}, + {file = "googleapis_common_protos-1.67.0.tar.gz", hash = "sha256:21398025365f138be356d5923e9168737d94d46a72aefee4a6110a1f23463c86"}, ] [package.dependencies] @@ -849,14 +849,14 @@ tomli = {version = "*", markers = "python_version > \"3.6\" and python_version < [[package]] name = "ipython" -version = "8.31.0" +version = "8.32.0" description = "IPython: Productive Interactive Computing" optional = false python-versions = ">=3.10" groups = ["integration"] files = [ - {file = "ipython-8.31.0-py3-none-any.whl", hash = "sha256:46ec58f8d3d076a61d128fe517a51eb730e3aaf0c184ea8c17d16e366660c6a6"}, - {file = "ipython-8.31.0.tar.gz", hash = "sha256:b6a2274606bec6166405ff05e54932ed6e5cfecaca1fc05f2cacde7bb074d70b"}, + {file = "ipython-8.32.0-py3-none-any.whl", hash = "sha256:cae85b0c61eff1fc48b0a8002de5958b6528fa9c8defb1894da63f42613708aa"}, + {file = "ipython-8.32.0.tar.gz", hash = "sha256:be2c91895b0b9ea7ba49d33b23e2040c352b33eb6a519cca7ce6e0c743444251"}, ] [package.dependencies] @@ -908,14 +908,14 @@ testing = ["Django", "attrs", "colorama", "docopt", "pytest (<9.0.0)"] [[package]] name = "jinja2" -version = "3.1.5" +version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" groups = ["main", "integration"] files = [ - {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, - {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, + {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, + {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, ] [package.dependencies] @@ -1335,14 +1335,14 @@ files = [ [[package]] name = "ops" -version = "2.17.1" +version = "2.18.1" description = "The Python library behind great charms" optional = false python-versions = ">=3.8" groups = ["main", "charm-libs"] files = [ - {file = "ops-2.17.1-py3-none-any.whl", hash = "sha256:0fabc45740d59619c3265328f51f71f99b06557e22493cdd32d10c2b25bcd553"}, - {file = "ops-2.17.1.tar.gz", hash = "sha256:de2d1dd382b4a5f3df3ba78a5266d59462644f3f8ea0f4e7479a248998862a3f"}, + {file = "ops-2.18.1-py3-none-any.whl", hash = "sha256:ba0312366e25b3ae90cf4b8d0af6ea6b612d4951500f856bce609cdb25c9bdeb"}, + {file = "ops-2.18.1.tar.gz", hash = "sha256:5619deb370c00ea851f9579b780a09b88b1a1d020e58e1ed81d31c8fb7b28c8a"}, ] [package.dependencies] @@ -1350,7 +1350,7 @@ PyYAML = "==6.*" websocket-client = "==1.*" [package.extras] -docs = ["canonical-sphinx-extensions", "furo", "linkify-it-py", "myst-parser", "ops-scenario (>=7.0.5,<8)", "pyspelling", "sphinx (>=8.0.0,<8.1.0)", "sphinx-autobuild", "sphinx-copybutton", "sphinx-design", "sphinx-notfound-page", "sphinx-tabs", "sphinxcontrib-jquery", "sphinxext-opengraph"] +docs = ["canonical-sphinx-extensions", "furo", "linkify-it-py", "myst-parser", "pyspelling", "sphinx (>=8.0.0,<8.1.0)", "sphinx-autobuild", "sphinx-copybutton", "sphinx-design", "sphinx-notfound-page", "sphinx-tabs", "sphinxcontrib-jquery", "sphinxext-opengraph"] testing = ["ops-scenario (>=7.0.5,<8)"] [[package]] @@ -1382,14 +1382,14 @@ dev = ["jinja2"] [[package]] name = "paramiko" -version = "3.5.0" +version = "3.5.1" description = "SSH2 protocol library" optional = false python-versions = ">=3.6" groups = ["integration"] files = [ - {file = "paramiko-3.5.0-py3-none-any.whl", hash = "sha256:1fedf06b085359051cd7d0d270cebe19e755a8a921cc2ddbfa647fb0cd7d68f9"}, - {file = "paramiko-3.5.0.tar.gz", hash = "sha256:ad11e540da4f55cedda52931f1a3f812a8238a7af7f62a60de538cd80bb28124"}, + {file = "paramiko-3.5.1-py3-none-any.whl", hash = "sha256:43b9a0501fc2b5e70680388d9346cf252cfb7d00b0667c39e80eb43a408b8f61"}, + {file = "paramiko-3.5.1.tar.gz", hash = "sha256:b2c665bc45b2b215bd7d7f039901b14b067da00f3a11e6640995fd58f2664822"}, ] [package.dependencies] @@ -1464,14 +1464,14 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "poetry-core" -version = "2.0.1" +version = "2.1.1" description = "Poetry PEP 517 Build Backend" optional = false python-versions = "<4.0,>=3.9" groups = ["charm-libs"] files = [ - {file = "poetry_core-2.0.1-py3-none-any.whl", hash = "sha256:a3c7009536522cda4eb0fb3805c9dc935b5537f8727dd01efb9c15e51a17552b"}, - {file = "poetry_core-2.0.1.tar.gz", hash = "sha256:10177c2772469d9032a49f0d8707af761b1c597cea3b4fb31546e5cd436eb157"}, + {file = "poetry_core-2.1.1-py3-none-any.whl", hash = "sha256:bc3b0382ab4d00d5d780277fd0aad1580eb4403613b37fc60fec407b5bee1fe6"}, + {file = "poetry_core-2.1.1.tar.gz", hash = "sha256:c1a1f6f00e4254742f40988a8caf665549101cf9991122cd5de1198897768b1a"}, ] [[package]] @@ -1512,33 +1512,26 @@ files = [ [[package]] name = "psutil" -version = "6.1.1" -description = "Cross-platform lib for process and system monitoring in Python." +version = "7.0.0" +description = "Cross-platform lib for process and system monitoring in Python. NOTE: the syntax of this script MUST be kept compatible with Python 2.7." optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +python-versions = ">=3.6" groups = ["main"] files = [ - {file = "psutil-6.1.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:9ccc4316f24409159897799b83004cb1e24f9819b0dcf9c0b68bdcb6cefee6a8"}, - {file = "psutil-6.1.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ca9609c77ea3b8481ab005da74ed894035936223422dc591d6772b147421f777"}, - {file = "psutil-6.1.1-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:8df0178ba8a9e5bc84fed9cfa61d54601b371fbec5c8eebad27575f1e105c0d4"}, - {file = "psutil-6.1.1-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:1924e659d6c19c647e763e78670a05dbb7feaf44a0e9c94bf9e14dfc6ba50468"}, - {file = "psutil-6.1.1-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:018aeae2af92d943fdf1da6b58665124897cfc94faa2ca92098838f83e1b1bca"}, - {file = "psutil-6.1.1-cp27-none-win32.whl", hash = "sha256:6d4281f5bbca041e2292be3380ec56a9413b790579b8e593b1784499d0005dac"}, - {file = "psutil-6.1.1-cp27-none-win_amd64.whl", hash = "sha256:c777eb75bb33c47377c9af68f30e9f11bc78e0f07fbf907be4a5d70b2fe5f030"}, - {file = "psutil-6.1.1-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:fc0ed7fe2231a444fc219b9c42d0376e0a9a1a72f16c5cfa0f68d19f1a0663e8"}, - {file = "psutil-6.1.1-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:0bdd4eab935276290ad3cb718e9809412895ca6b5b334f5a9111ee6d9aff9377"}, - {file = "psutil-6.1.1-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b6e06c20c05fe95a3d7302d74e7097756d4ba1247975ad6905441ae1b5b66003"}, - {file = "psutil-6.1.1-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97f7cb9921fbec4904f522d972f0c0e1f4fabbdd4e0287813b21215074a0f160"}, - {file = "psutil-6.1.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33431e84fee02bc84ea36d9e2c4a6d395d479c9dd9bba2376c1f6ee8f3a4e0b3"}, - {file = "psutil-6.1.1-cp36-cp36m-win32.whl", hash = "sha256:384636b1a64b47814437d1173be1427a7c83681b17a450bfc309a1953e329603"}, - {file = "psutil-6.1.1-cp36-cp36m-win_amd64.whl", hash = "sha256:8be07491f6ebe1a693f17d4f11e69d0dc1811fa082736500f649f79df7735303"}, - {file = "psutil-6.1.1-cp37-abi3-win32.whl", hash = "sha256:eaa912e0b11848c4d9279a93d7e2783df352b082f40111e078388701fd479e53"}, - {file = "psutil-6.1.1-cp37-abi3-win_amd64.whl", hash = "sha256:f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649"}, - {file = "psutil-6.1.1.tar.gz", hash = "sha256:cf8496728c18f2d0b45198f06895be52f36611711746b7f30c464b422b50e2f5"}, + {file = "psutil-7.0.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:101d71dc322e3cffd7cea0650b09b3d08b8e7c4109dd6809fe452dfd00e58b25"}, + {file = "psutil-7.0.0-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:39db632f6bb862eeccf56660871433e111b6ea58f2caea825571951d4b6aa3da"}, + {file = "psutil-7.0.0-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1fcee592b4c6f146991ca55919ea3d1f8926497a713ed7faaf8225e174581e91"}, + {file = "psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34"}, + {file = "psutil-7.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5f098451abc2828f7dc6b58d44b532b22f2088f4999a937557b603ce72b1993"}, + {file = "psutil-7.0.0-cp36-cp36m-win32.whl", hash = "sha256:84df4eb63e16849689f76b1ffcb36db7b8de703d1bc1fe41773db487621b6c17"}, + {file = "psutil-7.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:1e744154a6580bc968a0195fd25e80432d3afec619daf145b9e5ba16cc1d688e"}, + {file = "psutil-7.0.0-cp37-abi3-win32.whl", hash = "sha256:ba3fcef7523064a6c9da440fc4d6bd07da93ac726b5733c29027d7dc95b39d99"}, + {file = "psutil-7.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:4cf3d4eb1aa9b348dec30105c55cd9b7d4629285735a102beb4441e38db90553"}, + {file = "psutil-7.0.0.tar.gz", hash = "sha256:7be9c3eba38beccb6495ea33afd982a44074b78f28c434a1f51cc07fd315c456"}, ] [package.extras] -dev = ["abi3audit", "black", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest-cov", "requests", "rstcheck", "ruff", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "vulture", "wheel"] +dev = ["abi3audit", "black (==24.10.0)", "check-manifest", "coverage", "packaging", "pylint", "pyperf", "pypinfo", "pytest", "pytest-cov", "pytest-xdist", "requests", "rstcheck", "ruff", "setuptools", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "virtualenv", "vulture", "wheel"] test = ["pytest", "pytest-xdist", "setuptools"] [[package]] @@ -1849,13 +1842,13 @@ pytz = "*" [[package]] name = "pysyncobj" -version = "0.3.13" +version = "0.3.14" description = "A library for replicating your python class between multiple servers, based on raft protocol" optional = false python-versions = "*" groups = ["main"] files = [ - {file = "pysyncobj-0.3.13.tar.gz", hash = "sha256:1785930b738fa21af298ebb04c213af25c31af148faa32f53af337ed1492d5a2"}, + {file = "pysyncobj-0.3.14.tar.gz", hash = "sha256:69d34e672257694f83f50dfb5e6e7ce446a68dba09ed48e142c782380d6428d4"}, ] [[package]] @@ -1900,33 +1893,16 @@ pytest = ">=7.0.0" docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy (>=0.931)", "pytest-trio (>=0.7.0)"] -[[package]] -name = "pytest-github-secrets" -version = "0.1.0" -description = "" -optional = false -python-versions = ">=3.8" -groups = ["integration"] -files = [] -develop = false - -[package.source] -type = "git" -url = "https://github.com/canonical/data-platform-workflows" -reference = "v29.1.0" -resolved_reference = "cf3e292107a8d420c452e35cf7552c225add7fbd" -subdirectory = "python/pytest_plugins/github_secrets" - [[package]] name = "pytest-operator" -version = "0.39.0" +version = "0.40.0" description = "Fixtures for Operators" optional = false python-versions = "*" groups = ["integration"] files = [ - {file = "pytest_operator-0.39.0-py3-none-any.whl", hash = "sha256:ade76e1896eaf7f71704b537fd6661a705d81a045b8db71531d9e4741913fa19"}, - {file = "pytest_operator-0.39.0.tar.gz", hash = "sha256:b66bd8c6d161593c258a5714118a51e9f37721e7cd9e503299423d8a7d900f90"}, + {file = "pytest_operator-0.40.0-py3-none-any.whl", hash = "sha256:1cfa93ab61b11e8d7bf58dbb1a39e75fcbfcc084781bb571fde08fda7e236713"}, + {file = "pytest_operator-0.40.0.tar.gz", hash = "sha256:45394ade32b7765b6ba89871b676d1fb8aa7578589f74df26ff0fca4692d1c7b"}, ] [package.dependencies] @@ -1937,46 +1913,6 @@ pytest = "*" pytest-asyncio = "<0.23" pyyaml = "*" -[[package]] -name = "pytest-operator-cache" -version = "0.1.0" -description = "" -optional = false -python-versions = ">=3.8" -groups = ["integration"] -files = [] -develop = false - -[package.dependencies] -pyyaml = "*" - -[package.source] -type = "git" -url = "https://github.com/canonical/data-platform-workflows" -reference = "v29.1.0" -resolved_reference = "cf3e292107a8d420c452e35cf7552c225add7fbd" -subdirectory = "python/pytest_plugins/pytest_operator_cache" - -[[package]] -name = "pytest-operator-groups" -version = "0.1.0" -description = "" -optional = false -python-versions = ">=3.8" -groups = ["integration"] -files = [] -develop = false - -[package.dependencies] -pytest = "*" - -[package.source] -type = "git" -url = "https://github.com/canonical/data-platform-workflows" -reference = "v29.1.0" -resolved_reference = "cf3e292107a8d420c452e35cf7552c225add7fbd" -subdirectory = "python/pytest_plugins/pytest_operator_groups" - [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1994,14 +1930,14 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2024.2" +version = "2025.1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" groups = ["integration"] files = [ - {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, - {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, + {file = "pytz-2025.1-py2.py3-none-any.whl", hash = "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57"}, + {file = "pytz-2025.1.tar.gz", hash = "sha256:c2db42be2a2518b28e65f9207c4d05e6ff547d1efa4086469ef855e4ab70178e"}, ] [[package]] @@ -2255,30 +2191,30 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruff" -version = "0.9.3" +version = "0.9.6" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["format"] files = [ - {file = "ruff-0.9.3-py3-none-linux_armv6l.whl", hash = "sha256:7f39b879064c7d9670197d91124a75d118d00b0990586549949aae80cdc16624"}, - {file = "ruff-0.9.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a187171e7c09efa4b4cc30ee5d0d55a8d6c5311b3e1b74ac5cb96cc89bafc43c"}, - {file = "ruff-0.9.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c59ab92f8e92d6725b7ded9d4a31be3ef42688a115c6d3da9457a5bda140e2b4"}, - {file = "ruff-0.9.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dc153c25e715be41bb228bc651c1e9b1a88d5c6e5ed0194fa0dfea02b026439"}, - {file = "ruff-0.9.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:646909a1e25e0dc28fbc529eab8eb7bb583079628e8cbe738192853dbbe43af5"}, - {file = "ruff-0.9.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a5a46e09355695fbdbb30ed9889d6cf1c61b77b700a9fafc21b41f097bfbba4"}, - {file = "ruff-0.9.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:c4bb09d2bbb394e3730d0918c00276e79b2de70ec2a5231cd4ebb51a57df9ba1"}, - {file = "ruff-0.9.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:96a87ec31dc1044d8c2da2ebbed1c456d9b561e7d087734336518181b26b3aa5"}, - {file = "ruff-0.9.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bb7554aca6f842645022fe2d301c264e6925baa708b392867b7a62645304df4"}, - {file = "ruff-0.9.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cabc332b7075a914ecea912cd1f3d4370489c8018f2c945a30bcc934e3bc06a6"}, - {file = "ruff-0.9.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:33866c3cc2a575cbd546f2cd02bdd466fed65118e4365ee538a3deffd6fcb730"}, - {file = "ruff-0.9.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:006e5de2621304c8810bcd2ee101587712fa93b4f955ed0985907a36c427e0c2"}, - {file = "ruff-0.9.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:ba6eea4459dbd6b1be4e6bfc766079fb9b8dd2e5a35aff6baee4d9b1514ea519"}, - {file = "ruff-0.9.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:90230a6b8055ad47d3325e9ee8f8a9ae7e273078a66401ac66df68943ced029b"}, - {file = "ruff-0.9.3-py3-none-win32.whl", hash = "sha256:eabe5eb2c19a42f4808c03b82bd313fc84d4e395133fb3fc1b1516170a31213c"}, - {file = "ruff-0.9.3-py3-none-win_amd64.whl", hash = "sha256:040ceb7f20791dfa0e78b4230ee9dce23da3b64dd5848e40e3bf3ab76468dcf4"}, - {file = "ruff-0.9.3-py3-none-win_arm64.whl", hash = "sha256:800d773f6d4d33b0a3c60e2c6ae8f4c202ea2de056365acfa519aa48acf28e0b"}, - {file = "ruff-0.9.3.tar.gz", hash = "sha256:8293f89985a090ebc3ed1064df31f3b4b56320cdfcec8b60d3295bddb955c22a"}, + {file = "ruff-0.9.6-py3-none-linux_armv6l.whl", hash = "sha256:2f218f356dd2d995839f1941322ff021c72a492c470f0b26a34f844c29cdf5ba"}, + {file = "ruff-0.9.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b908ff4df65dad7b251c9968a2e4560836d8f5487c2f0cc238321ed951ea0504"}, + {file = "ruff-0.9.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b109c0ad2ececf42e75fa99dc4043ff72a357436bb171900714a9ea581ddef83"}, + {file = "ruff-0.9.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1de4367cca3dac99bcbd15c161404e849bb0bfd543664db39232648dc00112dc"}, + {file = "ruff-0.9.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac3ee4d7c2c92ddfdaedf0bf31b2b176fa7aa8950efc454628d477394d35638b"}, + {file = "ruff-0.9.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5dc1edd1775270e6aa2386119aea692039781429f0be1e0949ea5884e011aa8e"}, + {file = "ruff-0.9.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:4a091729086dffa4bd070aa5dab7e39cc6b9d62eb2bef8f3d91172d30d599666"}, + {file = "ruff-0.9.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1bbc6808bf7b15796cef0815e1dfb796fbd383e7dbd4334709642649625e7c5"}, + {file = "ruff-0.9.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:589d1d9f25b5754ff230dce914a174a7c951a85a4e9270613a2b74231fdac2f5"}, + {file = "ruff-0.9.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc61dd5131742e21103fbbdcad683a8813be0e3c204472d520d9a5021ca8b217"}, + {file = "ruff-0.9.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5e2d9126161d0357e5c8f30b0bd6168d2c3872372f14481136d13de9937f79b6"}, + {file = "ruff-0.9.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:68660eab1a8e65babb5229a1f97b46e3120923757a68b5413d8561f8a85d4897"}, + {file = "ruff-0.9.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c4cae6c4cc7b9b4017c71114115db0445b00a16de3bcde0946273e8392856f08"}, + {file = "ruff-0.9.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:19f505b643228b417c1111a2a536424ddde0db4ef9023b9e04a46ed8a1cb4656"}, + {file = "ruff-0.9.6-py3-none-win32.whl", hash = "sha256:194d8402bceef1b31164909540a597e0d913c0e4952015a5b40e28c146121b5d"}, + {file = "ruff-0.9.6-py3-none-win_amd64.whl", hash = "sha256:03482d5c09d90d4ee3f40d97578423698ad895c87314c4de39ed2af945633caa"}, + {file = "ruff-0.9.6-py3-none-win_arm64.whl", hash = "sha256:0e2bb706a2be7ddfea4a4af918562fdc1bcb16df255e5fa595bbd800ce322a5a"}, + {file = "ruff-0.9.6.tar.gz", hash = "sha256:81761592f72b620ec8fa1068a6fd00e98a5ebee342a3642efd84454f3031dca9"}, ] [[package]] @@ -2508,81 +2444,81 @@ test = ["websockets"] [[package]] name = "websockets" -version = "14.2" +version = "15.0" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" optional = false python-versions = ">=3.9" groups = ["integration"] files = [ - {file = "websockets-14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e8179f95323b9ab1c11723e5d91a89403903f7b001828161b480a7810b334885"}, - {file = "websockets-14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0d8c3e2cdb38f31d8bd7d9d28908005f6fa9def3324edb9bf336d7e4266fd397"}, - {file = "websockets-14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:714a9b682deb4339d39ffa674f7b674230227d981a37d5d174a4a83e3978a610"}, - {file = "websockets-14.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2e53c72052f2596fb792a7acd9704cbc549bf70fcde8a99e899311455974ca3"}, - {file = "websockets-14.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3fbd68850c837e57373d95c8fe352203a512b6e49eaae4c2f4088ef8cf21980"}, - {file = "websockets-14.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b27ece32f63150c268593d5fdb82819584831a83a3f5809b7521df0685cd5d8"}, - {file = "websockets-14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4daa0faea5424d8713142b33825fff03c736f781690d90652d2c8b053345b0e7"}, - {file = "websockets-14.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:bc63cee8596a6ec84d9753fd0fcfa0452ee12f317afe4beae6b157f0070c6c7f"}, - {file = "websockets-14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7a570862c325af2111343cc9b0257b7119b904823c675b22d4ac547163088d0d"}, - {file = "websockets-14.2-cp310-cp310-win32.whl", hash = "sha256:75862126b3d2d505e895893e3deac0a9339ce750bd27b4ba515f008b5acf832d"}, - {file = "websockets-14.2-cp310-cp310-win_amd64.whl", hash = "sha256:cc45afb9c9b2dc0852d5c8b5321759cf825f82a31bfaf506b65bf4668c96f8b2"}, - {file = "websockets-14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3bdc8c692c866ce5fefcaf07d2b55c91d6922ac397e031ef9b774e5b9ea42166"}, - {file = "websockets-14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c93215fac5dadc63e51bcc6dceca72e72267c11def401d6668622b47675b097f"}, - {file = "websockets-14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c9b6535c0e2cf8a6bf938064fb754aaceb1e6a4a51a80d884cd5db569886910"}, - {file = "websockets-14.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a52a6d7cf6938e04e9dceb949d35fbdf58ac14deea26e685ab6368e73744e4c"}, - {file = "websockets-14.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9f05702e93203a6ff5226e21d9b40c037761b2cfb637187c9802c10f58e40473"}, - {file = "websockets-14.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22441c81a6748a53bfcb98951d58d1af0661ab47a536af08920d129b4d1c3473"}, - {file = "websockets-14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd9b868d78b194790e6236d9cbc46d68aba4b75b22497eb4ab64fa640c3af56"}, - {file = "websockets-14.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1a5a20d5843886d34ff8c57424cc65a1deda4375729cbca4cb6b3353f3ce4142"}, - {file = "websockets-14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:34277a29f5303d54ec6468fb525d99c99938607bc96b8d72d675dee2b9f5bf1d"}, - {file = "websockets-14.2-cp311-cp311-win32.whl", hash = "sha256:02687db35dbc7d25fd541a602b5f8e451a238ffa033030b172ff86a93cb5dc2a"}, - {file = "websockets-14.2-cp311-cp311-win_amd64.whl", hash = "sha256:862e9967b46c07d4dcd2532e9e8e3c2825e004ffbf91a5ef9dde519ee2effb0b"}, - {file = "websockets-14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1f20522e624d7ffbdbe259c6b6a65d73c895045f76a93719aa10cd93b3de100c"}, - {file = "websockets-14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:647b573f7d3ada919fd60e64d533409a79dcf1ea21daeb4542d1d996519ca967"}, - {file = "websockets-14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6af99a38e49f66be5a64b1e890208ad026cda49355661549c507152113049990"}, - {file = "websockets-14.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:091ab63dfc8cea748cc22c1db2814eadb77ccbf82829bac6b2fbe3401d548eda"}, - {file = "websockets-14.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b374e8953ad477d17e4851cdc66d83fdc2db88d9e73abf755c94510ebddceb95"}, - {file = "websockets-14.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a39d7eceeea35db85b85e1169011bb4321c32e673920ae9c1b6e0978590012a3"}, - {file = "websockets-14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0a6f3efd47ffd0d12080594f434faf1cd2549b31e54870b8470b28cc1d3817d9"}, - {file = "websockets-14.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:065ce275e7c4ffb42cb738dd6b20726ac26ac9ad0a2a48e33ca632351a737267"}, - {file = "websockets-14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e9d0e53530ba7b8b5e389c02282f9d2aa47581514bd6049d3a7cffe1385cf5fe"}, - {file = "websockets-14.2-cp312-cp312-win32.whl", hash = "sha256:20e6dd0984d7ca3037afcb4494e48c74ffb51e8013cac71cf607fffe11df7205"}, - {file = "websockets-14.2-cp312-cp312-win_amd64.whl", hash = "sha256:44bba1a956c2c9d268bdcdf234d5e5ff4c9b6dc3e300545cbe99af59dda9dcce"}, - {file = "websockets-14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6f1372e511c7409a542291bce92d6c83320e02c9cf392223272287ce55bc224e"}, - {file = "websockets-14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4da98b72009836179bb596a92297b1a61bb5a830c0e483a7d0766d45070a08ad"}, - {file = "websockets-14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8a86a269759026d2bde227652b87be79f8a734e582debf64c9d302faa1e9f03"}, - {file = "websockets-14.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86cf1aaeca909bf6815ea714d5c5736c8d6dd3a13770e885aafe062ecbd04f1f"}, - {file = "websockets-14.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9b0f6c3ba3b1240f602ebb3971d45b02cc12bd1845466dd783496b3b05783a5"}, - {file = "websockets-14.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:669c3e101c246aa85bc8534e495952e2ca208bd87994650b90a23d745902db9a"}, - {file = "websockets-14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:eabdb28b972f3729348e632ab08f2a7b616c7e53d5414c12108c29972e655b20"}, - {file = "websockets-14.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2066dc4cbcc19f32c12a5a0e8cc1b7ac734e5b64ac0a325ff8353451c4b15ef2"}, - {file = "websockets-14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ab95d357cd471df61873dadf66dd05dd4709cae001dd6342edafc8dc6382f307"}, - {file = "websockets-14.2-cp313-cp313-win32.whl", hash = "sha256:a9e72fb63e5f3feacdcf5b4ff53199ec8c18d66e325c34ee4c551ca748623bbc"}, - {file = "websockets-14.2-cp313-cp313-win_amd64.whl", hash = "sha256:b439ea828c4ba99bb3176dc8d9b933392a2413c0f6b149fdcba48393f573377f"}, - {file = "websockets-14.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7cd5706caec1686c5d233bc76243ff64b1c0dc445339bd538f30547e787c11fe"}, - {file = "websockets-14.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ec607328ce95a2f12b595f7ae4c5d71bf502212bddcea528290b35c286932b12"}, - {file = "websockets-14.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:da85651270c6bfb630136423037dd4975199e5d4114cae6d3066641adcc9d1c7"}, - {file = "websockets-14.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ecadc7ce90accf39903815697917643f5b7cfb73c96702318a096c00aa71f5"}, - {file = "websockets-14.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1979bee04af6a78608024bad6dfcc0cc930ce819f9e10342a29a05b5320355d0"}, - {file = "websockets-14.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dddacad58e2614a24938a50b85969d56f88e620e3f897b7d80ac0d8a5800258"}, - {file = "websockets-14.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:89a71173caaf75fa71a09a5f614f450ba3ec84ad9fca47cb2422a860676716f0"}, - {file = "websockets-14.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:6af6a4b26eea4fc06c6818a6b962a952441e0e39548b44773502761ded8cc1d4"}, - {file = "websockets-14.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:80c8efa38957f20bba0117b48737993643204645e9ec45512579132508477cfc"}, - {file = "websockets-14.2-cp39-cp39-win32.whl", hash = "sha256:2e20c5f517e2163d76e2729104abc42639c41cf91f7b1839295be43302713661"}, - {file = "websockets-14.2-cp39-cp39-win_amd64.whl", hash = "sha256:b4c8cef610e8d7c70dea92e62b6814a8cd24fbd01d7103cc89308d2bfe1659ef"}, - {file = "websockets-14.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d7d9cafbccba46e768be8a8ad4635fa3eae1ffac4c6e7cb4eb276ba41297ed29"}, - {file = "websockets-14.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:c76193c1c044bd1e9b3316dcc34b174bbf9664598791e6fb606d8d29000e070c"}, - {file = "websockets-14.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd475a974d5352390baf865309fe37dec6831aafc3014ffac1eea99e84e83fc2"}, - {file = "websockets-14.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c6c0097a41968b2e2b54ed3424739aab0b762ca92af2379f152c1aef0187e1c"}, - {file = "websockets-14.2-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d7ff794c8b36bc402f2e07c0b2ceb4a2424147ed4785ff03e2a7af03711d60a"}, - {file = "websockets-14.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:dec254fcabc7bd488dab64846f588fc5b6fe0d78f641180030f8ea27b76d72c3"}, - {file = "websockets-14.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:bbe03eb853e17fd5b15448328b4ec7fb2407d45fb0245036d06a3af251f8e48f"}, - {file = "websockets-14.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a3c4aa3428b904d5404a0ed85f3644d37e2cb25996b7f096d77caeb0e96a3b42"}, - {file = "websockets-14.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:577a4cebf1ceaf0b65ffc42c54856214165fb8ceeba3935852fc33f6b0c55e7f"}, - {file = "websockets-14.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ad1c1d02357b7665e700eca43a31d52814ad9ad9b89b58118bdabc365454b574"}, - {file = "websockets-14.2-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f390024a47d904613577df83ba700bd189eedc09c57af0a904e5c39624621270"}, - {file = "websockets-14.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3c1426c021c38cf92b453cdf371228d3430acd775edee6bac5a4d577efc72365"}, - {file = "websockets-14.2-py3-none-any.whl", hash = "sha256:7a6ceec4ea84469f15cf15807a747e9efe57e369c384fa86e022b3bea679b79b"}, - {file = "websockets-14.2.tar.gz", hash = "sha256:5059ed9c54945efb321f097084b4c7e52c246f2c869815876a69d1efc4ad6eb5"}, + {file = "websockets-15.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5e6ee18a53dd5743e6155b8ff7e8e477c25b29b440f87f65be8165275c87fef0"}, + {file = "websockets-15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ee06405ea2e67366a661ed313e14cf2a86e84142a3462852eb96348f7219cee3"}, + {file = "websockets-15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8711682a629bbcaf492f5e0af72d378e976ea1d127a2d47584fa1c2c080b436b"}, + {file = "websockets-15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94c4a9b01eede952442c088d415861b0cf2053cbd696b863f6d5022d4e4e2453"}, + {file = "websockets-15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45535fead66e873f411c1d3cf0d3e175e66f4dd83c4f59d707d5b3e4c56541c4"}, + {file = "websockets-15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e389efe46ccb25a1f93d08c7a74e8123a2517f7b7458f043bd7529d1a63ffeb"}, + {file = "websockets-15.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:67a04754d121ea5ca39ddedc3f77071651fb5b0bc6b973c71c515415b44ed9c5"}, + {file = "websockets-15.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:bd66b4865c8b853b8cca7379afb692fc7f52cf898786537dfb5e5e2d64f0a47f"}, + {file = "websockets-15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a4cc73a6ae0a6751b76e69cece9d0311f054da9b22df6a12f2c53111735657c8"}, + {file = "websockets-15.0-cp310-cp310-win32.whl", hash = "sha256:89da58e4005e153b03fe8b8794330e3f6a9774ee9e1c3bd5bc52eb098c3b0c4f"}, + {file = "websockets-15.0-cp310-cp310-win_amd64.whl", hash = "sha256:4ff380aabd7a74a42a760ee76c68826a8f417ceb6ea415bd574a035a111fd133"}, + {file = "websockets-15.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:dd24c4d256558429aeeb8d6c24ebad4e982ac52c50bc3670ae8646c181263965"}, + {file = "websockets-15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f83eca8cbfd168e424dfa3b3b5c955d6c281e8fc09feb9d870886ff8d03683c7"}, + {file = "websockets-15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4095a1f2093002c2208becf6f9a178b336b7572512ee0a1179731acb7788e8ad"}, + {file = "websockets-15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb915101dfbf318486364ce85662bb7b020840f68138014972c08331458d41f3"}, + {file = "websockets-15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45d464622314973d78f364689d5dbb9144e559f93dca11b11af3f2480b5034e1"}, + {file = "websockets-15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ace960769d60037ca9625b4c578a6f28a14301bd2a1ff13bb00e824ac9f73e55"}, + {file = "websockets-15.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c7cd4b1015d2f60dfe539ee6c95bc968d5d5fad92ab01bb5501a77393da4f596"}, + {file = "websockets-15.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4f7290295794b5dec470867c7baa4a14182b9732603fd0caf2a5bf1dc3ccabf3"}, + {file = "websockets-15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3abd670ca7ce230d5a624fd3d55e055215d8d9b723adee0a348352f5d8d12ff4"}, + {file = "websockets-15.0-cp311-cp311-win32.whl", hash = "sha256:110a847085246ab8d4d119632145224d6b49e406c64f1bbeed45c6f05097b680"}, + {file = "websockets-15.0-cp311-cp311-win_amd64.whl", hash = "sha256:8d7bbbe2cd6ed80aceef2a14e9f1c1b61683194c216472ed5ff33b700e784e37"}, + {file = "websockets-15.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cccc18077acd34c8072578394ec79563664b1c205f7a86a62e94fafc7b59001f"}, + {file = "websockets-15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4c22992e24f12de340ca5f824121a5b3e1a37ad4360b4e1aaf15e9d1c42582d"}, + {file = "websockets-15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1206432cc6c644f6fc03374b264c5ff805d980311563202ed7fef91a38906276"}, + {file = "websockets-15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d3cc75ef3e17490042c47e0523aee1bcc4eacd2482796107fd59dd1100a44bc"}, + {file = "websockets-15.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b89504227a5311610e4be16071465885a0a3d6b0e82e305ef46d9b064ce5fb72"}, + {file = "websockets-15.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56e3efe356416bc67a8e093607315951d76910f03d2b3ad49c4ade9207bf710d"}, + {file = "websockets-15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0f2205cdb444a42a7919690238fb5979a05439b9dbb73dd47c863d39640d85ab"}, + {file = "websockets-15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:aea01f40995fa0945c020228ab919b8dfc93fc8a9f2d3d705ab5b793f32d9e99"}, + {file = "websockets-15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a9f8e33747b1332db11cf7fcf4a9512bef9748cb5eb4d3f7fbc8c30d75dc6ffc"}, + {file = "websockets-15.0-cp312-cp312-win32.whl", hash = "sha256:32e02a2d83f4954aa8c17e03fe8ec6962432c39aca4be7e8ee346b05a3476904"}, + {file = "websockets-15.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffc02b159b65c05f2ed9ec176b715b66918a674bd4daed48a9a7a590dd4be1aa"}, + {file = "websockets-15.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d2244d8ab24374bed366f9ff206e2619345f9cd7fe79aad5225f53faac28b6b1"}, + {file = "websockets-15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3a302241fbe825a3e4fe07666a2ab513edfdc6d43ce24b79691b45115273b5e7"}, + {file = "websockets-15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:10552fed076757a70ba2c18edcbc601c7637b30cdfe8c24b65171e824c7d6081"}, + {file = "websockets-15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c53f97032b87a406044a1c33d1e9290cc38b117a8062e8a8b285175d7e2f99c9"}, + {file = "websockets-15.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1caf951110ca757b8ad9c4974f5cac7b8413004d2f29707e4d03a65d54cedf2b"}, + {file = "websockets-15.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bf1ab71f9f23b0a1d52ec1682a3907e0c208c12fef9c3e99d2b80166b17905f"}, + {file = "websockets-15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bfcd3acc1a81f106abac6afd42327d2cf1e77ec905ae11dc1d9142a006a496b6"}, + {file = "websockets-15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c8c5c8e1bac05ef3c23722e591ef4f688f528235e2480f157a9cfe0a19081375"}, + {file = "websockets-15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:86bfb52a9cfbcc09aba2b71388b0a20ea5c52b6517c0b2e316222435a8cdab72"}, + {file = "websockets-15.0-cp313-cp313-win32.whl", hash = "sha256:26ba70fed190708551c19a360f9d7eca8e8c0f615d19a574292b7229e0ae324c"}, + {file = "websockets-15.0-cp313-cp313-win_amd64.whl", hash = "sha256:ae721bcc8e69846af00b7a77a220614d9b2ec57d25017a6bbde3a99473e41ce8"}, + {file = "websockets-15.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c348abc5924caa02a62896300e32ea80a81521f91d6db2e853e6b1994017c9f6"}, + {file = "websockets-15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5294fcb410ed0a45d5d1cdedc4e51a60aab5b2b3193999028ea94afc2f554b05"}, + {file = "websockets-15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c24ba103ecf45861e2e1f933d40b2d93f5d52d8228870c3e7bf1299cd1cb8ff1"}, + {file = "websockets-15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc8821a03bcfb36e4e4705316f6b66af28450357af8a575dc8f4b09bf02a3dee"}, + {file = "websockets-15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc5ae23ada6515f31604f700009e2df90b091b67d463a8401c1d8a37f76c1d7"}, + {file = "websockets-15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ac67b542505186b3bbdaffbc303292e1ee9c8729e5d5df243c1f20f4bb9057e"}, + {file = "websockets-15.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c86dc2068f1c5ca2065aca34f257bbf4f78caf566eb230f692ad347da191f0a1"}, + {file = "websockets-15.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:30cff3ef329682b6182c01c568f551481774c476722020b8f7d0daacbed07a17"}, + {file = "websockets-15.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:98dcf978d4c6048965d1762abd534c9d53bae981a035bfe486690ba11f49bbbb"}, + {file = "websockets-15.0-cp39-cp39-win32.whl", hash = "sha256:37d66646f929ae7c22c79bc73ec4074d6db45e6384500ee3e0d476daf55482a9"}, + {file = "websockets-15.0-cp39-cp39-win_amd64.whl", hash = "sha256:24d5333a9b2343330f0f4eb88546e2c32a7f5c280f8dd7d3cc079beb0901781b"}, + {file = "websockets-15.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b499caef4bca9cbd0bd23cd3386f5113ee7378094a3cb613a2fa543260fe9506"}, + {file = "websockets-15.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:17f2854c6bd9ee008c4b270f7010fe2da6c16eac5724a175e75010aacd905b31"}, + {file = "websockets-15.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89f72524033abbfde880ad338fd3c2c16e31ae232323ebdfbc745cbb1b3dcc03"}, + {file = "websockets-15.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1657a9eecb29d7838e3b415458cc494e6d1b194f7ac73a34aa55c6fb6c72d1f3"}, + {file = "websockets-15.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e413352a921f5ad5d66f9e2869b977e88d5103fc528b6deb8423028a2befd842"}, + {file = "websockets-15.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8561c48b0090993e3b2a54db480cab1d23eb2c5735067213bb90f402806339f5"}, + {file = "websockets-15.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:190bc6ef8690cd88232a038d1b15714c258f79653abad62f7048249b09438af3"}, + {file = "websockets-15.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:327adab7671f3726b0ba69be9e865bba23b37a605b585e65895c428f6e47e766"}, + {file = "websockets-15.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bd8ef197c87afe0a9009f7a28b5dc613bfc585d329f80b7af404e766aa9e8c7"}, + {file = "websockets-15.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:789c43bf4a10cd067c24c321238e800b8b2716c863ddb2294d2fed886fa5a689"}, + {file = "websockets-15.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7394c0b7d460569c9285fa089a429f58465db930012566c03046f9e3ab0ed181"}, + {file = "websockets-15.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ea4f210422b912ebe58ef0ad33088bc8e5c5ff9655a8822500690abc3b1232d"}, + {file = "websockets-15.0-py3-none-any.whl", hash = "sha256:51ffd53c53c4442415b613497a34ba0aa7b99ac07f1e4a62db5dcd640ae6c3c3"}, + {file = "websockets-15.0.tar.gz", hash = "sha256:ca36151289a15b39d8d683fd8b7abbe26fc50be311066c5f8dcf3cb8cee107ab"}, ] [[package]] @@ -2697,4 +2633,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "b89b458ed0f91e834c027be0d2541d464e97d65aaf56b0454c1ed07b004828e1" +content-hash = "32f3b67d60393e53e28f1c3856b0c6f5c32ea538d4ae6cae1847cdd37a001ceb" diff --git a/pyproject.toml b/pyproject.toml index 11a4e50151..3dc81824de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,16 +7,16 @@ requires-poetry = ">=2.0.0" [tool.poetry.dependencies] python = "^3.10" -ops = "^2.17.1" -boto3 = "^1.35.87" +ops = "^2.18.1" +boto3 = "^1.35.99" pgconnstr = "^1.0.1" requests = "^2.32.3" tenacity = "^9.0.0" psycopg2 = "^2.9.10" pydantic = "^1.10.21" jinja2 = "^3.1.5" -pysyncobj = "^0.3.13" -psutil = "^6.1.1" +pysyncobj = "^0.3.14" +psutil = "^7.0.0" [tool.poetry.group.charm-libs.dependencies] # data_platform_libs/v0/data_interfaces.py @@ -27,7 +27,7 @@ poetry-core = "*" # data_platform_libs/v0/data_models.py requires pydantic ^1.10 pydantic = "^1.10" # grafana_agent/v0/cos_agent.py -cosl = "*" +cosl = ">=0.0.50" # tls_certificates_interface/v2/tls_certificates.py cryptography = "*" jsonschema = "*" @@ -38,19 +38,19 @@ opentelemetry-exporter-otlp-proto-http = "1.21.0" optional = true [tool.poetry.group.format.dependencies] -ruff = "^0.9.3" +ruff = "^0.9.6" [tool.poetry.group.lint] optional = true [tool.poetry.group.lint.dependencies] -codespell = "^2.4.0" +codespell = "^2.4.1" [tool.poetry.group.unit] optional = true [tool.poetry.group.unit.dependencies] -coverage = {extras = ["toml"], version = "^7.6.10"} +coverage = {extras = ["toml"], version = "^7.6.12"} pytest = "^8.3.4" pytest-asyncio = "*" parameterized = "^0.9.0" @@ -61,10 +61,7 @@ optional = true [tool.poetry.group.integration.dependencies] pytest = "^8.3.4" -pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v29.1.0", subdirectory = "python/pytest_plugins/github_secrets"} -pytest-operator = "^0.39.0" -pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v29.1.0", subdirectory = "python/pytest_plugins/pytest_operator_cache"} -pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v29.1.0", subdirectory = "python/pytest_plugins/pytest_operator_groups"} +pytest-operator = "^0.40.0" # renovate caret doesn't work: https://github.com/renovatebot/renovate/issues/26940 juju = "<=3.6.1.0" boto3 = "*" @@ -73,7 +70,7 @@ landscape-api-py3 = "^0.9.0" mailmanclient = "^3.3.5" psycopg2-binary = "^2.9.10" allure-pytest = "^2.13.5" -allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v29.1.0", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"} +allure-pytest-default-results = "^0.1.2" # Testing tools configuration [tool.coverage.run] @@ -89,7 +86,7 @@ exclude_lines = [ minversion = "6.0" log_cli_level = "INFO" asyncio_mode = "auto" -markers = ["unstable", "juju2", "juju3", "juju_secrets"] +markers = ["juju3", "juju_secrets"] # Formatting tools configuration [tool.black] diff --git a/spread.yaml b/spread.yaml new file mode 100644 index 0000000000..8e1352e7b3 --- /dev/null +++ b/spread.yaml @@ -0,0 +1,128 @@ +project: postgresql-operator + +backends: + # Derived from https://github.com/jnsgruk/zinc-k8s-operator/blob/a21eae8399eb3b9df4ddb934b837af25ef831976/spread.yaml#L11 + lxd-vm: + # TODO: remove after https://github.com/canonical/spread/pull/185 merged & in charmcraft + type: adhoc + allocate: | + hash=$(python3 -c "import hashlib; print(hashlib.sha256('$SPREAD_PASSWORD'.encode()).hexdigest()[:6])") + VM_NAME="${VM_NAME:-${SPREAD_SYSTEM//./-}-${hash}}" + DISK="${DISK:-20}" + CPU="${CPU:-4}" + MEM="${MEM:-8}" + + cloud_config="#cloud-config + ssh_pwauth: true + users: + - default + - name: runner + plain_text_passwd: $SPREAD_PASSWORD + lock_passwd: false + sudo: ALL=(ALL) NOPASSWD:ALL + " + + lxc launch --vm \ + "${SPREAD_SYSTEM//-/:}" \ + "${VM_NAME}" \ + -c user.user-data="${cloud_config}" \ + -c limits.cpu="${CPU}" \ + -c limits.memory="${MEM}GiB" \ + -d root,size="${DISK}GiB" + + # Wait for the runner user + while ! lxc exec "${VM_NAME}" -- id -u runner &>/dev/null; do sleep 0.5; done + + # Set the instance address for spread + ADDRESS "$(lxc ls -f csv | grep "${VM_NAME}" | cut -d"," -f3 | cut -d" " -f1)" + discard: | + hash=$(python3 -c "import hashlib; print(hashlib.sha256('$SPREAD_PASSWORD'.encode()).hexdigest()[:6])") + VM_NAME="${VM_NAME:-${SPREAD_SYSTEM//./-}-${hash}}" + lxc delete --force "${VM_NAME}" + environment: + CONCIERGE_EXTRA_SNAPS: charmcraft + CONCIERGE_EXTRA_DEBS: pipx + systems: + - ubuntu-24.04: + username: runner + prepare: | + systemctl disable --now unattended-upgrades.service + systemctl mask unattended-upgrades.service + pipx install charmcraftcache + cd "$SPREAD_PATH" + charmcraftcache pack -v + restore-each: | + cd "$SPREAD_PATH" + # Revert python-libjuju version override + git restore pyproject.toml poetry.lock + + # Use instead of `concierge restore` to save time between tests + # For example, with microk8s, using `concierge restore` takes twice as long as this (e.g. 6 + # min instead of 3 min between every spread job) + juju destroy-model --force --no-wait --destroy-storage --no-prompt testing + juju kill-controller --no-prompt concierge-lxd + restore: | + rm -rf "$SPREAD_PATH" + + github-ci: + type: adhoc + # Only run on CI + manual: true + # HACK: spread requires runners to be accessible via SSH + # Configure local sshd & instruct spread to connect to the same machine spread is running on + # (spread cannot provision GitHub Actions runners, so we provision a GitHub Actions runner for + # each spread job & select a single job when running spread) + # Derived from https://github.com/jnsgruk/zinc-k8s-operator/blob/a21eae8399eb3b9df4ddb934b837af25ef831976/spread.yaml#L47 + allocate: | + sudo tee /etc/ssh/sshd_config.d/10-spread-github-ci.conf << 'EOF' + PasswordAuthentication yes + PermitEmptyPasswords yes + EOF + + ADDRESS localhost + # HACK: spread does not pass environment variables set on runner + # Manually pass specific environment variables + environment: + CI: '$(HOST: echo $CI)' + AWS_ACCESS_KEY: '$(HOST: echo $AWS_ACCESS_KEY)' + AWS_SECRET_KEY: '$(HOST: echo $AWS_SECRET_KEY)' + GCP_ACCESS_KEY: '$(HOST: echo $GCP_ACCESS_KEY)' + GCP_SECRET_KEY: '$(HOST: echo $GCP_SECRET_KEY)' + UBUNTU_PRO_TOKEN: '$(HOST: echo $UBUNTU_PRO_TOKEN)' + LANDSCAPE_ACCOUNT_NAME: '$(HOST: echo $LANDSCAPE_ACCOUNT_NAME)' + LANDSCAPE_REGISTRATION_KEY: '$(HOST: echo $LANDSCAPE_REGISTRATION_KEY)' + systems: + - ubuntu-24.04: + username: runner + - ubuntu-24.04-arm: + username: runner + +suites: + tests/spread/: + summary: Spread tests + +path: /root/spread_project + +kill-timeout: 3h +environment: + PATH: $PATH:$(pipx environment --value PIPX_BIN_DIR) + CONCIERGE_JUJU_CHANNEL/juju36: 3.6/stable +prepare: | + snap refresh --hold + chown -R root:root "$SPREAD_PATH" + cd "$SPREAD_PATH" + snap install --classic concierge + + # Install charmcraft & pipx (on lxd-vm backend) + concierge prepare --trace + + pipx install tox poetry +prepare-each: | + cd "$SPREAD_PATH" + # `concierge prepare` needs to be run for each spread job in case Juju version changed + concierge prepare --trace + + # Unable to set constraint on all models because of Juju bug: + # https://bugs.launchpad.net/juju/+bug/2065050 + juju set-model-constraints arch="$(dpkg --print-architecture)" +# Only restore on lxd backend—no need to restore on CI diff --git a/src/backups.py b/src/backups.py index 214fe7f082..903ba028f6 100644 --- a/src/backups.py +++ b/src/backups.py @@ -212,7 +212,10 @@ def can_use_s3_repository(self) -> tuple[bool, str | None]: for line in system_identifier_from_instance.splitlines() if "Database system identifier" in line ).split(" ")[-1] - system_identifier_from_stanza = str(stanza.get("db")[0]["system-id"]) + stanza_dbs = stanza.get("db") + system_identifier_from_stanza = ( + str(stanza_dbs[0]["system-id"]) if len(stanza_dbs) else None + ) if system_identifier_from_instance != system_identifier_from_stanza: logger.debug( f"can_use_s3_repository: incompatible system identifier s3={system_identifier_from_stanza}, local={system_identifier_from_instance}" diff --git a/src/charm.py b/src/charm.py index bcb5cbcac6..a814a710de 100755 --- a/src/charm.py +++ b/src/charm.py @@ -61,6 +61,7 @@ Patroni, RemoveRaftMemberFailedError, SwitchoverFailedError, + SwitchoverNotSyncError, ) from cluster_topology_observer import ( ClusterTopologyChangeCharmEvents, @@ -70,6 +71,7 @@ from constants import ( APP_SCOPE, BACKUP_USER, + DATABASE_DEFAULT_NAME, METRICS_PORT, MONITORING_PASSWORD_KEY, MONITORING_SNAP_SERVICE, @@ -101,7 +103,6 @@ REPLICATION_OFFER_RELATION, PostgreSQLAsyncReplication, ) -from relations.db import EXTENSIONS_BLOCKING_MESSAGE, DbProvides from relations.postgresql_provider import PostgreSQLProvider from rotate_logs import RotateLogs from upgrade import PostgreSQLUpgrade, get_postgresql_dependencies_model @@ -126,7 +127,6 @@ class CannotConnectError(Exception): extra_types=( ClusterTopologyObserver, COSAgentProvider, - DbProvides, Patroni, PostgreSQL, PostgreSQLAsyncReplication, @@ -181,7 +181,6 @@ def __init__(self, *args): self.framework.observe(self.on[PEER].relation_changed, self._on_peer_relation_changed) self.framework.observe(self.on.secret_changed, self._on_peer_relation_changed) self.framework.observe(self.on[PEER].relation_departed, self._on_peer_relation_departed) - self.framework.observe(self.on.pgdata_storage_detaching, self._on_pgdata_storage_detaching) self.framework.observe(self.on.start, self._on_start) self.framework.observe(self.on.get_password_action, self._on_get_password) self.framework.observe(self.on.set_password_action, self._on_set_password) @@ -198,8 +197,6 @@ def __init__(self, *args): substrate="vm", ) self.postgresql_client_relation = PostgreSQLProvider(self) - self.legacy_db_relation = DbProvides(self, admin=False) - self.legacy_db_admin_relation = DbProvides(self, admin=True) self.backup = PostgreSQLBackups(self, "s3-parameters") self.tls = PostgreSQLTLS(self, PEER) self.async_replication = PostgreSQLAsyncReplication(self) @@ -294,8 +291,6 @@ def peer_relation_data(self, scope: Scopes) -> DataPeerData: def _translate_field_to_secret_key(self, key: str) -> str: """Change 'key' to secrets-compatible key field.""" - if not JujuVersion.from_environ().has_secrets: - return key key = SECRET_KEY_OVERRIDES.get(key, key) new_key = key.replace("_", "-") return new_key.strip("-") @@ -308,10 +303,6 @@ def get_secret(self, scope: Scopes, key: str) -> str | None: if not (peers := self.model.get_relation(PEER)): return None secret_key = self._translate_field_to_secret_key(key) - # Old translation in databag is to be taken - if result := self.peer_relation_data(scope).fetch_my_relation_field(peers.id, key): - return result - return self.peer_relation_data(scope).get_secret(peers.id, secret_key) def set_secret(self, scope: Scopes, key: str, value: str | None) -> str | None: @@ -325,8 +316,6 @@ def set_secret(self, scope: Scopes, key: str, value: str | None) -> str | None: if not (peers := self.model.get_relation(PEER)): return None secret_key = self._translate_field_to_secret_key(key) - # Old translation in databag is to be deleted - self.scoped_peer_data(scope).pop(key, None) self.peer_relation_data(scope).set_secret(peers.id, secret_key, value) def remove_secret(self, scope: Scopes, key: str) -> None: @@ -337,7 +326,6 @@ def remove_secret(self, scope: Scopes, key: str) -> None: if not (peers := self.model.get_relation(PEER)): return None secret_key = self._translate_field_to_secret_key(key) - self.peer_relation_data(scope).delete_relation_data(peers.id, [secret_key]) @property @@ -373,7 +361,7 @@ def postgresql(self) -> PostgreSQL: current_host=self._unit_ip, user=USER, password=self.get_secret(APP_SCOPE, f"{USER}-password"), - database="postgres", + database=DATABASE_DEFAULT_NAME, system_users=SYSTEM_USERS, ) @@ -431,10 +419,10 @@ def _on_get_primary(self, event: ActionEvent) -> None: except RetryError as e: logger.error(f"failed to get primary with error {e}") - def _updated_synchronous_node_count(self, num_units: int | None = None) -> bool: + def updated_synchronous_node_count(self) -> bool: """Tries to update synchronous_node_count configuration and reports the result.""" try: - self._patroni.update_synchronous_node_count(num_units) + self._patroni.update_synchronous_node_count() return True except RetryError: logger.debug("Unable to set synchronous_node_count") @@ -472,9 +460,7 @@ def _on_peer_relation_departed(self, event: RelationDepartedEvent) -> None: if not self.unit.is_leader(): return - if not self.is_cluster_initialised or not self._updated_synchronous_node_count( - len(self._units_ips) - ): + if not self.is_cluster_initialised or not self.updated_synchronous_node_count(): logger.debug("Deferring on_peer_relation_departed: cluster not initialized") event.defer() return @@ -500,52 +486,6 @@ def _on_peer_relation_departed(self, event: RelationDepartedEvent) -> None: # Update the sync-standby endpoint in the async replication data. self.async_replication.update_async_replication_data() - def _on_pgdata_storage_detaching(self, _) -> None: - # Change the primary if it's the unit that is being removed. - try: - primary = self._patroni.get_primary(unit_name_pattern=True) - except RetryError: - # Ignore the event if the primary couldn't be retrieved. - # If a switchover is needed, an automatic failover will be triggered - # when the unit is removed. - logger.debug("Early exit on_pgdata_storage_detaching: primary cannot be retrieved") - return - - if self.unit.name != primary: - return - - if not self._patroni.are_all_members_ready(): - logger.warning( - "could not switchover because not all members are ready" - " - an automatic failover will be triggered" - ) - return - - # Try to switchover to another member and raise an exception if it doesn't succeed. - # If it doesn't happen on time, Patroni will automatically run a fail-over. - try: - # Get the current primary to check if it has changed later. - current_primary = self._patroni.get_primary() - - # Trigger the switchover. - self._patroni.switchover() - - # Wait for the switchover to complete. - self._patroni.primary_changed(current_primary) - - logger.info("successful switchover") - except (RetryError, SwitchoverFailedError) as e: - logger.warning( - f"switchover failed with reason: {e} - an automatic failover will be triggered" - ) - return - - # Only update the connection endpoints if there is a primary. - # A cluster can have all members as replicas for some time after - # a failed switchover, so wait until the primary is elected. - if self.primary_endpoint: - self._update_relation_endpoints() - def _stuck_raft_cluster_check(self) -> None: """Check for stuck raft cluster and reinitialise if safe.""" raft_stuck = False @@ -1183,6 +1123,11 @@ def _on_config_changed(self, event) -> None: logger.error("Invalid configuration: %s", str(e)) return + if not self.updated_synchronous_node_count(): + logger.debug("Defer on_config_changed: unable to set synchronous node count") + event.defer() + return + if self.is_blocked and "Configuration Error" in self.unit.status.message: self.unit.status = ActiveStatus() @@ -1195,21 +1140,6 @@ def _on_config_changed(self, event) -> None: # Enable and/or disable the extensions. self.enable_disable_extensions() - # Unblock the charm after extensions are enabled (only if it's blocked due to application - # charms requesting extensions). - if self.unit.status.message != EXTENSIONS_BLOCKING_MESSAGE: - return - - for relation in [ - *self.model.relations.get("db", []), - *self.model.relations.get("db-admin", []), - ]: - if not self.legacy_db_relation.set_up_relation(relation): - logger.debug( - "Early exit on_config_changed: legacy relation requested extensions that are still disabled" - ) - return - def enable_disable_extensions(self, database: str | None = None) -> None: """Enable/disable PostgreSQL extensions set through config options. @@ -1393,7 +1323,7 @@ def _start_primary(self, event: StartEvent) -> None: self.postgresql.create_user( MONITORING_USER, self.get_secret(APP_SCOPE, MONITORING_PASSWORD_KEY), - extra_user_roles="pg_monitor", + extra_user_roles=["pg_monitor"], ) except PostgreSQLCreateUserError as e: logger.exception(e) @@ -1552,8 +1482,10 @@ def promote_primary_unit(self, event: ActionEvent) -> None: return try: self._patroni.switchover(self._member_name) - except SwitchoverFailedError: + except SwitchoverNotSyncError: event.fail("Unit is not sync standby") + except SwitchoverFailedError: + event.fail("Switchover failed or timed out, check the logs for details") def _on_update_status(self, _) -> None: """Update the unit status message and users list in the database.""" @@ -1945,6 +1877,12 @@ def update_config(self, is_creating_backup: bool = False, no_peers: bool = False return True if not self._patroni.member_started: + if enable_tls: + logger.debug( + "Early exit update_config: patroni not responding but TLS is enabled." + ) + self._handle_postgresql_restart_need(True) + return True logger.debug("Early exit update_config: Patroni not started yet") return False @@ -2001,8 +1939,14 @@ def _validate_config_options(self) -> None: def _handle_postgresql_restart_need(self, enable_tls: bool) -> None: """Handle PostgreSQL restart need based on the TLS configuration and configuration changes.""" - restart_postgresql = self.is_tls_enabled != self.postgresql.is_tls_enabled() - self._patroni.reload_patroni_configuration() + if self._can_connect_to_postgresql: + restart_postgresql = self.is_tls_enabled != self.postgresql.is_tls_enabled() + else: + restart_postgresql = False + try: + self._patroni.reload_patroni_configuration() + except Exception as e: + logger.error(f"Reload patroni call failed! error: {e!s}") # Wait for some more time than the Patroni's loop_wait default value (10 seconds), # which tells how much time Patroni will wait before checking the configuration # file again to reload it. @@ -2028,8 +1972,6 @@ def _handle_postgresql_restart_need(self, enable_tls: bool) -> None: def _update_relation_endpoints(self) -> None: """Updates endpoints and read-only endpoint in all relations.""" self.postgresql_client_relation.update_endpoints() - self.legacy_db_relation.update_endpoints() - self.legacy_db_admin_relation.update_endpoints() def get_available_memory(self) -> int: """Returns the system available memory in bytes.""" @@ -2043,11 +1985,7 @@ def get_available_memory(self) -> int: @property def client_relations(self) -> list[Relation]: """Return the list of established client relations.""" - relations = [] - for relation_name in ["database", "db", "db-admin"]: - for relation in self.model.relations.get(relation_name, []): - relations.append(relation) - return relations + return self.model.relations.get("database", []) def override_patroni_restart_condition( self, new_condition: str, repeat_cause: str | None diff --git a/src/cluster.py b/src/cluster.py index cf7801b95d..6d227cda26 100644 --- a/src/cluster.py +++ b/src/cluster.py @@ -94,6 +94,10 @@ class SwitchoverFailedError(Exception): """Raised when a switchover failed for some reason.""" +class SwitchoverNotSyncError(SwitchoverFailedError): + """Raised when a switchover failed because node is not sync.""" + + class UpdateSyncNodeCountError(Exception): """Raised when updating synchronous_node_count failed for some reason.""" @@ -668,7 +672,7 @@ def render_patroni_yml_file( stanza=stanza, restore_stanza=restore_stanza, version=self.get_postgresql_version().split(".")[0], - minority_count=self.planned_units // 2, + synchronous_node_count=self._synchronous_node_count, pg_parameters=parameters, primary_cluster_endpoint=self.charm.async_replication.get_primary_cluster_endpoint(), extra_replication_endpoints=self.charm.async_replication.get_standby_endpoints(), @@ -766,6 +770,13 @@ def switchover(self, candidate: str | None = None) -> None: # Check whether the switchover was unsuccessful. if r.status_code != 200: + if ( + r.status_code == 412 + and r.text == "candidate name does not match with sync_standby" + ): + logger.debug("Unit is not sync standby") + raise SwitchoverNotSyncError() + logger.warning(f"Switchover call failed with code {r.status_code} {r.text}") raise SwitchoverFailedError(f"received {r.status_code}") @retry( @@ -915,9 +926,10 @@ def remove_raft_member(self, member_ip: str) -> None: raise RemoveRaftMemberFailedError() from None if not result.startswith("SUCCESS"): + logger.debug("Remove raft member: Remove call not successful") raise RemoveRaftMemberFailedError() - @retry(stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=10)) + @retry(stop=stop_after_attempt(20), wait=wait_exponential(multiplier=1, min=2, max=10)) def reload_patroni_configuration(self): """Reload Patroni configuration after it was changed.""" requests.post( @@ -977,16 +989,28 @@ def bulk_update_parameters_controller_by_patroni(self, parameters: dict[str, Any timeout=PATRONI_TIMEOUT, ) - def update_synchronous_node_count(self, units: int | None = None) -> None: + @property + def _synchronous_node_count(self) -> int: + planned_units = self.charm.app.planned_units() + if self.charm.config.synchronous_node_count == "all": + return planned_units - 1 + elif self.charm.config.synchronous_node_count == "majority": + return planned_units // 2 + # -1 for leader + return ( + self.charm.config.synchronous_node_count + if self.charm.config.synchronous_node_count < planned_units - 1 + else planned_units - 1 + ) + + def update_synchronous_node_count(self) -> None: """Update synchronous_node_count to the minority of the planned cluster.""" - if units is None: - units = self.planned_units # Try to update synchronous_node_count. for attempt in Retrying(stop=stop_after_delay(60), wait=wait_fixed(3)): with attempt: r = requests.patch( f"{self._patroni_url}/config", - json={"synchronous_node_count": units // 2}, + json={"synchronous_node_count": self._synchronous_node_count}, verify=self.verify, auth=self._patroni_auth, timeout=PATRONI_TIMEOUT, diff --git a/src/config.py b/src/config.py index 45b7b04566..6306243df2 100644 --- a/src/config.py +++ b/src/config.py @@ -8,7 +8,7 @@ from typing import Literal from charms.data_platform_libs.v0.data_models import BaseConfigModel -from pydantic import validator +from pydantic import PositiveInt, validator from locales import SNAP_LOCALES @@ -18,6 +18,7 @@ class CharmConfig(BaseConfigModel): """Manager for the structured configuration.""" + synchronous_node_count: Literal["all", "majority"] | PositiveInt durability_synchronous_commit: str | None instance_default_text_search_config: str | None instance_max_locks_per_transaction: int | None diff --git a/src/constants.py b/src/constants.py index ebced6479d..9eb1ac152c 100644 --- a/src/constants.py +++ b/src/constants.py @@ -6,12 +6,10 @@ BACKUP_ID_FORMAT = "%Y-%m-%dT%H:%M:%SZ" PGBACKREST_BACKUP_ID_FORMAT = "%Y%m%d-%H%M%S" DATABASE = "database" +DATABASE_DEFAULT_NAME = "postgres" DATABASE_PORT = "5432" -LEGACY_DB = "db" -LEGACY_DB_ADMIN = "db-admin" PEER = "database-peers" -ALL_CLIENT_RELATIONS = [DATABASE, LEGACY_DB, LEGACY_DB_ADMIN] -ALL_LEGACY_RELATIONS = [LEGACY_DB, LEGACY_DB_ADMIN] +ALL_CLIENT_RELATIONS = [DATABASE] API_REQUEST_TIMEOUT = 5 PATRONI_CLUSTER_STATUS_ENDPOINT = "cluster" BACKUP_USER = "backup" @@ -34,7 +32,7 @@ SNAP_PACKAGES = [ ( POSTGRESQL_SNAP_NAME, - {"revision": {"aarch64": "142", "x86_64": "143"}}, + {"revision": {"aarch64": "160", "x86_64": "161"}}, ) ] diff --git a/src/dependency.json b/src/dependency.json index 8071abbc37..cc4e9f524d 100644 --- a/src/dependency.json +++ b/src/dependency.json @@ -8,7 +8,7 @@ "snap": { "dependencies": {}, "name": "charmed-postgresql", - "upgrade_supported": "^14", - "version": "14.12" + "upgrade_supported": "^16", + "version": "16.6" } } diff --git a/src/locales.py b/src/locales.py index 7fd67a0ee8..d9422a958b 100644 --- a/src/locales.py +++ b/src/locales.py @@ -8,7 +8,6 @@ "aa_DJ", "aa_DJ.utf8", "aa_ER", - "aa_ER@saaho", "aa_ET", "af_ZA", "af_ZA.utf8", @@ -97,6 +96,7 @@ "chr_US", "ckb_IQ", "cmn_TW", + "crh_RU", "crh_UA", "cs_CZ", "cs_CZ.utf8", @@ -248,6 +248,7 @@ "ga_IE", "ga_IE.utf8", "ga_IE@euro", + "gbm_IN", "gd_GB", "gd_GB.utf8", "gez_ER", @@ -308,6 +309,7 @@ "ks_IN@devanagari", "ku_TR", "ku_TR.utf8", + "kv_RU", "kw_GB", "kw_GB.utf8", "ky_KG", @@ -389,6 +391,7 @@ "pt_PT@euro", "quz_PE", "raj_IN", + "rif_MA", "ro_RO", "ro_RO.utf8", "ru_RU", @@ -430,8 +433,10 @@ "sr_RS", "sr_RS@latin", "ss_ZA", + "ssy_ER", "st_ZA", "st_ZA.utf8", + "su_ID", "sv_FI", "sv_FI.utf8", "sv_FI@euro", @@ -440,6 +445,7 @@ "sv_SE.utf8", "sw_KE", "sw_TZ", + "syr", "szl_PL", "ta_IN", "ta_LK", @@ -458,6 +464,7 @@ "tl_PH.utf8", "tn_ZA", "to_TO", + "tok", "tpi_PG", "tr_CY", "tr_CY.utf8", @@ -491,6 +498,7 @@ "yo_NG", "yue_HK", "yuw_PG", + "zgh_MA", "zh_CN", "zh_CN.gb18030", "zh_CN.gbk", diff --git a/src/relations/db.py b/src/relations/db.py deleted file mode 100644 index 5f7d8f9ea5..0000000000 --- a/src/relations/db.py +++ /dev/null @@ -1,436 +0,0 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Library containing the implementation of the legacy db and db-admin relations.""" - -import logging -from collections.abc import Iterable - -from charms.postgresql_k8s.v0.postgresql import ( - PostgreSQLCreateDatabaseError, - PostgreSQLCreateUserError, - PostgreSQLGetPostgreSQLVersionError, -) -from ops.charm import ( - CharmBase, - RelationBrokenEvent, - RelationChangedEvent, - RelationDepartedEvent, -) -from ops.framework import Object -from ops.model import ActiveStatus, BlockedStatus, Relation, Unit -from pgconnstr import ConnectionString - -from constants import ( - ALL_LEGACY_RELATIONS, - APP_SCOPE, - DATABASE_PORT, - ENDPOINT_SIMULTANEOUSLY_BLOCKING_MESSAGE, -) -from utils import new_password - -logger = logging.getLogger(__name__) - -EXTENSIONS_BLOCKING_MESSAGE = ( - "extensions requested through relation, enable them through config options" -) - -ROLES_BLOCKING_MESSAGE = ( - "roles requested through relation, use postgresql_client interface instead" -) - - -class DbProvides(Object): - """Defines functionality for the 'provides' side of the 'db' relation. - - Hook events observed: - - relation-changed - - relation-departed - - relation-broken - """ - - def __init__(self, charm: CharmBase, admin: bool = False): - """Constructor for DbProvides object. - - Args: - charm: the charm for which this relation is provided - admin: a boolean defining whether this relation has admin permissions, switching - between "db" and "db-admin" relations. - """ - if admin: - self.relation_name = "db-admin" - else: - self.relation_name = "db" - - super().__init__(charm, self.relation_name) - - self.framework.observe( - charm.on[self.relation_name].relation_changed, self._on_relation_changed - ) - self.framework.observe( - charm.on[self.relation_name].relation_departed, self._on_relation_departed - ) - self.framework.observe( - charm.on[self.relation_name].relation_broken, self._on_relation_broken - ) - - self.admin = admin - self.charm = charm - - def _check_for_blocking_relations(self, relation_id: int) -> bool: - """Checks if there are relations with extensions or roles. - - Args: - relation_id: current relation to be skipped - """ - for relname in ["db", "db-admin"]: - for relation in self.charm.model.relations.get(relname, []): - if relation.id == relation_id: - continue - for data in relation.data.values(): - if "extensions" in data or "roles" in data: - return True - return False - - def _check_exist_current_relation(self) -> bool: - return any(r in ALL_LEGACY_RELATIONS for r in self.charm.client_relations) - - def _check_multiple_endpoints(self) -> bool: - """Checks if there are relations with other endpoints.""" - is_exist = self._check_exist_current_relation() - for relation in self.charm.client_relations: - if relation.name not in ALL_LEGACY_RELATIONS and is_exist: - return True - return False - - def _on_relation_changed(self, event: RelationChangedEvent) -> None: - """Handle the legacy db/db-admin relation changed event. - - Generate password and handle user and database creation for the related application. - """ - # Check for some conditions before trying to access the PostgreSQL instance. - if not self.charm.unit.is_leader(): - return - - if self._check_multiple_endpoints(): - self.charm.unit.status = BlockedStatus(ENDPOINT_SIMULTANEOUSLY_BLOCKING_MESSAGE) - return - - if ( - not self.charm.is_cluster_initialised - or not self.charm._patroni.member_started - or not self.charm.primary_endpoint - ): - logger.debug( - "Deferring on_relation_changed: cluster not initialized, Patroni not started or primary endpoint not available" - ) - event.defer() - return - - logger.warning(f"DEPRECATION WARNING - `{self.relation_name}` is a legacy interface") - - self.set_up_relation(event.relation) - - def _get_extensions(self, relation: Relation) -> tuple[list, set]: - """Returns the list of required and disabled extensions.""" - requested_extensions = relation.data.get(relation.app, {}).get("extensions", "").split(",") - for unit in relation.units: - requested_extensions.extend( - relation.data.get(unit, {}).get("extensions", "").split(",") - ) - required_extensions = [] - for extension in requested_extensions: - if extension != "" and extension not in required_extensions: - required_extensions.append(extension) - disabled_extensions = set() - if required_extensions: - for extension in required_extensions: - extension_name = extension.split(":")[0] - if not self.charm.model.config.get(f"plugin_{extension_name}_enable"): - disabled_extensions.add(extension_name) - return required_extensions, disabled_extensions - - def _get_roles(self, relation: Relation) -> bool: - """Checks if relation required roles.""" - return "roles" in relation.data.get(relation.app, {}) - - def set_up_relation(self, relation: Relation) -> bool: - """Set up the relation to be used by the application charm.""" - # Do not allow apps requesting extensions to be installed - # (let them now about config options). - _, disabled_extensions = self._get_extensions(relation) - if disabled_extensions: - logger.error( - f"ERROR - `extensions` ({', '.join(disabled_extensions)}) cannot be requested through relations" - " - Please enable extensions through `juju config` and add the relation again." - ) - self.charm.unit.status = BlockedStatus(EXTENSIONS_BLOCKING_MESSAGE) - return False - if self._get_roles(relation): - self.charm.unit.status = BlockedStatus(ROLES_BLOCKING_MESSAGE) - return False - - user = f"relation-{relation.id}" - database = relation.data.get(relation.app, {}).get( - "database", self.charm.get_secret(APP_SCOPE, f"{user}-database") - ) - if not database: - for unit in relation.units: - unit_database = relation.data.get(unit, {}).get("database") - if unit_database: - database = unit_database - break - - # Sometimes a relation changed event is triggered, and it doesn't have - # a database name in it (like the relation with Landscape server charm), - # so create a database with the other application name. - if not database: - database = relation.app.name - - try: - unit_relation_databag = relation.data[self.charm.unit] - - # Creates the user and the database for this specific relation if it was not already - # created in a previous relation changed event. - password = unit_relation_databag.get("password", new_password()) - - # Store the user, password and database name in the secret store to be accessible by - # non-leader units when the cluster topology changes. - self.charm.set_secret(APP_SCOPE, user, password) - self.charm.set_secret(APP_SCOPE, f"{user}-database", database) - - self.charm.postgresql.create_user(user, password, self.admin) - plugins = self.charm.get_plugins() - - self.charm.postgresql.create_database( - database, user, plugins=plugins, client_relations=self.charm.client_relations - ) - - except (PostgreSQLCreateDatabaseError, PostgreSQLCreateUserError) as e: - logger.exception(e) - self.charm.unit.status = BlockedStatus( - f"Failed to initialize {self.relation_name} relation" - ) - return False - - self.update_endpoints(relation) - - self._update_unit_status(relation) - - return True - - def _on_relation_departed(self, event: RelationDepartedEvent) -> None: - """Handle the departure of legacy db and db-admin relations. - - Remove unit name from allowed_units key. - """ - # Set a flag to avoid deleting database users when this unit - # is removed and receives relation broken events from related applications. - # This is needed because of https://bugs.launchpad.net/juju/+bug/1979811. - # Neither peer relation data nor stored state are good solutions, - # just a temporary solution. - if event.departing_unit == self.charm.unit: - self.charm._peers.data[self.charm.unit].update({"departing": "True"}) - # Just run the rest of the logic for departing of remote units. - logger.debug("Early exit on_relation_departed: Skipping departing unit") - return - - # Check for some conditions before trying to access the PostgreSQL instance. - if not self.charm.unit.is_leader(): - return - - if ( - not self.charm.is_cluster_initialised - or not self.charm._patroni.member_started - or not self.charm.primary_endpoint - ): - logger.debug( - "Deferring on_relation_departed: cluster not initialized, Patroni not started or primary endpoint not available" - ) - event.defer() - return - - departing_unit = event.departing_unit.name - local_unit_data = event.relation.data[self.charm.unit] - local_app_data = event.relation.data[self.charm.app] - - current_allowed_units = local_unit_data.get("allowed_units", "") - - logger.debug(f"Removing unit {departing_unit} from allowed_units") - local_app_data["allowed_units"] = local_unit_data["allowed_units"] = " ".join({ - unit for unit in current_allowed_units.split() if unit != departing_unit - }) - - def _on_relation_broken(self, event: RelationBrokenEvent) -> None: - """Remove the user created for this relation.""" - # Check for some conditions before trying to access the PostgreSQL instance. - if ( - not self.charm.unit.is_leader() - or not self.charm.is_cluster_initialised - or not self.charm._patroni.member_started - or not self.charm.primary_endpoint - ): - logger.debug( - "Early exit on_relation_broken: Not leader, cluster not initialized, Patroni not started or no primary endpoint" - ) - return - - # Run this event only if this unit isn't being - # removed while the others from this application - # are still alive. This check is needed because of - # https://bugs.launchpad.net/juju/+bug/1979811. - # Neither peer relation data nor stored state - # are good solutions, just a temporary solution. - if self.charm.is_unit_departing: - logger.debug("Early exit on_relation_broken: Skipping departing unit") - return - - self._update_unit_status(event.relation) - - def _update_unit_status(self, relation: Relation) -> None: - """Clean up Blocked status if it's due to extensions request.""" - if ( - self.charm.is_blocked - and self.charm.unit.status.message - in [ - EXTENSIONS_BLOCKING_MESSAGE, - ROLES_BLOCKING_MESSAGE, - ] - and not self._check_for_blocking_relations(relation.id) - ): - self.charm.unit.status = ActiveStatus() - self._update_unit_status_on_blocking_endpoint_simultaneously() - - def _update_unit_status_on_blocking_endpoint_simultaneously(self): - """Clean up Blocked status if this is due related of multiple endpoints.""" - if ( - self.charm.is_blocked - and self.charm.unit.status.message == ENDPOINT_SIMULTANEOUSLY_BLOCKING_MESSAGE - and not self._check_multiple_endpoints() - ): - self.charm.unit.status = ActiveStatus() - - def update_endpoints(self, relation: Relation = None) -> None: - """Set the read/write and read-only endpoints.""" - # Get the current relation or all the relations - # if this is triggered by another type of event. - relations = [relation] if relation else self.model.relations[self.relation_name] - if len(relations) == 0: - return - - postgresql_version = None - try: - postgresql_version = self.charm.postgresql.get_postgresql_version() - except PostgreSQLGetPostgreSQLVersionError: - logger.exception( - f"Failed to retrieve the PostgreSQL version to initialise/update {self.relation_name} relation" - ) - - # List the replicas endpoints. - replicas_endpoint = list(self.charm.members_ips - {self.charm.primary_endpoint}) - replicas_endpoint.sort() - - for relation in relations: - # Retrieve some data from the relation. - unit_relation_databag = relation.data[self.charm.unit] - user = f"relation-{relation.id}" - password = self.charm.get_secret(APP_SCOPE, user) - database = self.charm.get_secret(APP_SCOPE, f"{user}-database") - - # If the relation data is not complete, the relations was not initialised yet. - if not database or not user or not password: - continue - - # Build the primary's connection string. - primary_endpoint = str( - ConnectionString( - host=self.charm.primary_endpoint, - dbname=database, - port=DATABASE_PORT, - user=user, - password=password, - ) - ) - - # If there are no replicas, remove the read-only endpoint. - read_only_endpoints = ( - ",".join( - str( - ConnectionString( - host=replica_endpoint, - dbname=database, - port=DATABASE_PORT, - user=user, - password=password, - ) - ) - for replica_endpoint in replicas_endpoint - ) - if len(replicas_endpoint) > 0 - else "" - ) - - required_extensions, _ = self._get_extensions(relation) - # Set the read/write endpoint. - allowed_subnets = self._get_allowed_subnets(relation) - allowed_units = self._get_allowed_units(relation) - data = { - "allowed-subnets": allowed_subnets, - "allowed-units": allowed_units, - "host": self.charm.primary_endpoint, - "port": DATABASE_PORT, - "user": user, - "schema_user": user, - "password": password, - "schema_password": password, - "database": database, - "master": primary_endpoint, - "standbys": read_only_endpoints, - "state": self._get_state(), - "extensions": ",".join(required_extensions), - } - if postgresql_version: - data["version"] = postgresql_version - - # Set the data only in the unit databag. - unit_relation_databag.update(data) - - def _get_allowed_subnets(self, relation: Relation) -> str: - """Build the list of allowed subnets as in the legacy charm.""" - - def _comma_split(s) -> Iterable[str]: - if s: - for b in s.split(","): - b = b.strip() - if b: - yield b - - subnets = set() - for unit, relation_data in relation.data.items(): - if isinstance(unit, Unit) and not unit.name.startswith(self.model.app.name): - # Egress-subnets is not always available. - subnets.update(set(_comma_split(relation_data.get("egress-subnets", "")))) - return ",".join(sorted(subnets)) - - def _get_allowed_units(self, relation: Relation) -> str: - """Build the list of allowed units as in the legacy charm.""" - return " ".join( - sorted( - unit.name - for unit in relation.data - if isinstance(unit, Unit) and not unit.name.startswith(self.model.app.name) - ) - ) - - def _get_state(self) -> str: - """Gets the given state for this unit. - - Returns: - The state of this unit. Can be 'standalone', 'master', or 'standby'. - """ - if len(self.charm._peers.units) == 0: - return "standalone" - if self.charm._patroni.get_primary(unit_name_pattern=True) == self.charm.unit.name: - return "master" - else: - return "hot standby" diff --git a/src/relations/postgresql_provider.py b/src/relations/postgresql_provider.py index 487c6fb9e5..0f2fa040c1 100644 --- a/src/relations/postgresql_provider.py +++ b/src/relations/postgresql_provider.py @@ -65,6 +65,14 @@ def __init__(self, charm: CharmBase, relation_name: str = "database") -> None: self.database_provides.on.database_requested, self._on_database_requested ) + @staticmethod + def _sanitize_extra_roles(extra_roles: str | None) -> list[str]: + """Standardize and sanitize user extra-roles.""" + if extra_roles is None: + return [] + + return [role.lower() for role in extra_roles.split(",")] + def _on_database_requested(self, event: DatabaseRequestedEvent) -> None: """Generate password and handle user and database creation for the related application.""" # Check for some conditions before trying to access the PostgreSQL instance. @@ -84,7 +92,9 @@ def _on_database_requested(self, event: DatabaseRequestedEvent) -> None: # Retrieve the database name and extra user roles using the charm library. database = event.database - extra_user_roles = event.extra_user_roles + + # Make sure that certain groups are not in the list + extra_user_roles = self._sanitize_extra_roles(event.extra_user_roles) try: # Creates the user and the database for this specific relation. @@ -275,9 +285,7 @@ def check_for_invalid_extra_user_roles(self, relation_id: int) -> bool: continue for data in relation.data.values(): extra_user_roles = data.get("extra-user-roles") - if extra_user_roles is None: - continue - extra_user_roles = extra_user_roles.lower().split(",") + extra_user_roles = self._sanitize_extra_roles(extra_user_roles) for extra_user_role in extra_user_roles: if ( extra_user_role not in valid_privileges diff --git a/src/upgrade.py b/src/upgrade.py index 629ba06fa8..d2de69bfde 100644 --- a/src/upgrade.py +++ b/src/upgrade.py @@ -114,8 +114,8 @@ def _on_upgrade_charm_check_legacy(self) -> None: fixed_dependencies["snap"] = { "dependencies": {}, "name": "charmed-postgresql", - "upgrade_supported": "^14", - "version": "14.9", + "upgrade_supported": "^16", + "version": "16.6", } self.peer_relation.data[self.charm.app].update({ "dependencies": json.dumps(fixed_dependencies) @@ -146,6 +146,7 @@ def _on_upgrade_granted(self, event: UpgradeGrantedEvent) -> None: # Update the configuration. self.charm.unit.status = MaintenanceStatus("updating configuration") self.charm.update_config() + self.charm.updated_synchronous_node_count() self.charm.unit.status = MaintenanceStatus("refreshing the snap") self.charm._install_snap_packages(packages=SNAP_PACKAGES, refresh=True) diff --git a/templates/patroni.yml.j2 b/templates/patroni.yml.j2 index b072254c3a..9de9d71d86 100644 --- a/templates/patroni.yml.j2 +++ b/templates/patroni.yml.j2 @@ -59,7 +59,7 @@ bootstrap: retry_timeout: 10 maximum_lag_on_failover: 1048576 synchronous_mode: true - synchronous_node_count: {{ minority_count }} + synchronous_node_count: {{ synchronous_node_count }} postgresql: use_pg_rewind: true remove_data_directory_on_rewind_failure: true @@ -183,12 +183,27 @@ postgresql: replication: username: replication password: {{ replication_password }} + {%- if enable_tls %} + sslrootcert: {{ conf_path }}/ca.pem + sslcert: {{ conf_path }}/cert.pem + sslkey: {{ conf_path }}/key.pem + {%- endif %} rewind: username: {{ rewind_user }} password: {{ rewind_password }} + {%- if enable_tls %} + sslrootcert: {{ conf_path }}/ca.pem + sslcert: {{ conf_path }}/cert.pem + sslkey: {{ conf_path }}/key.pem + {%- endif %} superuser: username: {{ superuser }} password: {{ superuser_password }} + {%- if enable_tls %} + sslrootcert: {{ conf_path }}/ca.pem + sslcert: {{ conf_path }}/cert.pem + sslkey: {{ conf_path }}/key.pem + {%- endif %} use_unix_socket: true {%- if is_creating_backup %} tags: diff --git a/terraform/variables.tf b/terraform/variables.tf index ede475f37a..d3ceba0c1e 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -12,7 +12,7 @@ variable "app_name" { variable "channel" { description = "Charm channel to use when deploying" type = string - default = "14/stable" + default = "16/stable" } variable "revision" { @@ -24,7 +24,7 @@ variable "revision" { variable "base" { description = "Application base" type = string - default = "ubuntu@22.04" + default = "ubuntu@24.04" } variable "units" { diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 87bd24fb9b..18fd8c71e3 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,12 +1,96 @@ -#!/usr/bin/env python3 # Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. +import logging +import os +import uuid + +import boto3 import pytest from pytest_operator.plugin import OpsTest +from . import architecture +from .helpers import construct_endpoint + +AWS = "AWS" +GCP = "GCP" + +logger = logging.getLogger(__name__) + + +@pytest.fixture(scope="session") +def charm(): + # Return str instead of pathlib.Path since python-libjuju's model.deploy(), juju deploy, and + # juju bundle files expect local charms to begin with `./` or `/` to distinguish them from + # Charmhub charms. + return f"./postgresql_ubuntu@24.04-{architecture.architecture}.charm" + + +def get_cloud_config(cloud: str) -> tuple[dict[str, str], dict[str, str]]: + # Define some configurations and credentials. + if cloud == AWS: + return { + "endpoint": "https://s3.amazonaws.com", + "bucket": "data-charms-testing", + "path": f"/postgresql-k8s/{uuid.uuid1()}", + "region": "us-east-1", + }, { + "access-key": os.environ["AWS_ACCESS_KEY"], + "secret-key": os.environ["AWS_SECRET_KEY"], + } + elif cloud == GCP: + return { + "endpoint": "https://storage.googleapis.com", + "bucket": "data-charms-testing", + "path": f"/postgresql-k8s/{uuid.uuid1()}", + "region": "", + }, { + "access-key": os.environ["GCP_ACCESS_KEY"], + "secret-key": os.environ["GCP_SECRET_KEY"], + } + + +def cleanup_cloud(config: dict[str, str], credentials: dict[str, str]) -> None: + # Delete the previously created objects. + logger.info("deleting the previously created backups") + session = boto3.session.Session( + aws_access_key_id=credentials["access-key"], + aws_secret_access_key=credentials["secret-key"], + region_name=config["region"], + ) + s3 = session.resource( + "s3", endpoint_url=construct_endpoint(config["endpoint"], config["region"]) + ) + bucket = s3.Bucket(config["bucket"]) + # GCS doesn't support batch delete operation, so delete the objects one by one. + for bucket_object in bucket.objects.filter(Prefix=config["path"].lstrip("/")): + bucket_object.delete() + + +@pytest.fixture(scope="module") +async def aws_cloud_configs(ops_test: OpsTest) -> None: + if ( + not os.environ.get("AWS_ACCESS_KEY", "").strip() + or not os.environ.get("AWS_SECRET_KEY", "").strip() + ): + pytest.skip("AWS configs not set") + return + + config, credentials = get_cloud_config(AWS) + yield config, credentials + + cleanup_cloud(config, credentials) + @pytest.fixture(scope="module") -async def charm(ops_test: OpsTest): - """Build the charm-under-test.""" - # Build charm from local source folder. - yield await ops_test.build_charm(".") +async def gcp_cloud_configs(ops_test: OpsTest) -> None: + if ( + not os.environ.get("GCP_ACCESS_KEY", "").strip() + or not os.environ.get("GCP_SECRET_KEY", "").strip() + ): + pytest.skip("GCP configs not set") + return + + config, credentials = get_cloud_config(GCP) + yield config, credentials + + cleanup_cloud(config, credentials) diff --git a/tests/integration/ha_tests/helpers.py b/tests/integration/ha_tests/helpers.py index 57ddac6dd9..d9ea25543d 100644 --- a/tests/integration/ha_tests/helpers.py +++ b/tests/integration/ha_tests/helpers.py @@ -106,7 +106,7 @@ async def are_writes_increasing( with attempt: more_writes, _ = await count_writes( ops_test, - down_unit=down_unit, + down_unit=down_units[0], use_ip_from_inside=use_ip_from_inside, extra_model=extra_model, ) @@ -954,21 +954,21 @@ async def add_unit_with_storage(ops_test, app, storage): Note: this function exists as a temporary solution until this issue is resolved: https://github.com/juju/python-libjuju/issues/695 """ - expected_units = len(ops_test.model.applications[app].units) + 1 - prev_units = [unit.name for unit in ops_test.model.applications[app].units] + original_units = {unit.name for unit in ops_test.model.applications[app].units} model_name = ops_test.model.info.name add_unit_cmd = f"add-unit {app} --model={model_name} --attach-storage={storage}".split() return_code, _, _ = await ops_test.juju(*add_unit_cmd) assert return_code == 0, "Failed to add unit with storage" async with ops_test.fast_forward(): await ops_test.model.wait_for_idle(apps=[app], status="active", timeout=2000) - assert len(ops_test.model.applications[app].units) == expected_units, ( - "New unit not added to model" - ) + + # When removing all units sometimes the last unit remain in the list + current_units = {unit.name for unit in ops_test.model.applications[app].units} + original_units.intersection_update(current_units) + assert original_units.issubset(current_units), "New unit not added to model" # verify storage attached - curr_units = [unit.name for unit in ops_test.model.applications[app].units] - new_unit = next(unit for unit in set(curr_units) - set(prev_units)) + new_unit = (current_units - original_units).pop() assert storage_id(ops_test, new_unit) == storage, "unit added with incorrect storage" # return a reference to newly added unit diff --git a/tests/integration/ha_tests/test_async_replication.py b/tests/integration/ha_tests/test_async_replication.py index 7b76660bb3..b8ed2b30c8 100644 --- a/tests/integration/ha_tests/test_async_replication.py +++ b/tests/integration/ha_tests/test_async_replication.py @@ -12,7 +12,7 @@ from pytest_operator.plugin import OpsTest from tenacity import Retrying, stop_after_delay, wait_fixed -from .. import architecture, markers +from .. import architecture from ..helpers import ( APPLICATION_NAME, DATABASE_APP_NAME, @@ -99,15 +99,12 @@ async def second_model_continuous_writes(second_model) -> None: assert action.results["result"] == "True", "Unable to clear up continuous_writes table" -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_deploy_async_replication_setup( ops_test: OpsTest, first_model: Model, second_model: Model, charm ) -> None: """Build and deploy two PostgreSQL cluster in two separate models to test async replication.""" if not await app_name(ops_test): - charm = await ops_test.build_charm(".") await ops_test.model.deploy( charm, num_units=CLUSTER_SIZE, @@ -122,7 +119,6 @@ async def test_deploy_async_replication_setup( ) await ops_test.model.relate(DATABASE_APP_NAME, DATA_INTEGRATOR_APP_NAME) if not await app_name(ops_test, model=second_model): - charm = await ops_test.build_charm(".") await second_model.deploy( charm, num_units=CLUSTER_SIZE, @@ -146,8 +142,6 @@ async def test_deploy_async_replication_setup( ) -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_async_replication( ops_test: OpsTest, @@ -224,8 +218,6 @@ async def test_async_replication( await check_writes(ops_test, extra_model=second_model) -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_get_data_integrator_credentials( ops_test: OpsTest, @@ -237,8 +229,6 @@ async def test_get_data_integrator_credentials( data_integrator_credentials = result.results -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_switchover( ops_test: OpsTest, @@ -269,7 +259,7 @@ async def test_switchover( leader_unit = await get_leader_unit(ops_test, DATABASE_APP_NAME, model=second_model) assert leader_unit is not None, "No leader unit found" logger.info("promoting the second cluster") - run_action = await leader_unit.run_action("promote-to-primary", **{"force": True}) + run_action = await leader_unit.run_action("promote-to-primary", scope="cluster", force=True) await run_action.wait() assert (run_action.results.get("return-code", None) == 0) or ( run_action.results.get("Code", None) == "0" @@ -292,8 +282,6 @@ async def test_switchover( await are_writes_increasing(ops_test, extra_model=second_model) -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_data_integrator_creds_keep_on_working( ops_test: OpsTest, @@ -315,8 +303,6 @@ async def test_data_integrator_creds_keep_on_working( connection.close() -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_promote_standby( ops_test: OpsTest, @@ -350,7 +336,7 @@ async def test_promote_standby( leader_unit = await get_leader_unit(ops_test, DATABASE_APP_NAME) assert leader_unit is not None, "No leader unit found" logger.info("promoting the first cluster") - run_action = await leader_unit.run_action("promote-to-primary") + run_action = await leader_unit.run_action("promote-to-primary", scope="cluster") await run_action.wait() assert (run_action.results.get("return-code", None) == 0) or ( run_action.results.get("Code", None) == "0" @@ -393,8 +379,6 @@ async def test_promote_standby( await are_writes_increasing(ops_test) -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_reestablish_relation( ops_test: OpsTest, first_model: Model, second_model: Model, continuous_writes @@ -451,8 +435,6 @@ async def test_reestablish_relation( await check_writes(ops_test, extra_model=second_model) -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_async_replication_failover_in_main_cluster( ops_test: OpsTest, first_model: Model, second_model: Model, continuous_writes @@ -497,8 +479,6 @@ async def test_async_replication_failover_in_main_cluster( await check_writes(ops_test, extra_model=second_model) -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_async_replication_failover_in_secondary_cluster( ops_test: OpsTest, first_model: Model, second_model: Model, continuous_writes @@ -534,8 +514,6 @@ async def test_async_replication_failover_in_secondary_cluster( await check_writes(ops_test, extra_model=second_model) -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_scaling( ops_test: OpsTest, first_model: Model, second_model: Model, continuous_writes diff --git a/tests/integration/ha_tests/test_replication.py b/tests/integration/ha_tests/test_replication.py index dd924948da..fed67a8019 100644 --- a/tests/integration/ha_tests/test_replication.py +++ b/tests/integration/ha_tests/test_replication.py @@ -18,16 +18,14 @@ ) -@pytest.mark.group(1) @pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest) -> None: +async def test_build_and_deploy(ops_test: OpsTest, charm) -> None: """Build and deploy three unit of PostgreSQL.""" wait_for_apps = False # It is possible for users to provide their own cluster for HA testing. Hence, check if there # is a pre-existing cluster. if not await app_name(ops_test): wait_for_apps = True - charm = await ops_test.build_charm(".") async with ops_test.fast_forward(): await ops_test.model.deploy( charm, @@ -51,7 +49,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: await ops_test.model.wait_for_idle(status="active", timeout=1500) -@pytest.mark.group(1) async def test_reelection(ops_test: OpsTest, continuous_writes, primary_start_timeout) -> None: """Kill primary unit, check reelection.""" app = await app_name(ops_test) @@ -63,9 +60,7 @@ async def test_reelection(ops_test: OpsTest, continuous_writes, primary_start_ti # Remove the primary unit. primary_name = await get_primary(ops_test, app) - await ops_test.model.destroy_units( - primary_name, - ) + await ops_test.model.destroy_units(primary_name) # Wait and get the primary again (which can be any unit, including the previous primary). async with ops_test.fast_forward(): @@ -89,7 +84,6 @@ async def test_reelection(ops_test: OpsTest, continuous_writes, primary_start_ti await check_writes(ops_test) -@pytest.mark.group(1) async def test_consistency(ops_test: OpsTest, continuous_writes) -> None: """Write to primary, read data from secondaries (check consistency).""" # Locate primary unit. @@ -106,8 +100,9 @@ async def test_consistency(ops_test: OpsTest, continuous_writes) -> None: await check_writes(ops_test) -@pytest.mark.group(1) -async def test_no_data_replicated_between_clusters(ops_test: OpsTest, continuous_writes) -> None: +async def test_no_data_replicated_between_clusters( + ops_test: OpsTest, charm, continuous_writes +) -> None: """Check that writes in one cluster are not replicated to another cluster.""" # Locate primary unit. app = await app_name(ops_test) @@ -116,7 +111,6 @@ async def test_no_data_replicated_between_clusters(ops_test: OpsTest, continuous # Deploy another cluster. new_cluster_app = f"second-{app}" if not await app_name(ops_test, new_cluster_app): - charm = await ops_test.build_charm(".") async with ops_test.fast_forward(): await ops_test.model.deploy( charm, diff --git a/tests/integration/ha_tests/test_restore_cluster.py b/tests/integration/ha_tests/test_restore_cluster.py index 9542dbb850..8a26b15cb5 100644 --- a/tests/integration/ha_tests/test_restore_cluster.py +++ b/tests/integration/ha_tests/test_restore_cluster.py @@ -29,12 +29,10 @@ charm = None -@pytest.mark.group(1) @pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest) -> None: +async def test_build_and_deploy(ops_test: OpsTest, charm) -> None: """Build and deploy two PostgreSQL clusters.""" # This is a potentially destructive test, so it shouldn't be run against existing clusters - charm = await ops_test.build_charm(".") async with ops_test.fast_forward(): # Deploy the first cluster with reusable storage await ops_test.model.deploy( @@ -68,7 +66,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: await ops_test.model.destroy_unit(second_primary) -@pytest.mark.group(1) async def test_cluster_restore(ops_test): """Recreates the cluster from storage volumes.""" # Write some data. diff --git a/tests/integration/ha_tests/test_scaling.py b/tests/integration/ha_tests/test_scaling.py index f3105d38cd..0f48af1ffb 100644 --- a/tests/integration/ha_tests/test_scaling.py +++ b/tests/integration/ha_tests/test_scaling.py @@ -7,7 +7,6 @@ import pytest from pytest_operator.plugin import OpsTest -from .. import markers from ..helpers import ( CHARM_BASE, DATABASE_APP_NAME, @@ -27,13 +26,10 @@ charm = None -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest) -> None: +async def test_build_and_deploy(ops_test: OpsTest, charm) -> None: """Build and deploy two PostgreSQL clusters.""" # This is a potentially destructive test, so it shouldn't be run against existing clusters - charm = await ops_test.build_charm(".") async with ops_test.fast_forward(): # Deploy the first cluster with reusable storage await gather( @@ -55,8 +51,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: await ops_test.model.wait_for_idle(status="active", timeout=1500) -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_removing_stereo_primary(ops_test: OpsTest, continuous_writes) -> None: # Start an application that continuously writes data to the database. @@ -105,8 +99,6 @@ async def test_removing_stereo_primary(ops_test: OpsTest, continuous_writes) -> await check_writes(ops_test) -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_removing_stereo_sync_standby(ops_test: OpsTest, continuous_writes) -> None: # Start an application that continuously writes data to the database. @@ -140,16 +132,12 @@ async def test_removing_stereo_sync_standby(ops_test: OpsTest, continuous_writes await check_writes(ops_test) -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_scale_to_five_units(ops_test: OpsTest) -> None: await ops_test.model.applications[DATABASE_APP_NAME].add_unit(count=3) await ops_test.model.wait_for_idle(status="active", timeout=1500) -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail async def test_removing_raft_majority(ops_test: OpsTest, continuous_writes) -> None: # Start an application that continuously writes data to the database. @@ -164,73 +152,15 @@ async def test_removing_raft_majority(ops_test: OpsTest, continuous_writes) -> N ops_test.model.destroy_unit( original_roles["primaries"][0], force=True, destroy_storage=False, max_wait=1500 ), - ops_test.model.destroy_unit( - original_roles["replicas"][0], force=True, destroy_storage=False, max_wait=1500 - ), ops_test.model.destroy_unit( original_roles["sync_standbys"][0], force=True, destroy_storage=False, max_wait=1500 ), - ) - - left_unit = ops_test.model.units[original_roles["sync_standbys"][1]] - await ops_test.model.block_until( - lambda: left_unit.workload_status == "blocked" - and left_unit.workload_status_message == "Raft majority loss, run: promote-to-primary", - timeout=600, - ) - - run_action = await left_unit.run_action("promote-to-primary", scope="unit", force=True) - await run_action.wait() - - await ops_test.model.wait_for_idle(status="active", timeout=900, idle_period=45) - - await are_writes_increasing( - ops_test, - [ - original_roles["primaries"][0], - original_roles["replicas"][0], - original_roles["sync_standbys"][0], - ], - ) - - logger.info("Scaling back up") - await ops_test.model.applications[DATABASE_APP_NAME].add_unit(count=3) - await ops_test.model.wait_for_idle(status="active", timeout=1500) - - await check_writes(ops_test) - new_roles = await get_cluster_roles( - ops_test, ops_test.model.applications[DATABASE_APP_NAME].units[0].name - ) - assert len(new_roles["primaries"]) == 1 - assert len(new_roles["sync_standbys"]) == 2 - assert new_roles["primaries"][0] == original_roles["sync_standbys"][1] - - -@pytest.mark.group(1) -@markers.juju3 -@pytest.mark.abort_on_fail -async def test_removing_raft_majority_async(ops_test: OpsTest, continuous_writes) -> None: - # Start an application that continuously writes data to the database. - app = await app_name(ops_test) - original_roles = await get_cluster_roles( - ops_test, ops_test.model.applications[DATABASE_APP_NAME].units[0].name - ) - - await start_continuous_writes(ops_test, app) - logger.info("Deleting primary") - await gather( - ops_test.model.destroy_unit( - original_roles["primaries"][0], force=True, destroy_storage=False, max_wait=1500 - ), - ops_test.model.destroy_unit( - original_roles["replicas"][0], force=True, destroy_storage=False, max_wait=1500 - ), ops_test.model.destroy_unit( - original_roles["replicas"][1], force=True, destroy_storage=False, max_wait=1500 + original_roles["sync_standbys"][1], force=True, destroy_storage=False, max_wait=1500 ), ) - left_unit = ops_test.model.units[original_roles["sync_standbys"][0]] + left_unit = ops_test.model.units[original_roles["sync_standbys"][2]] await ops_test.model.block_until( lambda: left_unit.workload_status == "blocked" and left_unit.workload_status_message == "Raft majority loss, run: promote-to-primary", @@ -246,8 +176,8 @@ async def test_removing_raft_majority_async(ops_test: OpsTest, continuous_writes ops_test, [ original_roles["primaries"][0], - original_roles["replicas"][0], - original_roles["replicas"][1], + original_roles["sync_standbys"][0], + original_roles["sync_standbys"][1], ], ) @@ -260,8 +190,5 @@ async def test_removing_raft_majority_async(ops_test: OpsTest, continuous_writes ops_test, ops_test.model.applications[DATABASE_APP_NAME].units[0].name ) assert len(new_roles["primaries"]) == 1 - assert len(new_roles["sync_standbys"]) == 2 - assert ( - new_roles["primaries"][0] == original_roles["sync_standbys"][0] - or new_roles["primaries"][0] == original_roles["sync_standbys"][1] - ) + assert len(new_roles["sync_standbys"]) == 4 + assert new_roles["primaries"][0] == original_roles["sync_standbys"][2] diff --git a/tests/integration/ha_tests/test_scaling_three_units.py b/tests/integration/ha_tests/test_scaling_three_units.py index 6817cd238a..a5b95bb9d1 100644 --- a/tests/integration/ha_tests/test_scaling_three_units.py +++ b/tests/integration/ha_tests/test_scaling_three_units.py @@ -3,11 +3,11 @@ # See LICENSE file for licensing details. import logging from asyncio import exceptions, gather, sleep +from copy import deepcopy import pytest from pytest_operator.plugin import OpsTest -from .. import markers from ..helpers import ( CHARM_BASE, DATABASE_APP_NAME, @@ -28,13 +28,10 @@ charm = None -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest) -> None: +async def test_build_and_deploy(ops_test: OpsTest, charm) -> None: """Build and deploy two PostgreSQL clusters.""" # This is a potentially destructive test, so it shouldn't be run against existing clusters - charm = await ops_test.build_charm(".") async with ops_test.fast_forward(): # Deploy the first cluster with reusable storage await gather( @@ -56,16 +53,13 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: await ops_test.model.wait_for_idle(status="active", timeout=1500) -@pytest.mark.group(1) -@markers.juju3 @pytest.mark.parametrize( "roles", [ ["primaries"], ["sync_standbys"], - ["replicas"], - ["primaries", "replicas"], - ["sync_standbys", "replicas"], + ["primaries", "sync_standbys"], + ["sync_standbys", "sync_standbys"], ], ) @pytest.mark.abort_on_fail @@ -76,8 +70,9 @@ async def test_removing_unit(ops_test: OpsTest, roles: list[str], continuous_wri original_roles = await get_cluster_roles( ops_test, ops_test.model.applications[DATABASE_APP_NAME].units[0].name ) + copied_roles = deepcopy(original_roles) await start_continuous_writes(ops_test, app) - units = [original_roles[role][0] for role in roles] + units = [copied_roles[role].pop(0) for role in roles] for unit in units: logger.info(f"Stopping unit {unit}") await stop_machine(ops_test, await get_machine_from_unit(ops_test, unit)) @@ -124,10 +119,10 @@ async def test_removing_unit(ops_test: OpsTest, roles: list[str], continuous_wri ops_test, ops_test.model.applications[DATABASE_APP_NAME].units[0].name ) assert len(new_roles["primaries"]) == 1 - assert len(new_roles["sync_standbys"]) == 1 - assert len(new_roles["replicas"]) == 1 + assert len(new_roles["sync_standbys"]) == 2 + assert len(new_roles["replicas"]) == 0 if "primaries" in roles: - assert new_roles["primaries"][0] == original_roles["sync_standbys"][0] + assert new_roles["primaries"][0] in original_roles["sync_standbys"] else: assert new_roles["primaries"][0] == original_roles["primaries"][0] diff --git a/tests/integration/ha_tests/test_scaling_three_units_async.py b/tests/integration/ha_tests/test_scaling_three_units_async.py new file mode 100644 index 0000000000..96cecaa14f --- /dev/null +++ b/tests/integration/ha_tests/test_scaling_three_units_async.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +import logging +from asyncio import exceptions, gather, sleep +from copy import deepcopy + +import pytest +from pytest_operator.plugin import OpsTest + +from ..helpers import ( + CHARM_BASE, + DATABASE_APP_NAME, + get_machine_from_unit, + stop_machine, +) +from .conftest import APPLICATION_NAME +from .helpers import ( + app_name, + are_writes_increasing, + check_writes, + get_cluster_roles, + start_continuous_writes, +) + +logger = logging.getLogger(__name__) + +charm = None + + +@pytest.mark.abort_on_fail +async def test_build_and_deploy(ops_test: OpsTest, charm) -> None: + """Build and deploy two PostgreSQL clusters.""" + # This is a potentially destructive test, so it shouldn't be run against existing clusters + async with ops_test.fast_forward(): + # Deploy the first cluster with reusable storage + await gather( + ops_test.model.deploy( + charm, + application_name=DATABASE_APP_NAME, + num_units=3, + base=CHARM_BASE, + config={"profile": "testing", "synchronous_node_count": "majority"}, + ), + ops_test.model.deploy( + APPLICATION_NAME, + application_name=APPLICATION_NAME, + base=CHARM_BASE, + channel="edge", + ), + ) + + await ops_test.model.wait_for_idle(status="active", timeout=1500) + + +@pytest.mark.parametrize( + "roles", + [ + ["primaries"], + ["sync_standbys"], + ["replicas"], + ["primaries", "replicas"], + ["sync_standbys", "replicas"], + ], +) +@pytest.mark.abort_on_fail +async def test_removing_unit(ops_test: OpsTest, roles: list[str], continuous_writes) -> None: + logger.info(f"removing {', '.join(roles)}") + # Start an application that continuously writes data to the database. + app = await app_name(ops_test) + original_roles = await get_cluster_roles( + ops_test, ops_test.model.applications[DATABASE_APP_NAME].units[0].name + ) + copied_roles = deepcopy(original_roles) + await start_continuous_writes(ops_test, app) + units = [copied_roles[role].pop(0) for role in roles] + for unit in units: + logger.info(f"Stopping unit {unit}") + await stop_machine(ops_test, await get_machine_from_unit(ops_test, unit)) + await sleep(15) + for unit in units: + logger.info(f"Deleting unit {unit}") + await ops_test.model.destroy_unit(unit, force=True, destroy_storage=False, max_wait=1500) + + if len(roles) > 1: + for left_unit in ops_test.model.applications[DATABASE_APP_NAME].units: + if left_unit.name not in units: + break + try: + await ops_test.model.block_until( + lambda: left_unit.workload_status == "blocked" + and left_unit.workload_status_message + == "Raft majority loss, run: promote-to-primary", + timeout=600, + ) + + run_action = ( + await ops_test.model.applications[DATABASE_APP_NAME] + .units[0] + .run_action("promote-to-primary", scope="unit", force=True) + ) + await run_action.wait() + except exceptions.TimeoutError: + # Check if Patroni self healed + assert ( + left_unit.workload_status == "active" + and left_unit.workload_status_message == "Primary" + ) + logger.warning(f"Patroni self-healed without raft reinitialisation for roles {roles}") + + await ops_test.model.wait_for_idle(status="active", timeout=600, idle_period=45) + + await are_writes_increasing(ops_test, units) + + logger.info("Scaling back up") + await ops_test.model.applications[DATABASE_APP_NAME].add_unit(count=len(roles)) + await ops_test.model.wait_for_idle(status="active", timeout=1500) + + new_roles = await get_cluster_roles( + ops_test, ops_test.model.applications[DATABASE_APP_NAME].units[0].name + ) + assert len(new_roles["primaries"]) == 1 + assert len(new_roles["sync_standbys"]) == 1 + assert len(new_roles["replicas"]) == 1 + if "primaries" in roles: + assert new_roles["primaries"][0] in original_roles["sync_standbys"] + else: + assert new_roles["primaries"][0] == original_roles["primaries"][0] + + await check_writes(ops_test) diff --git a/tests/integration/ha_tests/test_self_healing.py b/tests/integration/ha_tests/test_self_healing.py index 12b61a4fd7..f3ddc6fe88 100644 --- a/tests/integration/ha_tests/test_self_healing.py +++ b/tests/integration/ha_tests/test_self_healing.py @@ -62,16 +62,14 @@ MEDIAN_ELECTION_TIME = 10 -@pytest.mark.group(1) @pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest) -> None: +async def test_build_and_deploy(ops_test: OpsTest, charm) -> None: """Build and deploy three unit of PostgreSQL.""" wait_for_apps = False # It is possible for users to provide their own cluster for HA testing. Hence, check if there # is a pre-existing cluster. if not await app_name(ops_test): wait_for_apps = True - charm = await ops_test.build_charm(".") async with ops_test.fast_forward(): await ops_test.model.deploy( charm, @@ -96,7 +94,6 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None: await ops_test.model.wait_for_idle(status="active", timeout=1500) -@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_storage_re_use(ops_test, continuous_writes): """Verifies that database units with attached storage correctly repurpose storage. @@ -144,7 +141,6 @@ async def test_storage_re_use(ops_test, continuous_writes): ) -@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.parametrize("process", DB_PROCESSES) @pytest.mark.parametrize("signal", ["SIGTERM", "SIGKILL"]) @@ -179,7 +175,6 @@ async def test_interruption_db_process( await is_cluster_updated(ops_test, primary_name) -@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.parametrize("process", DB_PROCESSES) async def test_freeze_db_process( @@ -221,7 +216,6 @@ async def test_freeze_db_process( await is_cluster_updated(ops_test, primary_name) -@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.parametrize("process", DB_PROCESSES) @pytest.mark.parametrize("signal", ["SIGTERM", "SIGKILL"]) @@ -309,9 +303,8 @@ async def test_full_cluster_restart( await check_writes(ops_test) -@pytest.mark.group(1) @pytest.mark.abort_on_fail -@pytest.mark.unstable +@pytest.mark.skip(reason="Unstable") async def test_forceful_restart_without_data_and_transaction_logs( ops_test: OpsTest, continuous_writes, @@ -386,7 +379,6 @@ async def test_forceful_restart_without_data_and_transaction_logs( await is_cluster_updated(ops_test, primary_name) -@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_network_cut(ops_test: OpsTest, continuous_writes, primary_start_timeout): """Completely cut and restore network.""" @@ -475,7 +467,6 @@ async def test_network_cut(ops_test: OpsTest, continuous_writes, primary_start_t await is_cluster_updated(ops_test, primary_name, use_ip_from_inside=True) -@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_network_cut_without_ip_change( ops_test: OpsTest, continuous_writes, primary_start_timeout diff --git a/tests/integration/ha_tests/test_smoke.py b/tests/integration/ha_tests/test_smoke.py index ea872d45d0..b160d38f2b 100644 --- a/tests/integration/ha_tests/test_smoke.py +++ b/tests/integration/ha_tests/test_smoke.py @@ -14,7 +14,6 @@ APPLICATION_NAME, CHARM_BASE, ) -from ..juju_ import juju_major_version from .helpers import ( add_unit_with_storage, check_db, @@ -23,7 +22,6 @@ get_any_deatached_storage, is_postgresql_ready, is_storage_exists, - remove_unit_force, storage_id, ) @@ -33,7 +31,6 @@ logger = logging.getLogger(__name__) -@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_app_force_removal(ops_test: OpsTest, charm: str): """Remove unit with force while storage is alive.""" @@ -79,12 +76,9 @@ async def test_app_force_removal(ops_test: OpsTest, charm: str): # Destroy charm logger.info("force removing charm") - if juju_major_version == 2: - await remove_unit_force(ops_test, primary_name) - else: - await ops_test.model.destroy_unit( - primary_name, force=True, destroy_storage=False, max_wait=1500 - ) + await ops_test.model.destroy_unit( + primary_name, force=True, destroy_storage=False, max_wait=1500 + ) # Storage should remain logger.info("werifing is storage exists") @@ -93,7 +87,6 @@ async def test_app_force_removal(ops_test: OpsTest, charm: str): assert await is_storage_exists(ops_test, storage_id_str) -@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_charm_garbage_ignorance(ops_test: OpsTest, charm: str): """Test charm deploy in dirty environment with garbage storage.""" @@ -133,9 +126,7 @@ async def test_charm_garbage_ignorance(ops_test: OpsTest, charm: str): await ops_test.model.destroy_unit(primary_name) -@pytest.mark.group(1) @pytest.mark.abort_on_fail -@pytest.mark.skipif(juju_major_version < 3, reason="Requires juju 3 or higher") async def test_app_resources_conflicts_v3(ops_test: OpsTest, charm: str): """Test application deploy in dirty environment with garbage storage from another application.""" async with ops_test.fast_forward(): @@ -171,51 +162,3 @@ async def test_app_resources_conflicts_v3(ops_test: OpsTest, charm: str): assert not await check_password_auth( ops_test, ops_test.model.applications[DUP_APPLICATION_NAME].units[0].name ) - - -@pytest.mark.group(1) -@pytest.mark.abort_on_fail -@pytest.mark.skipif(juju_major_version != 2, reason="Requires juju 2") -async def test_app_resources_conflicts_v2(ops_test: OpsTest, charm: str): - """Test application deploy in dirty environment with garbage storage from another application.""" - async with ops_test.fast_forward(): - logger.info("checking garbage storage") - garbage_storage = None - for attempt in Retrying(stop=stop_after_delay(30 * 3), wait=wait_fixed(3), reraise=True): - with attempt: - garbage_storage = await get_any_deatached_storage(ops_test) - - # Deploy duplicaate charm - logger.info("deploying duplicate application") - await ops_test.model.deploy( - charm, - application_name=DUP_APPLICATION_NAME, - num_units=1, - base=CHARM_BASE, - config={"profile": "testing"}, - ) - - logger.info("force removing charm") - await remove_unit_force( - ops_test, ops_test.model.applications[DUP_APPLICATION_NAME].units[0].name - ) - - # Add unit with garbage storage - logger.info("adding charm with attached storage") - add_unit_cmd = f"add-unit {DUP_APPLICATION_NAME} --model={ops_test.model.info.name} --attach-storage={garbage_storage}".split() - return_code, _, _ = await ops_test.juju(*add_unit_cmd) - assert return_code == 0, "Failed to add unit with storage" - - logger.info("waiting for duplicate application to be blocked") - try: - await ops_test.model.wait_for_idle( - apps=[DUP_APPLICATION_NAME], timeout=1000, status="blocked" - ) - except asyncio.TimeoutError: - logger.info("Application is not in blocked state. Checking logs...") - - # Since application have postgresql db in storage from external application it should not be able to connect due to new password - logger.info("checking operator password auth") - assert not await check_password_auth( - ops_test, ops_test.model.applications[DUP_APPLICATION_NAME].units[0].name - ) diff --git a/tests/integration/ha_tests/test_synchronous_policy.py b/tests/integration/ha_tests/test_synchronous_policy.py new file mode 100644 index 0000000000..920642d81e --- /dev/null +++ b/tests/integration/ha_tests/test_synchronous_policy.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +import pytest +from pytest_operator.plugin import OpsTest +from tenacity import Retrying, stop_after_attempt, wait_fixed + +from ..helpers import CHARM_BASE +from .helpers import app_name, get_cluster_roles + + +@pytest.mark.abort_on_fail +async def test_build_and_deploy(ops_test: OpsTest, charm: str) -> None: + """Build and deploy two PostgreSQL clusters.""" + async with ops_test.fast_forward(): + # Deploy the first cluster with reusable storage + await ops_test.model.deploy( + charm, + num_units=3, + base=CHARM_BASE, + config={"profile": "testing"}, + ) + await ops_test.model.wait_for_idle(status="active", timeout=1500) + + +async def test_default_all(ops_test: OpsTest) -> None: + app = await app_name(ops_test) + + async with ops_test.fast_forward(): + await ops_test.model.wait_for_idle(apps=[app], status="active", timeout=300) + + for attempt in Retrying(stop=stop_after_attempt(3), wait=wait_fixed(5), reraise=True): + with attempt: + roles = await get_cluster_roles( + ops_test, ops_test.model.applications[app].units[0].name + ) + + assert len(roles["primaries"]) == 1 + assert len(roles["sync_standbys"]) == 2 + assert len(roles["replicas"]) == 0 + + +async def test_majority(ops_test: OpsTest) -> None: + app = await app_name(ops_test) + + await ops_test.model.applications[app].set_config({"synchronous_node_count": "majority"}) + + async with ops_test.fast_forward(): + await ops_test.model.wait_for_idle(apps=[app], status="active") + + for attempt in Retrying(stop=stop_after_attempt(3), wait=wait_fixed(5), reraise=True): + with attempt: + roles = await get_cluster_roles( + ops_test, ops_test.model.applications[app].units[0].name + ) + + assert len(roles["primaries"]) == 1 + assert len(roles["sync_standbys"]) == 1 + assert len(roles["replicas"]) == 1 + + +async def test_constant(ops_test: OpsTest) -> None: + app = await app_name(ops_test) + + await ops_test.model.applications[app].set_config({"synchronous_node_count": "2"}) + + async with ops_test.fast_forward(): + await ops_test.model.wait_for_idle(apps=[app], status="active", timeout=300) + + for attempt in Retrying(stop=stop_after_attempt(3), wait=wait_fixed(5), reraise=True): + with attempt: + roles = await get_cluster_roles( + ops_test, ops_test.model.applications[app].units[0].name + ) + + assert len(roles["primaries"]) == 1 + assert len(roles["sync_standbys"]) == 2 + assert len(roles["replicas"]) == 0 diff --git a/tests/integration/ha_tests/test_upgrade.py b/tests/integration/ha_tests/test_upgrade.py index 497c7ace9a..e639b8ff60 100644 --- a/tests/integration/ha_tests/test_upgrade.py +++ b/tests/integration/ha_tests/test_upgrade.py @@ -29,14 +29,13 @@ TIMEOUT = 600 -@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_deploy_latest(ops_test: OpsTest) -> None: """Simple test to ensure that the PostgreSQL and application charms get deployed.""" await ops_test.model.deploy( DATABASE_APP_NAME, num_units=3, - channel="14/edge", + channel="16/edge", config={"profile": "testing"}, ) await ops_test.model.deploy( @@ -52,7 +51,6 @@ async def test_deploy_latest(ops_test: OpsTest) -> None: assert len(ops_test.model.applications[DATABASE_APP_NAME].units) == 3 -@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_pre_upgrade_check(ops_test: OpsTest) -> None: """Test that the pre-upgrade-check action runs successfully.""" @@ -65,9 +63,8 @@ async def test_pre_upgrade_check(ops_test: OpsTest) -> None: await action.wait() -@pytest.mark.group(1) @pytest.mark.abort_on_fail -async def test_upgrade_from_edge(ops_test: OpsTest, continuous_writes) -> None: +async def test_upgrade_from_edge(ops_test: OpsTest, continuous_writes, charm) -> None: # Start an application that continuously writes data to the database. logger.info("starting continuous writes to the database") await start_continuous_writes(ops_test, DATABASE_APP_NAME) @@ -81,9 +78,6 @@ async def test_upgrade_from_edge(ops_test: OpsTest, continuous_writes) -> None: application = ops_test.model.applications[DATABASE_APP_NAME] - logger.info("Build charm locally") - charm = await ops_test.build_charm(".") - logger.info("Refresh the charm") await application.refresh(path=charm) @@ -115,9 +109,8 @@ async def test_upgrade_from_edge(ops_test: OpsTest, continuous_writes) -> None: ) -@pytest.mark.group(1) @pytest.mark.abort_on_fail -async def test_fail_and_rollback(ops_test, continuous_writes) -> None: +async def test_fail_and_rollback(ops_test, charm, continuous_writes) -> None: # Start an application that continuously writes data to the database. logger.info("starting continuous writes to the database") await start_continuous_writes(ops_test, DATABASE_APP_NAME) @@ -134,10 +127,9 @@ async def test_fail_and_rollback(ops_test, continuous_writes) -> None: action = await leader_unit.run_action("pre-upgrade-check") await action.wait() - local_charm = await ops_test.build_charm(".") - filename = local_charm.split("/")[-1] if isinstance(local_charm, str) else local_charm.name + filename = Path(charm).name fault_charm = Path("/tmp/", filename) - shutil.copy(local_charm, fault_charm) + shutil.copy(charm, fault_charm) logger.info("Inject dependency fault") await inject_dependency_fault(ops_test, DATABASE_APP_NAME, fault_charm) @@ -162,7 +154,7 @@ async def test_fail_and_rollback(ops_test, continuous_writes) -> None: await action.wait() logger.info("Re-refresh the charm") - await application.refresh(path=local_charm) + await application.refresh(path=charm) logger.info("Wait for upgrade to start") await ops_test.model.block_until( diff --git a/tests/integration/ha_tests/test_upgrade_from_stable.py b/tests/integration/ha_tests/test_upgrade_from_stable.py index 977ec9c067..17f51a51ab 100644 --- a/tests/integration/ha_tests/test_upgrade_from_stable.py +++ b/tests/integration/ha_tests/test_upgrade_from_stable.py @@ -12,7 +12,6 @@ count_switchovers, get_leader_unit, get_primary, - remove_chown_workaround, ) from .helpers import ( are_writes_increasing, @@ -25,10 +24,11 @@ TIMEOUT = 900 -@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_deploy_stable(ops_test: OpsTest) -> None: """Simple test to ensure that the PostgreSQL and application charms get deployed.""" + # TODO remove once we release to stable + pytest.skip("No 16/stable yet.") return_code, charm_info, stderr = await ops_test.juju("info", "postgresql", "--format=json") if return_code != 0: raise Exception(f"failed to get charm info with error: {stderr}") @@ -40,29 +40,11 @@ async def test_deploy_stable(ops_test: OpsTest) -> None: else parsed_charm_info["channel-map"]["14/stable"]["revision"] ) logger.info(f"14/stable revision: {revision}") - if int(revision) < 315: - original_charm_name = "./postgresql.charm" - return_code, _, stderr = await ops_test.juju( - "download", - "postgresql", - "--channel=14/stable", - f"--filepath={original_charm_name}", - ) - if return_code != 0: - raise Exception( - f"failed to download charm from 14/stable channel with error: {stderr}" - ) - patched_charm_name = "./modified_postgresql.charm" - remove_chown_workaround(original_charm_name, patched_charm_name) - return_code, _, stderr = await ops_test.juju("deploy", patched_charm_name, "-n", "3") - if return_code != 0: - raise Exception(f"failed to deploy charm from 14/stable channel with error: {stderr}") - else: - await ops_test.model.deploy( - DATABASE_APP_NAME, - num_units=3, - channel="14/stable", - ) + await ops_test.model.deploy( + DATABASE_APP_NAME, + num_units=3, + channel="16/stable", + ) await ops_test.model.deploy( APPLICATION_NAME, num_units=1, @@ -76,10 +58,11 @@ async def test_deploy_stable(ops_test: OpsTest) -> None: assert len(ops_test.model.applications[DATABASE_APP_NAME].units) == 3 -@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_pre_upgrade_check(ops_test: OpsTest) -> None: """Test that the pre-upgrade-check action runs successfully.""" + # TODO remove once we release to stable + pytest.skip("No 16/stable yet.") application = ops_test.model.applications[DATABASE_APP_NAME] if "pre-upgrade-check" not in await application.get_actions(): logger.info("skipping the test because the charm from 14/stable doesn't support upgrade") @@ -94,10 +77,11 @@ async def test_pre_upgrade_check(ops_test: OpsTest) -> None: await action.wait() -@pytest.mark.group(1) @pytest.mark.abort_on_fail -async def test_upgrade_from_stable(ops_test: OpsTest): +async def test_upgrade_from_stable(ops_test: OpsTest, charm): """Test updating from stable channel.""" + # TODO remove once we release to stable + pytest.skip("No 16/stable yet.") # Start an application that continuously writes data to the database. logger.info("starting continuous writes to the database") await start_continuous_writes(ops_test, DATABASE_APP_NAME) @@ -112,9 +96,6 @@ async def test_upgrade_from_stable(ops_test: OpsTest): application = ops_test.model.applications[DATABASE_APP_NAME] actions = await application.get_actions() - logger.info("Build charm locally") - charm = await ops_test.build_charm(".") - logger.info("Refresh the charm") await application.refresh(path=charm) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index dfb0254bfc..d777c0bb81 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -30,6 +30,8 @@ wait_fixed, ) +from constants import DATABASE_DEFAULT_NAME + CHARM_BASE = "ubuntu@22.04" METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) DATABASE_APP_NAME = METADATA["name"] @@ -497,7 +499,7 @@ async def execute_query_on_unit( unit_address: str, password: str, query: str, - database: str = "postgres", + database: str = DATABASE_DEFAULT_NAME, sslmode: str | None = None, ): """Execute given PostgreSQL query on a unit. @@ -851,44 +853,6 @@ def has_relation_exited( return True -def remove_chown_workaround(original_charm_filename: str, patched_charm_filename: str) -> None: - """Remove the chown workaround from the charm.""" - with ( - zipfile.ZipFile(original_charm_filename, "r") as charm_file, - zipfile.ZipFile(patched_charm_filename, "w") as modified_charm_file, - ): - # Iterate the input files - unix_attributes = {} - for charm_info in charm_file.infolist(): - # Read input file - with charm_file.open(charm_info) as file: - if charm_info.filename == "src/charm.py": - content = file.read() - # Modify the content of the file by replacing a string - content = ( - content.decode() - .replace( - """ try: - self._patch_snap_seccomp_profile() - except subprocess.CalledProcessError as e: - logger.exception(e) - self.unit.status = BlockedStatus("failed to patch snap seccomp profile") - return""", - "", - ) - .encode() - ) - # Write content. - modified_charm_file.writestr(charm_info.filename, content) - else: # Other file, don't want to modify => just copy it. - content = file.read() - modified_charm_file.writestr(charm_info.filename, content) - unix_attributes[charm_info.filename] = charm_info.external_attr >> 16 - - for modified_charm_info in modified_charm_file.infolist(): - modified_charm_info.external_attr = unix_attributes[modified_charm_info.filename] << 16 - - @retry( retry=retry_if_result(lambda x: not x), stop=stop_after_attempt(10), @@ -1053,7 +1017,6 @@ def switchover( ) assert response.status_code == 200 app_name = current_primary.split("/")[0] - minority_count = len(ops_test.model.applications[app_name].units) // 2 for attempt in Retrying(stop=stop_after_attempt(30), wait=wait_fixed(2), reraise=True): with attempt: response = requests.get(f"http://{primary_ip}:8008/cluster") @@ -1061,7 +1024,7 @@ def switchover( standbys = len([ member for member in response.json()["members"] if member["role"] == "sync_standby" ]) - assert standbys >= minority_count + assert standbys == len(ops_test.model.applications[app_name].units) - 1 async def wait_for_idle_on_blocked( diff --git a/tests/integration/markers.py b/tests/integration/markers.py index 2cfeab1c4f..2f6cdd315c 100644 --- a/tests/integration/markers.py +++ b/tests/integration/markers.py @@ -7,7 +7,6 @@ from . import architecture from .juju_ import juju_major_version -juju2 = pytest.mark.skipif(juju_major_version != 2, reason="Requires juju 2") juju3 = pytest.mark.skipif(juju_major_version != 3, reason="Requires juju 3") juju_secrets = pytest.mark.skipif(juju_major_version < 3, reason="Requires juju secrets") amd64_only = pytest.mark.skipif( diff --git a/tests/integration/new_relations/test_new_relations.py b/tests/integration/new_relations/test_new_relations_1.py similarity index 89% rename from tests/integration/new_relations/test_new_relations.py rename to tests/integration/new_relations/test_new_relations_1.py index 70069e86e4..f8d6c0d611 100644 --- a/tests/integration/new_relations/test_new_relations.py +++ b/tests/integration/new_relations/test_new_relations_1.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. import asyncio @@ -13,7 +12,8 @@ from pytest_operator.plugin import OpsTest from tenacity import Retrying, stop_after_attempt, wait_fixed -from .. import markers +from constants import DATABASE_DEFAULT_NAME + from ..helpers import ( CHARM_BASE, assert_sync_standbys, @@ -24,7 +24,6 @@ start_machine, stop_machine, ) -from ..juju_ import juju_major_version from .helpers import ( build_connection_string, check_relation_data_existence, @@ -47,7 +46,6 @@ INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles" -@pytest.mark.group("new_relations_tests") @pytest.mark.abort_on_fail async def test_deploy_charms(ops_test: OpsTest, charm): """Deploy both charms (application and database) to use in the tests.""" @@ -81,7 +79,6 @@ async def test_deploy_charms(ops_test: OpsTest, charm): await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active", timeout=3000) -@pytest.mark.group("new_relations_tests") async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest): """Test that there is no read-only endpoint in a standalone cluster.""" async with ops_test.fast_forward(): @@ -98,24 +95,23 @@ async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest): await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active") # Check that on juju 3 we have secrets and no username and password in the rel databag - if juju_major_version > 2: - logger.info("checking for secrets") - secret_uri, password = await asyncio.gather( - get_application_relation_data( - ops_test, - APPLICATION_APP_NAME, - FIRST_DATABASE_RELATION_NAME, - "secret-user", - ), - get_application_relation_data( - ops_test, - APPLICATION_APP_NAME, - FIRST_DATABASE_RELATION_NAME, - "password", - ), - ) - assert secret_uri is not None - assert password is None + logger.info("checking for secrets") + secret_uri, password = await asyncio.gather( + get_application_relation_data( + ops_test, + APPLICATION_APP_NAME, + FIRST_DATABASE_RELATION_NAME, + "secret-user", + ), + get_application_relation_data( + ops_test, + APPLICATION_APP_NAME, + FIRST_DATABASE_RELATION_NAME, + "password", + ), + ) + assert secret_uri is not None + assert password is None # Try to get the connection string of the database using the read-only endpoint. # It should not be available. @@ -128,7 +124,6 @@ async def test_no_read_only_endpoint_in_standalone_cluster(ops_test: OpsTest): ) -@pytest.mark.group("new_relations_tests") async def test_read_only_endpoint_in_scaled_up_cluster(ops_test: OpsTest): """Test that there is read-only endpoint in a scaled up cluster.""" async with ops_test.fast_forward(): @@ -146,7 +141,6 @@ async def test_read_only_endpoint_in_scaled_up_cluster(ops_test: OpsTest): ) -@pytest.mark.group("new_relations_tests") async def test_database_relation_with_charm_libraries(ops_test: OpsTest): """Test basic functionality of database relation interface.""" # Get the connection string to connect to the database using the read/write endpoint. @@ -194,7 +188,6 @@ async def test_database_relation_with_charm_libraries(ops_test: OpsTest): cursor.execute("DROP TABLE test;") -@pytest.mark.group("new_relations_tests") @pytest.mark.abort_on_fail async def test_filter_out_degraded_replicas(ops_test: OpsTest): primary = await get_primary(ops_test, f"{DATABASE_APP_NAME}/0") @@ -225,7 +218,6 @@ async def test_filter_out_degraded_replicas(ops_test: OpsTest): ) -@pytest.mark.group("new_relations_tests") async def test_user_with_extra_roles(ops_test: OpsTest): """Test superuser actions and the request for more permissions.""" # Get the connection string to connect to the database. @@ -246,7 +238,6 @@ async def test_user_with_extra_roles(ops_test: OpsTest): connection.close() -@pytest.mark.group("new_relations_tests") async def test_two_applications_doesnt_share_the_same_relation_data(ops_test: OpsTest): """Test that two different application connect to the database with different credentials.""" # Set some variables to use in this test. @@ -286,7 +277,10 @@ async def test_two_applications_doesnt_share_the_same_relation_data(ops_test: Op (another_application_app_name, f"{APPLICATION_APP_NAME.replace('-', '_')}_database"), ]: connection_string = await build_connection_string( - ops_test, application, FIRST_DATABASE_RELATION_NAME, database="postgres" + ops_test, + application, + FIRST_DATABASE_RELATION_NAME, + database=DATABASE_DEFAULT_NAME, ) with pytest.raises(psycopg2.Error): psycopg2.connect(connection_string) @@ -300,7 +294,6 @@ async def test_two_applications_doesnt_share_the_same_relation_data(ops_test: Op psycopg2.connect(connection_string) -@pytest.mark.group("new_relations_tests") async def test_an_application_can_connect_to_multiple_database_clusters(ops_test: OpsTest): """Test that an application can connect to different clusters of the same database.""" # Relate the application with both database clusters @@ -331,7 +324,6 @@ async def test_an_application_can_connect_to_multiple_database_clusters(ops_test assert application_connection_string != another_application_connection_string -@pytest.mark.group("new_relations_tests") async def test_an_application_can_connect_to_multiple_aliased_database_clusters(ops_test: OpsTest): """Test that an application can connect to different clusters of the same database.""" # Relate the application with both database clusters @@ -365,7 +357,6 @@ async def test_an_application_can_connect_to_multiple_aliased_database_clusters( assert application_connection_string != another_application_connection_string -@pytest.mark.group("new_relations_tests") async def test_an_application_can_request_multiple_databases(ops_test: OpsTest): """Test that an application can request additional databases using the same interface.""" # Relate the charms using another relation and wait for them exchanging some connection data. @@ -386,7 +377,6 @@ async def test_an_application_can_request_multiple_databases(ops_test: OpsTest): assert first_database_connection_string != second_database_connection_string -@pytest.mark.group("new_relations_tests") @pytest.mark.abort_on_fail async def test_relation_data_is_updated_correctly_when_scaling(ops_test: OpsTest): """Test that relation data, like connection data, is updated correctly when scaling.""" @@ -467,7 +457,6 @@ async def test_relation_data_is_updated_correctly_when_scaling(ops_test: OpsTest psycopg2.connect(primary_connection_string) -@pytest.mark.group("new_relations_tests") async def test_relation_with_no_database_name(ops_test: OpsTest): """Test that a relation with no database name doesn't block the charm.""" async with ops_test.fast_forward(): @@ -484,7 +473,6 @@ async def test_relation_with_no_database_name(ops_test: OpsTest): await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active", raise_on_blocked=True) -@pytest.mark.group("new_relations_tests") async def test_admin_role(ops_test: OpsTest): """Test that the admin role gives access to all the databases.""" all_app_names = [DATA_INTEGRATOR_APP_NAME] @@ -502,7 +490,7 @@ async def test_admin_role(ops_test: OpsTest): # Check that the user can access all the databases. for database in [ - "postgres", + DATABASE_DEFAULT_NAME, f"{APPLICATION_APP_NAME.replace('-', '_')}_database", "another_application_database", ]: @@ -526,12 +514,12 @@ async def test_admin_role(ops_test: OpsTest): ) assert version == data - # Write some data (it should fail in the "postgres" database). + # Write some data (it should fail in the default database name). random_name = ( f"test_{''.join(secrets.choice(string.ascii_lowercase) for _ in range(10))}" ) - should_fail = database == "postgres" - cursor.execute(f"CREATE TABLE {random_name}(data TEXT);") + should_fail = database == DATABASE_DEFAULT_NAME + cursor.execute(f"CREATE SCHEMA test; CREATE TABLE test.{random_name}(data TEXT);") if should_fail: assert False, ( f"failed to run a statement in the following database: {database}" @@ -548,7 +536,7 @@ async def test_admin_role(ops_test: OpsTest): # Test the creation and deletion of databases. connection_string = await build_connection_string( - ops_test, DATA_INTEGRATOR_APP_NAME, "postgresql", database="postgres" + ops_test, DATA_INTEGRATOR_APP_NAME, "postgresql", database=DATABASE_DEFAULT_NAME ) connection = psycopg2.connect(connection_string) connection.autocommit = True @@ -557,8 +545,10 @@ async def test_admin_role(ops_test: OpsTest): cursor.execute(f"CREATE DATABASE {random_name};") cursor.execute(f"DROP DATABASE {random_name};") try: - cursor.execute("DROP DATABASE postgres;") - assert False, "the admin extra user role was able to drop the `postgres` system database" + cursor.execute(f"DROP DATABASE {DATABASE_DEFAULT_NAME};") + assert False, ( + f"the admin extra user role was able to drop the `{DATABASE_DEFAULT_NAME}` system database" + ) except psycopg2.errors.InsufficientPrivilege: # Ignore the error, as the admin extra user role mustn't be able to drop # the "postgres" system database. @@ -567,7 +557,6 @@ async def test_admin_role(ops_test: OpsTest): connection.close() -@pytest.mark.group("new_relations_tests") async def test_invalid_extra_user_roles(ops_test: OpsTest): async with ops_test.fast_forward(): # Remove the relation between the database and the first data integrator. @@ -627,43 +616,3 @@ async def test_invalid_extra_user_roles(ops_test: OpsTest): raise_on_blocked=False, timeout=1000, ) - - -@pytest.mark.group("nextcloud_blocked") -@markers.amd64_only # nextcloud charm not available for arm64 -async def test_nextcloud_db_blocked(ops_test: OpsTest, charm: str) -> None: - # Deploy Database Charm and Nextcloud - await asyncio.gather( - ops_test.model.deploy( - charm, - application_name=DATABASE_APP_NAME, - num_units=1, - base=CHARM_BASE, - config={"profile": "testing"}, - ), - ops_test.model.deploy( - "nextcloud", - channel="edge", - application_name="nextcloud", - num_units=1, - base=CHARM_BASE, - ), - ) - await asyncio.gather( - ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=2000), - ops_test.model.wait_for_idle( - apps=["nextcloud"], - status="blocked", - raise_on_blocked=False, - timeout=2000, - ), - ) - - await ops_test.model.relate("nextcloud:database", f"{DATABASE_APP_NAME}:database") - - await ops_test.model.wait_for_idle( - apps=[DATABASE_APP_NAME, "nextcloud"], - status="active", - raise_on_blocked=False, - timeout=1000, - ) diff --git a/tests/integration/new_relations/test_new_relations_2.py b/tests/integration/new_relations/test_new_relations_2.py new file mode 100644 index 0000000000..4a9134a2d1 --- /dev/null +++ b/tests/integration/new_relations/test_new_relations_2.py @@ -0,0 +1,69 @@ +# Copyright 2022 Canonical Ltd. +# See LICENSE file for licensing details. +import asyncio +import logging +from pathlib import Path + +import pytest +import yaml +from pytest_operator.plugin import OpsTest + +from .. import markers +from ..helpers import ( + CHARM_BASE, +) + +logger = logging.getLogger(__name__) + +APPLICATION_APP_NAME = "postgresql-test-app" +DATABASE_APP_NAME = "database" +ANOTHER_DATABASE_APP_NAME = "another-database" +DATA_INTEGRATOR_APP_NAME = "data-integrator" +APP_NAMES = [APPLICATION_APP_NAME, DATABASE_APP_NAME, ANOTHER_DATABASE_APP_NAME] +DATABASE_APP_METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) +FIRST_DATABASE_RELATION_NAME = "database" +SECOND_DATABASE_RELATION_NAME = "second-database" +MULTIPLE_DATABASE_CLUSTERS_RELATION_NAME = "multiple-database-clusters" +ALIASED_MULTIPLE_DATABASE_CLUSTERS_RELATION_NAME = "aliased-multiple-database-clusters" +NO_DATABASE_RELATION_NAME = "no-database" +INVALID_EXTRA_USER_ROLE_BLOCKING_MESSAGE = "invalid role(s) for extra user roles" + + +@markers.amd64_only # nextcloud charm not available for arm64 +@pytest.mark.skip(reason="Unstable") +async def test_nextcloud_db_blocked(ops_test: OpsTest, charm: str) -> None: + # Deploy Database Charm and Nextcloud + await asyncio.gather( + ops_test.model.deploy( + charm, + application_name=DATABASE_APP_NAME, + num_units=1, + base=CHARM_BASE, + config={"profile": "testing"}, + ), + ops_test.model.deploy( + "nextcloud", + channel="edge", + application_name="nextcloud", + num_units=1, + base=CHARM_BASE, + ), + ) + await asyncio.gather( + ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=2000), + ops_test.model.wait_for_idle( + apps=["nextcloud"], + status="blocked", + raise_on_blocked=False, + timeout=2000, + ), + ) + + await ops_test.model.relate("nextcloud:database", f"{DATABASE_APP_NAME}:database") + + await ops_test.model.wait_for_idle( + apps=[DATABASE_APP_NAME, "nextcloud"], + status="active", + raise_on_blocked=False, + timeout=1000, + ) diff --git a/tests/integration/new_relations/test_relations_coherence.py b/tests/integration/new_relations/test_relations_coherence.py index 1f2a751922..74fd202fa6 100644 --- a/tests/integration/new_relations/test_relations_coherence.py +++ b/tests/integration/new_relations/test_relations_coherence.py @@ -9,9 +9,11 @@ import pytest from pytest_operator.plugin import OpsTest +from constants import DATABASE_DEFAULT_NAME + from ..helpers import CHARM_BASE, DATABASE_APP_NAME from .helpers import build_connection_string -from .test_new_relations import DATA_INTEGRATOR_APP_NAME +from .test_new_relations_1 import DATA_INTEGRATOR_APP_NAME logger = logging.getLogger(__name__) @@ -20,7 +22,6 @@ FIRST_DATABASE_RELATION_NAME = "database" -@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_relations(ops_test: OpsTest, charm): """Test that check relation data.""" @@ -126,14 +127,14 @@ async def test_relations(ops_test: OpsTest, charm): for database in [ DATA_INTEGRATOR_APP_NAME.replace("-", "_"), - "postgres", + DATABASE_DEFAULT_NAME, ]: logger.info(f"connecting to the following database: {database}") connection_string = await build_connection_string( ops_test, DATA_INTEGRATOR_APP_NAME, "postgresql", database=database ) connection = None - should_fail = database == "postgres" + should_fail = database == DATABASE_DEFAULT_NAME try: with ( psycopg2.connect(connection_string) as connection, @@ -143,7 +144,7 @@ async def test_relations(ops_test: OpsTest, charm): data = cursor.fetchone() assert data[0] == "some data" - # Write some data (it should fail in the "postgres" database). + # Write some data (it should fail in the default database name). random_name = f"test_{''.join(secrets.choice(string.ascii_lowercase) for _ in range(10))}" cursor.execute(f"CREATE TABLE {random_name}(data TEXT);") if should_fail: diff --git a/tests/integration/relations/__init__.py b/tests/integration/relations/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/integration/relations/helpers.py b/tests/integration/relations/helpers.py deleted file mode 100644 index 1bafe6e79a..0000000000 --- a/tests/integration/relations/helpers.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. - -import yaml -from pytest_operator.plugin import OpsTest - - -async def get_legacy_db_connection_str( - ops_test: OpsTest, - application_name: str, - relation_name: str, - read_only_endpoint: bool = False, - remote_unit_name: str | None = None, -) -> str | None: - """Returns a PostgreSQL connection string. - - Args: - ops_test: The ops test framework instance - application_name: The name of the application - relation_name: name of the relation to get connection data from - read_only_endpoint: whether to choose the read-only endpoint - instead of the read/write endpoint - remote_unit_name: Optional remote unit name used to retrieve - unit data instead of application data - - Returns: - a PostgreSQL connection string - """ - unit_name = f"{application_name}/0" - raw_data = (await ops_test.juju("show-unit", unit_name))[1] - if not raw_data: - raise ValueError(f"no unit info could be grabbed for {unit_name}") - data = yaml.safe_load(raw_data) - # Filter the data based on the relation name. - relation_data = [ - v for v in data[unit_name]["relation-info"] if v["related-endpoint"] == relation_name - ] - if len(relation_data) == 0: - raise ValueError( - f"no relation data could be grabbed on relation with endpoint {relation_name}" - ) - if remote_unit_name: - data = relation_data[0]["related-units"][remote_unit_name]["data"] - else: - data = relation_data[0]["application-data"] - if read_only_endpoint: - if data.get("standbys") is None: - return None - return data.get("standbys").split(",")[0] - else: - return data.get("master") diff --git a/tests/integration/relations/test_relations.py b/tests/integration/relations/test_relations.py deleted file mode 100644 index b3a542decc..0000000000 --- a/tests/integration/relations/test_relations.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. -import asyncio -import logging - -import psycopg2 -import pytest -from pytest_operator.plugin import OpsTest -from tenacity import Retrying, stop_after_delay, wait_fixed - -from ..helpers import CHARM_BASE, METADATA -from ..new_relations.test_new_relations import APPLICATION_APP_NAME, build_connection_string -from ..relations.helpers import get_legacy_db_connection_str - -logger = logging.getLogger(__name__) - -APP_NAME = METADATA["name"] -# MAILMAN3_CORE_APP_NAME = "mailman3-core" -DB_RELATION = "db" -DATABASE_RELATION = "database" -FIRST_DATABASE_RELATION = "database" -DATABASE_APP_NAME = "database-app" -DB_APP_NAME = "db-app" -APP_NAMES = [APP_NAME, DATABASE_APP_NAME, DB_APP_NAME] - - -@pytest.mark.group(1) -@pytest.mark.abort_on_fail -async def test_deploy_charms(ops_test: OpsTest, charm): - """Deploy both charms (application and database) to use in the tests.""" - # Deploy both charms (multiple units for each application to test that later they correctly - # set data in the relation application databag using only the leader unit). - async with ops_test.fast_forward(): - await asyncio.gather( - ops_test.model.deploy( - APPLICATION_APP_NAME, - application_name=DATABASE_APP_NAME, - num_units=1, - base=CHARM_BASE, - channel="edge", - ), - ops_test.model.deploy( - charm, - application_name=APP_NAME, - num_units=1, - base=CHARM_BASE, - config={ - "profile": "testing", - "plugin_unaccent_enable": "True", - "plugin_pg_trgm_enable": "True", - }, - ), - ops_test.model.deploy( - APPLICATION_APP_NAME, - application_name=DB_APP_NAME, - num_units=1, - base=CHARM_BASE, - channel="edge", - ), - ) - - await ops_test.model.wait_for_idle(apps=APP_NAMES, status="active", timeout=3000) - - -@pytest.mark.group(1) -async def test_legacy_endpoint_with_multiple_related_endpoints(ops_test: OpsTest): - await ops_test.model.relate(f"{DB_APP_NAME}:{DB_RELATION}", f"{APP_NAME}:{DB_RELATION}") - await ops_test.model.relate(APP_NAME, f"{DATABASE_APP_NAME}:{FIRST_DATABASE_RELATION}") - - app = ops_test.model.applications[APP_NAME] - await ops_test.model.block_until( - lambda: "blocked" in {unit.workload_status for unit in app.units}, - timeout=1500, - ) - - logger.info(" remove relation with modern endpoints") - await ops_test.model.applications[APP_NAME].remove_relation( - f"{APP_NAME}:{DATABASE_RELATION}", f"{DATABASE_APP_NAME}:{FIRST_DATABASE_RELATION}" - ) - async with ops_test.fast_forward(): - await ops_test.model.wait_for_idle( - status="active", - timeout=1500, - raise_on_error=False, - ) - - legacy_interface_connect = await get_legacy_db_connection_str( - ops_test, DB_APP_NAME, DB_RELATION, remote_unit_name=f"{APP_NAME}/0" - ) - logger.info(f" check connect to = {legacy_interface_connect}") - for attempt in Retrying(stop=stop_after_delay(60 * 3), wait=wait_fixed(10)): - with attempt, psycopg2.connect(legacy_interface_connect) as connection: - assert connection.status == psycopg2.extensions.STATUS_READY - - logger.info(f" remove relation {DB_APP_NAME}:{DB_RELATION}") - async with ops_test.fast_forward(): - await ops_test.model.applications[APP_NAME].remove_relation( - f"{APP_NAME}:{DB_RELATION}", f"{DB_APP_NAME}:{DB_RELATION}" - ) - await ops_test.model.wait_for_idle(apps=[APP_NAME], status="active", timeout=1000) - for attempt in Retrying(stop=stop_after_delay(60 * 5), wait=wait_fixed(10)): - with attempt, pytest.raises(psycopg2.OperationalError): - psycopg2.connect(legacy_interface_connect) - - -@pytest.mark.group(1) -async def test_modern_endpoint_with_multiple_related_endpoints(ops_test: OpsTest): - await ops_test.model.relate(f"{DB_APP_NAME}:{DB_RELATION}", f"{APP_NAME}:{DB_RELATION}") - await ops_test.model.relate(APP_NAME, f"{DATABASE_APP_NAME}:{FIRST_DATABASE_RELATION}") - - app = ops_test.model.applications[APP_NAME] - await ops_test.model.block_until( - lambda: "blocked" in {unit.workload_status for unit in app.units}, - timeout=1500, - ) - - logger.info(" remove relation with legacy endpoints") - await ops_test.model.applications[APP_NAME].remove_relation( - f"{DB_APP_NAME}:{DB_RELATION}", f"{APP_NAME}:{DB_RELATION}" - ) - async with ops_test.fast_forward(): - await ops_test.model.wait_for_idle(status="active", timeout=3000, raise_on_error=False) - - modern_interface_connect = await build_connection_string( - ops_test, DATABASE_APP_NAME, FIRST_DATABASE_RELATION - ) - logger.info(f"check connect to = {modern_interface_connect}") - for attempt in Retrying(stop=stop_after_delay(60 * 3), wait=wait_fixed(10)): - with attempt, psycopg2.connect(modern_interface_connect) as connection: - assert connection.status == psycopg2.extensions.STATUS_READY - - logger.info(f"remove relation {DATABASE_APP_NAME}:{FIRST_DATABASE_RELATION}") - async with ops_test.fast_forward(): - await ops_test.model.applications[APP_NAME].remove_relation( - f"{APP_NAME}:{DATABASE_RELATION}", f"{DATABASE_APP_NAME}:{FIRST_DATABASE_RELATION}" - ) - await ops_test.model.wait_for_idle(apps=[APP_NAME], status="active", timeout=1000) - for attempt in Retrying(stop=stop_after_delay(60 * 5), wait=wait_fixed(10)): - with attempt, pytest.raises(psycopg2.OperationalError): - psycopg2.connect(modern_interface_connect) diff --git a/tests/integration/test_audit.py b/tests/integration/test_audit.py index 257ef9cf70..4b4c3ae5e0 100644 --- a/tests/integration/test_audit.py +++ b/tests/integration/test_audit.py @@ -21,7 +21,6 @@ RELATION_ENDPOINT = "database" -@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_audit_plugin(ops_test: OpsTest, charm) -> None: """Test the audit plugin.""" diff --git a/tests/integration/test_backups_aws.py b/tests/integration/test_backups_aws.py new file mode 100644 index 0000000000..a6cd122393 --- /dev/null +++ b/tests/integration/test_backups_aws.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. +import logging + +import pytest as pytest +from pytest_operator.plugin import OpsTest +from tenacity import Retrying, stop_after_attempt, wait_exponential + +from . import architecture +from .conftest import AWS +from .helpers import ( + DATABASE_APP_NAME, + backup_operations, + db_connect, + get_password, + get_primary, + get_unit_address, + scale_application, + switchover, +) + +ANOTHER_CLUSTER_REPOSITORY_ERROR_MESSAGE = "the S3 repository has backups from another cluster" +FAILED_TO_ACCESS_CREATE_BUCKET_ERROR_MESSAGE = ( + "failed to access/create the bucket, check your S3 settings" +) +S3_INTEGRATOR_APP_NAME = "s3-integrator" +tls_certificates_app_name = "self-signed-certificates" +tls_channel = "latest/edge" if architecture.architecture == "arm64" else "latest/stable" +tls_config = {"ca-common-name": "Test CA"} + +logger = logging.getLogger(__name__) + + +@pytest.mark.abort_on_fail +async def test_backup_aws(ops_test: OpsTest, aws_cloud_configs: tuple[dict, dict], charm) -> None: + """Build and deploy two units of PostgreSQL in AWS, test backup and restore actions.""" + config = aws_cloud_configs[0] + credentials = aws_cloud_configs[1] + + await backup_operations( + ops_test, + S3_INTEGRATOR_APP_NAME, + tls_certificates_app_name, + tls_config, + tls_channel, + credentials, + AWS, + config, + charm, + ) + database_app_name = f"{DATABASE_APP_NAME}-aws" + + # Remove the relation to the TLS certificates operator. + await ops_test.model.applications[database_app_name].remove_relation( + f"{database_app_name}:certificates", f"{tls_certificates_app_name}:certificates" + ) + + new_unit_name = f"{database_app_name}/2" + + # Scale up to be able to test primary and leader being different. + async with ops_test.fast_forward(): + await scale_application(ops_test, database_app_name, 2) + + # Ensure replication is working correctly. + address = get_unit_address(ops_test, new_unit_name) + password = await get_password(ops_test, new_unit_name) + patroni_password = await get_password(ops_test, new_unit_name, "patroni") + with db_connect(host=address, password=password) as connection, connection.cursor() as cursor: + cursor.execute( + "SELECT EXISTS (SELECT FROM information_schema.tables" + " WHERE table_schema = 'public' AND table_name = 'backup_table_1');" + ) + assert cursor.fetchone()[0], ( + f"replication isn't working correctly: table 'backup_table_1' doesn't exist in {new_unit_name}" + ) + cursor.execute( + "SELECT EXISTS (SELECT FROM information_schema.tables" + " WHERE table_schema = 'public' AND table_name = 'backup_table_2');" + ) + assert not cursor.fetchone()[0], ( + f"replication isn't working correctly: table 'backup_table_2' exists in {new_unit_name}" + ) + connection.close() + + old_primary = await get_primary(ops_test, new_unit_name) + switchover(ops_test, old_primary, patroni_password, new_unit_name) + + # Get the new primary unit. + primary = await get_primary(ops_test, new_unit_name) + # Check that the primary changed. + for attempt in Retrying( + stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=30) + ): + with attempt: + assert primary == new_unit_name + + # Ensure stanza is working correctly. + logger.info("listing the available backups") + action = await ops_test.model.units.get(new_unit_name).run_action("list-backups") + await action.wait() + backups = action.results.get("backups") + assert backups, "backups not outputted" + + await ops_test.model.wait_for_idle(status="active", timeout=1000) + + # Remove the database app. + await ops_test.model.remove_application(database_app_name, block_until_done=True) + + # Remove the TLS operator. + await ops_test.model.remove_application(tls_certificates_app_name, block_until_done=True) diff --git a/tests/integration/test_backups_ceph.py b/tests/integration/test_backups_ceph.py index 99a57fb1e1..3b3741e37a 100644 --- a/tests/integration/test_backups_ceph.py +++ b/tests/integration/test_backups_ceph.py @@ -18,19 +18,13 @@ from .helpers import ( backup_operations, ) -from .juju_ import juju_major_version logger = logging.getLogger(__name__) S3_INTEGRATOR_APP_NAME = "s3-integrator" -if juju_major_version < 3: - tls_certificates_app_name = "tls-certificates-operator" - tls_channel = "legacy/edge" if architecture.architecture == "arm64" else "legacy/stable" - tls_config = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"} -else: - tls_certificates_app_name = "self-signed-certificates" - tls_channel = "latest/edge" if architecture.architecture == "arm64" else "latest/stable" - tls_config = {"ca-common-name": "Test CA"} +tls_certificates_app_name = "self-signed-certificates" +tls_channel = "latest/edge" if architecture.architecture == "arm64" else "latest/stable" +tls_config = {"ca-common-name": "Test CA"} backup_id, value_before_backup, value_after_backup = "", None, None @@ -190,7 +184,6 @@ def cloud_configs(microceph: ConnectionInformation): } -@pytest.mark.group("ceph") @markers.amd64_only async def test_backup_ceph(ops_test: OpsTest, cloud_configs, cloud_credentials, charm) -> None: """Build and deploy two units of PostgreSQL in microceph, test backup and restore actions.""" diff --git a/tests/integration/test_backups.py b/tests/integration/test_backups_gcp.py similarity index 55% rename from tests/integration/test_backups.py rename to tests/integration/test_backups_gcp.py index e087abd5b3..3f8aa3111f 100644 --- a/tests/integration/test_backups.py +++ b/tests/integration/test_backups_gcp.py @@ -4,178 +4,39 @@ import logging import uuid -import boto3 import pytest as pytest from pytest_operator.plugin import OpsTest from tenacity import Retrying, stop_after_attempt, wait_exponential from . import architecture +from .conftest import GCP from .helpers import ( CHARM_BASE, DATABASE_APP_NAME, backup_operations, - construct_endpoint, db_connect, get_password, - get_primary, get_unit_address, - scale_application, - switchover, wait_for_idle_on_blocked, ) -from .juju_ import juju_major_version ANOTHER_CLUSTER_REPOSITORY_ERROR_MESSAGE = "the S3 repository has backups from another cluster" FAILED_TO_ACCESS_CREATE_BUCKET_ERROR_MESSAGE = ( "failed to access/create the bucket, check your S3 settings" ) S3_INTEGRATOR_APP_NAME = "s3-integrator" -if juju_major_version < 3: - tls_certificates_app_name = "tls-certificates-operator" - tls_channel = "legacy/edge" if architecture.architecture == "arm64" else "legacy/stable" - tls_config = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"} -else: - tls_certificates_app_name = "self-signed-certificates" - tls_channel = "latest/edge" if architecture.architecture == "arm64" else "latest/stable" - tls_config = {"ca-common-name": "Test CA"} +tls_certificates_app_name = "self-signed-certificates" +tls_channel = "latest/edge" if architecture.architecture == "arm64" else "latest/stable" +tls_config = {"ca-common-name": "Test CA"} logger = logging.getLogger(__name__) -AWS = "AWS" -GCP = "GCP" - -@pytest.fixture(scope="module") -async def cloud_configs(github_secrets) -> None: - # Define some configurations and credentials. - configs = { - AWS: { - "endpoint": "https://s3.amazonaws.com", - "bucket": "data-charms-testing", - "path": f"/postgresql-vm/{uuid.uuid1()}", - "region": "us-east-1", - }, - GCP: { - "endpoint": "https://storage.googleapis.com", - "bucket": "data-charms-testing", - "path": f"/postgresql-vm/{uuid.uuid1()}", - "region": "", - }, - } - credentials = { - AWS: { - "access-key": github_secrets["AWS_ACCESS_KEY"], - "secret-key": github_secrets["AWS_SECRET_KEY"], - }, - GCP: { - "access-key": github_secrets["GCP_ACCESS_KEY"], - "secret-key": github_secrets["GCP_SECRET_KEY"], - }, - } - yield configs, credentials - # Delete the previously created objects. - logger.info("deleting the previously created backups") - for cloud, config in configs.items(): - session = boto3.session.Session( - aws_access_key_id=credentials[cloud]["access-key"], - aws_secret_access_key=credentials[cloud]["secret-key"], - region_name=config["region"], - ) - s3 = session.resource( - "s3", endpoint_url=construct_endpoint(config["endpoint"], config["region"]) - ) - bucket = s3.Bucket(config["bucket"]) - # GCS doesn't support batch delete operation, so delete the objects one by one. - for bucket_object in bucket.objects.filter(Prefix=config["path"].lstrip("/")): - bucket_object.delete() - - -@pytest.mark.group("AWS") @pytest.mark.abort_on_fail -async def test_backup_aws(ops_test: OpsTest, cloud_configs: tuple[dict, dict], charm) -> None: - """Build and deploy two units of PostgreSQL in AWS, test backup and restore actions.""" - config = cloud_configs[0][AWS] - credentials = cloud_configs[1][AWS] - - await backup_operations( - ops_test, - S3_INTEGRATOR_APP_NAME, - tls_certificates_app_name, - tls_config, - tls_channel, - credentials, - AWS, - config, - charm, - ) - database_app_name = f"{DATABASE_APP_NAME}-aws" - - # Remove the relation to the TLS certificates operator. - await ops_test.model.applications[database_app_name].remove_relation( - f"{database_app_name}:certificates", f"{tls_certificates_app_name}:certificates" - ) - - new_unit_name = f"{database_app_name}/2" - - # Scale up to be able to test primary and leader being different. - async with ops_test.fast_forward(): - await scale_application(ops_test, database_app_name, 2) - - # Ensure replication is working correctly. - address = get_unit_address(ops_test, new_unit_name) - password = await get_password(ops_test, new_unit_name) - patroni_password = await get_password(ops_test, new_unit_name, "patroni") - with db_connect(host=address, password=password) as connection, connection.cursor() as cursor: - cursor.execute( - "SELECT EXISTS (SELECT FROM information_schema.tables" - " WHERE table_schema = 'public' AND table_name = 'backup_table_1');" - ) - assert cursor.fetchone()[0], ( - f"replication isn't working correctly: table 'backup_table_1' doesn't exist in {new_unit_name}" - ) - cursor.execute( - "SELECT EXISTS (SELECT FROM information_schema.tables" - " WHERE table_schema = 'public' AND table_name = 'backup_table_2');" - ) - assert not cursor.fetchone()[0], ( - f"replication isn't working correctly: table 'backup_table_2' exists in {new_unit_name}" - ) - connection.close() - - old_primary = await get_primary(ops_test, new_unit_name) - switchover(ops_test, old_primary, patroni_password, new_unit_name) - - # Get the new primary unit. - primary = await get_primary(ops_test, new_unit_name) - # Check that the primary changed. - for attempt in Retrying( - stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=30) - ): - with attempt: - assert primary == new_unit_name - - # Ensure stanza is working correctly. - logger.info("listing the available backups") - action = await ops_test.model.units.get(new_unit_name).run_action("list-backups") - await action.wait() - backups = action.results.get("backups") - assert backups, "backups not outputted" - - await ops_test.model.wait_for_idle(status="active", timeout=1000) - - # Remove the database app. - await ops_test.model.remove_application(database_app_name, block_until_done=True) - - # Remove the TLS operator. - await ops_test.model.remove_application(tls_certificates_app_name, block_until_done=True) - - -@pytest.mark.group("GCP") -@pytest.mark.abort_on_fail -async def test_backup_gcp(ops_test: OpsTest, cloud_configs: tuple[dict, dict], charm) -> None: +async def test_backup_gcp(ops_test: OpsTest, gcp_cloud_configs: tuple[dict, dict], charm) -> None: """Build and deploy two units of PostgreSQL in GCP, test backup and restore actions.""" - config = cloud_configs[0][GCP] - credentials = cloud_configs[1][GCP] + config = gcp_cloud_configs[0] + credentials = gcp_cloud_configs[1] await backup_operations( ops_test, @@ -197,8 +58,9 @@ async def test_backup_gcp(ops_test: OpsTest, cloud_configs: tuple[dict, dict], c await ops_test.model.remove_application(tls_certificates_app_name, block_until_done=True) -@pytest.mark.group("GCP") -async def test_restore_on_new_cluster(ops_test: OpsTest, github_secrets, charm) -> None: +async def test_restore_on_new_cluster( + ops_test: OpsTest, charm, gcp_cloud_configs: tuple[dict, dict] +) -> None: """Test that is possible to restore a backup to another PostgreSQL cluster.""" previous_database_app_name = f"{DATABASE_APP_NAME}-gcp" database_app_name = f"new-{DATABASE_APP_NAME}" @@ -295,9 +157,8 @@ async def test_restore_on_new_cluster(ops_test: OpsTest, github_secrets, charm) connection.close() -@pytest.mark.group("GCP") async def test_invalid_config_and_recovery_after_fixing_it( - ops_test: OpsTest, cloud_configs: tuple[dict, dict] + ops_test: OpsTest, gcp_cloud_configs: tuple[dict, dict] ) -> None: """Test that the charm can handle invalid and valid backup configurations.""" database_app_name = f"new-{DATABASE_APP_NAME}" @@ -328,10 +189,10 @@ async def test_invalid_config_and_recovery_after_fixing_it( logger.info( "configuring S3 integrator for a valid cloud, but with the path of another cluster repository" ) - await ops_test.model.applications[S3_INTEGRATOR_APP_NAME].set_config(cloud_configs[0][GCP]) + await ops_test.model.applications[S3_INTEGRATOR_APP_NAME].set_config(gcp_cloud_configs[0]) action = await ops_test.model.units.get(f"{S3_INTEGRATOR_APP_NAME}/0").run_action( "sync-s3-credentials", - **cloud_configs[1][GCP], + **gcp_cloud_configs[1], ) await action.wait() logger.info("waiting for the database charm to become blocked") @@ -342,7 +203,7 @@ async def test_invalid_config_and_recovery_after_fixing_it( # Provide valid backup configurations, with another path in the S3 bucket. logger.info("configuring S3 integrator for a valid cloud") - config = cloud_configs[0][GCP].copy() + config = gcp_cloud_configs[0].copy() config["path"] = f"/postgresql/{uuid.uuid1()}" await ops_test.model.applications[S3_INTEGRATOR_APP_NAME].set_config(config) logger.info("waiting for the database charm to become active") diff --git a/tests/integration/test_backups_pitr.py b/tests/integration/test_backups_pitr_aws.py similarity index 82% rename from tests/integration/test_backups_pitr.py rename to tests/integration/test_backups_pitr_aws.py index 9a76ad3a5a..5a671b9623 100644 --- a/tests/integration/test_backups_pitr.py +++ b/tests/integration/test_backups_pitr_aws.py @@ -2,86 +2,30 @@ # Copyright 2024 Canonical Ltd. # See LICENSE file for licensing details. import logging -import uuid -import boto3 import pytest as pytest from pytest_operator.plugin import OpsTest from tenacity import Retrying, stop_after_attempt, wait_exponential from . import architecture +from .conftest import AWS from .helpers import ( CHARM_BASE, DATABASE_APP_NAME, - construct_endpoint, db_connect, get_password, get_primary, get_unit_address, ) -from .juju_ import juju_major_version CANNOT_RESTORE_PITR = "cannot restore PITR, juju debug-log for details" S3_INTEGRATOR_APP_NAME = "s3-integrator" -if juju_major_version < 3: - TLS_CERTIFICATES_APP_NAME = "tls-certificates-operator" - TLS_CHANNEL = "legacy/edge" if architecture.architecture == "arm64" else "legacy/stable" - TLS_CONFIG = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"} -else: - TLS_CERTIFICATES_APP_NAME = "self-signed-certificates" - TLS_CHANNEL = "latest/edge" if architecture.architecture == "arm64" else "latest/stable" - TLS_CONFIG = {"ca-common-name": "Test CA"} +TLS_CERTIFICATES_APP_NAME = "self-signed-certificates" +TLS_CHANNEL = "latest/edge" if architecture.architecture == "arm64" else "latest/stable" +TLS_CONFIG = {"ca-common-name": "Test CA"} logger = logging.getLogger(__name__) -AWS = "AWS" -GCP = "GCP" - - -@pytest.fixture(scope="module") -async def cloud_configs(github_secrets) -> None: - # Define some configurations and credentials. - configs = { - AWS: { - "endpoint": "https://s3.amazonaws.com", - "bucket": "data-charms-testing", - "path": f"/postgresql-vm/{uuid.uuid1()}", - "region": "us-east-1", - }, - GCP: { - "endpoint": "https://storage.googleapis.com", - "bucket": "data-charms-testing", - "path": f"/postgresql-vm/{uuid.uuid1()}", - "region": "", - }, - } - credentials = { - AWS: { - "access-key": github_secrets["AWS_ACCESS_KEY"], - "secret-key": github_secrets["AWS_SECRET_KEY"], - }, - GCP: { - "access-key": github_secrets["GCP_ACCESS_KEY"], - "secret-key": github_secrets["GCP_SECRET_KEY"], - }, - } - yield configs, credentials - # Delete the previously created objects. - logger.info("deleting the previously created backups") - for cloud, config in configs.items(): - session = boto3.session.Session( - aws_access_key_id=credentials[cloud]["access-key"], - aws_secret_access_key=credentials[cloud]["secret-key"], - region_name=config["region"], - ) - s3 = session.resource( - "s3", endpoint_url=construct_endpoint(config["endpoint"], config["region"]) - ) - bucket = s3.Bucket(config["bucket"]) - # GCS doesn't support batch delete operation, so delete the objects one by one. - for bucket_object in bucket.objects.filter(Prefix=config["path"].lstrip("/")): - bucket_object.delete() - async def pitr_backup_operations( ops_test: OpsTest, @@ -374,12 +318,12 @@ async def pitr_backup_operations( await ops_test.model.remove_application(tls_certificates_app_name, block_until_done=True) -@pytest.mark.group("AWS") @pytest.mark.abort_on_fail -async def test_pitr_backup_aws(ops_test: OpsTest, cloud_configs: tuple[dict, dict], charm) -> None: +async def test_pitr_backup_aws( + ops_test: OpsTest, aws_cloud_configs: tuple[dict, dict], charm +) -> None: """Build, deploy two units of PostgreSQL and do backup in AWS. Then, write new data into DB, switch WAL file and test point-in-time-recovery restore action.""" - config = cloud_configs[0][AWS] - credentials = cloud_configs[1][AWS] + config, credentials = aws_cloud_configs await pitr_backup_operations( ops_test, @@ -394,26 +338,6 @@ async def test_pitr_backup_aws(ops_test: OpsTest, cloud_configs: tuple[dict, dic ) -@pytest.mark.group("GCP") -@pytest.mark.abort_on_fail -async def test_pitr_backup_gcp(ops_test: OpsTest, cloud_configs: tuple[dict, dict], charm) -> None: - """Build, deploy two units of PostgreSQL and do backup in GCP. Then, write new data into DB, switch WAL file and test point-in-time-recovery restore action.""" - config = cloud_configs[0][GCP] - credentials = cloud_configs[1][GCP] - - await pitr_backup_operations( - ops_test, - S3_INTEGRATOR_APP_NAME, - TLS_CERTIFICATES_APP_NAME, - TLS_CONFIG, - TLS_CHANNEL, - credentials, - GCP, - config, - charm, - ) - - def _create_table(host: str, password: str): with db_connect(host=host, password=password) as connection: connection.autocommit = True diff --git a/tests/integration/test_backups_pitr_gcp.py b/tests/integration/test_backups_pitr_gcp.py new file mode 100644 index 0000000000..4e5e03d582 --- /dev/null +++ b/tests/integration/test_backups_pitr_gcp.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python3 +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. +import logging + +import pytest as pytest +from pytest_operator.plugin import OpsTest +from tenacity import Retrying, stop_after_attempt, wait_exponential + +from . import architecture +from .conftest import GCP +from .helpers import ( + CHARM_BASE, + DATABASE_APP_NAME, + db_connect, + get_password, + get_primary, + get_unit_address, +) + +CANNOT_RESTORE_PITR = "cannot restore PITR, juju debug-log for details" +S3_INTEGRATOR_APP_NAME = "s3-integrator" +TLS_CERTIFICATES_APP_NAME = "self-signed-certificates" +TLS_CHANNEL = "latest/edge" if architecture.architecture == "arm64" else "latest/stable" +TLS_CONFIG = {"ca-common-name": "Test CA"} + +logger = logging.getLogger(__name__) + + +async def pitr_backup_operations( + ops_test: OpsTest, + s3_integrator_app_name: str, + tls_certificates_app_name: str, + tls_config, + tls_channel, + credentials, + cloud, + config, + charm, +) -> None: + """Basic set of operations for PITR backup and timelines management testing. + + Below is presented algorithm in the next format: "(timeline): action_1 -> action_2". + 1: table -> backup_b1 -> test_data_td1 -> timestamp_ts1 -> test_data_td2 -> restore_ts1 => 2 + 2: check_td1 -> check_not_td2 -> test_data_td3 -> restore_b1_latest => 3 + 3: check_td1 -> check_td2 -> check_not_td3 -> test_data_td4 -> restore_t2_latest => 4 + 4: check_td1 -> check_not_td2 -> check_td3 -> check_not_td4 + """ + # Set-up environment + database_app_name = f"{DATABASE_APP_NAME}-{cloud.lower()}" + + logger.info("deploying the next charms: s3-integrator, self-signed-certificates, postgresql") + await ops_test.model.deploy(s3_integrator_app_name) + await ops_test.model.deploy(tls_certificates_app_name, config=tls_config, channel=tls_channel) + await ops_test.model.deploy( + charm, + application_name=database_app_name, + num_units=2, + base=CHARM_BASE, + config={"profile": "testing"}, + ) + + logger.info( + "integrating self-signed-certificates with postgresql and waiting them to stabilize" + ) + await ops_test.model.relate(database_app_name, tls_certificates_app_name) + async with ops_test.fast_forward(fast_interval="60s"): + await ops_test.model.wait_for_idle( + apps=[database_app_name, tls_certificates_app_name], status="active", timeout=1000 + ) + + logger.info(f"configuring s3-integrator for {cloud}") + await ops_test.model.applications[s3_integrator_app_name].set_config(config) + action = await ops_test.model.units.get(f"{s3_integrator_app_name}/0").run_action( + "sync-s3-credentials", + **credentials, + ) + await action.wait() + + logger.info("integrating s3-integrator with postgresql and waiting model to stabilize") + await ops_test.model.relate(database_app_name, s3_integrator_app_name) + async with ops_test.fast_forward(fast_interval="60s"): + await ops_test.model.wait_for_idle(status="active", timeout=1000) + + primary = await get_primary(ops_test, f"{database_app_name}/0") + for unit in ops_test.model.applications[database_app_name].units: + if unit.name != primary: + replica = unit.name + break + password = await get_password(ops_test, primary) + address = get_unit_address(ops_test, primary) + + logger.info("1: creating table") + _create_table(address, password) + + logger.info("1: creating backup b1") + action = await ops_test.model.units.get(replica).run_action("create-backup") + await action.wait() + backup_status = action.results.get("backup-status") + assert backup_status, "backup hasn't succeeded" + await ops_test.model.wait_for_idle(status="active", timeout=1000) + backup_b1 = await _get_most_recent_backup(ops_test, ops_test.model.units.get(replica)) + + logger.info("1: creating test data td1") + _insert_test_data("test_data_td1", address, password) + + logger.info("1: get timestamp ts1") + with db_connect(host=address, password=password) as connection, connection.cursor() as cursor: + cursor.execute("SELECT current_timestamp;") + timestamp_ts1 = str(cursor.fetchone()[0]) + connection.close() + # Wrong timestamp pointing to one year ahead + unreachable_timestamp_ts1 = timestamp_ts1.replace( + timestamp_ts1[:4], str(int(timestamp_ts1[:4]) + 1), 1 + ) + + logger.info("1: creating test data td2") + _insert_test_data("test_data_td2", address, password) + + logger.info("1: switching wal") + _switch_wal(address, password) + + logger.info("1: scaling down to do restore") + async with ops_test.fast_forward(): + await ops_test.model.destroy_unit(replica) + await ops_test.model.wait_for_idle(status="active", timeout=1000) + for unit in ops_test.model.applications[database_app_name].units: + remaining_unit = unit + break + + logger.info("1: restoring the backup b1 with bad restore-to-time parameter") + action = await remaining_unit.run_action( + "restore", **{"backup-id": backup_b1, "restore-to-time": "bad data"} + ) + await action.wait() + assert action.status == "failed", ( + "1: restore must fail with bad restore-to-time parameter, but that action succeeded" + ) + + logger.info("1: restoring the backup b1 with unreachable restore-to-time parameter") + action = await remaining_unit.run_action( + "restore", **{"backup-id": backup_b1, "restore-to-time": unreachable_timestamp_ts1} + ) + await action.wait() + logger.info("1: waiting for the database charm to become blocked after restore") + async with ops_test.fast_forward(): + await ops_test.model.block_until( + lambda: remaining_unit.workload_status_message == CANNOT_RESTORE_PITR, + timeout=1000, + ) + logger.info( + "1: database charm become in blocked state after restore, as supposed to be with unreachable PITR parameter" + ) + + for attempt in Retrying( + stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30) + ): + with attempt: + logger.info("1: restoring to the timestamp ts1") + action = await remaining_unit.run_action( + "restore", **{"restore-to-time": timestamp_ts1} + ) + await action.wait() + restore_status = action.results.get("restore-status") + assert restore_status, "1: restore to the timestamp ts1 hasn't succeeded" + await ops_test.model.wait_for_idle(status="active", timeout=1000, idle_period=30) + + logger.info("2: successful restore") + primary = await get_primary(ops_test, remaining_unit.name) + address = get_unit_address(ops_test, primary) + timeline_t2 = await _get_most_recent_backup(ops_test, remaining_unit) + assert backup_b1 != timeline_t2, "2: timeline 2 do not exist in list-backups action or bad" + + logger.info("2: checking test data td1") + assert _check_test_data("test_data_td1", address, password), "2: test data td1 should exist" + + logger.info("2: checking not test data td2") + assert not _check_test_data("test_data_td2", address, password), ( + "2: test data td2 shouldn't exist" + ) + + logger.info("2: creating test data td3") + _insert_test_data("test_data_td3", address, password) + + logger.info("2: get timestamp ts2") + with db_connect(host=address, password=password) as connection, connection.cursor() as cursor: + cursor.execute("SELECT current_timestamp;") + timestamp_ts2 = str(cursor.fetchone()[0]) + connection.close() + + logger.info("2: creating test data td4") + _insert_test_data("test_data_td4", address, password) + + logger.info("2: switching wal") + _switch_wal(address, password) + + for attempt in Retrying( + stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30) + ): + with attempt: + logger.info("2: restoring the backup b1 to the latest") + action = await remaining_unit.run_action( + "restore", **{"backup-id": backup_b1, "restore-to-time": "latest"} + ) + await action.wait() + restore_status = action.results.get("restore-status") + assert restore_status, "2: restore the backup b1 to the latest hasn't succeeded" + await ops_test.model.wait_for_idle(status="active", timeout=1000, idle_period=30) + + logger.info("3: successful restore") + primary = await get_primary(ops_test, remaining_unit.name) + address = get_unit_address(ops_test, primary) + timeline_t3 = await _get_most_recent_backup(ops_test, remaining_unit) + assert backup_b1 != timeline_t3 and timeline_t2 != timeline_t3, ( + "3: timeline 3 do not exist in list-backups action or bad" + ) + + logger.info("3: checking test data td1") + assert _check_test_data("test_data_td1", address, password), "3: test data td1 should exist" + + logger.info("3: checking test data td2") + assert _check_test_data("test_data_td2", address, password), "3: test data td2 should exist" + + logger.info("3: checking not test data td3") + assert not _check_test_data("test_data_td3", address, password), ( + "3: test data td3 shouldn't exist" + ) + + logger.info("3: checking not test data td4") + assert not _check_test_data("test_data_td4", address, password), ( + "3: test data td4 shouldn't exist" + ) + + logger.info("3: switching wal") + _switch_wal(address, password) + + for attempt in Retrying( + stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30) + ): + with attempt: + logger.info("3: restoring the timeline 2 to the latest") + action = await remaining_unit.run_action( + "restore", **{"backup-id": timeline_t2, "restore-to-time": "latest"} + ) + await action.wait() + restore_status = action.results.get("restore-status") + assert restore_status, "3: restore the timeline 2 to the latest hasn't succeeded" + await ops_test.model.wait_for_idle(status="active", timeout=1000, idle_period=30) + + logger.info("4: successful restore") + primary = await get_primary(ops_test, remaining_unit.name) + address = get_unit_address(ops_test, primary) + timeline_t4 = await _get_most_recent_backup(ops_test, remaining_unit) + assert ( + backup_b1 != timeline_t4 and timeline_t2 != timeline_t4 and timeline_t3 != timeline_t4 + ), "4: timeline 4 do not exist in list-backups action or bad" + + logger.info("4: checking test data td1") + assert _check_test_data("test_data_td1", address, password), "4: test data td1 should exist" + + logger.info("4: checking not test data td2") + assert not _check_test_data("test_data_td2", address, password), ( + "4: test data td2 shouldn't exist" + ) + + logger.info("4: checking test data td3") + assert _check_test_data("test_data_td3", address, password), "4: test data td3 should exist" + + logger.info("4: checking test data td4") + assert _check_test_data("test_data_td4", address, password), "4: test data td4 should exist" + + logger.info("4: switching wal") + _switch_wal(address, password) + + for attempt in Retrying( + stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30) + ): + with attempt: + logger.info("4: restoring to the timestamp ts2") + action = await remaining_unit.run_action( + "restore", **{"restore-to-time": timestamp_ts2} + ) + await action.wait() + restore_status = action.results.get("restore-status") + assert restore_status, "4: restore to the timestamp ts2 hasn't succeeded" + await ops_test.model.wait_for_idle(status="active", timeout=1000, idle_period=30) + + logger.info("5: successful restore") + primary = await get_primary(ops_test, remaining_unit.name) + address = get_unit_address(ops_test, primary) + timeline_t5 = await _get_most_recent_backup(ops_test, remaining_unit) + assert ( + backup_b1 != timeline_t5 + and timeline_t2 != timeline_t5 + and timeline_t3 != timeline_t5 + and timeline_t4 != timeline_t5 + ), "5: timeline 5 do not exist in list-backups action or bad" + + logger.info("5: checking test data td1") + assert _check_test_data("test_data_td1", address, password), "5: test data td1 should exist" + + logger.info("5: checking not test data td2") + assert not _check_test_data("test_data_td2", address, password), ( + "5: test data td2 shouldn't exist" + ) + + logger.info("5: checking test data td3") + assert _check_test_data("test_data_td3", address, password), "5: test data td3 should exist" + + logger.info("5: checking not test data td4") + assert not _check_test_data("test_data_td4", address, password), ( + "5: test data td4 shouldn't exist" + ) + + # Remove the database app. + await ops_test.model.remove_application(database_app_name, block_until_done=True) + # Remove the TLS operator. + await ops_test.model.remove_application(tls_certificates_app_name, block_until_done=True) + + +@pytest.mark.abort_on_fail +async def test_pitr_backup_gcp( + ops_test: OpsTest, gcp_cloud_configs: tuple[dict, dict], charm +) -> None: + """Build, deploy two units of PostgreSQL and do backup in GCP. Then, write new data into DB, switch WAL file and test point-in-time-recovery restore action.""" + config, credentials = gcp_cloud_configs + + await pitr_backup_operations( + ops_test, + S3_INTEGRATOR_APP_NAME, + TLS_CERTIFICATES_APP_NAME, + TLS_CONFIG, + TLS_CHANNEL, + credentials, + GCP, + config, + charm, + ) + + +def _create_table(host: str, password: str): + with db_connect(host=host, password=password) as connection: + connection.autocommit = True + connection.cursor().execute("CREATE TABLE IF NOT EXISTS backup_table (test_column TEXT);") + connection.close() + + +def _insert_test_data(td: str, host: str, password: str): + with db_connect(host=host, password=password) as connection: + connection.autocommit = True + connection.cursor().execute( + "INSERT INTO backup_table (test_column) VALUES (%s);", + (td,), + ) + connection.close() + + +def _check_test_data(td: str, host: str, password: str) -> bool: + with db_connect(host=host, password=password) as connection, connection.cursor() as cursor: + cursor.execute( + "SELECT EXISTS (SELECT 1 FROM backup_table WHERE test_column = %s);", + (td,), + ) + res = cursor.fetchone()[0] + connection.close() + return res + + +def _switch_wal(host: str, password: str): + with db_connect(host=host, password=password) as connection: + connection.autocommit = True + connection.cursor().execute("SELECT pg_switch_wal();") + connection.close() + + +async def _get_most_recent_backup(ops_test: OpsTest, unit: any) -> str: + logger.info("listing the available backups") + action = await unit.run_action("list-backups") + await action.wait() + backups = action.results.get("backups") + assert backups, "backups not outputted" + await ops_test.model.wait_for_idle(status="active", timeout=1000) + most_recent_backup = backups.split("\n")[-1] + return most_recent_backup.split()[0] diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 53ac76e4b2..743bbdb242 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -37,7 +37,6 @@ UNIT_IDS = [0, 1, 2] -@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed async def test_deploy(ops_test: OpsTest, charm: str): @@ -61,7 +60,6 @@ async def test_deploy(ops_test: OpsTest, charm: str): assert ops_test.model.applications[DATABASE_APP_NAME].units[0].workload_status == "active" -@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.parametrize("unit_id", UNIT_IDS) async def test_database_is_up(ops_test: OpsTest, unit_id: int): @@ -72,7 +70,6 @@ async def test_database_is_up(ops_test: OpsTest, unit_id: int): assert result.status_code == 200 -@pytest.mark.group(1) @pytest.mark.parametrize("unit_id", UNIT_IDS) async def test_exporter_is_up(ops_test: OpsTest, unit_id: int): # Query Patroni REST API and check the status that indicates @@ -85,7 +82,6 @@ async def test_exporter_is_up(ops_test: OpsTest, unit_id: int): ) -@pytest.mark.group(1) @pytest.mark.parametrize("unit_id", UNIT_IDS) async def test_settings_are_correct(ops_test: OpsTest, unit_id: int): # Connect to the PostgreSQL instance. @@ -171,7 +167,6 @@ async def test_settings_are_correct(ops_test: OpsTest, unit_id: int): assert unit.data["port-ranges"][0]["protocol"] == "tcp" -@pytest.mark.group(1) async def test_postgresql_locales(ops_test: OpsTest) -> None: raw_locales = await run_command_on_unit( ops_test, @@ -188,7 +183,6 @@ async def test_postgresql_locales(ops_test: OpsTest) -> None: assert locales == SNAP_LOCALES -@pytest.mark.group(1) async def test_postgresql_parameters_change(ops_test: OpsTest) -> None: """Test that's possible to change PostgreSQL parameters.""" await ops_test.model.applications[DATABASE_APP_NAME].set_config({ @@ -236,7 +230,6 @@ async def test_postgresql_parameters_change(ops_test: OpsTest) -> None: connection.close() -@pytest.mark.group(1) async def test_scale_down_and_up(ops_test: OpsTest): """Test data is replicated to new units after a scale up.""" # Ensure the initial number of units in the application. @@ -324,7 +317,6 @@ async def test_scale_down_and_up(ops_test: OpsTest): await scale_application(ops_test, DATABASE_APP_NAME, initial_scale) -@pytest.mark.group(1) async def test_switchover_sync_standby(ops_test: OpsTest): original_roles = await get_cluster_roles( ops_test, ops_test.model.applications[DATABASE_APP_NAME].units[0].name @@ -342,7 +334,6 @@ async def test_switchover_sync_standby(ops_test: OpsTest): assert new_roles["primaries"][0] == original_roles["sync_standbys"][0] -@pytest.mark.group(1) async def test_persist_data_through_primary_deletion(ops_test: OpsTest): """Test data persists through a primary deletion.""" # Set a composite application name in order to test in more than one series at the same time. diff --git a/tests/integration/test_config.py b/tests/integration/test_config.py index 622264c6c4..f9dba0acbb 100644 --- a/tests/integration/test_config.py +++ b/tests/integration/test_config.py @@ -15,13 +15,11 @@ logger = logging.getLogger(__name__) -@pytest.mark.group(1) @pytest.mark.abort_on_fail -async def test_config_parameters(ops_test: OpsTest) -> None: +async def test_config_parameters(ops_test: OpsTest, charm) -> None: """Build and deploy one unit of PostgreSQL and then test config with wrong parameters.""" # Build and deploy the PostgreSQL charm. async with ops_test.fast_forward(): - charm = await ops_test.build_charm(".") await ops_test.model.deploy( charm, num_units=1, @@ -34,6 +32,10 @@ async def test_config_parameters(ops_test: OpsTest) -> None: test_string = "abcXYZ123" configs = [ + {"synchronous_node_count": ["0", "1"]}, # config option is greater than 0 + { + "synchronous_node_count": [test_string, "all"] + }, # config option is one of `all`, `minority` or `majority` { "durability_synchronous_commit": [test_string, "on"] }, # config option is one of `on`, `remote_apply` or `remote_write` diff --git a/tests/integration/test_db.py b/tests/integration/test_db.py deleted file mode 100644 index 88f87c1536..0000000000 --- a/tests/integration/test_db.py +++ /dev/null @@ -1,319 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. -import asyncio -import logging - -import psycopg2 as psycopg2 -import pytest as pytest -from mailmanclient import Client -from pytest_operator.plugin import OpsTest -from tenacity import Retrying, stop_after_delay, wait_fixed - -from . import markers -from .helpers import ( - APPLICATION_NAME, - CHARM_BASE, - DATABASE_APP_NAME, - assert_sync_standbys, - build_connection_string, - check_database_users_existence, - check_databases_creation, - deploy_and_relate_application_with_postgresql, - deploy_and_relate_bundle_with_postgresql, - get_leader_unit, - run_command_on_unit, -) - -logger = logging.getLogger(__name__) - -LIVEPATCH_APP_NAME = "livepatch" -MAILMAN3_CORE_APP_NAME = "mailman3-core" -APPLICATION_UNITS = 1 -DATABASE_UNITS = 2 -RELATION_NAME = "db" - -EXTENSIONS_BLOCKING_MESSAGE = ( - "extensions requested through relation, enable them through config options" -) -ROLES_BLOCKING_MESSAGE = ( - "roles requested through relation, use postgresql_client interface instead" -) - - -@pytest.mark.group(1) -@pytest.mark.abort_on_fail -async def test_mailman3_core_db(ops_test: OpsTest, charm: str) -> None: - """Deploy Mailman3 Core to test the 'db' relation.""" - async with ops_test.fast_forward(): - await ops_test.model.deploy( - charm, - application_name=DATABASE_APP_NAME, - num_units=DATABASE_UNITS, - base=CHARM_BASE, - config={"profile": "testing"}, - ) - - # Wait until the PostgreSQL charm is successfully deployed. - await ops_test.model.wait_for_idle( - apps=[DATABASE_APP_NAME], - status="active", - timeout=1500, - wait_for_exact_units=DATABASE_UNITS, - ) - - # Extra config option for Mailman3 Core. - config = {"hostname": "example.org"} - # Deploy and test the deployment of Mailman3 Core. - relation_id = await deploy_and_relate_application_with_postgresql( - ops_test, - "mailman3-core", - MAILMAN3_CORE_APP_NAME, - APPLICATION_UNITS, - config, - series="focal", - ) - await check_databases_creation(ops_test, ["mailman3"]) - - mailman3_core_users = [f"relation-{relation_id}"] - - await check_database_users_existence(ops_test, mailman3_core_users, []) - - # Assert Mailman3 Core is configured to use PostgreSQL instead of SQLite. - mailman_unit = ops_test.model.applications[MAILMAN3_CORE_APP_NAME].units[0] - result = await run_command_on_unit(ops_test, mailman_unit.name, "mailman info") - assert "db url: postgres://" in result - - # Do some CRUD operations using Mailman3 Core client. - domain_name = "canonical.com" - list_name = "postgresql-list" - credentials = ( - result.split("credentials: ")[1].strip().split(":") - ) # This outputs a list containing username and password. - client = Client( - f"http://{mailman_unit.public_address}:8001/3.1", credentials[0], credentials[1] - ) - - # Create a domain and list the domains to check that the new one is there. - domain = client.create_domain(domain_name) - assert domain_name in [domain.mail_host for domain in client.domains] - - # Update the domain by creating a mailing list into it. - mailing_list = domain.create_list(list_name) - assert mailing_list.fqdn_listname in [ - mailing_list.fqdn_listname for mailing_list in domain.lists - ] - - # Delete the domain and check that the change was persisted. - domain.delete() - assert domain_name not in [domain.mail_host for domain in client.domains] - - -@pytest.mark.group(1) -@pytest.mark.abort_on_fail -async def test_relation_data_is_updated_correctly_when_scaling(ops_test: OpsTest): - """Test that relation data, like connection data, is updated correctly when scaling.""" - # Retrieve the list of current database unit names. - units_to_remove = [unit.name for unit in ops_test.model.applications[DATABASE_APP_NAME].units] - - async with ops_test.fast_forward(): - # Add two more units. - await ops_test.model.applications[DATABASE_APP_NAME].add_units(2) - await ops_test.model.wait_for_idle( - apps=[DATABASE_APP_NAME], status="active", timeout=1500, wait_for_exact_units=4 - ) - - assert_sync_standbys( - ops_test.model.applications[DATABASE_APP_NAME].units[0].public_address, 2 - ) - - # Remove the original units. - leader_unit = await get_leader_unit(ops_test, DATABASE_APP_NAME) - await ops_test.model.applications[DATABASE_APP_NAME].destroy_units(*[ - unit for unit in units_to_remove if unit != leader_unit.name - ]) - await ops_test.model.wait_for_idle( - apps=[DATABASE_APP_NAME], status="active", timeout=600, wait_for_exact_units=3 - ) - await ops_test.model.applications[DATABASE_APP_NAME].destroy_units(leader_unit.name) - await ops_test.model.wait_for_idle( - apps=[DATABASE_APP_NAME], status="active", timeout=600, wait_for_exact_units=2 - ) - - # Get the updated connection data and assert it can be used - # to write and read some data properly. - database_unit_name = ops_test.model.applications[DATABASE_APP_NAME].units[0].name - primary_connection_string = await build_connection_string( - ops_test, MAILMAN3_CORE_APP_NAME, RELATION_NAME, remote_unit_name=database_unit_name - ) - replica_connection_string = await build_connection_string( - ops_test, - MAILMAN3_CORE_APP_NAME, - RELATION_NAME, - read_only_endpoint=True, - remote_unit_name=database_unit_name, - ) - - # Connect to the database using the primary connection string. - with psycopg2.connect(primary_connection_string) as connection: - connection.autocommit = True - with connection.cursor() as cursor: - # Check that it's possible to write and read data from the database that - # was created for the application. - cursor.execute("DROP TABLE IF EXISTS test;") - cursor.execute("CREATE TABLE test(data TEXT);") - cursor.execute("INSERT INTO test(data) VALUES('some data');") - cursor.execute("SELECT data FROM test;") - data = cursor.fetchone() - assert data[0] == "some data" - connection.close() - - # Connect to the database using the replica endpoint. - with psycopg2.connect(replica_connection_string) as connection, connection.cursor() as cursor: - # Read some data. - cursor.execute("SELECT data FROM test;") - data = cursor.fetchone() - assert data[0] == "some data" - - # Try to alter some data in a read-only transaction. - with pytest.raises(psycopg2.errors.ReadOnlySqlTransaction): - cursor.execute("DROP TABLE test;") - connection.close() - - # Remove the relation and test that its user was deleted - # (by checking that the connection string doesn't work anymore). - async with ops_test.fast_forward(): - await ops_test.model.applications[DATABASE_APP_NAME].remove_relation( - f"{DATABASE_APP_NAME}:{RELATION_NAME}", f"{MAILMAN3_CORE_APP_NAME}:{RELATION_NAME}" - ) - await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=1000) - for attempt in Retrying(stop=stop_after_delay(60 * 3), wait=wait_fixed(10)): - with attempt, pytest.raises(psycopg2.OperationalError): - psycopg2.connect(primary_connection_string) - - -@pytest.mark.group(1) -async def test_roles_blocking(ops_test: OpsTest, charm: str) -> None: - await ops_test.model.deploy( - APPLICATION_NAME, - application_name=APPLICATION_NAME, - config={"legacy_roles": True}, - base=CHARM_BASE, - channel="edge", - ) - await ops_test.model.deploy( - APPLICATION_NAME, - application_name=f"{APPLICATION_NAME}2", - config={"legacy_roles": True}, - base=CHARM_BASE, - channel="edge", - ) - - await ops_test.model.wait_for_idle( - apps=[DATABASE_APP_NAME, APPLICATION_NAME, f"{APPLICATION_NAME}2"], - status="active", - timeout=1000, - ) - - await asyncio.gather( - ops_test.model.relate(f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}:db"), - ops_test.model.relate(f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}2:db"), - ) - - leader_unit = await get_leader_unit(ops_test, DATABASE_APP_NAME) - await ops_test.model.block_until( - lambda: leader_unit.workload_status_message == ROLES_BLOCKING_MESSAGE, timeout=1000 - ) - - assert leader_unit.workload_status_message == ROLES_BLOCKING_MESSAGE - - logger.info("Verify that the charm remains blocked if there are other blocking relations") - await ops_test.model.applications[DATABASE_APP_NAME].destroy_relation( - f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}:db" - ) - - await ops_test.model.block_until( - lambda: leader_unit.workload_status_message == ROLES_BLOCKING_MESSAGE, timeout=1000 - ) - - assert leader_unit.workload_status_message == ROLES_BLOCKING_MESSAGE - - logger.info("Verify that active status is restored when all blocking relations are gone") - await ops_test.model.applications[DATABASE_APP_NAME].destroy_relation( - f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}2:db" - ) - - await ops_test.model.wait_for_idle( - apps=[DATABASE_APP_NAME], - status="active", - timeout=1000, - ) - - -@pytest.mark.group(1) -async def test_extensions_blocking(ops_test: OpsTest, charm: str) -> None: - await asyncio.gather( - ops_test.model.applications[APPLICATION_NAME].set_config({"legacy_roles": "False"}), - ops_test.model.applications[f"{APPLICATION_NAME}2"].set_config({"legacy_roles": "False"}), - ) - await ops_test.model.wait_for_idle( - apps=[APPLICATION_NAME, f"{APPLICATION_NAME}2"], - status="active", - timeout=1000, - ) - - await asyncio.gather( - ops_test.model.relate(f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}:db"), - ops_test.model.relate(f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}2:db"), - ) - - leader_unit = await get_leader_unit(ops_test, DATABASE_APP_NAME) - await ops_test.model.block_until( - lambda: leader_unit.workload_status_message == EXTENSIONS_BLOCKING_MESSAGE, timeout=1000 - ) - - assert leader_unit.workload_status_message == EXTENSIONS_BLOCKING_MESSAGE - - logger.info("Verify that the charm remains blocked if there are other blocking relations") - await ops_test.model.applications[DATABASE_APP_NAME].destroy_relation( - f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}:db" - ) - - await ops_test.model.block_until( - lambda: leader_unit.workload_status_message == EXTENSIONS_BLOCKING_MESSAGE, timeout=1000 - ) - - assert leader_unit.workload_status_message == EXTENSIONS_BLOCKING_MESSAGE - - logger.info("Verify that active status is restored when all blocking relations are gone") - await ops_test.model.applications[DATABASE_APP_NAME].destroy_relation( - f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}2:db" - ) - - -@markers.juju2 -@pytest.mark.group(1) -@pytest.mark.unstable -@markers.amd64_only # canonical-livepatch-server charm (in bundle) not available for arm64 -async def test_canonical_livepatch_onprem_bundle_db(ops_test: OpsTest) -> None: - # Deploy and test the Livepatch onprem bundle (using this PostgreSQL charm - # and an overlay to make the Ubuntu Advantage charm work with PostgreSQL). - # We intentionally wait for the `✘ sync_token not set` status message as we - # aren't providing an Ubuntu Pro token (as this is just a test to ensure - # the database works in the context of the relation with the Livepatch charm). - overlay = { - "applications": {"ubuntu-advantage": {"charm": "ubuntu-advantage", "base": CHARM_BASE}} - } - await deploy_and_relate_bundle_with_postgresql( - ops_test, - "canonical-livepatch-onprem", - LIVEPATCH_APP_NAME, - relation_name="db", - status="blocked", - status_message="✘ sync_token not set", - overlay=overlay, - ) - - action = await ops_test.model.units.get(f"{LIVEPATCH_APP_NAME}/0").run_action("schema-upgrade") - await action.wait() - assert action.results.get("Code") == "0", "schema-upgrade action hasn't succeeded" diff --git a/tests/integration/test_db_admin.py b/tests/integration/test_db_admin.py deleted file mode 100644 index b95d38d70d..0000000000 --- a/tests/integration/test_db_admin.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. -import json -import logging - -import psycopg2 -import pytest -from landscape_api.base import HTTPError, run_query -from pytest_operator.plugin import OpsTest -from tenacity import Retrying, stop_after_delay, wait_fixed - -from .helpers import ( - CHARM_BASE, - DATABASE_APP_NAME, - build_connection_string, - check_database_users_existence, - check_databases_creation, - deploy_and_relate_bundle_with_postgresql, - ensure_correct_relation_data, - get_landscape_api_credentials, - get_machine_from_unit, - get_password, - get_primary, - primary_changed, - start_machine, - stop_machine, - switchover, -) - -logger = logging.getLogger(__name__) - -HAPROXY_APP_NAME = "haproxy" -LANDSCAPE_APP_NAME = "landscape-server" -RABBITMQ_APP_NAME = "rabbitmq-server" -DATABASE_UNITS = 3 -RELATION_NAME = "db-admin" - - -@pytest.mark.group(1) -async def test_landscape_scalable_bundle_db(ops_test: OpsTest, charm: str) -> None: - """Deploy Landscape Scalable Bundle to test the 'db-admin' relation.""" - await ops_test.model.deploy( - charm, - application_name=DATABASE_APP_NAME, - num_units=DATABASE_UNITS, - base=CHARM_BASE, - config={"profile": "testing"}, - ) - - # Deploy and test the Landscape Scalable bundle (using this PostgreSQL charm). - relation_id = await deploy_and_relate_bundle_with_postgresql( - ops_test, - "ch:landscape-scalable", - LANDSCAPE_APP_NAME, - main_application_num_units=2, - relation_name=RELATION_NAME, - timeout=3000, - ) - await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active") - - await check_databases_creation( - ops_test, - [ - "landscape-standalone-account-1", - "landscape-standalone-knowledge", - "landscape-standalone-main", - "landscape-standalone-package", - "landscape-standalone-resource-1", - "landscape-standalone-session", - ], - ) - - landscape_users = [f"relation-{relation_id}"] - - await check_database_users_existence(ops_test, landscape_users, []) - - # Create the admin user on Landscape through configs. - await ops_test.model.applications["landscape-server"].set_config({ - "admin_email": "admin@canonical.com", - "admin_name": "Admin", - "admin_password": "test1234", - }) - await ops_test.model.wait_for_idle( - apps=["landscape-server", DATABASE_APP_NAME], - status="active", - timeout=1200, - ) - - # Connect to the Landscape API through HAProxy and do some CRUD calls (without the update). - key, secret = await get_landscape_api_credentials(ops_test) - haproxy_unit = ops_test.model.applications[HAPROXY_APP_NAME].units[0] - api_uri = f"https://{haproxy_unit.public_address}/api/" - - # Create a role and list the available roles later to check that the new one is there. - role_name = "User1" - run_query(key, secret, "CreateRole", {"name": role_name}, api_uri, False) - api_response = run_query(key, secret, "GetRoles", {}, api_uri, False) - assert role_name in [user["name"] for user in json.loads(api_response)] - - # Remove the role and assert it isn't part of the roles list anymore. - run_query(key, secret, "RemoveRole", {"name": role_name}, api_uri, False) - api_response = run_query(key, secret, "GetRoles", {}, api_uri, False) - assert role_name not in [user["name"] for user in json.loads(api_response)] - - await ensure_correct_relation_data(ops_test, DATABASE_UNITS, LANDSCAPE_APP_NAME, RELATION_NAME) - - # Stop the primary unit machine. - logger.info("restarting primary") - former_primary = await get_primary(ops_test, f"{DATABASE_APP_NAME}/0") - former_primary_machine = await get_machine_from_unit(ops_test, former_primary) - patroni_password = await get_password(ops_test, former_primary, "patroni") - - await stop_machine(ops_test, former_primary_machine) - - # Await for a new primary to be elected. - assert await primary_changed(ops_test, former_primary) - - # Start the former primary unit machine again. - await start_machine(ops_test, former_primary_machine) - - # Wait for the unit to be ready again. Some errors in the start hook may happen due to - # rebooting the unit machine in the middle of a hook (what is needed when the issue from - # https://bugs.launchpad.net/juju/+bug/1999758 happens). - await ops_test.model.wait_for_idle( - apps=[DATABASE_APP_NAME], status="active", timeout=1500, raise_on_error=False - ) - - await ensure_correct_relation_data(ops_test, DATABASE_UNITS, LANDSCAPE_APP_NAME, RELATION_NAME) - - # Trigger a switchover. - logger.info("triggering a switchover") - primary = await get_primary(ops_test, f"{DATABASE_APP_NAME}/0") - switchover(ops_test, primary, patroni_password) - - # Await for a new primary to be elected. - assert await primary_changed(ops_test, primary) - - await ensure_correct_relation_data(ops_test, DATABASE_UNITS, LANDSCAPE_APP_NAME, RELATION_NAME) - - # Trigger a config change to start the Landscape API service again. - # The Landscape API was stopped after a new primary (postgresql) was elected. - await ops_test.model.applications["landscape-server"].set_config({ - "admin_name": "Admin 1", - }) - await ops_test.model.wait_for_idle( - apps=["landscape-server", DATABASE_APP_NAME], timeout=1500, status="active" - ) - - # Create a role and list the available roles later to check that the new one is there. - role_name = "User2" - try: - run_query(key, secret, "CreateRole", {"name": role_name}, api_uri, False) - except HTTPError as e: - assert False, f"error when trying to create role on Landscape: {e}" - - database_unit_name = ops_test.model.applications[DATABASE_APP_NAME].units[0].name - connection_string = await build_connection_string( - ops_test, LANDSCAPE_APP_NAME, RELATION_NAME, remote_unit_name=database_unit_name - ) - - # Remove the applications from the bundle. - await ops_test.model.remove_application(LANDSCAPE_APP_NAME, block_until_done=True) - await ops_test.model.remove_application(HAPROXY_APP_NAME, block_until_done=True) - await ops_test.model.remove_application(RABBITMQ_APP_NAME, block_until_done=True) - - # Remove the relation and test that its user was deleted - # (by checking that the connection string doesn't work anymore). - async with ops_test.fast_forward(): - await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=1000) - for attempt in Retrying(stop=stop_after_delay(60 * 3), wait=wait_fixed(10)): - with attempt, pytest.raises(psycopg2.OperationalError): - psycopg2.connect(connection_string) - - # Remove the PostgreSQL application. - await ops_test.model.remove_application(DATABASE_APP_NAME, block_until_done=True) diff --git a/tests/integration/test_password_rotation.py b/tests/integration/test_password_rotation.py index 0cf2f6c26c..563626b229 100644 --- a/tests/integration/test_password_rotation.py +++ b/tests/integration/test_password_rotation.py @@ -27,12 +27,10 @@ APP_NAME = METADATA["name"] -@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed -async def test_deploy_active(ops_test: OpsTest): +async def test_deploy_active(ops_test: OpsTest, charm): """Build the charm and deploy it.""" - charm = await ops_test.build_charm(".") async with ops_test.fast_forward(): await ops_test.model.deploy( charm, @@ -44,7 +42,6 @@ async def test_deploy_active(ops_test: OpsTest): await ops_test.model.wait_for_idle(apps=[APP_NAME], status="active", timeout=1500) -@pytest.mark.group(1) async def test_password_rotation(ops_test: OpsTest): """Test password rotation action.""" # Get the initial passwords set for the system users. @@ -120,7 +117,6 @@ async def test_password_rotation(ops_test: OpsTest): assert check_patroni(ops_test, unit.name, restart_time) -@pytest.mark.group(1) @markers.juju_secrets async def test_password_from_secret_same_as_cli(ops_test: OpsTest): """Checking if password is same as returned by CLI. @@ -147,7 +143,6 @@ async def test_password_from_secret_same_as_cli(ops_test: OpsTest): assert data[secret_id]["content"]["Data"]["replication-password"] == password -@pytest.mark.group(1) async def test_empty_password(ops_test: OpsTest) -> None: """Test that the password can't be set to an empty string.""" leader_unit = await get_leader_unit(ops_test, APP_NAME) @@ -160,7 +155,6 @@ async def test_empty_password(ops_test: OpsTest) -> None: assert password == "None" -@pytest.mark.group(1) async def test_db_connection_with_empty_password(ops_test: OpsTest): """Test that user can't connect with empty password.""" primary = await get_primary(ops_test, f"{APP_NAME}/0") @@ -169,7 +163,6 @@ async def test_db_connection_with_empty_password(ops_test: OpsTest): connection.close() -@pytest.mark.group(1) async def test_no_password_change_on_invalid_password(ops_test: OpsTest) -> None: """Test that in general, there is no change when password validation fails.""" leader_unit = await get_leader_unit(ops_test, APP_NAME) @@ -182,7 +175,6 @@ async def test_no_password_change_on_invalid_password(ops_test: OpsTest) -> None assert password1 == password2 -@pytest.mark.group(1) async def test_no_password_exposed_on_logs(ops_test: OpsTest) -> None: """Test that passwords don't get exposed on postgresql logs.""" for unit in ops_test.model.applications[APP_NAME].units: diff --git a/tests/integration/test_plugins.py b/tests/integration/test_plugins.py index 7b3d5d3ce1..4dedc11a7e 100644 --- a/tests/integration/test_plugins.py +++ b/tests/integration/test_plugins.py @@ -62,8 +62,10 @@ HYPOPG_EXTENSION_STATEMENT = "CREATE TABLE hypopg_test (id integer, val text); SELECT hypopg_create_index('CREATE INDEX ON hypopg_test (id)');" IP4R_EXTENSION_STATEMENT = "CREATE TABLE ip4r_test (ip ip4);" JSONB_PLPERL_EXTENSION_STATEMENT = "CREATE OR REPLACE FUNCTION jsonb_plperl_test(val jsonb) RETURNS jsonb TRANSFORM FOR TYPE jsonb LANGUAGE plperl as $$ return $_[0]; $$;" -ORAFCE_EXTENSION_STATEMENT = "SELECT add_months(date '2005-05-31',1);" -PG_SIMILARITY_EXTENSION_STATEMENT = "SHOW pg_similarity.levenshtein_threshold;" +ORAFCE_EXTENSION_STATEMENT = "SELECT oracle.add_months(date '2005-05-31',1);" +PG_SIMILARITY_EXTENSION_STATEMENT = ( + "SET pg_similarity.levenshtein_threshold = 0.7; SELECT 'aaa', 'aab', lev('aaa','aab');" +) PLPERL_EXTENSION_STATEMENT = "CREATE OR REPLACE FUNCTION plperl_test(name text) RETURNS text AS $$ return $_SHARED{$_[0]}; $$ LANGUAGE plperl;" PREFIX_EXTENSION_STATEMENT = "SELECT '123'::prefix_range @> '123456';" RDKIT_EXTENSION_STATEMENT = "SELECT is_valid_smiles('CCC');" @@ -88,18 +90,17 @@ TIMESCALEDB_EXTENSION_STATEMENT = "CREATE TABLE test_timescaledb (time TIMESTAMPTZ NOT NULL); SELECT create_hypertable('test_timescaledb', 'time');" -@pytest.mark.group(1) @pytest.mark.abort_on_fail -async def test_plugins(ops_test: OpsTest) -> None: +async def test_plugins(ops_test: OpsTest, charm) -> None: """Build and deploy one unit of PostgreSQL and then test the available plugins.""" # Build and deploy the PostgreSQL charm. async with ops_test.fast_forward(): - charm = await ops_test.build_charm(".") await ops_test.model.deploy( charm, num_units=2, base=CHARM_BASE, - config={"profile": "testing"}, + # TODO Figure out how to deal with pgaudit + config={"profile": "testing", "plugin_audit_enable": "False"}, ) await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active", timeout=1500) @@ -210,7 +211,6 @@ def enable_disable_config(enabled: False): connection.close() -@pytest.mark.group(1) async def test_plugin_objects(ops_test: OpsTest) -> None: """Checks if charm gets blocked when trying to disable a plugin in use.""" primary = await get_primary(ops_test, f"{DATABASE_APP_NAME}/0") diff --git a/tests/integration/test_subordinates.py b/tests/integration/test_subordinates.py index be9be926cc..c3caeb2d0f 100644 --- a/tests/integration/test_subordinates.py +++ b/tests/integration/test_subordinates.py @@ -3,6 +3,7 @@ # See LICENSE file for licensing details. import logging +import os from asyncio import gather import pytest @@ -20,9 +21,18 @@ logger = logging.getLogger(__name__) -@pytest.mark.group(1) +@pytest.fixture(scope="module") +async def check_subordinate_env_vars(ops_test: OpsTest) -> None: + if ( + not os.environ.get("UBUNTU_PRO_TOKEN", "").strip() + or not os.environ.get("LANDSCAPE_ACCOUNT_NAME", "").strip() + or not os.environ.get("LANDSCAPE_REGISTRATION_KEY", "").strip() + ): + pytest.skip("Subordinate configs not set") + + @pytest.mark.abort_on_fail -async def test_deploy(ops_test: OpsTest, charm: str, github_secrets): +async def test_deploy(ops_test: OpsTest, charm: str, check_subordinate_env_vars): await gather( ops_test.model.deploy( charm, @@ -32,16 +42,18 @@ async def test_deploy(ops_test: OpsTest, charm: str, github_secrets): ), ops_test.model.deploy( UBUNTU_PRO_APP_NAME, - config={"token": github_secrets["UBUNTU_PRO_TOKEN"]}, + config={"token": os.environ["UBUNTU_PRO_TOKEN"]}, channel="latest/edge", num_units=0, - base=CHARM_BASE, + # TODO switch back to series when pylib juju can figure out the base: + # https://github.com/juju/python-libjuju/issues/1240 + series="noble", ), ops_test.model.deploy( LS_CLIENT, config={ - "account-name": github_secrets["LANDSCAPE_ACCOUNT_NAME"], - "registration-key": github_secrets["LANDSCAPE_REGISTRATION_KEY"], + "account-name": os.environ["LANDSCAPE_ACCOUNT_NAME"], + "registration-key": os.environ["LANDSCAPE_REGISTRATION_KEY"], "ppa": "ppa:landscape/self-hosted-beta", }, channel="latest/edge", @@ -60,8 +72,7 @@ async def test_deploy(ops_test: OpsTest, charm: str, github_secrets): ) -@pytest.mark.group(1) -async def test_scale_up(ops_test: OpsTest, github_secrets): +async def test_scale_up(ops_test: OpsTest, check_subordinate_env_vars): await scale_application(ops_test, DATABASE_APP_NAME, 4) await ops_test.model.wait_for_idle( @@ -69,8 +80,7 @@ async def test_scale_up(ops_test: OpsTest, github_secrets): ) -@pytest.mark.group(1) -async def test_scale_down(ops_test: OpsTest, github_secrets): +async def test_scale_down(ops_test: OpsTest, check_subordinate_env_vars): await scale_application(ops_test, DATABASE_APP_NAME, 3) await ops_test.model.wait_for_idle( diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 7408a8352f..1052df7900 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -26,27 +26,19 @@ primary_changed, run_command_on_unit, ) -from .juju_ import juju_major_version logger = logging.getLogger(__name__) APP_NAME = METADATA["name"] -if juju_major_version < 3: - tls_certificates_app_name = "tls-certificates-operator" - tls_channel = "legacy/edge" if architecture.architecture == "arm64" else "legacy/stable" - tls_config = {"generate-self-signed-certificates": "true", "ca-common-name": "Test CA"} -else: - tls_certificates_app_name = "self-signed-certificates" - tls_channel = "latest/edge" if architecture.architecture == "arm64" else "latest/stable" - tls_config = {"ca-common-name": "Test CA"} +tls_certificates_app_name = "self-signed-certificates" +tls_channel = "latest/edge" if architecture.architecture == "arm64" else "latest/stable" +tls_config = {"ca-common-name": "Test CA"} -@pytest.mark.group(1) @pytest.mark.abort_on_fail @pytest.mark.skip_if_deployed -async def test_deploy_active(ops_test: OpsTest): +async def test_deploy_active(ops_test: OpsTest, charm): """Build the charm and deploy it.""" - charm = await ops_test.build_charm(".") async with ops_test.fast_forward(): await ops_test.model.deploy( charm, @@ -59,7 +51,6 @@ async def test_deploy_active(ops_test: OpsTest): # bundles don't wait between deploying charms. -@pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_tls_enabled(ops_test: OpsTest) -> None: """Test that TLS is enabled when relating to the TLS Certificates Operator.""" @@ -148,7 +139,7 @@ async def test_tls_enabled(ops_test: OpsTest) -> None: await run_command_on_unit( ops_test, primary, - "pkill --signal SIGKILL -f /snap/charmed-postgresql/current/usr/lib/postgresql/14/bin/postgres", + "pkill --signal SIGKILL -f /snap/charmed-postgresql/current/usr/lib/postgresql/16/bin/postgres", ) await run_command_on_unit( ops_test, diff --git a/tests/spread/test_async_replication.py/task.yaml b/tests/spread/test_async_replication.py/task.yaml new file mode 100644 index 0000000000..4fbf3b6b36 --- /dev/null +++ b/tests/spread/test_async_replication.py/task.yaml @@ -0,0 +1,9 @@ +summary: test_async_replication.py +environment: + TEST_MODULE: ha_tests/test_async_replication.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results +variants: + - -juju29 diff --git a/tests/spread/test_audit.py/task.yaml b/tests/spread/test_audit.py/task.yaml new file mode 100644 index 0000000000..9cbc84e43d --- /dev/null +++ b/tests/spread/test_audit.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_audit.py +environment: + TEST_MODULE: test_audit.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_backups_aws.py/task.yaml b/tests/spread/test_backups_aws.py/task.yaml new file mode 100644 index 0000000000..c7eb541232 --- /dev/null +++ b/tests/spread/test_backups_aws.py/task.yaml @@ -0,0 +1,9 @@ +summary: test_backups_aws.py +environment: + TEST_MODULE: test_backups_aws.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results +backends: + - -lxd-vm # Requires CI secrets diff --git a/tests/spread/test_backups_ceph.py/task.yaml b/tests/spread/test_backups_ceph.py/task.yaml new file mode 100644 index 0000000000..8f6c8a387d --- /dev/null +++ b/tests/spread/test_backups_ceph.py/task.yaml @@ -0,0 +1,9 @@ +summary: test_backups_ceph.py +environment: + TEST_MODULE: test_backups_ceph.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results +systems: + - -ubuntu-24.04-arm diff --git a/tests/spread/test_backups_gcp.py/task.yaml b/tests/spread/test_backups_gcp.py/task.yaml new file mode 100644 index 0000000000..c0dc3ac976 --- /dev/null +++ b/tests/spread/test_backups_gcp.py/task.yaml @@ -0,0 +1,9 @@ +summary: test_backups_gcp.py +environment: + TEST_MODULE: test_backups_gcp.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results +backends: + - -lxd-vm # Requires CI secrets diff --git a/tests/spread/test_backups_pitr_aws.py/task.yaml b/tests/spread/test_backups_pitr_aws.py/task.yaml new file mode 100644 index 0000000000..4ac59fbf85 --- /dev/null +++ b/tests/spread/test_backups_pitr_aws.py/task.yaml @@ -0,0 +1,9 @@ +summary: test_backups_pitr_aws.py +environment: + TEST_MODULE: test_backups_pitr_aws.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results +backends: + - -lxd-vm # Requires CI secrets diff --git a/tests/spread/test_backups_pitr_gcp.py/task.yaml b/tests/spread/test_backups_pitr_gcp.py/task.yaml new file mode 100644 index 0000000000..a6b31a59a6 --- /dev/null +++ b/tests/spread/test_backups_pitr_gcp.py/task.yaml @@ -0,0 +1,9 @@ +summary: test_backups_pitr_gcp.py +environment: + TEST_MODULE: test_backups_pitr_gcp.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results +backends: + - -lxd-vm # Requires CI secrets diff --git a/tests/spread/test_charm.py/task.yaml b/tests/spread/test_charm.py/task.yaml new file mode 100644 index 0000000000..96450bdc32 --- /dev/null +++ b/tests/spread/test_charm.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_charm.py +environment: + TEST_MODULE: test_charm.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_config.py/task.yaml b/tests/spread/test_config.py/task.yaml new file mode 100644 index 0000000000..f330f89b38 --- /dev/null +++ b/tests/spread/test_config.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_config.py +environment: + TEST_MODULE: test_config.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_new_relations_1.py/task.yaml b/tests/spread/test_new_relations_1.py/task.yaml new file mode 100644 index 0000000000..0c64fe771f --- /dev/null +++ b/tests/spread/test_new_relations_1.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_new_relations_1.py +environment: + TEST_MODULE: new_relations/test_new_relations_1.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_new_relations_2.py/task.yaml b/tests/spread/test_new_relations_2.py/task.yaml new file mode 100644 index 0000000000..0b7af326a4 --- /dev/null +++ b/tests/spread/test_new_relations_2.py/task.yaml @@ -0,0 +1,9 @@ +summary: test_new_relations_2.py +environment: + TEST_MODULE: new_relations/test_new_relations_2.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results +systems: + - -ubuntu-24.04-arm diff --git a/tests/spread/test_password_rotation.py/task.yaml b/tests/spread/test_password_rotation.py/task.yaml new file mode 100644 index 0000000000..439559b4e6 --- /dev/null +++ b/tests/spread/test_password_rotation.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_password_rotation.py +environment: + TEST_MODULE: test_password_rotation.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_plugins.py/task.yaml b/tests/spread/test_plugins.py/task.yaml new file mode 100644 index 0000000000..e9dce8e28f --- /dev/null +++ b/tests/spread/test_plugins.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_plugins.py +environment: + TEST_MODULE: test_plugins.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_relations_coherence.py/task.yaml b/tests/spread/test_relations_coherence.py/task.yaml new file mode 100644 index 0000000000..bff0e492b3 --- /dev/null +++ b/tests/spread/test_relations_coherence.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_relations_coherence.py +environment: + TEST_MODULE: new_relations/test_relations_coherence.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_replication.py/task.yaml b/tests/spread/test_replication.py/task.yaml new file mode 100644 index 0000000000..237cc3981b --- /dev/null +++ b/tests/spread/test_replication.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_replication.py +environment: + TEST_MODULE: ha_tests/test_replication.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_restore_cluster.py/task.yaml b/tests/spread/test_restore_cluster.py/task.yaml new file mode 100644 index 0000000000..bce2ec14d4 --- /dev/null +++ b/tests/spread/test_restore_cluster.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_restore_cluster.py +environment: + TEST_MODULE: ha_tests/test_restore_cluster.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_scaling.py/task.yaml b/tests/spread/test_scaling.py/task.yaml new file mode 100644 index 0000000000..32358243db --- /dev/null +++ b/tests/spread/test_scaling.py/task.yaml @@ -0,0 +1,9 @@ +summary: test_scaling.py +environment: + TEST_MODULE: ha_tests/test_scaling.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results +variants: + - -juju29 diff --git a/tests/spread/test_scaling_three_units.py/task.yaml b/tests/spread/test_scaling_three_units.py/task.yaml new file mode 100644 index 0000000000..ae8dcc1006 --- /dev/null +++ b/tests/spread/test_scaling_three_units.py/task.yaml @@ -0,0 +1,9 @@ +summary: test_scaling_three_units.py +environment: + TEST_MODULE: ha_tests/test_scaling_three_units.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results +variants: + - -juju29 diff --git a/tests/spread/test_scaling_three_units_async.py/task.yaml b/tests/spread/test_scaling_three_units_async.py/task.yaml new file mode 100644 index 0000000000..cd8a7ba5aa --- /dev/null +++ b/tests/spread/test_scaling_three_units_async.py/task.yaml @@ -0,0 +1,9 @@ +summary: test_scaling_three_units.py +environment: + TEST_MODULE: ha_tests/test_scaling_three_units_async.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results +variants: + - -juju29 diff --git a/tests/spread/test_self_healing.py/task.yaml b/tests/spread/test_self_healing.py/task.yaml new file mode 100644 index 0000000000..d8fca3acea --- /dev/null +++ b/tests/spread/test_self_healing.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_self_healing.py +environment: + TEST_MODULE: ha_tests/test_self_healing.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_smoke.py/task.yaml b/tests/spread/test_smoke.py/task.yaml new file mode 100644 index 0000000000..d2fe9793d1 --- /dev/null +++ b/tests/spread/test_smoke.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_smoke.py +environment: + TEST_MODULE: ha_tests/test_smoke.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_subordinates.py/task.yaml b/tests/spread/test_subordinates.py/task.yaml new file mode 100644 index 0000000000..a7477d7bab --- /dev/null +++ b/tests/spread/test_subordinates.py/task.yaml @@ -0,0 +1,9 @@ +summary: test_subordinates.py +environment: + TEST_MODULE: test_subordinates.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results +backends: + - -lxd-vm # Requires CI secrets diff --git a/tests/spread/test_synchronous_policy.py/task.yaml b/tests/spread/test_synchronous_policy.py/task.yaml new file mode 100644 index 0000000000..e4ce19458f --- /dev/null +++ b/tests/spread/test_synchronous_policy.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_scaling.py +environment: + TEST_MODULE: ha_tests/test_synchronous_policy.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_tls.py/task.yaml b/tests/spread/test_tls.py/task.yaml new file mode 100644 index 0000000000..a605744913 --- /dev/null +++ b/tests/spread/test_tls.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_tls.py +environment: + TEST_MODULE: test_tls.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_upgrade.py/task.yaml b/tests/spread/test_upgrade.py/task.yaml new file mode 100644 index 0000000000..b3be366921 --- /dev/null +++ b/tests/spread/test_upgrade.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_upgrade.py +environment: + TEST_MODULE: ha_tests/test_upgrade.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/spread/test_upgrade_from_stable.py/task.yaml b/tests/spread/test_upgrade_from_stable.py/task.yaml new file mode 100644 index 0000000000..047617ab39 --- /dev/null +++ b/tests/spread/test_upgrade_from_stable.py/task.yaml @@ -0,0 +1,7 @@ +summary: test_upgrade_from_stable.py +environment: + TEST_MODULE: ha_tests/test_upgrade_from_stable.py +execute: | + tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" +artifacts: + - allure-results diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 03c68492e9..479919633d 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -9,27 +9,9 @@ # This causes every test defined in this file to run 2 times, each with # charm.JujuVersion.has_secrets set as True or as False -@pytest.fixture(params=[True, False], autouse=True) +@pytest.fixture(autouse=True) def _has_secrets(request, monkeypatch): - monkeypatch.setattr("charm.JujuVersion.has_secrets", PropertyMock(return_value=request.param)) - return request.param - - -@pytest.fixture -def only_with_juju_secrets(_has_secrets): - """Pretty way to skip Juju 3 tests.""" - if not _has_secrets: - pytest.skip("Secrets test only applies on Juju 3.x") - - -@pytest.fixture -def only_without_juju_secrets(_has_secrets): - """Pretty way to skip Juju 2-specific tests. - - Typically: to save CI time, when the same check were executed in a Juju 3-specific way already - """ - if _has_secrets: - pytest.skip("Skipping legacy secrets tests") + monkeypatch.setattr("charm.JujuVersion.has_secrets", PropertyMock(return_value=True)) @pytest.fixture(autouse=True) diff --git a/tests/unit/test_backups.py b/tests/unit/test_backups.py index f7931348a9..27320474a3 100644 --- a/tests/unit/test_backups.py +++ b/tests/unit/test_backups.py @@ -202,7 +202,7 @@ def test_can_use_s3_repository(harness): patch("charm.Patroni.member_started", new_callable=PropertyMock) as _member_started, patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, patch( - "charm.Patroni.get_postgresql_version", return_value="14.10" + "charm.Patroni.get_postgresql_version", return_value="16.6" ) as _get_postgresql_version, patch("charm.PostgresqlOperatorCharm.postgresql") as _postgresql, patch( @@ -297,6 +297,19 @@ def test_can_use_s3_repository(harness): ] assert harness.charm.backup.can_use_s3_repository() == (True, None) + # Empty db + _execute_command.side_effect = None + _execute_command.return_value = (1, "", "") + pgbackrest_info_other_cluster_name_backup_output = ( + 0, + f'[{{"db": [], "name": "another-model.{harness.charm.cluster_name}"}}]', + "", + ) + assert harness.charm.backup.can_use_s3_repository() == ( + False, + FAILED_TO_INITIALIZE_STANZA_ERROR_MESSAGE, + ) + def test_construct_endpoint(harness): # Test with an AWS endpoint without region. diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 77b62a08dc..2e7f1cfcda 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -35,7 +35,12 @@ PRIMARY_NOT_REACHABLE_MESSAGE, PostgresqlOperatorCharm, ) -from cluster import NotReadyError, RemoveRaftMemberFailedError, SwitchoverFailedError +from cluster import ( + NotReadyError, + RemoveRaftMemberFailedError, + SwitchoverFailedError, + SwitchoverNotSyncError, +) from constants import PEER, POSTGRESQL_SNAP_NAME, SECRET_INTERNAL_LABEL, SNAP_PACKAGES CREATE_CLUSTER_CONF_PATH = "/etc/postgresql-common/createcluster.d/pgcharm.conf" @@ -271,7 +276,9 @@ def test_on_config_changed(harness): "charm.PostgresqlOperatorCharm._validate_config_options" ) as _validate_config_options, patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, - patch("relations.db.DbProvides.set_up_relation") as _set_up_relation, + patch( + "charm.PostgresqlOperatorCharm.updated_synchronous_node_count", return_value=True + ) as _updated_synchronous_node_count, patch( "charm.PostgresqlOperatorCharm.enable_disable_extensions" ) as _enable_disable_extensions, @@ -283,14 +290,12 @@ def test_on_config_changed(harness): _is_cluster_initialised.return_value = False harness.charm.on.config_changed.emit() _enable_disable_extensions.assert_not_called() - _set_up_relation.assert_not_called() # Test when the unit is not the leader. _is_cluster_initialised.return_value = True harness.charm.on.config_changed.emit() _validate_config_options.assert_called_once() _enable_disable_extensions.assert_not_called() - _set_up_relation.assert_not_called() # Test unable to connect to db _update_config.reset_mock() @@ -298,13 +303,13 @@ def test_on_config_changed(harness): harness.charm.on.config_changed.emit() assert not _update_config.called _validate_config_options.side_effect = None + _updated_synchronous_node_count.assert_called_once_with() # Test after the cluster was initialised. with harness.hooks_disabled(): harness.set_leader() harness.charm.on.config_changed.emit() _enable_disable_extensions.assert_called_once() - _set_up_relation.assert_not_called() # Test when the unit is in a blocked state due to extensions request, # but there are no established legacy relations. @@ -314,34 +319,6 @@ def test_on_config_changed(harness): ) harness.charm.on.config_changed.emit() _enable_disable_extensions.assert_called_once() - _set_up_relation.assert_not_called() - - # Test when the unit is in a blocked state due to extensions request, - # but there are established legacy relations. - _enable_disable_extensions.reset_mock() - _set_up_relation.return_value = False - db_relation_id = harness.add_relation("db", "application") - harness.charm.on.config_changed.emit() - _enable_disable_extensions.assert_called_once() - _set_up_relation.assert_called_once() - harness.remove_relation(db_relation_id) - - _enable_disable_extensions.reset_mock() - _set_up_relation.reset_mock() - harness.add_relation("db-admin", "application") - harness.charm.on.config_changed.emit() - _enable_disable_extensions.assert_called_once() - _set_up_relation.assert_called_once() - - # Test when there are established legacy relations, - # but the charm fails to set up one of them. - _enable_disable_extensions.reset_mock() - _set_up_relation.reset_mock() - _set_up_relation.return_value = False - harness.add_relation("db", "application") - harness.charm.on.config_changed.emit() - _enable_disable_extensions.assert_called_once() - _set_up_relation.assert_called_once() def test_check_extension_dependencies(harness): @@ -409,6 +386,9 @@ def test_enable_disable_extensions(harness, caplog): postgresql_mock.enable_disable_extensions.side_effect = None with caplog.at_level(logging.ERROR): config = """options: + synchronous_node_count: + type: string + default: "all" plugin_citext_enable: default: false type: boolean @@ -623,7 +603,7 @@ def test_on_start(harness): side_effect=[False, True, True, True, True, True], ) as _is_storage_attached, ): - _get_postgresql_version.return_value = "14.0" + _get_postgresql_version.return_value = "16.6" # Test without storage. harness.charm.on.start.emit() @@ -709,7 +689,7 @@ def test_on_start_replica(harness): return_value=True, ) as _is_storage_attached, ): - _get_postgresql_version.return_value = "14.0" + _get_postgresql_version.return_value = "16.6" # Set the current unit to be a replica (non leader unit). harness.set_leader(False) @@ -768,7 +748,7 @@ def test_on_start_no_patroni_member(harness): bootstrap_cluster = patroni.return_value.bootstrap_cluster bootstrap_cluster.return_value = True - patroni.return_value.get_postgresql_version.return_value = "14.0" + patroni.return_value.get_postgresql_version.return_value = "16.6" harness.set_leader() harness.charm.on.start.emit() @@ -1119,29 +1099,29 @@ def test_install_snap_packages(harness): # Test for problem with snap update. with pytest.raises(snap.SnapError): - harness.charm._install_snap_packages([("postgresql", {"channel": "14/edge"})]) + harness.charm._install_snap_packages([("postgresql", {"channel": "16/edge"})]) _snap_cache.return_value.__getitem__.assert_called_once_with("postgresql") _snap_cache.assert_called_once_with() - _snap_package.ensure.assert_called_once_with(snap.SnapState.Latest, channel="14/edge") + _snap_package.ensure.assert_called_once_with(snap.SnapState.Latest, channel="16/edge") # Test with a not found package. _snap_cache.reset_mock() _snap_package.reset_mock() _snap_package.ensure.side_effect = snap.SnapNotFoundError with pytest.raises(snap.SnapNotFoundError): - harness.charm._install_snap_packages([("postgresql", {"channel": "14/edge"})]) + harness.charm._install_snap_packages([("postgresql", {"channel": "16/edge"})]) _snap_cache.return_value.__getitem__.assert_called_once_with("postgresql") _snap_cache.assert_called_once_with() - _snap_package.ensure.assert_called_once_with(snap.SnapState.Latest, channel="14/edge") + _snap_package.ensure.assert_called_once_with(snap.SnapState.Latest, channel="16/edge") # Then test a valid one. _snap_cache.reset_mock() _snap_package.reset_mock() _snap_package.ensure.side_effect = None - harness.charm._install_snap_packages([("postgresql", {"channel": "14/edge"})]) + harness.charm._install_snap_packages([("postgresql", {"channel": "16/edge"})]) _snap_cache.assert_called_once_with() _snap_cache.return_value.__getitem__.assert_called_once_with("postgresql") - _snap_package.ensure.assert_called_once_with(snap.SnapState.Latest, channel="14/edge") + _snap_package.ensure.assert_called_once_with(snap.SnapState.Latest, channel="16/edge") _snap_package.hold.assert_not_called() # Test revision @@ -1340,6 +1320,7 @@ def test_update_config(harness): harness.update_relation_data( rel_id, harness.charm.unit.name, {"tls": ""} ) # Mock some data in the relation to test that it doesn't change. + _is_tls_enabled.return_value = False harness.charm.update_config() _handle_postgresql_restart_need.assert_not_called() assert "tls" not in harness.get_relation_data(rel_id, harness.charm.unit.name) @@ -1813,35 +1794,8 @@ def test_client_relations(harness): # Test when the charm has some relations. harness.add_relation("database", "application") - harness.add_relation("db", "legacy-application") - harness.add_relation("db-admin", "legacy-admin-application") database_relation = harness.model.get_relation("database") - db_relation = harness.model.get_relation("db") - db_admin_relation = harness.model.get_relation("db-admin") - assert harness.charm.client_relations == [database_relation, db_relation, db_admin_relation] - - -def test_on_pgdata_storage_detaching(harness): - with ( - patch( - "charm.PostgresqlOperatorCharm._update_relation_endpoints" - ) as _update_relation_endpoints, - patch("charm.PostgresqlOperatorCharm.primary_endpoint", new_callable=PropertyMock), - patch("charm.Patroni.are_all_members_ready") as _are_all_members_ready, - patch("charm.Patroni.get_primary", return_value="primary") as _get_primary, - patch("charm.Patroni.switchover") as _switchover, - patch("charm.Patroni.primary_changed") as _primary_changed, - ): - # Early exit if not primary - event = Mock() - harness.charm._on_pgdata_storage_detaching(event) - assert not _are_all_members_ready.called - - _get_primary.side_effect = [harness.charm.unit.name, "primary"] - harness.charm._on_pgdata_storage_detaching(event) - _switchover.assert_called_once_with() - _primary_changed.assert_called_once_with("primary") - _update_relation_endpoints.assert_called_once_with() + assert harness.charm.client_relations == [database_relation] def test_add_cluster_member(harness): @@ -2055,32 +2009,6 @@ def test_scope_obj(harness): assert harness.charm._scope_obj("test") is None -def test_get_secret_from_databag(harness): - """Asserts that get_secret method can read secrets from databag. - - This must be backwards-compatible so it runs on both juju2 and juju3. - """ - with patch("charm.PostgresqlOperatorCharm._on_leader_elected"): - rel_id = harness.model.get_relation(PEER).id - # App level changes require leader privileges - harness.set_leader() - # Test application scope. - assert harness.charm.get_secret("app", "operator_password") is None - harness.update_relation_data( - rel_id, harness.charm.app.name, {"operator_password": "test-password"} - ) - assert harness.charm.get_secret("app", "operator_password") == "test-password" - - # Unit level changes don't require leader privileges - harness.set_leader(False) - # Test unit scope. - assert harness.charm.get_secret("unit", "operator_password") is None - harness.update_relation_data( - rel_id, harness.charm.unit.name, {"operator_password": "test-password"} - ) - assert harness.charm.get_secret("unit", "operator_password") == "test-password" - - def test_on_get_password_secrets(harness): with ( patch("charm.PostgresqlOperatorCharm._on_leader_elected"), @@ -2122,39 +2050,6 @@ def test_get_secret_secrets(harness, scope, field): assert harness.charm.get_secret(scope, field) == "test" -def test_set_secret_in_databag(harness, only_without_juju_secrets): - """Asserts that set_secret method writes to relation databag. - - This is juju2 specific. In juju3, set_secret writes to juju secrets. - """ - with patch("charm.PostgresqlOperatorCharm._on_leader_elected"): - rel_id = harness.model.get_relation(PEER).id - harness.set_leader() - - # Test application scope. - assert "password" not in harness.get_relation_data(rel_id, harness.charm.app.name) - harness.charm.set_secret("app", "password", "test-password") - assert ( - harness.get_relation_data(rel_id, harness.charm.app.name)["password"] - == "test-password" - ) - harness.charm.set_secret("app", "password", None) - assert "password" not in harness.get_relation_data(rel_id, harness.charm.app.name) - - # Test unit scope. - assert "password" not in harness.get_relation_data(rel_id, harness.charm.unit.name) - harness.charm.set_secret("unit", "password", "test-password") - assert ( - harness.get_relation_data(rel_id, harness.charm.unit.name)["password"] - == "test-password" - ) - harness.charm.set_secret("unit", "password", None) - assert "password" not in harness.get_relation_data(rel_id, harness.charm.unit.name) - - with pytest.raises(RuntimeError): - harness.charm.set_secret("test", "password", "test") - - @pytest.mark.parametrize("scope,is_leader", [("app", True), ("unit", True), ("unit", False)]) def test_set_reset_new_secret(harness, scope, is_leader): with ( @@ -2191,7 +2086,7 @@ def test_invalid_secret(harness, scope, is_leader): assert harness.charm.get_secret(scope, "somekey") is None -def test_delete_password(harness, _has_secrets, caplog): +def test_delete_password(harness, caplog): with ( patch("charm.PostgresqlOperatorCharm._on_leader_elected"), ): @@ -2208,14 +2103,7 @@ def test_delete_password(harness, _has_secrets, caplog): harness.set_leader(True) with caplog.at_level(logging.DEBUG): - if _has_secrets: - error_message = ( - "Non-existing secret operator-password was attempted to be removed." - ) - else: - error_message = ( - "Non-existing field 'operator-password' was attempted to be removed" - ) + error_message = "Non-existing secret operator-password was attempted to be removed." harness.charm.remove_secret("app", "operator-password") assert error_message in caplog.text @@ -2237,34 +2125,7 @@ def test_delete_password(harness, _has_secrets, caplog): @pytest.mark.parametrize("scope,is_leader", [("app", True), ("unit", True), ("unit", False)]) -def test_migration_from_databag(harness, only_with_juju_secrets, scope, is_leader): - """Check if we're moving on to use secrets when live upgrade from databag to Secrets usage. - - Since it checks for a migration from databag to juju secrets, it's specific to juju3. - """ - with ( - patch("charm.PostgresqlOperatorCharm._on_leader_elected"), - ): - rel_id = harness.model.get_relation(PEER).id - # App has to be leader, unit can be either - harness.set_leader(is_leader) - - # Getting current password - entity = getattr(harness.charm, scope) - harness.update_relation_data(rel_id, entity.name, {"operator_password": "bla"}) - assert harness.charm.get_secret(scope, "operator_password") == "bla" - - # Reset new secret - harness.charm.set_secret(scope, "operator-password", "blablabla") - assert harness.charm.model.get_secret(label=f"{PEER}.postgresql.{scope}") - assert harness.charm.get_secret(scope, "operator-password") == "blablabla" - assert "operator-password" not in harness.get_relation_data( - rel_id, getattr(harness.charm, scope).name - ) - - -@pytest.mark.parametrize("scope,is_leader", [("app", True), ("unit", True), ("unit", False)]) -def test_migration_from_single_secret(harness, only_with_juju_secrets, scope, is_leader): +def test_migration_from_single_secret(harness, scope, is_leader): """Check if we're moving on to use secrets when live upgrade from databag to Secrets usage. Since it checks for a migration from databag to juju secrets, it's specific to juju3. @@ -2366,7 +2227,7 @@ def test_on_peer_relation_departed(harness): patch("charm.Patroni.are_all_members_ready") as _are_all_members_ready, patch("charm.PostgresqlOperatorCharm._get_ips_to_remove") as _get_ips_to_remove, patch( - "charm.PostgresqlOperatorCharm._updated_synchronous_node_count" + "charm.PostgresqlOperatorCharm.updated_synchronous_node_count" ) as _updated_synchronous_node_count, patch("charm.Patroni.remove_raft_member") as _remove_raft_member, patch("charm.PostgresqlOperatorCharm._unit_ip") as _unit_ip, @@ -2445,7 +2306,7 @@ def test_on_peer_relation_departed(harness): harness.charm._on_peer_relation_departed(event) _remove_raft_member.assert_called_once_with(mock_ip_address) event.defer.assert_called_once() - _updated_synchronous_node_count.assert_called_once_with(1) + _updated_synchronous_node_count.assert_called_once_with() _get_ips_to_remove.assert_not_called() _remove_from_members_ips.assert_not_called() _update_config.assert_not_called() @@ -2460,7 +2321,7 @@ def test_on_peer_relation_departed(harness): harness.charm._on_peer_relation_departed(event) _remove_raft_member.assert_called_once_with(mock_ip_address) event.defer.assert_called_once() - _updated_synchronous_node_count.assert_called_once_with(2) + _updated_synchronous_node_count.assert_called_once_with() _get_ips_to_remove.assert_not_called() _remove_from_members_ips.assert_not_called() _update_config.assert_not_called() @@ -2476,7 +2337,7 @@ def test_on_peer_relation_departed(harness): harness.charm._on_peer_relation_departed(event) _remove_raft_member.assert_called_once_with(mock_ip_address) event.defer.assert_not_called() - _updated_synchronous_node_count.assert_called_once_with(2) + _updated_synchronous_node_count.assert_called_once_with() _get_ips_to_remove.assert_called_once() _remove_from_members_ips.assert_not_called() _update_config.assert_not_called() @@ -2494,7 +2355,7 @@ def test_on_peer_relation_departed(harness): harness.charm._on_peer_relation_departed(event) _remove_raft_member.assert_called_once_with(mock_ip_address) event.defer.assert_called_once() - _updated_synchronous_node_count.assert_called_once_with(2) + _updated_synchronous_node_count.assert_called_once_with() _get_ips_to_remove.assert_called_once() _remove_from_members_ips.assert_not_called() _update_config.assert_not_called() @@ -2510,7 +2371,7 @@ def test_on_peer_relation_departed(harness): harness.charm._on_peer_relation_departed(event) _remove_raft_member.assert_called_once_with(mock_ip_address) event.defer.assert_not_called() - _updated_synchronous_node_count.assert_called_once_with(2) + _updated_synchronous_node_count.assert_called_once_with() _get_ips_to_remove.assert_called_once() _remove_from_members_ips.assert_has_calls([call(ips_to_remove[0]), call(ips_to_remove[1])]) assert _update_config.call_count == 2 @@ -2529,7 +2390,7 @@ def test_on_peer_relation_departed(harness): harness.charm._on_peer_relation_departed(event) _remove_raft_member.assert_called_once_with(mock_ip_address) event.defer.assert_not_called() - _updated_synchronous_node_count.assert_called_once_with(2) + _updated_synchronous_node_count.assert_called_once_with() _get_ips_to_remove.assert_called_once() _remove_from_members_ips.assert_called_once() _update_config.assert_called_once() @@ -2787,6 +2648,17 @@ def test_on_promote_to_primary(harness): harness.charm._on_promote_to_primary(event) + event.fail.assert_called_once_with( + "Switchover failed or timed out, check the logs for details" + ) + event.fail.reset_mock() + + # Unit, no force, not sync + event.params = {"scope": "unit"} + _switchover.side_effect = SwitchoverNotSyncError + + harness.charm._on_promote_to_primary(event) + event.fail.assert_called_once_with("Unit is not sync standby") event.fail.reset_mock() diff --git a/tests/unit/test_cluster.py b/tests/unit/test_cluster.py index 07f01eed47..92ab7dc212 100644 --- a/tests/unit/test_cluster.py +++ b/tests/unit/test_cluster.py @@ -20,7 +20,13 @@ ) from charm import PostgresqlOperatorCharm -from cluster import PATRONI_TIMEOUT, Patroni, RemoveRaftMemberFailedError +from cluster import ( + PATRONI_TIMEOUT, + Patroni, + RemoveRaftMemberFailedError, + SwitchoverFailedError, + SwitchoverNotSyncError, +) from constants import ( PATRONI_CONF_PATH, PATRONI_LOGS_PATH, @@ -166,11 +172,11 @@ def test_get_postgresql_version(peers_ips, patroni): _get_installed_snaps = _snap_client.return_value.get_installed_snaps _get_installed_snaps.return_value = [ {"name": "something"}, - {"name": "charmed-postgresql", "version": "14.0"}, + {"name": "charmed-postgresql", "version": "16.6"}, ] version = patroni.get_postgresql_version() - assert version == "14.0" + assert version == "16.6" _snap_client.assert_called_once_with() _get_installed_snaps.assert_called_once_with() @@ -315,7 +321,7 @@ def test_render_patroni_yml_file(peers_ips, patroni): patch("charm.Patroni.render_file") as _render_file, patch("charm.Patroni._create_directory"), ): - _get_postgresql_version.return_value = "14.7" + _get_postgresql_version.return_value = "16.6" # Define variables to render in the template. member_name = "postgresql-0" @@ -325,7 +331,7 @@ def test_render_patroni_yml_file(peers_ips, patroni): rewind_password = "fake-rewind-password" raft_password = "fake-raft-password" patroni_password = "fake-patroni-password" - postgresql_version = "14" + postgresql_version = "16" # Get the expected content from a file. with open("templates/patroni.yml.j2") as file: @@ -346,7 +352,7 @@ def test_render_patroni_yml_file(peers_ips, patroni): rewind_user=REWIND_USER, rewind_password=rewind_password, version=postgresql_version, - minority_count=patroni.planned_units // 2, + synchronous_node_count=0, raft_password=raft_password, patroni_password=patroni_password, ) @@ -473,6 +479,22 @@ def test_switchover(peers_ips, patroni): timeout=PATRONI_TIMEOUT, ) + # Test candidate, not sync + response = _post.return_value + response.status_code = 412 + response.text = "candidate name does not match with sync_standby" + with pytest.raises(SwitchoverNotSyncError): + patroni.switchover("candidate") + assert False + + # Test general error + response = _post.return_value + response.status_code = 412 + response.text = "something else " + with pytest.raises(SwitchoverFailedError): + patroni.switchover() + assert False + def test_update_synchronous_node_count(peers_ips, patroni): with ( diff --git a/tests/unit/test_db.py b/tests/unit/test_db.py deleted file mode 100644 index d5ebd95e97..0000000000 --- a/tests/unit/test_db.py +++ /dev/null @@ -1,638 +0,0 @@ -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. - -from unittest.mock import Mock, PropertyMock, patch - -import pytest -from charms.postgresql_k8s.v0.postgresql import ( - PostgreSQLCreateDatabaseError, - PostgreSQLCreateUserError, - PostgreSQLGetPostgreSQLVersionError, -) -from ops.framework import EventBase -from ops.model import ActiveStatus, BlockedStatus -from ops.testing import Harness - -from charm import PostgresqlOperatorCharm -from constants import DATABASE_PORT, PEER - -DATABASE = "test_database" -RELATION_NAME = "db" -POSTGRESQL_VERSION = "12" - - -@pytest.fixture(autouse=True) -def harness(): - harness = Harness(PostgresqlOperatorCharm) - - # Set up the initial relation and hooks. - harness.set_leader(True) - harness.begin() - - # Define some relations. - rel_id = harness.add_relation(RELATION_NAME, "application") - harness.add_relation_unit(rel_id, "application/0") - peer_rel_id = harness.add_relation(PEER, harness.charm.app.name) - harness.add_relation_unit(peer_rel_id, f"{harness.charm.app.name}/1") - harness.add_relation_unit(peer_rel_id, harness.charm.unit.name) - harness.update_relation_data( - peer_rel_id, - harness.charm.app.name, - {"cluster_initialised": "True"}, - ) - yield harness - harness.cleanup() - - -def request_database(_harness): - # Reset the charm status. - _harness.model.unit.status = ActiveStatus() - rel_id = _harness.model.get_relation(RELATION_NAME).id - - with _harness.hooks_disabled(): - # Reset the application databag. - _harness.update_relation_data( - rel_id, - "application/0", - {"database": ""}, - ) - - # Reset the database databag. - _harness.update_relation_data( - rel_id, - _harness.charm.app.name, - { - "allowed-subnets": "", - "allowed-units": "", - "port": "", - "version": "", - "user": "", - "password": "", - "database": "", - }, - ) - - # Simulate the request of a new database. - _harness.update_relation_data( - rel_id, - "application/0", - {"database": DATABASE}, - ) - - -def test_on_relation_changed(harness): - with ( - patch("charm.DbProvides.set_up_relation") as _set_up_relation, - patch.object(EventBase, "defer") as _defer, - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - ) as _primary_endpoint, - patch("charm.Patroni.member_started", new_callable=PropertyMock) as _member_started, - ): - # Set some side effects to test multiple situations. - _member_started.side_effect = [False, True, True, True, True, True] - _primary_endpoint.side_effect = [ - None, - {"1.1.1.1"}, - {"1.1.1.1"}, - {"1.1.1.1"}, - {"1.1.1.1"}, - ] - # Request a database to a non leader unit. - with harness.hooks_disabled(): - harness.set_leader(False) - request_database(harness) - _defer.assert_not_called() - _set_up_relation.assert_not_called() - - # Request a database before the database is ready. - with harness.hooks_disabled(): - harness.set_leader() - request_database(harness) - _defer.assert_called_once() - _set_up_relation.assert_not_called() - - # Request a database before primary endpoint is available. - request_database(harness) - assert _defer.call_count == 2 - _set_up_relation.assert_not_called() - - # Request it again when the database is ready. - _defer.reset_mock() - request_database(harness) - _defer.assert_not_called() - _set_up_relation.assert_called_once() - - -def test_get_extensions(harness): - # Test when there are no extensions in the relation databags. - rel_id = harness.model.get_relation(RELATION_NAME).id - relation = harness.model.get_relation(RELATION_NAME, rel_id) - assert harness.charm.legacy_db_relation._get_extensions(relation) == ([], set()) - - # Test when there are extensions in the application relation databag. - extensions = ["", "citext:public", "debversion"] - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - "application", - {"extensions": ",".join(extensions)}, - ) - assert harness.charm.legacy_db_relation._get_extensions(relation) == ( - [extensions[1], extensions[2]], - {extensions[1].split(":")[0], extensions[2]}, - ) - - # Test when there are extensions in the unit relation databag. - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - "application", - {"extensions": ""}, - ) - harness.update_relation_data( - rel_id, - "application/0", - {"extensions": ",".join(extensions)}, - ) - assert harness.charm.legacy_db_relation._get_extensions(relation) == ( - [extensions[1], extensions[2]], - {extensions[1].split(":")[0], extensions[2]}, - ) - - # Test when one of the plugins/extensions is enabled. - config = """options: - plugin_citext_enable: - default: true - type: boolean - plugin_debversion_enable: - default: false - type: boolean""" - harness = Harness(PostgresqlOperatorCharm, config=config) - harness.cleanup() - harness.begin() - assert harness.charm.legacy_db_relation._get_extensions(relation) == ( - [extensions[1], extensions[2]], - {extensions[2]}, - ) - - -def test_set_up_relation(harness): - with ( - patch.object(PostgresqlOperatorCharm, "postgresql", Mock()) as postgresql_mock, - patch("subprocess.check_output", return_value=b"C"), - patch("relations.db.DbProvides._update_unit_status") as _update_unit_status, - patch("relations.db.DbProvides.update_endpoints") as _update_endpoints, - patch("relations.db.new_password", return_value="test-password") as _new_password, - patch("relations.db.DbProvides._get_extensions") as _get_extensions, - ): - rel_id = harness.model.get_relation(RELATION_NAME).id - # Define some mocks' side effects. - extensions = ["citext:public", "debversion"] - _get_extensions.side_effect = [ - (extensions, {"debversion"}), - (extensions, set()), - (extensions, set()), - (extensions, set()), - (extensions, set()), - (extensions, set()), - (extensions, set()), - ] - postgresql_mock.create_user = PropertyMock( - side_effect=[None, None, None, PostgreSQLCreateUserError, None, None] - ) - postgresql_mock.create_database = PropertyMock( - side_effect=[None, None, None, PostgreSQLCreateDatabaseError, None] - ) - - # Assert no operation is done when at least one of the requested extensions - # is disabled. - relation = harness.model.get_relation(RELATION_NAME, rel_id) - assert not harness.charm.legacy_db_relation.set_up_relation(relation) - postgresql_mock.create_user.assert_not_called() - postgresql_mock.create_database.assert_not_called() - postgresql_mock.get_postgresql_version.assert_not_called() - _update_endpoints.assert_not_called() - _update_unit_status.assert_not_called() - - # Assert that the correct calls were made in a successful setup. - harness.charm.unit.status = ActiveStatus() - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - "application", - {"database": DATABASE}, - ) - assert harness.charm.legacy_db_relation.set_up_relation(relation) - user = f"relation-{rel_id}" - postgresql_mock.create_user.assert_called_once_with(user, "test-password", False) - postgresql_mock.create_database.assert_called_once_with( - DATABASE, user, plugins=["pgaudit"], client_relations=[relation] - ) - _update_endpoints.assert_called_once() - _update_unit_status.assert_called_once() - assert not isinstance(harness.model.unit.status, BlockedStatus) - - # Assert that the correct calls were made when the database name is not - # provided in both application and unit databags. - postgresql_mock.create_user.reset_mock() - postgresql_mock.create_database.reset_mock() - postgresql_mock.get_postgresql_version.reset_mock() - _update_endpoints.reset_mock() - _update_unit_status.reset_mock() - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - "application", - {"database": ""}, - ) - harness.update_relation_data( - rel_id, - "application/0", - {"database": DATABASE}, - ) - assert harness.charm.legacy_db_relation.set_up_relation(relation) - postgresql_mock.create_user.assert_called_once_with(user, "test-password", False) - postgresql_mock.create_database.assert_called_once_with( - DATABASE, user, plugins=["pgaudit"], client_relations=[relation] - ) - _update_endpoints.assert_called_once() - _update_unit_status.assert_called_once() - assert not isinstance(harness.model.unit.status, BlockedStatus) - - # Assert that the correct calls were made when the database name is not provided. - postgresql_mock.create_user.reset_mock() - postgresql_mock.create_database.reset_mock() - postgresql_mock.get_postgresql_version.reset_mock() - _update_endpoints.reset_mock() - _update_unit_status.reset_mock() - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - "application/0", - {"database": ""}, - ) - assert harness.charm.legacy_db_relation.set_up_relation(relation) - postgresql_mock.create_user.assert_called_once_with(user, "test-password", False) - postgresql_mock.create_database.assert_called_once_with( - "test_database", user, plugins=["pgaudit"], client_relations=[relation] - ) - _update_endpoints.assert_called_once() - _update_unit_status.assert_called_once() - assert not isinstance(harness.model.unit.status, BlockedStatus) - - # BlockedStatus due to a PostgreSQLCreateUserError. - postgresql_mock.create_database.reset_mock() - postgresql_mock.get_postgresql_version.reset_mock() - _update_endpoints.reset_mock() - _update_unit_status.reset_mock() - assert not harness.charm.legacy_db_relation.set_up_relation(relation) - postgresql_mock.create_database.assert_not_called() - _update_endpoints.assert_not_called() - _update_unit_status.assert_not_called() - assert isinstance(harness.model.unit.status, BlockedStatus) - - # BlockedStatus due to a PostgreSQLCreateDatabaseError. - harness.charm.unit.status = ActiveStatus() - assert not harness.charm.legacy_db_relation.set_up_relation(relation) - _update_endpoints.assert_not_called() - _update_unit_status.assert_not_called() - assert isinstance(harness.model.unit.status, BlockedStatus) - - -def test_update_unit_status(harness): - with ( - patch( - "relations.db.DbProvides._check_for_blocking_relations" - ) as _check_for_blocking_relations, - patch( - "charm.PostgresqlOperatorCharm.is_blocked", new_callable=PropertyMock - ) as _is_blocked, - ): - # Test when the charm is not blocked. - rel_id = harness.model.get_relation(RELATION_NAME).id - relation = harness.model.get_relation(RELATION_NAME, rel_id) - _is_blocked.return_value = False - harness.charm.legacy_db_relation._update_unit_status(relation) - _check_for_blocking_relations.assert_not_called() - assert not isinstance(harness.charm.unit.status, ActiveStatus) - - # Test when the charm is blocked but not due to extensions request. - _is_blocked.return_value = True - harness.charm.unit.status = BlockedStatus("fake message") - harness.charm.legacy_db_relation._update_unit_status(relation) - _check_for_blocking_relations.assert_not_called() - assert not isinstance(harness.charm.unit.status, ActiveStatus) - - # Test when there are relations causing the blocked status. - harness.charm.unit.status = BlockedStatus( - "extensions requested through relation, enable them through config options" - ) - _check_for_blocking_relations.return_value = True - harness.charm.legacy_db_relation._update_unit_status(relation) - _check_for_blocking_relations.assert_called_once_with(relation.id) - assert not isinstance(harness.charm.unit.status, ActiveStatus) - - # Test when there are no relations causing the blocked status anymore. - _check_for_blocking_relations.reset_mock() - _check_for_blocking_relations.return_value = False - harness.charm.legacy_db_relation._update_unit_status(relation) - _check_for_blocking_relations.assert_called_once_with(relation.id) - assert isinstance(harness.charm.unit.status, ActiveStatus) - - -def test_on_relation_broken_extensions_unblock(harness): - with ( - patch.object(PostgresqlOperatorCharm, "postgresql", Mock()) as postgresql_mock, - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - ) as _primary_endpoint, - patch("charm.PostgresqlOperatorCharm.is_blocked", new_callable=PropertyMock) as is_blocked, - patch("charm.Patroni.member_started", new_callable=PropertyMock) as _member_started, - patch("charm.DbProvides._on_relation_departed") as _on_relation_departed, - ): - # Set some side effects to test multiple situations. - rel_id = harness.model.get_relation(RELATION_NAME).id - is_blocked.return_value = True - _member_started.return_value = True - _primary_endpoint.return_value = {"1.1.1.1"} - postgresql_mock.delete_user = PropertyMock(return_value=None) - harness.model.unit.status = BlockedStatus( - "extensions requested through relation, enable them through config options" - ) - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - "application", - {"database": DATABASE, "extensions": "test"}, - ) - - # Break the relation that blocked the charm. - harness.remove_relation(rel_id) - assert isinstance(harness.model.unit.status, ActiveStatus) - - -def test_on_relation_broken_extensions_keep_block(harness): - with ( - patch.object(PostgresqlOperatorCharm, "postgresql", Mock()) as postgresql_mock, - patch("charm.DbProvides._on_relation_departed") as _on_relation_departed, - patch("charm.Patroni.member_started", new_callable=PropertyMock) as _member_started, - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - ) as _primary_endpoint, - patch("charm.PostgresqlOperatorCharm.is_blocked", new_callable=PropertyMock) as is_blocked, - ): - # Set some side effects to test multiple situations. - is_blocked.return_value = True - _member_started.return_value = True - _primary_endpoint.return_value = {"1.1.1.1"} - postgresql_mock.delete_user = PropertyMock(return_value=None) - harness.model.unit.status = BlockedStatus( - "extensions requested through relation, enable them through config options" - ) - with harness.hooks_disabled(): - first_rel_id = harness.add_relation(RELATION_NAME, "application1") - harness.update_relation_data( - first_rel_id, - "application1", - {"database": DATABASE, "extensions": "test"}, - ) - second_rel_id = harness.add_relation(RELATION_NAME, "application2") - harness.update_relation_data( - second_rel_id, - "application2", - {"database": DATABASE, "extensions": "test"}, - ) - - event = Mock() - event.relation.id = first_rel_id - # Break one of the relations that block the charm. - harness.charm.legacy_db_relation._on_relation_broken(event) - assert isinstance(harness.model.unit.status, BlockedStatus) - - -def test_update_endpoints_with_relation(harness): - with ( - patch.object(PostgresqlOperatorCharm, "postgresql", Mock()) as postgresql_mock, - patch("charm.Patroni.get_primary") as _get_primary, - patch( - "relations.db.logger", - ) as _logger, - patch( - "charm.PostgresqlOperatorCharm.members_ips", - new_callable=PropertyMock, - ) as _members_ips, - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock(return_value="1.1.1.1"), - ) as _primary_endpoint, - patch( - "charm.DbProvides._get_state", - side_effect="postgresql/0", - ) as _get_state, - ): - # Set some side effects to test multiple situations. - postgresql_mock.get_postgresql_version = PropertyMock( - side_effect=[ - POSTGRESQL_VERSION, - POSTGRESQL_VERSION, - PostgreSQLGetPostgreSQLVersionError, - ] - ) - - # Mock the members_ips list to simulate different scenarios - # (with and without a replica). - _members_ips.side_effect = [ - {"1.1.1.1", "2.2.2.2"}, - {"1.1.1.1", "2.2.2.2"}, - {"1.1.1.1"}, - {"1.1.1.1"}, - ] - - # Add two different relations. - rel_id = harness.add_relation(RELATION_NAME, "application") - another_rel_id = harness.add_relation(RELATION_NAME, "application") - - # Get the relation to be used in the subsequent update endpoints calls. - relation = harness.model.get_relation(RELATION_NAME, rel_id) - - # Set some data to be used and compared in the relations. - password = "test-password" - master = f"dbname={DATABASE} host=1.1.1.1 password={password} port={DATABASE_PORT} user=" - standbys = f"dbname={DATABASE} host=2.2.2.2 password={password} port={DATABASE_PORT} user=" - - # Set some required data before update_endpoints is called. - for rel in [rel_id, another_rel_id]: - user = f"relation-{rel}" - harness.charm.set_secret("app", user, password) - harness.charm.set_secret("app", f"{user}-database", DATABASE) - - # Test with both a primary and a replica. - # Update the endpoints with the event and check that it updated only - # the right relation databags (the app and unit databags from the event). - harness.charm.legacy_db_relation.update_endpoints(relation) - for rel in [rel_id, another_rel_id]: - # Set the expected username based on the relation id. - user = f"relation-{rel}" - - # Check that the unit relation databag contains (or not) the endpoints. - unit_relation_data = harness.get_relation_data(rel, harness.charm.unit.name) - if rel == rel_id: - assert ( - "master" in unit_relation_data - and master + user == unit_relation_data["master"] - ) - assert ( - "standbys" in unit_relation_data - and standbys + user == unit_relation_data["standbys"] - ) - else: - assert not ( - "master" in unit_relation_data - and master + user == unit_relation_data["master"] - ) - assert not ( - "standbys" in unit_relation_data - and standbys + user == unit_relation_data["standbys"] - ) - - # Also test with only a primary instance. - harness.charm.legacy_db_relation.update_endpoints(relation) - for rel in [rel_id, another_rel_id]: - # Set the expected username based on the relation id. - user = f"relation-{rel}" - - # Check that the unit relation databag contains the endpoints. - unit_relation_data = harness.get_relation_data(rel, harness.charm.unit.name) - if rel == rel_id: - assert ( - "master" in unit_relation_data - and master + user == unit_relation_data["master"] - ) - assert ( - "standbys" in unit_relation_data - and standbys + user == unit_relation_data["standbys"] - ) - else: - assert not ( - "master" in unit_relation_data - and master + user == unit_relation_data["master"] - ) - assert not ( - "standbys" in unit_relation_data - and standbys + user == unit_relation_data["standbys"] - ) - - # version is not updated due to a PostgreSQLGetPostgreSQLVersionError. - harness.charm.legacy_db_relation.update_endpoints() - _logger.exception.assert_called_once_with( - "Failed to retrieve the PostgreSQL version to initialise/update db relation" - ) - - -def test_update_endpoints_without_relation(harness): - with ( - patch.object(PostgresqlOperatorCharm, "postgresql", Mock()) as postgresql_mock, - patch("charm.Patroni.get_primary") as _get_primary, - patch( - "relations.db.logger", - ) as _logger, - patch( - "charm.PostgresqlOperatorCharm.members_ips", - new_callable=PropertyMock, - ) as _members_ips, - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock(return_value="1.1.1.1"), - ) as _primary_endpoint, - patch( - "charm.DbProvides._get_state", - side_effect="postgresql/0", - ) as _get_state, - ): - # Set some side effects to test multiple situations. - postgresql_mock.get_postgresql_version = PropertyMock( - side_effect=[ - POSTGRESQL_VERSION, - POSTGRESQL_VERSION, - PostgreSQLGetPostgreSQLVersionError, - ] - ) - _get_primary.return_value = harness.charm.unit.name - # Mock the members_ips list to simulate different scenarios - # (with and without a replica). - _members_ips.side_effect = [ - {"1.1.1.1", "2.2.2.2"}, - {"1.1.1.1", "2.2.2.2"}, - {"1.1.1.1"}, - {"1.1.1.1"}, - ] - - # Add two different relations. - rel_id = harness.add_relation(RELATION_NAME, "application") - another_rel_id = harness.add_relation(RELATION_NAME, "application") - - # Set some data to be used and compared in the relations. - password = "test-password" - master = f"dbname={DATABASE} host=1.1.1.1 password={password} port={DATABASE_PORT} user=" - standbys = f"dbname={DATABASE} host=2.2.2.2 password={password} port={DATABASE_PORT} user=" - - # Set some required data before update_endpoints is called. - for rel in [rel_id, another_rel_id]: - user = f"relation-{rel}" - harness.charm.set_secret("app", user, password) - harness.charm.set_secret("app", f"{user}-database", DATABASE) - - # Test with both a primary and a replica. - # Update the endpoints and check that all relations' databags are updated. - harness.charm.legacy_db_relation.update_endpoints() - for rel in [rel_id, another_rel_id]: - # Set the expected username based on the relation id. - user = f"relation-{rel}" - - # Check that the unit relation databag contains the endpoints. - unit_relation_data = harness.get_relation_data(rel, harness.charm.unit.name) - assert "master" in unit_relation_data and master + user == unit_relation_data["master"] - assert ( - "standbys" in unit_relation_data - and standbys + user == unit_relation_data["standbys"] - ) - - # Also test with only a primary instance. - harness.charm.legacy_db_relation.update_endpoints() - for rel in [rel_id, another_rel_id]: - # Set the expected username based on the relation id. - user = f"relation-{rel}" - - # Check that the unit relation databag contains the endpoints. - unit_relation_data = harness.get_relation_data(rel, harness.charm.unit.name) - assert "master" in unit_relation_data and master + user == unit_relation_data["master"] - assert ( - "standbys" in unit_relation_data - and standbys + user == unit_relation_data["standbys"] - ) - - # version is not updated due to a PostgreSQLGetPostgreSQLVersionError. - harness.charm.legacy_db_relation.update_endpoints() - _logger.exception.assert_called_once_with( - "Failed to retrieve the PostgreSQL version to initialise/update db relation" - ) - - -def test_get_allowed_units(harness): - # No allowed units from the current database application. - peer_rel_id = harness.model.get_relation(PEER).id - rel_id = harness.model.get_relation(RELATION_NAME).id - peer_relation = harness.model.get_relation(PEER, peer_rel_id) - assert harness.charm.legacy_db_relation._get_allowed_units(peer_relation) == "" - - # List of space separated allowed units from the other application. - harness.add_relation_unit(rel_id, "application/1") - db_relation = harness.model.get_relation(RELATION_NAME, rel_id) - assert ( - harness.charm.legacy_db_relation._get_allowed_units(db_relation) - == "application/0 application/1" - ) diff --git a/tests/unit/test_postgresql_provider.py b/tests/unit/test_postgresql_provider.py index 2aaed01083..399ebc2759 100644 --- a/tests/unit/test_postgresql_provider.py +++ b/tests/unit/test_postgresql_provider.py @@ -125,12 +125,17 @@ def test_on_database_requested(harness): # Assert that the correct calls were made. user = f"relation-{rel_id}" postgresql_mock.create_user.assert_called_once_with( - user, "test-password", extra_user_roles=EXTRA_USER_ROLES + user, + "test-password", + extra_user_roles=[role.lower() for role in EXTRA_USER_ROLES.split(",")], ) database_relation = harness.model.get_relation(RELATION_NAME) client_relations = [database_relation] postgresql_mock.create_database.assert_called_once_with( - DATABASE, user, plugins=["pgaudit"], client_relations=client_relations + DATABASE, + user, + plugins=["pgaudit"], + client_relations=client_relations, ) postgresql_mock.get_postgresql_version.assert_called_once() _update_endpoints.assert_called_once() diff --git a/tests/unit/test_upgrade.py b/tests/unit/test_upgrade.py index 7dfbfc521b..1baac2da9b 100644 --- a/tests/unit/test_upgrade.py +++ b/tests/unit/test_upgrade.py @@ -106,6 +106,9 @@ def test_on_upgrade_granted(harness): patch("charm.Patroni.start_patroni") as _start_patroni, patch("charm.PostgresqlOperatorCharm._install_snap_packages") as _install_snap_packages, patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, + patch( + "charm.PostgresqlOperatorCharm.updated_synchronous_node_count" + ) as _updated_synchronous_node_count, ): # Test when the charm fails to start Patroni. mock_event = MagicMock() @@ -174,6 +177,7 @@ def test_on_upgrade_granted(harness): _member_started.reset_mock() _cluster_members.reset_mock() mock_event.defer.reset_mock() + _updated_synchronous_node_count.reset_mock() _is_replication_healthy.return_value = True with harness.hooks_disabled(): harness.set_leader(True) @@ -184,6 +188,7 @@ def test_on_upgrade_granted(harness): _set_unit_completed.assert_called_once() _set_unit_failed.assert_not_called() _on_upgrade_changed.assert_called_once() + _updated_synchronous_node_count.assert_called_once_with() def test_pre_upgrade_check(harness): diff --git a/tox.ini b/tox.ini index 508d1a645f..0f7b4d4bd4 100644 --- a/tox.ini +++ b/tox.ini @@ -57,8 +57,13 @@ commands = description = Run integration tests pass_env = CI - GITHUB_OUTPUT - SECRETS_FROM_GITHUB + AWS_ACCESS_KEY + AWS_SECRET_KEY + GCP_ACCESS_KEY + GCP_SECRET_KEY + UBUNTU_PRO_TOKEN + LANDSCAPE_ACCOUNT_NAME + LANDSCAPE_REGISTRATION_KEY commands_pre = poetry install --only integration --no-root commands =