feat: uv and versioned ci (#378)

* feat: uv and versioned ci

* point workflows at v1

* pin markupsafe and fix things

* add cargo and rustc

* add binary packages again

* add deps

* reuse packed charm

* use platforms syntax in charmcraft.yaml

* try to fix grafana tester build

* use sh to build grafana tester

* move scenario under unit and use sh

* chore: trigger ci

* try to fix itests

* try to fix itests again

* comment things

* tox fmt

* fix itests

* remove extra file

* try again

* fix lint

* multiple things

* chore: trigger ci

* chore: trigger ci

* fix git describe

* run release on track/**

* try to fix last itest

* try to fix last itest

* chore: trigger ci

* use subprocess.run instead of ops_test.run

* try to fix last itest

* tox fmt

* chore: trigger ci

* chore: trigger ci

* chore: trigger ci

* chore: trigger ci

* try againg

* tox fmt

* re-add the sg command

* try test changes

* tryg

* remove sudo

* chore: trigger ci

* use workflow tags instead of branch

* use the full platforms notation

* fix static checks on tests

* tox fmt

* change requires-python to >=3.8

* python-version to 3.8

* chore: trigger ci

---------

Signed-off-by: Luca Bello <36242061+lucabello@users.noreply.github.com>

.github/workflows/promote.yaml

@@ -14,7 +14,7 @@ on:
 jobs:
   promote:
     name: Promote
-    uses: canonical/observability/.github/workflows/charm-promote.yaml@main
+    uses: canonical/observability/.github/workflows/charm-promote.yaml@v1
     with:
       promotion: ${{ github.event.inputs.promotion }}
     secrets: inherit

.github/workflows/pull-request.yaml

@@ -8,11 +8,5 @@ on:
 jobs:
   pull-request:
     name: PR
-    uses: canonical/observability/.github/workflows/charm-pull-request.yaml@main
+    uses: canonical/observability/.github/workflows/charm-pull-request.yaml@v1
     secrets: inherit
-    with:
-      ip-range: 10.64.140.43/30
-
-  terraform-checks:
-    name: Terraform
-    uses: canonical/observability/.github/workflows/terraform-quality-checks.yaml@main

.github/workflows/quality-gates.yaml

@@ -0,0 +1,16 @@
+name: Quality Gates
+
+on:
+  # Manual trigger
+  workflow_dispatch:
+  # Run the quality checks periodically
+  # FIXME: adjust the frequency as needed once we have actual gates in place
+  schedule:
+    - cron: "0 0 * * Tue"
+
+
+jobs:
+  quality-gates:
+    name: Run quality gates
+    uses: canonical/observability/.github/workflows/charm-quality-gates.yaml@v1
+    secrets: inherit

.github/workflows/release.yaml

@@ -4,10 +4,9 @@ on:
   push:
     branches:
       - main
+      - track/**
 
 jobs:
   release:
-    uses: canonical/observability/.github/workflows/charm-release.yaml@main
+    uses: canonical/observability/.github/workflows/charm-release.yaml@v1
     secrets: inherit
-    with:
-      ip-range: 10.64.140.43/30

.github/workflows/update-libs.yaml

@@ -9,6 +9,6 @@ on:
 jobs:
   update-lib:
     name: Check libraries
-    uses: canonical/observability/.github/workflows/charm-update-libs.yaml@main
+    uses: canonical/observability/.github/workflows/charm-update-libs.yaml@v1
     secrets: inherit
 

charmcraft.yaml

@@ -110,21 +110,19 @@ resources:
     description: upstream image for sqlite streaming
     upstream-source: docker.io/litestream/litestream:0.4.0-beta.2
 
-bases:
-  - build-on:
-      - name: "ubuntu"
-        channel: "20.04"
-    run-on:
-      - name: "ubuntu"
-        channel: "20.04"
+platforms:
+  ubuntu@20.04:amd64:
 
 parts:
   charm:
-    # This is currently necessary because it cuts on packing times.
-    # It can be removed when we have at least multithreading on charmcraft pack.
     charm-binary-python-packages: [cryptography, jsonschema, pydantic, pydantic-core, maturin]
-    build-packages:
-      - git
+    build-packages: [git]
+    build-snaps: [astral-uv]
+    override-build: |
+      uv export --frozen --no-hashes --format=requirements-txt -o requirements.txt
+      git describe --always > version
+      craftctl default
+    charm-requirements: [requirements.txt]
   static-sqlite3:
     plugin: dump
     source: .

pyproject.toml

@@ -1,5 +1,54 @@
 # Copyright 2021 Canonical Ltd.
 # See LICENSE file for licensing details.
+[project]
+name = "grafana-k8s"
+version = "0.0"
+requires-python = "~=3.8"
+
+dependencies = [
+  "ops>=2.17",
+  "pyyaml",
+  "urllib3",
+  "jsonschema",
+  "cryptography",
+  "jinja2<3",
+  "markupsafe==2.0.1", # https://github.com/pallets/markupsafe/issues/371
+  "lightkube>=0.11",
+  # lib/charms/grafana_k8s/v0/grafana_dashboard.py
+  # lib/charms/temo_k8s/v1/charm_tracing.py
+  "cosl", 
+  # lib/charms/tempo_k8s/v1/charm_tracing.py
+  "opentelemetry-exporter-otlp-proto-http",
+  "pydantic",
+]
+
+[project.optional-dependencies]
+dev = [
+  # Linting
+  "ruff",
+  "codespell",
+  # Static
+  "pyright",
+  # Unit
+  "pytest",
+  "coverage[toml]",
+  "responses",
+  "cosl",
+  "pydantic",
+  # Integration
+  "juju<=3.3.0,>=3.0",
+  "websockets<14.0",
+  "oauth_tools@git+https://github.com/canonical/iam-bundle@671a33419869fab1fe63d81873b41f6b181498e3#egg=oauth_tools",
+  "lightkube",
+  "minio",
+  "pytest-operator>=0.39",
+  "ops[testing]",
+  "pytest-asyncio",
+  "aiohttp",
+  "asyncstdlib",
+  "sh",
+  "tenacity",
+]
 
 # Testing tools configuration
 [tool.coverage.run]
@@ -36,6 +85,7 @@ pythonPlatform = "All"
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
+addopts = "--tb=native --verbose --capture=no --log-cli-level=INFO"
 
 [tool.codespell]
 skip = ".git,.tox,build,venv*"

tests/integration/conftest.py

@@ -3,6 +3,7 @@
 # See LICENSE file for licensing details.
 import functools
 import logging
+import os
 import shutil
 from collections import defaultdict
 from datetime import datetime
@@ -73,6 +74,9 @@ def copy_grafana_libraries_into_tester_charm(ops_test: OpsTest) -> None:
 @timed_memoizer
 async def grafana_charm(ops_test: OpsTest) -> Path:
     """Grafana charm used for integration testing."""
+    if charm_file := os.environ.get("CHARM_PATH"):
+        return Path(charm_file)
+
     charm = await ops_test.build_charm(".")
     return charm
 

tests/integration/helpers.py

@@ -1,13 +1,10 @@
 #!/usr/bin/env python3
 # Copyright 2021 Canonical Ltd.
 # See LICENSE file for licensing details.
-import asyncio
-import grp
 import json
 import logging
-import subprocess
 from pathlib import Path
-from typing import Tuple
+from typing import Optional, Tuple
 
 import requests
 import yaml
@@ -34,13 +31,14 @@ async def grafana_password(ops_test: OpsTest, app_name: str) -> str:
     Returns:
         admin password as a string
     """
-    leader = None  # type: Unit
-    for unit in ops_test.model.applications[app_name].units:
+    leader: Optional[Unit] = None
+    for unit in ops_test.model.applications[app_name].units:  # type: ignore
         is_leader = await unit.is_leader_from_status()
         if is_leader:
             leader = unit
             break
 
+    assert leader
     action = await leader.run_action("get-admin-password")
     action = await action.wait()
     return action.results["admin-password"]
@@ -57,6 +55,7 @@ async def unit_address(ops_test: OpsTest, app_name: str, unit_num: int) -> str:
     Returns:
         unit address as a string
     """
+    assert ops_test.model
     status = await ops_test.model.get_status()
     return status["applications"][app_name]["units"]["{}/{}".format(app_name, unit_num)]["address"]
 
@@ -295,50 +294,7 @@ async def get_grafana_environment_variable(
 
     # If we do find one, split it into parts around `foo=bar` and return the value
     value = next(iter([env for env in stdout.splitlines() if env_var in env])).split("=")[-1] or ""
-    return rc, value, stderr.strip
-
-
-def uk8s_group() -> str:
-    try:
-        # Classically confined microk8s
-        uk8s_group = grp.getgrnam("microk8s").gr_name
-    except KeyError:
-        # Strictly confined microk8s
-        uk8s_group = "snap_microk8s"
-    return uk8s_group
-
-
-async def reenable_metallb() -> str:
-    # Set up microk8s metallb addon, needed by traefik
-    logger.info("(Re)-enabling metallb")
-    cmd = [
-        "sh",
-        "-c",
-        "ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc'",
-    ]
-    result = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    ip = result.stdout.decode("utf-8").strip()
-
-    logger.info("First, disable metallb, just in case")
-    try:
-        cmd = ["sg", uk8s_group(), "-c", "microk8s disable metallb"]
-        subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    except Exception as e:
-        print(e)
-        raise
-
-    await asyncio.sleep(30)  # why? just because, for now
-
-    logger.info("Now enable metallb")
-    try:
-        cmd = ["sg", uk8s_group(), "-c", f"microk8s enable metallb:{ip}-{ip}"]
-        subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    except Exception as e:
-        print(e)
-        raise
-
-    await asyncio.sleep(30)  # why? just because, for now
-    return ip
+    return str(rc), value, stderr.strip()
 
 
 async def deploy_literal_bundle(ops_test: OpsTest, bundle: str):
@@ -389,6 +345,7 @@ async def deploy_and_configure_minio(ops_test: OpsTest) -> None:
         "access-key": "accesskey",
         "secret-key": "secretkey",
     }
+    assert ops_test.model
     await ops_test.model.deploy("minio", channel="edge", trust=True, config=config)
     await ops_test.model.wait_for_idle(apps=["minio"], status="active", timeout=2000)
     minio_addr = await unit_address(ops_test, "minio", 0)
@@ -406,7 +363,7 @@ async def deploy_and_configure_minio(ops_test: OpsTest) -> None:
         mc_client.make_bucket("tempo")
 
     # configure s3-integrator
-    s3_integrator_app: Application = ops_test.model.applications["s3-integrator"]
+    s3_integrator_app: Application = ops_test.model.applications["s3-integrator"]  # type: ignore
     s3_integrator_leader: Unit = s3_integrator_app.units[0]
 
     await s3_integrator_app.set_config(
@@ -427,6 +384,7 @@ async def deploy_tempo_cluster(ops_test: OpsTest):
     worker_app = "tempo-worker"
     tempo_worker_charm_url, worker_channel = "tempo-worker-k8s", "edge"
     tempo_coordinator_charm_url, coordinator_channel = "tempo-coordinator-k8s", "edge"
+    assert ops_test.model
     await ops_test.model.deploy(
         tempo_worker_charm_url, application_name=worker_app, channel=worker_channel, trust=True
     )
@@ -477,6 +435,7 @@ async def get_traces_patiently(tempo_host, service_name="tracegen-otlp_http", tl
 
 async def get_application_ip(ops_test: OpsTest, app_name: str) -> str:
     """Get the application IP address."""
+    assert ops_test.model
     status = await ops_test.model.get_status()
     app = status["applications"][app_name]
     return app.public_address
@@ -491,11 +450,13 @@ def __init__(self, ops_test: OpsTest, config: dict):
 
     async def __aenter__(self):
         """On entry, the config is set to the user provided custom values."""
+        assert self.ops_test.model
         config = await self.ops_test.model.get_config()
         self.revert_to = {k: config[k] for k in self.change_to.keys()}
         await self.ops_test.model.set_config(self.change_to)
         return self
 
     async def __aexit__(self, exc_type, exc_value, exc_traceback):
         """On exit, the modified config options are reverted to their original values."""
+        assert self.ops_test.model
         await self.ops_test.model.set_config(self.revert_to)

tests/integration/test_external_url.py

@@ -6,7 +6,7 @@
 import logging
 
 import pytest
-from helpers import ModelConfigChange, grafana_password, oci_image, reenable_metallb
+from helpers import ModelConfigChange, grafana_password, oci_image
 from pytest_operator.plugin import OpsTest
 from workload import Grafana
 
@@ -54,8 +54,9 @@ async def test_deploy(ops_test, grafana_charm):
 
 @pytest.mark.xfail
 async def test_grafana_is_reachable_via_traefik(ops_test: OpsTest):
+    assert ops_test.model
     # GIVEN metallb is ready
-    ip = await reenable_metallb()
+    ip = "10.64.140.43"  # default in concierge: https://github.com/jnsgruk/concierge/blob/1fbe3c55cc8b53eadfa5782f57d1f60e8fb5504b/README.md?plain=1#L313
 
     # WHEN grafana is related to traefik
     await ops_test.model.add_relation(f"{grafana_app_name}:ingress", "traefik")

tests/integration/test_get_password.py

@@ -19,10 +19,6 @@
 }
 
 
-async def test_setup_env(ops_test):
-    await ops_test.model.set_config({"logging-config": "<root>=WARNING; unit=DEBUG"})
-
-
 @pytest.mark.abort_on_fail
 async def test_password_returns_correct_value_after_upgrading(ops_test, grafana_charm):
     """Deploy from charmhub and then upgrade with the charm-under-test."""