ci(ovn): add verification tests for database

.gitignore

@@ -17,3 +17,5 @@ junit.xml
 junit-go.xml
 ansible-lint.xml
 output
+.stestr
+.tox

.stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=./atmosphere/tests
+top_dir=./

zuul.d/playbooks/molecule-aio/run.yml → atmosphere/tests/roles/test_openvswitch.py

@@ -12,9 +12,9 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-- hosts: all
-  become: true
-  roles:
-    - tox
-  vars:
-    tox_envlist: "molecule-aio-{{ network_backend }}"
+from oslotest import base  # type: ignore
+
+
+class TestOpenVswitch(base.BaseTestCase):
+    def test_noop(self):
+        pass

atmosphere/tests/roles/test_ovn.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2024 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+from datetime import datetime, timedelta
+
+import testscenarios  # type: ignore
+from kubernetes import client, config  # type: ignore
+from kubernetes.stream import stream  # type: ignore
+from oslotest import base  # type: ignore
+from testtools import matchers, testcase  # type: ignore
+
+config.load_kube_config()
+
+
+class TestOVN(testscenarios.WithScenarios, testcase.WithAttributes, base.BaseTestCase):
+    scenarios = [
+        ("ovsdb-nb", {"statefulset": "ovn-ovsdb-nb", "filename": "ovnnb_db.db"}),
+        ("ovsdb-sb", {"statefulset": "ovn-ovsdb-sb", "filename": "ovnsb_db.db"}),
+    ]
+
+    def setUp(self):
+        super(TestOVN, self).setUp()
+
+        self.corev1 = client.CoreV1Api()
+        self.appsv1 = client.AppsV1Api()
+
+        if os.environ["ATMOSPHERE_NETWORK_BACKEND"] != "ovn":
+            self.skipTest("OVN is not deployed on this cloud.")
+
+    def _read_stateful_set(self):
+        return self.appsv1.read_namespaced_stateful_set(
+            name=self.statefulset, namespace="openstack"
+        )
+
+    def test_ovn_is_running(self):
+        sts = self._read_stateful_set()
+        self.assertEqual(sts.spec.replicas, sts.status.replicas)
+
+    def test_ovn_is_using_correct_database_folder(self):
+        sts = self._read_stateful_set()
+        self.assertThat(sts.spec.volume_claim_templates, matchers.HasLength(1))
+        self.assertEqual(sts.spec.volume_claim_templates[0].metadata.name, "data")
+
+        volume_mounts = sts.spec.template.spec.containers[0].volume_mounts
+        volume_mount = next(
+            (
+                vm
+                for vm in volume_mounts
+                if vm.name == sts.spec.volume_claim_templates[0].metadata.name
+            )
+        )
+
+        self.assertEqual(volume_mount.mount_path, "/etc/ovn")
+
+        db_last_modified = stream(
+            self.corev1.connect_get_namespaced_pod_exec,
+            f"{sts.metadata.name}-0",
+            sts.metadata.namespace,
+            command=["stat", "-c", "%Y", f"{volume_mount.mount_path}/{self.filename}"],
+            stdin=False,
+            stderr=True,
+            stdout=True,
+            tty=False,
+        )
+
+        db_last_modified = datetime.fromtimestamp(int(db_last_modified))
+        self.assertTrue(db_last_modified > datetime.now() - timedelta(minutes=1))

flake.nix

@@ -15,14 +15,15 @@
         {
           devShell = pkgs.mkShell {
             buildInputs = with pkgs; [
+              bashInteractive
               earthly
+              glibcLocales
               go
               kubernetes-helm
               nixpkgs-fmt
               poetry
+              python311Packages.tox
               vendir
-              glibcLocales
-              bashInteractive
             ];
           };
         }

molecule/aio/verify.yml

@@ -12,5 +12,17 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+- name: Run tests
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Run "stestr" tests
+      become: true
+      ansible.builtin.shell: stestr run
+      args:
+        chdir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}"
+      environment:
+        KUBECONFIG: /root/.kube/config
+
 - name: Run Tempest tests
   ansible.builtin.import_playbook: vexxhost.atmosphere.tempest

test-requirements.txt

@@ -0,0 +1,6 @@
+kubernetes
+molecule
+molecule-plugins[docker]
+oslotest
+stestr
+testscenarios

tox.ini

@@ -1,37 +1,39 @@
 [tox]
 minversion = 4
-skipsdist = True
 
-[testenv:molecule]
+[testenv]
+usedevelop = True
+setenv =
+  VIRTUAL_ENV={envdir}
+  LANGUAGE=en_US
+  LC_ALL=en_US.utf-8
+  OS_STDOUT_CAPTURE=1
+  OS_STDERR_CAPTURE=1
+  OS_TEST_TIMEOUT=160
+  PYTHONDONTWRITEBYTECODE=1
 deps =
-  molecule
-  molecule-plugins[docker]
-  # TODO(mnaser): We should remove these once we have an actual installable
-  #               Python package and drop `skipsdist`.
-  docker-image-py
-  jmespath
-  netaddr
-  openstacksdk<0.99.0
-  rjsonnet
+  -r{toxinidir}/test-requirements.txt
+passenv =
+  KUBECONFIG
+
+[testenv:venv]
+passenv =
+  ATMOSPHERE_NETWORK_BACKEND
+commands =
+  {posargs}
 
 [testenv:molecule-keycloak]
-deps =
-  {[testenv:molecule]deps}
 commands =
   molecule test -s keycloak
 
 [testenv:molecule-csi-{rbd,local-path-provisioner}]
-deps =
-  {[testenv:molecule]deps}
 setenv =
   rbd: MOLECULE_CSI_DRIVER = rbd
   local-path-provisioner: MOLECULE_CSI_DRIVER = local-path-provisioner
 commands =
   molecule test -s csi
 
 [testenv:molecule-aio-{openvswitch,ovn}]
-deps =
-  {[testenv:molecule]deps}
 setenv =
   ATMOSPHERE_DEBUG = true
   openvswitch: ATMOSPHERE_NETWORK_BACKEND = openvswitch

zuul.d/jobs.yaml

@@ -17,6 +17,8 @@
     parent: tox
     abstract: true
     pre-run: zuul.d/playbooks/molecule/pre.yml
+    run: zuul.d/playbooks/molecule/run.yml
+    post-run: zuul.d/playbooks/molecule/post.yml
 
 - job:
     name: atmosphere-molecule-keycloak
@@ -28,28 +30,26 @@
 - job:
     name: atmosphere-molecule-csi
     parent: atmosphere-molecule
-    run: zuul.d/playbooks/molecule-csi/run.yml
     abstract: true
 
 - job:
     name: atmosphere-molecule-csi-local-path-provisioner
     parent: atmosphere-molecule-csi
     vars:
-      csi_driver: local-path-provisioner
+      tox_envlist: molecule-csi-local-path-provisioner
 
 - job:
     name: atmosphere-molecule-csi-rbd
     parent: atmosphere-molecule-csi
     vars:
-      csi_driver: rbd
+      tox_envlist: molecule-csi-rbd
 
 - job:
     name: atmosphere-molecule-aio
     parent: atmosphere-molecule
     abstract: true
     timeout: 7200
     pre-run: zuul.d/playbooks/molecule-aio/pre.yml
-    run: zuul.d/playbooks/molecule-aio/run.yml
     post-run: zuul.d/playbooks/molecule-aio/post.yml
     roles:
       - zuul: openstack/openstack-helm-infra
@@ -62,10 +62,10 @@
     name: atmosphere-molecule-aio-openvswitch
     parent: atmosphere-molecule-aio
     vars:
-      network_backend: openvswitch
+      tox_envlist: molecule-aio-openvswitch
 
 - job:
     name: atmosphere-molecule-aio-ovn
     parent: atmosphere-molecule-aio
     vars:
-      network_backend: ovn
+      tox_envlist: molecule-aio-ovn

zuul.d/playbooks/molecule/post.yml

@@ -0,0 +1,29 @@
+# Copyright (c) 2024 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Finalize Molecule tests
+  hosts: all
+  tasks:
+    # NOTE(mnaser): Since we need to run Molecule using `sudo` because
+    #               of a bug with the Kubernetes collection, we need to
+    #               restore the permissions of the repository to the
+    #               user that is running the tests.
+    - name: Fix permissions for repository
+      become: true
+      ansible.builtin.file:
+        path: "{{ zuul.project.src_dir }}"
+        state: directory
+        recurse: yes
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"

zuul.d/playbooks/molecule/pre.yml

@@ -25,3 +25,11 @@
         sed -i 's/  \(.*\): \(.*\)$/  \1: registry.atmosphere.dev\/\2/' roles/defaults/vars/main.yml
       args:
         chdir: "{{ zuul.project.src_dir }}"
+
+    # TODO(mnaser): Drop this when we move to PBR
+    - name: Add current folder to Git's safe directories
+      become: true
+      ansible.builtin.shell: |
+        git config --global --add safe.directory $(pwd)
+      args:
+        chdir: "{{ zuul.project.src_dir }}"

zuul.d/playbooks/molecule-csi/run.yml → zuul.d/playbooks/molecule/run.yml

@@ -16,5 +16,3 @@
   become: true
   roles:
     - tox
-  vars:
-    tox_envlist: "molecule-csi-{{ csi_driver }}"