ci(ovn): add verification tests for database

We realized that we actually were pointing to the wrong
folder for our database.  This adds tests to make sure
that we verify that the database is at the right place.

Depends-On: #1044

Signed-off-by: Mohammed Naser <mnaser@vexxhost.com>

.gitignore

@@ -17,3 +17,5 @@ junit.xml
 junit-go.xml
 ansible-lint.xml
 output
+.stestr
+.tox

.stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=./atmosphere/tests
+top_dir=./

zuul.d/playbooks/molecule-aio/run.yml → atmosphere/tests/roles/test_openvswitch.py

@@ -12,9 +12,9 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-- hosts: all
-  become: true
-  roles:
-    - tox
-  vars:
-    tox_envlist: "molecule-aio-{{ network_backend }}"
+from oslotest import base  # type: ignore
+
+
+class TestOpenVswitch(base.BaseTestCase):
+    def test_noop(self):
+        pass

atmosphere/tests/roles/test_ovn.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2024 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+from datetime import datetime, timedelta
+
+import testscenarios  # type: ignore
+from kubernetes import client, config  # type: ignore
+from kubernetes.stream import stream  # type: ignore
+from oslotest import base  # type: ignore
+from testtools import matchers, testcase  # type: ignore
+
+config.load_kube_config()
+
+
+class TestOVN(testscenarios.WithScenarios, testcase.WithAttributes, base.BaseTestCase):
+    scenarios = [
+        ("ovsdb-nb", {"statefulset": "ovn-ovsdb-nb", "filename": "ovnnb_db.db"}),
+        ("ovsdb-sb", {"statefulset": "ovn-ovsdb-sb", "filename": "ovnsb_db.db"}),
+    ]
+
+    def setUp(self):
+        super(TestOVN, self).setUp()
+
+        self.corev1 = client.CoreV1Api()
+        self.appsv1 = client.AppsV1Api()
+
+        if os.environ["ATMOSPHERE_NETWORK_BACKEND"] != "ovn":
+            self.skipTest("OVN is not deployed on this cloud.")
+
+    def _read_stateful_set(self):
+        return self.appsv1.read_namespaced_stateful_set(
+            name=self.statefulset, namespace="openstack"
+        )
+
+    def test_ovn_is_running(self):
+        sts = self._read_stateful_set()
+        self.assertEqual(sts.spec.replicas, sts.status.replicas)
+
+    def test_ovn_is_using_correct_database_folder(self):
+        sts = self._read_stateful_set()
+        self.assertThat(sts.spec.volume_claim_templates, matchers.HasLength(1))
+        self.assertEqual(sts.spec.volume_claim_templates[0].metadata.name, "data")
+
+        volume_mounts = sts.spec.template.spec.containers[0].volume_mounts
+        volume_mount = next(
+            (
+                vm
+                for vm in volume_mounts
+                if vm.name == sts.spec.volume_claim_templates[0].metadata.name
+            )
+        )
+
+        self.assertEqual(volume_mount.mount_path, "/etc/ovn")
+
+        db_last_modified = stream(
+            self.corev1.connect_get_namespaced_pod_exec,
+            f"{sts.metadata.name}-0",
+            sts.metadata.namespace,
+            command=["stat", "-c", "%Y", f"{volume_mount.mount_path}/{self.filename}"],
+            stdin=False,
+            stderr=True,
+            stdout=True,
+            tty=False,
+        )
+
+        db_last_modified = datetime.fromtimestamp(int(db_last_modified))
+        self.assertTrue(db_last_modified > datetime.now() - timedelta(minutes=1))

flake.nix

@@ -15,14 +15,15 @@
         {
           devShell = pkgs.mkShell {
             buildInputs = with pkgs; [
+              bashInteractive
               earthly
+              glibcLocales
               go
               kubernetes-helm
               nixpkgs-fmt
               poetry
+              python311Packages.tox
               vendir
-              glibcLocales
-              bashInteractive
             ];
           };
         }

molecule/aio/verify.yml

@@ -12,5 +12,17 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+- name: Run tests
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Run "stestr" tests
+      become: true
+      ansible.builtin.shell: env; stestr run
+      args:
+        chdir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}"
+      environment:
+        KUBECONFIG: /root/.kube/config
+
 - name: Run Tempest tests
   ansible.builtin.import_playbook: vexxhost.atmosphere.tempest

test-requirements.txt

@@ -0,0 +1,6 @@
+kubernetes
+molecule
+molecule-plugins[docker]
+oslotest
+stestr
+testscenarios

tox.ini

@@ -1,40 +1,46 @@
 [tox]
 minversion = 4
-skipsdist = True
 
-[testenv:molecule]
+[testenv]
+usedevelop = True
+setenv =
+  VIRTUAL_ENV={envdir}
+  LANGUAGE=en_US
+  LC_ALL=en_US.utf-8
+  OS_STDOUT_CAPTURE=1
+  OS_STDERR_CAPTURE=1
+  OS_TEST_TIMEOUT=160
+  PYTHONDONTWRITEBYTECODE=1
 deps =
-  molecule
-  molecule-plugins[docker]
-  # TODO(mnaser): We should remove these once we have an actual installable
-  #               Python package and drop `skipsdist`.
-  docker-image-py
-  jmespath
-  netaddr
-  openstacksdk<0.99.0
-  rjsonnet
+  -r{toxinidir}/test-requirements.txt
+passenv =
+  KUBECONFIG
+
+[testenv:venv]
+passenv =
+  ATMOSPHERE_NETWORK_BACKEND
+commands =
+  {posargs}
 
 [testenv:molecule-keycloak]
-deps =
-  {[testenv:molecule]deps}
 commands =
   molecule test -s keycloak
 
 [testenv:molecule-csi-{rbd,local-path-provisioner}]
-deps =
-  {[testenv:molecule]deps}
+allowlist_externals =
+  sudo
 setenv =
   rbd: MOLECULE_CSI_DRIVER = rbd
   local-path-provisioner: MOLECULE_CSI_DRIVER = local-path-provisioner
 commands =
-  molecule test -s csi
+  sudo --preserve-env=ATMOSPHERE_DEBUG,MOLECULE_CSI_DRIVER molecule test -s csi
 
 [testenv:molecule-aio-{openvswitch,ovn}]
-deps =
-  {[testenv:molecule]deps}
+allowlist_externals =
+  sudo
 setenv =
   ATMOSPHERE_DEBUG = true
   openvswitch: ATMOSPHERE_NETWORK_BACKEND = openvswitch
   ovn: ATMOSPHERE_NETWORK_BACKEND = ovn
 commands =
-  molecule test -s aio
+  sudo --preserve-env=ATMOSPHERE_DEBUG,ATMOSPHERE_NETWORK_BACKEND molecule test -s aio

zuul.d/jobs.yaml

@@ -17,6 +17,8 @@
     parent: tox
     abstract: true
     pre-run: zuul.d/playbooks/molecule/pre.yml
+    run: zuul.d/playbooks/molecule/run.yml
+    post-run: zuul.d/playbooks/molecule/post.yml
 
 - job:
     name: atmosphere-molecule-keycloak
@@ -28,28 +30,26 @@
 - job:
     name: atmosphere-molecule-csi
     parent: atmosphere-molecule
-    run: zuul.d/playbooks/molecule-csi/run.yml
     abstract: true
 
 - job:
     name: atmosphere-molecule-csi-local-path-provisioner
     parent: atmosphere-molecule-csi
     vars:
-      csi_driver: local-path-provisioner
+      tox_envlist: molecule-csi-local-path-provisioner
 
 - job:
     name: atmosphere-molecule-csi-rbd
     parent: atmosphere-molecule-csi
     vars:
-      csi_driver: rbd
+      tox_envlist: molecule-csi-rbd
 
 - job:
     name: atmosphere-molecule-aio
     parent: atmosphere-molecule
     abstract: true
     timeout: 7200
     pre-run: zuul.d/playbooks/molecule-aio/pre.yml
-    run: zuul.d/playbooks/molecule-aio/run.yml
     post-run: zuul.d/playbooks/molecule-aio/post.yml
     roles:
       - zuul: openstack/openstack-helm-infra
@@ -62,10 +62,10 @@
     name: atmosphere-molecule-aio-openvswitch
     parent: atmosphere-molecule-aio
     vars:
-      network_backend: openvswitch
+      tox_envlist: molecule-aio-openvswitch
 
 - job:
     name: atmosphere-molecule-aio-ovn
     parent: atmosphere-molecule-aio
     vars:
-      network_backend: ovn
+      tox_envlist: molecule-aio-ovn

zuul.d/playbooks/molecule/post.yml

@@ -0,0 +1,28 @@
+# Copyright (c) 2024 VEXXHOST, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+- name: Finalize Molecule tests
+  hosts: all
+  tasks:
+    # NOTE(mnaser): Since we need to run Molecule using `sudo` because
+    #               of a bug with the Kubernetes collection, we need to
+    #               restore the permissions of the repository to the
+    #               user that is running the tests.
+    - name: Fix permissions for repository
+      ansible.builtin.file:
+        path: "{{ zuul.project.src_dir }}"
+        state: directory
+        recurse: yes
+        owner: "{{ ansible_user }}"
+        group: "{{ ansible_user }}"

zuul.d/playbooks/molecule-csi/run.yml → zuul.d/playbooks/molecule/run.yml

@@ -13,8 +13,5 @@
 # under the License.
 
 - hosts: all
-  become: true
   roles:
     - tox
-  vars:
-    tox_envlist: "molecule-csi-{{ csi_driver }}"