Fix test suite

.github/workflows/test-module.yml

@@ -2,6 +2,11 @@ name: Test module
 
 on:
   workflow_dispatch:
+    inputs:
+      debug_shell:
+        description: "Debug shell"
+        required: true
+        type: boolean
   workflow_run:
     workflows: ["Publish images"]
     types: [completed]
@@ -10,11 +15,12 @@ jobs:
   module:
     if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == '' }}
     uses: NethServer/ns8-github-actions/.github/workflows/module-info.yml@main
-  run_tests_on_do:
+  run_tests:
     needs: module
     uses: NethServer/ns8-github-actions/.github/workflows/test-on-digitalocean-infra.yml@main
     with:
-      coremodules: "ghcr.io/${{needs.module.outputs.owner}}/${{needs.module.outputs.name}}:${{needs.module.outputs.tag}}"
+      args: "ghcr.io/${{needs.module.outputs.owner}}/${{needs.module.outputs.name}}:${{needs.module.outputs.tag}}"
       repo_ref: ${{needs.module.outputs.sha}}
+      debug_shell: ${{ github.event.inputs.debug_shell == 'true' || false }}
     secrets:
       do_token: ${{ secrets.do_token }}

test-module.sh

@@ -1,30 +1,42 @@
 #!/bin/bash
 
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+set -e
+
 LEADER_NODE=$1
+IMAGE_URL=$2
+shift 2
 SSH_KEYFILE=${SSH_KEYFILE:-$HOME/.ssh/id_rsa}
 
-ssh_key="$(cat $SSH_KEYFILE)"
+ssh_key="$(< $SSH_KEYFILE)"
 
+cleanup() {
+    set +e
+    podman cp rf-core-runner:/home/pwuser/outputs tests/
+    podman stop rf-core-runner
+    podman rm rf-core-runner
+}
+
+trap cleanup EXIT
 podman run -i \
-    --volume=site-packages:/home/pwuser/.local/lib/python3.8/site-packages:Z \
+    --network=host \
     -v .:/home/pwuser/ns8-module:z \
-    --name rf-core-runner ghcr.io/marketsquare/robotframework-browser/rfbrowser-stable:v10.0.3 \
+    --volume=site-packages:/home/pwuser/.local/lib/python3.8/site-packages:Z \
+    --name rf-core-runner ghcr.io/marketsquare/robotframework-browser/rfbrowser-stable:18.6.3 \
     bash -l -s <<EOF
-    set -e
-    echo "$ssh_key" > /home/pwuser/ns8-key
-    set -x
-    pip install -r /home/pwuser/ns8-module/tests/pythonreq.txt
-    mkdir ~/outputs
-    cd /home/pwuser/ns8-module
-    robot -v NODE_ADDR:${LEADER_NODE} \
-        -v SSH_KEYFILE:/home/pwuser/ns8-key \
-	-d ~/outputs /home/pwuser/ns8-module/tests/
+set -e
+echo "$ssh_key" > /home/pwuser/ns8-key
+pip install -q -r /home/pwuser/ns8-module/tests/pythonreq.txt
+mkdir ~/outputs
+cd /home/pwuser/ns8-module
+exec robot -v NODE_ADDR:${LEADER_NODE} \
+    -v IMAGE_URL:${IMAGE_URL} \
+    -v SSH_KEYFILE:/home/pwuser/ns8-key \
+    --name loki \
+    --skiponfailure unstable \
+    -d ~/outputs ${@} /home/pwuser/ns8-module/tests/
 EOF
-
-tests_res=$?
-
-podman cp rf-core-runner:/home/pwuser/outputs tests/
-podman stop rf-core-runner
-podman rm rf-core-runner
-
-exit ${tests_res}

tests/loki.robot → tests/10__install.robot

@@ -2,20 +2,27 @@
 Library    SSHLibrary
 
 *** Test Cases ***
+Module installation
+    ${output}  ${rc} =    Execute Command    add-module ${IMAGE_URL} 1
+    ...    return_rc=True
+    Should Be Equal As Integers    ${rc}  0
+    &{output} =    Evaluate    ${output}
+    Set Global Variable    ${MID}    ${output.module_id}
+
 Check if loki service is loaded correctly
-    ${output}  ${rc} =    Execute Command    ssh -o StrictHostKeyChecking=no loki1@localhost systemctl --user show --property=LoadState loki
+    ${output}  ${rc} =    Execute Command    runagent -m ${MID} systemctl --user show --property=LoadState loki
     ...    return_rc=True
     Should Be Equal As Integers    ${rc}  0
     Should Be Equal As Strings    ${output}    LoadState=loaded
 
 Check if loki-server service is loaded correctly
-    ${output}  ${rc} =    Execute Command    ssh -o StrictHostKeyChecking=no loki1@localhost systemctl --user show --property=LoadState loki-server
+    ${output}  ${rc} =    Execute Command    runagent -m ${MID} systemctl --user show --property=LoadState loki-server
     ...    return_rc=True
     Should Be Equal As Integers    ${rc}  0
     Should Be Equal As Strings    ${output}    LoadState=loaded
 
 Check if internal traefik service is loaded correctly
-    ${output}  ${rc} =    Execute Command    ssh -o StrictHostKeyChecking=no loki1@localhost systemctl --user show --property=LoadState traefik
+    ${output}  ${rc} =    Execute Command    runagent -m ${MID} systemctl --user show --property=LoadState traefik
     ...    return_rc=True
     Should Be Equal As Integers    ${rc}  0
     Should Be Equal As Strings    ${output}    LoadState=loaded

tests/90__uninstall.robot

@@ -0,0 +1,12 @@
+*** Settings ***
+Library    SSHLibrary
+
+*** Variables ***
+${MID}
+
+*** Test Cases ***
+Module removal
+    [Tags]    module    remove
+    ${rc} =    Execute Command    remove-module --no-preserve ${MID}
+    ...    return_rc=True  return_stdout=False
+    Should Be Equal As Integers    ${rc}  0

tests/__init__.robot

@@ -1,15 +1,35 @@
 *** Settings ***
 Library           SSHLibrary
+Library           DateTime
 
 *** Variables ***
 ${SSH_KEYFILE}    %{HOME}/.ssh/id_ecdsa
+${NODE_ADDR}      127.0.0.1
+${MID}            loki0
+${IMAGE_URL}      ghcr.io/nethserver/loki:latest
 
 *** Keywords ***
 Connect to the node
     Open Connection   ${NODE_ADDR}
     Login With Public Key    root    ${SSH_KEYFILE}
-    ${output} =    Execute Command    systemctl is-system-running --wait
+
+Wait until boot completes
+    ${output} =    Execute Command    systemctl is-system-running  --wait
     Should Be True    '${output}' == 'running' or '${output}' == 'degraded'
 
+Save the journal begin timestamp
+    ${tsnow} =    Get Current Date    result_format=epoch
+    Set Global Variable    ${JOURNAL_SINCE}    ${tsnow}
+
+Collect the suite journal
+    Execute Command    journalctl -S @${JOURNAL_SINCE} >journal-dump.log
+    Get File    journal-dump.log    ${OUTPUT DIR}/journal-${SUITE NAME}.log
+
 *** Settings ***
-Suite Setup       Connect to the Node
+Suite Setup       Run Keywords
+                  ...    Connect to the Node
+                  ...    Wait until boot completes
+                  ...    Save the journal begin timestamp
+
+Suite Teardown    Run Keywords
+                  ...    Collect the suite journal

tests/pythonreq.txt

@@ -1,10 +1,2 @@
-robotframework==4.1.2
-robotframework-sshlibrary==3.8.0
-robotframework-requests==0.9.2
-bcrypt==3.2.0
-cffi==1.15.0
-cryptography==36.0.1
-paramiko==2.9.2
-pycparser==2.21
-pynacl==1.5.0
-scp==0.14.2
+robotframework
+robotframework-sshlibrary