From 03728976005f6851e64e20f064eed8c4540817e5 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Wed, 11 Sep 2024 11:02:10 +0200 Subject: [PATCH 1/3] Use Compactor for Loki 2.4+ Upgrade our configuration according to Loki 2.4.0 release notes. The table-manager no longer runs by default within the single binary mode and can be removed from config. --- imageroot/loki-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/imageroot/loki-config.yaml b/imageroot/loki-config.yaml index e254ea6..e34fee2 100644 --- a/imageroot/loki-config.yaml +++ b/imageroot/loki-config.yaml @@ -41,18 +41,18 @@ storage_config: compactor: working_directory: /loki/boltdb-shipper-compactor shared_store: filesystem + retention_enabled: true + retention_delete_delay: 30m + delete_request_store: filesystem limits_config: reject_old_samples: true reject_old_samples_max_age: 168h + retention_period: ${LOKI_RETENTION_PERIOD:-365}d chunk_store_config: max_look_back_period: 0s -table_manager: - retention_deletes_enabled: false - retention_period: ${LOKI_RETENTION_PERIOD:-365}d - ruler: storage: type: local From 3a48d2c5feea662fb81d65487207fd7d9bb91ad5 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Wed, 11 Sep 2024 11:53:20 +0200 Subject: [PATCH 2/3] Bump Loki 2.9.10 We need to raise Loki version up to 2.8.3+ to get the fix for the `json` filter in LogQL, used by api-server log queries. See grafana/loki PR 3983. --- build-images.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-images.sh b/build-images.sh index 5ef7ffd..dba8416 100755 --- a/build-images.sh +++ b/build-images.sh @@ -10,7 +10,7 @@ container=$(buildah from scratch) buildah add "${container}" imageroot /imageroot buildah add "${container}" ui /ui buildah config --entrypoint=/ \ - --label="org.nethserver.images=docker.io/traefik:v2.11.6 docker.io/grafana/loki:2.7.5" \ + --label="org.nethserver.images=docker.io/traefik:v2.11.6 docker.io/grafana/loki:2.9.10" \ --label="org.nethserver.tcp-ports-demand=1" \ --label='org.nethserver.flags=core_module' \ "${container}" From b20cbfbd7ebeb53f06315f86c0f2cc8afc6a94b1 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Wed, 11 Sep 2024 12:43:51 +0200 Subject: [PATCH 3/3] Fix test suite --- .github/workflows/test-module.yml | 10 ++++- test-module.sh | 52 +++++++++++++++---------- tests/{loki.robot => 10__install.robot} | 13 +++++-- tests/90__uninstall.robot | 12 ++++++ tests/__init__.robot | 24 +++++++++++- tests/pythonreq.txt | 12 +----- 6 files changed, 86 insertions(+), 37 deletions(-) rename tests/{loki.robot => 10__install.robot} (50%) create mode 100644 tests/90__uninstall.robot diff --git a/.github/workflows/test-module.yml b/.github/workflows/test-module.yml index a60dac0..91c4413 100644 --- a/.github/workflows/test-module.yml +++ b/.github/workflows/test-module.yml @@ -2,6 +2,11 @@ name: Test module on: workflow_dispatch: + inputs: + debug_shell: + description: "Debug shell" + required: true + type: boolean workflow_run: workflows: ["Publish images"] types: [completed] @@ -10,11 +15,12 @@ jobs: module: if: ${{ github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == '' }} uses: NethServer/ns8-github-actions/.github/workflows/module-info.yml@main - run_tests_on_do: + run_tests: needs: module uses: NethServer/ns8-github-actions/.github/workflows/test-on-digitalocean-infra.yml@main with: - coremodules: "ghcr.io/${{needs.module.outputs.owner}}/${{needs.module.outputs.name}}:${{needs.module.outputs.tag}}" + args: "ghcr.io/${{needs.module.outputs.owner}}/${{needs.module.outputs.name}}:${{needs.module.outputs.tag}}" repo_ref: ${{needs.module.outputs.sha}} + debug_shell: ${{ github.event.inputs.debug_shell == 'true' || false }} secrets: do_token: ${{ secrets.do_token }} diff --git a/test-module.sh b/test-module.sh index ed9df60..026b36a 100755 --- a/test-module.sh +++ b/test-module.sh @@ -1,30 +1,42 @@ #!/bin/bash +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +set -e + LEADER_NODE=$1 +IMAGE_URL=$2 +shift 2 SSH_KEYFILE=${SSH_KEYFILE:-$HOME/.ssh/id_rsa} -ssh_key="$(cat $SSH_KEYFILE)" +ssh_key="$(< $SSH_KEYFILE)" +cleanup() { + set +e + podman cp rf-core-runner:/home/pwuser/outputs tests/ + podman stop rf-core-runner + podman rm rf-core-runner +} + +trap cleanup EXIT podman run -i \ - --volume=site-packages:/home/pwuser/.local/lib/python3.8/site-packages:Z \ + --network=host \ -v .:/home/pwuser/ns8-module:z \ - --name rf-core-runner ghcr.io/marketsquare/robotframework-browser/rfbrowser-stable:v10.0.3 \ + --volume=site-packages:/home/pwuser/.local/lib/python3.8/site-packages:Z \ + --name rf-core-runner ghcr.io/marketsquare/robotframework-browser/rfbrowser-stable:18.6.3 \ bash -l -s < /home/pwuser/ns8-key - set -x - pip install -r /home/pwuser/ns8-module/tests/pythonreq.txt - mkdir ~/outputs - cd /home/pwuser/ns8-module - robot -v NODE_ADDR:${LEADER_NODE} \ - -v SSH_KEYFILE:/home/pwuser/ns8-key \ - -d ~/outputs /home/pwuser/ns8-module/tests/ +set -e +echo "$ssh_key" > /home/pwuser/ns8-key +pip install -q -r /home/pwuser/ns8-module/tests/pythonreq.txt +mkdir ~/outputs +cd /home/pwuser/ns8-module +exec robot -v NODE_ADDR:${LEADER_NODE} \ + -v IMAGE_URL:${IMAGE_URL} \ + -v SSH_KEYFILE:/home/pwuser/ns8-key \ + --name loki \ + --skiponfailure unstable \ + -d ~/outputs ${@} /home/pwuser/ns8-module/tests/ EOF - -tests_res=$? - -podman cp rf-core-runner:/home/pwuser/outputs tests/ -podman stop rf-core-runner -podman rm rf-core-runner - -exit ${tests_res} diff --git a/tests/loki.robot b/tests/10__install.robot similarity index 50% rename from tests/loki.robot rename to tests/10__install.robot index e8b043b..3354181 100644 --- a/tests/loki.robot +++ b/tests/10__install.robot @@ -2,20 +2,27 @@ Library SSHLibrary *** Test Cases *** +Module installation + ${output} ${rc} = Execute Command add-module ${IMAGE_URL} 1 + ... return_rc=True + Should Be Equal As Integers ${rc} 0 + &{output} = Evaluate ${output} + Set Global Variable ${MID} ${output.module_id} + Check if loki service is loaded correctly - ${output} ${rc} = Execute Command ssh -o StrictHostKeyChecking=no loki1@localhost systemctl --user show --property=LoadState loki + ${output} ${rc} = Execute Command runagent -m ${MID} systemctl --user show --property=LoadState loki ... return_rc=True Should Be Equal As Integers ${rc} 0 Should Be Equal As Strings ${output} LoadState=loaded Check if loki-server service is loaded correctly - ${output} ${rc} = Execute Command ssh -o StrictHostKeyChecking=no loki1@localhost systemctl --user show --property=LoadState loki-server + ${output} ${rc} = Execute Command runagent -m ${MID} systemctl --user show --property=LoadState loki-server ... return_rc=True Should Be Equal As Integers ${rc} 0 Should Be Equal As Strings ${output} LoadState=loaded Check if internal traefik service is loaded correctly - ${output} ${rc} = Execute Command ssh -o StrictHostKeyChecking=no loki1@localhost systemctl --user show --property=LoadState traefik + ${output} ${rc} = Execute Command runagent -m ${MID} systemctl --user show --property=LoadState traefik ... return_rc=True Should Be Equal As Integers ${rc} 0 Should Be Equal As Strings ${output} LoadState=loaded diff --git a/tests/90__uninstall.robot b/tests/90__uninstall.robot new file mode 100644 index 0000000..3a16fd4 --- /dev/null +++ b/tests/90__uninstall.robot @@ -0,0 +1,12 @@ +*** Settings *** +Library SSHLibrary + +*** Variables *** +${MID} + +*** Test Cases *** +Module removal + [Tags] module remove + ${rc} = Execute Command remove-module --no-preserve ${MID} + ... return_rc=True return_stdout=False + Should Be Equal As Integers ${rc} 0 diff --git a/tests/__init__.robot b/tests/__init__.robot index 72d73ec..7a7f02f 100644 --- a/tests/__init__.robot +++ b/tests/__init__.robot @@ -1,15 +1,35 @@ *** Settings *** Library SSHLibrary +Library DateTime *** Variables *** ${SSH_KEYFILE} %{HOME}/.ssh/id_ecdsa +${NODE_ADDR} 127.0.0.1 +${MID} loki0 +${IMAGE_URL} ghcr.io/nethserver/loki:latest *** Keywords *** Connect to the node Open Connection ${NODE_ADDR} Login With Public Key root ${SSH_KEYFILE} - ${output} = Execute Command systemctl is-system-running --wait + +Wait until boot completes + ${output} = Execute Command systemctl is-system-running --wait Should Be True '${output}' == 'running' or '${output}' == 'degraded' +Save the journal begin timestamp + ${tsnow} = Get Current Date result_format=epoch + Set Global Variable ${JOURNAL_SINCE} ${tsnow} + +Collect the suite journal + Execute Command journalctl -S @${JOURNAL_SINCE} >journal-dump.log + Get File journal-dump.log ${OUTPUT DIR}/journal-${SUITE NAME}.log + *** Settings *** -Suite Setup Connect to the Node +Suite Setup Run Keywords + ... Connect to the Node + ... Wait until boot completes + ... Save the journal begin timestamp + +Suite Teardown Run Keywords + ... Collect the suite journal diff --git a/tests/pythonreq.txt b/tests/pythonreq.txt index 14326fb..af1d9bf 100644 --- a/tests/pythonreq.txt +++ b/tests/pythonreq.txt @@ -1,10 +1,2 @@ -robotframework==4.1.2 -robotframework-sshlibrary==3.8.0 -robotframework-requests==0.9.2 -bcrypt==3.2.0 -cffi==1.15.0 -cryptography==36.0.1 -paramiko==2.9.2 -pycparser==2.21 -pynacl==1.5.0 -scp==0.14.2 +robotframework +robotframework-sshlibrary