diff --git a/testsuite/Rakefile b/testsuite/Rakefile index 1c7c5fae2637..fce4c0892f48 100644 --- a/testsuite/Rakefile +++ b/testsuite/Rakefile @@ -62,6 +62,7 @@ namespace :utils do task :generate_smoke_tests do clients = Rake::FileList['features/build_validation/init_clients/*.feature'] clients.reject! { |client| client.include? 'proxy' } + clients.reject! { |client| client.include? 'monitoring_server' } clients.reject! { |client| (client.include? 'buildhost') || (client.include? 'terminal') } clients.each do |client| Rake::Task['utils:generate_feature'].invoke('features/build_validation/smoke_tests/smoke_tests.template', @@ -76,6 +77,7 @@ namespace :utils do task :generate_add_maintenance_update_repository_features do clients = Rake::FileList['features/build_validation/init_clients/*.feature'] clients.reject! { |client| client.include? 'ssh'} + clients.reject! { |client| client.include? 'monitoring_server' } clients.each do |client| Rake::Task['utils:generate_feature'].invoke('features/build_validation/add_custom_repositories/add_maintenance_update_repository.template', File.basename(client,'.feature'), diff --git a/testsuite/documentation/cucumber-steps.md b/testsuite/documentation/cucumber-steps.md index e66ce2af5ea0..ff5fc10ff8c5 100644 --- a/testsuite/documentation/cucumber-steps.md +++ b/testsuite/documentation/cucumber-steps.md @@ -30,17 +30,18 @@ The corresponding notion in Cucumber steps is "step host names". Possible values are currently: -| Test host | Ruby target | Bash environment variable | Step host name | sumaform module | -| --------- | ----------- | -------------------------- | -------------- | --------------- | -| Uyuni server | ```$server``` | ```$SERVER``` | | ```"suse_manager"``` | -| Uyuni proxy | ```$proxy``` | ```$PROXY``` | ```"proxy"``` | ```"suse_manager_proxy"``` | -| SLES Salt minion | ```$minion``` | ```$MINION``` | ```"sle_minion"``` | ```"minion"``` | -| SLES Docker and Kiwi build host | ```$build_host``` | ```$BUILD_HOST``` | ```"build_host"``` | ```"minion"``` | -| SLES Salt SSH minion | ```$ssh_minion``` | ```$SSH_MINION``` | ```"ssh_minion"``` | ```"minion"``` | -| Red Hat-like Salt minion | ```$rhlike_minion``` | ```$RHLIKE_MINION``` | ```"rhlike_minion"``` | ```"minion"``` | -| Debian-like Salt minion | ```$deblike_minion``` | ```$DEBLIKE_MINION``` | ```"deblike_minion"``` | ```"minion"``` | -| PXE-boot minion | None | ```$PXEBOOT_MAC``` | ```"pxeboot_minion"``` | ```"pxeboot"``` | -| KVM virtual host minion | ```$kvm_server``` | ```$VIRTHOST_KVM_URL``` and ```$VIRTHOST_KVM_PASSWORD``` | ```"kvm_server"``` | ```"virthost"``` | +| Test host | Ruby target | Bash environment variable | Step host name | sumaform module | +|---------------------------------|--------------------------|----------------------------------------------------------|--------------------------|----------------------------| +| Uyuni server | ```$server``` | ```$SERVER``` | | ```"suse_manager"``` | +| Uyuni proxy | ```$proxy``` | ```$PROXY``` | ```"proxy"``` | ```"suse_manager_proxy"``` | +| SLES Salt minion | ```$minion``` | ```$MINION``` | ```"sle_minion"``` | ```"minion"``` | +| SLES Docker and Kiwi build host | ```$build_host``` | ```$BUILD_HOST``` | ```"build_host"``` | ```"build_host"``` | +| Monitoring Server | ```$monitoring_server``` | ```$MONITORING_SERVER``` | ```"monitoring_server``` | ```"minion"``` | +| SLES Salt SSH minion | ```$ssh_minion``` | ```$SSH_MINION``` | ```"ssh_minion"``` | ```"minion"``` | +| Red Hat-like Salt minion | ```$rhlike_minion``` | ```$RHLIKE_MINION``` | ```"rhlike_minion"``` | ```"minion"``` | +| Debian-like Salt minion | ```$deblike_minion``` | ```$DEBLIKE_MINION``` | ```"deblike_minion"``` | ```"minion"``` | +| PXE-boot minion | None | ```$PXEBOOT_MAC``` | ```"pxeboot_minion"``` | ```"pxeboot"``` | +| KVM virtual host minion | ```$kvm_server``` | ```$VIRTHOST_KVM_URL``` and ```$VIRTHOST_KVM_PASSWORD``` | ```"kvm_server"``` | ```"virthost"``` | These names are such for historical reasons and might be made better in the future. diff --git a/testsuite/features/build_validation/core/allcli_sanity.feature b/testsuite/features/build_validation/core/allcli_sanity.feature index 4a9a5d7626bb..b7c8880612ee 100644 --- a/testsuite/features/build_validation/core/allcli_sanity.feature +++ b/testsuite/features/build_validation/core/allcli_sanity.feature @@ -284,3 +284,10 @@ Feature: Sanity checks And reverse resolution should work for "sle15sp4_buildhost" And "sle15sp4_buildhost" should communicate with the server using public interface And the clock from "sle15sp4_buildhost" should be exact + +@monitoring_server + Scenario: The monitoring server is healthy + Then "monitoring_server" should have a FQDN + And reverse resolution should work for "monitoring_server" + And "monitoring_server" should communicate with the server using public interface + And the clock from "monitoring_server" should be exact diff --git a/testsuite/features/build_validation/create_bootstrap_repositories/srv_create_bootstrap_repositories.feature b/testsuite/features/build_validation/create_bootstrap_repositories/srv_create_bootstrap_repositories.feature index c6a51f510adf..1f475e26b62c 100644 --- a/testsuite/features/build_validation/create_bootstrap_repositories/srv_create_bootstrap_repositories.feature +++ b/testsuite/features/build_validation/create_bootstrap_repositories/srv_create_bootstrap_repositories.feature @@ -38,6 +38,10 @@ Feature: Create bootstrap repositories Scenario: Create the bootstrap repository for a SLES 15 SP4 minion When I create the bootstrap repository for "sle15sp4_minion" on the server +@monitoring_server + Scenario: Create the bootstrap repository for the monitoring server + When I create the bootstrap repository for "monitoring_server" on the server + @centos7_minion Scenario: Create the bootstrap repository for a CentOS 7 Salt minion When I create the bootstrap repository for "centos7_minion" on the server diff --git a/testsuite/features/build_validation/init_clients/monitoring_server.feature b/testsuite/features/build_validation/init_clients/monitoring_server.feature new file mode 100644 index 000000000000..0a1bf5340ff0 --- /dev/null +++ b/testsuite/features/build_validation/init_clients/monitoring_server.feature @@ -0,0 +1,94 @@ +# Copyright (c) 2022 SUSE LLC +# Licensed under the terms of the MIT license. + +@monitoring_server +Feature: Bootstrap the monitoring server + + Scenario: Clean up sumaform leftovers on the monitoring server + When I perform a full salt minion cleanup on "monitoring_server" + + Scenario: Log in as admin user + Given I am authorized for the "Admin" section + + Scenario: Bootstrap the monitoring server + When I follow the left menu "Systems > Bootstrapping" + Then I should see a "Bootstrap Minions" text + When I enter the hostname of "monitoring_server" as "hostname" + And I enter "22" as "port" + And I enter "root" as "user" + And I enter "linux" as "password" + And I select "1-monitoring_server_key" from "activationKeys" + And I select the hostname of "proxy" from "proxies" if present + And I click on "Bootstrap" + And I wait until I see "Successfully bootstrapped host!" text + And I wait until onboarding is completed for "monitoring_server" + + Scenario: Check the new bootstrapped monitoring server in System Overview page + When I follow the left menu "Salt > Keys" + Then I should see a "accepted" text + And the Salt master can reach "monitoring_server" + +@proxy + Scenario: Check connection from monitoring server to proxy + Given I am on the Systems overview page of this "monitoring_server" + When I follow "Details" in the content area + And I follow "Connection" in the content area + Then I should see "proxy" short hostname + +@proxy + Scenario: Check registration on proxy of monitoring server + Given I am on the Systems overview page of this "proxy" + When I follow "Details" in the content area + And I follow "Proxy" in the content area + Then I should see "monitoring_server" hostname + + Scenario: Check events history for failures on monitoring server + Given I am on the Systems overview page of this "monitoring_server" + Then I check for failed events on history event page + + Scenario: Test Prometheus formula on monitoring server + Given I am on the Systems overview page of this "monitoring_server" + When I follow "Formulas" in the content area + And I check the "prometheus" formula + And I click on "Save" + Then I wait until I see "Formula saved" text + When I follow "Prometheus" in the content area + And I click on "Expand All Sections" + And I enter "admin" as "Username" + And I enter "admin" as "Password" + And I click on "Save Formula" + Then I should see a "Formula saved" text + When I follow "States" in the content area + And I click on "Apply Highstate" + Then I should see a "Applying the highstate has been scheduled." text + And I wait until event "Apply highstate scheduled by admin" is completed + # Visit monitoring endpoints on the minion + When I wait until "prometheus" service is active on "monitoring_server" + And I visit "Prometheus" endpoint of this "monitoring_server" + + Scenario: Test Grafana formula on monitoring server + Given I am on the Systems overview page of this "monitoring_server" + When I follow "Formulas" in the content area + And I check the "grafana" formula + And I click on "Save" + Then I wait until I see "Formula saved" text + When I follow "Grafana" in the content area + And I click on "Expand All Sections" + And I enter the "monitoring_server" hostname as the Prometheus URL + And I click on "Save Formula" + When I follow "States" in the content area + And I click on "Apply Highstate" + Then I should see a "Applying the highstate has been scheduled." text + And I wait until event "Apply highstate scheduled by admin" is completed + # Visit monitoring endpoints on the minion + When I wait until "grafana-server" service is active on "monitoring_server" + And I visit "Grafana" endpoint of this "monitoring_server" + + Scenario: Test Grafana dashboards of monitoring server + When I visit the grafana dashboards of this "monitoring_server" + And I wait until I do not see "Loading Grafana" text + # These are the 4 dashboards created by default when enabling the Grafana formula + Then I should see a "Apache2" text + And I should see a "PostgreSQL database insights" text + And I should see a "Client Systems" text + And I should see a "Server" text diff --git a/testsuite/features/build_validation/smoke_tests/smoke_tests.template b/testsuite/features/build_validation/smoke_tests/smoke_tests.template index a2894396361a..86a3bbaa84f1 100644 --- a/testsuite/features/build_validation/smoke_tests/smoke_tests.template +++ b/testsuite/features/build_validation/smoke_tests/smoke_tests.template @@ -137,27 +137,6 @@ Feature: Smoke tests for Then I should see a "1 package install has been scheduled for" text And I wait until event "Package Install/Upgrade scheduled by admin" is completed -@suse_minion - Scenario: Test Prometheus formula on the - Given I am on the Systems overview page of this "" - When I follow "Formulas" in the content area - And I check the "prometheus" formula - And I click on "Save" - Then I wait until I see "Formula saved" text - When I follow "Prometheus" in the content area - And I click on "Expand All Sections" - And I enter "admin" as "Username" - And I enter "admin" as "Password" - And I click on "Save Formula" - Then I should see a "Formula saved" text - When I follow "States" in the content area - And I click on "Apply Highstate" - Then I should see a "Applying the highstate has been scheduled." text - And I wait until event "Apply highstate scheduled by admin" is completed - # Visit monitoring endpoints on the minion - When I wait until "prometheus" service is active on "" - And I visit "Prometheus" endpoint of this "" - Scenario: Test Prometheus exporter formula on the Given I am on the Systems overview page of this "" When I follow "Formulas" in the content area diff --git a/testsuite/features/step_definitions/navigation_steps.rb b/testsuite/features/step_definitions/navigation_steps.rb index 67994d5cf899..a1d9a4dd0add 100644 --- a/testsuite/features/step_definitions/navigation_steps.rb +++ b/testsuite/features/step_definitions/navigation_steps.rb @@ -999,15 +999,27 @@ When(/^I visit "([^"]*)" endpoint of this "([^"]*)"$/) do |service, host| node = get_target(host) system_name = get_system_name(host) + os_family = node.os_family port, protocol, path, text = case service when 'Proxy' then [443, 'https', '/pub/', 'Index of /pub'] when 'Prometheus' then [9090, 'http', '', 'graph'] when 'Prometheus node exporter' then [9100, 'http', '', 'Node Exporter'] when 'Prometheus apache exporter' then [9117, 'http', '', 'Apache Exporter'] when 'Prometheus postgres exporter' then [9187, 'http', '', 'Postgres Exporter'] + when 'Grafana' then [3000, 'http', '', 'Grafana Labs'] else raise "Unknown port for service #{service}" end - node.run_until_ok("curl -s -k #{protocol}://#{system_name}:#{port}#{path} | grep -i '#{text}'") + # debian based systems don't come with curl installed + if (os_family.include? 'debian') || (os_family.include? 'ubuntu') + node.run_until_ok("wget --no-check-certificate -qO- #{protocol}://#{system_name}:#{port}#{path} | grep -i '#{text}'") + else + node.run_until_ok("curl -s -k #{protocol}://#{system_name}:#{port}#{path} | grep -i '#{text}'") + end +end + +When(/^I enter the "([^"]*)" hostname as the Prometheus URL$/) do |host| + node = get_target(host) + step %(I enter "http://#{node.full_hostname}:9090" as "Prometheus URL") end When(/^I select the next maintenance window$/) do @@ -1130,3 +1142,8 @@ end end end + +When(/^I visit the grafana dashboards of this "([^"]*)"$/) do |host| + node = get_target(host) + visit("http://#{node.public_ip}:3000/dashboards") +end diff --git a/testsuite/features/support/constants.rb b/testsuite/features/support/constants.rb index 7780ec7be4b7..92593f47f852 100644 --- a/testsuite/features/support/constants.rb +++ b/testsuite/features/support/constants.rb @@ -131,6 +131,7 @@ 'sle15sp4_minion' => 'SLES15-SP4-Pool', 'sle15sp4_ssh_minion' => 'SLES15-SP4-Pool', 'sle15sp4_buildhost' => 'SLES15-SP4-Pool', + 'monitoring_server' => 'SLES15-SP4-Pool', 'sle15sp4_terminal' => 'SLES15-SP4-Pool', 'centos7_minion' => 'RHEL x86_64 Server 7', 'centos7_ssh_minion' => 'RHEL x86_64 Server 7', diff --git a/testsuite/features/support/env.rb b/testsuite/features/support/env.rb index bed5d6cbd3ba..de6baed93782 100644 --- a/testsuite/features/support/env.rb +++ b/testsuite/features/support/env.rb @@ -320,6 +320,10 @@ def enable_assertions skip_this_scenario unless $sle15sp4_buildhost end +Before('@monitoring_server') do + skip_this_scenario unless $monitoring_server +end + Before('@sle15sp4_terminal') do skip_this_scenario unless $sle15sp4_terminal_mac end diff --git a/testsuite/features/support/twopence_init.rb b/testsuite/features/support/twopence_init.rb index b149bb3cb3c9..4714ff7ef5ab 100644 --- a/testsuite/features/support/twopence_init.rb +++ b/testsuite/features/support/twopence_init.rb @@ -72,6 +72,7 @@ def twopence_init(target) $opensuse154arm_ssh_minion = twopence_init("ssh:#{ENV['OPENSUSE154ARM_SSHMINION']}") if ENV['OPENSUSE154ARM_SSHMINION'] $sle12sp5_buildhost = twopence_init("ssh:#{ENV['SLE12SP5_BUILDHOST']}") if ENV['SLE12SP5_BUILDHOST'] $sle15sp4_buildhost = twopence_init("ssh:#{ENV['SLE15SP4_BUILDHOST']}") if ENV['SLE15SP4_BUILDHOST'] + $monitoring_server = twopence_init("ssh:#{ENV['MONITORING_SERVER']}") if ENV['MONITORING_SERVER'] $nodes += [$sle12sp4_minion, $sle12sp4_ssh_minion, $sle12sp5_minion, $sle12sp5_ssh_minion, $sle15_minion, $sle15_ssh_minion, @@ -90,7 +91,8 @@ def twopence_init(target) $debian11_minion, $debian11_ssh_minion, $opensuse154arm_minion, $opensuse154arm_ssh_minion, $sle12sp5_buildhost, - $sle15sp4_buildhost] + $sle15sp4_buildhost, + $monitoring_server] else # Define twopence objects for QA environment $minion = twopence_init("ssh:#{ENV['MINION']}") if ENV['MINION'] @@ -295,7 +297,8 @@ def file_inject(node, local_file, remote_file) 'opensuse154arm_minion' => $opensuse154arm_minion, 'opensuse154arm_ssh_minion' => $opensuse154arm_ssh_minion, 'sle12sp5_buildhost' => $sle12sp5_buildhost, - 'sle15sp4_buildhost' => $sle15sp4_buildhost } + 'sle15sp4_buildhost' => $sle15sp4_buildhost, + 'monitoring_server' => $monitoring_server } # This is the inverse of `node_by_host`. $host_by_node = {} diff --git a/testsuite/run_sets/build_validation_init_monitoring.yml b/testsuite/run_sets/build_validation_init_monitoring.yml new file mode 100644 index 000000000000..b1691ea502d5 --- /dev/null +++ b/testsuite/run_sets/build_validation_init_monitoring.yml @@ -0,0 +1,7 @@ +# This file describes the order of features in a Build Validation test suite run. + +## Init monitoring server BEGIN ### + +- features/build_validation/init_clients/monitoring_server.feature + +## Init monitoring server END ###