QE: Add smoke tests for Grafana in BV

testsuite/Rakefile

@@ -62,6 +62,7 @@ namespace :utils do
  task :generate_smoke_tests do
  clients = Rake::FileList['features/build_validation/init_clients/*.feature']
  clients.reject! { |client| client.include? 'proxy' }
+ clients.reject! { |client| client.include? 'monitoring_server' }
  clients.reject! { |client| (client.include? 'buildhost') || (client.include? 'terminal') }
  clients.each do |client|
  Rake::Task['utils:generate_feature'].invoke('features/build_validation/smoke_tests/smoke_tests.template',
@@ -76,6 +77,7 @@ namespace :utils do
  task :generate_add_maintenance_update_repository_features do
  clients = Rake::FileList['features/build_validation/init_clients/*.feature']
  clients.reject! { |client| client.include? 'ssh'}
+ clients.reject! { |client| client.include? 'monitoring_server' }
  clients.each do |client|
  Rake::Task['utils:generate_feature'].invoke('features/build_validation/add_custom_repositories/add_maintenance_update_repository.template',
  File.basename(client,'.feature'),

testsuite/documentation/cucumber-steps.md

@@ -30,17 +30,18 @@ The corresponding notion in Cucumber steps is "step host names".
 
 Possible values are currently:
 
-| Test host | Ruby target | Bash environment variable | Step host name | sumaform module |
-| --------- | ----------- | -------------------------- | -------------- | --------------- |
-| Uyuni server | ```$server``` | ```$SERVER``` | | ```"suse_manager"``` |
-| Uyuni proxy | ```$proxy``` | ```$PROXY``` | ```"proxy"``` | ```"suse_manager_proxy"``` |
-| SLES Salt minion | ```$minion``` | ```$MINION``` | ```"sle_minion"``` | ```"minion"``` |
-| SLES Docker and Kiwi build host | ```$build_host``` | ```$BUILD_HOST``` | ```"build_host"``` | ```"minion"``` |
-| SLES Salt SSH minion | ```$ssh_minion``` | ```$SSH_MINION``` | ```"ssh_minion"``` | ```"minion"``` |
-| Red Hat-like Salt minion | ```$rhlike_minion``` | ```$RHLIKE_MINION``` | ```"rhlike_minion"``` | ```"minion"``` |
-| Debian-like Salt minion | ```$deblike_minion``` | ```$DEBLIKE_MINION``` | ```"deblike_minion"``` | ```"minion"``` |
-| PXE-boot minion | None | ```$PXEBOOT_MAC``` | ```"pxeboot_minion"``` | ```"pxeboot"``` |
-| KVM virtual host minion | ```$kvm_server``` | ```$VIRTHOST_KVM_URL``` and ```$VIRTHOST_KVM_PASSWORD``` | ```"kvm_server"``` | ```"virthost"``` |
+| Test host | Ruby target | Bash environment variable | Step host name | sumaform module |
+|---------------------------------|--------------------------|----------------------------------------------------------|--------------------------|----------------------------|
+| Uyuni server | ```$server``` | ```$SERVER``` | | ```"suse_manager"``` |
+| Uyuni proxy | ```$proxy``` | ```$PROXY``` | ```"proxy"``` | ```"suse_manager_proxy"``` |
+| SLES Salt minion | ```$minion``` | ```$MINION``` | ```"sle_minion"``` | ```"minion"``` |
+| SLES Docker and Kiwi build host | ```$build_host``` | ```$BUILD_HOST``` | ```"build_host"``` | ```"build_host"``` |
+| Monitoring Server | ```$monitoring_server``` | ```$MONITORING_SERVER``` | ```"monitoring_server``` | ```"minion"``` |
+| SLES Salt SSH minion | ```$ssh_minion``` | ```$SSH_MINION``` | ```"ssh_minion"``` | ```"minion"``` |
+| Red Hat-like Salt minion | ```$rhlike_minion``` | ```$RHLIKE_MINION``` | ```"rhlike_minion"``` | ```"minion"``` |
+| Debian-like Salt minion | ```$deblike_minion``` | ```$DEBLIKE_MINION``` | ```"deblike_minion"``` | ```"minion"``` |
+| PXE-boot minion | None | ```$PXEBOOT_MAC``` | ```"pxeboot_minion"``` | ```"pxeboot"``` |
+| KVM virtual host minion | ```$kvm_server``` | ```$VIRTHOST_KVM_URL``` and ```$VIRTHOST_KVM_PASSWORD``` | ```"kvm_server"``` | ```"virthost"``` |
 
 These names are such for historical reasons and might be made better in the future.
 

testsuite/features/build_validation/core/allcli_sanity.feature

@@ -284,3 +284,10 @@ Feature: Sanity checks
  And reverse resolution should work for "sle15sp4_buildhost"
  And "sle15sp4_buildhost" should communicate with the server using public interface
  And the clock from "sle15sp4_buildhost" should be exact
+
+@monitoring_server
+ Scenario: The monitoring server is healthy
+ Then "monitoring_server" should have a FQDN
+ And reverse resolution should work for "monitoring_server"
+ And "monitoring_server" should communicate with the server using public interface
+ And the clock from "monitoring_server" should be exact

testsuite/features/build_validation/create_bootstrap_repositories/srv_create_bootstrap_repositories.feature

@@ -38,6 +38,10 @@ Feature: Create bootstrap repositories
  Scenario: Create the bootstrap repository for a SLES 15 SP4 minion
  When I create the bootstrap repository for "sle15sp4_minion" on the server
 
+@monitoring_server
+ Scenario: Create the bootstrap repository for the monitoring server
+ When I create the bootstrap repository for "monitoring_server" on the server
+
 @centos7_minion
  Scenario: Create the bootstrap repository for a CentOS 7 Salt minion
  When I create the bootstrap repository for "centos7_minion" on the server

testsuite/features/build_validation/init_clients/monitoring_server.feature

@@ -0,0 +1,94 @@
+# Copyright (c) 2022 SUSE LLC
+# Licensed under the terms of the MIT license.
+
+@monitoring_server
+Feature: Bootstrap the monitoring server
+
+ Scenario: Clean up sumaform leftovers on the monitoring server
+ When I perform a full salt minion cleanup on "monitoring_server"
+
+ Scenario: Log in as admin user
+ Given I am authorized for the "Admin" section
+
+ Scenario: Bootstrap the monitoring server
+ When I follow the left menu "Systems > Bootstrapping"
+ Then I should see a "Bootstrap Minions" text
+ When I enter the hostname of "monitoring_server" as "hostname"
+ And I enter "22" as "port"
+ And I enter "root" as "user"
+ And I enter "linux" as "password"
+ And I select "1-monitoring_server_key" from "activationKeys"
+ And I select the hostname of "proxy" from "proxies" if present
+ And I click on "Bootstrap"
+ And I wait until I see "Successfully bootstrapped host!" text
+ And I wait until onboarding is completed for "monitoring_server"
+
+ Scenario: Check the new bootstrapped monitoring server in System Overview page
+ When I follow the left menu "Salt > Keys"
+ Then I should see a "accepted" text
+ And the Salt master can reach "monitoring_server"
+
+@proxy
+ Scenario: Check connection from monitoring server to proxy
+ Given I am on the Systems overview page of this "monitoring_server"
+ When I follow "Details" in the content area
+ And I follow "Connection" in the content area
+ Then I should see "proxy" short hostname
+
+@proxy
+ Scenario: Check registration on proxy of monitoring server
+ Given I am on the Systems overview page of this "proxy"
+ When I follow "Details" in the content area
+ And I follow "Proxy" in the content area
+ Then I should see "monitoring_server" hostname
+
+ Scenario: Check events history for failures on monitoring server
+ Given I am on the Systems overview page of this "monitoring_server"
+ Then I check for failed events on history event page
+
+ Scenario: Test Prometheus formula on monitoring server
+ Given I am on the Systems overview page of this "monitoring_server"
+ When I follow "Formulas" in the content area
+ And I check the "prometheus" formula
+ And I click on "Save"
+ Then I wait until I see "Formula saved" text
+ When I follow "Prometheus" in the content area
+ And I click on "Expand All Sections"
+ And I enter "admin" as "Username"
+ And I enter "admin" as "Password"
+ And I click on "Save Formula"
+ Then I should see a "Formula saved" text
+ When I follow "States" in the content area
+ And I click on "Apply Highstate"
+ Then I should see a "Applying the highstate has been scheduled." text
+ And I wait until event "Apply highstate scheduled by admin" is completed
+ # Visit monitoring endpoints on the minion
+ When I wait until "prometheus" service is active on "monitoring_server"
+ And I visit "Prometheus" endpoint of this "monitoring_server"
+
+ Scenario: Test Grafana formula on monitoring server
+ Given I am on the Systems overview page of this "monitoring_server"
+ When I follow "Formulas" in the content area
+ And I check the "grafana" formula
+ And I click on "Save"
+ Then I wait until I see "Formula saved" text
+ When I follow "Grafana" in the content area
+ And I click on "Expand All Sections"
+ And I enter the "monitoring_server" hostname as the Prometheus URL
+ And I click on "Save Formula"
+ When I follow "States" in the content area
+ And I click on "Apply Highstate"
+ Then I should see a "Applying the highstate has been scheduled." text
+ And I wait until event "Apply highstate scheduled by admin" is completed
+ # Visit monitoring endpoints on the minion
+ When I wait until "grafana-server" service is active on "monitoring_server"
+ And I visit "Grafana" endpoint of this "monitoring_server"
+
+ Scenario: Test Grafana dashboards of monitoring server
+ When I visit the grafana dashboards of this "monitoring_server"
+ And I wait until I do not see "Loading Grafana" text
+ # These are the 4 dashboards created by default when enabling the Grafana formula
+ Then I should see a "Apache2" text
+ And I should see a "PostgreSQL database insights" text
+ And I should see a "Client Systems" text
+ And I should see a "Server" text

testsuite/features/build_validation/smoke_tests/smoke_tests.template

@@ -137,27 +137,6 @@ Feature: Smoke tests for <client>
  Then I should see a "1 package install has been scheduled for" text
  And I wait until event "Package Install/Upgrade scheduled by admin" is completed
 
-@suse_minion
- Scenario: Test Prometheus formula on the <client>
- Given I am on the Systems overview page of this "<client>"
- When I follow "Formulas" in the content area
- And I check the "prometheus" formula
- And I click on "Save"
- Then I wait until I see "Formula saved" text
- When I follow "Prometheus" in the content area
- And I click on "Expand All Sections"
- And I enter "admin" as "Username"
- And I enter "admin" as "Password"
- And I click on "Save Formula"
- Then I should see a "Formula saved" text
- When I follow "States" in the content area
- And I click on "Apply Highstate"
- Then I should see a "Applying the highstate has been scheduled." text
- And I wait until event "Apply highstate scheduled by admin" is completed
- # Visit monitoring endpoints on the minion
- When I wait until "prometheus" service is active on "<client>"
- And I visit "Prometheus" endpoint of this "<client>"
-
  Scenario: Test Prometheus exporter formula on the <client>
  Given I am on the Systems overview page of this "<client>"
  When I follow "Formulas" in the content area

testsuite/features/step_definitions/navigation_steps.rb

@@ -999,15 +999,27 @@
 When(/^I visit "([^"]*)" endpoint of this "([^"]*)"$/) do |service, host|
  node = get_target(host)
  system_name = get_system_name(host)
+ os_family = node.os_family
  port, protocol, path, text = case service
  when 'Proxy' then [443, 'https', '/pub/', 'Index of /pub']
  when 'Prometheus' then [9090, 'http', '', 'graph']
  when 'Prometheus node exporter' then [9100, 'http', '', 'Node Exporter']
  when 'Prometheus apache exporter' then [9117, 'http', '', 'Apache Exporter']
  when 'Prometheus postgres exporter' then [9187, 'http', '', 'Postgres Exporter']
+ when 'Grafana' then [3000, 'http', '', 'Grafana Labs']
  else raise "Unknown port for service #{service}"
  end
- node.run_until_ok("curl -s -k #{protocol}://#{system_name}:#{port}#{path} | grep -i '#{text}'")
+ # debian based systems don't come with curl installed
+ if (os_family.include? 'debian') || (os_family.include? 'ubuntu')
+ node.run_until_ok("wget --no-check-certificate -qO- #{protocol}://#{system_name}:#{port}#{path} | grep -i '#{text}'")
+ else
+ node.run_until_ok("curl -s -k #{protocol}://#{system_name}:#{port}#{path} | grep -i '#{text}'")
+ end
+end
+
+When(/^I enter the "([^"]*)" hostname as the Prometheus URL$/) do |host|
+ node = get_target(host)
+ step %(I enter "http://#{node.full_hostname}:9090" as "Prometheus URL")
 end
 
 When(/^I select the next maintenance window$/) do
@@ -1130,3 +1142,8 @@
  end
  end
 end
+
+When(/^I visit the grafana dashboards of this "([^"]*)"$/) do |host|
+ node = get_target(host)
+ visit("http://#{node.public_ip}:3000/dashboards")
+end

testsuite/features/support/constants.rb

@@ -131,6 +131,7 @@
  'sle15sp4_minion' => 'SLES15-SP4-Pool',
  'sle15sp4_ssh_minion' => 'SLES15-SP4-Pool',
  'sle15sp4_buildhost' => 'SLES15-SP4-Pool',
+ 'monitoring_server' => 'SLES15-SP4-Pool',
  'sle15sp4_terminal' => 'SLES15-SP4-Pool',
  'centos7_minion' => 'RHEL x86_64 Server 7',
  'centos7_ssh_minion' => 'RHEL x86_64 Server 7',

testsuite/features/support/env.rb

@@ -320,6 +320,10 @@ def enable_assertions
  skip_this_scenario unless $sle15sp4_buildhost
 end
 
+Before('@monitoring_server') do
+ skip_this_scenario unless $monitoring_server
+end
+
 Before('@sle15sp4_terminal') do
  skip_this_scenario unless $sle15sp4_terminal_mac
 end

testsuite/features/support/twopence_init.rb

@@ -72,6 +72,7 @@ def twopence_init(target)
  $opensuse154arm_ssh_minion = twopence_init("ssh:#{ENV['OPENSUSE154ARM_SSHMINION']}") if ENV['OPENSUSE154ARM_SSHMINION']
  $sle12sp5_buildhost = twopence_init("ssh:#{ENV['SLE12SP5_BUILDHOST']}") if ENV['SLE12SP5_BUILDHOST']
  $sle15sp4_buildhost = twopence_init("ssh:#{ENV['SLE15SP4_BUILDHOST']}") if ENV['SLE15SP4_BUILDHOST']
+ $monitoring_server = twopence_init("ssh:#{ENV['MONITORING_SERVER']}") if ENV['MONITORING_SERVER']
  $nodes += [$sle12sp4_minion, $sle12sp4_ssh_minion,
  $sle12sp5_minion, $sle12sp5_ssh_minion,
  $sle15_minion, $sle15_ssh_minion,
@@ -90,7 +91,8 @@ def twopence_init(target)
  $debian11_minion, $debian11_ssh_minion,
  $opensuse154arm_minion, $opensuse154arm_ssh_minion,
  $sle12sp5_buildhost,
- $sle15sp4_buildhost]
+ $sle15sp4_buildhost,
+ $monitoring_server]
 else
  # Define twopence objects for QA environment
  $minion = twopence_init("ssh:#{ENV['MINION']}") if ENV['MINION']
@@ -295,7 +297,8 @@ def file_inject(node, local_file, remote_file)
  'opensuse154arm_minion' => $opensuse154arm_minion,
  'opensuse154arm_ssh_minion' => $opensuse154arm_ssh_minion,
  'sle12sp5_buildhost' => $sle12sp5_buildhost,
- 'sle15sp4_buildhost' => $sle15sp4_buildhost }
+ 'sle15sp4_buildhost' => $sle15sp4_buildhost,
+ 'monitoring_server' => $monitoring_server }
 
 # This is the inverse of `node_by_host`.
 $host_by_node = {}

testsuite/run_sets/build_validation_init_monitoring.yml

@@ -0,0 +1,7 @@
+# This file describes the order of features in a Build Validation test suite run.
+
+## Init monitoring server BEGIN ###
+
+- features/build_validation/init_clients/monitoring_server.feature
+
+## Init monitoring server END ###