Skip to content

Commit

Permalink
Feat: Update compatibility tests for new installation process
Browse files Browse the repository at this point in the history 
Refs: #2153
- Removed functions that will not be available with new installation
process.
- Tests now account for multiple deployments in one CNF and check
accordingly. In case of any failure the whole task exists with FAILED,
additional information about which chart failed can be found in the
logs.

Signed-off-by: svteb <slavo.valko@tietoevry.com>
  • Loading branch information
@svteb
svteb committed Oct 22, 2024
1 parent bd35849 commit 3435af2
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 65 deletions.
8 changes: 4 additions & 4 deletions spec/workload/installability_spec.cr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ describe CnfTestSuite do
ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose")
result = ShellCmd.run_testsuite("helm_chart_valid verbose")
result[:status].success?.should be_true
(/Lint Passed/ =~ result[:output]).should_not be_nil
(/Helm chart lint passed on all charts/ =~ result[:output]).should_not be_nil
ensure
result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose")
end
Expand All @@ -37,7 +37,7 @@ describe CnfTestSuite do
ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-bad_helm_coredns-cnf/cnf-testsuite.yml verbose skip_wait_for_install", expect_failure: true)
result = ShellCmd.run_testsuite("helm_chart_valid")
result[:status].success?.should be_true
(/Lint Failed/ =~ result[:output]).should_not be_nil
(/Helm chart lint failed on one or more charts/ =~ result[:output]).should_not be_nil
ensure
result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose")
end
Expand All @@ -48,7 +48,7 @@ describe CnfTestSuite do
ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-coredns-cnf")
result = ShellCmd.run_testsuite("helm_chart_published")
result[:status].success?.should be_true
(/(PASSED).*(Published Helm Chart Found)/ =~ result[:output]).should_not be_nil
(/(PASSED).*(All Helm charts are published)/ =~ result[:output]).should_not be_nil
ensure
result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-coredns-cnf")
end
Expand All @@ -61,7 +61,7 @@ describe CnfTestSuite do
result = ShellCmd.run("helm search repo stable/coredns", force_output: true)
result = ShellCmd.run_testsuite("helm_chart_published verbose")
result[:status].success?.should be_true
(/(FAILED).*(Published Helm Chart Not Found)/ =~ result[:output]).should_not be_nil
(/(FAILED).*(One or more Helm charts are not published)/ =~ result[:output]).should_not be_nil
ensure
result = ShellCmd.run("#{Helm::BinarySingleton.helm} repo remove badrepo")
result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-bad-helm-repo")
Expand Down
139 changes: 78 additions & 61 deletions src/tasks/workload/compatibility.cr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ require "docker_client"
require "../utils/utils.cr"



desc "The CNF test suite checks to see if CNFs support horizontal scaling (across multiple machines) and vertical scaling (between sizes of machines) by using the native K8s kubectl"
task "compatibility", ["helm_chart_valid", "helm_chart_published", "helm_deploy", "cni_compatible", "increase_decrease_capacity", "rollback"].concat(ROLLING_VERSION_CHANGE_TEST_NAMES) do |_, args|
stdout_score("compatibility", "Compatibility, Installability, and Upgradeability")
Expand Down Expand Up @@ -408,88 +407,106 @@ end

task "helm_chart_published", ["helm_local_install"] do |t, args|
CNFManager::Task.task_runner(args, task: t) do |args, config|
if check_verbose(args)
Log.for("verbose").debug { "helm_chart_published args.raw: #{args.raw}" }
Log.for("verbose").debug { "helm_chart_published args.named: #{args.named}" }
helm = Helm::BinarySingleton.helm

# Store chart search commands in an array
chart_searches = [] of String

# Collect helm chart search queries from deployments
config.deployments.helm_charts.each do |deployment|
helm_repo_name = deployment.helm_repo_name
helm_chart_name = deployment.helm_chart_name

helm_chart_full_name = "#{helm_repo_name}/#{helm_chart_name}"
chart_searches << helm_chart_full_name
end

helm_chart = config.deployments.get_deployment_param(:helm_chart)
current_dir = FileUtils.pwd
helm = Helm::BinarySingleton.helm
Log.for("verbose").debug { helm } if check_verbose(args)

if CNFManager.helm_repo_add(args: args)
unless helm_chart.empty?
helm_search_cmd = "#{helm} search repo #{helm_chart}"
Log.for(t.name).info { "helm search command: #{helm_search_cmd}" }
Process.run(
helm_search_cmd,
shell: true,
output: helm_search_stdout = IO::Memory.new,
error: helm_search_stderr = IO::Memory.new
)
helm_search = helm_search_stdout.to_s
Log.for("verbose").debug { "#{helm_search}" } if check_verbose(args)
unless helm_search =~ /No results found/
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Passed, "Published Helm Chart Found")
else
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "Published Helm Chart Not Found")
end
else
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "Published Helm Chart Not Found")
# Initialize flags to track the state of the task
charts_found = !chart_searches.empty?
all_published = true

# Process the helm chart searches and log results for each search
chart_searches.each do |helm_chart_full_name|
helm_search_cmd = "#{helm} search repo #{helm_chart_full_name}"
helm_search_status = Process.run(helm_search_cmd, shell: true, output: helm_search_stdout = IO::Memory.new, error: helm_search_stderr = IO::Memory.new)
helm_search_output = helm_search_stdout.to_s
Log.for(t.name).info { "Searching Helm chart: #{helm_chart_full_name}" }
Log.for(t.name).info { "Helm search output:\n#{helm_search_output}" }

# Check if the chart was found
if helm_search_output =~ /No results found/
all_published = false
end
end

# Handle the case where no charts were specified
unless charts_found
Log.for(t.name).info { "No Helm charts found for searching." }
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Skipped, "No Helm charts found to search")
else
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "Published Helm Chart Not Found")
# Return appropriate results based on search outcomes
if all_published
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Passed, "All Helm charts are published")
else
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "One or more Helm charts are not published")
end
end
end
end

task "helm_chart_valid", ["helm_local_install"] do |t, args|
CNFManager::Task.task_runner(args, task: t) do |args, config|
if check_verbose(args)
Log.for("verbose").debug { "helm_chart_valid args.raw: #{args.raw}" }
Log.for("verbose").debug { "helm_chart_valid args.named: #{args.named}" }
end
current_dir = FileUtils.pwd
helm = Helm::BinarySingleton.helm
Log.info { "Current directory: #{current_dir}" }

response = String::Builder.new
# Store chart locations in an array
chart_dirs = [] of Tuple(String, String)

helm_directory = config.deployments.get_deployment_param(:helm_directory)
if helm_directory.empty?
working_chart_directory = "exported_chart"
else
working_chart_directory = helm_directory
# Collect helm chart paths
config.deployments.helm_charts.each do |deployment|
chart_dirs << {"#{current_dir}/#{CNF_DIR}/#{deployment.name}/exported_chart", deployment.name}
end

if args.named.keys.includes? "cnf_chart_path"
working_chart_directory = args.named["cnf_chart_path"]
# Collect helm directory paths
config.deployments.helm_dirs.each do |deployment|
chart_dirs << {"#{current_dir}/#{CNF_DIR}/#{deployment.name}/chart", deployment.name}
end

Log.for("verbose").debug { "working_chart_directory: #{working_chart_directory}" } if check_verbose(args)

current_dir = FileUtils.pwd
Log.for(t.name).debug { "current dir: #{current_dir}" }
helm = Helm::BinarySingleton.helm

destination_cnf_dir = config.dynamic.destination_cnf_dir

helm_lint_cmd = "#{helm} lint #{destination_cnf_dir}/#{working_chart_directory}"
helm_lint_status = Process.run(
helm_lint_cmd,
shell: true,
output: helm_lint_stdout = IO::Memory.new,
error: helm_link_stderr = IO::Memory.new
)
helm_lint = helm_lint_stdout.to_s
Log.for(t.name).debug { "helm_lint: #{helm_lint}" } if check_verbose(args)
# Initialize flags to track the task state
charts_found = !chart_dirs.empty?
all_passed = true

# Iterate over chart directories and store the results for each lint
chart_dirs.each do |chart_dir, deployment_name|
helm_lint_cmd = "#{helm} lint #{chart_dir}"
helm_lint_status = Process.run(helm_lint_cmd, shell: true, output: helm_lint_stdout = IO::Memory.new, error: helm_lint_stderr = IO::Memory.new)
helm_lint_output = helm_lint_stdout.to_s
Log.for(t.name).info { "Linting helm chart for deployment: #{deployment_name}" }
Log.for(t.name).info { "Helm Lint Output:\n#{helm_lint_output}" }

# If the linting failed, mark as failed
if !helm_lint_status.success?
all_passed = false
end
end

if helm_lint_status.success?
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Passed, "Helm Chart #{working_chart_directory} Lint Passed")
# Handle case where no charts were found
unless charts_found
Log.for(t.name).info { "No Helm charts or directories found for linting." }
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Skipped, "No Helm charts found to lint")
else
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "Helm Chart #{working_chart_directory} Lint Failed")
# Return appropriate results based on linting outcomes
if all_passed
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Passed, "Helm chart lint passed on all charts")
else
CNFManager::TestcaseResult.new(CNFManager::ResultStatus::Failed, "Helm chart lint failed on one or more charts")
end
end
end
end


def setup_calico_cluster(cluster_name : String) : KindManager::Cluster
Log.info { "Running cni_compatible(Cluster Creation)" }
Helm.helm_repo_add("projectcalico","https://docs.projectcalico.org/charts")
Expand Down

0 comments on commit 3435af2

Please sign in to comment.