diff --git a/tools/interu/fixtures/interu.yaml b/tools/interu/fixtures/interu.yaml index 279ee08..42b3ecf 100644 --- a/tools/interu/fixtures/interu.yaml +++ b/tools/interu/fixtures/interu.yaml @@ -6,7 +6,7 @@ runners: - name: default arch: amd64 size: large - disk: 50 + disk-gb: 50 nodes: 3 default-arm64: @@ -16,7 +16,7 @@ runners: - name: default arch: arm64 size: large - disk: 50 + disk-gb: 50 nodes: 3 default-mixed: @@ -26,12 +26,12 @@ runners: - name: amd64-nodes arch: amd64 size: large - disk: 50 + disk-gb: 50 nodes: 3 - name: arm64-nodes arch: arm64 size: large - disk: 50 + disk-gb: 50 nodes: 3 profiles: diff --git a/tools/interu/fixtures/test-definition.yaml b/tools/interu/fixtures/test-definition.yaml new file mode 100644 index 0000000..bc8508f --- /dev/null +++ b/tools/interu/fixtures/test-definition.yaml @@ -0,0 +1,98 @@ +# COPIED FROM AIRFLOW-OPERATOR +--- +dimensions: + - name: airflow + values: + - 2.9.2 + - 2.9.3 + - 2.10.2 + # To use a custom image, add a comma and the full name after the product version + # - 2.8.1,docker.stackable.tech/sandbox/airflow:2.8.1-stackable0.0.0-dev + - name: airflow-latest + values: + - 2.10.2 + # To use a custom image, add a comma and the full name after the product version + # - 2.8.1,docker.stackable.tech/sandbox/airflow:2.8.1-stackable0.0.0-dev + - name: ldap-authentication + values: + - no-tls + - insecure-tls + - server-verification-tls + - name: openshift + values: + - "false" + - name: executor + values: + - celery + - kubernetes +tests: + - name: smoke + dimensions: + - airflow + - openshift + - executor + - name: mount-dags-configmap + dimensions: + - airflow-latest + - openshift + - executor + - name: mount-dags-gitsync + dimensions: + - airflow-latest + - openshift + - executor + - name: ldap + dimensions: + - airflow-latest + - openshift + - ldap-authentication + - executor + - name: oidc + dimensions: + - airflow + - openshift + - name: resources + dimensions: + - airflow-latest + - openshift + - name: orphaned-resources + dimensions: + - airflow-latest + - openshift + - name: logging + dimensions: + - airflow + - openshift + - executor + - name: cluster-operation + dimensions: + - airflow-latest + - openshift + - name: overrides + dimensions: + - airflow-latest + - openshift +suites: + - name: nightly + # Run nightly with the latest airflow + patch: + - dimensions: + - name: airflow + expr: last + - name: smoke-latest + # Useful for development + select: + - smoke + patch: + - dimensions: + - expr: last + - name: openshift + # Run on OpenShift with latest airflow + patch: + - dimensions: + - expr: last + - dimensions: + - name: airflow + expr: last + - name: openshift + expr: "true" diff --git a/tools/interu/src/cli.rs b/tools/interu/src/cli.rs index 7ce6577..c4b1af9 100644 --- a/tools/interu/src/cli.rs +++ b/tools/interu/src/cli.rs @@ -23,8 +23,7 @@ pub struct Cli { option, short = 'o', long = "output", - description = "write configuration key=value pairs separated by newlines to file - Useful for CI tools which give a file to write env vars and outputs to which are used in subsequent steps" + description = "write configuration key=value pairs separated by newlines to file. Useful for CI tools which give a file to write env vars and outputs to which are used in subsequent steps" )] pub output: Option, @@ -33,6 +32,19 @@ pub struct Cli { #[argh(switch, short = 'q', long = "quiet")] pub quiet: bool, + /// validate the beku test definition of the selected profile + #[argh(switch, long = "check-test-definitions")] + pub check_test_definitions: bool, + + /// path to beku test-definition file [default = tests/test-definition.yaml] + #[argh( + option, + short = 't', + long = "test-definitions", + default = r#"PathBuf::from("tests/test-definition.yaml")"# + )] + pub test_definitions: PathBuf, + /// which test profile to use #[argh(positional)] pub profile: String, diff --git a/tools/interu/src/config/mod.rs b/tools/interu/src/config/mod.rs index fbcb5fd..a76e04a 100644 --- a/tools/interu/src/config/mod.rs +++ b/tools/interu/src/config/mod.rs @@ -11,7 +11,7 @@ use tracing::instrument; use crate::{ config::{ - profile::{Profile, StrategyValidationError, TestRun}, + profile::{Profile, StrategyValidationError, TestOptions, TestRun}, runner::{ ConvertNodeGroupError, Distribution, ReplicatedNodeGroup, Runner, RunnerValidationError, }, @@ -21,6 +21,7 @@ use crate::{ pub mod profile; pub mod runner; +pub mod test; /// Errors which can be encountered when reading and validating the config file. #[derive(Debug, Snafu)] @@ -43,6 +44,9 @@ pub enum Error { path: PathBuf, }, + #[snafu(display("failed to validate test options"))] + ValidateTestOptions { source: StrategyValidationError }, + #[snafu(display("failed to find profile named {profile_name:?}"))] UnknownProfileName { profile_name: String }, @@ -92,25 +96,19 @@ impl Config { Ok(config) } - #[instrument(name = "validate_config", skip(self))] - fn validate(&self) -> Result<(), ValidationError> { - for (runner_name, runner) in &self.runners { - tracing::debug!(runner_name, "validate runner"); - - runner - .validate(runner_name) - .context(InvalidRunnerConfigSnafu)?; - } - - for (profile_name, profile) in &self.profiles { - tracing::debug!(profile_name, "validate profile"); - - profile - .validate(profile_name, &self.runners) - .context(InvalidProfileConfigSnafu)?; - } + pub fn get_profile(&self, profile_name: &String) -> Result<&Profile, Error> { + self.profiles + .get(profile_name) + .context(UnknownProfileNameSnafu { profile_name }) + } - Ok(()) + pub fn validate_test_options

(&self, profile_name: &String, path: P) -> Result<(), Error> + where + P: AsRef, + { + self.get_profile(profile_name)? + .validate_test_options(&profile_name, path) + .context(ValidateTestOptionsSnafu) } /// Determines the final expanded parameters based on the provided profile. @@ -120,10 +118,7 @@ impl Config { instances: &'a Instances, ) -> Result, Error> { // First, lookup the profile by name. Error if the profile does't exist. - let profile = self - .profiles - .get(profile_name) - .context(UnknownProfileNameSnafu { profile_name })?; + let profile = self.get_profile(profile_name)?; // Next, lookup the runner ref based on the profile strategy let runner_ref = match &profile.strategy { @@ -144,7 +139,11 @@ impl Config { let runner = self.runners.get(runner_ref).unwrap(); // Get test options - let (test_parallelism, test_run, test_parameter) = profile.strategy.get_test_options(); + let TestOptions { + parallelism, + test_run, + test_parameter, + } = profile.strategy.get_test_options(); // Convert our node groups to replicated node groups let node_groups = runner @@ -158,13 +157,34 @@ impl Config { Ok(Parameters { kubernetes_distribution: &runner.platform.distribution, kubernetes_version: &runner.platform.version, + test_parallelism: *parallelism, cluster_ttl: &runner.ttl, - test_parallelism, test_parameter, node_groups, test_run, }) } + + #[instrument(name = "validate_config", skip(self))] + fn validate(&self) -> Result<(), ValidationError> { + for (runner_name, runner) in &self.runners { + tracing::debug!(runner_name, "validate runner"); + + runner + .validate(runner_name) + .context(InvalidRunnerConfigSnafu)?; + } + + for (profile_name, profile) in &self.profiles { + tracing::debug!(profile_name, "validate profile"); + + profile + .validate(profile_name, &self.runners) + .context(InvalidProfileConfigSnafu)?; + } + + Ok(()) + } } /// Parameters which will be expanded into environment variables via the [`Display`] implementation. @@ -221,7 +241,7 @@ impl<'a> Display for Parameters<'a> { } #[cfg(test)] -mod test { +mod tests { use std::path::PathBuf; use super::*; diff --git a/tools/interu/src/config/profile.rs b/tools/interu/src/config/profile.rs index 14bb98a..fe53ebf 100644 --- a/tools/interu/src/config/profile.rs +++ b/tools/interu/src/config/profile.rs @@ -1,9 +1,12 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, path::Path}; use serde::{Deserialize, Serialize}; -use snafu::{ensure, Snafu}; +use snafu::{ensure, ResultExt, Snafu}; -use crate::config::runner::Runner; +use crate::config::{ + runner::Runner, + test::{self, TestDefinition}, +}; #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] @@ -20,6 +23,17 @@ impl Profile { ) -> Result<(), StrategyValidationError> { self.strategy.validate(profile_name, runners) } + + pub fn validate_test_options

( + &self, + profile_name: &str, + path: P, + ) -> Result<(), StrategyValidationError> + where + P: AsRef, + { + self.strategy.validate_test_options(profile_name, path) + } } #[derive(Debug, Snafu)] @@ -34,6 +48,12 @@ pub enum StrategyValidationError { #[snafu(display("strategy {at} references a runner already referenced by another weight"))] NonUniqueWeightRunner { at: String }, + + #[snafu(display("strategy {at} defines invalid test options"))] + InvalidTestOptions { + source: TestOptionsValidationError, + at: String, + }, } #[derive(Debug, Deserialize, Serialize)] @@ -59,18 +79,25 @@ impl Strategy { } } - pub fn get_test_options(&self) -> (usize, &TestRun, &str) { - match self { - Strategy::Weighted(options) => ( - options.options.parallelism, - &options.options.test_run, - options.options.test_parameter.as_str(), - ), - Strategy::UseRunner(options) => ( - options.options.parallelism, - &options.options.test_run, - options.options.test_parameter.as_str(), - ), + pub fn validate_test_options

( + &self, + profile_name: &str, + path: P, + ) -> Result<(), StrategyValidationError> + where + P: AsRef, + { + self.get_test_options() + .validate(path) + .context(InvalidTestOptionsSnafu { + at: format!("profiles.{profile_name}.options"), + }) + } + + pub fn get_test_options(&self) -> &TestOptions { + match &self { + Strategy::Weighted(weighted) => &weighted.options, + Strategy::UseRunner(use_runner) => &use_runner.options, } } } @@ -156,6 +183,18 @@ impl UseRunnerOptions { } } +#[derive(Debug, Snafu)] +pub enum TestOptionsValidationError { + #[snafu(display("failed to load test definition file"))] + ReadFile { source: test::Error }, + + #[snafu(display("encountered unknown test {test_name:?}"))] + UnknownTest { test_name: String }, + + #[snafu(display("encountered unknown test-suite {test_suite:?}"))] + UnknownTestSuite { test_suite: String }, +} + #[derive(Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub struct TestOptions { @@ -168,6 +207,49 @@ pub struct TestOptions { pub test_parameter: String, } +impl TestOptions { + pub fn validate

(&self, path: P) -> Result<(), TestOptionsValidationError> + where + P: AsRef, + { + match self.test_run { + TestRun::TestSuite => { + let test_definition = TestDefinition::from_file(path).context(ReadFileSnafu)?; + + if !test_definition + .suites + .iter() + .any(|s| s.name == self.test_parameter) + { + return UnknownTestSuiteSnafu { + test_suite: self.test_parameter.clone(), + } + .fail(); + } + + Ok(()) + } + TestRun::Test => { + let test_definition = TestDefinition::from_file(path).context(ReadFileSnafu)?; + + if !test_definition + .tests + .iter() + .any(|s| s.name == self.test_parameter) + { + return UnknownTestSnafu { + test_name: self.test_parameter.clone(), + } + .fail(); + } + + Ok(()) + } + TestRun::All => Ok(()), + } + } +} + #[derive(Clone, Debug, Default, Deserialize, Serialize, strum::Display)] #[strum(serialize_all = "kebab-case")] #[serde(rename_all = "kebab-case")] diff --git a/tools/interu/src/config/test.rs b/tools/interu/src/config/test.rs new file mode 100644 index 0000000..5a2d731 --- /dev/null +++ b/tools/interu/src/config/test.rs @@ -0,0 +1,73 @@ +use std::path::{Path, PathBuf}; + +use serde::Deserialize; +use snafu::{ResultExt as _, Snafu}; +use tracing::instrument; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to read test definition file at {path}", path = path.display()))] + ReadFile { + source: std::io::Error, + path: PathBuf, + }, + + #[snafu(display("failed to deserialize test definition file at {path} as yaml", path = path.display()))] + Deserialize { + source: serde_yaml::Error, + path: PathBuf, + }, +} + +#[derive(Debug, Deserialize)] +pub struct TestDefinition { + #[serde(default)] + pub tests: Vec, + + #[serde(default)] + pub suites: Vec, +} + +impl TestDefinition { + /// Read and deserialize test definition from a file located at `path`. + #[instrument(name = "load_test_definition_from_file", skip(path), fields(path = %path.as_ref().display()))] + pub fn from_file

(path: P) -> Result + where + P: AsRef, + { + let contents = std::fs::read_to_string(&path).context(ReadFileSnafu { + path: path.as_ref(), + })?; + + tracing::debug!("deserialize file contents"); + let test_definition: Self = serde_yaml::from_str(&contents).context(DeserializeSnafu { + path: path.as_ref(), + })?; + + Ok(test_definition) + } +} + +#[derive(Debug, Deserialize)] +pub struct Test { + pub name: String, +} + +#[derive(Debug, Deserialize)] +pub struct Suite { + pub name: String, +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + fn from_file(#[files("fixtures/test-definition.yaml")] path: PathBuf) { + let td = TestDefinition::from_file(path).unwrap(); + + assert_eq!(td.suites.len(), 3); + assert_eq!(td.tests.len(), 10); + } +} diff --git a/tools/interu/src/main.rs b/tools/interu/src/main.rs index 0ccc6be..82fbbf6 100644 --- a/tools/interu/src/main.rs +++ b/tools/interu/src/main.rs @@ -13,6 +13,9 @@ enum Error { #[snafu(display("failed to load config"))] LoadConfig { source: config::Error }, + #[snafu(display("failed to validate test options"))] + ValidateTestOptions { source: config::Error }, + #[snafu(display("failed to determine expanded parameters"))] DetermineParameters { source: config::Error }, @@ -32,6 +35,12 @@ fn main() -> Result<(), Error> { let config = Config::from_file(&cli.config).context(LoadConfigSnafu)?; let instances = Instances::from_file(&cli.instances).context(LoadInstancesSnafu)?; + if cli.check_test_definitions { + config + .validate_test_options(&cli.profile, &cli.test_definitions) + .context(ValidateTestOptionsSnafu)?; + } + tracing::info!("determine parameters"); let parameters = config .determine_parameters(&cli.profile, &instances)