diff --git a/CHANGELOG.md b/CHANGELOG.md index cc01942f..2cc4baba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Allowing arbitrary python code as EXPERIMENTAL_FILE_HEADER and EXPERIMENTAL_FILE_FOOTER in superset_config.py ([#530]) + ### Changed - Reduce CRD size from `472KB` to `45KB` by accepting arbitrary YAML input instead of the underlying schema for the following fields ([#528]): @@ -9,6 +13,7 @@ - `affinity` [#528]: https://github.com/stackabletech/superset-operator/pull/528 +[#530]: https://github.com/stackabletech/superset-operator/pull/530 ## [24.7.0] - 2024-07-24 diff --git a/Cargo.lock b/Cargo.lock index 62b603bd..34558aea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2056,8 +2056,8 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "stackable-operator" -version = "0.73.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.73.0#4d98a29b08a7d959e5e287f774cf064c02ffbd62" +version = "0.74.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.74.0#c77a5423b66bc1667b63af7d8bec00de88a5303f" dependencies = [ "chrono", "clap", @@ -2070,7 +2070,6 @@ dependencies = [ "json-patch", "k8s-openapi", "kube", - "lazy_static", "opentelemetry-jaeger", "opentelemetry_sdk", "product-config", @@ -2094,7 +2093,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.73.0#4d98a29b08a7d959e5e287f774cf064c02ffbd62" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.74.0#c77a5423b66bc1667b63af7d8bec00de88a5303f" dependencies = [ "darling", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 51d1f9d7..8ab7d385 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" snafu = "0.8" -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.73.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.74.0" } strum = { version = "0.26", features = ["derive"] } tokio = { version = "1.39", features = ["full"] } tracing = "0.1" diff --git a/docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc b/docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc index 4594d753..d8d6fa49 100644 --- a/docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc +++ b/docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc @@ -59,6 +59,33 @@ be taken to produce a valid configuration. For a full list of configuration options we refer to the https://github.com/apache/superset/blob/master/superset/config.py[main config file for Superset]. +As Superset can be configured with python code too, arbitrary code can be added to the `superset_conf.py`. +You can use either `EXPERIMENTAL_FILE_HEADER` to add code to the top or `EXPERIMENTAL_FILE_FOOTER` to add to the bottom. + +IMPORTANT: This is an experimental feature + +[source,yaml] +---- +nodes: + configOverrides: + superset_config.py: + CSV_EXPORT: "{'encoding': 'utf-8'}" + EXPERIMENTAL_FILE_HEADER: | + from modules.my_module import my_class + EXPERIMENTAL_FILE_FOOTER: | + import logging + from superset.security import SupersetSecurityManager + + class myCustomSecurityManger(SupersetSecurityManager): + def __init__(): + init() + + CUSTOM_SECURITY_MANAGER = myCustomSecurityManger + roleGroups: + default: + config: {} +---- + == Environment Variables In a similar fashion, environment variables can be (over)written. For example per role group: @@ -85,4 +112,5 @@ nodes: config: {} ---- + // cliOverrides don't make sense for this operator, so the feature is omitted for now diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 94edcb13..0cbf3284 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -369,7 +369,8 @@ impl SupersetConfig { logging: product_logging::spec::default_logging(), affinity: get_affinity(cluster_name, role), graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT), - ..Default::default() + row_limit: None, + webserver_timeout: None, } } } diff --git a/rust/operator-binary/src/superset_controller.rs b/rust/operator-binary/src/superset_controller.rs index a559c7e3..42abd6e6 100644 --- a/rust/operator-binary/src/superset_controller.rs +++ b/rust/operator-binary/src/superset_controller.rs @@ -2,6 +2,7 @@ use std::{ borrow::Cow, collections::{BTreeMap, BTreeSet, HashMap}, + io::Write, sync::Arc, }; @@ -39,7 +40,10 @@ use stackable_operator::{ kube::{runtime::controller::Action, Resource, ResourceExt}, kvp::{Label, Labels}, logging::controller::ReconcilerError, - product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, + product_config_utils::{ + transform_all_roles_to_config, validate_all_roles_and_groups_config, + CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, + }, product_logging::{ self, framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, @@ -250,6 +254,11 @@ pub enum Error { AddTlsVolumesAndVolumeMounts { source: stackable_operator::commons::authentication::tls::TlsClientDetailsError, }, + + #[snafu(display( + "failed to write to String (Vec to be precise) containing superset config" + ))] + WriteToConfigFileString { source: std::io::Error }, } type Result = std::result::Result; @@ -523,6 +532,14 @@ fn build_rolegroup_config_map( ); let mut config_file = Vec::new(); + + // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well + // (which would be bad) + if let Some(header) = config_properties.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { + writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; + } + let temp_file_footer = config_properties.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); + flask_app_config_writer::write::( &mut config_file, config_properties.iter(), @@ -532,6 +549,10 @@ fn build_rolegroup_config_map( rolegroup: rolegroup.clone(), })?; + if let Some(footer) = temp_file_footer { + writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; + } + let mut cm_builder = ConfigMapBuilder::new(); cm_builder