Allow arbitrary python code (#530)

* Starting with Superset-Operator

* Add snafu error

* Add snafu error message

* Add comment

* Fixing footer variable

* Adding comments on open todo to reference footer and header with vars from operator-rs

* Comment about removing key and write later

* Removing comments

* Bump operator-rs to 0.74.0, using constants rather then strings for header and footer keys

* Remove todo's

* Updating changelog

* Apply suggestions from code review

Co-authored-by: Sebastian Bernauer <sebastian.bernauer@stackable.de>

* Adding docs

* Adding tests configs respect role and group level configs

* Revert: tests for conf overrides

* retrigger pipeline

---------

Co-authored-by: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Co-authored-by: Sebastian Bernauer <sebastian.bernauer@stackable.de>

CHANGELOG.md

@@ -2,13 +2,18 @@
 
 ## [Unreleased]
 
+### Added
+
+- Allowing arbitrary python code as EXPERIMENTAL_FILE_HEADER and EXPERIMENTAL_FILE_FOOTER in superset_config.py ([#530])
+
 ### Changed
 
 - Reduce CRD size from `472KB` to `45KB` by accepting arbitrary YAML input instead of the underlying schema for the following fields ([#528]):
   - `podOverrides`
   - `affinity`
 
 [#528]: https://github.com/stackabletech/superset-operator/pull/528
+[#530]: https://github.com/stackabletech/superset-operator/pull/530
 
 ## [24.7.0] - 2024-07-24
 

Cargo.toml

@@ -21,7 +21,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.8"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.73.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.74.0" }
 strum = { version = "0.26", features = ["derive"] }
 tokio = { version = "1.39", features = ["full"] }
 tracing = "0.1"

docs/modules/superset/pages/usage-guide/configuration-environment-overrides.adoc

@@ -59,6 +59,33 @@ be taken to produce a valid configuration.
 For a full list of configuration options we refer to the
 https://github.com/apache/superset/blob/master/superset/config.py[main config file for Superset].
 
+As Superset can be configured with python code too, arbitrary code can be added to the `superset_conf.py`.
+You can use either `EXPERIMENTAL_FILE_HEADER` to add code to the top or `EXPERIMENTAL_FILE_FOOTER` to add to the bottom.
+
+IMPORTANT: This is an experimental feature
+
+[source,yaml]
+----
+nodes:
+  configOverrides:
+    superset_config.py:
+      CSV_EXPORT: "{'encoding': 'utf-8'}"
+      EXPERIMENTAL_FILE_HEADER: |
+        from modules.my_module import my_class
+      EXPERIMENTAL_FILE_FOOTER: |
+        import logging
+        from superset.security import SupersetSecurityManager
+
+        class myCustomSecurityManger(SupersetSecurityManager):
+          def __init__():
+            init()
+
+        CUSTOM_SECURITY_MANAGER = myCustomSecurityManger
+  roleGroups:
+    default:
+      config: {}
+----
+
 == Environment Variables
 
 In a similar fashion, environment variables can be (over)written. For example per role group:
@@ -85,4 +112,5 @@ nodes:
       config: {}
 ----
 
+
 // cliOverrides don't make sense for this operator, so the feature is omitted for now

rust/crd/src/lib.rs

@@ -369,7 +369,8 @@ impl SupersetConfig {
             logging: product_logging::spec::default_logging(),
             affinity: get_affinity(cluster_name, role),
             graceful_shutdown_timeout: Some(DEFAULT_NODE_GRACEFUL_SHUTDOWN_TIMEOUT),
-            ..Default::default()
+            row_limit: None,
+            webserver_timeout: None,
         }
     }
 }

rust/operator-binary/src/superset_controller.rs

@@ -2,6 +2,7 @@
 use std::{
     borrow::Cow,
     collections::{BTreeMap, BTreeSet, HashMap},
+    io::Write,
     sync::Arc,
 };
 
@@ -39,7 +40,10 @@ use stackable_operator::{
     kube::{runtime::controller::Action, Resource, ResourceExt},
     kvp::{Label, Labels},
     logging::controller::ReconcilerError,
-    product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config},
+    product_config_utils::{
+        transform_all_roles_to_config, validate_all_roles_and_groups_config,
+        CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY,
+    },
     product_logging::{
         self,
         framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command},
@@ -250,6 +254,11 @@ pub enum Error {
     AddTlsVolumesAndVolumeMounts {
         source: stackable_operator::commons::authentication::tls::TlsClientDetailsError,
     },
+
+    #[snafu(display(
+        "failed to write to String (Vec<u8> to be precise) containing superset config"
+    ))]
+    WriteToConfigFileString { source: std::io::Error },
 }
 
 type Result<T, E = Error> = std::result::Result<T, E>;
@@ -523,6 +532,14 @@ fn build_rolegroup_config_map(
     );
 
     let mut config_file = Vec::new();
+
+    // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well
+    // (which would be bad)
+    if let Some(header) = config_properties.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) {
+        writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?;
+    }
+    let temp_file_footer = config_properties.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY);
+
     flask_app_config_writer::write::<SupersetConfigOptions, _, _>(
         &mut config_file,
         config_properties.iter(),
@@ -532,6 +549,10 @@ fn build_rolegroup_config_map(
         rolegroup: rolegroup.clone(),
     })?;
 
+    if let Some(footer) = temp_file_footer {
+        writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?;
+    }
+
     let mut cm_builder = ConfigMapBuilder::new();
 
     cm_builder