refactor!: Don't parse resolv.conf for cluster domain auto-detection (#896)

* WIP spike: Store cluster domain in Client

* Store cluster domain in separate KubernetesClusterInfo struct

* Remove src/utils/cluster_domain.rs

* Dont pass all CLI args

* changelog

* typo

* Move CLI arg into KubernetesClusterInfoCliOpts struct

* KubernetesClusterInfoCliOpts -> KubernetesClusterInfoOpts

* Derive Default for KubernetesClusterInfoOpts

* Fix cli leftovers

* Add rust docs

* Mark as breaking in changelog

Author: sbernauer

crates/stackable-operator/CHANGELOG.md

@@ -4,12 +4,19 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Changed
+
+- BREAKING: Don't parse `/etc/resolv.conf` to auto-detect the Kubernetes cluster domain in case it is not explicitly configured.
+  Instead the operator will default to `cluster.local`. We revert this now after some concerns where raised, we will
+  create a follow-up decision instead addressing how we will continue with this ([#896]).
+
 ### Fixed
 
 - Fix Kubernetes cluster domain parsing from resolv.conf, e.g. on AWS EKS.
   We now only consider Kubernetes services domains instead of all domains (which could include non-Kubernetes domains) ([#895]).
 
 [#895]: https://github.com/stackabletech/operator-rs/pull/895
+[#896]: https://github.com/stackabletech/operator-rs/pull/896
 
 ## [0.79.0] - 2024-10-18
 

crates/stackable-operator/fixtures/cluster_domain/fail/invalid.resolv.conf

@@ -115,7 +115,10 @@ use clap::Args;
 use product_config::ProductConfigManager;
 use snafu::{ResultExt, Snafu};
 
-use crate::{logging::TracingTarget, namespace::WatchNamespace};
+use crate::{
+    logging::TracingTarget, namespace::WatchNamespace,
+    utils::cluster_info::KubernetesClusterInfoOpts,
+};
 
 pub const AUTHOR: &str = "Stackable GmbH - info@stackable.tech";
 
@@ -171,15 +174,19 @@ pub enum Command<Run: Args = ProductOperatorRun> {
 ///     common: ProductOperatorRun,
 /// }
 /// use clap::Parser;
-/// use stackable_operator::logging::TracingTarget;
-/// use stackable_operator::namespace::WatchNamespace;
+/// use stackable_operator::{
+///     logging::TracingTarget,
+///     namespace::WatchNamespace,
+/// };
+///
 /// let opts = Command::<Run>::parse_from(["foobar-operator", "run", "--name", "foo", "--product-config", "bar", "--watch-namespace", "foobar"]);
 /// assert_eq!(opts, Command::Run(Run {
 ///     name: "foo".to_string(),
 ///     common: ProductOperatorRun {
 ///         product_config: ProductConfigPath::from("bar".as_ref()),
 ///         watch_namespace: WatchNamespace::One("foobar".to_string()),
-///         tracing_target: TracingTarget::None
+///         tracing_target: TracingTarget::None,
+///         cluster_info_opts: Default::default(),
 ///     },
 /// }));
 /// ```
@@ -205,12 +212,17 @@ pub struct ProductOperatorRun {
     /// Provides the path to a product-config file
     #[arg(long, short = 'p', value_name = "FILE", default_value = "", env)]
     pub product_config: ProductConfigPath,
+
     /// Provides a specific namespace to watch (instead of watching all namespaces)
     #[arg(long, env, default_value = "")]
     pub watch_namespace: WatchNamespace,
+
     /// Tracing log collector system
     #[arg(long, env, default_value_t, value_enum)]
     pub tracing_target: TracingTarget,
+
+    #[command(flatten)]
+    pub cluster_info_opts: KubernetesClusterInfoOpts,
 }
 
 /// A path to a [`ProductConfigManager`] spec file
@@ -384,6 +396,7 @@ mod tests {
                 product_config: ProductConfigPath::from("bar".as_ref()),
                 watch_namespace: WatchNamespace::One("foo".to_string()),
                 tracing_target: TracingTarget::None,
+                cluster_info_opts: Default::default(),
             }
         );
 
@@ -395,6 +408,7 @@ mod tests {
                 product_config: ProductConfigPath::from("bar".as_ref()),
                 watch_namespace: WatchNamespace::All,
                 tracing_target: TracingTarget::None,
+                cluster_info_opts: Default::default(),
             }
         );
 
@@ -407,6 +421,7 @@ mod tests {
                 product_config: ProductConfigPath::from("bar".as_ref()),
                 watch_namespace: WatchNamespace::One("foo".to_string()),
                 tracing_target: TracingTarget::None,
+                cluster_info_opts: Default::default(),
             }
         );
     }

crates/stackable-operator/fixtures/cluster_domain/pass/kubernetes-multiple.resolv.conf

@@ -1,23 +1,29 @@
-use crate::kvp::LabelSelectorExt;
-use crate::utils::cluster_domain::{self, retrieve_cluster_domain, KUBERNETES_CLUSTER_DOMAIN};
+use std::{
+    convert::TryFrom,
+    fmt::{Debug, Display},
+};
 
 use either::Either;
 use futures::StreamExt;
-use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
-use k8s_openapi::{ClusterResourceScope, NamespaceResourceScope};
-use kube::api::{DeleteParams, ListParams, Patch, PatchParams, PostParams, Resource, ResourceExt};
-use kube::client::Client as KubeClient;
-use kube::core::Status;
-use kube::runtime::wait::delete::delete_and_finalize;
-use kube::runtime::{watcher, WatchStreamExt};
-use kube::{Api, Config};
-use serde::de::DeserializeOwned;
-use serde::Serialize;
+use k8s_openapi::{
+    apimachinery::pkg::apis::meta::v1::LabelSelector, ClusterResourceScope, NamespaceResourceScope,
+};
+use kube::{
+    api::{DeleteParams, ListParams, Patch, PatchParams, PostParams, Resource, ResourceExt},
+    client::Client as KubeClient,
+    core::Status,
+    runtime::{wait::delete::delete_and_finalize, watcher, WatchStreamExt},
+    Api, Config,
+};
+use serde::{de::DeserializeOwned, Serialize};
 use snafu::{OptionExt, ResultExt, Snafu};
-use std::convert::TryFrom;
-use std::fmt::{Debug, Display};
 use tracing::trace;
 
+use crate::{
+    kvp::LabelSelectorExt,
+    utils::cluster_info::{KubernetesClusterInfo, KubernetesClusterInfoOpts},
+};
+
 pub type Result<T, E = Error> = std::result::Result<T, E>;
 
 #[derive(Debug, Snafu)]
@@ -78,9 +84,6 @@ pub enum Error {
 
     #[snafu(display("unable to create kubernetes client"))]
     CreateKubeClient { source: kube::Error },
-
-    #[snafu(display("unable to to resolve kubernetes cluster domain"))]
-    ResolveKubernetesClusterDomain { source: cluster_domain::Error },
 }
 
 /// This `Client` can be used to access Kubernetes.
@@ -93,13 +96,16 @@ pub struct Client {
     delete_params: DeleteParams,
     /// Default namespace as defined in the kubeconfig this client has been created from.
     pub default_namespace: String,
+
+    pub kubernetes_cluster_info: KubernetesClusterInfo,
 }
 
 impl Client {
     pub fn new(
         client: KubeClient,
         field_manager: Option<String>,
         default_namespace: String,
+        kubernetes_cluster_info: KubernetesClusterInfo,
     ) -> Self {
         Client {
             client,
@@ -113,6 +119,7 @@ impl Client {
             },
             delete_params: DeleteParams::default(),
             default_namespace,
+            kubernetes_cluster_info,
         }
     }
 
@@ -517,9 +524,11 @@ impl Client {
     /// use stackable_operator::client::{Client, initialize_operator};
     ///
     /// #[tokio::main]
-    /// async fn main(){
+    /// async fn main() {
     ///
-    /// let client: Client = initialize_operator(None).await.expect("Unable to construct client.");
+    /// let client = initialize_operator(None, &Default::default())
+    ///     .await
+    ///     .expect("Unable to construct client.");
     /// let watcher_config: watcher::Config =
     ///         watcher::Config::default().fields(&format!("metadata.name=nonexistent-pod"));
     ///
@@ -626,38 +635,45 @@ where
     }
 }
 
-pub async fn initialize_operator(field_manager: Option<String>) -> Result<Client> {
-    let _ = KUBERNETES_CLUSTER_DOMAIN
-        .set(retrieve_cluster_domain().context(ResolveKubernetesClusterDomainSnafu)?);
-    create_client(field_manager).await
-}
-
-async fn create_client(field_manager: Option<String>) -> Result<Client> {
+pub async fn initialize_operator(
+    field_manager: Option<String>,
+    cluster_info_opts: &KubernetesClusterInfoOpts,
+) -> Result<Client> {
     let kubeconfig: Config = kube::Config::infer()
         .await
         .map_err(kube::Error::InferConfig)
         .context(InferKubeConfigSnafu)?;
     let default_namespace = kubeconfig.default_namespace.clone();
     let client = kube::Client::try_from(kubeconfig).context(CreateKubeClientSnafu)?;
-    Ok(Client::new(client, field_manager, default_namespace))
+    let cluster_info = KubernetesClusterInfo::new(cluster_info_opts);
+
+    Ok(Client::new(
+        client,
+        field_manager,
+        default_namespace,
+        cluster_info,
+    ))
 }
 
 #[cfg(test)]
 mod tests {
+    use std::{collections::BTreeMap, time::Duration};
+
     use futures::StreamExt;
-    use k8s_openapi::api::core::v1::{Container, Pod, PodSpec};
-    use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
-    use kube::api::{ObjectMeta, PostParams, ResourceExt};
-    use kube::runtime::watcher;
-    use kube::runtime::watcher::Event;
-    use std::collections::BTreeMap;
-    use std::time::Duration;
+    use k8s_openapi::{
+        api::core::v1::{Container, Pod, PodSpec},
+        apimachinery::pkg::apis::meta::v1::LabelSelector,
+    };
+    use kube::{
+        api::{ObjectMeta, PostParams, ResourceExt},
+        runtime::watcher::{self, Event},
+    };
     use tokio::time::error::Elapsed;
 
     #[tokio::test]
     #[ignore = "Tests depending on Kubernetes are not ran by default"]
     async fn k8s_test_wait_created() {
-        let client = super::create_client(None)
+        let client = super::initialize_operator(None, &Default::default())
             .await
             .expect("KUBECONFIG variable must be configured.");
 
@@ -735,7 +751,7 @@ mod tests {
     #[tokio::test]
     #[ignore = "Tests depending on Kubernetes are not ran by default"]
     async fn k8s_test_wait_created_timeout() {
-        let client = super::create_client(None)
+        let client = super::initialize_operator(None, &Default::default())
             .await
             .expect("KUBECONFIG variable must be configured.");
 
@@ -755,7 +771,7 @@ mod tests {
     #[tokio::test]
     #[ignore = "Tests depending on Kubernetes are not ran by default"]
     async fn k8s_test_list_with_label_selector() {
-        let client = super::create_client(None)
+        let client = super::initialize_operator(None, &Default::default())
             .await
             .expect("KUBECONFIG variable must be configured.");