diff --git a/CHANGELOG.md b/CHANGELOG.md index dfd7e24e..a1965e4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,18 +6,20 @@ All notable changes to this project will be documented in this file. ### Added -- The possibility to specifiy `configOverrides` and `envOverrides` ([#122]). +- The possibility to specify `configOverrides` and `envOverrides` ([#122]). - Reconciliation errors are now reported as Kubernetes events ([#130]). - Use cli argument `watch-namespace` / env var `WATCH_NAMESPACE` to specify a single namespace to watch ([#134]). ### Changed -- `operator-rs` `0.10.0` -> `0.13.0` ([#130],[#134]). +- `operator-rs` `0.10.0` -> `0.15.0` ([#130], [#134], [#148]). +- `HADOOP_OPTS` for jmx exporter specified to `HADOOP_NAMENODE_OPTS`, `HADOOP_DATANODE_OPTS` and `HADOOP_JOURNALNODE_OPTS` to fix cli tool ([#148]). [#122]: https://github.com/stackabletech/hdfs-operator/pull/122 [#130]: https://github.com/stackabletech/hdfs-operator/pull/130 [#134]: https://github.com/stackabletech/hdfs-operator/pull/134 +[#148]: https://github.com/stackabletech/hdfs-operator/pull/148 ## [0.3.0] - 2022-02-14 diff --git a/Cargo.lock b/Cargo.lock index bb5ea7ce..0d36ecfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,6 +65,21 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bit-set" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -152,7 +167,7 @@ version = "3.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da95d038ede1a964ce99f49cbe27a7fb538d1da595e4b4f70b8c8f338d17bf16" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro-error", "proc-macro2", "quote", @@ -338,6 +353,16 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" +[[package]] +name = "fancy-regex" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95b4efe5be9104a4a18a9916e86654319895138be727b229820c39257c30dda" +dependencies = [ + "bit-set", + "regex", +] + [[package]] name = "fastrand" version = "1.7.0" @@ -497,15 +522,6 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.4.0" @@ -710,9 +726,9 @@ dependencies = [ [[package]] name = "kube" -version = "0.69.1" +version = "0.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86835e9147615457c1edc2b11c723acd1d21372e9cefa80a9d2742f6d69042d0" +checksum = "4dcc72fdf0c491160a34d4a1bfb03f96da8a5054288d61c816d514b5c2fa49ea" dependencies = [ "k8s-openapi", "kube-client", @@ -723,9 +739,9 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.69.1" +version = "0.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71f8aa916ee8290fbd0af76b3a28093a8786fccb6286902095a5243484447971" +checksum = "94b01c722d55ffedec74cbc259b4508d8a59bf19540006ec87618f76ab156579" dependencies = [ "base64", "bytes", @@ -751,7 +767,7 @@ dependencies = [ "thiserror", "tokio", "tokio-native-tls", - "tokio-util 0.6.9", + "tokio-util", "tower", "tower-http", "tracing", @@ -759,9 +775,9 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.69.1" +version = "0.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da41f98f213796a68fcd4510c88b5fd36de131c1ea30d152474c3f2e1bef1a72" +checksum = "7dd9e3535777edd122cc26fe3fe6357066b33eff63d8b919862edbe7a956a679" dependencies = [ "chrono", "form_urlencoded", @@ -777,9 +793,9 @@ dependencies = [ [[package]] name = "kube-derive" -version = "0.69.1" +version = "0.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd6fdc2e1615a3aecd7e90fe4bda3ba32379e38193251d4ddaccba26579fe22" +checksum = "1322e25c20dd6f18ca6baecc88bb130331e99d988df9d7a9a207f15819e05bff" dependencies = [ "darling", "proc-macro2", @@ -790,9 +806,9 @@ dependencies = [ [[package]] name = "kube-runtime" -version = "0.69.1" +version = "0.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e8bf2258850dbc0a062b561656a1d7020fa8040d552e149f0a0361f0d6794e" +checksum = "816c8c086f8bbcf9a4db0b7a68db90b784ef6292a57de35c64cccb90d5edfbe5" dependencies = [ "ahash", "backoff", @@ -808,7 +824,7 @@ dependencies = [ "smallvec", "thiserror", "tokio", - "tokio-util 0.6.9", + "tokio-util", "tracing", ] @@ -1031,27 +1047,25 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.11.2" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ - "instant", "lock_api", "parking_lot_core", ] [[package]] name = "parking_lot_core" -version = "0.8.5" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" dependencies = [ "cfg-if", - "instant", "libc", "redox_syscall", "smallvec", - "winapi", + "windows-sys", ] [[package]] @@ -1148,11 +1162,11 @@ dependencies = [ [[package]] name = "product-config" -version = "0.3.0" -source = "git+https://github.com/stackabletech/product-config.git?tag=0.3.0#602e7b8e1c235dd93fb3289662c1200eaa1a83db" +version = "0.3.1" +source = "git+https://github.com/stackabletech/product-config.git?tag=0.3.1#40c93e5283beef100c9fecdb6368f1e1480db3e8" dependencies = [ + "fancy-regex", "java-properties", - "regex", "schemars", "semver", "serde", @@ -1465,7 +1479,7 @@ dependencies = [ "serde_json", "serde_yaml", "stackable-operator", - "strum 0.24.0", + "strum", "thiserror", "tracing", ] @@ -1503,8 +1517,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.13.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.13.0#401f4053d8c32d0fcd507d94799956181f66105d" +version = "0.15.0" +source = "git+https://github.com/stackabletech/operator-rs.git?tag=0.15.0#c7c408d476c0b7ba06833c19e5c9d2aefa5875ae" dependencies = [ "backoff", "chrono", @@ -1524,7 +1538,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "strum 0.23.0", + "strum", "thiserror", "tokio", "tracing", @@ -1537,35 +1551,13 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "strum" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" -dependencies = [ - "strum_macros 0.23.1", -] - [[package]] name = "strum" version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e96acfc1b70604b8b2f1ffa4c57e59176c7dbb05d556c71ecd2f5498a1dee7f8" dependencies = [ - "strum_macros 0.24.0", -] - -[[package]] -name = "strum_macros" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" -dependencies = [ - "heck 0.3.3", - "proc-macro2", - "quote", - "rustversion", - "syn", + "strum_macros", ] [[package]] @@ -1574,7 +1566,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6878079b17446e4d3eba6192bb0a2950d5b14f0ed8424b852310e5a94345d0ef" dependencies = [ - "heck 0.4.0", + "heck", "proc-macro2", "quote", "rustversion", @@ -1723,21 +1715,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-util" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "slab", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.0" @@ -1749,6 +1726,7 @@ dependencies = [ "futures-sink", "log", "pin-project-lite", + "slab", "tokio", ] @@ -1772,7 +1750,7 @@ dependencies = [ "pin-project", "pin-project-lite", "tokio", - "tokio-util 0.7.0", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -1915,12 +1893,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-segmentation" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" - [[package]] name = "unicode-xid" version = "0.2.2" @@ -2010,6 +1982,49 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" + +[[package]] +name = "windows_i686_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" + +[[package]] +name = "windows_i686_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" + [[package]] name = "xml-rs" version = "0.8.4" diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml index c1bba5ed..8f381ecf 100644 --- a/rust/crd/Cargo.toml +++ b/rust/crd/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/stackabletech/hdfs-operator" version = "0.4.0-nightly" [dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.13.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } semver = "1.0" serde = { version = "1.0", features = ["derive"] } diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index ce3a7fca..2590d158 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,7 @@ version = "0.4.0-nightly" build = "build.rs" [dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.13.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } stackable-hdfs-crd = { path = "../crd" } stackable-hdfs-operator = { path = "../operator" } clap = { version = "3.1", features = ["derive"] } @@ -19,7 +19,7 @@ serde_yaml = "0.8" [build-dependencies] built = { version = "0.5", features = ["chrono", "git2"] } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.13.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } stackable-hdfs-crd = { path = "../crd" } [[bin]] diff --git a/rust/operator/Cargo.toml b/rust/operator/Cargo.toml index abcb0c95..3d1a1ced 100644 --- a/rust/operator/Cargo.toml +++ b/rust/operator/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/stackabletech/hdfs-operator" version = "0.4.0-nightly" [dependencies] -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.13.0" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.15.0" } stackable-hdfs-crd = { path = "../crd" } futures = "0.3" diff --git a/rust/operator/src/hdfs_controller.rs b/rust/operator/src/hdfs_controller.rs index 5283b019..e124f229 100644 --- a/rust/operator/src/hdfs_controller.rs +++ b/rust/operator/src/hdfs_controller.rs @@ -20,7 +20,7 @@ use stackable_operator::k8s_openapi::apimachinery::pkg::{ api::resource::Quantity, apis::meta::v1::LabelSelector, }; use stackable_operator::kube::api::ObjectMeta; -use stackable_operator::kube::runtime::controller::{Context, ReconcilerAction}; +use stackable_operator::kube::runtime::controller::{Action, Context}; use stackable_operator::kube::runtime::reflector::ObjectRef; use stackable_operator::kube::ResourceExt; use stackable_operator::labels::role_group_selector_labels; @@ -54,7 +54,7 @@ pub struct Ctx { pub async fn reconcile_hdfs( hdfs: Arc, ctx: Context, -) -> HdfsOperatorResult { +) -> HdfsOperatorResult { tracing::info!("Starting reconcile"); let client = &ctx.get_ref().client; @@ -126,15 +126,11 @@ pub async fn reconcile_hdfs( } } - Ok(ReconcilerAction { - requeue_after: None, - }) + Ok(Action::await_change()) } -pub fn error_policy(_error: &Error, _ctx: Context) -> ReconcilerAction { - ReconcilerAction { - requeue_after: Some(Duration::from_secs(5)), - } +pub fn error_policy(_error: &Error, _ctx: Context) -> Action { + Action::requeue(Duration::from_secs(5)) } fn rolegroup_service( @@ -458,7 +454,7 @@ fn journalnode_containers( ) -> Vec { let mut env: Vec = hadoop_container.clone().env.unwrap(); env.push(EnvVar { - name: "HADOOP_OPTS".to_string(), + name: "HADOOP_JOURNALNODE_OPTS".to_string(), value: Some( format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={}:/stackable/jmx/{}.yaml", DEFAULT_JOURNAL_NODE_METRICS_PORT, @@ -487,7 +483,7 @@ fn namenode_containers( ) -> Vec { let mut env: Vec = hadoop_container.clone().env.unwrap(); env.push(EnvVar { - name: "HADOOP_OPTS".to_string(), + name: "HADOOP_NAMENODE_OPTS".to_string(), value: Some( format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={}:/stackable/jmx/{}.yaml", DEFAULT_NAME_NODE_METRICS_PORT, @@ -509,7 +505,7 @@ fn namenode_containers( liveness_probe: Some(PROBE.clone()), ..hadoop_container.clone() }, - // Note that we don't add the HADOOP_OPTS env var to this container (zkfc) + // Note that we don't add the HADOOP_OPTS / HADOOP_NAMENODE_OPTS env var to this container (zkfc) // Here it would cause an "address already in use" error and prevent the namenode container from starting. // Because the jmx exporter is not enabled here, also the readiness probes are not enabled. Container { @@ -529,7 +525,7 @@ fn datanode_containers( ) -> Vec { let mut env: Vec = hadoop_container.clone().env.unwrap(); env.push(EnvVar { - name: "HADOOP_OPTS".to_string(), + name: "HADOOP_DATANODE_OPTS".to_string(), value: Some( format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={}:/stackable/jmx/{}.yaml", DEFAULT_DATA_NODE_METRICS_PORT, diff --git a/rust/operator/src/pod_svc_controller.rs b/rust/operator/src/pod_svc_controller.rs index 02223f97..8d6bb710 100644 --- a/rust/operator/src/pod_svc_controller.rs +++ b/rust/operator/src/pod_svc_controller.rs @@ -10,7 +10,7 @@ use stackable_operator::{ }, kube::{ core::ObjectMeta, - runtime::controller::{Context, ReconcilerAction}, + runtime::controller::{Action, Context}, }, }; use std::sync::Arc; @@ -20,10 +20,7 @@ pub struct Ctx { pub client: stackable_operator::client::Client, } -pub async fn reconcile_pod( - pod: Arc, - ctx: Context, -) -> HdfsOperatorResult { +pub async fn reconcile_pod(pod: Arc, ctx: Context) -> HdfsOperatorResult { tracing::info!("Starting reconcile"); let name = pod.metadata.name.clone().ok_or(Error::PodHasNoName)?; @@ -89,13 +86,9 @@ pub async fn reconcile_pod( .apply_patch(FIELD_MANAGER_SCOPE_POD, &svc, &svc) .await .map_err(|source| Error::ApplyPodServiceFailed { source, name })?; - Ok(ReconcilerAction { - requeue_after: None, - }) + Ok(Action::await_change()) } -pub fn error_policy(_error: &Error, _ctx: Context) -> ReconcilerAction { - ReconcilerAction { - requeue_after: Some(Duration::from_secs(5)), - } +pub fn error_policy(_error: &Error, _ctx: Context) -> Action { + Action::requeue(Duration::from_secs(5)) }