From 571bf23d7ddb85a6eed1005ff6b590208ac1a04e Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:24:56 +0200 Subject: [PATCH 01/13] feat: add quorum-manager sidecar for dynamic KRaft voter membership Controllers now run a quorum-manager sidecar that admits itself into the KRaft voter set on startup (add-controller) and removes itself before termination (remove-controller via preStop), so controller role groups can be scaled up/down on a running cluster without a full rolling restart. controller.quorum.bootstrap.servers now points at each controller role group's headless Service DNS name instead of individual pod addresses, keeping container commands stable across replica changes. Co-Authored-By: Claude Sonnet 5 --- .../src/controller/build/command.rs | 664 +++++++++++++++++- .../src/controller/build/mod.rs | 161 ++++- .../src/controller/build/properties/mod.rs | 151 +++- .../controller/build/resource/config_map.rs | 30 +- .../controller/build/resource/statefulset.rs | 500 ++++++++++++- .../src/controller/build/security.rs | 95 ++- rust/operator-binary/src/crd/mod.rs | 36 - 7 files changed, 1517 insertions(+), 120 deletions(-) diff --git a/rust/operator-binary/src/controller/build/command.rs b/rust/operator-binary/src/controller/build/command.rs index 4805a623..ebd097a1 100644 --- a/rust/operator-binary/src/controller/build/command.rs +++ b/rust/operator-binary/src/controller/build/command.rs @@ -11,8 +11,8 @@ use super::properties::ConfigFileName; use crate::{ controller::{build::security::copy_opa_tls_cert_command, security::ValidatedKafkaSecurity}, crd::{ - BROKER_ID_POD_MAP_DIR, KafkaPodDescriptor, STACKABLE_CONFIG_DIR, - STACKABLE_KERBEROS_KRB5_PATH, STACKABLE_LOG_CONFIG_DIR, + BROKER_ID_POD_MAP_DIR, KafkaPodDescriptor, METRICS_PORT, STACKABLE_CONFIG_DIR, + STACKABLE_KERBEROS_KRB5_PATH, STACKABLE_LOG_CONFIG_DIR, role::KafkaRole, }, }; @@ -37,12 +37,26 @@ pub fn kafka_log_opts_env_var() -> String { "KAFKA_LOG4J_OPTS".to_string() } +/// Shell snippet setting `$POD_INDEX` to this pod's ordinal, parsed from the trailing digits +/// of `$POD_NAME` (e.g. `2` for `..-controller-default-2`). +/// +/// Paired with [`EXPORT_REPLICA_ID`] (see there for why the split): used, in some combination, +/// by four call sites that used to each duplicate this derivation with slightly drifted +/// whitespace — the broker and controller `kafka` containers' own entrypoints, and the +/// `quorum-manager` sidecar's main loop and `preStop` hook. +const DERIVE_POD_INDEX: &str = r#"POD_INDEX=$(echo "$POD_NAME" | grep -oE '[0-9]+$')"#; + +/// Shell snippet exporting `$REPLICA_ID` (this container's KRaft node id) from `$POD_INDEX` +/// (see [`DERIVE_POD_INDEX`], which must run first) and `$NODE_ID_OFFSET`. Exported (rather +/// than a plain assignment) because every caller either runs `config-utils template` or the +/// `quorum-manager` sidecar's `kafka-metadata-quorum.sh`/`curl` calls as a *subprocess*, which +/// need `REPLICA_ID` in their environment, not just this shell's. +const EXPORT_REPLICA_ID: &str = "export REPLICA_ID=$((POD_INDEX + NODE_ID_OFFSET))"; + /// Returns the commands to start the main Kafka container pub fn broker_kafka_container_commands( kraft_mode: bool, - controller_descriptors: Vec, kafka_security: &ValidatedKafkaSecurity, - product_version: &str, ) -> String { formatdoc! {" {COMMON_BASH_TRAP_FUNCTIONS} @@ -65,18 +79,14 @@ pub fn broker_kafka_container_commands( false => "".to_string(), }, import_opa_tls_cert = copy_opa_tls_cert_command(kafka_security), - broker_start_command = broker_start_command(kraft_mode, controller_descriptors, product_version), + broker_start_command = broker_start_command(kraft_mode), } } -fn broker_start_command( - kraft_mode: bool, - controller_descriptors: Vec, - product_version: &str, -) -> String { +fn broker_start_command(kraft_mode: bool) -> String { let common_command = formatdoc! {" - export POD_INDEX=$(echo \"$POD_NAME\" | grep -oE '[0-9]+$') - export REPLICA_ID=$((POD_INDEX+NODE_ID_OFFSET)) + {derive_pod_index} + {export_replica_id} if [ -f \"{broker_id_pod_map_dir}/$POD_NAME\" ]; then REPLICA_ID=$(cat \"{broker_id_pod_map_dir}/$POD_NAME\") @@ -88,6 +98,8 @@ fn broker_start_command( cp {config_dir}/{jaas_file} /tmp/{jaas_file} config-utils template /tmp/{jaas_file} ", + derive_pod_index = DERIVE_POD_INDEX, + export_replica_id = EXPORT_REPLICA_ID, broker_id_pod_map_dir = BROKER_ID_POD_MAP_DIR, config_dir = STACKABLE_CONFIG_DIR, properties_file = ConfigFileName::BrokerProperties, @@ -98,11 +110,10 @@ fn broker_start_command( formatdoc! {" {common_command} - bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/{properties_file} --ignore-formatted {initial_controller_command} + bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/{properties_file} --ignore-formatted --no-initial-controllers bin/kafka-server-start.sh /tmp/{properties_file} & ", properties_file = ConfigFileName::BrokerProperties, - initial_controller_command = initial_controllers_command(&controller_descriptors, product_version), } } else { formatdoc! {" @@ -156,9 +167,48 @@ wait_for_termination() } "#; +/// Chooses exactly one controller (the one with the numerically lowest KRaft `node_id` among +/// all controller pod descriptors, a value that is stable across scale-up/down of an existing +/// controller role group, since new replicas only ever get higher node ids) to bootstrap the +/// dynamic KRaft quorum by itself, via `kafka-storage.sh format --standalone`, the first time +/// it is ever formatted. +/// +/// Every other controller — whether it is part of the cluster's initial desired replica count +/// or added later on scale-up — is formatted with `--no-initial-controllers` and relies +/// entirely on the `quorum-manager` sidecar's `add-controller` loop to join the quorum. This is +/// what keeps the controller container's command identical across replica-count changes (no +/// voter list baked into it), and what makes "admit a new controller" solely the sidecar's +/// concern rather than something the format step also has a hand in. +/// +/// Known limitation: this rule is only safe for a cluster's *original* bootstrap. If the +/// designated node's persistent volume is ever lost and needs to reformat after the cluster has +/// already formed a quorum elsewhere, reformatting it with `--standalone` would bootstrap a +/// second, conflicting one-node quorum instead of rejoining the existing one — the same class +/// of manual-recovery scenario as losing enough voters to break quorum in any Raft-based +/// system, not something this operator (which deliberately has no live-cluster awareness) +/// can detect or repair automatically. See `kraft-controller.adoc`. +fn controller_quorum_format_flag(controller_descriptors: &[KafkaPodDescriptor]) -> String { + let bootstrap_node_id = controller_descriptors + .iter() + .filter(|descriptor| descriptor.role == KafkaRole::Controller) + .map(|descriptor| descriptor.node_id) + .min() + .expect( + "a controller StatefulSet is always built with at least one controller pod descriptor", + ); + + formatdoc! {" + if [ \"$REPLICA_ID\" = \"{bootstrap_node_id}\" ]; then + FORMAT_QUORUM_FLAG=--standalone + else + FORMAT_QUORUM_FLAG=--no-initial-controllers + fi + " + } +} + pub fn controller_kafka_container_command( controller_descriptors: Vec, - product_version: &str, ) -> String { formatdoc! {" {BASH_TRAP_FUNCTIONS} @@ -166,44 +216,588 @@ pub fn controller_kafka_container_command( prepare_signal_handlers containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & - POD_INDEX=$(echo \"$POD_NAME\" | grep -oE '[0-9]+$') - export REPLICA_ID=$((POD_INDEX+NODE_ID_OFFSET)) + {derive_pod_index} + {export_replica_id} cp {config_dir}/{properties_file} /tmp/{properties_file} config-utils template /tmp/{properties_file} - bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/{properties_file} --ignore-formatted {initial_controller_command} + {quorum_format_flag} + bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/{properties_file} --ignore-formatted \"$FORMAT_QUORUM_FLAG\" bin/kafka-server-start.sh /tmp/{properties_file} & wait_for_termination $! {create_vector_shutdown_file_command} ", remove_vector_shutdown_file_command = remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), + derive_pod_index = DERIVE_POD_INDEX, + export_replica_id = EXPORT_REPLICA_ID, config_dir = STACKABLE_CONFIG_DIR, properties_file = ConfigFileName::ControllerProperties, - initial_controller_command = initial_controllers_command(&controller_descriptors, product_version), + quorum_format_flag = controller_quorum_format_flag(&controller_descriptors), create_vector_shutdown_file_command = create_vector_shutdown_file_command(STACKABLE_LOG_DIR) } } -fn to_initial_controllers(controller_descriptors: &[KafkaPodDescriptor]) -> String { - controller_descriptors - .iter() - .map(|desc| desc.as_voter()) - .collect::>() - .join(",") +/// The `kafka-metadata-quorum.sh` binary, referenced by its absolute path (matching every +/// other exec-into-pod usage of a Kafka CLI tool in this repo, e.g. the kuttl test scripts +/// under `tests/templates/kuttl/*/*.sh`), rather than the relative `bin/...` form used only +/// inside the `kafka` container's own entrypoint (which runs with the Kafka install dir as +/// its working directory). +const KAFKA_METADATA_QUORUM_BINARY: &str = "/stackable/kafka/bin/kafka-metadata-quorum.sh"; + +const ADMIN_CLIENT_PROPERTIES_PATH: &str = "/stackable/config/admin-client.properties"; + +/// The merged config used only for `add-controller` (self-registration). +/// +/// `add-controller` is not a plain admin-client call: the same process that connects to the +/// quorum also reads `node.id` and its own `listeners`/`controller.listener.names` from the +/// **same** `--command-config` file to build the voter registration payload (confirmed live: +/// pointed at the plain [`ADMIN_CLIENT_PROPERTIES_PATH`], every attempt failed with `node.id +/// not found in configuration file`, so no controller was ever able to admit itself as a +/// voter). But that rendered `controller.properties` has no bare `security.protocol`/`ssl.*` +/// keys of its own — only the `listener.name..ssl.*`-prefixed ones the server process +/// uses for its listeners — so using it *instead of* the admin-client config leaves the +/// AdminClient with no TLS config and unable to reach the (TLS-only) bootstrap controller. +/// Concatenating both files (also confirmed live) gives `add-controller` everything it reads: +/// the bare `ssl.*`/`security.protocol` keys for its own connection, plus `node.id` and the +/// listener keys for the registration payload. +/// +/// **Order matters.** There is no key overlap between the two files today, but +/// `controller.properties` accepts unconditional `configOverrides` merged into it (see +/// `controller_properties::build`), so a user override there could add a colliding key. Java +/// properties parsing lets a later occurrence of the same key win, so `controller.properties` +/// is concatenated *first* and [`ADMIN_CLIENT_PROPERTIES_PATH`] *last* — that way the client +/// TLS config `add-controller` connects with always wins by construction, rather than +/// depending on there being no collision today. +const ADD_CONTROLLER_PROPERTIES_PATH: &str = "/tmp/add-controller.properties"; + +/// Wall-clock bound (seconds) applied to every individual `kafka-metadata-quorum.sh` +/// invocation via `timeout`. The Java AdminClient can otherwise retry internally for far +/// longer than any of this file's own script-level deadlines, which matters most in +/// `quorum_manager_pre_stop_command`: it runs exactly when peers may be unreachable, and a +/// hung admin-client call there would burn into `terminationGracePeriodSeconds` (default: +/// 30 minutes) rather than the script's own 25s budget. +const CLI_CALL_TIMEOUT_SECONDS: u32 = 15; + +/// Grace period (seconds) after [`CLI_CALL_TIMEOUT_SECONDS`] elapses before `timeout` sends +/// `SIGKILL`, via `--kill-after`. +/// +/// `timeout N cmd` (GNU coreutils) without `--kill-after` only *sends* `SIGTERM` once `N` +/// seconds pass — it does not force-kill `cmd`, so if `cmd` doesn't honor the signal +/// promptly, the whole call can run far longer than `N` seconds. Confirmed directly, +/// independent of Kafka: `timeout 3 bash -c 'trap "" TERM; sleep 30'` takes the full 30s, not +/// 3s. Confirmed live, with Kafka: during a full namespace deletion (every controller +/// terminating concurrently, so a peer's `describe`/`add-controller`/`remove-controller` call +/// can hit a blackholed rather than actively-refused connection), a controller's sidecar kept +/// running well past its own `preStop` script's ~25-40s design budget — the `timeout` wrapper +/// around its `kafka-metadata-quorum.sh` calls was not actually bounding them. +const CLI_CALL_KILL_AFTER_SECONDS: u32 = 5; + +/// Shell snippet setting `$BOOTSTRAP_SERVERS` by extracting +/// `controller.quorum.bootstrap.servers` from the static, un-rendered `controller.properties` +/// ConfigMap file, un-escaping the `\:` that `to_java_properties_string` applies to colons. +/// This value has no `${env:...}` placeholders — every `host:port` pair is already fully +/// resolved at build time from pod descriptors (see `kraft_controllers` in +/// `build/properties/mod.rs`) — so it can be read directly without running `config-utils +/// template` first. +/// +/// Reading this at runtime, rather than baking the peer list into this script as a Rust +/// literal, keeps both sidecar scripts' content — and therefore the controller pod +/// template — identical across changes to an existing controller role group's *replica +/// count*. Confirmed live: without this, scaling controllers up/down rolled every +/// already-existing controller pod, not just the ones actually being added/removed — the +/// same class of problem `--initial-controllers` caused before it was removed from the +/// `kafka` container's own format step (see `controller_quorum_format_flag`), just via this +/// sidecar's command instead. +fn extract_bootstrap_servers_command() -> String { + format!( + r#"BOOTSTRAP_SERVERS=$(grep '^controller.quorum.bootstrap.servers=' {config_dir}/{controller_properties_file} | cut -d= -f2- | sed 's/\\:/:/g')"#, + config_dir = STACKABLE_CONFIG_DIR, + controller_properties_file = ConfigFileName::ControllerProperties, + ) } -fn initial_controllers_command( - controller_descriptors: &[KafkaPodDescriptor], - product_version: &str, -) -> String { - match product_version.starts_with("3.7") { - true => "".to_string(), - false => format!( - "--initial-controllers {initial_controllers}", - initial_controllers = to_initial_controllers(controller_descriptors), - ), +/// The sidecar's main-loop command: while this controller's local Raft state is +/// `observer`, repeatedly attempt to admit it into the quorum's voter set. +/// +/// Explicitly traps `TERM` and exits: this script runs as the container's PID 1, and the +/// kernel suppresses the default action of unhandled signals for PID 1, so without this +/// trap the loop below would never notice `SIGTERM` and would run until Kubernetes gives up +/// waiting and sends `SIGKILL` after the full `terminationGracePeriodSeconds` (confirmed +/// live: with no trap, this container kept looping — and its pod kept reporting as +/// `Terminating` — long after the `kafka` container in the same pod had shut down +/// gracefully). The `sleep 10 &`/`wait $!` pair (rather than a plain `sleep 10`) lets the +/// trap fire immediately: bash's `wait` builtin is interrupted as soon as a trapped signal +/// arrives, whereas a foreground `sleep` would only be noticed once it finished. +/// +/// Renders [`ADD_CONTROLLER_PROPERTIES_PATH`] once at startup (this controller's identity +/// and listener address don't change for the container's lifetime) by reusing the same +/// `$POD_NAME`/`NODE_ID_OFFSET` → `REPLICA_ID` derivation ([`DERIVE_POD_INDEX`]/ +/// [`EXPORT_REPLICA_ID`]), and the same `config-utils template` render step, as the `kafka` +/// container's own entrypoint (see [`controller_kafka_container_command`]) — see +/// [`ADD_CONTROLLER_PROPERTIES_PATH`] for why `add-controller` needs this merged file rather +/// than the plain admin-client config, and for why the concatenation order matters. +/// +/// The render/merge preamble's inputs are static, operator-rendered config (env vars set +/// once at pod creation), so a failure there is a genuine misconfiguration that retrying +/// won't fix. It must still be loud in the logs, but it must *not* crash the container: a +/// container with no readiness probe is only `Ready` while `Running`, and (with +/// `OrderedReady` pod management on every non-Kerberos controller `StatefulSet`) a +/// crash-looping sidecar would make its whole pod `NotReady` and block scale/update +/// progress for every sibling pod in the role, not just the broken one. So on failure this +/// falls into a "degraded" loop that repeats a clear error every 30s and never attempts +/// `add-controller` (there is no valid rendered config to use), keeping the container alive +/// and `Running` while the problem stays visible via `kubectl logs`. This deliberately does +/// *not* retry the render/merge step itself — that would look like it might eventually +/// succeed, when the actual cause is a misconfiguration that only a human or a new rollout +/// can fix. +pub fn quorum_manager_container_command() -> String { + format!( + r#" + set -uo pipefail + trap 'exit 0' TERM + {derive_pod_index} + [ -n "$POD_INDEX" ] || exit 0 + {export_replica_id} + {extract_bootstrap_servers} + + if cp {config_dir}/{controller_properties_file} /tmp/{controller_properties_file} \ + && config-utils template /tmp/{controller_properties_file} \ + && cat /tmp/{controller_properties_file} {admin_client_config} > {add_controller_config}; then + echo "Starting KRaft voter admission loop against bootstrap servers: $BOOTSTRAP_SERVERS" + while true; do + state=$(curl -s --max-time 5 --connect-timeout 2 localhost:{metrics_port}/metrics | grep -oE 'kafka_server_raft_metrics_current_state\{{state="[a-z]+"\}}' | grep -oE '"[a-z]+"' | tr -d '"') + if [ "$state" = "observer" ]; then + echo "Local Raft state is observer, attempting add-controller..." + timeout --kill-after={cli_kill_after} {cli_timeout} {binary} --bootstrap-controller "$BOOTSTRAP_SERVERS" --command-config {add_controller_config} add-controller \ + || echo "add-controller attempt failed (this is expected if it already succeeded or a leader election is in progress), will retry" + elif [ -z "$state" ]; then + echo "Could not determine local Raft state (metrics scrape returned nothing), will retry" + else + echo "Local Raft state is '$state', nothing to do" + fi + sleep 10 & + wait $! + done + else + echo "ERROR: quorum-manager failed to render or merge its configuration (see errors above); this looks like a genuine misconfiguration, not a transient failure." + while true; do + echo "ERROR: quorum-manager is degraded and will NOT attempt add-controller: configuration render/merge failed at startup and this container is not retrying it. Check the errors above and the operator-rendered config; this pod likely needs manual investigation or a new rollout." + sleep 30 & + wait $! + done + fi + "#, + metrics_port = METRICS_PORT, + binary = KAFKA_METADATA_QUORUM_BINARY, + derive_pod_index = DERIVE_POD_INDEX, + export_replica_id = EXPORT_REPLICA_ID, + extract_bootstrap_servers = extract_bootstrap_servers_command(), + config_dir = STACKABLE_CONFIG_DIR, + controller_properties_file = ConfigFileName::ControllerProperties, + admin_client_config = ADMIN_CLIENT_PROPERTIES_PATH, + add_controller_config = ADD_CONTROLLER_PROPERTIES_PATH, + cli_timeout = CLI_CALL_TIMEOUT_SECONDS, + cli_kill_after = CLI_CALL_KILL_AFTER_SECONDS, + ) +} + +/// The sidecar's `preStop` command: before this controller pod terminates, check that +/// removing it would not remove the *last* remaining voter from the quorum, and if so, +/// remove it from the voter set. Always exits 0 — a stuck or failed check must never block +/// pod termination. +/// +/// Removing a departing voter only ever *lowers* the majority threshold for the remaining +/// set, and the `remove-controller` RPC itself needs the *current* quorum to already commit +/// it — if peers are unreachable the call simply fails, it can't corrupt anything. So the +/// only real invariant worth enforcing here is "never remove the last voter": a 1-voter +/// quorum can't be reduced further without permanently losing all fault tolerance (there +/// would be no other voter left to ever add a replacement to). +/// +/// This controller's own KRaft node id is derived at runtime from `$POD_NAME` and +/// `$NODE_ID_OFFSET` ([`DERIVE_POD_INDEX`]/[`EXPORT_REPLICA_ID`]), exactly as the `kafka` +/// container's own entrypoint does — see `controller_kafka_container_command`. +/// +/// `describe --replication`'s column layout (`NodeId` as column 1, `DirectoryId` as column +/// 2, `Status` as the last column, with `Status` one of `Leader`/`Follower`/`Observer`) is +/// the *documented* KIP-853 tabular format, but has not been confirmed against a live +/// cluster (see Task 4's brief, Step 5 — deferred to Task 7's kuttl run, which has one). +/// Filtering is deliberately conservative: only rows whose `Status` is a recognized voter +/// value (`Leader`/`Follower`) count towards `total_voters`, and if that filter yields zero +/// voters (e.g. because the real column layout differs from what's assumed here), the +/// check simply retries rather than treating "no known voters" as "safe to remove" — i.e. +/// this fails closed (skips removal) rather than open on a parsing mismatch. +/// +/// The "would leave zero voters" case is the one exception that does *not* retry: once a +/// `describe` shows this pod is the last remaining voter, retrying for the rest of the +/// `DEADLINE` can't make it safe to remove — nothing else is going to add a voter for it +/// while it terminates. Confirmed live: before this early `break`, a controller pod that +/// became the last voter (e.g. scaling controllers down to 1, or the last surviving pod +/// during a full teardown) always burned the entire 25s `DEADLINE` here for no benefit. +pub fn quorum_manager_pre_stop_command() -> String { + format!( + r#" + set -uo pipefail + {derive_pod_index} + [ -n "$POD_INDEX" ] || exit 0 + {export_replica_id} + {extract_bootstrap_servers} + DEADLINE=$((SECONDS + 25)) + while [ "$SECONDS" -lt "$DEADLINE" ]; do + describe=$(timeout --kill-after={cli_kill_after} {cli_timeout} {binary} --bootstrap-controller "$BOOTSTRAP_SERVERS" --command-config {config} describe --replication 2>/dev/null) + if [ -n "$describe" ]; then + voters=$(echo "$describe" | tail -n +2 | awk '$NF == "Leader" || $NF == "Follower"') + total_voters=$(echo "$voters" | grep -c .) + if [ "$total_voters" -gt 0 ]; then + remaining_after_removal=$(( total_voters - 1 )) + if [ "$remaining_after_removal" -ge 1 ]; then + directory_id=$(echo "$voters" | awk -v id="$REPLICA_ID" '$1 == id {{ print $2 }}') + if [ -n "$directory_id" ]; then + echo "Removing self (node $REPLICA_ID, directory $directory_id) from the voter set..." + timeout --kill-after={cli_kill_after} {cli_timeout} {binary} --bootstrap-controller "$BOOTSTRAP_SERVERS" --command-config {config} remove-controller \ + --controller-id "$REPLICA_ID" --controller-directory-id "$directory_id" \ + || echo "remove-controller failed, proceeding with termination anyway" + else + echo "Could not find own node $REPLICA_ID among current voters (already removed?), nothing to do" + fi + break + else + echo "Removing self would leave zero voters, skipping (this can't become safe later during my own termination -- nothing else will add a voter for me)" + break + fi + else + echo "Could not identify any voters in the describe output (unrecognized format), skipping removal for safety and retrying..." + fi + fi + sleep 2 + done + exit 0 + "#, + binary = KAFKA_METADATA_QUORUM_BINARY, + config = ADMIN_CLIENT_PROPERTIES_PATH, + cli_timeout = CLI_CALL_TIMEOUT_SECONDS, + cli_kill_after = CLI_CALL_KILL_AFTER_SECONDS, + derive_pod_index = DERIVE_POD_INDEX, + export_replica_id = EXPORT_REPLICA_ID, + extract_bootstrap_servers = extract_bootstrap_servers_command(), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn quorum_manager_container_command_targets_the_bootstrap_servers_not_localhost() { + let command = quorum_manager_container_command(); + assert!(command.contains( + "grep '^controller.quorum.bootstrap.servers=' /stackable/config/controller.properties" + )); + assert!(command.contains(r#"--bootstrap-controller "$BOOTSTRAP_SERVERS""#)); + assert!(command.contains("add-controller")); + assert!(!command.contains("--bootstrap-controller 'localhost")); + assert!(!command.contains(r#"--bootstrap-controller "localhost"#)); + } + + /// Checks only that the trap and the interruptible-sleep pair are present in the + /// generated command *string* — it does not execute the script, so it cannot verify the + /// trap actually fires promptly under a real `SIGTERM`. That was confirmed separately on + /// a live cluster: without a `TERM` trap, this loop runs as the container's PID 1, whose + /// unhandled signals the kernel suppresses by default — so the `kafka` container in the + /// same pod shut down promptly on `SIGTERM` while this sidecar kept looping (curl + /// connection-refused every ~15s) until Kubernetes gave up and sent `SIGKILL` after the + /// full `terminationGracePeriodSeconds` (1800s), holding the whole pod in `Terminating` + /// well past kuttl's step timeout. The trap plus `sleep 10 &` / `wait $!` (rather than a + /// foreground `sleep 10`) let bash notice and act on `SIGTERM` immediately instead of + /// only after the next blocking command returns. + #[test] + fn quorum_manager_container_command_traps_term_and_sleeps_interruptibly() { + let command = quorum_manager_container_command(); + assert!(command.contains("trap 'exit 0' TERM")); + assert!(command.contains("sleep 10 &")); + assert!(command.contains("wait $!")); + } + + /// Checks only that the generated command *string* concatenates the two config files in + /// the order that makes `add-controller` self-register successfully — it does not + /// execute the script, so it cannot verify runtime behavior. That was confirmed + /// separately on a live cluster: `add-controller` reads `node.id` and its own + /// `listeners`/`controller.listener.names` from the *same* `--command-config` file it + /// connects with, to build the voter registration payload — pointed at the plain + /// admin-client config (which has no `node.id`), every attempt failed with `node.id not + /// found in configuration file`, so no controller was ever admitted as a voter. See + /// [`ADD_CONTROLLER_PROPERTIES_PATH`] for why the fix is a merged file (in this specific + /// order) rather than switching to `controller.properties` outright (that file has no + /// bare `ssl.*`/`security.protocol`, so the AdminClient couldn't reach the TLS-only + /// bootstrap controller at all). + #[test] + fn quorum_manager_container_command_string_merges_controller_and_admin_client_properties_for_add_controller() + { + let command = quorum_manager_container_command(); + // Renders this controller's own `controller.properties` (carries `node.id` and + // `listeners`) via the same REPLICA_ID derivation used by the `kafka` container. + assert!(command.contains("export REPLICA_ID=$((POD_INDEX + NODE_ID_OFFSET))")); + assert!(command.contains("config-utils template /tmp/controller.properties")); + // Merges it with the plain admin-client config (carries `security.protocol`/`ssl.*`), + // controller.properties first so the client TLS config in admin-client.properties + // wins on any key collision (see `ADD_CONTROLLER_PROPERTIES_PATH`'s doc comment). + assert!(command.contains( + "cat /tmp/controller.properties /stackable/config/admin-client.properties > /tmp/add-controller.properties" + )); + assert!(command.contains("--command-config /tmp/add-controller.properties add-controller")); + } + + #[test] + fn quorum_manager_pre_stop_command_always_exits_zero() { + let command = quorum_manager_pre_stop_command(); + assert!(command.trim_end().ends_with("exit 0")); + assert!(command.contains("remove-controller")); + } + + /// The old majority-based guard (`majority=$(( total_voters / 2 + 1 ))`, + /// `remaining_after_removal -ge majority`) always blocked the last safe removal of a + /// 2-voter quorum (2 -> 1): `majority` was 2, `remaining_after_removal` was 1, and + /// `1 -ge 2` is false. That left a 2-voter quorum with only 1 live member — a dead + /// quorum requiring manual recovery, exactly the outage this feature exists to prevent. + /// The only invariant that actually matters is "never remove the last voter", so this + /// asserts the generated script uses that condition instead. + #[test] + fn quorum_manager_pre_stop_command_allows_removing_the_second_to_last_voter() { + let command = quorum_manager_pre_stop_command(); + assert!( + command.contains(r#"remaining_after_removal" -ge 1 ]"#), + "expected the guard to allow removal whenever at least one voter remains \ + afterwards, command was: {command}" + ); + assert!( + !command.contains("majority"), + "the old majority-based guard variable should be gone entirely, command was: \ + {command}" + ); + } + + /// Directly exercises the corrected guard's arithmetic (mirrored from the generated + /// script) end to end in bash: a 2-voter quorum must allow removing the departing voter + /// (leaving 1), while a 1-voter quorum must not (that would leave zero). + #[test] + fn quorum_manager_pre_stop_guard_arithmetic_allows_two_to_one_but_not_one_to_zero() { + fn removal_allowed(total_voters: u32) -> bool { + let script = format!( + r#" + total_voters={total_voters} + remaining_after_removal=$(( total_voters - 1 )) + [ "$remaining_after_removal" -ge 1 ] + "# + ); + std::process::Command::new("bash") + .arg("-c") + .arg(script) + .status() + .expect("bash is available to run this test") + .success() + } + + assert!( + removal_allowed(2), + "removing the second-to-last voter of a 2-voter quorum must be allowed" + ); + assert!( + !removal_allowed(1), + "removing the last voter of a 1-voter quorum must never be allowed" + ); + } + + /// Confirmed live: a controller pod that is the last remaining voter when it terminates + /// (e.g. scaling controllers down to 1, or the last survivor of a full teardown) hit the + /// "would leave zero voters" branch and, before this fix, kept retrying every 2s until + /// the full 25s `DEADLINE` elapsed for no benefit -- nothing else adds a voter for this + /// pod while it's terminating, so the outcome can never change. The branch must `break` + /// immediately instead of falling through to the loop's `sleep 2`. + #[test] + fn quorum_manager_pre_stop_command_gives_up_immediately_on_the_last_voter() { + let command = quorum_manager_pre_stop_command(); + let zero_voters_branch = command + .split("Removing self would leave zero voters") + .nth(1) + .expect("the zero-voters message is present in the generated script"); + let next_fi = zero_voters_branch + .find("fi") + .expect("an `fi` closes this branch"); + assert!( + zero_voters_branch[..next_fi].contains("break"), + "the zero-voters branch must break out of the retry loop immediately instead of \ + retrying until DEADLINE, branch was: {}", + &zero_voters_branch[..next_fi] + ); + } + + /// The `preStop` hook already guarded its `REPLICA_ID` derivation against an empty + /// `POD_INDEX`; the main loop's derivation must have the same guard, or an empty + /// `POD_INDEX` would silently produce a wrong `node.id` instead of the sidecar noticing. + #[test] + fn quorum_manager_container_command_guards_against_empty_pod_index() { + let command = quorum_manager_container_command(); + assert!(command.contains(r#"[ -n "$POD_INDEX" ] || exit 0"#)); + } + + /// The render/merge preamble (`cp`/`config-utils template`/`cat`) must log loudly on + /// failure, but must not crash-loop the container: it falls into a degraded loop instead + /// of exiting, and never attempts `add-controller` once degraded. + #[test] + fn quorum_manager_container_command_preamble_is_loud_but_does_not_crash_on_error() { + let command = quorum_manager_container_command(); + // A failed render/merge must not crash-loop the container (that would make the pod + // NotReady and, under OrderedReady pod management, block every sibling pod in the + // role too) — it must log loudly instead and stay Running. + assert!( + !command.contains("set -e"), + "the preamble must not opt into `set -e` (that would crash-loop the container), \ + command was: {command}" + ); + assert!( + command.contains("ERROR"), + "expected a clear error message on a failed render/merge, command was: {command}" + ); + // On failure it must degrade into a loop rather than exiting (which would also crash + // the container) and must never attempt add-controller once degraded. + let error_branch_start = command + .find("echo \"ERROR: quorum-manager failed to render or merge") + .expect("the command has a degraded-mode error branch"); + let degraded_branch = &command[error_branch_start..]; + assert!(degraded_branch.contains("while true")); + // The degraded branch must never invoke the CLI tool (there is no valid rendered + // config to use) — check for the actual invocation, not just the word + // "add-controller" (which also appears inside the degraded branch's own log + // message, explaining what it is *not* doing). + assert!(!degraded_branch.contains(KAFKA_METADATA_QUORUM_BINARY)); + } + + /// The SIGTERM-handling fix's whole point is prompt shutdown, but an unresponsive (not + /// refused) connection to the metrics port would otherwise block the loop body + /// indefinitely — the trap can only fire between commands or during `wait` — reintroducing + /// the exact stall the fix targeted. + #[test] + fn quorum_manager_container_command_metrics_curl_has_timeouts() { + let command = quorum_manager_container_command(); + assert!(command.contains("curl -s --max-time 5 --connect-timeout 2 localhost")); + } + + /// `timeout N cmd` (GNU coreutils, no `--kill-after`) only *sends* the signal after `N` + /// seconds — it does not force-kill the process, so if `cmd` doesn't honor the signal + /// promptly, the whole call can run far longer than `N` seconds. Confirmed directly, + /// independent of Kafka: `timeout 3 bash -c 'trap "" TERM; sleep 30'` takes the full 30s, + /// not 3s, while `timeout --kill-after=2 3 bash -c 'trap "" TERM; sleep 30'` is correctly + /// bounded to ~5s. This matters most for `quorum_manager_pre_stop_command`, which runs + /// exactly when peers may be mid-termination (a blackholed, not actively-refused, + /// connection is exactly the kind of thing a JVM AdminClient can hang on past its own + /// `timeout` wrapper) — confirmed live: during a full namespace deletion, a controller's + /// sidecar kept running (past `preStop`, so its `SIGTERM` hadn't even been delivered to + /// the main loop yet) for 100+ seconds, far past the script's own ~25-40s design budget. + #[test] + fn every_cli_call_has_a_kill_after_so_timeout_is_actually_enforced() { + let container_command = quorum_manager_container_command(); + let pre_stop_command = quorum_manager_pre_stop_command(); + + for command in [&container_command, &pre_stop_command] { + for line in command + .lines() + .filter(|line| line.contains(KAFKA_METADATA_QUORUM_BINARY)) + { + assert!( + line.contains("timeout --kill-after="), + "every kafka-metadata-quorum.sh invocation must use `timeout --kill-after=...` \ + so a hung call is actually bounded, not just signaled — offending line: {line}" + ); + } + } + } + + /// Builds a minimal [`KafkaPodDescriptor`] for the given role and replica. + /// + /// `KafkaPodDescriptor`'s fields are `pub(crate)`, which is crate-wide (not + /// module-scoped) visibility in Rust, so this direct construction is legal from any + /// module inside `stackable-kafka-operator` — mirrors the identically-named helper in + /// `build/properties/mod.rs`'s own test module. + fn pod_descriptor(role: KafkaRole, replica: u16, node_id: u32) -> KafkaPodDescriptor { + KafkaPodDescriptor { + namespace: "default".parse().expect("valid namespace name"), + role_group_statefulset_name: "kafka-controller-default" + .parse() + .expect("valid statefulset name"), + role_group_service_name: "kafka-controller-default-headless" + .parse() + .expect("valid service name"), + replica, + cluster_domain: stackable_operator::commons::networking::DomainName::try_from( + "cluster.local", + ) + .expect("valid domain"), + node_id, + role, + client_port: 9093.into(), + } + } + + /// The controller with the lowest `node_id` bootstraps the quorum by itself + /// (`--standalone`); every other controller joins via the `quorum-manager` sidecar's + /// `add-controller` loop (`--no-initial-controllers`) — this is the runtime branch that + /// replaces baking a fixed `--initial-controllers ` into the format command. + #[test] + fn controller_kafka_container_command_branches_on_the_lowest_node_id() { + let descriptors = vec![ + pod_descriptor(KafkaRole::Controller, 0, 5), + pod_descriptor(KafkaRole::Controller, 1, 6), + pod_descriptor(KafkaRole::Controller, 2, 7), + ]; + let command = controller_kafka_container_command(descriptors); + + assert!(command.contains(r#"if [ "$REPLICA_ID" = "5" ]; then"#)); + assert!(command.contains("FORMAT_QUORUM_FLAG=--standalone")); + assert!(command.contains("FORMAT_QUORUM_FLAG=--no-initial-controllers")); + assert!(command.contains( + "bin/kafka-storage.sh format --cluster-id \"$KAFKA_CLUSTER_ID\" --config /tmp/controller.properties --ignore-formatted \"$FORMAT_QUORUM_FLAG\"" + )); + // The old `--initial-controllers ` scheme is gone entirely, including its + // synthetic directory-id suffix. + assert!(!command.contains("--initial-controllers")); + assert!(!command.contains("0000000000-")); + } + + /// The whole point of removing the baked-in voter list: the container command must stay + /// byte-for-byte identical when only the *replica count* of an existing controller role + /// group changes (new replicas only ever get higher node ids), so scaling up/down no + /// longer forces Kubernetes to roll every already-existing controller pod just to pick up + /// an unchanged (`--ignore-formatted` no-ops it anyway) format command. + #[test] + fn controller_kafka_container_command_is_stable_across_replica_count_changes() { + let three_replicas = vec![ + pod_descriptor(KafkaRole::Controller, 0, 5), + pod_descriptor(KafkaRole::Controller, 1, 6), + pod_descriptor(KafkaRole::Controller, 2, 7), + ]; + let five_replicas = vec![ + pod_descriptor(KafkaRole::Controller, 0, 5), + pod_descriptor(KafkaRole::Controller, 1, 6), + pod_descriptor(KafkaRole::Controller, 2, 7), + pod_descriptor(KafkaRole::Controller, 3, 8), + pod_descriptor(KafkaRole::Controller, 4, 9), + ]; + + assert_eq!( + controller_kafka_container_command(three_replicas), + controller_kafka_container_command(five_replicas) + ); + } + + /// Brokers are never voters and never the bootstrap candidate — they always join (or, for + /// a fresh cluster, simply never assert any voter membership) via `--no-initial-controllers`. + #[test] + fn broker_start_command_always_uses_no_initial_controllers_in_kraft_mode() { + let command = broker_start_command(true); + assert!(command.contains("--no-initial-controllers")); + assert!(!command.contains("--initial-controllers")); + assert!(!command.contains("--standalone")); } } diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 9bf436bb..7ea36cad 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -163,13 +163,17 @@ pub fn build(cluster: &ValidatedCluster) -> Result mod tests { use stackable_operator::kube::Resource; - use super::build; - use crate::controller::{ - ValidatedCluster, - test_support::{ - bootstrap_listener, ingress_address, minimal_kafka, validated_cluster, - zookeeper_mode_cluster, + use super::{build, security::STACKABLE_TLS_KAFKA_INTERNAL_DIR}; + use crate::{ + controller::{ + ValidatedCluster, + node_id_hasher::node_id_hash32_offset, + test_support::{ + bootstrap_listener, ingress_address, minimal_kafka, validated_cluster, + zookeeper_mode_cluster, + }, }, + crd::{STACKABLE_CONFIG_DIR, STACKABLE_DATA_DIR, role::KafkaRole}, }; /// Sorted `metadata.name`s of the given resources, for order-independent assertions. @@ -299,6 +303,151 @@ mod tests { ); } + /// The `quorum-manager` sidecar's admin-client calls need every directory that + /// `controller_admin_client_properties` (see `build/security.rs`) writes paths into: + /// the config volume (for `admin-client.properties` itself) and the internal TLS + /// volume (for the keystore/truststore the properties file points at). Missing either + /// mount makes every `add-controller`/`remove-controller` invocation fail SSL init. + #[test] + fn quorum_manager_sidecar_mounts_every_directory_referenced_by_admin_client_properties() { + let cluster = kraft_mode_cluster(); + let resources = build(&cluster).expect("build succeeds"); + + let controller_sts = resources + .stateful_sets + .iter() + .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-controller-default")) + .expect("the controller StatefulSet should be built"); + let pod_spec = controller_sts + .spec + .as_ref() + .expect("the StatefulSet should have a spec") + .template + .spec + .as_ref() + .expect("the pod template should have a spec"); + let quorum_manager = pod_spec + .containers + .iter() + .find(|c| c.name == "quorum-manager") + .expect("the controller pod should have a quorum-manager sidecar"); + + let mount_paths: Vec<&str> = quorum_manager + .volume_mounts + .as_ref() + .expect("the sidecar should have volume mounts") + .iter() + .map(|vm| vm.mount_path.as_str()) + .collect(); + assert!( + mount_paths.contains(&STACKABLE_CONFIG_DIR), + "the sidecar must mount the config directory carrying admin-client.properties, got: {mount_paths:?}" + ); + assert!( + mount_paths.contains(&STACKABLE_TLS_KAFKA_INTERNAL_DIR), + "the sidecar must mount the internal TLS directory admin-client.properties points its keystore/truststore at, got: {mount_paths:?}" + ); + // `add-controller` reads this controller's own on-disk `meta.properties` (written by + // `kafka-storage.sh format`, and pointed at by `log.dirs` in the merged config it + // connects with) to build the voter registration payload. Confirmed live: without + // this mount, every `add-controller` attempt fails with "Unable to read + // meta.properties from /stackable/data/kraft" — the path simply doesn't exist in + // this container without it. + assert!( + mount_paths.contains(&STACKABLE_DATA_DIR), + "the sidecar must mount the data directory holding its own meta.properties, or add-controller can never read its own identity, got: {mount_paths:?}" + ); + } + + /// Confirmed live: the `smoke-kraft` test cluster's admission control rejects any pod + /// whose memory limit-to-request ratio isn't exactly 1 ("memory max limit to request + /// ratio per Container is 1, but provided ratio is 2.000000"), which is also what the + /// operator's own `stackable_operator::builder::pod` warning already flags. Every + /// container's memory request must equal its memory limit, not just the `quorum-manager` + /// sidecar's — this test covers all containers in both the broker and controller pods so + /// a future container addition can't reintroduce this for either role. + #[test] + fn every_container_has_a_1_to_1_memory_limit_to_request_ratio() { + let cluster = kraft_mode_cluster(); + let resources = build(&cluster).expect("build succeeds"); + + for sts in &resources.stateful_sets { + let pod_spec = sts + .spec + .as_ref() + .expect("the StatefulSet should have a spec") + .template + .spec + .as_ref() + .expect("the pod template should have a spec"); + for container in &pod_spec.containers { + let resources = container + .resources + .as_ref() + .unwrap_or_else(|| panic!("container {} has no resources set", container.name)); + let request = resources + .requests + .as_ref() + .and_then(|r| r.get("memory")) + .unwrap_or_else(|| { + panic!("container {} has no memory request set", container.name) + }); + let limit = resources + .limits + .as_ref() + .and_then(|l| l.get("memory")) + .unwrap_or_else(|| { + panic!("container {} has no memory limit set", container.name) + }); + assert_eq!( + request, limit, + "container {} must have memory request == memory limit (ratio 1:1); \ + the smoke-kraft test cluster's admission control rejects anything else", + container.name + ); + } + } + } + + /// Guards against `add_common_kafka_env`'s refactor (accepting a pre-computed + /// `node_id_offset: &str` instead of computing it internally) silently changing the + /// broker's own `NODE_ID_OFFSET` env var value. + #[test] + fn broker_node_id_offset_env_var_is_unchanged_by_the_shared_computation_refactor() { + let cluster = kraft_mode_cluster(); + let resources = build(&cluster).expect("build succeeds"); + + let broker_sts = resources + .stateful_sets + .iter() + .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-broker-default")) + .expect("the broker StatefulSet should be built"); + let kafka_container = broker_sts + .spec + .as_ref() + .expect("the StatefulSet should have a spec") + .template + .spec + .as_ref() + .expect("the pod template should have a spec") + .containers + .iter() + .find(|c| c.name == "kafka") + .expect("the broker pod should have a kafka container"); + + let node_id_offset_value = kafka_container + .env + .as_ref() + .expect("the kafka container should have env vars") + .iter() + .find(|env_var| env_var.name == "NODE_ID_OFFSET") + .and_then(|env_var| env_var.value.as_deref()) + .expect("NODE_ID_OFFSET should be set"); + + let expected = node_id_hash32_offset(&KafkaRole::Broker, "default").to_string(); + assert_eq!(node_id_offset_value, expected); + } + /// ZooKeeper mode has no `controller` role, so `build()` emits no controller resources while /// still producing the broker's bootstrap Listener. #[test] diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index 2f3951f5..c253f79b 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -25,6 +25,11 @@ pub enum ConfigFileName { Security, #[strum(serialize = "client.properties")] Client, + /// Client-side (unprefixed `security.protocol`/`ssl.*`) properties for an admin CLI tool + /// (e.g. `kafka-metadata-quorum.sh`) running inside a controller pod. Only written to + /// controller rolegroup `ConfigMap`s. + #[strum(serialize = "admin-client.properties")] + AdminClient, /// JAAS configuration for Kerberos authentication. It has the `.properties` /// extension but is not a Java properties file. #[strum(serialize = "jaas.properties")] @@ -56,18 +61,34 @@ pub fn uses_legacy_log4j(product_version: &str) -> bool { product_version.starts_with("3.") } +/// `controller.quorum.bootstrap.servers` addresses, one per distinct controller role group, +/// pointing at each role group's own headless Service DNS name rather than individual pod +/// FQDNs. +/// +/// A headless Service's own DNS name (no pod prefix) resolves to every backing pod's IP — +/// exactly what Kafka's own `client.dns.lookup=use_all_dns_ips` default already expects — and +/// the operator's headless Service sets `publishNotReadyAddresses: true`, so this also +/// resolves correctly during initial cluster formation before any pod is Ready. This is what +/// makes the value invariant to an existing controller role group's replica count: adding or +/// removing replicas within a role group never changes that role group's own Service name. +/// Only adding or removing a whole role group changes this list. pub(crate) fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Vec { - pod_descriptors + let mut role_group_addresses: Vec = pod_descriptors .iter() .filter(|pd| pd.role == KafkaRole::Controller) .map(|desc| { format!( - "{fqdn}:{client_port}", - fqdn = desc.fqdn(), - client_port = desc.client_port + "{service}.{namespace}.svc.{cluster_domain}:{client_port}", + service = desc.role_group_service_name, + namespace = desc.namespace, + cluster_domain = desc.cluster_domain, + client_port = desc.client_port, ) }) - .collect::>() + .collect(); + role_group_addresses.sort(); + role_group_addresses.dedup(); + role_group_addresses } #[cfg(test)] @@ -86,8 +107,128 @@ mod tests { ); assert_eq!(ConfigFileName::Security.to_string(), "security.properties"); assert_eq!(ConfigFileName::Client.to_string(), "client.properties"); + assert_eq!( + ConfigFileName::AdminClient.to_string(), + "admin-client.properties" + ); assert_eq!(ConfigFileName::Jaas.to_string(), "jaas.properties"); assert_eq!(ConfigFileName::Log4j.to_string(), "log4j.properties"); assert_eq!(ConfigFileName::Log4j2.to_string(), "log4j2.properties"); } + + /// Builds a minimal [`KafkaPodDescriptor`] for the given role, replica and client port. + /// + /// `KafkaPodDescriptor`'s fields are `pub(crate)`, which is crate-wide (not + /// module-scoped) visibility in Rust, so this direct construction is legal from any + /// module inside `stackable-kafka-operator`, including this one. + fn pod_descriptor(role: KafkaRole, replica: u16, client_port: u16) -> KafkaPodDescriptor { + KafkaPodDescriptor { + namespace: "default".parse().expect("valid namespace name"), + role_group_statefulset_name: "kafka-controller-default" + .parse() + .expect("valid statefulset name"), + role_group_service_name: "kafka-controller-default-headless" + .parse() + .expect("valid service name"), + replica, + cluster_domain: stackable_operator::commons::networking::DomainName::try_from( + "cluster.local", + ) + .expect("valid domain"), + node_id: replica.into(), + role, + client_port: client_port.into(), + } + } + + /// `kraft_controllers` points at the controller role group's *headless Service* DNS name + /// (no pod prefix), not individual pod FQDNs. A headless Service's own DNS name resolves + /// to every backing pod's IP (Kafka's own AdminClient default, + /// `client.dns.lookup=use_all_dns_ips`, already expects exactly this), and the + /// operator's headless Service sets `publishNotReadyAddresses: true`, so this also works + /// during initial cluster formation before any pod is Ready. This is what makes + /// `controller.quorum.bootstrap.servers` invariant to the controller role group's + /// replica count: adding or removing replicas within an existing role group never + /// changes the role group's own Service name. + #[test] + fn kraft_controllers_points_at_the_role_group_headless_service_not_individual_pods() { + let pod_descriptors = vec![ + pod_descriptor(KafkaRole::Controller, 0, 9093), + pod_descriptor(KafkaRole::Controller, 1, 9093), + pod_descriptor(KafkaRole::Controller, 2, 9093), + // Brokers must be filtered out of the controller quorum bootstrap servers list. + pod_descriptor(KafkaRole::Broker, 0, 9092), + ]; + + let quorum_bootstrap_servers = kraft_controllers(&pod_descriptors).join(","); + + assert_eq!( + quorum_bootstrap_servers, + "kafka-controller-default-headless.default.svc.cluster.local:9093" + ); + } + + /// The whole point: scaling an existing controller role group up or down must not change + /// `kraft_controllers`'s output at all, since it no longer depends on which replicas + /// currently exist — the role group's Service name is stable regardless. + #[test] + fn kraft_controllers_is_stable_across_replica_count_changes() { + let three_replicas = vec![ + pod_descriptor(KafkaRole::Controller, 0, 9093), + pod_descriptor(KafkaRole::Controller, 1, 9093), + pod_descriptor(KafkaRole::Controller, 2, 9093), + ]; + let five_replicas = vec![ + pod_descriptor(KafkaRole::Controller, 0, 9093), + pod_descriptor(KafkaRole::Controller, 1, 9093), + pod_descriptor(KafkaRole::Controller, 2, 9093), + pod_descriptor(KafkaRole::Controller, 3, 9093), + pod_descriptor(KafkaRole::Controller, 4, 9093), + ]; + + assert_eq!( + kraft_controllers(&three_replicas), + kraft_controllers(&five_replicas) + ); + } + + /// Multiple controller role groups each have their own headless Service, so each must + /// still get its own bootstrap-servers entry — deduplication is per-Service, not a + /// blanket "collapse everything to one entry". + #[test] + fn kraft_controllers_lists_every_distinct_role_groups_service_once() { + let mut default_group_pod = pod_descriptor(KafkaRole::Controller, 0, 9093); + let mut other_group_pod = pod_descriptor(KafkaRole::Controller, 0, 9093); + other_group_pod.role_group_statefulset_name = "kafka-controller-other" + .parse() + .expect("valid statefulset name"); + other_group_pod.role_group_service_name = "kafka-controller-other-headless" + .parse() + .expect("valid service name"); + // Second replica of the *same* role group as `default_group_pod` — must not produce + // a second entry for that Service. + let default_group_pod_replica_1 = { + let mut pod = pod_descriptor(KafkaRole::Controller, 1, 9093); + pod.node_id = 1; + pod + }; + default_group_pod.node_id = 0; + + let pod_descriptors = vec![ + default_group_pod, + default_group_pod_replica_1, + other_group_pod, + ]; + + let mut quorum_bootstrap_servers = kraft_controllers(&pod_descriptors); + quorum_bootstrap_servers.sort(); + + assert_eq!( + quorum_bootstrap_servers, + vec![ + "kafka-controller-default-headless.default.svc.cluster.local:9093".to_string(), + "kafka-controller-other-headless.default.svc.cluster.local:9093".to_string(), + ] + ); + } } diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index 284a281b..1869b5e6 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -17,7 +17,7 @@ use crate::{ properties::{ ConfigFileName, config_file_name, product_logging::role_group_config_map_data, }, - security::client_properties, + security::{client_properties, controller_admin_client_properties}, }, }, crd::{ @@ -50,6 +50,16 @@ pub enum Error { role_group: RoleGroupName, }, + #[snafu(display( + "failed to serialize client-side connection properties ([{}] or [{}]) for role group {role_group}", + ConfigFileName::Client, + ConfigFileName::AdminClient + ))] + ClientProperties { + source: PropertiesWriterError, + role_group: RoleGroupName, + }, + #[snafu(display("failed to build pod descriptors"))] BuildPodDescriptors { source: crate::controller::PodDescriptorsError, @@ -159,7 +169,7 @@ pub fn build_rolegroup_config_map( .iter() .filter_map(|(k, v)| v.as_ref().map(|v| (k, v))), ) - .with_context(|_| JvmSecurityPropertiesSnafu { + .with_context(|_| ClientPropertiesSnafu { role_group: role_group_name.clone(), })?, ) @@ -172,6 +182,22 @@ pub fn build_rolegroup_config_map( jaas_config_file(kafka_security.has_kerberos_enabled()), ); + // `admin-client.properties` is only needed by the controller-side sidecar running + // `kafka-metadata-quorum.sh` against the CONTROLLER listener; brokers don't need it. + if let AnyConfig::Controller(_) = &validated_rg.config.config { + cm_builder.add_data( + ConfigFileName::AdminClient.to_string(), + to_java_properties_string( + controller_admin_client_properties(kafka_security) + .iter() + .filter_map(|(k, v)| v.as_ref().map(|v| (k, v))), + ) + .with_context(|_| ClientPropertiesSnafu { + role_group: role_group_name.clone(), + })?, + ); + } + tracing::debug!(?kafka_config, "Applied kafka config"); tracing::debug!(?jvm_sec_props, "Applied JVM config"); diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index c9147c55..669e3b27 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -20,7 +20,7 @@ use stackable_operator::{ apps::v1::{StatefulSet, StatefulSetSpec, StatefulSetUpdateStrategy}, core::v1::{ ConfigMapVolumeSource, ContainerPort, EnvVar, EnvVarSource, ExecAction, - ObjectFieldSelector, PodSpec, Probe, TCPSocketAction, Volume, + LifecycleHandler, ObjectFieldSelector, PodSpec, Probe, TCPSocketAction, Volume, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, @@ -49,12 +49,14 @@ use crate::{ build::{ command::{ broker_kafka_container_commands, controller_kafka_container_command, - kafka_log_opts, kafka_log_opts_env_var, + kafka_log_opts, kafka_log_opts_env_var, quorum_manager_container_command, + quorum_manager_pre_stop_command, }, graceful_shutdown::add_graceful_shutdown_config, kerberos::add_kerberos_pod_config, properties::product_logging::MAX_KAFKA_LOG_FILES_SIZE, security::{ + STACKABLE_TLS_KAFKA_INTERNAL_DIR, STACKABLE_TLS_KAFKA_INTERNAL_VOLUME_NAME, add_broker_volume_and_volume_mounts, add_controller_volume_and_volume_mounts, kcat_prober_container_commands, }, @@ -124,6 +126,39 @@ fn common_operator_env_vars( env } +/// Environment variables the operator sets that are common to *every* container in a +/// **controller** pod: today that's the `kafka` server process and, when present, the +/// `quorum-manager` sidecar. +/// +/// The sidecar renders the very same `controller.properties` template (see +/// `properties/controller_properties.rs`) that the `kafka` container's own entrypoint does, to +/// build its own `add-controller`/`remove-controller` config — so it needs every +/// `${env:...}` placeholder that template references (`POD_NAME`, `KAFKA_CLIENT_PORT`, +/// `NAMESPACE`, `ROLEGROUP_HEADLESS_SERVICE_NAME`, `CLUSTER_DOMAIN`). Building this set once +/// and handing it to both containers means they can't silently drift apart over time (a real +/// bug found in review: the sidecar was originally given only `POD_NAME`/`NODE_ID_OFFSET`, +/// so its `controller.properties` render most likely produced a broken `listeners` value). +/// +/// The caller merges the user's `envOverrides` on top (so a user override wins on a name +/// collision) and, for the `kafka` container only, adds container-specific env vars such as +/// `PRE_STOP_CONTROLLER_SLEEP_SECONDS`. +fn controller_pod_shared_env_vars( + validated_cluster: &ValidatedCluster, + kafka_security: &ValidatedKafkaSecurity, + resource_names: &ResourceNames, +) -> EnvVarSet { + common_operator_env_vars(validated_cluster, kafka_security) + .with_field_path(&env_var_name("NAMESPACE"), &FieldPathEnvVar::Namespace) + .with_value( + &env_var_name("ROLEGROUP_HEADLESS_SERVICE_NAME"), + resource_names.headless_service_name().to_string(), + ) + .with_value( + &env_var_name("CLUSTER_DOMAIN"), + validated_cluster.cluster_domain.to_string(), + ) +} + const POD_MANAGEMENT_POLICY_PARALLEL: &str = "Parallel"; #[derive(Snafu, Debug)] @@ -266,14 +301,11 @@ pub fn build_broker_rolegroup_statefulset( ]) .args(vec![broker_kafka_container_commands( validated_cluster.cluster_config.is_kraft_mode(), - // we need controller pods - validated_cluster - .pod_descriptors(Some(&KafkaRole::Controller)) - .context(BuildPodDescriptorsSnafu)?, kafka_security, - &resolved_product_image.product_version, )]); + let node_id_offset = node_id_hash32_offset(kafka_role, role_group_name.as_ref()).to_string(); + add_common_kafka_env( &mut cb_kafka, merged_config, @@ -281,8 +313,7 @@ pub fn build_broker_rolegroup_statefulset( .product_specific_common_config .jvm_argument_overrides, resolved_product_image, - kafka_role, - role_group_name, + &node_id_offset, )?; cb_kafka @@ -472,22 +503,30 @@ pub fn build_controller_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); + let node_id_offset = node_id_hash32_offset(kafka_role, role_group_name.as_ref()).to_string(); + // Operator-set env vars first (common + controller-specific); the user's `envOverrides` - // are merged on top and win. - let env: Vec = common_operator_env_vars(validated_cluster, kafka_security) - .with_field_path(&env_var_name("NAMESPACE"), &FieldPathEnvVar::Namespace) - .with_value( - &env_var_name("ROLEGROUP_HEADLESS_SERVICE_NAME"), - resource_names.headless_service_name().to_string(), - ) - .with_value( - &env_var_name("CLUSTER_DOMAIN"), - validated_cluster.cluster_domain.to_string(), - ) + // are merged on top and win. Shared between the `kafka` container and the + // `quorum-manager` sidecar (see `controller_pod_shared_env_vars`) so they can't drift + // apart; each container then layers its own additions on top. + let controller_shared_env = + controller_pod_shared_env_vars(validated_cluster, kafka_security, &resource_names); + + let env: Vec = controller_shared_env + .clone() .with_value(&env_var_name("PRE_STOP_CONTROLLER_SLEEP_SECONDS"), "10") .merge(validated_rg.env_overrides.clone()) .into(); + let quorum_manager_env: Vec = controller_shared_env + .with_value(&env_var_name(KAFKA_NODE_ID_OFFSET), &node_id_offset) + .merge(validated_rg.env_overrides.clone()) + .into(); + + let controller_pod_descriptors = validated_cluster + .pod_descriptors(Some(kafka_role)) + .context(BuildPodDescriptorsSnafu)?; + cb_kafka .image_from_product_image(resolved_product_image) .command(vec![ @@ -498,10 +537,7 @@ pub fn build_controller_rolegroup_statefulset( "-c".to_string(), ]) .args(vec![controller_kafka_container_command( - validated_cluster - .pod_descriptors(Some(kafka_role)) - .context(BuildPodDescriptorsSnafu)?, - &resolved_product_image.product_version, + controller_pod_descriptors, )]); add_common_kafka_env( @@ -511,8 +547,7 @@ pub fn build_controller_rolegroup_statefulset( .product_specific_common_config .jvm_argument_overrides, resolved_product_image, - kafka_role, - role_group_name, + &node_id_offset, )?; cb_kafka @@ -580,6 +615,12 @@ pub fn build_controller_rolegroup_statefulset( .add_container(kafka_container) .affinity(&merged_config.affinity); + if let Some(quorum_manager_container) = + build_quorum_manager_container(resolved_product_image, kafka_security, quorum_manager_env)? + { + pod_builder.add_container(quorum_manager_container); + } + add_common_pod_config( &mut pod_builder, &resource_names, @@ -669,13 +710,17 @@ fn container_ports(kafka_security: &ValidatedKafkaSecurity) -> Vec Result<(), Error> { cb_kafka .add_env_var( @@ -703,10 +748,7 @@ fn add_common_kafka_env( "CONTAINERDEBUG_LOG_DIRECTORY", format!("{STACKABLE_LOG_DIR}/containerdebug"), ) - .add_env_var( - KAFKA_NODE_ID_OFFSET, - node_id_hash32_offset(kafka_role, role_group_name.as_ref()).to_string(), - ); + .add_env_var(KAFKA_NODE_ID_OFFSET, node_id_offset); Ok(()) } @@ -781,6 +823,93 @@ fn container_name(container: impl std::fmt::Display) -> ContainerName { .expect("a container enum variant is always a valid ContainerName") } +/// Name of the controller's `quorum-manager` sidecar container. +const QUORUM_MANAGER_CONTAINER_NAME: &str = "quorum-manager"; + +/// Builds the `quorum-manager` sidecar for a controller pod. Returns `None` when Kerberos is +/// enabled (the sidecar's admin-client properties file only covers the TLS/SSL case). +/// +/// `env` is expected to be [`controller_pod_shared_env_vars`] (plus `NODE_ID_OFFSET` and the +/// rolegroup's `envOverrides`) — the same base the `kafka` container in this pod gets — so +/// this sidecar's `controller.properties` render has every env var it references. See +/// [`controller_pod_shared_env_vars`] for why that matters. +fn build_quorum_manager_container( + resolved_product_image: &ResolvedProductImage, + kafka_security: &ValidatedKafkaSecurity, + env: Vec, +) -> Result, Error> { + if kafka_security.has_kerberos_enabled() { + return Ok(None); + } + + let mut cb = ContainerBuilder::new(QUORUM_MANAGER_CONTAINER_NAME).context( + InvalidContainerNameSnafu { + name: QUORUM_MANAGER_CONTAINER_NAME, + }, + )?; + + cb.image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-c".to_string(), + quorum_manager_container_command(), + ]) + .add_env_vars(env) + // `kafka-metadata-quorum.sh` goes through `kafka-run-class.sh`, which defaults + // `KAFKA_HEAP_OPTS` to `-Xmx256M` when unset. Set an explicit, modest heap so the + // JVM's max heap plus its base/metaspace/SSL-buffer overhead stays comfortably + // under the container's memory limit below. + .add_env_var(KAFKA_HEAP_OPTS, "-Xmx128M") + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + // A JVM cold start plus an SSL handshake and an admin-client round-trip all + // need to happen inside this sidecar's existing `timeout 15`/`25s preStop` + // budgets (see `CLI_CALL_TIMEOUT_SECONDS` in `command.rs`). + .with_cpu_limit("500m") + // Request must equal limit: the Stackable platform's admission control + // rejects any container whose memory limit-to-request ratio isn't exactly 1 + // (confirmed live: "memory max limit to request ratio per Container is 1, + // but provided ratio is 2.000000"). + .with_memory_request("512Mi") + .with_memory_limit("512Mi") + .build(), + ) + .add_volume_mount(STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_DIR) + .context(AddVolumeMountSnafu)? + // `controller_admin_client_properties` (see `build/security.rs`) always points + // its keystore/truststore at this directory, so the sidecar's admin-client calls + // need it mounted here too, not just on the `kafka` container. + .add_volume_mount( + STACKABLE_TLS_KAFKA_INTERNAL_VOLUME_NAME, + STACKABLE_TLS_KAFKA_INTERNAL_DIR, + ) + .context(AddVolumeMountSnafu)? + // `add-controller` reads this controller's own on-disk `meta.properties` (its + // `node.id`/`directory.id`, written by `kafka-storage.sh format`) from `log.dirs` in + // the merged config it connects with — confirmed live: without this mount, every + // `add-controller` attempt failed with "Unable to read meta.properties from + // /stackable/data/kraft", since that path doesn't exist in this container's + // filesystem at all without it. This mounts the *same* per-pod PVC the `kafka` + // container itself writes `meta.properties` into, read-write for parity with it + // (the CLI tool doesn't document a read-only requirement, and this repo has no + // read-only-mount helper to reach for). + .add_volume_mount(LOG_DIRS_VOLUME_NAME, STACKABLE_DATA_DIR) + .context(AddVolumeMountSnafu)? + .lifecycle_pre_stop(LifecycleHandler { + exec: Some(ExecAction { + command: Some(vec![ + "/bin/bash".to_string(), + "-c".to_string(), + quorum_manager_pre_stop_command(), + ]), + }), + ..LifecycleHandler::default() + }); + + Ok(Some(cb.build())) +} + fn add_vector_container( pod_builder: &mut PodBuilder, vector_container_name: &ContainerName, @@ -801,3 +930,310 @@ fn add_vector_container( )); } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::controller::test_support::{minimal_kafka, validated_cluster}; + + /// A minimal KRaft cluster with one controller role group, resolved through the real + /// validate step (mirroring the fixtures in `build/mod.rs`'s own tests), since + /// `ValidatedCluster` carries several resolved types that are impractical to construct by + /// hand. + fn kraft_mode_cluster() -> crate::controller::ValidatedCluster { + let kafka = minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + metadataManager: kraft + controllers: + roleGroups: + default: + replicas: 3 + brokers: + roleGroups: + default: + replicas: 3 + "#, + ); + validated_cluster(&kafka) + } + + /// End-to-end regression covering the whole point of removing `--initial-controllers` + /// (and the sidecar's own baked-in bootstrap-servers literal) from the controller pod + /// template: scaling an existing controller role group's replica count must not change + /// either container's `command`, or Kubernetes will roll every already-existing + /// controller pod on every scale-up/down, not just the ones actually being added or + /// removed. Confirmed live: before this fix, both the `kafka` container's format command + /// and the `quorum-manager` sidecar's bootstrap-servers literal changed with replica + /// count, forcing a full rolling restart on every scale operation. + #[test] + fn controller_pod_template_is_stable_across_replica_count_changes() { + let three_replicas = kraft_mode_cluster(); + let five_replicas = crate::controller::test_support::validated_cluster( + &crate::controller::test_support::minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + metadataManager: kraft + controllers: + roleGroups: + default: + replicas: 5 + brokers: + roleGroups: + default: + replicas: 3 + "#, + ), + ); + + let three_containers = controller_containers(&three_replicas); + let five_containers = controller_containers(&five_replicas); + + for name in ["kafka", QUORUM_MANAGER_CONTAINER_NAME] { + let three_command = three_containers + .iter() + .find(|c| c.name == name) + .unwrap_or_else(|| panic!("the {name} container is built (3 replicas)")) + .command + .clone(); + let five_command = five_containers + .iter() + .find(|c| c.name == name) + .unwrap_or_else(|| panic!("the {name} container is built (5 replicas)")) + .command + .clone(); + assert_eq!( + three_command, five_command, + "the {name} container's command must not change when only the replica count \ + of an existing controller role group changes" + ); + } + } + + fn controller_containers( + cluster: &crate::controller::ValidatedCluster, + ) -> Vec { + let resources = crate::controller::build::build(cluster).expect("build succeeds"); + let sts = resources + .stateful_sets + .into_iter() + .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-controller-default")) + .expect("the controller StatefulSet is built"); + sts.spec + .expect("the StatefulSet has a spec") + .template + .spec + .expect("the pod template has a spec") + .containers + } + + #[test] + fn controller_pods_get_a_quorum_manager_sidecar_on_supported_versions() { + let cluster = kraft_mode_cluster(); + let containers = controller_containers(&cluster); + + assert!( + containers + .iter() + .any(|c| c.name == QUORUM_MANAGER_CONTAINER_NAME), + "expected a quorum-manager sidecar, got containers: {:?}", + containers.iter().map(|c| &c.name).collect::>() + ); + } + + #[test] + fn quorum_manager_sidecar_targets_bootstrap_servers_in_its_command() { + let cluster = kraft_mode_cluster(); + let containers = controller_containers(&cluster); + let sidecar = containers + .iter() + .find(|c| c.name == QUORUM_MANAGER_CONTAINER_NAME) + .expect("the quorum-manager sidecar is built"); + + let command = sidecar + .command + .as_ref() + .expect("the sidecar has a command") + .join(" "); + assert!(command.contains("add-controller")); + + let pre_stop_command = sidecar + .lifecycle + .as_ref() + .and_then(|l| l.pre_stop.as_ref()) + .and_then(|h| h.exec.as_ref()) + .and_then(|e| e.command.as_ref()) + .expect("the sidecar has a preStop exec hook") + .join(" "); + assert!(pre_stop_command.contains("remove-controller")); + assert!(pre_stop_command.trim_end().ends_with("exit 0")); + } + + /// Every `${env:NAME}` placeholder found in a rendered Java properties (or similar) + /// string, in first-seen order, de-duplicated. + /// + /// The Java properties writer used to serialize the rendered `controller.properties` + /// escapes `:` as `\:` (`:` otherwise separates a properties key from its value), so a + /// placeholder actually appears as `${env\:NAME}` in the rendered ConfigMap content — + /// this accepts either form. + fn extract_env_placeholders(rendered: &str) -> Vec { + let mut result = Vec::new(); + let mut rest = rendered; + while let Some(start) = rest.find("${env") { + rest = &rest[start + "${env".len()..]; + rest = rest.strip_prefix('\\').unwrap_or(rest); + let Some(rest_after_colon) = rest.strip_prefix(':') else { + continue; + }; + rest = rest_after_colon; + let Some(end) = rest.find('}') else { + break; + }; + let name = rest[..end].to_string(); + if !result.contains(&name) { + result.push(name); + } + rest = &rest[end + 1..]; + } + result + } + + /// Regression test for a real bug found in review: `build_quorum_manager_container` once + /// set only `POD_NAME`/`NODE_ID_OFFSET` on the sidecar, while its own + /// `controller.properties` render (used to build the `add-controller` config, see + /// `command.rs`) needs `POD_NAME`, `ROLEGROUP_HEADLESS_SERVICE_NAME`, `NAMESPACE`, + /// `CLUSTER_DOMAIN` and `KAFKA_CLIENT_PORT` — so the rendered `listeners` value was most + /// likely broken (unresolved `${env:...}` placeholders). This asserts, from the actual + /// rendered `controller.properties` content, that every placeholder it references has a + /// matching env var on the sidecar container. + #[test] + fn quorum_manager_sidecar_has_every_env_var_controller_properties_rendering_references() { + let cluster = kraft_mode_cluster(); + let resources = crate::controller::build::build(&cluster).expect("build succeeds"); + + let controller_properties = resources + .config_maps + .iter() + .find(|cm| cm.metadata.name.as_deref() == Some("simple-kafka-controller-default")) + .expect("the controller rolegroup ConfigMap is built") + .data + .as_ref() + .expect("the ConfigMap carries data") + .get("controller.properties") + .expect("controller.properties is rendered into the ConfigMap") + .clone(); + + let placeholders = extract_env_placeholders(&controller_properties); + assert!( + placeholders.len() > 1, + "sanity check failed: expected multiple ${{env:...}} placeholders in the rendered \ + controller.properties, got: {placeholders:?}" + ); + + let containers = controller_containers(&cluster); + let sidecar = containers + .iter() + .find(|c| c.name == QUORUM_MANAGER_CONTAINER_NAME) + .expect("the quorum-manager sidecar is built"); + let sidecar_env_names: Vec<&str> = sidecar + .env + .as_ref() + .expect("the sidecar has env vars") + .iter() + .map(|e| e.name.as_str()) + .collect(); + + for placeholder in &placeholders { + // REPLICA_ID is not a Kubernetes-injected env var: both the `kafka` container's + // entrypoint and this sidecar's main-loop script derive and `export` it + // themselves from `$POD_NAME`/`$NODE_ID_OFFSET` before rendering the template + // (see `command.rs`), so it's expected to be absent from the container spec's + // `env` list. + if placeholder == "REPLICA_ID" { + continue; + } + assert!( + sidecar_env_names.contains(&placeholder.as_str()), + "quorum-manager sidecar is missing env var {placeholder:?}, which is \ + referenced by controller.properties's rendering; sidecar env vars: \ + {sidecar_env_names:?}" + ); + } + + // Targeted assertion (rather than relying on it only showing up incidentally among + // `placeholders` above): NODE_ID_OFFSET is consumed directly by the sidecar's own + // `EXPORT_REPLICA_ID` bash logic under `set -u` (see `command.rs`), so a regression + // here would break the sidecar's main loop and its `preStop` hook silently (an unset + // variable under `set -u` aborts the script). + assert!( + sidecar_env_names.contains(&KAFKA_NODE_ID_OFFSET), + "quorum-manager sidecar is missing the {KAFKA_NODE_ID_OFFSET} env var, needed by \ + its EXPORT_REPLICA_ID derivation under `set -u`; sidecar env vars: \ + {sidecar_env_names:?}" + ); + } + + #[test] + fn controller_pods_get_no_quorum_manager_sidecar_when_kerberos_is_enabled() { + // This is a Global Constraint (see the plan header): the sidecar's admin-client + // properties file only covers the TLS/SSL case, so it must never be added when + // Kerberos is enabled, even on an otherwise-supported Kafka version. + // + // Rather than building a full CRD-level Kerberos fixture (which needs a resolved + // AuthenticationClass threaded through `DereferencedObjects`, more than this test + // needs), call `build_quorum_manager_container` directly — it already takes + // `&ValidatedKafkaSecurity` as a parameter, so a fixture at that level is enough. + // Reuse the `kerberos()` fixture from `security.rs`'s existing test module (see + // Task 2). + let cluster = kraft_mode_cluster(); + let kerberos_security = crate::controller::build::security::tests::kerberos(); + + let result = build_quorum_manager_container(&cluster.image, &kerberos_security, Vec::new()) + .expect("build_quorum_manager_container does not error for a kerberos security value"); + + assert!(result.is_none()); + } + + #[test] + fn broker_pods_never_get_a_quorum_manager_sidecar() { + let cluster = kraft_mode_cluster(); + let resources = crate::controller::build::build(&cluster).expect("build succeeds"); + let sts = resources + .stateful_sets + .into_iter() + .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-broker-default")) + .expect("the broker StatefulSet is built"); + let containers = sts + .spec + .expect("the StatefulSet has a spec") + .template + .spec + .expect("the pod template has a spec") + .containers; + + assert!( + !containers + .iter() + .any(|c| c.name == QUORUM_MANAGER_CONTAINER_NAME) + ); + } + +} diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index e36191c1..87693836 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -49,8 +49,11 @@ const PROPERTY_SECURITY_PROTOCOL: &str = "security.protocol"; const PROPERTY_SASL_ENABLED_MECHANISMS: &str = "sasl.enabled.mechanisms"; const PROPERTY_SASL_KERBEROS_SERVICE_NAME: &str = "sasl.kerberos.service.name"; const PROPERTY_SASL_INTER_BROKER_MECHANISM: &str = "sasl.mechanism.inter.broker.protocol"; -const STACKABLE_TLS_KAFKA_INTERNAL_DIR: &str = "/stackable/tls-kafka-internal"; -const STACKABLE_TLS_KAFKA_INTERNAL_VOLUME_NAME: &str = "tls-kafka-internal"; +// Also mounted on the controller's `quorum-manager` sidecar (see +// `build_quorum_manager_container` in `build/resource/statefulset.rs`), since +// `controller_admin_client_properties` points its keystore/truststore here. +pub(crate) const STACKABLE_TLS_KAFKA_INTERNAL_DIR: &str = "/stackable/tls-kafka-internal"; +pub(crate) const STACKABLE_TLS_KAFKA_INTERNAL_VOLUME_NAME: &str = "tls-kafka-internal"; const STACKABLE_TLS_KAFKA_SERVER_DIR: &str = "/stackable/tls-kafka-server"; const STACKABLE_TLS_KAFKA_SERVER_VOLUME_NAME: &str = "tls-kafka-server"; // directories @@ -221,6 +224,34 @@ pub fn client_properties(security: &ValidatedKafkaSecurity) -> Vec<(String, Opti props } +/// Client-side (unprefixed `security.protocol`/`ssl.*`) properties for an admin CLI tool +/// (e.g. `kafka-metadata-quorum.sh`) talking to the CONTROLLER listener from *inside* a +/// controller pod, over the `tls-kafka-internal` volume mounted by +/// `add_controller_volume_and_volume_mounts`. +/// +/// This is deliberately separate from `client_properties()`: that function points at +/// `/stackable/tls-kafka-server`, a directory that is only mounted on broker pods. +/// +/// Internal (broker/controller) TLS is mandatory (see [`ValidatedKafkaSecurity::tls_internal_secret_class`]), +/// and `add_controller_volume_and_volume_mounts` unconditionally mounts both the keystore and +/// truststore on controller pods, independent of the external client TLS/authentication +/// settings. So, mirroring how `controller_config_settings` unconditionally writes the +/// CONTROLLER listener's keystore/truststore settings, this function always returns SSL +/// properties - there is no plaintext variant for the CONTROLLER listener. +pub fn controller_admin_client_properties( + _security: &ValidatedKafkaSecurity, +) -> Vec<(String, Option)> { + let mut properties = vec![]; + + properties.push(( + PROPERTY_SECURITY_PROTOCOL.to_string(), + Some(KafkaListenerProtocol::Ssl.to_string()), + )); + push_client_ssl_stores(&mut properties, STACKABLE_TLS_KAFKA_INTERNAL_DIR); + + properties +} + /// Adds required volumes and volume mounts to the broker pod and container builders /// depending on the tls and authentication settings. pub fn add_broker_volume_and_volume_mounts( @@ -651,7 +682,7 @@ fn kcat_client_sasl_ssl(cert_directory: &str, service_name: &str) -> Vec } #[cfg(test)] -mod tests { +pub(crate) mod tests { use std::{collections::BTreeMap, str::FromStr}; use stackable_operator::{ @@ -724,7 +755,7 @@ mod tests { } /// Kerberos, which also requires server and internal TLS. - fn kerberos() -> ValidatedKafkaSecurity { + pub(crate) fn kerberos() -> ValidatedKafkaSecurity { ValidatedKafkaSecurity::new( ResolvedAuthenticationClasses::new(vec![kerberos_auth_class()]), SecretClassName::from_str("tls").expect("tls secret class name is valid"), @@ -888,6 +919,62 @@ mod tests { assert!(props.contains_key("sasl.jaas.config")); } + // ---- controller_admin_client_properties ---- + + #[test] + fn controller_admin_client_properties_uses_the_internal_tls_directory() { + let security = server_tls(); + let props = as_map(controller_admin_client_properties(&security)); + + assert_eq!( + props.get("security.protocol"), + Some(&Some("SSL".to_string())) + ); + assert_eq!( + props.get("ssl.truststore.location"), + Some(&Some( + "/stackable/tls-kafka-internal/truststore.p12".to_string() + )) + ); + assert_eq!( + props.get("ssl.truststore.type"), + Some(&Some("PKCS12".to_string())) + ); + } + + #[test] + fn controller_admin_client_properties_includes_keystore_when_client_auth_is_required() { + let security = client_auth_tls(); + let props = as_map(controller_admin_client_properties(&security)); + + assert_eq!( + props.get("ssl.keystore.location"), + Some(&Some( + "/stackable/tls-kafka-internal/keystore.p12".to_string() + )) + ); + } + + #[test] + fn controller_admin_client_properties_always_uses_tls_even_without_external_client_tls() { + // Internal (broker/controller) TLS is mandatory (`tls_internal_secret_class()` always + // returns a SecretClass, defaulting to "tls"), and `add_controller_volume_and_volume_mounts` + // unconditionally mounts both the keystore and truststore on controller pods, independent + // of the external client TLS/authentication settings. So even the "plaintext" fixture + // (no external client TLS, no client-cert auth) still needs SSL to reach the CONTROLLER + // listener - mirroring `controller_config_settings`'s unconditional treatment of the same + // listener (see `controller_config_plaintext_has_internal_tls`). + let security = plaintext(); + let props = as_map(controller_admin_client_properties(&security)); + + assert_eq!( + props.get("security.protocol"), + Some(&Some("SSL".to_string())) + ); + assert!(props.contains_key("ssl.truststore.location")); + assert!(props.contains_key("ssl.keystore.location")); + } + // ---- broker_config_settings ---- #[test] diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 487a478c..bd25a842 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -334,42 +334,6 @@ pub struct KafkaPodDescriptor { pub client_port: Port, } -impl KafkaPodDescriptor { - /// Return the fully qualified domain name - /// Format: `...svc.` - pub fn fqdn(&self) -> String { - format!( - "{pod_name}.{service_name}.{namespace}.svc.{cluster_domain}", - pod_name = self.pod_name(), - service_name = self.role_group_service_name, - namespace = self.namespace, - cluster_domain = self.cluster_domain - ) - } - - pub fn pod_name(&self) -> String { - format!("{}-{}", self.role_group_statefulset_name, self.replica) - } - - /// Build the Kraft voter String - /// See: - /// Example: 0@controller-0:1234:0000000000-00000000000 - /// * 0 is the replica id - /// * 0000000000-00000000000 is the replica directory id (even though the used Uuid states to be type 4 it does not work) - /// See: - /// * controller-0 is the replica's host, - /// * 1234 is the replica's port. - // NOTE(@maltesander): Even though the used Uuid states to be type 4 it does not work... 0000000000-00000000000 works... - pub fn as_voter(&self) -> String { - format!( - "{node_id}@{fqdn}:{port}:0000000000-{node_id:0>11}", - node_id = self.node_id, - port = self.client_port, - fqdn = self.fqdn(), - ) - } -} - #[derive(Clone, Default, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KafkaClusterStatus { From 059e7d9b2664fe4e4ee4786ed4a51a8d59bf3645 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:26:24 +0200 Subject: [PATCH 02/13] feat: add startup/liveness/readiness probes for KRaft controllers Controllers get a startupProbe (plain TCP, generous failure threshold for slow metadata-log replay on boot), a plain-TCP livenessProbe, and a readinessProbe that checks the node's Raft state via its metrics endpoint instead of a bare TCP check, so a controller stuck rejoining the quorum is correctly reported as not ready. Co-Authored-By: Claude Sonnet 5 --- .../controller/build/resource/statefulset.rs | 173 +++++++++++++++--- 1 file changed, 151 insertions(+), 22 deletions(-) diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 669e3b27..9a1e39b6 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -39,7 +39,10 @@ use stackable_operator::{ STACKABLE_LOG_DIR, ValidatedContainerLogConfigChoice, vector_container, }, role_group_utils::ResourceNames, - types::kubernetes::{ConfigMapKey, ContainerName, PersistentVolumeClaimName, VolumeName}, + types::{ + common::Port, + kubernetes::{ConfigMapKey, ContainerName, PersistentVolumeClaimName, VolumeName}, + }, }, }; @@ -562,27 +565,30 @@ pub fn build_controller_rolegroup_statefulset( .add_volume_mount(STACKABLE_LOG_DIR_NAME, STACKABLE_LOG_DIR) .context(AddVolumeMountSnafu)? .resources(merged_config.resources().clone().into()) - // TODO: improve probes - .liveness_probe(Probe { - tcp_socket: Some(TCPSocketAction { - port: IntOrString::Int(kafka_security.client_port().into()), - ..Default::default() - }), - timeout_seconds: Some(10), - period_seconds: Some(10), - failure_threshold: Some(6), - ..Probe::default() - }) - .readiness_probe(Probe { - tcp_socket: Some(TCPSocketAction { - port: IntOrString::Int(kafka_security.client_port().into()), - ..Default::default() - }), - timeout_seconds: Some(10), - period_seconds: Some(10), - failure_threshold: Some(6), - ..Probe::default() - }); + // The controller listener socket only opens once the KRaft node has finished replaying + // its metadata log, which can take a while on a slow first boot or after a long outage. + // The startupProbe gives it up to 5 minutes (60 * 5s) before the liveness probe is + // allowed to start counting failures at all, so a slow (but progressing) boot is never + // mistaken for a stuck process. + .startup_probe(controller_tcp_probe( + kafka_security.client_port(), + /* timeout_seconds */ 5, + /* period_seconds */ 5, + /* failure_threshold */ 60, + )) + // Liveness intentionally stays a plain TCP check, same as startupProbe + .liveness_probe(controller_tcp_probe( + kafka_security.client_port(), + /* timeout_seconds */ 10, + /* period_seconds */ 10, + /* failure_threshold */ 6, + )) + .readiness_probe(controller_raft_state_probe( + METRICS_PORT, + /* timeout_seconds */ 10, + /* period_seconds */ 10, + /* failure_threshold */ 6, + )); add_log_config_volume( &mut pod_builder, @@ -681,6 +687,57 @@ pub fn build_controller_rolegroup_statefulset( }) } +/// A `Probe` that dials the controller's KRaft listener socket via a plain TCP connect. +/// +/// This only proves the socket is open, not that the node has a healthy Raft state (leader, +/// follower, or voted). It is intentionally still used for `startupProbe` (there is no +/// meaningful Raft state to check yet while the process is still starting) and for +/// `livenessProbe` (an unhealthy Raft state, e.g. `candidate`/`unattached`, means the node +/// cannot currently reach its peers, which restarting this pod cannot fix on its own). +fn controller_tcp_probe( + port: Port, + timeout_seconds: i32, + period_seconds: i32, + failure_threshold: i32, +) -> Probe { + Probe { + tcp_socket: Some(TCPSocketAction { + port: IntOrString::Int(port.into()), + ..Default::default() + }), + timeout_seconds: Some(timeout_seconds), + period_seconds: Some(period_seconds), + failure_threshold: Some(failure_threshold), + ..Probe::default() + } +} + +/// A `Probe` that curls the JMX Prometheus exporter's `/metrics` endpoint and checks that the +/// controller's Raft state is one of the healthy states (`leader`, `follower`, or `voted`) +/// rather than stuck in `unattached` or `candidate`. +fn controller_raft_state_probe( + metrics_port: Port, + timeout_seconds: i32, + period_seconds: i32, + failure_threshold: i32, +) -> Probe { + Probe { + exec: Some(ExecAction { + command: Some(vec![ + "bash".to_string(), + "-c".to_string(), + format!( + "curl -s localhost:{metrics_port}/metrics | grep -E 'kafka_server_raft_metrics_current_state\\{{state=\"(leader|follower|voted)\",?\\}}'" + ), + ]), + }), + timeout_seconds: Some(timeout_seconds), + period_seconds: Some(period_seconds), + failure_threshold: Some(failure_threshold), + ..Probe::default() + } +} + /// We only expose client HTTP / HTTPS and Metrics ports. fn container_ports(kafka_security: &ValidatedKafkaSecurity) -> Vec { let mut ports = vec![ @@ -933,6 +990,8 @@ fn add_vector_container( #[cfg(test)] mod tests { + use stackable_operator::k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; + use super::*; use crate::controller::test_support::{minimal_kafka, validated_cluster}; @@ -1236,4 +1295,74 @@ mod tests { ); } + fn controller_kafka_container( + cluster: &crate::controller::ValidatedCluster, + ) -> stackable_operator::k8s_openapi::api::core::v1::Container { + controller_containers(cluster) + .into_iter() + .find(|c| c.name == "kafka") + .expect("the kafka container is built") + } + + #[test] + fn controller_kafka_container_has_a_startup_probe() { + let cluster = kraft_mode_cluster(); + let container = controller_kafka_container(&cluster); + let client_port = cluster.cluster_config.kafka_security.client_port(); + + let startup_probe = container + .startup_probe + .expect("the controller kafka container must have a startupProbe"); + let tcp_socket = startup_probe + .tcp_socket + .expect("the startupProbe must be a tcpSocket check"); + assert_eq!(tcp_socket.port, IntOrString::Int(client_port.into())); + assert_eq!(startup_probe.timeout_seconds, Some(5)); + assert_eq!(startup_probe.period_seconds, Some(5)); + assert_eq!(startup_probe.failure_threshold, Some(60)); + } + + #[test] + fn controller_kafka_container_liveness_probe_is_a_plain_tcp_check() { + let cluster = kraft_mode_cluster(); + let container = controller_kafka_container(&cluster); + let client_port = cluster.cluster_config.kafka_security.client_port(); + + // Liveness intentionally stays a bare TCP check, not the Raft-state exec probe used for + // readiness: an unreachable-quorum Raft state is not something restarting this pod can + // fix, so liveness must not fail on it. + let liveness_probe = container + .liveness_probe + .expect("the controller kafka container must have a livenessProbe"); + let tcp_socket = liveness_probe + .tcp_socket + .expect("the livenessProbe must be a tcpSocket check, not an exec check"); + assert_eq!(tcp_socket.port, IntOrString::Int(client_port.into())); + assert_eq!(liveness_probe.timeout_seconds, Some(10)); + assert_eq!(liveness_probe.period_seconds, Some(10)); + assert_eq!(liveness_probe.failure_threshold, Some(6)); + } + + #[test] + fn controller_kafka_container_readiness_probe_checks_raft_state() { + let cluster = kraft_mode_cluster(); + let container = controller_kafka_container(&cluster); + + let readiness_probe = container.readiness_probe.expect("readiness probe is set"); + let exec = readiness_probe + .exec + .expect("readiness probe is an exec check"); + let command = exec.command.expect("exec has a command"); + assert_eq!( + command, + vec![ + "bash".to_string(), + "-c".to_string(), + "curl -s localhost:9606/metrics | grep -E 'kafka_server_raft_metrics_current_state\\{state=\"(leader|follower|voted)\",?\\}'".to_string(), + ] + ); + assert_eq!(readiness_probe.timeout_seconds, Some(10)); + assert_eq!(readiness_probe.period_seconds, Some(10)); + assert_eq!(readiness_probe.failure_threshold, Some(6)); + } } From f98c117194880c7a7ca913e68cfcf51cfbd25558 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:27:15 +0200 Subject: [PATCH 03/13] feat: use OrderedReady pod management for controller StatefulSets Controller pods now start/scale sequentially (OrderedReady) instead of in parallel, since the quorum-manager sidecar's admission flow assumes one voter joins at a time. Brokers are unaffected and keep Parallel. Co-Authored-By: Claude Sonnet 5 --- .../controller/build/resource/statefulset.rs | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 9a1e39b6..3708eea9 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -163,6 +163,7 @@ fn controller_pod_shared_env_vars( } const POD_MANAGEMENT_POLICY_PARALLEL: &str = "Parallel"; +const POD_MANAGEMENT_POLICY_ORDERED_READY: &str = "OrderedReady"; #[derive(Snafu, Debug)] pub enum Error { @@ -664,7 +665,7 @@ pub fn build_controller_rolegroup_statefulset( .with_label(RESTART_CONTROLLER_ENABLED_LABEL.to_owned()) .build(), spec: Some(StatefulSetSpec { - pod_management_policy: Some(POD_MANAGEMENT_POLICY_PARALLEL.to_string()), + pod_management_policy: Some(POD_MANAGEMENT_POLICY_ORDERED_READY.to_string()), update_strategy: Some(StatefulSetUpdateStrategy { type_: Some("RollingUpdate".to_string()), ..StatefulSetUpdateStrategy::default() @@ -1026,6 +1027,42 @@ mod tests { validated_cluster(&kafka) } + #[test] + fn controller_statefulset_uses_ordered_ready_pod_management() { + let cluster = kraft_mode_cluster(); + let resources = crate::controller::build::build(&cluster).expect("build succeeds"); + let sts = resources + .stateful_sets + .into_iter() + .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-controller-default")) + .expect("the controller StatefulSet is built"); + + assert_eq!( + sts.spec + .expect("the StatefulSet has a spec") + .pod_management_policy, + Some("OrderedReady".to_string()) + ); + } + + #[test] + fn broker_statefulset_still_uses_parallel_pod_management() { + let cluster = kraft_mode_cluster(); + let resources = crate::controller::build::build(&cluster).expect("build succeeds"); + let sts = resources + .stateful_sets + .into_iter() + .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-broker-default")) + .expect("the broker StatefulSet is built"); + + assert_eq!( + sts.spec + .expect("the StatefulSet has a spec") + .pod_management_policy, + Some("Parallel".to_string()) + ); + } + /// End-to-end regression covering the whole point of removing `--initial-controllers` /// (and the sidecar's own baked-in bootstrap-servers literal) from the controller pod /// template: scaling an existing controller role group's replica count must not change From 645061b2cdaf4ee6ab81289050920373e316087d Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:28:03 +0200 Subject: [PATCH 04/13] fix: reject and gracefully handle KRaft controllers scaled to zero Scaling controllers to 0 replicas while brokers keep running is now rejected at validation time with an actionable error, instead of failing much later and confusingly while building the broker's ConfigMap. Scaling controllers and brokers to 0 together (a coordinated whole-cluster stop) is still allowed and now actually builds, since downstream resource builders no longer assume a non-empty controller quorum whenever KRaft mode is active. Co-Authored-By: Claude Sonnet 5 --- rust/operator-binary/src/controller.rs | 9 +- .../src/controller/build/command.rs | 9 +- .../src/controller/build/mod.rs | 39 ++++ .../controller/build/resource/config_map.rs | 13 +- .../src/controller/validate.rs | 204 +++++++++++++++++- 5 files changed, 258 insertions(+), 16 deletions(-) diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 2e3ca3a2..f04814ba 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -667,6 +667,14 @@ pub(crate) mod test_support { /// Runs the real validate step against a minimal (auth/OPA-free) fixture. pub fn validated_cluster(kafka: &v1alpha1::KafkaCluster) -> ValidatedCluster { + validate_err(kafka).expect("validate should succeed for the test fixture") + } + + /// Runs the real validate step against a minimal (auth/OPA-free) fixture, without unwrapping + /// the result -- for tests asserting on a specific validation failure. + pub fn validate_err( + kafka: &v1alpha1::KafkaCluster, + ) -> Result { validate( kafka, DereferencedObjects { @@ -677,7 +685,6 @@ pub(crate) mod test_support { }, &operator_environment(), ) - .expect("validate should succeed for the test fixture") } } diff --git a/rust/operator-binary/src/controller/build/command.rs b/rust/operator-binary/src/controller/build/command.rs index ebd097a1..dd985237 100644 --- a/rust/operator-binary/src/controller/build/command.rs +++ b/rust/operator-binary/src/controller/build/command.rs @@ -188,14 +188,17 @@ wait_for_termination() /// system, not something this operator (which deliberately has no live-cluster awareness) /// can detect or repair automatically. See `kraft-controller.adoc`. fn controller_quorum_format_flag(controller_descriptors: &[KafkaPodDescriptor]) -> String { + // Empty only when the controller role group itself is scaled to 0 replicas -- which + // `validate` only allows together with brokers also at 0 (a coordinated whole-cluster + // stop, see `NoKraftControllerReplicas` in `controller/validate.rs`). The StatefulSet is + // still built in that case (just with 0 replicas), so this command template is assembled + // but never actually run by any pod; the placeholder node id is never observed. let bootstrap_node_id = controller_descriptors .iter() .filter(|descriptor| descriptor.role == KafkaRole::Controller) .map(|descriptor| descriptor.node_id) .min() - .expect( - "a controller StatefulSet is always built with at least one controller pod descriptor", - ); + .unwrap_or(0); formatdoc! {" if [ \"$REPLICA_ID\" = \"{bootstrap_node_id}\" ]; then diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 7ea36cad..4e5f4e09 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -216,6 +216,45 @@ mod tests { validated_cluster(&kafka) } + /// Confirmed live via the `operations-kraft` kuttl test: scaling both the controller and + /// broker role groups down to 0 replicas together (a coordinated whole-cluster stop, which + /// `validate` allows -- see `NoKraftControllerReplicas` in `controller/validate.rs`) used to + /// still fail to *build*, as `NoKraftControllersFound` while building the (unused) rolegroup + /// ConfigMaps: `pod_descriptors` comes back empty once every role group is at 0 replicas, + /// and `build_rolegroup_config_map` treated that as always broken in KRaft mode, without + /// distinguishing it from the "controllers at 0, brokers still running" case `validate` + /// actually rejects. No pod will ever read these ConfigMaps, so building them with an empty + /// controller quorum is harmless. + #[test] + fn build_succeeds_when_every_kraft_role_group_is_scaled_to_zero() { + let kafka = minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + metadataManager: kraft + controllers: + roleGroups: + default: + replicas: 0 + brokers: + roleGroups: + default: + replicas: 0 + "#, + ); + let cluster = validated_cluster(&kafka); + + build(&cluster).expect("build succeeds when the whole KRaft cluster is stopped"); + } + #[test] fn build_produces_expected_resource_names() { let cluster = kraft_mode_cluster(); diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index 1869b5e6..45b37d04 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -64,9 +64,6 @@ pub enum Error { BuildPodDescriptors { source: crate::controller::PodDescriptorsError, }, - - #[snafu(display("no Kraft controllers found to build"))] - NoKraftControllersFound, } /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator. @@ -92,14 +89,16 @@ pub fn build_rolegroup_config_map( .overrides .clone(); + // In KRaft mode, `pod_descriptors` can only be empty when *every* controller and broker + // role group is scaled to 0 replicas: `validate` rejects any other combination of zero + // controllers with running brokers before this point is ever reached (see + // `NoKraftControllerReplicas`), so a positive broker replica count anywhere guarantees a + // positive controller replica count, and vice versa. A whole-cluster-at-zero ConfigMap is + // harmless to build (no pod will ever read it), so there is nothing to reject here. let pod_descriptors = validated_cluster .pod_descriptors(None) .context(BuildPodDescriptorsSnafu)?; - if cluster_config.is_kraft_mode() && pod_descriptors.is_empty() { - return NoKraftControllersFoundSnafu.fail(); - } - let kafka_config = match &validated_rg.config.config { AnyConfig::Broker(_) => crate::controller::build::properties::broker_properties::build( cluster_config, diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 35405e63..be56d81d 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -105,6 +105,14 @@ pub enum Error { "the Vector aggregator discovery ConfigMap name is required when the Vector agent is enabled" ))] MissingVectorAggregatorConfigMapName, + + #[snafu(display( + "at least one KRaft controller replica is required while any broker replicas are \ + configured; a KRaft cluster with zero controllers has no metadata quorum for its \ + brokers to use. Scale brokers to 0 as well (or use `clusterOperation.stopped`) for a \ + coordinated full stop" + ))] + NoKraftControllerReplicas, } /// Validated logging configuration for a Kafka role group's Kafka and (optional) Vector @@ -265,6 +273,10 @@ pub fn validate( ); role_group_configs.insert(KafkaRole::Broker, broker_groups); + let metadata_manager = kafka + .effective_metadata_manager() + .context(InvalidMetadataManagerSnafu)?; + // Controllers are optional: ZooKeeper-mode clusters have none, in which case they are simply // absent from both maps and not reconciled. if let Some(controller_role) = kafka.spec.controllers.as_ref() { @@ -277,6 +289,39 @@ pub fn validate( validate_controller_logging, &vector_aggregator_config_map_name, )?; + + // A KRaft cluster with zero controller replicas *and running brokers* is a broken + // half-state: the brokers expect a live metadata quorum that no longer exists, and + // every resource that reads the controller quorum's pod descriptors (including the + // broker's own ConfigMap, which renders `controller.quorum.bootstrap.servers` from + // them) would fail to build. Reject that combination here, at validation time, with an + // actionable message, instead of letting it surface downstream as + // `NoKraftControllersFound` while building an unrelated ConfigMap. + // + // Controllers *and* brokers at zero together is not rejected: that is exactly what + // `clusterOperation.stopped` already does today, unconditionally, for every Stackable + // operator (scaling every managed StatefulSet's replicas to 0 at apply time, bypassing + // this check entirely since it only inspects the raw, pre-`stopped` spec) -- so a + // coordinated whole-cluster stop is already a supported shape, not one this check can + // meaningfully forbid. + // + // `replicas: None` (left for a HorizontalPodAutoscaler to own) is never treated as + // zero, for either role -- only an explicit, summed-to-zero replica count is. + let controller_replicas: u16 = controller_groups + .values() + .map(|rg| rg.replicas.unwrap_or(1)) + .sum(); + let broker_replicas: u16 = role_group_configs[&KafkaRole::Broker] + .values() + .map(|rg| rg.replicas.unwrap_or(1)) + .sum(); + if metadata_manager == crate::crd::MetadataManager::KRaft + && controller_replicas == 0 + && broker_replicas > 0 + { + return NoKraftControllerReplicasSnafu.fail(); + } + role_configs.insert( KafkaRole::Controller, ValidatedRoleConfig { @@ -286,10 +331,6 @@ pub fn validate( role_group_configs.insert(KafkaRole::Controller, controller_groups); } - let metadata_manager = kafka - .effective_metadata_manager() - .context(InvalidMetadataManagerSnafu)?; - let name = get_cluster_name(kafka).context(ResolveClusterNameSnafu)?; let namespace = get_namespace(kafka).context(ResolveNamespaceSnafu)?; let uid = get_uid(kafka).context(ResolveUidSnafu)?; @@ -420,7 +461,7 @@ mod tests { types::operator::RoleGroupName, }; - use super::{KAFKA_CLUSTER_ID_ENV, inject_cluster_id}; + use super::{Error, KAFKA_CLUSTER_ID_ENV, inject_cluster_id}; use crate::{ controller::test_support::{app_version_label, minimal_kafka, validated_cluster}, crd::role::KafkaRole, @@ -540,4 +581,157 @@ mod tests { let env = inject_cluster_id(EnvVarSet::new(), None).unwrap(); assert_eq!(cluster_id_value(&env), None); } + + /// Confirmed live: scaling a KRaft cluster's only controller role group down to 0 replicas + /// while brokers keep running used to pass validation and fail much later and much more + /// confusingly, as `NoKraftControllersFound` while building the *broker* role group's + /// ConfigMap (which also reads the controller quorum's pod descriptors, to render + /// `controller.quorum.bootstrap.servers`). Brokers with zero controllers have no metadata + /// quorum to talk to at all, so this combination must be rejected here, at validation time, + /// with a message that actually names the real problem. + #[test] + fn kraft_mode_rejects_zero_controller_replicas_while_brokers_are_running() { + let kafka = minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + metadataManager: kraft + controllers: + roleGroups: + default: + replicas: 0 + brokers: + roleGroups: + default: + replicas: 3 + "#, + ); + + let result = crate::controller::test_support::validate_err(&kafka); + let Err(error) = result else { + panic!( + "validate should reject zero controller replicas while brokers are running in KRaft mode" + ); + }; + + assert!( + matches!(error, Error::NoKraftControllerReplicas), + "expected NoKraftControllerReplicas, got: {error:?}" + ); + } + + /// The same zero-sum check, but split across two controller role groups (0 + 0): neither + /// group alone looks suspicious, only their sum does. + #[test] + fn kraft_mode_rejects_zero_controller_replicas_summed_across_role_groups() { + let kafka = minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + metadataManager: kraft + controllers: + roleGroups: + a: + replicas: 0 + b: + replicas: 0 + brokers: + roleGroups: + default: + replicas: 3 + "#, + ); + + let result = crate::controller::test_support::validate_err(&kafka); + let Err(error) = result else { + panic!( + "validate should reject zero controller replicas while brokers are running in KRaft mode" + ); + }; + + assert!( + matches!(error, Error::NoKraftControllerReplicas), + "expected NoKraftControllerReplicas, got: {error:?}" + ); + } + + /// Controllers *and* brokers at zero together is not rejected: that is exactly what + /// `clusterOperation.stopped` already does today, unconditionally, for every Stackable + /// operator, bypassing this check entirely -- a coordinated whole-cluster stop is already a + /// supported shape, not a broken half-state the way controllers-only-at-zero is. + #[test] + fn kraft_mode_allows_controllers_and_brokers_at_zero_together() { + let kafka = minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + metadataManager: kraft + controllers: + roleGroups: + default: + replicas: 0 + brokers: + roleGroups: + default: + replicas: 0 + "#, + ); + + let _cluster = validated_cluster(&kafka); + } + + /// A `replicas: 0` controller role group is fine in ZooKeeper mode: the check only applies + /// to KRaft, where controllers *are* the metadata quorum. + #[test] + fn zookeeper_mode_allows_zero_controller_replicas() { + let kafka = minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.9.2 + clusterConfig: + zookeeperConfigMapName: zk-discovery + controllers: + roleGroups: + default: + replicas: 0 + brokers: + roleGroups: + default: + replicas: 3 + "#, + ); + + let _cluster = validated_cluster(&kafka); + } } From 930a49740f9ab1b4cccff836b20d185c06f26e3c Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:28:17 +0200 Subject: [PATCH 05/13] test: cover KRaft controller scale-up/down/shutdown in kuttl operations tests Enable the previously version-gated scale-up/down steps (Kafka 3.7 no longer needs special-casing), assert quorum voter counts via kafka-metadata-quorum.sh after each scale, and add a final step scaling both controllers and brokers to 0 to exercise the whole-cluster-stop path before namespace teardown. Co-Authored-By: Claude Sonnet 5 --- .../kuttl/operations-kraft/60-assert.yaml.j2 | 14 +++++- .../60-scale-controller-up.yaml.j2 | 2 - .../kuttl/operations-kraft/70-assert.yaml.j2 | 14 +++++- .../70-scale-controller-down.yaml.j2 | 2 - .../kuttl/operations-kraft/80-assert.yaml.j2 | 2 - .../80-scale-broker-down.yaml.j2 | 2 - .../kuttl/operations-kraft/90-assert.yaml.j2 | 18 ++++++++ .../90-controller-shutdown.yaml.j2 | 45 +++++++++++++++++++ .../kuttl/operations-kraft/README.md | 14 ------ 9 files changed, 87 insertions(+), 26 deletions(-) create mode 100644 tests/templates/kuttl/operations-kraft/90-assert.yaml.j2 create mode 100644 tests/templates/kuttl/operations-kraft/90-controller-shutdown.yaml.j2 delete mode 100644 tests/templates/kuttl/operations-kraft/README.md diff --git a/tests/templates/kuttl/operations-kraft/60-assert.yaml.j2 b/tests/templates/kuttl/operations-kraft/60-assert.yaml.j2 index 61968a8a..0dce51c1 100644 --- a/tests/templates/kuttl/operations-kraft/60-assert.yaml.j2 +++ b/tests/templates/kuttl/operations-kraft/60-assert.yaml.j2 @@ -1,10 +1,20 @@ -{% if not test_scenario['values']['kafka-kraft'].startswith("3.7") %} --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 commands: - script: kubectl -n $NAMESPACE wait --for=condition=available kafkaclusters.kafka.stackable.tech/test-kafka --timeout 301s + - script: | + # :9093 is the TLS client port of this test fixture's default security config, not a + # fixed Kafka port - if the fixture's TLS/port config changes, update this too. + kubectl exec -n $NAMESPACE test-kafka-controller-default-0 -c kafka -- \ + /stackable/kafka/bin/kafka-metadata-quorum.sh \ + --bootstrap-controller test-kafka-controller-default-0.test-kafka-controller-default-headless.$NAMESPACE.svc.cluster.local:9093 \ + --command-config /stackable/config/admin-client.properties \ + describe --replication | tail -n +2 | awk '$NF == "Leader" || $NF == "Follower"' | wc -l | grep -q '^5$' + # `timeout` is known-inert here: kuttl's TestAssert `commands` don't read this field (only + # TestStep commands do); left in place only as documentation of the intended budget. + timeout: 30 --- apiVersion: apps/v1 kind: StatefulSet @@ -13,6 +23,7 @@ metadata: status: readyReplicas: 3 replicas: 3 +--- apiVersion: apps/v1 kind: StatefulSet metadata: @@ -20,4 +31,3 @@ metadata: status: readyReplicas: 5 replicas: 5 -{% endif %} diff --git a/tests/templates/kuttl/operations-kraft/60-scale-controller-up.yaml.j2 b/tests/templates/kuttl/operations-kraft/60-scale-controller-up.yaml.j2 index 3fdc5c4d..5ce9614a 100644 --- a/tests/templates/kuttl/operations-kraft/60-scale-controller-up.yaml.j2 +++ b/tests/templates/kuttl/operations-kraft/60-scale-controller-up.yaml.j2 @@ -1,4 +1,3 @@ -{% if not test_scenario['values']['kafka-kraft'].startswith("3.7") %} --- apiVersion: kuttl.dev/v1beta1 kind: TestStep @@ -38,4 +37,3 @@ spec: clusterOperation: stopped: false reconciliationPaused: false -{% endif %} diff --git a/tests/templates/kuttl/operations-kraft/70-assert.yaml.j2 b/tests/templates/kuttl/operations-kraft/70-assert.yaml.j2 index cd8c8ae2..b4de15cf 100644 --- a/tests/templates/kuttl/operations-kraft/70-assert.yaml.j2 +++ b/tests/templates/kuttl/operations-kraft/70-assert.yaml.j2 @@ -1,10 +1,20 @@ -{% if not test_scenario['values']['kafka-kraft'].startswith("3.7") %} --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 commands: - script: kubectl -n $NAMESPACE wait --for=condition=available kafkaclusters.kafka.stackable.tech/test-kafka --timeout 301s + - script: | + # :9093 is the TLS client port of this test fixture's default security config, not a + # fixed Kafka port - if the fixture's TLS/port config changes, update this too. + kubectl exec -n $NAMESPACE test-kafka-controller-default-0 -c kafka -- \ + /stackable/kafka/bin/kafka-metadata-quorum.sh \ + --bootstrap-controller test-kafka-controller-default-0.test-kafka-controller-default-headless.$NAMESPACE.svc.cluster.local:9093 \ + --command-config /stackable/config/admin-client.properties \ + describe --replication | tail -n +2 | awk '$NF == "Leader" || $NF == "Follower"' | wc -l | grep -q '^3$' + # `timeout` is known-inert here: kuttl's TestAssert `commands` don't read this field (only + # TestStep commands do); left in place only as documentation of the intended budget. + timeout: 30 --- apiVersion: apps/v1 kind: StatefulSet @@ -13,6 +23,7 @@ metadata: status: readyReplicas: 3 replicas: 3 +--- apiVersion: apps/v1 kind: StatefulSet metadata: @@ -20,4 +31,3 @@ metadata: status: readyReplicas: 3 replicas: 3 -{% endif %} diff --git a/tests/templates/kuttl/operations-kraft/70-scale-controller-down.yaml.j2 b/tests/templates/kuttl/operations-kraft/70-scale-controller-down.yaml.j2 index a077213b..a6ad4ec2 100644 --- a/tests/templates/kuttl/operations-kraft/70-scale-controller-down.yaml.j2 +++ b/tests/templates/kuttl/operations-kraft/70-scale-controller-down.yaml.j2 @@ -1,4 +1,3 @@ -{% if not test_scenario['values']['kafka-kraft'].startswith("3.7") %} --- apiVersion: kuttl.dev/v1beta1 kind: TestStep @@ -38,4 +37,3 @@ spec: clusterOperation: stopped: false reconciliationPaused: false -{% endif %} diff --git a/tests/templates/kuttl/operations-kraft/80-assert.yaml.j2 b/tests/templates/kuttl/operations-kraft/80-assert.yaml.j2 index a1d7088f..793f8aad 100644 --- a/tests/templates/kuttl/operations-kraft/80-assert.yaml.j2 +++ b/tests/templates/kuttl/operations-kraft/80-assert.yaml.j2 @@ -1,4 +1,3 @@ -{% if not test_scenario['values']['kafka-kraft'].startswith("3.7") %} --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert @@ -18,4 +17,3 @@ metadata: status: readyReplicas: 3 replicas: 3 -{% endif %} diff --git a/tests/templates/kuttl/operations-kraft/80-scale-broker-down.yaml.j2 b/tests/templates/kuttl/operations-kraft/80-scale-broker-down.yaml.j2 index d788a9c9..ee4cb139 100644 --- a/tests/templates/kuttl/operations-kraft/80-scale-broker-down.yaml.j2 +++ b/tests/templates/kuttl/operations-kraft/80-scale-broker-down.yaml.j2 @@ -4,7 +4,6 @@ # The brokers must be deleted because otherwise they are left dangling until # the test timeouts and fails. # -{% if not test_scenario['values']['kafka-kraft'].startswith("3.7") %} --- apiVersion: kuttl.dev/v1beta1 kind: TestStep @@ -44,4 +43,3 @@ spec: clusterOperation: stopped: false reconciliationPaused: false -{% endif %} diff --git a/tests/templates/kuttl/operations-kraft/90-assert.yaml.j2 b/tests/templates/kuttl/operations-kraft/90-assert.yaml.j2 new file mode 100644 index 00000000..41e7d53d --- /dev/null +++ b/tests/templates/kuttl/operations-kraft/90-assert.yaml.j2 @@ -0,0 +1,18 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-broker-default +status: + replicas: 0 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-controller-default +status: + replicas: 0 diff --git a/tests/templates/kuttl/operations-kraft/90-controller-shutdown.yaml.j2 b/tests/templates/kuttl/operations-kraft/90-controller-shutdown.yaml.j2 new file mode 100644 index 00000000..6ccff4ad --- /dev/null +++ b/tests/templates/kuttl/operations-kraft/90-controller-shutdown.yaml.j2 @@ -0,0 +1,45 @@ +# +# This is a test helper to ensure that all broker pods are deleted before +# the test namespace is terminated. +# The brokers must be deleted because otherwise they are left dangling until +# the test timeouts and fails. +# +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 600 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + image: +{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: + metadataManager: kraft + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + controllers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 0 + brokers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 0 + clusterOperation: + stopped: false + reconciliationPaused: false diff --git a/tests/templates/kuttl/operations-kraft/README.md b/tests/templates/kuttl/operations-kraft/README.md deleted file mode 100644 index 5c0fa86b..00000000 --- a/tests/templates/kuttl/operations-kraft/README.md +++ /dev/null @@ -1,14 +0,0 @@ -Tests Kraft cluster operations: - -- Cluster stop/pause/restart -- Scale brokers up/down -- Scale controllers up/down - -Notes - -- Kafka 3.7 controllers do not scale at all. - The scaling test steps are disabled for this version. -- Scaling controllers from 3 -> 1 doesn't work. - Both brokers and controllers try to communicate with old controllers. - This is why, the last step scales from 5 -> 3 controllers. - This at least, leaves the cluster in a working state. From 3074345ec24442ee30810fc8d0dc229399fa90a1 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:28:31 +0200 Subject: [PATCH 06/13] docs: document KRaft dynamic voter membership and update CHANGELOG Update the KRaft controller usage guide for scale-up/down support, record the design spec and implementation plan, add the CHANGELOG entries for this branch's changes, and ignore .worktrees/ for local worktree checkouts. Co-Authored-By: Claude Sonnet 5 --- .gitignore | 2 + CHANGELOG.md | 41 + .../pages/usage-guide/kraft-controller.adoc | 61 +- ...26-08-14-kraft-dynamic-voter-membership.md | 1008 +++++++++++++++++ ...4-kraft-dynamic-voter-membership-design.md | 197 ++++ 5 files changed, 1304 insertions(+), 5 deletions(-) create mode 100644 docs/superpowers/plans/2026-08-14-kraft-dynamic-voter-membership.md create mode 100644 docs/superpowers/specs/2026-08-14-kraft-dynamic-voter-membership-design.md diff --git a/.gitignore b/.gitignore index 696bc411..0ff5d1ec 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ tilt_options.json .envrc .DS_Store + +.worktrees/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e3b1a1a..ada29037 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,25 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- KRaft controller replicas can now be scaled up and down on a running cluster: a new + `quorum-manager` sidecar container on each controller pod is solely responsible for + admitting itself into the KRaft voter set on startup and removing itself before + termination. Exactly one controller bootstraps the quorum standalone at format time + (`kafka-storage.sh format --standalone`); every other controller, whether present from + the start or added later, formats with `--no-initial-controllers` and joins purely + through the sidecar. `controller.quorum.bootstrap.servers` now points at each controller + role group's headless Service DNS name instead of individual pod addresses, so neither + the container commands nor that ConfigMap value change with the replica count anymore. + Confirmed live: scaling a controller role group up or down leaves every already-existing + controller pod completely untouched (same UID, zero restarts, no `StatefulSet` revision + change) — only the pods actually being added or removed are touched ([#NNNN]). +- A `startupProbe` and a plain TCP `livenessProbe` for KRaft controllers, and a new + `readinessProbe` that checks the controller's Raft state (`leader`/`follower`/`voted`) via + its metrics endpoint instead of a bare TCP check, so a controller that can't join or + rejoin the quorum is correctly reported as not ready ([#1006]). + ### Changed - Internal operator refactoring: introduce a build() step in the reconciler that @@ -16,18 +35,40 @@ All notable changes to this project will be documented in this file. - All product containers now run with `securityContext.runAsNonRoot` set to `true` to improve security ([#998]). - The reconciler now applies resources and derives the cluster status in discrete apply and update_status steps ([#1000]). +- BREAKING: KRaft mode now requires Kafka 3.9.0 or later; Kafka 3.7.x is no longer supported and its previous + special-casing has been removed entirely, rather than narrowed. Running KRaft mode on an unsupported Kafka + version is undefined behavior ([#NNNN]). ### Fixed - Fix a longstanding problem of including empty `categories`, `shortNames` and `additionalPrinterColumns` in the CRDs, which could cause problems with GitOps tools (e.g. ArgoCD) reporting a diff in the custom resources. See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#998]). +- The `quorum-manager` sidecar's `preStop` hook no longer retries for the full 25s timeout + when a controller pod is the last remaining voter at termination (e.g. scaling controllers + down to a single replica, or the last surviving pod of a full teardown): removal is + correctly refused in that case, and confirmed live that retrying can never change that + outcome, so the hook now gives up immediately instead of retrying until the deadline ([#NNNN]). +- Scaling a KRaft cluster's controller role group(s) down to a total of 0 replicas while any + broker replicas are configured is now rejected up front, during validation, with an actionable + error message. Previously it passed validation and failed much later and much more + confusingly, as `no Kraft controllers found to build` while building the unrelated *broker* + role group's `ConfigMap`. Controllers and brokers at 0 replicas together is unaffected, since + that is what `clusterOperation.stopped` already does today ([#NNNN]). +- Scaling a KRaft cluster's controller *and* broker role groups down to 0 replicas together (a + coordinated whole-cluster stop, which the check above deliberately still allows) no longer + fails to build resources with `no Kraft controllers found to build`. That check only ever + guarded against a genuinely broken half-state; a whole-cluster-at-zero build is harmless since + no pod ever reads the resulting `ConfigMap`s or `StatefulSet`s, so it is no longer rejected + ([#NNNN]). [#985]: https://github.com/stackabletech/kafka-operator/pull/985 [#990]: https://github.com/stackabletech/kafka-operator/pull/990 [#994]: https://github.com/stackabletech/kafka-operator/pull/994 [#998]: https://github.com/stackabletech/kafka-operator/pull/998 [#1000]: https://github.com/stackabletech/kafka-operator/pull/1000 +[#1006]: https://github.com/stackabletech/kafka-operator/pull/1006 +[#NNNN]: https://github.com/stackabletech/kafka-operator/pull/NNNN ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc index 455188c9..3acdf8b5 100644 --- a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc +++ b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc @@ -16,6 +16,12 @@ WARNING: The Stackable Operator for Apache Kafka currently does not support auto * Full Replacement: Kafka 4.0.0 (2025) removes ZooKeeper completely. * Migration: Tools exist to migrate from ZooKeeper to KRaft, but new deployments should start with KRaft. +IMPORTANT: The Stackable Operator for Apache Kafka requires Kafka 3.9.0 or later for KRaft mode. Kafka 3.7.x is not +supported: the operator relies on the KIP-853 dynamic quorum tooling (`kafka-storage.sh format --standalone` / +`--no-initial-controllers`, and `kafka-metadata-quorum.sh add-controller` / `remove-controller`), which requires +3.9.0+. Running KRaft mode on an unsupported Kafka version is undefined behavior, up to and including +`kafka-storage.sh` rejecting the operator-generated formatting command outright. + == Configuration The Stackable Kafka operator introduces a new xref:concepts:roles-and-role-groups.adoc[role] in the KafkaCluster CRD called KRaft `Controller`. @@ -85,13 +91,49 @@ KRaft mode requires major configuration changes compared to ZooKeeper: * `cluster-id`: This is set to the `metadata.name` of the KafkaCluster resource during initial formatting * `node.id`: This is a calculated integer, hashed from the `role` and `rolegroup` and added `replica` id. * `process.roles`: Will always only be `broker` or `controller`. Mixed `broker,controller` servers are not supported. -* The operator configures a static voter list containing the controller pods. Controllers are not dynamically managed. +* Controller pods have a `startupProbe` and a plain TCP `livenessProbe` on the KRaft listener port, and a + `readinessProbe` that checks the controller's Raft state (`leader`, `follower`, or `voted`) via its metrics + endpoint rather than a bare TCP check, so a controller that cannot join or rejoin the quorum is correctly + reported as not ready instead of appearing healthy. +* Admitting a controller into the KRaft voter set is *solely* the concern of the `quorum-manager` sidecar container + (see below) — the format step never asserts a voter list. Exactly one controller (the one with the numerically + lowest `node.id` among all controller pod descriptors) formats with `kafka-storage.sh format --standalone`, + bootstrapping a single-node quorum by itself. Every other controller — whether it is part of the cluster's initial + desired replica count or added later on scale-up — formats with `--no-initial-controllers` and joins purely + through the sidecar's `add-controller` call. Brokers always format with `--no-initial-controllers` too; they are + never voters. Because no voter list is baked into any container's command, the command is identical regardless of + the current replica count. +* `controller.quorum.bootstrap.servers` (used by the `kafka` process itself to find the controller quorum, and by + the `quorum-manager` sidecar for its own `add-controller`/`remove-controller` calls) points at each controller + role group's own headless Service DNS name, not individual pod addresses. A headless Service's own DNS name + resolves to every backing pod's IP — exactly what Kafka's `client.dns.lookup=use_all_dns_ips` default already + expects — and the Service sets `publishNotReadyAddresses: true`, so this also resolves correctly during initial + cluster formation before any pod is `Ready`. Combined with the previous point, this makes the whole controller + pod template invariant to `spec.controllers.roleGroups..replicas`: confirmed live, scaling a controller + role group up or down leaves every already-existing controller pod completely untouched (same UID, zero + restarts, no `StatefulSet` revision change) — only the pods actually being added or removed are touched. +* When Kerberos is not enabled, each controller pod runs an additional `quorum-manager` sidecar container (requires + Kafka 3.9.0 or later, see the minimum-version note above). On startup it admits the pod into the KRaft voter set + (`kafka-metadata-quorum.sh add-controller`), and on pod termination (`preStop`) it removes the pod from the voter + set again (`remove-controller`), but only if doing so would not remove the last remaining voter. == Known Issues * Automatic migration from Apache ZooKeeper to KRaft is not supported. -* Scaling controller replicas might lead to unstable clusters. * Kerberos is currently not supported for KRaft in all versions. +* Scaling controllers down to a single replica is not verified under the sidecar-based mechanism described above; + only scale-downs that keep an odd number of controllers greater than one have been tested. +* If a `remove-controller` call fails or times out during pod termination (for example, no reachable leader within + the pod's grace period), the pod terminates anyway and can leave a stale voter entry in the quorum behind. This is + not fully automatic in every case and may require manual cleanup with `kafka-metadata-quorum.sh remove-controller`. +* The single controller chosen to bootstrap the quorum standalone (see "Internal operator details" above) is picked + by a stable, deterministic rule (lowest `node.id`), which is safe only for a cluster's *original* bootstrap. If + that specific controller's persistent volume is ever lost and needs to reformat after the cluster has already + formed a quorum elsewhere, reformatting it with `--standalone` would bootstrap a second, conflicting one-node + quorum instead of rejoining the existing one. This is the same class of manual-recovery scenario as losing enough + voters to break quorum in any Raft-based system, and the operator has no way to detect or repair it automatically + (it deliberately has no live-cluster awareness). If this happens, recovery requires manual intervention with + Kafka's own KRaft tooling. == Troubleshooting @@ -108,10 +150,19 @@ Likely caused by controller resource starvation or unstable Kubernetes schedulin Ensure Kafka version 3.9.x and higher and follow the official migration documentation. The Stackable Kafka operator currently does not support the migration. -=== Scaling issues +=== Scaling controllers + +Controller replicas can be scaled up and down on a running cluster. A per-pod `quorum-manager` sidecar admits and +removes the pod from the KRaft voter set as described under "Internal operator details" above. This requires Kafka +3.9.0 or later (see the minimum-version note in "Overview"), which supports the +https://developers.redhat.com/articles/2024/11/27/dynamic-kafka-controller-quorum[KIP-853 dynamic quorum tooling]. + +Scaling more than one controller down at a time is processed one pod at a time (`OrderedReady` pod management), not +in parallel, so that each pod's removal from the voter set can complete before the next one is terminated. -The https://developers.redhat.com/articles/2024/11/27/dynamic-kafka-controller-quorum[Dynamic scaling] is only supported from Kafka version 3.9.0. -If you are using older versions, automatic scaling may not work properly (e.g. adding or removing controller replicas). +This has been validated on a live cluster and is covered by unit tests, but a full end-to-end scale-up/scale-down +kuttl test suite run has not yet produced a clean pass; treat controller scaling as functional but not yet fully +verified end-to-end. == Kraft migration guide diff --git a/docs/superpowers/plans/2026-08-14-kraft-dynamic-voter-membership.md b/docs/superpowers/plans/2026-08-14-kraft-dynamic-voter-membership.md new file mode 100644 index 00000000..723f7612 --- /dev/null +++ b/docs/superpowers/plans/2026-08-14-kraft-dynamic-voter-membership.md @@ -0,0 +1,1008 @@ +# KRaft Dynamic Voter Membership Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let KRaft controller replicas be scaled up and down on a running Kafka cluster, by having each controller pod manage its own quorum membership via a sidecar container instead of the operator talking to the live cluster. + +**Architecture:** A new `quorum-manager` sidecar container (reusing the Kafka product image) runs alongside the `kafka` container on controller pods only. Its main process loops, admitting itself as a voter (`kafka-metadata-quorum.sh add-controller`) while its local Raft state is `observer`. Its `preStop` hook checks a majority-safety condition and removes itself (`remove-controller`) before termination. The controller StatefulSet switches `podManagementPolicy` to `OrderedReady` so Kubernetes drains controllers one at a time on scale-down. All of this is gated to Kafka versions that support KIP-853 dynamic quorum tooling (everything except the `3.7.x` line, mirroring the existing `--initial-controllers` version check). + +**Tech Stack:** Rust (`stackable-operator`, `kube-rs` builder types), Bash (sidecar scripts), Kafka's `kafka-metadata-quorum.sh` CLI, kuttl (integration tests). + +**Spec:** `docs/superpowers/specs/2026-08-14-kraft-dynamic-voter-membership-design.md` + +## Global Constraints + +- Sidecar is added only for the controller role (`build_controller_rolegroup_statefulset`), never brokers. +- Sidecar is added only when `!resolved_product_image.product_version.starts_with("3.7")` — same literal-prefix check style as `initial_controllers_command` (`rust/operator-binary/src/controller/build/command.rs:198-211`) and `uses_legacy_log4j` (`rust/operator-binary/src/controller/build/properties/mod.rs:55-57`). +- Sidecar is not added when `kafka_security.has_kerberos_enabled()` is true — Kerberos for KRaft is already a documented unsupported combination (`docs/modules/kafka/pages/usage-guide/kraft-controller.adoc`, Known Issues), and the sidecar's admin-client properties file only handles the TLS/SSL case. +- The sidecar's `preStop` script must always exit `0`, regardless of whether `remove-controller` succeeded — it must never block pod termination. +- The sidecar targets the quorum's bootstrap servers (its peers), never `localhost` for the admin-client calls — its own `kafka` container may be concurrently shutting down. +- No new Kubernetes RBAC, no new reconcile phase, no CRD status field. All Rust changes are confined to the `build` phase. +- Every `rust/` change must pass `cargo build`, `cargo test`, `cargo clippy --all-targets -- -D warnings`, `cargo fmt --check` before being considered done, per this repo's existing verification gate. CRDs/docs are regenerated (`make regenerate-charts`) whenever the CRD schema changes. +- A CHANGELOG entry is added in the same commit as the change it documents, under a new `### Added` section (there is currently no `### Added` section under `## [Unreleased]` in `CHANGELOG.md`). + +--- + +## Task 1: Version gate helper — `supports_dynamic_quorum` + +**Files:** + +- Modify: `rust/operator-binary/src/controller/build/properties/mod.rs` +- Test: same file, `#[cfg(test)] mod tests` (create if absent — check first, this file may already have one) + +**Interfaces:** + +- Produces: `pub fn supports_dynamic_quorum(product_version: &str) -> bool` — used by Task 4 (sidecar container gating) and Task 7 (kuttl test gating verification). + +- [ ] **Step 1: Check for an existing test module in this file** + +Run: `grep -n "mod tests" rust/operator-binary/src/controller/build/properties/mod.rs` + +If it exists, note the line number — new tests go inside it. If not, one will be created in Step 3. + +- [ ] **Step 2: Write the failing test** + +Add near the existing `uses_legacy_log4j` function (`rust/operator-binary/src/controller/build/properties/mod.rs:55-57`): + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dynamic_quorum_is_supported_from_3_9_onwards() { + assert!(supports_dynamic_quorum("3.9.2")); + assert!(supports_dynamic_quorum("4.1.1")); + assert!(supports_dynamic_quorum("4.2.1")); + } + + #[test] + fn dynamic_quorum_is_not_supported_on_3_7() { + assert!(!supports_dynamic_quorum("3.7.2")); + } +} +``` + +If a `mod tests` block already exists in this file, add these two `#[test]` functions inside it instead of writing a new module, and skip the `use super::*;` line if it's already present. + +- [ ] **Step 3: Run the test to verify it fails** + +Run: `cargo test -p stackable-kafka-operator-binary supports_dynamic_quorum 2>&1 | tail -30` +Expected: compile error, `supports_dynamic_quorum` not found. + +- [ ] **Step 4: Write the minimal implementation** + +Add next to `uses_legacy_log4j`: + +```rust +/// Whether this Kafka version supports the KIP-853 dynamic KRaft quorum tooling +/// (`kafka-metadata-quorum.sh add-controller` / `remove-controller`) needed to change +/// the voter set of an already-formed quorum. Mirrors the existing 3.7.x carve-out +/// already used for `--initial-controllers` (see `initial_controllers_command` in +/// `build/command.rs`). +pub fn supports_dynamic_quorum(product_version: &str) -> bool { + !product_version.starts_with("3.7") +} +``` + +- [ ] **Step 5: Run the test to verify it passes** + +Run: `cargo test -p stackable-kafka-operator-binary supports_dynamic_quorum 2>&1 | tail -30` +Expected: both tests PASS. + +- [ ] **Step 6: Commit** + +```bash +git add rust/operator-binary/src/controller/build/properties/mod.rs +git commit -m "feat: add supports_dynamic_quorum version gate + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +## Task 2: Admin-client properties file for the sidecar's TLS config + +**Files:** + +- Modify: `rust/operator-binary/src/controller/build/security.rs` +- Modify: `rust/operator-binary/src/controller/build/resource/config_map.rs` +- Test: `rust/operator-binary/src/controller/build/security.rs`, existing `#[cfg(test)] mod tests` (lines 653-992) + +**Interfaces:** + +- Consumes: `push_client_ssl_stores` (`security.rs:374-388`), `push_client_ssl_truststore` (`security.rs:392-405`), `STACKABLE_TLS_KAFKA_INTERNAL_DIR` (`security.rs:44`), `PROPERTY_SECURITY_PROTOCOL` (`security.rs:41`), `ValidatedKafkaSecurity` (already used throughout this file). +- Produces: `pub fn controller_admin_client_properties(security: &ValidatedKafkaSecurity) -> Vec<(String, Option)>` — consumed by Task 3's ConfigMap wiring and referenced by path (`/stackable/config/admin-client.properties`) in Task 4's sidecar scripts. + +The existing `client.properties` (built by `client_properties()`, `security.rs:165-222`) is unusable for the sidecar: it points at `/stackable/tls-kafka-server`, a directory that is only mounted on broker pods (`add_broker_volume_and_volume_mounts`), never on controller pods. Controller pods only mount `/stackable/tls-kafka-internal` (`add_controller_volume_and_volume_mounts`, `security.rs:301-337`). This task adds a new properties builder pointed at that directory instead, using unprefixed `security.protocol`/`ssl.*` keys (the ones a plain Kafka admin client / `--command-config` needs), as opposed to the `listener.name.controller.ssl.*`-prefixed keys `controller_config_settings()` writes for the broker/controller's own server-side listener config. + +- [ ] **Step 1: Write the failing test** + +Add inside the existing `#[cfg(test)] mod tests` block in `security.rs` (near the other `*_properties`-style tests — check the existing fixtures `plaintext()`, `server_tls()`, `client_auth_tls()`, `as_map()` around lines 653-992 and reuse them): + +```rust + #[test] + fn controller_admin_client_properties_uses_the_internal_tls_directory() { + let security = server_tls(); + let props = as_map(controller_admin_client_properties(&security)); + + assert_eq!(props.get("security.protocol"), Some(&"SSL".to_string())); + assert_eq!( + props.get("ssl.truststore.location"), + Some(&"/stackable/tls-kafka-internal/truststore.p12".to_string()) + ); + assert_eq!(props.get("ssl.truststore.type"), Some(&"PKCS12".to_string())); + } + + #[test] + fn controller_admin_client_properties_includes_keystore_when_client_auth_is_required() { + let security = client_auth_tls(); + let props = as_map(controller_admin_client_properties(&security)); + + assert_eq!( + props.get("ssl.keystore.location"), + Some(&"/stackable/tls-kafka-internal/keystore.p12".to_string()) + ); + } + + #[test] + fn controller_admin_client_properties_is_plaintext_when_no_tls_is_configured() { + let security = plaintext(); + let props = as_map(controller_admin_client_properties(&security)); + + assert_eq!(props.get("security.protocol"), None); + assert_eq!(props.get("ssl.truststore.location"), None); + } +``` + +Check the exact names/signatures of `server_tls()`, `client_auth_tls()`, `plaintext()`, and `as_map()` in the existing test module before using them — copy their exact fixture-building style if these names differ slightly. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `cargo test -p stackable-kafka-operator-binary controller_admin_client_properties 2>&1 | tail -40` +Expected: compile error, `controller_admin_client_properties` not found. + +- [ ] **Step 3: Write the minimal implementation** + +Add to `security.rs`, near `client_properties()` (around line 165), following the same shape (`Vec<(String, Option)>` of key/value pairs, `None` values filtered out by the caller): + +```rust +/// Client-side (unprefixed `security.protocol`/`ssl.*`) properties for an admin CLI tool +/// (e.g. `kafka-metadata-quorum.sh`) talking to the CONTROLLER listener from *inside* a +/// controller pod, over the `tls-kafka-internal` volume mounted by +/// `add_controller_volume_and_volume_mounts`. +/// +/// This is deliberately separate from `client_properties()`: that function points at +/// `/stackable/tls-kafka-server`, a directory that is only mounted on broker pods. +pub fn controller_admin_client_properties( + security: &ValidatedKafkaSecurity, +) -> Vec<(String, Option)> { + let mut properties = vec![]; + + if security.tls_internal_secret_class().is_some() { + properties.push(( + PROPERTY_SECURITY_PROTOCOL.to_string(), + Some("SSL".to_string()), + )); + push_client_ssl_truststore(&mut properties, STACKABLE_TLS_KAFKA_INTERNAL_DIR); + if security.tls_client_authentication_class().is_some() { + push_client_ssl_stores(&mut properties, STACKABLE_TLS_KAFKA_INTERNAL_DIR); + } + } + + properties +} +``` + +Check the exact method name for "is internal TLS configured" (`tls_internal_secret_class()` is a guess based on the sibling `tls_server_secret_class()`/`tls_client_authentication_class()` naming seen in `kcat_prober_container_commands`, `security.rs:95-161`) — grep for the real accessor: + +Run: `grep -n "fn tls_.*secret_class\|fn tls_client_authentication_class" rust/operator-binary/src/crd/security.rs rust/operator-binary/src/controller/security.rs 2>/dev/null` + +Adjust the method name used above to match what actually exists on `ValidatedKafkaSecurity`. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `cargo test -p stackable-kafka-operator-binary controller_admin_client_properties 2>&1 | tail -40` +Expected: all three PASS. + +- [ ] **Step 5: Wire the new properties into the controller rolegroup ConfigMap** + +Read `rust/operator-binary/src/controller/build/resource/config_map.rs:150-175` first to see exactly how `client.properties` is added, then add a sibling entry for the controller role only. Find the `ConfigFileName` enum (grep `enum ConfigFileName`) and add a variant: + +Run: `grep -n "enum ConfigFileName" -A 10 rust/operator-binary/src/controller/build/resource/config_map.rs rust/operator-binary/src/crd/mod.rs 2>/dev/null` + +Add a variant named `AdminClient` (kebab-case via the same derive macros the enum already uses) that serializes to `admin-client.properties`, then add, guarded to the controller role group's `add_data` block (mirroring the `client.properties` call at `config_map.rs:155-165`): + +```rust + .add_data( + ConfigFileName::AdminClient.to_string(), + to_java_properties_string( + controller_admin_client_properties(kafka_security) + .iter() + .filter_map(|(k, v)| v.as_ref().map(|v| (k, v))), + ) + .context(SerializePropertiesSnafu)?, + ) +``` + +Place this call only in the function that builds the **controller** rolegroup ConfigMap, not the broker one — check the function name/boundary by reading the file's structure first (`grep -n "^pub fn\|^fn" rust/operator-binary/src/controller/build/resource/config_map.rs`). + +- [ ] **Step 6: Run the full properties/config_map test suite** + +Run: `cargo test -p stackable-kafka-operator-binary --lib config_map security 2>&1 | tail -60` +Expected: all PASS, no regressions in existing `client.properties`/`controller.properties` tests. + +- [ ] **Step 7: Commit** + +```bash +git add rust/operator-binary/src/controller/build/security.rs rust/operator-binary/src/controller/build/resource/config_map.rs +git commit -m "feat: add admin-client.properties for controller-pod CLI tools + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +## Task 3: Expose bootstrap servers and node id to the sidecar + +**Files:** + +- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs` + +**Interfaces:** + +- Consumes: `kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Vec` (`rust/operator-binary/src/controller/build/properties/mod.rs:59-71`, already `pub(crate)`), `validated_cluster.pod_descriptors(Some(kafka_role))` (already used at `statefulset.rs:275`, `:505`), `KAFKA_NODE_ID_OFFSET` env var name and `node_id_hash32_offset(...)` (already used at `statefulset.rs:706-709` on the `kafka` container — read that exact block before duplicating it). +- Produces: two env vars the sidecar container (Task 4) will read: `KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS` (comma-joined `host:port` list) and the same `REPLICA_ID`-deriving inputs (`POD_NAME` via downward API, `NODE_ID_OFFSET`) already present on the `kafka` container, so the sidecar's script can compute its own replica/node id exactly as `command.rs:169-170` does inside the `kafka` container's entrypoint. + +This task only adds env vars to the (not-yet-created) sidecar container's builder; Task 4 creates that builder. Do this task by extending `build_controller_rolegroup_statefulset` to compute the values once and store them in local variables the Task 4 diff will consume — do not create the sidecar container yet, since that would make this task's diff untestable on its own. Instead, write a small pure helper function now, unit-test it in isolation, and call it from Task 4. + +- [ ] **Step 1: Write the failing test** + +Add near wherever `kraft_controllers` is exported from (`rust/operator-binary/src/controller/build/properties/mod.rs`), or create a new test in `statefulset.rs` if a test module doesn't exist yet there (check first: `grep -n "mod tests" rust/operator-binary/src/controller/build/resource/statefulset.rs`; if absent, this task creates the module, which Task 6 will also extend): + +```rust +#[cfg(test)] +mod tests { + use crate::controller::build::properties::kraft_controllers; + use crate::crd::mod::KafkaPodDescriptor; // adjust path once the real module path is confirmed + + #[test] + fn quorum_manager_bootstrap_servers_env_value_is_comma_joined_host_ports() { + // Build two minimal KafkaPodDescriptor values for controllers and assert + // kraft_controllers(...).join(",") produces "host1:9093,host2:9093". + // Fill in with the real KafkaPodDescriptor construction used in + // crd/mod.rs's own tests, since its fields are crate-private (pub(crate)). + } +} +``` + +Before writing this test for real, run: + +Run: `grep -n "KafkaPodDescriptor {" rust/operator-binary/src/crd/mod.rs` + +to find an existing test or construction site building a `KafkaPodDescriptor` by hand (its fields are `pub(crate)`, so this must be done from within the `crd` module or via a test already inside `crd/mod.rs`). If no direct constructor is accessible from `statefulset.rs`'s test module, skip a standalone unit test for the joining logic here (it's a one-line `.join(",")` over an already-tested function) and instead verify this wiring via the integration-style test added in Task 6, which builds a full `ValidatedCluster` through the public `validate()` path and inspects the sidecar container's env vars directly. Note that decision in the commit message for this task. + +- [ ] **Step 2: Add the env var to `build_controller_rolegroup_statefulset`** + +In `rust/operator-binary/src/controller/build/resource/statefulset.rs`, inside `build_controller_rolegroup_statefulset` (around line 505, right after the existing `pod_descriptors(Some(kafka_role))` call used for `controller_kafka_container_command`), compute: + +```rust + let controller_pod_descriptors = validated_cluster + .pod_descriptors(Some(kafka_role)) + .context(BuildPodDescriptorsSnafu)?; + let quorum_bootstrap_servers = + crate::controller::build::properties::kraft_controllers(&controller_pod_descriptors) + .join(","); +``` + +Reuse the existing `pod_descriptors(...)` call already present at line 505 rather than calling it twice — read the surrounding code first and thread `controller_pod_descriptors` through to both the existing `controller_kafka_container_command(...)` call and this new binding, instead of calling `pod_descriptors` a second time. + +Store `quorum_bootstrap_servers` in a local variable for Task 4 to consume when building the sidecar container's env vars — do not add it to the `kafka` container's env vars in this task (it's only needed by the sidecar). + +- [ ] **Step 3: Run the build to confirm it still compiles** + +Run: `cargo build -p stackable-kafka-operator-binary 2>&1 | tail -40` +Expected: compiles cleanly. `quorum_bootstrap_servers` will show an "unused variable" warning until Task 4 consumes it — that's expected and acceptable to leave as a `#[allow(unused)]`-free warning between these two tasks only if they're implemented back-to-back in the same session; otherwise prefix with `_` temporarily. Prefer implementing Task 4 immediately after this task in the same sitting so the warning never needs suppressing. + +- [ ] **Step 4: Commit** + +```bash +git add rust/operator-binary/src/controller/build/resource/statefulset.rs +git commit -m "feat: compute quorum bootstrap servers for the controller sidecar + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +## Task 4: Build the `quorum-manager` sidecar container + +**Files:** + +- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs` +- Modify: `rust/operator-binary/src/controller/build/command.rs` + +**Interfaces:** + +- Consumes: `supports_dynamic_quorum` (Task 1), `controller_admin_client_properties` path convention `/stackable/config/admin-client.properties` (Task 2 — the file this properties struct serializes to, mounted via the existing `STACKABLE_CONFIG_DIR_NAME` volume mount already present on the `kafka` container at `statefulset.rs:293`), `quorum_bootstrap_servers` local variable (Task 3), `METRICS_PORT`/`METRICS_PORT_NAME` (`crd/mod.rs:45-46`), `kafka_security.has_kerberos_enabled()` (already used at `container_ports`, `statefulset.rs:644-668`). +- Produces: the sidecar `Container`, added to the pod via `pod_builder.add_container(...)` — consumed by Task 6's unit tests (which inspect it by container name `"quorum-manager"`) and Task 7's kuttl assertions (which observe its effect on the live cluster). + +- [ ] **Step 1: Add the two script-building functions to `command.rs`** + +Read `rust/operator-binary/src/controller/build/command.rs` in full first (it's 209 lines) to match its existing style (plain `String`/`format!`, no templating engine). Add two new functions near `controller_kafka_container_command`: + +```rust +/// The `kafka-metadata-quorum.sh` binary, referenced by its absolute path (matching every +/// other exec-into-pod usage of a Kafka CLI tool in this repo, e.g. the kuttl test scripts +/// under `tests/templates/kuttl/*/*.sh`), rather than the relative `bin/...` form used only +/// inside the `kafka` container's own entrypoint (which runs with the Kafka install dir as +/// its working directory). +const KAFKA_METADATA_QUORUM_BINARY: &str = "/stackable/kafka/bin/kafka-metadata-quorum.sh"; + +const ADMIN_CLIENT_PROPERTIES_PATH: &str = "/stackable/config/admin-client.properties"; + +/// The sidecar's main-loop command: while this controller's local Raft state is +/// `observer`, repeatedly attempt to admit it into the quorum's voter set. +/// +/// `bootstrap_servers` is the comma-joined `host:port` list produced by +/// `kraft_controllers(...)` (see `build/properties/mod.rs`). +pub fn quorum_manager_container_command(bootstrap_servers: &str) -> String { + format!( + r#" + set -uo pipefail + echo "Starting KRaft voter admission loop against bootstrap servers: {bootstrap_servers}" + while true; do + state=$(curl -s localhost:{metrics_port}/metrics | grep -oE 'kafka_server_raft_metrics_current_state\{{state="[a-z]+"\}}' | grep -oE '"[a-z]+"' | tr -d '"') + if [ "$state" = "observer" ]; then + echo "Local Raft state is observer, attempting add-controller..." + {binary} --bootstrap-controller '{bootstrap_servers}' --command-config {config} add-controller \ + || echo "add-controller attempt failed (this is expected if it already succeeded or a leader election is in progress), will retry" + fi + sleep 10 + done + "#, + bootstrap_servers = bootstrap_servers, + metrics_port = METRICS_PORT, + binary = KAFKA_METADATA_QUORUM_BINARY, + config = ADMIN_CLIENT_PROPERTIES_PATH, + ) +} + +/// The sidecar's `preStop` command: before this controller pod terminates, check that +/// removing it still leaves the quorum with a majority of its *current* voter count, and +/// if so, remove it from the voter set. Always exits 0 — a stuck or failed check must +/// never block pod termination. +/// +/// `node_id` is this controller's own KRaft node id (the same value written to +/// `node.id` in `controller.properties`, derived from `$POD_NAME` and `NODE_ID_OFFSET` +/// exactly as the `kafka` container's own entrypoint does — see `controller_kafka_container_command`). +pub fn quorum_manager_pre_stop_command(bootstrap_servers: &str) -> String { + format!( + r#" + set -uo pipefail + POD_INDEX=$(echo "$POD_NAME" | grep -oE '[0-9]+$') + REPLICA_ID=$((POD_INDEX + NODE_ID_OFFSET)) + DEADLINE=$((SECONDS + 25)) + while [ "$SECONDS" -lt "$DEADLINE" ]; do + describe=$({binary} --bootstrap-controller '{bootstrap_servers}' --command-config {config} describe --replication 2>/dev/null) + if [ -n "$describe" ]; then + # NOTE: this parsing was written against the documented `describe --replication` + # tabular output (one voter per line, NodeId as the first column) and must be + # confirmed/adjusted against a live cluster's real output before this is + # considered done -- see Task 4 Step 4 below. + total_voters=$(echo "$describe" | tail -n +2 | grep -c .) + majority=$(( total_voters / 2 + 1 )) + remaining_after_removal=$(( total_voters - 1 )) + if [ "$remaining_after_removal" -ge "$majority" ]; then + directory_id=$(echo "$describe" | tail -n +2 | awk -v id="$REPLICA_ID" '$1 == id {{ print $2 }}') + if [ -n "$directory_id" ]; then + echo "Removing self (node $REPLICA_ID, directory $directory_id) from the voter set..." + {binary} --bootstrap-controller '{bootstrap_servers}' --command-config {config} remove-controller \ + --controller-id "$REPLICA_ID" --controller-directory-id "$directory_id" \ + || echo "remove-controller failed, proceeding with termination anyway" + fi + else + echo "Removing self would break quorum majority ($remaining_after_removal remaining of $majority needed), skipping and retrying..." + fi + break + fi + sleep 2 + done + exit 0 + "#, + bootstrap_servers = bootstrap_servers, + binary = KAFKA_METADATA_QUORUM_BINARY, + config = ADMIN_CLIENT_PROPERTIES_PATH, + ) +} +``` + +Import `METRICS_PORT` at the top of `command.rs` if not already imported (`grep -n "METRICS_PORT" rust/operator-binary/src/controller/build/command.rs`). + +- [ ] **Step 2: Write the failing unit tests for the two command strings** + +Add to (or create) a `#[cfg(test)] mod tests` in `command.rs`: + +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn quorum_manager_container_command_targets_the_bootstrap_servers_not_localhost() { + let command = quorum_manager_container_command("controller-0:9093,controller-1:9093"); + assert!(command.contains("--bootstrap-controller 'controller-0:9093,controller-1:9093'")); + assert!(command.contains("add-controller")); + assert!(!command.contains("--bootstrap-controller 'localhost")); + } + + #[test] + fn quorum_manager_pre_stop_command_always_exits_zero() { + let command = quorum_manager_pre_stop_command("controller-0:9093,controller-1:9093"); + assert!(command.trim_end().ends_with("exit 0")); + assert!(command.contains("remove-controller")); + } +} +``` + +If `command.rs` already has a test module, add these two functions inside it instead. + +- [ ] **Step 3: Run the tests to verify they pass** + +Run: `cargo test -p stackable-kafka-operator-binary quorum_manager 2>&1 | tail -40` +Expected: both PASS (these are just string-content assertions, so they should pass immediately once Step 1's functions compile — this is a case where writing the test after the implementation is acceptable, since the "test" here is really a guard against a future accidental typo in the command string, not driving the design). + +- [ ] **Step 4: Build the sidecar container in `statefulset.rs`** + +In `rust/operator-binary/src/controller/build/resource/statefulset.rs`, add a new function near `add_vector_container` (bottom of file): + +```rust +/// Builds the `quorum-manager` sidecar for a controller pod. Returns `None` when this +/// Kafka version doesn't support KIP-853 dynamic quorum tooling, or when Kerberos is +/// enabled (the sidecar's admin-client properties file only covers the TLS/SSL case). +fn build_quorum_manager_container( + resolved_product_image: &ResolvedProductImage, + kafka_security: &ValidatedKafkaSecurity, + quorum_bootstrap_servers: &str, +) -> Result, Error> { + if !supports_dynamic_quorum(&resolved_product_image.product_version) + || kafka_security.has_kerberos_enabled() + { + return Ok(None); + } + + let container_name = "quorum-manager".to_string(); + let mut cb = ContainerBuilder::new(&container_name).context(InvalidContainerNameSnafu { + name: container_name.clone(), + })?; + + cb.image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-c".to_string(), + quorum_manager_container_command(quorum_bootstrap_servers), + ]) + .add_env_vars(vec![EnvVar { + name: "POD_NAME".to_string(), + value_from: Some(EnvVarSource { + field_ref: Some(ObjectFieldSelector { + api_version: Some("v1".to_string()), + field_path: "metadata.name".to_string(), + }), + ..EnvVarSource::default() + }), + ..EnvVar::default() + }]) + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + .with_cpu_limit("200m") + .with_memory_request("128Mi") + .with_memory_limit("128Mi") + .build(), + ) + .add_volume_mount(STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_DIR) + .context(AddVolumeMountSnafu)? + .lifecycle_pre_stop(LifecycleHandler { + exec: Some(ExecAction { + command: Some(vec![ + "/bin/bash".to_string(), + "-c".to_string(), + quorum_manager_pre_stop_command(quorum_bootstrap_servers), + ]), + }), + ..LifecycleHandler::default() + }); + + Ok(Some(cb.build())) +} +``` + +Add the required imports at the top of `statefulset.rs`: `LifecycleHandler` from `stackable_operator::k8s_openapi::api::core::v1` (alongside the existing `ExecAction`, `EnvVar`, `EnvVarSource`, `ObjectFieldSelector` imports at lines 22-25), `ResourceRequirementsBuilder` (already imported at line 10 for the broker's kcat-prober container — reuse it), and `supports_dynamic_quorum`, `quorum_manager_container_command`, `quorum_manager_pre_stop_command` from `crate::controller::build::{properties, command}`. + +Also add the `Q` sidecar needs the `NODE_ID_OFFSET` env var referenced by its `preStop` script (`$NODE_ID_OFFSET`) — read `statefulset.rs:706-709` (the `kafka` container's own `NODE_ID_OFFSET` env var construction) and add the identical `EnvVar` to the sidecar's `add_env_vars` call in the snippet above, rather than duplicating the whole block — extract the shared computation into a local variable used by both containers if it isn't already. + +Then, inside `build_controller_rolegroup_statefulset`, right after the existing `pod_builder.add_container(kafka_container)` call (around line 579), add: + +```rust + if let Some(quorum_manager_container) = build_quorum_manager_container( + resolved_product_image, + kafka_security, + &quorum_bootstrap_servers, + )? { + pod_builder.add_container(quorum_manager_container); + } +``` + +using the `quorum_bootstrap_servers` binding from Task 3. + +- [ ] **Step 5: Verify the CLI's actual `describe --replication` output shape** + +This step is a real verification action, not a placeholder — the `preStop` script's `awk`/`grep` parsing in Step 1 was written against Kafka's documented tabular format and has not been checked against a live cluster. + +Run: `kubectl exec -n test-kafka-controller-default-0 -c kafka -- /stackable/kafka/bin/kafka-metadata-quorum.sh --bootstrap-controller :9093 --command-config /stackable/config/admin-client.properties describe --replication` + +(This requires Task 2's `admin-client.properties` to already be deployed — run this verification after Tasks 2-4 are all merged into a real running cluster, e.g. via a manual `./scripts/run-tests` smoke-kraft run, before considering this task done.) Compare the real column layout (which column holds `NodeId`, which holds `DirectoryId`) against the `awk -v id="$REPLICA_ID" '$1 == id { print $2 }'` assumption in Step 1, and adjust the column indices in `quorum_manager_pre_stop_command` if they don't match. Re-run the unit tests from Step 3 after any change (they assert command *structure*, not the exact awk column numbers, so they should still pass, but re-run them anyway to be safe). + +- [ ] **Step 6: Build and run the full test suite** + +Run: `cargo build -p stackable-kafka-operator-binary 2>&1 | tail -60` +Run: `cargo test -p stackable-kafka-operator-binary 2>&1 | tail -80` +Expected: builds cleanly, all tests PASS (this also exercises every existing `statefulset.rs`/`config_map.rs` test, confirming the new sidecar doesn't break broker-pod builds, which must never get this container). + +- [ ] **Step 7: Regenerate CRDs and check for unexpected diffs** + +Run: `make regenerate-charts 2>&1 | tail -40` +Expected: no diff, since this task adds a container by string literal name rather than a new `ContainerName`-enum variant, so the CRD's `logging.containers` schema is unchanged. If `make regenerate-charts` produces an unexpected diff, investigate before proceeding — it likely means a CRD-visible type changed somewhere in this task's diff. + +- [ ] **Step 8: Commit** + +```bash +git add rust/operator-binary/src/controller/build/resource/statefulset.rs rust/operator-binary/src/controller/build/command.rs +git commit -m "feat: add quorum-manager sidecar to controller pods + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +## Task 5: Switch controller StatefulSet to `OrderedReady` pod management + +**Files:** + +- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs` + +**Interfaces:** + +- Consumes: nothing new. +- Produces: `POD_MANAGEMENT_POLICY_ORDERED_READY` constant, consumed only by this task's own change and asserted by Task 6's unit test. + +- [ ] **Step 1: Write the failing unit test** + +In the `statefulset.rs` test module (created in Task 3 or already present), add: + +```rust + #[test] + fn controller_statefulset_uses_ordered_ready_pod_management() { + let cluster = kraft_mode_cluster(); + let resources = crate::controller::build::build(&cluster).expect("build succeeds"); + let sts = resources + .stateful_sets + .into_iter() + .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-controller-default")) + .expect("the controller StatefulSet is built"); + + assert_eq!( + sts.spec.expect("the StatefulSet has a spec").pod_management_policy, + Some("OrderedReady".to_string()) + ); + } + + #[test] + fn broker_statefulset_still_uses_parallel_pod_management() { + let cluster = kraft_mode_cluster(); + let resources = crate::controller::build::build(&cluster).expect("build succeeds"); + let sts = resources + .stateful_sets + .into_iter() + .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-broker-default")) + .expect("the broker StatefulSet is built"); + + assert_eq!( + sts.spec.expect("the StatefulSet has a spec").pod_management_policy, + Some("Parallel".to_string()) + ); + } +``` + +This uses a `kraft_mode_cluster()` fixture. Task 3 deliberately deferred creating this fixture (see its Step 1) in favor of this task owning it. **This task creates `kraft_mode_cluster()`** in this file's test module, copied from the pattern shown in the exploration: a minimal `KafkaCluster` YAML with `clusterConfig.metadataManager: kraft`, one controller role group of 3 replicas, one broker role group of 3 replicas, resolved via `crate::controller::test_support::{minimal_kafka, validated_cluster}`. + +- [ ] **Step 2: Run the tests to verify the controller one fails** + +Run: `cargo test -p stackable-kafka-operator-binary pod_management 2>&1 | tail -30` +Expected: `broker_statefulset_still_uses_parallel_pod_management` PASSes (no change yet), `controller_statefulset_uses_ordered_ready_pod_management` FAILs (`Parallel` != `OrderedReady`). + +- [ ] **Step 3: Make the change** + +In `build_controller_rolegroup_statefulset`, add a new constant near the existing `POD_MANAGEMENT_POLICY_PARALLEL` (`statefulset.rs:127`): + +```rust +const POD_MANAGEMENT_POLICY_ORDERED_READY: &str = "OrderedReady"; +``` + +And change the controller `StatefulSetSpec` construction (`statefulset.rs:620`) from: + +```rust + pod_management_policy: Some(POD_MANAGEMENT_POLICY_PARALLEL.to_string()), +``` + +to: + +```rust + pod_management_policy: Some(POD_MANAGEMENT_POLICY_ORDERED_READY.to_string()), +``` + +Leave the broker StatefulSet's construction (`statefulset.rs:435`) unchanged — it must keep using `POD_MANAGEMENT_POLICY_PARALLEL`. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `cargo test -p stackable-kafka-operator-binary pod_management 2>&1 | tail -30` +Expected: both PASS. + +- [ ] **Step 5: Commit** + +```bash +git add rust/operator-binary/src/controller/build/resource/statefulset.rs +git commit -m "feat: use OrderedReady pod management for controller StatefulSets + +Serializes scale-down so each controller's preStop hook (self-removal +from the KRaft voter set) completes before the next pod terminates. + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +## Task 6: Unit tests for sidecar presence/absence and version gating + +**Files:** + +- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs` + +**Interfaces:** + +- Consumes: `kraft_mode_cluster()` fixture (created by Task 5 — Task 3 deliberately deferred it), `build_quorum_manager_container` / the sidecar's presence in the built `StatefulSet` (Task 4), the `kerberos()` security fixture from `security.rs`'s test module (Task 2 — may need its visibility bumped to `pub(crate)` for this task to reach it). + +- [ ] **Step 1: Write the failing tests** + +```rust + fn controller_containers( + cluster: &crate::controller::ValidatedCluster, + ) -> Vec { + let resources = crate::controller::build::build(cluster).expect("build succeeds"); + let sts = resources + .stateful_sets + .into_iter() + .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-controller-default")) + .expect("the controller StatefulSet is built"); + sts.spec + .expect("the StatefulSet has a spec") + .template + .spec + .expect("the pod template has a spec") + .containers + } + + #[test] + fn controller_pods_get_a_quorum_manager_sidecar_on_supported_versions() { + let cluster = kraft_mode_cluster(); + let containers = controller_containers(&cluster); + + assert!( + containers.iter().any(|c| c.name == "quorum-manager"), + "expected a quorum-manager sidecar, got containers: {:?}", + containers.iter().map(|c| &c.name).collect::>() + ); + } + + #[test] + fn quorum_manager_sidecar_targets_bootstrap_servers_in_its_command() { + let cluster = kraft_mode_cluster(); + let containers = controller_containers(&cluster); + let sidecar = containers + .iter() + .find(|c| c.name == "quorum-manager") + .expect("the quorum-manager sidecar is built"); + + let command = sidecar + .command + .as_ref() + .expect("the sidecar has a command") + .join(" "); + assert!(command.contains("add-controller")); + + let pre_stop_command = sidecar + .lifecycle + .as_ref() + .and_then(|l| l.pre_stop.as_ref()) + .and_then(|h| h.exec.as_ref()) + .and_then(|e| e.command.as_ref()) + .expect("the sidecar has a preStop exec hook") + .join(" "); + assert!(pre_stop_command.contains("remove-controller")); + assert!(pre_stop_command.trim_end().ends_with("exit 0")); + } + + #[test] + fn controller_pods_get_no_quorum_manager_sidecar_on_kafka_3_7() { + let kafka = crate::controller::test_support::minimal_kafka( + r#" + apiVersion: kafka.stackable.tech/v1alpha1 + kind: KafkaCluster + metadata: + name: simple-kafka + namespace: default + uid: 12345678-1234-1234-1234-123456789012 + spec: + image: + productVersion: 3.7.2 + clusterConfig: + metadataManager: kraft + controllers: + roleGroups: + default: + replicas: 3 + brokers: + roleGroups: + default: + replicas: 3 + "#, + ); + let cluster = crate::controller::test_support::validated_cluster(&kafka); + let containers = controller_containers(&cluster); + + assert!(!containers.iter().any(|c| c.name == "quorum-manager")); + } + + #[test] + fn controller_pods_get_no_quorum_manager_sidecar_when_kerberos_is_enabled() { + // This is a Global Constraint (see the plan header): the sidecar's admin-client + // properties file only covers the TLS/SSL case, so it must never be added when + // Kerberos is enabled, even on an otherwise-supported Kafka version. + // + // Rather than building a full CRD-level Kerberos fixture (which needs a resolved + // AuthenticationClass threaded through `DereferencedObjects`, more than this test + // needs), call `build_quorum_manager_container` directly — it already takes + // `&ValidatedKafkaSecurity` as a parameter, so a fixture at that level is enough. + // Reuse the `kerberos()` fixture from `security.rs`'s existing test module (see + // Task 2) for a security value with Kerberos enabled; import it, adjusting its + // visibility to `pub(crate)` in `security.rs` if it is not already visible here. + let cluster = kraft_mode_cluster(); + let kerberos_security = crate::controller::build::security::tests::kerberos(); + + let result = build_quorum_manager_container( + &cluster.image, + &kerberos_security, + "controller-0:9093", + ) + .expect("build_quorum_manager_container does not error for a kerberos security value"); + + assert!(result.is_none()); + } + + #[test] + fn broker_pods_never_get_a_quorum_manager_sidecar() { + let cluster = kraft_mode_cluster(); + let resources = crate::controller::build::build(&cluster).expect("build succeeds"); + let sts = resources + .stateful_sets + .into_iter() + .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-broker-default")) + .expect("the broker StatefulSet is built"); + let containers = sts + .spec + .expect("the StatefulSet has a spec") + .template + .spec + .expect("the pod template has a spec") + .containers; + + assert!(!containers.iter().any(|c| c.name == "quorum-manager")); + } +``` + +Verify `productVersion: 3.7.2` is actually a version accepted by this repo's product-version validation (some operators restrict to an exact known list) — check: + +Run: `grep -rn "3.7" rust/crd/src/ tests/test-definition.yaml 2>/dev/null | head -20` + +If `3.7.2` isn't a recognized version, use whatever 3.7.x version is used elsewhere in this repo's own tests/fixtures instead. + +- [ ] **Step 2: Run the tests to verify they fail (or pass, if Task 4/5 already got this right)** + +Run: `cargo test -p stackable-kafka-operator-binary quorum_manager 2>&1 | tail -60` + +If Tasks 4-5 were implemented correctly, these should already PASS since they're testing behavior those tasks already built — this task exists to lock that behavior in with explicit regression coverage, not to drive new implementation. If any fail, fix the implementation in `statefulset.rs` from Task 4/5 (not the test) unless the test itself has a mistaken assumption — re-read Task 4/5's code before changing either. + +- [ ] **Step 3: Run the full test suite one more time** + +Run: `cargo test -p stackable-kafka-operator-binary 2>&1 | tail -80` +Expected: all PASS. + +- [ ] **Step 4: Commit** + +```bash +git add rust/operator-binary/src/controller/build/resource/statefulset.rs +git commit -m "test: cover quorum-manager sidecar presence, version gate, and commands + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +## Task 7: Fix and strengthen the kuttl scale-up/scale-down assertions + +**Files:** + +- Modify: `tests/templates/kuttl/operations-kraft/60-assert.yaml.j2` +- Modify: `tests/templates/kuttl/operations-kraft/70-assert.yaml.j2` + +**Interfaces:** none (test-only, no Rust interfaces). + +Both files currently have a real bug (found during design exploration): the two YAML documents for the broker and controller `StatefulSet` assertions are missing a `---` separator between them, which likely means the second document (the controller assertion) is silently ignored by the YAML parser or produces unexpected behavior. This task fixes that bug and adds a voter-count check. + +- [ ] **Step 1: Read both files in full** + +Run: `cat tests/templates/kuttl/operations-kraft/60-assert.yaml.j2 tests/templates/kuttl/operations-kraft/70-assert.yaml.j2` + +Confirm the missing `---` before the second `apiVersion: apps/v1` block in each file. + +- [ ] **Step 2: Fix the missing document separator in `60-assert.yaml.j2`** + +Insert a `---` line immediately before the second `apiVersion: apps/v1` (the `test-kafka-controller-default` StatefulSet assertion), so the file has three `---`-separated documents: the `TestAssert` header/commands block, the broker StatefulSet assertion, and the controller StatefulSet assertion. + +- [ ] **Step 3: Apply the identical fix to `70-assert.yaml.j2`** + +Same change, same reasoning. + +- [ ] **Step 4: Add a voter-count assertion command to `60-assert.yaml.j2`** + +In the `commands:` list of the `TestAssert` document (alongside the existing `kubectl -n $NAMESPACE wait --for=condition=available ...` command), add: + +```yaml + - script: | + kubectl exec -n $NAMESPACE test-kafka-controller-default-0 -c kafka -- \ + /stackable/kafka/bin/kafka-metadata-quorum.sh \ + --bootstrap-controller test-kafka-controller-default-0.test-kafka-controller-default-headless.$NAMESPACE.svc.cluster.local:9093 \ + --command-config /stackable/config/admin-client.properties \ + describe --replication | tail -n +2 | wc -l | grep -q '^5$' +``` + +Verify the exact headless service name pattern (`test-kafka-controller-default-headless`) against how the FQDN is actually constructed elsewhere in this test suite — grep other files in `tests/templates/kuttl/operations-kraft/` for an existing `--bootstrap-server`/FQDN reference to copy the exact naming convention rather than guessing it: + +Run: `grep -rn "headless\|bootstrap-server" tests/templates/kuttl/operations-kraft/*.j2 tests/templates/kuttl/smoke-kraft/*.j2 2>/dev/null | head -20` + +Adjust the hostname in the command above to match whatever convention those files actually use. + +- [ ] **Step 5: Add the equivalent assertion to `70-assert.yaml.j2`, expecting 3 voters** + +Same command, with `grep -q '^3$'` instead of `'^5$'`, matching the scaled-down replica count. + +- [ ] **Step 6: Run the kuttl test manually** + +Run: `./scripts/run-tests --skip-release --test operations-kraft_kafka-kraft-4.2.1_openshift-false 2>&1 | tail -100` + +Expected: PASS, including the new voter-count checks in steps 60 and 70. If the voter-count check fails while the StatefulSet readiness check passes, that's a real signal the sidecar (Task 4) isn't actually admitting/removing voters correctly — go back to Task 4 and debug using the same `vector tap` / `kubectl logs -c quorum-manager` techniques, rather than loosening this assertion. + +- [ ] **Step 7: Commit** + +```bash +git add tests/templates/kuttl/operations-kraft/60-assert.yaml.j2 tests/templates/kuttl/operations-kraft/70-assert.yaml.j2 +git commit -m "test: fix missing YAML separator and assert voter count in scale tests + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +## Task 8: Update documentation and remove the "unsupported" claim + +**Files:** + +- Modify: `docs/modules/kafka/pages/usage-guide/kraft-controller.adoc` +- Modify: `tests/templates/kuttl/operations-kraft/README.md` +- Modify: `CHANGELOG.md` + +**Interfaces:** none. + +- [ ] **Step 1: Update `kraft-controller.adoc`** + +Read the file in full (already read during brainstorming). Remove or rewrite the "Scaling controller replicas up is not supported" bullet under "Known Issues" and the entire "Scaling issues" subsection under "Troubleshooting", replacing them with a short description of the new behavior: + +- Controllers can now be scaled up and down on a running cluster. +- A per-pod `quorum-manager` sidecar handles admitting/removing the pod from the KRaft voter set. +- This requires a Kafka version that supports KIP-853 dynamic quorum tooling (everything except `3.7.x`). +- Scaling more than one controller down at a time is processed one pod at a time (`OrderedReady`), not in parallel. +- If a `remove-controller` call fails during pod termination (e.g. no reachable leader within the grace period), a stale voter entry can be left behind and requires manual cleanup — state this as a known limitation, do not imply it's fully automatic in every case. + +Also update the "Internal operator details" bullet that currently says "the operator does not perform the follow-up step required to add a controller to an already-formed quorum's voter set" — this is no longer true. + +- [ ] **Step 2: Update the kuttl README** + +`tests/templates/kuttl/operations-kraft/README.md` currently states "Scaling controllers from 3 -> 1 doesn't work. Both brokers and controllers try to communicate with old controllers." Verify whether this specific limitation (scaling below a certain floor) is still expected to hold after this change — if scaling controllers down to 1 was never exercised by these tests (they only go 3→5→3), leave this caveat in place rather than removing an unverified claim; do not claim a scenario is fixed that this plan's tests don't actually cover. + +- [ ] **Step 3: Add the CHANGELOG entry** + +In `CHANGELOG.md`, insert a new `### Added` section between `## [Unreleased]` and the existing `### Changed` section: + +```markdown +### Added + +- KRaft controller replicas can now be scaled up and down on a running cluster: a new + `quorum-manager` sidecar container on each controller pod admits itself into the KRaft + voter set on startup and removes itself before termination ([#NNNN]). +``` + +Add the corresponding link reference at the bottom of the file, in ascending numeric order alongside the existing `[#985]`/`[#990]`/etc. links: + +```markdown +[#NNNN]: https://github.com/stackabletech/kafka-operator/pull/NNNN +``` + +Leave `NNNN` as a literal placeholder for the real PR number — fill it in when the PR is actually opened (this is the one acceptable use of a placeholder in this plan, since the number doesn't exist until the PR is created; every other file in this plan has zero placeholders). + +- [ ] **Step 4: Commit** + +```bash +git add docs/modules/kafka/pages/usage-guide/kraft-controller.adoc tests/templates/kuttl/operations-kraft/README.md CHANGELOG.md +git commit -m "docs: document KRaft controller scale-up/down support + +Co-Authored-By: Claude Sonnet 5 " +``` + +--- + +## Task 9: Full verification gate + +**Files:** none (verification only). + +- [ ] **Step 1: Full build** + +Run: `cargo build --workspace 2>&1 | tail -60` +Expected: clean build. + +- [ ] **Step 2: Full test suite** + +Run: `cargo test --workspace 2>&1 | tail -100` +Expected: all PASS. + +- [ ] **Step 3: Clippy** + +Run: `cargo clippy --all-targets -- -D warnings 2>&1 | tail -100` +Expected: no warnings/errors. Fix anything that comes up before proceeding. + +- [ ] **Step 4: Format check** + +Run: `cargo fmt --check 2>&1 | tail -60` +Expected: no diff. If there is one, run `cargo fmt` and amend the relevant task's commit. + +- [ ] **Step 5: Regenerate charts/CRDs one final time** + +Run: `make regenerate-charts 2>&1 | tail -60` +Expected: no diff (confirmed already in Task 4, re-checked here after all subsequent tasks in case anything else drifted). + +- [ ] **Step 6: Full kuttl run for the affected test suite** + +Run: `./scripts/run-tests --skip-release --test operations-kraft_kafka-kraft-4.2.1_openshift-false 2>&1 | tail -150` +Run: `./scripts/run-tests --skip-release --test operations-kraft_kafka-kraft-3.9.2_openshift-false 2>&1 | tail -150` +Expected: both PASS. + +- [ ] **Step 7: Commit any fixups from this task as a single commit, if any were needed** + +```bash +git add -A +git commit -m "chore: fix clippy/fmt findings from verification pass + +Co-Authored-By: Claude Sonnet 5 " +``` + +If nothing needed fixing, skip this commit — don't create an empty one. diff --git a/docs/superpowers/specs/2026-08-14-kraft-dynamic-voter-membership-design.md b/docs/superpowers/specs/2026-08-14-kraft-dynamic-voter-membership-design.md new file mode 100644 index 00000000..d7c52714 --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-kraft-dynamic-voter-membership-design.md @@ -0,0 +1,197 @@ +# KRaft dynamic voter membership (scale-up / scale-down) + +Status: approved for planning +Date: 2026-08-14 +Branch this was designed on: `main` @ `5211842` + +## Problem + +Apache Kafka's KRaft dynamic quorum (KIP-853) requires an explicit +follow-up step to change the voter set of an already-formed quorum: +`kafka-metadata-quorum.sh add-controller` to admit a new controller, +`remove-controller` to retire one. The Stackable Kafka operator +currently only performs the one-time `--initial-controllers` step at +`kafka-storage.sh format` time. Any controller pod added after initial +cluster formation registers itself and starts up, but never leaves the +Raft `observer` state — it can never become `leader`/`follower`/`voted`, +so it never becomes healthy, and there is no supported way to remove a +controller from the voter set either. This is documented today as a +flat "do not scale controller replicas on a running cluster" limitation +in `docs/modules/kafka/pages/usage-guide/kraft-controller.adoc`. + +Goal: let `spec.controllers.roleGroups..replicas` be scaled up +and down on a running cluster, with the voter set kept in sync +automatically. + +## Scope + +- In scope: admitting new controllers to the voter set on scale-up, + removing controllers from the voter set on scale-down, for live + replica-count changes on an already-formed cluster. +- Out of scope: whole-cluster graceful-deletion draining (a + finalizer-based mechanism is a separate concern from live replica + changes and is not addressed here). ZooKeeper-to-KRaft migration is + unaffected. Kerberos support for KRaft is unaffected. +- This design was developed independently of, and does not reuse code + from, any prior unmerged work on a KRaft "quorum health gate" or + graceful-teardown finalizer — that work was discarded (local + branches deleted) before this design was started. + +## Non-goals / explicitly rejected approaches + +- **Operator-side kube-exec.** An earlier direction had the operator + itself exec into pods (`pods/exec`) to run + `kafka-metadata-quorum.sh`, gated by new reconcile phases (a + pre-`build` gate clamping the effective replica count on scale-down, + a post-`apply` phase admitting new voters on scale-up). This was + rejected in favor of the sidecar approach below: it needed new RBAC, + a new "live cluster" client capability the operator has never had, + and two new reconcile phases, none of which are needed once the pods + manage their own membership. +- **`controller.quorum.auto.join.enable`.** Delegates scale-up + self-promotion entirely to Kafka with zero new operator capability, + but doesn't address scale-down at all (still needs an active + `remove-controller` step), and gives up visibility into *why* + admission might be stuck. Not chosen because scale-down still needs + the same sidecar mechanism anyway, so this would only save the + add-controller half of the problem while adding a version dependency + to check. + +## Design + +### Architecture + +The operator gains **no new awareness of live quorum state**. The +existing reconcile pipeline (`dereference → validate → build → apply → +update_status`) is untouched. All quorum membership management is +delegated to the controller pods themselves, via: + +1. A new sidecar container, controller-role-only, reusing the `kafka` + product image (so `kafka-metadata-quorum.sh` and the TLS trust + material already mounted for the `kafka` container are available + without new volumes). +2. A `preStop` lifecycle hook on that sidecar. +3. `podManagementPolicy: OrderedReady` on the controller StatefulSet + (currently `Parallel`). + +Both the sidecar and the `preStop` hook are only added for Kafka +versions that support KIP-853 dynamic quorum tooling — mirrors the +existing per-version special-casing already present around +`--initial-controllers` for 3.7.x. Older versions get no sidecar at +all and keep today's documented "unsupported" behavior. + +### Components + +**Add-loop script** (the sidecar's main process, runs for the pod's +whole lifetime): + +- Polls the local JMX Prometheus metrics endpoint (the same + `kafka_server_raft_metrics_current_state` series the existing + readiness probe already reads) on a short interval. +- While state is `observer`, runs + `kafka-metadata-quorum.sh add-controller` against + `controller.quorum.bootstrap.servers` (this must be invoked locally + on the joining node — it reads local KRaft directory state + automatically, which also sidesteps the fake placeholder directory-id + used by `KafkaPodDescriptor::as_voter()` at format time; that + placeholder was flagged during design exploration as a hazard for + any tooling that validates directory ids, but `add-controller` does + not consume it). +- Treats "already a voter" responses as success and keeps polling at + the same interval indefinitely (cheap, idempotent, self-healing — + no persisted state, no operator involvement). + +This also resolves what looked like a circular dependency during +design: the existing readiness probe can only pass once raft state +leaves `observer`, so gating admission on pod-readiness would be +circular. The sidecar's loop is independent of the pod's own readiness +state, so there is no cycle. + +**Remove script** (the sidecar's `preStop` hook, runs once at +termination): + +1. Runs `kafka-metadata-quorum.sh describe --replication` to get the + current voter list. +2. Checks that removing itself would still leave a majority of the + *pre-removal* voter count. This check is done explicitly by the + script — the design does not assume `remove-controller` refuses an + unsafe removal on Kafka's side. +3. If safe, calls `remove-controller` for itself. +4. The whole hook is bounded by a timeout comfortably inside + `terminationGracePeriodSeconds`, and always exits `0` — a stuck or + failed check must never block pod termination indefinitely. + +### Data flow + +**Scale-up:** an ordinary declarative replica increase on the +controller StatefulSet (no change from today) creates a new pod. Its +`kafka` container boots exactly as today (format + start). Its sidecar +independently loops until it observes itself admitted. The existing +readiness probe starts passing once raft state leaves `observer`. If +multiple controllers are added at once, each pod's sidecar self-admits +independently; Kafka's leader serializes the actual `AddVoter` +application, so no operator-side coordination is required. + +**Scale-down:** an ordinary declarative replica decrease (no change +from today). `OrderedReady` means Kubernetes terminates exactly the +highest-ordinal pod, runs its `preStop` hook (self-removal via the +script above), and waits for full termination before considering the +next pod — this is what gives one-at-a-time, majority-checked draining +for a decrease of any size, entirely via a StatefulSet setting. No +Rust-side "gate the effective replica count" logic is needed. + +### Error handling + +- Transient `add-controller` / `describe` failures (e.g. a leader + election in flight) are simply retried by the loop on its normal + interval. There is no alerting path today: the sidecar has no + Kubernetes API access by design (that's the point — no new RBAC), + so failures are visible only via `kubectl logs` on the sidecar + container. +- **Known observability gap:** the sidecar's stdout will *not* be + picked up by the existing vector log-aggregation pipeline, which + only tails structured `*.log4j.xml` / `*.log4j2.xml` files written + by the JVM's own logging config (confirmed by direct inspection of + the deployed `vector.yaml` ConfigMaps during an unrelated + investigation). This is a real, known limitation of this design, not + something papered over — a future iteration could have the sidecar + write structured lines to a file under the shared log directory to + get picked up, but that is not included in this design's initial + scope. +- `preStop` removal timing out or failing (e.g. no reachable leader + within the grace period): the pod still terminates on schedule. A + stale voter entry can be left behind in the quorum in that case. + This is a genuine, stated limitation — recovery in that scenario is + manual (the same "no supported automated path" caveat that already + exists in the current docs for quorum-reconfiguration edge cases). + +### Testing + +- The existing `tests/templates/kuttl/operations-kraft/60-scale-controller-up.yaml.j2` + / `70-scale-controller-down.yaml.j2` kuttl tests (active on `main`, + not disabled) should pass once this is implemented. Strengthen their + `*-assert.yaml.j2` counterparts beyond "StatefulSet reports N/N + ready" to also verify voter count matches replica count post-scale + (e.g. via `describe --replication` run from the test's `python-0` + pod), so the test catches a silently-stuck-in-`observer` regression, + not just a stuck-not-ready one. +- Unit tests in `rust/operator-binary/src/controller/build/resource/statefulset.rs`, + mirroring the existing probe tests added in `ec59dab`: + - the sidecar container is present only on the controller role, and + only for Kafka versions that support dynamic quorum tooling; + - the sidecar's `preStop` command matches the expected removal + script invocation; + - the controller StatefulSet's `podManagementPolicy` is + `OrderedReady`. + +## Open questions for implementation planning + +- Exact minimum Kafka version for the version gate (needs verification + against Kafka's own KIP-853 tooling maturity, not assumed here). +- Exact script implementation (shell, embedded via ConfigMap vs. an + inline `bash -c` command similar to the existing probe commands in + `statefulset.rs`) and its `--command-config` security settings + (matching whatever TLS/SASL configuration the `kafka` container + already uses for its internal listener). +- Whether to close the sidecar-log observability gap noted above as + part of this work or as explicit follow-up. From fc21e2e60667244ba7fe360a85717508fc2af2cc Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:16:16 +0200 Subject: [PATCH 07/13] Cleanup inline comments and documentation --- CHANGELOG.md | 26 +- .../pages/usage-guide/kraft-controller.adoc | 49 +- ...26-08-14-kraft-dynamic-voter-membership.md | 1008 ----------------- ...4-kraft-dynamic-voter-membership-design.md | 197 ---- .../src/controller/build/command.rs | 20 +- .../src/controller/build/mod.rs | 14 - .../src/controller/build/properties/mod.rs | 6 - .../controller/build/resource/config_map.rs | 6 - .../controller/build/resource/statefulset.rs | 24 +- .../src/controller/build/security.rs | 3 - .../src/controller/validate.rs | 16 +- tests/test-definition.yaml | 7 +- 12 files changed, 29 insertions(+), 1347 deletions(-) delete mode 100644 docs/superpowers/plans/2026-08-14-kraft-dynamic-voter-membership.md delete mode 100644 docs/superpowers/specs/2026-08-14-kraft-dynamic-voter-membership-design.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ada29037..e1c469c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,22 +6,13 @@ All notable changes to this project will be documented in this file. ### Added -- KRaft controller replicas can now be scaled up and down on a running cluster: a new - `quorum-manager` sidecar container on each controller pod is solely responsible for +- A new `quorum-manager` sidecar container on each controller pod is solely responsible for admitting itself into the KRaft voter set on startup and removing itself before termination. Exactly one controller bootstraps the quorum standalone at format time (`kafka-storage.sh format --standalone`); every other controller, whether present from the start or added later, formats with `--no-initial-controllers` and joins purely - through the sidecar. `controller.quorum.bootstrap.servers` now points at each controller - role group's headless Service DNS name instead of individual pod addresses, so neither - the container commands nor that ConfigMap value change with the replica count anymore. - Confirmed live: scaling a controller role group up or down leaves every already-existing - controller pod completely untouched (same UID, zero restarts, no `StatefulSet` revision - change) — only the pods actually being added or removed are touched ([#NNNN]). -- A `startupProbe` and a plain TCP `livenessProbe` for KRaft controllers, and a new - `readinessProbe` that checks the controller's Raft state (`leader`/`follower`/`voted`) via - its metrics endpoint instead of a bare TCP check, so a controller that can't join or - rejoin the quorum is correctly reported as not ready ([#1006]). + through the sidecar ([#1010]). +- A new `readinessProbe` for KRaft controllers that fails when new pods cannot join the quorum ([#1010]). ### Changed @@ -35,9 +26,11 @@ All notable changes to this project will be documented in this file. - All product containers now run with `securityContext.runAsNonRoot` set to `true` to improve security ([#998]). - The reconciler now applies resources and derives the cluster status in discrete apply and update_status steps ([#1000]). -- BREAKING: KRaft mode now requires Kafka 3.9.0 or later; Kafka 3.7.x is no longer supported and its previous - special-casing has been removed entirely, rather than narrowed. Running KRaft mode on an unsupported Kafka - version is undefined behavior ([#NNNN]). +- `controller.quorum.bootstrap.servers` now points at each controller role group's + headless Service DNS name instead of individual pod addresses, so neither the container + commands nor that ConfigMap value change with the replica count anymore ([#1010]). +- The controller's StatefulSet now scales sequentially (`OrderedBy`) instead of parallel. + This ensures that one voter joins the quorum at a time ([#1010]). ### Fixed @@ -67,8 +60,7 @@ All notable changes to this project will be documented in this file. [#994]: https://github.com/stackabletech/kafka-operator/pull/994 [#998]: https://github.com/stackabletech/kafka-operator/pull/998 [#1000]: https://github.com/stackabletech/kafka-operator/pull/1000 -[#1006]: https://github.com/stackabletech/kafka-operator/pull/1006 -[#NNNN]: https://github.com/stackabletech/kafka-operator/pull/NNNN +[#1010]: https://github.com/stackabletech/kafka-operator/pull/1010 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc index 3acdf8b5..ebf6ffd0 100644 --- a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc +++ b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc @@ -16,12 +16,6 @@ WARNING: The Stackable Operator for Apache Kafka currently does not support auto * Full Replacement: Kafka 4.0.0 (2025) removes ZooKeeper completely. * Migration: Tools exist to migrate from ZooKeeper to KRaft, but new deployments should start with KRaft. -IMPORTANT: The Stackable Operator for Apache Kafka requires Kafka 3.9.0 or later for KRaft mode. Kafka 3.7.x is not -supported: the operator relies on the KIP-853 dynamic quorum tooling (`kafka-storage.sh format --standalone` / -`--no-initial-controllers`, and `kafka-metadata-quorum.sh add-controller` / `remove-controller`), which requires -3.9.0+. Running KRaft mode on an unsupported Kafka version is undefined behavior, up to and including -`kafka-storage.sh` rejecting the operator-generated formatting command outright. - == Configuration The Stackable Kafka operator introduces a new xref:concepts:roles-and-role-groups.adoc[role] in the KafkaCluster CRD called KRaft `Controller`. @@ -91,12 +85,15 @@ KRaft mode requires major configuration changes compared to ZooKeeper: * `cluster-id`: This is set to the `metadata.name` of the KafkaCluster resource during initial formatting * `node.id`: This is a calculated integer, hashed from the `role` and `rolegroup` and added `replica` id. * `process.roles`: Will always only be `broker` or `controller`. Mixed `broker,controller` servers are not supported. +* Each controller pod runs an additional `quorum-manager` sidecar container. On startup it admits the pod into + the KRaft voter set (`kafka-metadata-quorum.sh add-controller`), and on pod termination (`preStop`) it removes + the pod from the voter set again (`remove-controller`). * Controller pods have a `startupProbe` and a plain TCP `livenessProbe` on the KRaft listener port, and a `readinessProbe` that checks the controller's Raft state (`leader`, `follower`, or `voted`) via its metrics endpoint rather than a bare TCP check, so a controller that cannot join or rejoin the quorum is correctly reported as not ready instead of appearing healthy. * Admitting a controller into the KRaft voter set is *solely* the concern of the `quorum-manager` sidecar container - (see below) — the format step never asserts a voter list. Exactly one controller (the one with the numerically + — the format step never asserts a voter list. Exactly one controller (the one with the numerically lowest `node.id` among all controller pod descriptors) formats with `kafka-storage.sh format --standalone`, bootstrapping a single-node quorum by itself. Every other controller — whether it is part of the cluster's initial desired replica count or added later on scale-up — formats with `--no-initial-controllers` and joins purely @@ -105,35 +102,17 @@ KRaft mode requires major configuration changes compared to ZooKeeper: the current replica count. * `controller.quorum.bootstrap.servers` (used by the `kafka` process itself to find the controller quorum, and by the `quorum-manager` sidecar for its own `add-controller`/`remove-controller` calls) points at each controller - role group's own headless Service DNS name, not individual pod addresses. A headless Service's own DNS name - resolves to every backing pod's IP — exactly what Kafka's `client.dns.lookup=use_all_dns_ips` default already - expects — and the Service sets `publishNotReadyAddresses: true`, so this also resolves correctly during initial - cluster formation before any pod is `Ready`. Combined with the previous point, this makes the whole controller - pod template invariant to `spec.controllers.roleGroups..replicas`: confirmed live, scaling a controller - role group up or down leaves every already-existing controller pod completely untouched (same UID, zero - restarts, no `StatefulSet` revision change) — only the pods actually being added or removed are touched. -* When Kerberos is not enabled, each controller pod runs an additional `quorum-manager` sidecar container (requires - Kafka 3.9.0 or later, see the minimum-version note above). On startup it admits the pod into the KRaft voter set - (`kafka-metadata-quorum.sh add-controller`), and on pod termination (`preStop`) it removes the pod from the voter - set again (`remove-controller`), but only if doing so would not remove the last remaining voter. + role group's own headless Service DNS name, not individual pod addresses. == Known Issues * Automatic migration from Apache ZooKeeper to KRaft is not supported. * Kerberos is currently not supported for KRaft in all versions. -* Scaling controllers down to a single replica is not verified under the sidecar-based mechanism described above; - only scale-downs that keep an odd number of controllers greater than one have been tested. -* If a `remove-controller` call fails or times out during pod termination (for example, no reachable leader within - the pod's grace period), the pod terminates anyway and can leave a stale voter entry in the quorum behind. This is - not fully automatic in every case and may require manual cleanup with `kafka-metadata-quorum.sh remove-controller`. -* The single controller chosen to bootstrap the quorum standalone (see "Internal operator details" above) is picked - by a stable, deterministic rule (lowest `node.id`), which is safe only for a cluster's *original* bootstrap. If - that specific controller's persistent volume is ever lost and needs to reformat after the cluster has already - formed a quorum elsewhere, reformatting it with `--standalone` would bootstrap a second, conflicting one-node - quorum instead of rejoining the existing one. This is the same class of manual-recovery scenario as losing enough - voters to break quorum in any Raft-based system, and the operator has no way to detect or repair it automatically - (it deliberately has no live-cluster awareness). If this happens, recovery requires manual intervention with - Kafka's own KRaft tooling. +* The single controller chosen to bootstrap the quorum standalone is picked by a stable, deterministic rule + (lowest `node.id`), which is safe only for a cluster's *original* bootstrap. If that specific controller's + persistent volume is ever lost and needs to reformat after the cluster has already formed a quorum elsewhere, + reformatting it with `--standalone` would bootstrap a second, conflicting one-node quorum instead of rejoining + the existing one. If this happens, recovery requires manual intervention with Kafka's own KRaft tooling. == Troubleshooting @@ -153,17 +132,11 @@ The Stackable Kafka operator currently does not support the migration. === Scaling controllers Controller replicas can be scaled up and down on a running cluster. A per-pod `quorum-manager` sidecar admits and -removes the pod from the KRaft voter set as described under "Internal operator details" above. This requires Kafka -3.9.0 or later (see the minimum-version note in "Overview"), which supports the -https://developers.redhat.com/articles/2024/11/27/dynamic-kafka-controller-quorum[KIP-853 dynamic quorum tooling]. +removes the pod from the KRaft voter set as described under "Internal operator details" above. Scaling more than one controller down at a time is processed one pod at a time (`OrderedReady` pod management), not in parallel, so that each pod's removal from the voter set can complete before the next one is terminated. -This has been validated on a live cluster and is covered by unit tests, but a full end-to-end scale-up/scale-down -kuttl test suite run has not yet produced a clean pass; treat controller scaling as functional but not yet fully -verified end-to-end. - == Kraft migration guide The operator version `26.3.0` adds support for migrating Kafka clusters from ZooKeeper to KRaft mode. diff --git a/docs/superpowers/plans/2026-08-14-kraft-dynamic-voter-membership.md b/docs/superpowers/plans/2026-08-14-kraft-dynamic-voter-membership.md deleted file mode 100644 index 723f7612..00000000 --- a/docs/superpowers/plans/2026-08-14-kraft-dynamic-voter-membership.md +++ /dev/null @@ -1,1008 +0,0 @@ -# KRaft Dynamic Voter Membership Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Let KRaft controller replicas be scaled up and down on a running Kafka cluster, by having each controller pod manage its own quorum membership via a sidecar container instead of the operator talking to the live cluster. - -**Architecture:** A new `quorum-manager` sidecar container (reusing the Kafka product image) runs alongside the `kafka` container on controller pods only. Its main process loops, admitting itself as a voter (`kafka-metadata-quorum.sh add-controller`) while its local Raft state is `observer`. Its `preStop` hook checks a majority-safety condition and removes itself (`remove-controller`) before termination. The controller StatefulSet switches `podManagementPolicy` to `OrderedReady` so Kubernetes drains controllers one at a time on scale-down. All of this is gated to Kafka versions that support KIP-853 dynamic quorum tooling (everything except the `3.7.x` line, mirroring the existing `--initial-controllers` version check). - -**Tech Stack:** Rust (`stackable-operator`, `kube-rs` builder types), Bash (sidecar scripts), Kafka's `kafka-metadata-quorum.sh` CLI, kuttl (integration tests). - -**Spec:** `docs/superpowers/specs/2026-08-14-kraft-dynamic-voter-membership-design.md` - -## Global Constraints - -- Sidecar is added only for the controller role (`build_controller_rolegroup_statefulset`), never brokers. -- Sidecar is added only when `!resolved_product_image.product_version.starts_with("3.7")` — same literal-prefix check style as `initial_controllers_command` (`rust/operator-binary/src/controller/build/command.rs:198-211`) and `uses_legacy_log4j` (`rust/operator-binary/src/controller/build/properties/mod.rs:55-57`). -- Sidecar is not added when `kafka_security.has_kerberos_enabled()` is true — Kerberos for KRaft is already a documented unsupported combination (`docs/modules/kafka/pages/usage-guide/kraft-controller.adoc`, Known Issues), and the sidecar's admin-client properties file only handles the TLS/SSL case. -- The sidecar's `preStop` script must always exit `0`, regardless of whether `remove-controller` succeeded — it must never block pod termination. -- The sidecar targets the quorum's bootstrap servers (its peers), never `localhost` for the admin-client calls — its own `kafka` container may be concurrently shutting down. -- No new Kubernetes RBAC, no new reconcile phase, no CRD status field. All Rust changes are confined to the `build` phase. -- Every `rust/` change must pass `cargo build`, `cargo test`, `cargo clippy --all-targets -- -D warnings`, `cargo fmt --check` before being considered done, per this repo's existing verification gate. CRDs/docs are regenerated (`make regenerate-charts`) whenever the CRD schema changes. -- A CHANGELOG entry is added in the same commit as the change it documents, under a new `### Added` section (there is currently no `### Added` section under `## [Unreleased]` in `CHANGELOG.md`). - ---- - -## Task 1: Version gate helper — `supports_dynamic_quorum` - -**Files:** - -- Modify: `rust/operator-binary/src/controller/build/properties/mod.rs` -- Test: same file, `#[cfg(test)] mod tests` (create if absent — check first, this file may already have one) - -**Interfaces:** - -- Produces: `pub fn supports_dynamic_quorum(product_version: &str) -> bool` — used by Task 4 (sidecar container gating) and Task 7 (kuttl test gating verification). - -- [ ] **Step 1: Check for an existing test module in this file** - -Run: `grep -n "mod tests" rust/operator-binary/src/controller/build/properties/mod.rs` - -If it exists, note the line number — new tests go inside it. If not, one will be created in Step 3. - -- [ ] **Step 2: Write the failing test** - -Add near the existing `uses_legacy_log4j` function (`rust/operator-binary/src/controller/build/properties/mod.rs:55-57`): - -```rust -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dynamic_quorum_is_supported_from_3_9_onwards() { - assert!(supports_dynamic_quorum("3.9.2")); - assert!(supports_dynamic_quorum("4.1.1")); - assert!(supports_dynamic_quorum("4.2.1")); - } - - #[test] - fn dynamic_quorum_is_not_supported_on_3_7() { - assert!(!supports_dynamic_quorum("3.7.2")); - } -} -``` - -If a `mod tests` block already exists in this file, add these two `#[test]` functions inside it instead of writing a new module, and skip the `use super::*;` line if it's already present. - -- [ ] **Step 3: Run the test to verify it fails** - -Run: `cargo test -p stackable-kafka-operator-binary supports_dynamic_quorum 2>&1 | tail -30` -Expected: compile error, `supports_dynamic_quorum` not found. - -- [ ] **Step 4: Write the minimal implementation** - -Add next to `uses_legacy_log4j`: - -```rust -/// Whether this Kafka version supports the KIP-853 dynamic KRaft quorum tooling -/// (`kafka-metadata-quorum.sh add-controller` / `remove-controller`) needed to change -/// the voter set of an already-formed quorum. Mirrors the existing 3.7.x carve-out -/// already used for `--initial-controllers` (see `initial_controllers_command` in -/// `build/command.rs`). -pub fn supports_dynamic_quorum(product_version: &str) -> bool { - !product_version.starts_with("3.7") -} -``` - -- [ ] **Step 5: Run the test to verify it passes** - -Run: `cargo test -p stackable-kafka-operator-binary supports_dynamic_quorum 2>&1 | tail -30` -Expected: both tests PASS. - -- [ ] **Step 6: Commit** - -```bash -git add rust/operator-binary/src/controller/build/properties/mod.rs -git commit -m "feat: add supports_dynamic_quorum version gate - -Co-Authored-By: Claude Sonnet 5 " -``` - ---- - -## Task 2: Admin-client properties file for the sidecar's TLS config - -**Files:** - -- Modify: `rust/operator-binary/src/controller/build/security.rs` -- Modify: `rust/operator-binary/src/controller/build/resource/config_map.rs` -- Test: `rust/operator-binary/src/controller/build/security.rs`, existing `#[cfg(test)] mod tests` (lines 653-992) - -**Interfaces:** - -- Consumes: `push_client_ssl_stores` (`security.rs:374-388`), `push_client_ssl_truststore` (`security.rs:392-405`), `STACKABLE_TLS_KAFKA_INTERNAL_DIR` (`security.rs:44`), `PROPERTY_SECURITY_PROTOCOL` (`security.rs:41`), `ValidatedKafkaSecurity` (already used throughout this file). -- Produces: `pub fn controller_admin_client_properties(security: &ValidatedKafkaSecurity) -> Vec<(String, Option)>` — consumed by Task 3's ConfigMap wiring and referenced by path (`/stackable/config/admin-client.properties`) in Task 4's sidecar scripts. - -The existing `client.properties` (built by `client_properties()`, `security.rs:165-222`) is unusable for the sidecar: it points at `/stackable/tls-kafka-server`, a directory that is only mounted on broker pods (`add_broker_volume_and_volume_mounts`), never on controller pods. Controller pods only mount `/stackable/tls-kafka-internal` (`add_controller_volume_and_volume_mounts`, `security.rs:301-337`). This task adds a new properties builder pointed at that directory instead, using unprefixed `security.protocol`/`ssl.*` keys (the ones a plain Kafka admin client / `--command-config` needs), as opposed to the `listener.name.controller.ssl.*`-prefixed keys `controller_config_settings()` writes for the broker/controller's own server-side listener config. - -- [ ] **Step 1: Write the failing test** - -Add inside the existing `#[cfg(test)] mod tests` block in `security.rs` (near the other `*_properties`-style tests — check the existing fixtures `plaintext()`, `server_tls()`, `client_auth_tls()`, `as_map()` around lines 653-992 and reuse them): - -```rust - #[test] - fn controller_admin_client_properties_uses_the_internal_tls_directory() { - let security = server_tls(); - let props = as_map(controller_admin_client_properties(&security)); - - assert_eq!(props.get("security.protocol"), Some(&"SSL".to_string())); - assert_eq!( - props.get("ssl.truststore.location"), - Some(&"/stackable/tls-kafka-internal/truststore.p12".to_string()) - ); - assert_eq!(props.get("ssl.truststore.type"), Some(&"PKCS12".to_string())); - } - - #[test] - fn controller_admin_client_properties_includes_keystore_when_client_auth_is_required() { - let security = client_auth_tls(); - let props = as_map(controller_admin_client_properties(&security)); - - assert_eq!( - props.get("ssl.keystore.location"), - Some(&"/stackable/tls-kafka-internal/keystore.p12".to_string()) - ); - } - - #[test] - fn controller_admin_client_properties_is_plaintext_when_no_tls_is_configured() { - let security = plaintext(); - let props = as_map(controller_admin_client_properties(&security)); - - assert_eq!(props.get("security.protocol"), None); - assert_eq!(props.get("ssl.truststore.location"), None); - } -``` - -Check the exact names/signatures of `server_tls()`, `client_auth_tls()`, `plaintext()`, and `as_map()` in the existing test module before using them — copy their exact fixture-building style if these names differ slightly. - -- [ ] **Step 2: Run the tests to verify they fail** - -Run: `cargo test -p stackable-kafka-operator-binary controller_admin_client_properties 2>&1 | tail -40` -Expected: compile error, `controller_admin_client_properties` not found. - -- [ ] **Step 3: Write the minimal implementation** - -Add to `security.rs`, near `client_properties()` (around line 165), following the same shape (`Vec<(String, Option)>` of key/value pairs, `None` values filtered out by the caller): - -```rust -/// Client-side (unprefixed `security.protocol`/`ssl.*`) properties for an admin CLI tool -/// (e.g. `kafka-metadata-quorum.sh`) talking to the CONTROLLER listener from *inside* a -/// controller pod, over the `tls-kafka-internal` volume mounted by -/// `add_controller_volume_and_volume_mounts`. -/// -/// This is deliberately separate from `client_properties()`: that function points at -/// `/stackable/tls-kafka-server`, a directory that is only mounted on broker pods. -pub fn controller_admin_client_properties( - security: &ValidatedKafkaSecurity, -) -> Vec<(String, Option)> { - let mut properties = vec![]; - - if security.tls_internal_secret_class().is_some() { - properties.push(( - PROPERTY_SECURITY_PROTOCOL.to_string(), - Some("SSL".to_string()), - )); - push_client_ssl_truststore(&mut properties, STACKABLE_TLS_KAFKA_INTERNAL_DIR); - if security.tls_client_authentication_class().is_some() { - push_client_ssl_stores(&mut properties, STACKABLE_TLS_KAFKA_INTERNAL_DIR); - } - } - - properties -} -``` - -Check the exact method name for "is internal TLS configured" (`tls_internal_secret_class()` is a guess based on the sibling `tls_server_secret_class()`/`tls_client_authentication_class()` naming seen in `kcat_prober_container_commands`, `security.rs:95-161`) — grep for the real accessor: - -Run: `grep -n "fn tls_.*secret_class\|fn tls_client_authentication_class" rust/operator-binary/src/crd/security.rs rust/operator-binary/src/controller/security.rs 2>/dev/null` - -Adjust the method name used above to match what actually exists on `ValidatedKafkaSecurity`. - -- [ ] **Step 4: Run the tests to verify they pass** - -Run: `cargo test -p stackable-kafka-operator-binary controller_admin_client_properties 2>&1 | tail -40` -Expected: all three PASS. - -- [ ] **Step 5: Wire the new properties into the controller rolegroup ConfigMap** - -Read `rust/operator-binary/src/controller/build/resource/config_map.rs:150-175` first to see exactly how `client.properties` is added, then add a sibling entry for the controller role only. Find the `ConfigFileName` enum (grep `enum ConfigFileName`) and add a variant: - -Run: `grep -n "enum ConfigFileName" -A 10 rust/operator-binary/src/controller/build/resource/config_map.rs rust/operator-binary/src/crd/mod.rs 2>/dev/null` - -Add a variant named `AdminClient` (kebab-case via the same derive macros the enum already uses) that serializes to `admin-client.properties`, then add, guarded to the controller role group's `add_data` block (mirroring the `client.properties` call at `config_map.rs:155-165`): - -```rust - .add_data( - ConfigFileName::AdminClient.to_string(), - to_java_properties_string( - controller_admin_client_properties(kafka_security) - .iter() - .filter_map(|(k, v)| v.as_ref().map(|v| (k, v))), - ) - .context(SerializePropertiesSnafu)?, - ) -``` - -Place this call only in the function that builds the **controller** rolegroup ConfigMap, not the broker one — check the function name/boundary by reading the file's structure first (`grep -n "^pub fn\|^fn" rust/operator-binary/src/controller/build/resource/config_map.rs`). - -- [ ] **Step 6: Run the full properties/config_map test suite** - -Run: `cargo test -p stackable-kafka-operator-binary --lib config_map security 2>&1 | tail -60` -Expected: all PASS, no regressions in existing `client.properties`/`controller.properties` tests. - -- [ ] **Step 7: Commit** - -```bash -git add rust/operator-binary/src/controller/build/security.rs rust/operator-binary/src/controller/build/resource/config_map.rs -git commit -m "feat: add admin-client.properties for controller-pod CLI tools - -Co-Authored-By: Claude Sonnet 5 " -``` - ---- - -## Task 3: Expose bootstrap servers and node id to the sidecar - -**Files:** - -- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs` - -**Interfaces:** - -- Consumes: `kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Vec` (`rust/operator-binary/src/controller/build/properties/mod.rs:59-71`, already `pub(crate)`), `validated_cluster.pod_descriptors(Some(kafka_role))` (already used at `statefulset.rs:275`, `:505`), `KAFKA_NODE_ID_OFFSET` env var name and `node_id_hash32_offset(...)` (already used at `statefulset.rs:706-709` on the `kafka` container — read that exact block before duplicating it). -- Produces: two env vars the sidecar container (Task 4) will read: `KAFKA_CONTROLLER_QUORUM_BOOTSTRAP_SERVERS` (comma-joined `host:port` list) and the same `REPLICA_ID`-deriving inputs (`POD_NAME` via downward API, `NODE_ID_OFFSET`) already present on the `kafka` container, so the sidecar's script can compute its own replica/node id exactly as `command.rs:169-170` does inside the `kafka` container's entrypoint. - -This task only adds env vars to the (not-yet-created) sidecar container's builder; Task 4 creates that builder. Do this task by extending `build_controller_rolegroup_statefulset` to compute the values once and store them in local variables the Task 4 diff will consume — do not create the sidecar container yet, since that would make this task's diff untestable on its own. Instead, write a small pure helper function now, unit-test it in isolation, and call it from Task 4. - -- [ ] **Step 1: Write the failing test** - -Add near wherever `kraft_controllers` is exported from (`rust/operator-binary/src/controller/build/properties/mod.rs`), or create a new test in `statefulset.rs` if a test module doesn't exist yet there (check first: `grep -n "mod tests" rust/operator-binary/src/controller/build/resource/statefulset.rs`; if absent, this task creates the module, which Task 6 will also extend): - -```rust -#[cfg(test)] -mod tests { - use crate::controller::build::properties::kraft_controllers; - use crate::crd::mod::KafkaPodDescriptor; // adjust path once the real module path is confirmed - - #[test] - fn quorum_manager_bootstrap_servers_env_value_is_comma_joined_host_ports() { - // Build two minimal KafkaPodDescriptor values for controllers and assert - // kraft_controllers(...).join(",") produces "host1:9093,host2:9093". - // Fill in with the real KafkaPodDescriptor construction used in - // crd/mod.rs's own tests, since its fields are crate-private (pub(crate)). - } -} -``` - -Before writing this test for real, run: - -Run: `grep -n "KafkaPodDescriptor {" rust/operator-binary/src/crd/mod.rs` - -to find an existing test or construction site building a `KafkaPodDescriptor` by hand (its fields are `pub(crate)`, so this must be done from within the `crd` module or via a test already inside `crd/mod.rs`). If no direct constructor is accessible from `statefulset.rs`'s test module, skip a standalone unit test for the joining logic here (it's a one-line `.join(",")` over an already-tested function) and instead verify this wiring via the integration-style test added in Task 6, which builds a full `ValidatedCluster` through the public `validate()` path and inspects the sidecar container's env vars directly. Note that decision in the commit message for this task. - -- [ ] **Step 2: Add the env var to `build_controller_rolegroup_statefulset`** - -In `rust/operator-binary/src/controller/build/resource/statefulset.rs`, inside `build_controller_rolegroup_statefulset` (around line 505, right after the existing `pod_descriptors(Some(kafka_role))` call used for `controller_kafka_container_command`), compute: - -```rust - let controller_pod_descriptors = validated_cluster - .pod_descriptors(Some(kafka_role)) - .context(BuildPodDescriptorsSnafu)?; - let quorum_bootstrap_servers = - crate::controller::build::properties::kraft_controllers(&controller_pod_descriptors) - .join(","); -``` - -Reuse the existing `pod_descriptors(...)` call already present at line 505 rather than calling it twice — read the surrounding code first and thread `controller_pod_descriptors` through to both the existing `controller_kafka_container_command(...)` call and this new binding, instead of calling `pod_descriptors` a second time. - -Store `quorum_bootstrap_servers` in a local variable for Task 4 to consume when building the sidecar container's env vars — do not add it to the `kafka` container's env vars in this task (it's only needed by the sidecar). - -- [ ] **Step 3: Run the build to confirm it still compiles** - -Run: `cargo build -p stackable-kafka-operator-binary 2>&1 | tail -40` -Expected: compiles cleanly. `quorum_bootstrap_servers` will show an "unused variable" warning until Task 4 consumes it — that's expected and acceptable to leave as a `#[allow(unused)]`-free warning between these two tasks only if they're implemented back-to-back in the same session; otherwise prefix with `_` temporarily. Prefer implementing Task 4 immediately after this task in the same sitting so the warning never needs suppressing. - -- [ ] **Step 4: Commit** - -```bash -git add rust/operator-binary/src/controller/build/resource/statefulset.rs -git commit -m "feat: compute quorum bootstrap servers for the controller sidecar - -Co-Authored-By: Claude Sonnet 5 " -``` - ---- - -## Task 4: Build the `quorum-manager` sidecar container - -**Files:** - -- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs` -- Modify: `rust/operator-binary/src/controller/build/command.rs` - -**Interfaces:** - -- Consumes: `supports_dynamic_quorum` (Task 1), `controller_admin_client_properties` path convention `/stackable/config/admin-client.properties` (Task 2 — the file this properties struct serializes to, mounted via the existing `STACKABLE_CONFIG_DIR_NAME` volume mount already present on the `kafka` container at `statefulset.rs:293`), `quorum_bootstrap_servers` local variable (Task 3), `METRICS_PORT`/`METRICS_PORT_NAME` (`crd/mod.rs:45-46`), `kafka_security.has_kerberos_enabled()` (already used at `container_ports`, `statefulset.rs:644-668`). -- Produces: the sidecar `Container`, added to the pod via `pod_builder.add_container(...)` — consumed by Task 6's unit tests (which inspect it by container name `"quorum-manager"`) and Task 7's kuttl assertions (which observe its effect on the live cluster). - -- [ ] **Step 1: Add the two script-building functions to `command.rs`** - -Read `rust/operator-binary/src/controller/build/command.rs` in full first (it's 209 lines) to match its existing style (plain `String`/`format!`, no templating engine). Add two new functions near `controller_kafka_container_command`: - -```rust -/// The `kafka-metadata-quorum.sh` binary, referenced by its absolute path (matching every -/// other exec-into-pod usage of a Kafka CLI tool in this repo, e.g. the kuttl test scripts -/// under `tests/templates/kuttl/*/*.sh`), rather than the relative `bin/...` form used only -/// inside the `kafka` container's own entrypoint (which runs with the Kafka install dir as -/// its working directory). -const KAFKA_METADATA_QUORUM_BINARY: &str = "/stackable/kafka/bin/kafka-metadata-quorum.sh"; - -const ADMIN_CLIENT_PROPERTIES_PATH: &str = "/stackable/config/admin-client.properties"; - -/// The sidecar's main-loop command: while this controller's local Raft state is -/// `observer`, repeatedly attempt to admit it into the quorum's voter set. -/// -/// `bootstrap_servers` is the comma-joined `host:port` list produced by -/// `kraft_controllers(...)` (see `build/properties/mod.rs`). -pub fn quorum_manager_container_command(bootstrap_servers: &str) -> String { - format!( - r#" - set -uo pipefail - echo "Starting KRaft voter admission loop against bootstrap servers: {bootstrap_servers}" - while true; do - state=$(curl -s localhost:{metrics_port}/metrics | grep -oE 'kafka_server_raft_metrics_current_state\{{state="[a-z]+"\}}' | grep -oE '"[a-z]+"' | tr -d '"') - if [ "$state" = "observer" ]; then - echo "Local Raft state is observer, attempting add-controller..." - {binary} --bootstrap-controller '{bootstrap_servers}' --command-config {config} add-controller \ - || echo "add-controller attempt failed (this is expected if it already succeeded or a leader election is in progress), will retry" - fi - sleep 10 - done - "#, - bootstrap_servers = bootstrap_servers, - metrics_port = METRICS_PORT, - binary = KAFKA_METADATA_QUORUM_BINARY, - config = ADMIN_CLIENT_PROPERTIES_PATH, - ) -} - -/// The sidecar's `preStop` command: before this controller pod terminates, check that -/// removing it still leaves the quorum with a majority of its *current* voter count, and -/// if so, remove it from the voter set. Always exits 0 — a stuck or failed check must -/// never block pod termination. -/// -/// `node_id` is this controller's own KRaft node id (the same value written to -/// `node.id` in `controller.properties`, derived from `$POD_NAME` and `NODE_ID_OFFSET` -/// exactly as the `kafka` container's own entrypoint does — see `controller_kafka_container_command`). -pub fn quorum_manager_pre_stop_command(bootstrap_servers: &str) -> String { - format!( - r#" - set -uo pipefail - POD_INDEX=$(echo "$POD_NAME" | grep -oE '[0-9]+$') - REPLICA_ID=$((POD_INDEX + NODE_ID_OFFSET)) - DEADLINE=$((SECONDS + 25)) - while [ "$SECONDS" -lt "$DEADLINE" ]; do - describe=$({binary} --bootstrap-controller '{bootstrap_servers}' --command-config {config} describe --replication 2>/dev/null) - if [ -n "$describe" ]; then - # NOTE: this parsing was written against the documented `describe --replication` - # tabular output (one voter per line, NodeId as the first column) and must be - # confirmed/adjusted against a live cluster's real output before this is - # considered done -- see Task 4 Step 4 below. - total_voters=$(echo "$describe" | tail -n +2 | grep -c .) - majority=$(( total_voters / 2 + 1 )) - remaining_after_removal=$(( total_voters - 1 )) - if [ "$remaining_after_removal" -ge "$majority" ]; then - directory_id=$(echo "$describe" | tail -n +2 | awk -v id="$REPLICA_ID" '$1 == id {{ print $2 }}') - if [ -n "$directory_id" ]; then - echo "Removing self (node $REPLICA_ID, directory $directory_id) from the voter set..." - {binary} --bootstrap-controller '{bootstrap_servers}' --command-config {config} remove-controller \ - --controller-id "$REPLICA_ID" --controller-directory-id "$directory_id" \ - || echo "remove-controller failed, proceeding with termination anyway" - fi - else - echo "Removing self would break quorum majority ($remaining_after_removal remaining of $majority needed), skipping and retrying..." - fi - break - fi - sleep 2 - done - exit 0 - "#, - bootstrap_servers = bootstrap_servers, - binary = KAFKA_METADATA_QUORUM_BINARY, - config = ADMIN_CLIENT_PROPERTIES_PATH, - ) -} -``` - -Import `METRICS_PORT` at the top of `command.rs` if not already imported (`grep -n "METRICS_PORT" rust/operator-binary/src/controller/build/command.rs`). - -- [ ] **Step 2: Write the failing unit tests for the two command strings** - -Add to (or create) a `#[cfg(test)] mod tests` in `command.rs`: - -```rust -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn quorum_manager_container_command_targets_the_bootstrap_servers_not_localhost() { - let command = quorum_manager_container_command("controller-0:9093,controller-1:9093"); - assert!(command.contains("--bootstrap-controller 'controller-0:9093,controller-1:9093'")); - assert!(command.contains("add-controller")); - assert!(!command.contains("--bootstrap-controller 'localhost")); - } - - #[test] - fn quorum_manager_pre_stop_command_always_exits_zero() { - let command = quorum_manager_pre_stop_command("controller-0:9093,controller-1:9093"); - assert!(command.trim_end().ends_with("exit 0")); - assert!(command.contains("remove-controller")); - } -} -``` - -If `command.rs` already has a test module, add these two functions inside it instead. - -- [ ] **Step 3: Run the tests to verify they pass** - -Run: `cargo test -p stackable-kafka-operator-binary quorum_manager 2>&1 | tail -40` -Expected: both PASS (these are just string-content assertions, so they should pass immediately once Step 1's functions compile — this is a case where writing the test after the implementation is acceptable, since the "test" here is really a guard against a future accidental typo in the command string, not driving the design). - -- [ ] **Step 4: Build the sidecar container in `statefulset.rs`** - -In `rust/operator-binary/src/controller/build/resource/statefulset.rs`, add a new function near `add_vector_container` (bottom of file): - -```rust -/// Builds the `quorum-manager` sidecar for a controller pod. Returns `None` when this -/// Kafka version doesn't support KIP-853 dynamic quorum tooling, or when Kerberos is -/// enabled (the sidecar's admin-client properties file only covers the TLS/SSL case). -fn build_quorum_manager_container( - resolved_product_image: &ResolvedProductImage, - kafka_security: &ValidatedKafkaSecurity, - quorum_bootstrap_servers: &str, -) -> Result, Error> { - if !supports_dynamic_quorum(&resolved_product_image.product_version) - || kafka_security.has_kerberos_enabled() - { - return Ok(None); - } - - let container_name = "quorum-manager".to_string(); - let mut cb = ContainerBuilder::new(&container_name).context(InvalidContainerNameSnafu { - name: container_name.clone(), - })?; - - cb.image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-c".to_string(), - quorum_manager_container_command(quorum_bootstrap_servers), - ]) - .add_env_vars(vec![EnvVar { - name: "POD_NAME".to_string(), - value_from: Some(EnvVarSource { - field_ref: Some(ObjectFieldSelector { - api_version: Some("v1".to_string()), - field_path: "metadata.name".to_string(), - }), - ..EnvVarSource::default() - }), - ..EnvVar::default() - }]) - .resources( - ResourceRequirementsBuilder::new() - .with_cpu_request("100m") - .with_cpu_limit("200m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - ) - .add_volume_mount(STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_DIR) - .context(AddVolumeMountSnafu)? - .lifecycle_pre_stop(LifecycleHandler { - exec: Some(ExecAction { - command: Some(vec![ - "/bin/bash".to_string(), - "-c".to_string(), - quorum_manager_pre_stop_command(quorum_bootstrap_servers), - ]), - }), - ..LifecycleHandler::default() - }); - - Ok(Some(cb.build())) -} -``` - -Add the required imports at the top of `statefulset.rs`: `LifecycleHandler` from `stackable_operator::k8s_openapi::api::core::v1` (alongside the existing `ExecAction`, `EnvVar`, `EnvVarSource`, `ObjectFieldSelector` imports at lines 22-25), `ResourceRequirementsBuilder` (already imported at line 10 for the broker's kcat-prober container — reuse it), and `supports_dynamic_quorum`, `quorum_manager_container_command`, `quorum_manager_pre_stop_command` from `crate::controller::build::{properties, command}`. - -Also add the `Q` sidecar needs the `NODE_ID_OFFSET` env var referenced by its `preStop` script (`$NODE_ID_OFFSET`) — read `statefulset.rs:706-709` (the `kafka` container's own `NODE_ID_OFFSET` env var construction) and add the identical `EnvVar` to the sidecar's `add_env_vars` call in the snippet above, rather than duplicating the whole block — extract the shared computation into a local variable used by both containers if it isn't already. - -Then, inside `build_controller_rolegroup_statefulset`, right after the existing `pod_builder.add_container(kafka_container)` call (around line 579), add: - -```rust - if let Some(quorum_manager_container) = build_quorum_manager_container( - resolved_product_image, - kafka_security, - &quorum_bootstrap_servers, - )? { - pod_builder.add_container(quorum_manager_container); - } -``` - -using the `quorum_bootstrap_servers` binding from Task 3. - -- [ ] **Step 5: Verify the CLI's actual `describe --replication` output shape** - -This step is a real verification action, not a placeholder — the `preStop` script's `awk`/`grep` parsing in Step 1 was written against Kafka's documented tabular format and has not been checked against a live cluster. - -Run: `kubectl exec -n test-kafka-controller-default-0 -c kafka -- /stackable/kafka/bin/kafka-metadata-quorum.sh --bootstrap-controller :9093 --command-config /stackable/config/admin-client.properties describe --replication` - -(This requires Task 2's `admin-client.properties` to already be deployed — run this verification after Tasks 2-4 are all merged into a real running cluster, e.g. via a manual `./scripts/run-tests` smoke-kraft run, before considering this task done.) Compare the real column layout (which column holds `NodeId`, which holds `DirectoryId`) against the `awk -v id="$REPLICA_ID" '$1 == id { print $2 }'` assumption in Step 1, and adjust the column indices in `quorum_manager_pre_stop_command` if they don't match. Re-run the unit tests from Step 3 after any change (they assert command *structure*, not the exact awk column numbers, so they should still pass, but re-run them anyway to be safe). - -- [ ] **Step 6: Build and run the full test suite** - -Run: `cargo build -p stackable-kafka-operator-binary 2>&1 | tail -60` -Run: `cargo test -p stackable-kafka-operator-binary 2>&1 | tail -80` -Expected: builds cleanly, all tests PASS (this also exercises every existing `statefulset.rs`/`config_map.rs` test, confirming the new sidecar doesn't break broker-pod builds, which must never get this container). - -- [ ] **Step 7: Regenerate CRDs and check for unexpected diffs** - -Run: `make regenerate-charts 2>&1 | tail -40` -Expected: no diff, since this task adds a container by string literal name rather than a new `ContainerName`-enum variant, so the CRD's `logging.containers` schema is unchanged. If `make regenerate-charts` produces an unexpected diff, investigate before proceeding — it likely means a CRD-visible type changed somewhere in this task's diff. - -- [ ] **Step 8: Commit** - -```bash -git add rust/operator-binary/src/controller/build/resource/statefulset.rs rust/operator-binary/src/controller/build/command.rs -git commit -m "feat: add quorum-manager sidecar to controller pods - -Co-Authored-By: Claude Sonnet 5 " -``` - ---- - -## Task 5: Switch controller StatefulSet to `OrderedReady` pod management - -**Files:** - -- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs` - -**Interfaces:** - -- Consumes: nothing new. -- Produces: `POD_MANAGEMENT_POLICY_ORDERED_READY` constant, consumed only by this task's own change and asserted by Task 6's unit test. - -- [ ] **Step 1: Write the failing unit test** - -In the `statefulset.rs` test module (created in Task 3 or already present), add: - -```rust - #[test] - fn controller_statefulset_uses_ordered_ready_pod_management() { - let cluster = kraft_mode_cluster(); - let resources = crate::controller::build::build(&cluster).expect("build succeeds"); - let sts = resources - .stateful_sets - .into_iter() - .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-controller-default")) - .expect("the controller StatefulSet is built"); - - assert_eq!( - sts.spec.expect("the StatefulSet has a spec").pod_management_policy, - Some("OrderedReady".to_string()) - ); - } - - #[test] - fn broker_statefulset_still_uses_parallel_pod_management() { - let cluster = kraft_mode_cluster(); - let resources = crate::controller::build::build(&cluster).expect("build succeeds"); - let sts = resources - .stateful_sets - .into_iter() - .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-broker-default")) - .expect("the broker StatefulSet is built"); - - assert_eq!( - sts.spec.expect("the StatefulSet has a spec").pod_management_policy, - Some("Parallel".to_string()) - ); - } -``` - -This uses a `kraft_mode_cluster()` fixture. Task 3 deliberately deferred creating this fixture (see its Step 1) in favor of this task owning it. **This task creates `kraft_mode_cluster()`** in this file's test module, copied from the pattern shown in the exploration: a minimal `KafkaCluster` YAML with `clusterConfig.metadataManager: kraft`, one controller role group of 3 replicas, one broker role group of 3 replicas, resolved via `crate::controller::test_support::{minimal_kafka, validated_cluster}`. - -- [ ] **Step 2: Run the tests to verify the controller one fails** - -Run: `cargo test -p stackable-kafka-operator-binary pod_management 2>&1 | tail -30` -Expected: `broker_statefulset_still_uses_parallel_pod_management` PASSes (no change yet), `controller_statefulset_uses_ordered_ready_pod_management` FAILs (`Parallel` != `OrderedReady`). - -- [ ] **Step 3: Make the change** - -In `build_controller_rolegroup_statefulset`, add a new constant near the existing `POD_MANAGEMENT_POLICY_PARALLEL` (`statefulset.rs:127`): - -```rust -const POD_MANAGEMENT_POLICY_ORDERED_READY: &str = "OrderedReady"; -``` - -And change the controller `StatefulSetSpec` construction (`statefulset.rs:620`) from: - -```rust - pod_management_policy: Some(POD_MANAGEMENT_POLICY_PARALLEL.to_string()), -``` - -to: - -```rust - pod_management_policy: Some(POD_MANAGEMENT_POLICY_ORDERED_READY.to_string()), -``` - -Leave the broker StatefulSet's construction (`statefulset.rs:435`) unchanged — it must keep using `POD_MANAGEMENT_POLICY_PARALLEL`. - -- [ ] **Step 4: Run the tests to verify they pass** - -Run: `cargo test -p stackable-kafka-operator-binary pod_management 2>&1 | tail -30` -Expected: both PASS. - -- [ ] **Step 5: Commit** - -```bash -git add rust/operator-binary/src/controller/build/resource/statefulset.rs -git commit -m "feat: use OrderedReady pod management for controller StatefulSets - -Serializes scale-down so each controller's preStop hook (self-removal -from the KRaft voter set) completes before the next pod terminates. - -Co-Authored-By: Claude Sonnet 5 " -``` - ---- - -## Task 6: Unit tests for sidecar presence/absence and version gating - -**Files:** - -- Modify: `rust/operator-binary/src/controller/build/resource/statefulset.rs` - -**Interfaces:** - -- Consumes: `kraft_mode_cluster()` fixture (created by Task 5 — Task 3 deliberately deferred it), `build_quorum_manager_container` / the sidecar's presence in the built `StatefulSet` (Task 4), the `kerberos()` security fixture from `security.rs`'s test module (Task 2 — may need its visibility bumped to `pub(crate)` for this task to reach it). - -- [ ] **Step 1: Write the failing tests** - -```rust - fn controller_containers( - cluster: &crate::controller::ValidatedCluster, - ) -> Vec { - let resources = crate::controller::build::build(cluster).expect("build succeeds"); - let sts = resources - .stateful_sets - .into_iter() - .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-controller-default")) - .expect("the controller StatefulSet is built"); - sts.spec - .expect("the StatefulSet has a spec") - .template - .spec - .expect("the pod template has a spec") - .containers - } - - #[test] - fn controller_pods_get_a_quorum_manager_sidecar_on_supported_versions() { - let cluster = kraft_mode_cluster(); - let containers = controller_containers(&cluster); - - assert!( - containers.iter().any(|c| c.name == "quorum-manager"), - "expected a quorum-manager sidecar, got containers: {:?}", - containers.iter().map(|c| &c.name).collect::>() - ); - } - - #[test] - fn quorum_manager_sidecar_targets_bootstrap_servers_in_its_command() { - let cluster = kraft_mode_cluster(); - let containers = controller_containers(&cluster); - let sidecar = containers - .iter() - .find(|c| c.name == "quorum-manager") - .expect("the quorum-manager sidecar is built"); - - let command = sidecar - .command - .as_ref() - .expect("the sidecar has a command") - .join(" "); - assert!(command.contains("add-controller")); - - let pre_stop_command = sidecar - .lifecycle - .as_ref() - .and_then(|l| l.pre_stop.as_ref()) - .and_then(|h| h.exec.as_ref()) - .and_then(|e| e.command.as_ref()) - .expect("the sidecar has a preStop exec hook") - .join(" "); - assert!(pre_stop_command.contains("remove-controller")); - assert!(pre_stop_command.trim_end().ends_with("exit 0")); - } - - #[test] - fn controller_pods_get_no_quorum_manager_sidecar_on_kafka_3_7() { - let kafka = crate::controller::test_support::minimal_kafka( - r#" - apiVersion: kafka.stackable.tech/v1alpha1 - kind: KafkaCluster - metadata: - name: simple-kafka - namespace: default - uid: 12345678-1234-1234-1234-123456789012 - spec: - image: - productVersion: 3.7.2 - clusterConfig: - metadataManager: kraft - controllers: - roleGroups: - default: - replicas: 3 - brokers: - roleGroups: - default: - replicas: 3 - "#, - ); - let cluster = crate::controller::test_support::validated_cluster(&kafka); - let containers = controller_containers(&cluster); - - assert!(!containers.iter().any(|c| c.name == "quorum-manager")); - } - - #[test] - fn controller_pods_get_no_quorum_manager_sidecar_when_kerberos_is_enabled() { - // This is a Global Constraint (see the plan header): the sidecar's admin-client - // properties file only covers the TLS/SSL case, so it must never be added when - // Kerberos is enabled, even on an otherwise-supported Kafka version. - // - // Rather than building a full CRD-level Kerberos fixture (which needs a resolved - // AuthenticationClass threaded through `DereferencedObjects`, more than this test - // needs), call `build_quorum_manager_container` directly — it already takes - // `&ValidatedKafkaSecurity` as a parameter, so a fixture at that level is enough. - // Reuse the `kerberos()` fixture from `security.rs`'s existing test module (see - // Task 2) for a security value with Kerberos enabled; import it, adjusting its - // visibility to `pub(crate)` in `security.rs` if it is not already visible here. - let cluster = kraft_mode_cluster(); - let kerberos_security = crate::controller::build::security::tests::kerberos(); - - let result = build_quorum_manager_container( - &cluster.image, - &kerberos_security, - "controller-0:9093", - ) - .expect("build_quorum_manager_container does not error for a kerberos security value"); - - assert!(result.is_none()); - } - - #[test] - fn broker_pods_never_get_a_quorum_manager_sidecar() { - let cluster = kraft_mode_cluster(); - let resources = crate::controller::build::build(&cluster).expect("build succeeds"); - let sts = resources - .stateful_sets - .into_iter() - .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-broker-default")) - .expect("the broker StatefulSet is built"); - let containers = sts - .spec - .expect("the StatefulSet has a spec") - .template - .spec - .expect("the pod template has a spec") - .containers; - - assert!(!containers.iter().any(|c| c.name == "quorum-manager")); - } -``` - -Verify `productVersion: 3.7.2` is actually a version accepted by this repo's product-version validation (some operators restrict to an exact known list) — check: - -Run: `grep -rn "3.7" rust/crd/src/ tests/test-definition.yaml 2>/dev/null | head -20` - -If `3.7.2` isn't a recognized version, use whatever 3.7.x version is used elsewhere in this repo's own tests/fixtures instead. - -- [ ] **Step 2: Run the tests to verify they fail (or pass, if Task 4/5 already got this right)** - -Run: `cargo test -p stackable-kafka-operator-binary quorum_manager 2>&1 | tail -60` - -If Tasks 4-5 were implemented correctly, these should already PASS since they're testing behavior those tasks already built — this task exists to lock that behavior in with explicit regression coverage, not to drive new implementation. If any fail, fix the implementation in `statefulset.rs` from Task 4/5 (not the test) unless the test itself has a mistaken assumption — re-read Task 4/5's code before changing either. - -- [ ] **Step 3: Run the full test suite one more time** - -Run: `cargo test -p stackable-kafka-operator-binary 2>&1 | tail -80` -Expected: all PASS. - -- [ ] **Step 4: Commit** - -```bash -git add rust/operator-binary/src/controller/build/resource/statefulset.rs -git commit -m "test: cover quorum-manager sidecar presence, version gate, and commands - -Co-Authored-By: Claude Sonnet 5 " -``` - ---- - -## Task 7: Fix and strengthen the kuttl scale-up/scale-down assertions - -**Files:** - -- Modify: `tests/templates/kuttl/operations-kraft/60-assert.yaml.j2` -- Modify: `tests/templates/kuttl/operations-kraft/70-assert.yaml.j2` - -**Interfaces:** none (test-only, no Rust interfaces). - -Both files currently have a real bug (found during design exploration): the two YAML documents for the broker and controller `StatefulSet` assertions are missing a `---` separator between them, which likely means the second document (the controller assertion) is silently ignored by the YAML parser or produces unexpected behavior. This task fixes that bug and adds a voter-count check. - -- [ ] **Step 1: Read both files in full** - -Run: `cat tests/templates/kuttl/operations-kraft/60-assert.yaml.j2 tests/templates/kuttl/operations-kraft/70-assert.yaml.j2` - -Confirm the missing `---` before the second `apiVersion: apps/v1` block in each file. - -- [ ] **Step 2: Fix the missing document separator in `60-assert.yaml.j2`** - -Insert a `---` line immediately before the second `apiVersion: apps/v1` (the `test-kafka-controller-default` StatefulSet assertion), so the file has three `---`-separated documents: the `TestAssert` header/commands block, the broker StatefulSet assertion, and the controller StatefulSet assertion. - -- [ ] **Step 3: Apply the identical fix to `70-assert.yaml.j2`** - -Same change, same reasoning. - -- [ ] **Step 4: Add a voter-count assertion command to `60-assert.yaml.j2`** - -In the `commands:` list of the `TestAssert` document (alongside the existing `kubectl -n $NAMESPACE wait --for=condition=available ...` command), add: - -```yaml - - script: | - kubectl exec -n $NAMESPACE test-kafka-controller-default-0 -c kafka -- \ - /stackable/kafka/bin/kafka-metadata-quorum.sh \ - --bootstrap-controller test-kafka-controller-default-0.test-kafka-controller-default-headless.$NAMESPACE.svc.cluster.local:9093 \ - --command-config /stackable/config/admin-client.properties \ - describe --replication | tail -n +2 | wc -l | grep -q '^5$' -``` - -Verify the exact headless service name pattern (`test-kafka-controller-default-headless`) against how the FQDN is actually constructed elsewhere in this test suite — grep other files in `tests/templates/kuttl/operations-kraft/` for an existing `--bootstrap-server`/FQDN reference to copy the exact naming convention rather than guessing it: - -Run: `grep -rn "headless\|bootstrap-server" tests/templates/kuttl/operations-kraft/*.j2 tests/templates/kuttl/smoke-kraft/*.j2 2>/dev/null | head -20` - -Adjust the hostname in the command above to match whatever convention those files actually use. - -- [ ] **Step 5: Add the equivalent assertion to `70-assert.yaml.j2`, expecting 3 voters** - -Same command, with `grep -q '^3$'` instead of `'^5$'`, matching the scaled-down replica count. - -- [ ] **Step 6: Run the kuttl test manually** - -Run: `./scripts/run-tests --skip-release --test operations-kraft_kafka-kraft-4.2.1_openshift-false 2>&1 | tail -100` - -Expected: PASS, including the new voter-count checks in steps 60 and 70. If the voter-count check fails while the StatefulSet readiness check passes, that's a real signal the sidecar (Task 4) isn't actually admitting/removing voters correctly — go back to Task 4 and debug using the same `vector tap` / `kubectl logs -c quorum-manager` techniques, rather than loosening this assertion. - -- [ ] **Step 7: Commit** - -```bash -git add tests/templates/kuttl/operations-kraft/60-assert.yaml.j2 tests/templates/kuttl/operations-kraft/70-assert.yaml.j2 -git commit -m "test: fix missing YAML separator and assert voter count in scale tests - -Co-Authored-By: Claude Sonnet 5 " -``` - ---- - -## Task 8: Update documentation and remove the "unsupported" claim - -**Files:** - -- Modify: `docs/modules/kafka/pages/usage-guide/kraft-controller.adoc` -- Modify: `tests/templates/kuttl/operations-kraft/README.md` -- Modify: `CHANGELOG.md` - -**Interfaces:** none. - -- [ ] **Step 1: Update `kraft-controller.adoc`** - -Read the file in full (already read during brainstorming). Remove or rewrite the "Scaling controller replicas up is not supported" bullet under "Known Issues" and the entire "Scaling issues" subsection under "Troubleshooting", replacing them with a short description of the new behavior: - -- Controllers can now be scaled up and down on a running cluster. -- A per-pod `quorum-manager` sidecar handles admitting/removing the pod from the KRaft voter set. -- This requires a Kafka version that supports KIP-853 dynamic quorum tooling (everything except `3.7.x`). -- Scaling more than one controller down at a time is processed one pod at a time (`OrderedReady`), not in parallel. -- If a `remove-controller` call fails during pod termination (e.g. no reachable leader within the grace period), a stale voter entry can be left behind and requires manual cleanup — state this as a known limitation, do not imply it's fully automatic in every case. - -Also update the "Internal operator details" bullet that currently says "the operator does not perform the follow-up step required to add a controller to an already-formed quorum's voter set" — this is no longer true. - -- [ ] **Step 2: Update the kuttl README** - -`tests/templates/kuttl/operations-kraft/README.md` currently states "Scaling controllers from 3 -> 1 doesn't work. Both brokers and controllers try to communicate with old controllers." Verify whether this specific limitation (scaling below a certain floor) is still expected to hold after this change — if scaling controllers down to 1 was never exercised by these tests (they only go 3→5→3), leave this caveat in place rather than removing an unverified claim; do not claim a scenario is fixed that this plan's tests don't actually cover. - -- [ ] **Step 3: Add the CHANGELOG entry** - -In `CHANGELOG.md`, insert a new `### Added` section between `## [Unreleased]` and the existing `### Changed` section: - -```markdown -### Added - -- KRaft controller replicas can now be scaled up and down on a running cluster: a new - `quorum-manager` sidecar container on each controller pod admits itself into the KRaft - voter set on startup and removes itself before termination ([#NNNN]). -``` - -Add the corresponding link reference at the bottom of the file, in ascending numeric order alongside the existing `[#985]`/`[#990]`/etc. links: - -```markdown -[#NNNN]: https://github.com/stackabletech/kafka-operator/pull/NNNN -``` - -Leave `NNNN` as a literal placeholder for the real PR number — fill it in when the PR is actually opened (this is the one acceptable use of a placeholder in this plan, since the number doesn't exist until the PR is created; every other file in this plan has zero placeholders). - -- [ ] **Step 4: Commit** - -```bash -git add docs/modules/kafka/pages/usage-guide/kraft-controller.adoc tests/templates/kuttl/operations-kraft/README.md CHANGELOG.md -git commit -m "docs: document KRaft controller scale-up/down support - -Co-Authored-By: Claude Sonnet 5 " -``` - ---- - -## Task 9: Full verification gate - -**Files:** none (verification only). - -- [ ] **Step 1: Full build** - -Run: `cargo build --workspace 2>&1 | tail -60` -Expected: clean build. - -- [ ] **Step 2: Full test suite** - -Run: `cargo test --workspace 2>&1 | tail -100` -Expected: all PASS. - -- [ ] **Step 3: Clippy** - -Run: `cargo clippy --all-targets -- -D warnings 2>&1 | tail -100` -Expected: no warnings/errors. Fix anything that comes up before proceeding. - -- [ ] **Step 4: Format check** - -Run: `cargo fmt --check 2>&1 | tail -60` -Expected: no diff. If there is one, run `cargo fmt` and amend the relevant task's commit. - -- [ ] **Step 5: Regenerate charts/CRDs one final time** - -Run: `make regenerate-charts 2>&1 | tail -60` -Expected: no diff (confirmed already in Task 4, re-checked here after all subsequent tasks in case anything else drifted). - -- [ ] **Step 6: Full kuttl run for the affected test suite** - -Run: `./scripts/run-tests --skip-release --test operations-kraft_kafka-kraft-4.2.1_openshift-false 2>&1 | tail -150` -Run: `./scripts/run-tests --skip-release --test operations-kraft_kafka-kraft-3.9.2_openshift-false 2>&1 | tail -150` -Expected: both PASS. - -- [ ] **Step 7: Commit any fixups from this task as a single commit, if any were needed** - -```bash -git add -A -git commit -m "chore: fix clippy/fmt findings from verification pass - -Co-Authored-By: Claude Sonnet 5 " -``` - -If nothing needed fixing, skip this commit — don't create an empty one. diff --git a/docs/superpowers/specs/2026-08-14-kraft-dynamic-voter-membership-design.md b/docs/superpowers/specs/2026-08-14-kraft-dynamic-voter-membership-design.md deleted file mode 100644 index d7c52714..00000000 --- a/docs/superpowers/specs/2026-08-14-kraft-dynamic-voter-membership-design.md +++ /dev/null @@ -1,197 +0,0 @@ -# KRaft dynamic voter membership (scale-up / scale-down) - -Status: approved for planning -Date: 2026-08-14 -Branch this was designed on: `main` @ `5211842` - -## Problem - -Apache Kafka's KRaft dynamic quorum (KIP-853) requires an explicit -follow-up step to change the voter set of an already-formed quorum: -`kafka-metadata-quorum.sh add-controller` to admit a new controller, -`remove-controller` to retire one. The Stackable Kafka operator -currently only performs the one-time `--initial-controllers` step at -`kafka-storage.sh format` time. Any controller pod added after initial -cluster formation registers itself and starts up, but never leaves the -Raft `observer` state — it can never become `leader`/`follower`/`voted`, -so it never becomes healthy, and there is no supported way to remove a -controller from the voter set either. This is documented today as a -flat "do not scale controller replicas on a running cluster" limitation -in `docs/modules/kafka/pages/usage-guide/kraft-controller.adoc`. - -Goal: let `spec.controllers.roleGroups..replicas` be scaled up -and down on a running cluster, with the voter set kept in sync -automatically. - -## Scope - -- In scope: admitting new controllers to the voter set on scale-up, - removing controllers from the voter set on scale-down, for live - replica-count changes on an already-formed cluster. -- Out of scope: whole-cluster graceful-deletion draining (a - finalizer-based mechanism is a separate concern from live replica - changes and is not addressed here). ZooKeeper-to-KRaft migration is - unaffected. Kerberos support for KRaft is unaffected. -- This design was developed independently of, and does not reuse code - from, any prior unmerged work on a KRaft "quorum health gate" or - graceful-teardown finalizer — that work was discarded (local - branches deleted) before this design was started. - -## Non-goals / explicitly rejected approaches - -- **Operator-side kube-exec.** An earlier direction had the operator - itself exec into pods (`pods/exec`) to run - `kafka-metadata-quorum.sh`, gated by new reconcile phases (a - pre-`build` gate clamping the effective replica count on scale-down, - a post-`apply` phase admitting new voters on scale-up). This was - rejected in favor of the sidecar approach below: it needed new RBAC, - a new "live cluster" client capability the operator has never had, - and two new reconcile phases, none of which are needed once the pods - manage their own membership. -- **`controller.quorum.auto.join.enable`.** Delegates scale-up - self-promotion entirely to Kafka with zero new operator capability, - but doesn't address scale-down at all (still needs an active - `remove-controller` step), and gives up visibility into *why* - admission might be stuck. Not chosen because scale-down still needs - the same sidecar mechanism anyway, so this would only save the - add-controller half of the problem while adding a version dependency - to check. - -## Design - -### Architecture - -The operator gains **no new awareness of live quorum state**. The -existing reconcile pipeline (`dereference → validate → build → apply → -update_status`) is untouched. All quorum membership management is -delegated to the controller pods themselves, via: - -1. A new sidecar container, controller-role-only, reusing the `kafka` - product image (so `kafka-metadata-quorum.sh` and the TLS trust - material already mounted for the `kafka` container are available - without new volumes). -2. A `preStop` lifecycle hook on that sidecar. -3. `podManagementPolicy: OrderedReady` on the controller StatefulSet - (currently `Parallel`). - -Both the sidecar and the `preStop` hook are only added for Kafka -versions that support KIP-853 dynamic quorum tooling — mirrors the -existing per-version special-casing already present around -`--initial-controllers` for 3.7.x. Older versions get no sidecar at -all and keep today's documented "unsupported" behavior. - -### Components - -**Add-loop script** (the sidecar's main process, runs for the pod's -whole lifetime): - -- Polls the local JMX Prometheus metrics endpoint (the same - `kafka_server_raft_metrics_current_state` series the existing - readiness probe already reads) on a short interval. -- While state is `observer`, runs - `kafka-metadata-quorum.sh add-controller` against - `controller.quorum.bootstrap.servers` (this must be invoked locally - on the joining node — it reads local KRaft directory state - automatically, which also sidesteps the fake placeholder directory-id - used by `KafkaPodDescriptor::as_voter()` at format time; that - placeholder was flagged during design exploration as a hazard for - any tooling that validates directory ids, but `add-controller` does - not consume it). -- Treats "already a voter" responses as success and keeps polling at - the same interval indefinitely (cheap, idempotent, self-healing — - no persisted state, no operator involvement). - -This also resolves what looked like a circular dependency during -design: the existing readiness probe can only pass once raft state -leaves `observer`, so gating admission on pod-readiness would be -circular. The sidecar's loop is independent of the pod's own readiness -state, so there is no cycle. - -**Remove script** (the sidecar's `preStop` hook, runs once at -termination): - -1. Runs `kafka-metadata-quorum.sh describe --replication` to get the - current voter list. -2. Checks that removing itself would still leave a majority of the - *pre-removal* voter count. This check is done explicitly by the - script — the design does not assume `remove-controller` refuses an - unsafe removal on Kafka's side. -3. If safe, calls `remove-controller` for itself. -4. The whole hook is bounded by a timeout comfortably inside - `terminationGracePeriodSeconds`, and always exits `0` — a stuck or - failed check must never block pod termination indefinitely. - -### Data flow - -**Scale-up:** an ordinary declarative replica increase on the -controller StatefulSet (no change from today) creates a new pod. Its -`kafka` container boots exactly as today (format + start). Its sidecar -independently loops until it observes itself admitted. The existing -readiness probe starts passing once raft state leaves `observer`. If -multiple controllers are added at once, each pod's sidecar self-admits -independently; Kafka's leader serializes the actual `AddVoter` -application, so no operator-side coordination is required. - -**Scale-down:** an ordinary declarative replica decrease (no change -from today). `OrderedReady` means Kubernetes terminates exactly the -highest-ordinal pod, runs its `preStop` hook (self-removal via the -script above), and waits for full termination before considering the -next pod — this is what gives one-at-a-time, majority-checked draining -for a decrease of any size, entirely via a StatefulSet setting. No -Rust-side "gate the effective replica count" logic is needed. - -### Error handling - -- Transient `add-controller` / `describe` failures (e.g. a leader - election in flight) are simply retried by the loop on its normal - interval. There is no alerting path today: the sidecar has no - Kubernetes API access by design (that's the point — no new RBAC), - so failures are visible only via `kubectl logs` on the sidecar - container. -- **Known observability gap:** the sidecar's stdout will *not* be - picked up by the existing vector log-aggregation pipeline, which - only tails structured `*.log4j.xml` / `*.log4j2.xml` files written - by the JVM's own logging config (confirmed by direct inspection of - the deployed `vector.yaml` ConfigMaps during an unrelated - investigation). This is a real, known limitation of this design, not - something papered over — a future iteration could have the sidecar - write structured lines to a file under the shared log directory to - get picked up, but that is not included in this design's initial - scope. -- `preStop` removal timing out or failing (e.g. no reachable leader - within the grace period): the pod still terminates on schedule. A - stale voter entry can be left behind in the quorum in that case. - This is a genuine, stated limitation — recovery in that scenario is - manual (the same "no supported automated path" caveat that already - exists in the current docs for quorum-reconfiguration edge cases). - -### Testing - -- The existing `tests/templates/kuttl/operations-kraft/60-scale-controller-up.yaml.j2` - / `70-scale-controller-down.yaml.j2` kuttl tests (active on `main`, - not disabled) should pass once this is implemented. Strengthen their - `*-assert.yaml.j2` counterparts beyond "StatefulSet reports N/N - ready" to also verify voter count matches replica count post-scale - (e.g. via `describe --replication` run from the test's `python-0` - pod), so the test catches a silently-stuck-in-`observer` regression, - not just a stuck-not-ready one. -- Unit tests in `rust/operator-binary/src/controller/build/resource/statefulset.rs`, - mirroring the existing probe tests added in `ec59dab`: - - the sidecar container is present only on the controller role, and - only for Kafka versions that support dynamic quorum tooling; - - the sidecar's `preStop` command matches the expected removal - script invocation; - - the controller StatefulSet's `podManagementPolicy` is - `OrderedReady`. - -## Open questions for implementation planning - -- Exact minimum Kafka version for the version gate (needs verification - against Kafka's own KIP-853 tooling maturity, not assumed here). -- Exact script implementation (shell, embedded via ConfigMap vs. an - inline `bash -c` command similar to the existing probe commands in - `statefulset.rs`) and its `--command-config` security settings - (matching whatever TLS/SASL configuration the `kafka` container - already uses for its internal listener). -- Whether to close the sidecar-log observability gap noted above as - part of this work or as explicit follow-up. diff --git a/rust/operator-binary/src/controller/build/command.rs b/rust/operator-binary/src/controller/build/command.rs index dd985237..aaafb7d5 100644 --- a/rust/operator-binary/src/controller/build/command.rs +++ b/rust/operator-binary/src/controller/build/command.rs @@ -39,18 +39,8 @@ pub fn kafka_log_opts_env_var() -> String { /// Shell snippet setting `$POD_INDEX` to this pod's ordinal, parsed from the trailing digits /// of `$POD_NAME` (e.g. `2` for `..-controller-default-2`). -/// -/// Paired with [`EXPORT_REPLICA_ID`] (see there for why the split): used, in some combination, -/// by four call sites that used to each duplicate this derivation with slightly drifted -/// whitespace — the broker and controller `kafka` containers' own entrypoints, and the -/// `quorum-manager` sidecar's main loop and `preStop` hook. const DERIVE_POD_INDEX: &str = r#"POD_INDEX=$(echo "$POD_NAME" | grep -oE '[0-9]+$')"#; -/// Shell snippet exporting `$REPLICA_ID` (this container's KRaft node id) from `$POD_INDEX` -/// (see [`DERIVE_POD_INDEX`], which must run first) and `$NODE_ID_OFFSET`. Exported (rather -/// than a plain assignment) because every caller either runs `config-utils template` or the -/// `quorum-manager` sidecar's `kafka-metadata-quorum.sh`/`curl` calls as a *subprocess*, which -/// need `REPLICA_ID` in their environment, not just this shell's. const EXPORT_REPLICA_ID: &str = "export REPLICA_ID=$((POD_INDEX + NODE_ID_OFFSET))"; /// Returns the commands to start the main Kafka container @@ -175,10 +165,7 @@ wait_for_termination() /// /// Every other controller — whether it is part of the cluster's initial desired replica count /// or added later on scale-up — is formatted with `--no-initial-controllers` and relies -/// entirely on the `quorum-manager` sidecar's `add-controller` loop to join the quorum. This is -/// what keeps the controller container's command identical across replica-count changes (no -/// voter list baked into it), and what makes "admit a new controller" solely the sidecar's -/// concern rather than something the format step also has a hand in. +/// entirely on the `quorum-manager` sidecar's `add-controller` loop to join the quorum. /// /// Known limitation: this rule is only safe for a cluster's *original* bootstrap. If the /// designated node's persistent volume is ever lost and needs to reformat after the cluster has @@ -188,11 +175,6 @@ wait_for_termination() /// system, not something this operator (which deliberately has no live-cluster awareness) /// can detect or repair automatically. See `kraft-controller.adoc`. fn controller_quorum_format_flag(controller_descriptors: &[KafkaPodDescriptor]) -> String { - // Empty only when the controller role group itself is scaled to 0 replicas -- which - // `validate` only allows together with brokers also at 0 (a coordinated whole-cluster - // stop, see `NoKraftControllerReplicas` in `controller/validate.rs`). The StatefulSet is - // still built in that case (just with 0 replicas), so this command template is assembled - // but never actually run by any pod; the placeholder node id is never observed. let bootstrap_node_id = controller_descriptors .iter() .filter(|descriptor| descriptor.role == KafkaRole::Controller) diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index 4e5f4e09..3b59312f 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -216,15 +216,6 @@ mod tests { validated_cluster(&kafka) } - /// Confirmed live via the `operations-kraft` kuttl test: scaling both the controller and - /// broker role groups down to 0 replicas together (a coordinated whole-cluster stop, which - /// `validate` allows -- see `NoKraftControllerReplicas` in `controller/validate.rs`) used to - /// still fail to *build*, as `NoKraftControllersFound` while building the (unused) rolegroup - /// ConfigMaps: `pod_descriptors` comes back empty once every role group is at 0 replicas, - /// and `build_rolegroup_config_map` treated that as always broken in KRaft mode, without - /// distinguishing it from the "controllers at 0, brokers still running" case `validate` - /// actually rejects. No pod will ever read these ConfigMaps, so building them with an empty - /// controller quorum is harmless. #[test] fn build_succeeds_when_every_kraft_role_group_is_scaled_to_zero() { let kafka = minimal_kafka( @@ -342,11 +333,6 @@ mod tests { ); } - /// The `quorum-manager` sidecar's admin-client calls need every directory that - /// `controller_admin_client_properties` (see `build/security.rs`) writes paths into: - /// the config volume (for `admin-client.properties` itself) and the internal TLS - /// volume (for the keystore/truststore the properties file points at). Missing either - /// mount makes every `add-controller`/`remove-controller` invocation fail SSL init. #[test] fn quorum_manager_sidecar_mounts_every_directory_referenced_by_admin_client_properties() { let cluster = kraft_mode_cluster(); diff --git a/rust/operator-binary/src/controller/build/properties/mod.rs b/rust/operator-binary/src/controller/build/properties/mod.rs index c253f79b..3ce39b94 100644 --- a/rust/operator-binary/src/controller/build/properties/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/mod.rs @@ -65,12 +65,6 @@ pub fn uses_legacy_log4j(product_version: &str) -> bool { /// pointing at each role group's own headless Service DNS name rather than individual pod /// FQDNs. /// -/// A headless Service's own DNS name (no pod prefix) resolves to every backing pod's IP — -/// exactly what Kafka's own `client.dns.lookup=use_all_dns_ips` default already expects — and -/// the operator's headless Service sets `publishNotReadyAddresses: true`, so this also -/// resolves correctly during initial cluster formation before any pod is Ready. This is what -/// makes the value invariant to an existing controller role group's replica count: adding or -/// removing replicas within a role group never changes that role group's own Service name. /// Only adding or removing a whole role group changes this list. pub(crate) fn kraft_controllers(pod_descriptors: &[KafkaPodDescriptor]) -> Vec { let mut role_group_addresses: Vec = pod_descriptors diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index 45b37d04..c99c02b7 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -89,12 +89,6 @@ pub fn build_rolegroup_config_map( .overrides .clone(); - // In KRaft mode, `pod_descriptors` can only be empty when *every* controller and broker - // role group is scaled to 0 replicas: `validate` rejects any other combination of zero - // controllers with running brokers before this point is ever reached (see - // `NoKraftControllerReplicas`), so a positive broker replica count anywhere guarantees a - // positive controller replica count, and vice versa. A whole-cluster-at-zero ConfigMap is - // harmless to build (no pod will ever read it), so there is nothing to reject here. let pod_descriptors = validated_cluster .pod_descriptors(None) .context(BuildPodDescriptorsSnafu)?; diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 3708eea9..5533ffa5 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -133,15 +133,6 @@ fn common_operator_env_vars( /// **controller** pod: today that's the `kafka` server process and, when present, the /// `quorum-manager` sidecar. /// -/// The sidecar renders the very same `controller.properties` template (see -/// `properties/controller_properties.rs`) that the `kafka` container's own entrypoint does, to -/// build its own `add-controller`/`remove-controller` config — so it needs every -/// `${env:...}` placeholder that template references (`POD_NAME`, `KAFKA_CLIENT_PORT`, -/// `NAMESPACE`, `ROLEGROUP_HEADLESS_SERVICE_NAME`, `CLUSTER_DOMAIN`). Building this set once -/// and handing it to both containers means they can't silently drift apart over time (a real -/// bug found in review: the sidecar was originally given only `POD_NAME`/`NODE_ID_OFFSET`, -/// so its `controller.properties` render most likely produced a broken `listeners` value). -/// /// The caller merges the user's `envOverrides` on top (so a user override wins on a name /// collision) and, for the `kafka` container only, adds container-specific env vars such as /// `PRE_STOP_CONTROLLER_SLEEP_SECONDS`. @@ -922,22 +913,17 @@ fn build_quorum_manager_container( ResourceRequirementsBuilder::new() .with_cpu_request("100m") // A JVM cold start plus an SSL handshake and an admin-client round-trip all - // need to happen inside this sidecar's existing `timeout 15`/`25s preStop` - // budgets (see `CLI_CALL_TIMEOUT_SECONDS` in `command.rs`). + // need to happen inside this sidecar's existing budgets .with_cpu_limit("500m") - // Request must equal limit: the Stackable platform's admission control - // rejects any container whose memory limit-to-request ratio isn't exactly 1 - // (confirmed live: "memory max limit to request ratio per Container is 1, - // but provided ratio is 2.000000"). .with_memory_request("512Mi") .with_memory_limit("512Mi") .build(), ) .add_volume_mount(STACKABLE_CONFIG_DIR_NAME, STACKABLE_CONFIG_DIR) .context(AddVolumeMountSnafu)? - // `controller_admin_client_properties` (see `build/security.rs`) always points - // its keystore/truststore at this directory, so the sidecar's admin-client calls - // need it mounted here too, not just on the `kafka` container. + // `controller_admin_client_properties` always points its keystore/truststore + // at this directory, so the sidecar's admin-client calls need it mounted + // here too, not just on the `kafka` container. .add_volume_mount( STACKABLE_TLS_KAFKA_INTERNAL_VOLUME_NAME, STACKABLE_TLS_KAFKA_INTERNAL_DIR, @@ -945,7 +931,7 @@ fn build_quorum_manager_container( .context(AddVolumeMountSnafu)? // `add-controller` reads this controller's own on-disk `meta.properties` (its // `node.id`/`directory.id`, written by `kafka-storage.sh format`) from `log.dirs` in - // the merged config it connects with — confirmed live: without this mount, every + // the merged config it connects with - without this mount, every // `add-controller` attempt failed with "Unable to read meta.properties from // /stackable/data/kraft", since that path doesn't exist in this container's // filesystem at all without it. This mounts the *same* per-pod PVC the `kafka` diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index 87693836..db3b9f28 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -49,9 +49,6 @@ const PROPERTY_SECURITY_PROTOCOL: &str = "security.protocol"; const PROPERTY_SASL_ENABLED_MECHANISMS: &str = "sasl.enabled.mechanisms"; const PROPERTY_SASL_KERBEROS_SERVICE_NAME: &str = "sasl.kerberos.service.name"; const PROPERTY_SASL_INTER_BROKER_MECHANISM: &str = "sasl.mechanism.inter.broker.protocol"; -// Also mounted on the controller's `quorum-manager` sidecar (see -// `build_quorum_manager_container` in `build/resource/statefulset.rs`), since -// `controller_admin_client_properties` points its keystore/truststore here. pub(crate) const STACKABLE_TLS_KAFKA_INTERNAL_DIR: &str = "/stackable/tls-kafka-internal"; pub(crate) const STACKABLE_TLS_KAFKA_INTERNAL_VOLUME_NAME: &str = "tls-kafka-internal"; const STACKABLE_TLS_KAFKA_SERVER_DIR: &str = "/stackable/tls-kafka-server"; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index be56d81d..033e126f 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -291,22 +291,10 @@ pub fn validate( )?; // A KRaft cluster with zero controller replicas *and running brokers* is a broken - // half-state: the brokers expect a live metadata quorum that no longer exists, and - // every resource that reads the controller quorum's pod descriptors (including the - // broker's own ConfigMap, which renders `controller.quorum.bootstrap.servers` from - // them) would fail to build. Reject that combination here, at validation time, with an - // actionable message, instead of letting it surface downstream as - // `NoKraftControllersFound` while building an unrelated ConfigMap. + // half-state. Reject that combination here. // // Controllers *and* brokers at zero together is not rejected: that is exactly what - // `clusterOperation.stopped` already does today, unconditionally, for every Stackable - // operator (scaling every managed StatefulSet's replicas to 0 at apply time, bypassing - // this check entirely since it only inspects the raw, pre-`stopped` spec) -- so a - // coordinated whole-cluster stop is already a supported shape, not one this check can - // meaningfully forbid. - // - // `replicas: None` (left for a HorizontalPodAutoscaler to own) is never treated as - // zero, for either role -- only an explicit, summed-to-zero replica count is. + // `clusterOperation.stopped` already does today. let controller_replicas: u16 = controller_groups .values() .map(|rg| rg.replicas.unwrap_or(1)) diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 3cb4633f..9b47243a 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -146,12 +146,7 @@ suites: - name: nightly patch: - dimensions: - - name: kafka - expr: last - - name: zookeeper - expr: last - - name: upgrade_old - expr: last + - expr: last - name: smoke-latest select: - smoke From d02b920d66bfa17aeb606e20c63ed1b85782c612 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:03:45 +0200 Subject: [PATCH 08/13] Remove the custom bash trap functions --- CHANGELOG.md | 17 ------- .../src/controller/build/command.rs | 44 +------------------ .../controller/build/resource/statefulset.rs | 5 +-- 3 files changed, 3 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1c469c2..6863ebbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,23 +37,6 @@ All notable changes to this project will be documented in this file. - Fix a longstanding problem of including empty `categories`, `shortNames` and `additionalPrinterColumns` in the CRDs, which could cause problems with GitOps tools (e.g. ArgoCD) reporting a diff in the custom resources. See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#998]). -- The `quorum-manager` sidecar's `preStop` hook no longer retries for the full 25s timeout - when a controller pod is the last remaining voter at termination (e.g. scaling controllers - down to a single replica, or the last surviving pod of a full teardown): removal is - correctly refused in that case, and confirmed live that retrying can never change that - outcome, so the hook now gives up immediately instead of retrying until the deadline ([#NNNN]). -- Scaling a KRaft cluster's controller role group(s) down to a total of 0 replicas while any - broker replicas are configured is now rejected up front, during validation, with an actionable - error message. Previously it passed validation and failed much later and much more - confusingly, as `no Kraft controllers found to build` while building the unrelated *broker* - role group's `ConfigMap`. Controllers and brokers at 0 replicas together is unaffected, since - that is what `clusterOperation.stopped` already does today ([#NNNN]). -- Scaling a KRaft cluster's controller *and* broker role groups down to 0 replicas together (a - coordinated whole-cluster stop, which the check above deliberately still allows) no longer - fails to build resources with `no Kraft controllers found to build`. That check only ever - guarded against a genuinely broken half-state; a whole-cluster-at-zero build is harmless since - no pod ever reads the resulting `ConfigMap`s or `StatefulSet`s, so it is no longer rejected - ([#NNNN]). [#985]: https://github.com/stackabletech/kafka-operator/pull/985 [#990]: https://github.com/stackabletech/kafka-operator/pull/990 diff --git a/rust/operator-binary/src/controller/build/command.rs b/rust/operator-binary/src/controller/build/command.rs index aaafb7d5..32643a5e 100644 --- a/rust/operator-binary/src/controller/build/command.rs +++ b/rust/operator-binary/src/controller/build/command.rs @@ -115,48 +115,6 @@ fn broker_start_command(kraft_mode: bool) -> String { } } -// During a namespace or stacklet delete the Kafka controllers shut down too fast leaving the brokers -// in a bad state. -// Brokers try to connect to controllers before gracefully shutting down but by that time, all -// controllers are already gone. -// The broker pods are then kept alive until the value of `gracefulShutdownTimeout` is reached. -// The environment variable `PRE_STOP_CONTROLLER_SLEEP_SECONDS` delays the termination of the -// controller processes to give the brokers more time to offload data and shutdown gracefully. -// Kubernetes has a built in `pre-stop` hook feature that is not yet generally available on all platforms -// supported by the operator. -const BASH_TRAP_FUNCTIONS: &str = r#" -prepare_signal_handlers() -{ - unset term_child_pid - unset term_kill_needed - trap 'handle_term_signal' TERM -} - -handle_term_signal() -{ - if [ "${term_child_pid}" ]; then - [ -n "$PRE_STOP_CONTROLLER_SLEEP_SECONDS" ] && sleep "$PRE_STOP_CONTROLLER_SLEEP_SECONDS" - kill -TERM "${term_child_pid}" 2>/dev/null - else - term_kill_needed="yes" - fi -} - -wait_for_termination() -{ - set +e - term_child_pid=$1 - if [[ -v term_kill_needed ]]; then - [ -n "$PRE_STOP_CONTROLLER_SLEEP_SECONDS" ] && sleep "$PRE_STOP_CONTROLLER_SLEEP_SECONDS" - kill -TERM "${term_child_pid}" 2>/dev/null - fi - wait ${term_child_pid} 2>/dev/null - trap - TERM - wait ${term_child_pid} 2>/dev/null - set -e -} -"#; - /// Chooses exactly one controller (the one with the numerically lowest KRaft `node_id` among /// all controller pod descriptors, a value that is stable across scale-up/down of an existing /// controller role group, since new replicas only ever get higher node ids) to bootstrap the @@ -196,7 +154,7 @@ pub fn controller_kafka_container_command( controller_descriptors: Vec, ) -> String { formatdoc! {" - {BASH_TRAP_FUNCTIONS} + {COMMON_BASH_TRAP_FUNCTIONS} {remove_vector_shutdown_file_command} prepare_signal_handlers containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 5533ffa5..16f2f4b9 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -134,8 +134,8 @@ fn common_operator_env_vars( /// `quorum-manager` sidecar. /// /// The caller merges the user's `envOverrides` on top (so a user override wins on a name -/// collision) and, for the `kafka` container only, adds container-specific env vars such as -/// `PRE_STOP_CONTROLLER_SLEEP_SECONDS`. +/// collision); the `quorum-manager` sidecar additionally gets its own container-specific env +/// vars layered on top (see [`KAFKA_NODE_ID_OFFSET`]). fn controller_pod_shared_env_vars( validated_cluster: &ValidatedCluster, kafka_security: &ValidatedKafkaSecurity, @@ -509,7 +509,6 @@ pub fn build_controller_rolegroup_statefulset( let env: Vec = controller_shared_env .clone() - .with_value(&env_var_name("PRE_STOP_CONTROLLER_SLEEP_SECONDS"), "10") .merge(validated_rg.env_overrides.clone()) .into(); From 62adbff7066b13e037bb9cfd3023b4cd7671d736 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:03:52 +0200 Subject: [PATCH 09/13] Comments abd documentation cleanups --- CHANGELOG.md | 17 +- .../pages/usage-guide/kraft-controller.adoc | 32 +- .../src/controller/build/command.rs | 358 +++++++++++------- .../controller/build/resource/statefulset.rs | 146 +++++-- 4 files changed, 364 insertions(+), 189 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6863ebbf..7ea8915f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,22 @@ All notable changes to this project will be documented in this file. ### Added -- A new `quorum-manager` sidecar container on each controller pod is solely responsible for - admitting itself into the KRaft voter set on startup and removing itself before - termination. Exactly one controller bootstraps the quorum standalone at format time +- A new `quorum-manager` sidecar container on each controller pod admits itself into the KRaft + voter set on startup. Exactly one controller bootstraps the quorum standalone at format time (`kafka-storage.sh format --standalone`); every other controller, whether present from the start or added later, formats with `--no-initial-controllers` and joins purely through the sidecar ([#1010]). -- A new `readinessProbe` for KRaft controllers that fails when new pods cannot join the quorum ([#1010]). +- A new `readinessProbe` on the controller's `kafka` container that fails when new pods cannot + join the quorum ([#1010]). +- A new `livenessProbe` on the controller's `kafka` container that fails when the state has + been stuck `unattached` for an extended period ([#1010]). +- A new `preStop` hook on the `kafka` container that removes the controller from the voters list. + This is the oposite step to what the `quorum-manager` does ([#1010]). +- The `quorum-manager` sidecar now kills an in-flight `add-controller` attempt as soon as its + pod starts terminating, instead of letting it run to completion. Without this, a call already + in flight could succeed after the `kafka` container's `preStop` had already checked the voter + list and found nothing to remove, re-adding a pod that was simultaneously being removed and + leaving it stuck in the on-disk voter list ([#1010]). ### Changed diff --git a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc index ebf6ffd0..27ae3494 100644 --- a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc +++ b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc @@ -85,13 +85,19 @@ KRaft mode requires major configuration changes compared to ZooKeeper: * `cluster-id`: This is set to the `metadata.name` of the KafkaCluster resource during initial formatting * `node.id`: This is a calculated integer, hashed from the `role` and `rolegroup` and added `replica` id. * `process.roles`: Will always only be `broker` or `controller`. Mixed `broker,controller` servers are not supported. -* Each controller pod runs an additional `quorum-manager` sidecar container. On startup it admits the pod into - the KRaft voter set (`kafka-metadata-quorum.sh add-controller`), and on pod termination (`preStop`) it removes - the pod from the voter set again (`remove-controller`). -* Controller pods have a `startupProbe` and a plain TCP `livenessProbe` on the KRaft listener port, and a - `readinessProbe` that checks the controller's Raft state (`leader`, `follower`, or `voted`) via its metrics - endpoint rather than a bare TCP check, so a controller that cannot join or rejoin the quorum is correctly - reported as not ready instead of appearing healthy. +* Each controller pod runs an additional `quorum-manager` sidecar container that, on startup, admits the pod into + the KRaft voter set (`kafka-metadata-quorum.sh add-controller`). + Removing the pod from the voter set again (`remove-controller`) on termination runs as the `kafka` container's + *own* `preStop` hook instead, not the sidecar's: `preStop` only delays that same container's `SIGTERM`, and it's + the `kafka` container's own Raft process — the thing actually leaving the voter set — that needs to stay alive + while removal is attempted, which matters most when the departing pod is the current leader. +* Controller pods have a `startupProbe` (a plain TCP check on the KRaft listener port) and a `livenessProbe` that + combines that same TCP check with a check that the controller's local Raft state hasn't been stuck `unattached` + for an extended period — a symptom of a dynamically-joining controller resolving + `controller.quorum.bootstrap.servers` to its own pod address and getting wedged fetching from itself; a restart + forces a fresh DNS resolution attempt. A `readinessProbe` separately checks that the Raft state is one of + `leader`, `follower`, or `voted` via the controller's metrics endpoint, so a controller that cannot join or + rejoin the quorum is correctly reported as not ready instead of appearing healthy. * Admitting a controller into the KRaft voter set is *solely* the concern of the `quorum-manager` sidecar container — the format step never asserts a voter list. Exactly one controller (the one with the numerically lowest `node.id` among all controller pod descriptors) formats with `kafka-storage.sh format --standalone`, @@ -100,9 +106,10 @@ KRaft mode requires major configuration changes compared to ZooKeeper: through the sidecar's `add-controller` call. Brokers always format with `--no-initial-controllers` too; they are never voters. Because no voter list is baked into any container's command, the command is identical regardless of the current replica count. -* `controller.quorum.bootstrap.servers` (used by the `kafka` process itself to find the controller quorum, and by - the `quorum-manager` sidecar for its own `add-controller`/`remove-controller` calls) points at each controller - role group's own headless Service DNS name, not individual pod addresses. +* `controller.quorum.bootstrap.servers` (used by the `kafka` process itself to find the controller quorum, by the + `quorum-manager` sidecar for its own `add-controller` calls, and by the `kafka` container's own `preStop` hook + for its `remove-controller` call) points at each controller role group's own headless Service DNS name, not + individual pod addresses. == Known Issues @@ -131,8 +138,9 @@ The Stackable Kafka operator currently does not support the migration. === Scaling controllers -Controller replicas can be scaled up and down on a running cluster. A per-pod `quorum-manager` sidecar admits and -removes the pod from the KRaft voter set as described under "Internal operator details" above. +Controller replicas can be scaled up and down on a running cluster. A per-pod `quorum-manager` sidecar admits the +pod into the KRaft voter set on startup, and the `kafka` container's own `preStop` hook removes it again on +termination, as described under "Internal operator details" above. Scaling more than one controller down at a time is processed one pod at a time (`OrderedReady` pod management), not in parallel, so that each pod's removal from the voter set can complete before the next one is terminated. diff --git a/rust/operator-binary/src/controller/build/command.rs b/rust/operator-binary/src/controller/build/command.rs index 32643a5e..b0b58663 100644 --- a/rust/operator-binary/src/controller/build/command.rs +++ b/rust/operator-binary/src/controller/build/command.rs @@ -3,6 +3,7 @@ use stackable_operator::{ product_logging::framework::{ create_vector_shutdown_file_command, remove_vector_shutdown_file_command, }, + shared::time::Duration, utils::COMMON_BASH_TRAP_FUNCTIONS, v2::product_logging::framework::STACKABLE_LOG_DIR, }; @@ -128,10 +129,7 @@ fn broker_start_command(kraft_mode: bool) -> String { /// Known limitation: this rule is only safe for a cluster's *original* bootstrap. If the /// designated node's persistent volume is ever lost and needs to reformat after the cluster has /// already formed a quorum elsewhere, reformatting it with `--standalone` would bootstrap a -/// second, conflicting one-node quorum instead of rejoining the existing one — the same class -/// of manual-recovery scenario as losing enough voters to break quorum in any Raft-based -/// system, not something this operator (which deliberately has no live-cluster awareness) -/// can detect or repair automatically. See `kraft-controller.adoc`. +/// second, conflicting one-node quorum instead of rejoining the existing one. fn controller_quorum_format_flag(controller_descriptors: &[KafkaPodDescriptor]) -> String { let bootstrap_node_id = controller_descriptors .iter() @@ -183,11 +181,6 @@ pub fn controller_kafka_container_command( } } -/// The `kafka-metadata-quorum.sh` binary, referenced by its absolute path (matching every -/// other exec-into-pod usage of a Kafka CLI tool in this repo, e.g. the kuttl test scripts -/// under `tests/templates/kuttl/*/*.sh`), rather than the relative `bin/...` form used only -/// inside the `kafka` container's own entrypoint (which runs with the Kafka install dir as -/// its working directory). const KAFKA_METADATA_QUORUM_BINARY: &str = "/stackable/kafka/bin/kafka-metadata-quorum.sh"; const ADMIN_CLIENT_PROPERTIES_PATH: &str = "/stackable/config/admin-client.properties"; @@ -196,64 +189,29 @@ const ADMIN_CLIENT_PROPERTIES_PATH: &str = "/stackable/config/admin-client.prope /// /// `add-controller` is not a plain admin-client call: the same process that connects to the /// quorum also reads `node.id` and its own `listeners`/`controller.listener.names` from the -/// **same** `--command-config` file to build the voter registration payload (confirmed live: -/// pointed at the plain [`ADMIN_CLIENT_PROPERTIES_PATH`], every attempt failed with `node.id -/// not found in configuration file`, so no controller was ever able to admit itself as a -/// voter). But that rendered `controller.properties` has no bare `security.protocol`/`ssl.*` -/// keys of its own — only the `listener.name..ssl.*`-prefixed ones the server process -/// uses for its listeners — so using it *instead of* the admin-client config leaves the -/// AdminClient with no TLS config and unable to reach the (TLS-only) bootstrap controller. -/// Concatenating both files (also confirmed live) gives `add-controller` everything it reads: -/// the bare `ssl.*`/`security.protocol` keys for its own connection, plus `node.id` and the -/// listener keys for the registration payload. +/// **same** `--command-config` file to build the voter registration payload. /// /// **Order matters.** There is no key overlap between the two files today, but -/// `controller.properties` accepts unconditional `configOverrides` merged into it (see -/// `controller_properties::build`), so a user override there could add a colliding key. Java -/// properties parsing lets a later occurrence of the same key win, so `controller.properties` -/// is concatenated *first* and [`ADMIN_CLIENT_PROPERTIES_PATH`] *last* — that way the client -/// TLS config `add-controller` connects with always wins by construction, rather than -/// depending on there being no collision today. +/// `controller.properties` accepts unconditional `configOverrides` merged into it, +/// so a user override there could add a colliding key. const ADD_CONTROLLER_PROPERTIES_PATH: &str = "/tmp/add-controller.properties"; /// Wall-clock bound (seconds) applied to every individual `kafka-metadata-quorum.sh` -/// invocation via `timeout`. The Java AdminClient can otherwise retry internally for far -/// longer than any of this file's own script-level deadlines, which matters most in -/// `quorum_manager_pre_stop_command`: it runs exactly when peers may be unreachable, and a -/// hung admin-client call there would burn into `terminationGracePeriodSeconds` (default: -/// 30 minutes) rather than the script's own 25s budget. +/// invocation via `timeout`. const CLI_CALL_TIMEOUT_SECONDS: u32 = 15; /// Grace period (seconds) after [`CLI_CALL_TIMEOUT_SECONDS`] elapses before `timeout` sends /// `SIGKILL`, via `--kill-after`. -/// -/// `timeout N cmd` (GNU coreutils) without `--kill-after` only *sends* `SIGTERM` once `N` -/// seconds pass — it does not force-kill `cmd`, so if `cmd` doesn't honor the signal -/// promptly, the whole call can run far longer than `N` seconds. Confirmed directly, -/// independent of Kafka: `timeout 3 bash -c 'trap "" TERM; sleep 30'` takes the full 30s, not -/// 3s. Confirmed live, with Kafka: during a full namespace deletion (every controller -/// terminating concurrently, so a peer's `describe`/`add-controller`/`remove-controller` call -/// can hit a blackholed rather than actively-refused connection), a controller's sidecar kept -/// running well past its own `preStop` script's ~25-40s design budget — the `timeout` wrapper -/// around its `kafka-metadata-quorum.sh` calls was not actually bounding them. const CLI_CALL_KILL_AFTER_SECONDS: u32 = 5; /// Shell snippet setting `$BOOTSTRAP_SERVERS` by extracting /// `controller.quorum.bootstrap.servers` from the static, un-rendered `controller.properties` -/// ConfigMap file, un-escaping the `\:` that `to_java_properties_string` applies to colons. -/// This value has no `${env:...}` placeholders — every `host:port` pair is already fully -/// resolved at build time from pod descriptors (see `kraft_controllers` in -/// `build/properties/mod.rs`) — so it can be read directly without running `config-utils -/// template` first. +/// ConfigMap file. /// /// Reading this at runtime, rather than baking the peer list into this script as a Rust /// literal, keeps both sidecar scripts' content — and therefore the controller pod /// template — identical across changes to an existing controller role group's *replica -/// count*. Confirmed live: without this, scaling controllers up/down rolled every -/// already-existing controller pod, not just the ones actually being added/removed — the -/// same class of problem `--initial-controllers` caused before it was removed from the -/// `kafka` container's own format step (see `controller_quorum_format_flag`), just via this -/// sidecar's command instead. +/// count*. fn extract_bootstrap_servers_command() -> String { format!( r#"BOOTSTRAP_SERVERS=$(grep '^controller.quorum.bootstrap.servers=' {config_dir}/{controller_properties_file} | cut -d= -f2- | sed 's/\\:/:/g')"#, @@ -264,43 +222,19 @@ fn extract_bootstrap_servers_command() -> String { /// The sidecar's main-loop command: while this controller's local Raft state is /// `observer`, repeatedly attempt to admit it into the quorum's voter set. -/// -/// Explicitly traps `TERM` and exits: this script runs as the container's PID 1, and the -/// kernel suppresses the default action of unhandled signals for PID 1, so without this -/// trap the loop below would never notice `SIGTERM` and would run until Kubernetes gives up -/// waiting and sends `SIGKILL` after the full `terminationGracePeriodSeconds` (confirmed -/// live: with no trap, this container kept looping — and its pod kept reporting as -/// `Terminating` — long after the `kafka` container in the same pod had shut down -/// gracefully). The `sleep 10 &`/`wait $!` pair (rather than a plain `sleep 10`) lets the -/// trap fire immediately: bash's `wait` builtin is interrupted as soon as a trapped signal -/// arrives, whereas a foreground `sleep` would only be noticed once it finished. -/// -/// Renders [`ADD_CONTROLLER_PROPERTIES_PATH`] once at startup (this controller's identity -/// and listener address don't change for the container's lifetime) by reusing the same -/// `$POD_NAME`/`NODE_ID_OFFSET` → `REPLICA_ID` derivation ([`DERIVE_POD_INDEX`]/ -/// [`EXPORT_REPLICA_ID`]), and the same `config-utils template` render step, as the `kafka` -/// container's own entrypoint (see [`controller_kafka_container_command`]) — see -/// [`ADD_CONTROLLER_PROPERTIES_PATH`] for why `add-controller` needs this merged file rather -/// than the plain admin-client config, and for why the concatenation order matters. -/// -/// The render/merge preamble's inputs are static, operator-rendered config (env vars set -/// once at pod creation), so a failure there is a genuine misconfiguration that retrying -/// won't fix. It must still be loud in the logs, but it must *not* crash the container: a -/// container with no readiness probe is only `Ready` while `Running`, and (with -/// `OrderedReady` pod management on every non-Kerberos controller `StatefulSet`) a -/// crash-looping sidecar would make its whole pod `NotReady` and block scale/update -/// progress for every sibling pod in the role, not just the broken one. So on failure this -/// falls into a "degraded" loop that repeats a clear error every 30s and never attempts -/// `add-controller` (there is no valid rendered config to use), keeping the container alive -/// and `Running` while the problem stays visible via `kubectl logs`. This deliberately does -/// *not* retry the render/merge step itself — that would look like it might eventually -/// succeed, when the actual cause is a misconfiguration that only a human or a new rollout -/// can fix. pub fn quorum_manager_container_command() -> String { format!( r#" set -uo pipefail - trap 'exit 0' TERM + ADD_CONTROLLER_PID="" + trap 'handle_term_signal' TERM + + handle_term_signal() + {{ + [ -n "$ADD_CONTROLLER_PID" ] && kill -TERM "$ADD_CONTROLLER_PID" 2>/dev/null + exit 0 + }} + {derive_pod_index} [ -n "$POD_INDEX" ] || exit 0 {export_replica_id} @@ -314,8 +248,11 @@ pub fn quorum_manager_container_command() -> String { state=$(curl -s --max-time 5 --connect-timeout 2 localhost:{metrics_port}/metrics | grep -oE 'kafka_server_raft_metrics_current_state\{{state="[a-z]+"\}}' | grep -oE '"[a-z]+"' | tr -d '"') if [ "$state" = "observer" ]; then echo "Local Raft state is observer, attempting add-controller..." - timeout --kill-after={cli_kill_after} {cli_timeout} {binary} --bootstrap-controller "$BOOTSTRAP_SERVERS" --command-config {add_controller_config} add-controller \ + timeout --kill-after={cli_kill_after} {cli_timeout} {binary} --bootstrap-controller "$BOOTSTRAP_SERVERS" --command-config {add_controller_config} add-controller & + ADD_CONTROLLER_PID=$! + wait "$ADD_CONTROLLER_PID" \ || echo "add-controller attempt failed (this is expected if it already succeeded or a leader election is in progress), will retry" + ADD_CONTROLLER_PID="" elif [ -z "$state" ]; then echo "Could not determine local Raft state (metrics scrape returned nothing), will retry" else @@ -347,39 +284,54 @@ pub fn quorum_manager_container_command() -> String { ) } -/// The sidecar's `preStop` command: before this controller pod terminates, check that -/// removing it would not remove the *last* remaining voter from the quorum, and if so, +/// Reserved (seconds), out of the pod's total `terminationGracePeriodSeconds`, for the `kafka` +/// process's *own* `SIGTERM`-triggered shutdown after this preStop hook finishes or gives up. +const PRE_STOP_RESERVED_FOR_KAFKA_SHUTDOWN_SECONDS: u64 = 30; + +/// Floor for [`pre_stop_deadline_seconds`]: never worse than the original fixed budget, even +/// for a user-configured `gracefulShutdownTimeout` too short to leave +/// [`PRE_STOP_RESERVED_FOR_KAFKA_SHUTDOWN_SECONDS`] of headroom. +const PRE_STOP_MIN_DEADLINE_SECONDS: u64 = 25; + +/// Cap for [`pre_stop_deadline_seconds`]: even against a generous `gracefulShutdownTimeout` +/// (the operator's own default is 30 minutes), a single pod's voter removal shouldn't +/// plausibly hang for tens of minutes during a routine scale-down. +const PRE_STOP_MAX_DEADLINE_SECONDS: u64 = 120; + +/// The wall-clock budget (seconds) [`controller_remove_self_pre_stop_command`] retries voter removal +/// for, derived from the pod's actual `gracefulShutdownTimeout` rather than a single fixed +/// constant. +fn pre_stop_deadline_seconds(graceful_shutdown_timeout: Option) -> u64 { + graceful_shutdown_timeout + .map(|timeout| { + timeout + .as_secs() + .saturating_sub(PRE_STOP_RESERVED_FOR_KAFKA_SHUTDOWN_SECONDS) + .clamp(PRE_STOP_MIN_DEADLINE_SECONDS, PRE_STOP_MAX_DEADLINE_SECONDS) + }) + .unwrap_or(PRE_STOP_MIN_DEADLINE_SECONDS) +} + +/// The `kafka` container's own `preStop` command: before this controller pod terminates, check +/// that removing it would not remove the *last* remaining voter from the quorum, and if so, /// remove it from the voter set. Always exits 0 — a stuck or failed check must never block /// pod termination. /// -/// Removing a departing voter only ever *lowers* the majority threshold for the remaining -/// set, and the `remove-controller` RPC itself needs the *current* quorum to already commit -/// it — if peers are unreachable the call simply fails, it can't corrupt anything. So the -/// only real invariant worth enforcing here is "never remove the last voter": a 1-voter -/// quorum can't be reduced further without permanently losing all fault tolerance (there -/// would be no other voter left to ever add a replacement to). -/// -/// This controller's own KRaft node id is derived at runtime from `$POD_NAME` and -/// `$NODE_ID_OFFSET` ([`DERIVE_POD_INDEX`]/[`EXPORT_REPLICA_ID`]), exactly as the `kafka` -/// container's own entrypoint does — see `controller_kafka_container_command`. +/// The "would leave zero voters" case is the one exception that does *not* retry: once a +/// `describe` shows this pod is the last remaining voter, stop. /// -/// `describe --replication`'s column layout (`NodeId` as column 1, `DirectoryId` as column -/// 2, `Status` as the last column, with `Status` one of `Leader`/`Follower`/`Observer`) is -/// the *documented* KIP-853 tabular format, but has not been confirmed against a live -/// cluster (see Task 4's brief, Step 5 — deferred to Task 7's kuttl run, which has one). -/// Filtering is deliberately conservative: only rows whose `Status` is a recognized voter -/// value (`Leader`/`Follower`) count towards `total_voters`, and if that filter yields zero -/// voters (e.g. because the real column layout differs from what's assumed here), the -/// check simply retries rather than treating "no known voters" as "safe to remove" — i.e. -/// this fails closed (skips removal) rather than open on a parsing mismatch. +/// IMPORTANT: the last voter must never be removed from the quorum because that would break +/// cluster restarts. In that situation a restart would reformat the Raft metadata effectively +/// losing all information from the previous iteration. /// -/// The "would leave zero voters" case is the one exception that does *not* retry: once a -/// `describe` shows this pod is the last remaining voter, retrying for the rest of the -/// `DEADLINE` can't make it safe to remove — nothing else is going to add a voter for it -/// while it terminates. Confirmed live: before this early `break`, a controller pod that -/// became the last voter (e.g. scaling controllers down to 1, or the last surviving pod -/// during a full teardown) always burned the entire 25s `DEADLINE` here for no benefit. -pub fn quorum_manager_pre_stop_command() -> String { +/// If every retry within `DEADLINE` fails, the loop falls through with the voter never +/// actually removed; the final `echo "ERROR: ..."` makes that failure loud (grep/alert-able in +/// container logs) rather than a plain, easy-to-miss log line, since a stale voter entry left +/// behind here is exactly the kind of thing that can strand a later restart-from-zero (see +/// `controller_stuck_unattached_liveness_probe`'s doc comment in `resource/statefulset.rs`). +pub fn controller_remove_self_pre_stop_command( + graceful_shutdown_timeout: Option, +) -> String { format!( r#" set -uo pipefail @@ -387,7 +339,8 @@ pub fn quorum_manager_pre_stop_command() -> String { [ -n "$POD_INDEX" ] || exit 0 {export_replica_id} {extract_bootstrap_servers} - DEADLINE=$((SECONDS + 25)) + DEADLINE=$((SECONDS + {deadline_seconds})) + finished=false while [ "$SECONDS" -lt "$DEADLINE" ]; do describe=$(timeout --kill-after={cli_kill_after} {cli_timeout} {binary} --bootstrap-controller "$BOOTSTRAP_SERVERS" --command-config {config} describe --replication 2>/dev/null) if [ -n "$describe" ]; then @@ -399,25 +352,33 @@ pub fn quorum_manager_pre_stop_command() -> String { directory_id=$(echo "$voters" | awk -v id="$REPLICA_ID" '$1 == id {{ print $2 }}') if [ -n "$directory_id" ]; then echo "Removing self (node $REPLICA_ID, directory $directory_id) from the voter set..." - timeout --kill-after={cli_kill_after} {cli_timeout} {binary} --bootstrap-controller "$BOOTSTRAP_SERVERS" --command-config {config} remove-controller \ - --controller-id "$REPLICA_ID" --controller-directory-id "$directory_id" \ - || echo "remove-controller failed, proceeding with termination anyway" + if timeout --kill-after={cli_kill_after} {cli_timeout} {binary} --bootstrap-controller "$BOOTSTRAP_SERVERS" --command-config {config} remove-controller \ + --controller-id "$REPLICA_ID" --controller-directory-id "$directory_id"; then + finished=true + else + echo "remove-controller attempt failed, will retry if time remains" + fi else echo "Could not find own node $REPLICA_ID among current voters (already removed?), nothing to do" + finished=true fi - break else echo "Removing self would leave zero voters, skipping (this can't become safe later during my own termination -- nothing else will add a voter for me)" - break + finished=true fi + [ "$finished" = true ] && break else echo "Could not identify any voters in the describe output (unrecognized format), skipping removal for safety and retrying..." fi fi sleep 2 done + if [ "$finished" != true ]; then + echo "ERROR: could not remove self (node $REPLICA_ID) from the voter set before terminating (every attempt within ${{DEADLINE}}s failed or the quorum was unreachable throughout); the on-disk voter set may now list this pod even though it is gone -- if nothing else corrects this, a later restart may get stuck and require manual recovery, see kraft-controller.adoc" + fi exit 0 "#, + deadline_seconds = pre_stop_deadline_seconds(graceful_shutdown_timeout), binary = KAFKA_METADATA_QUORUM_BINARY, config = ADMIN_CLIENT_PROPERTIES_PATH, cli_timeout = CLI_CALL_TIMEOUT_SECONDS, @@ -458,11 +419,36 @@ mod tests { #[test] fn quorum_manager_container_command_traps_term_and_sleeps_interruptibly() { let command = quorum_manager_container_command(); - assert!(command.contains("trap 'exit 0' TERM")); + assert!(command.contains("trap 'handle_term_signal' TERM")); assert!(command.contains("sleep 10 &")); assert!(command.contains("wait $!")); } + /// The whole point of backgrounding `add-controller`: a plain foreground `timeout ...` + /// call is not interrupted by an arriving `TERM` — bash only checks/runs traps between + /// commands or during the interruptible `wait` builtin — so a call already in flight when + /// the pod starts terminating could otherwise run to completion and race the `kafka` + /// container's own `preStop` removal, re-adding a pod that is simultaneously being + /// removed. Backgrounding it and having the trap actively `kill` it closes that window. + #[test] + fn quorum_manager_container_command_kills_an_in_flight_add_controller_attempt_on_term() { + let command = quorum_manager_container_command(); + assert!(command.contains("ADD_CONTROLLER_PID=$!")); + assert!(command.contains(r#"wait "$ADD_CONTROLLER_PID""#)); + assert!(command.contains(r#"kill -TERM "$ADD_CONTROLLER_PID""#)); + // The `add-controller` invocation itself must actually be backgrounded (not a plain + // foreground call) for the above to have any effect. + let add_controller_line = command + .lines() + .find(|line| line.contains("timeout") && line.contains("add-controller")) + .expect("the add-controller invocation is present"); + assert!( + add_controller_line.trim_end().ends_with('&'), + "add-controller must be backgrounded so TERM can interrupt `wait` immediately, \ + line was: {add_controller_line}" + ); + } + /// Checks only that the generated command *string* concatenates the two config files in /// the order that makes `add-controller` self-register successfully — it does not /// execute the script, so it cannot verify runtime behavior. That was confirmed @@ -493,12 +479,102 @@ mod tests { } #[test] - fn quorum_manager_pre_stop_command_always_exits_zero() { - let command = quorum_manager_pre_stop_command(); + fn controller_remove_self_pre_stop_command_always_exits_zero() { + let command = controller_remove_self_pre_stop_command(None); assert!(command.trim_end().ends_with("exit 0")); assert!(command.contains("remove-controller")); } + /// A failed `remove-controller` attempt must not `break` out of the retry loop — unlike + /// the "already removed" and "would leave zero voters" cases, a failure is exactly the + /// situation the retry loop exists for. Only success (`finished=true` on that path) may + /// exit early. + #[test] + fn controller_remove_self_pre_stop_command_retries_a_failed_remove_controller_attempt() { + let command = controller_remove_self_pre_stop_command(None); + + // The loop's exit check is conditional on success (`finished=true`), not an + // unconditional `break` - so a failed attempt, which never reaches `finished=true`, + // falls through to the loop's retry instead of exiting immediately. + assert!(command.contains(r#"[ "$finished" = true ] && break"#)); + + // The failed-attempt branch itself must not set `finished=true` or `break` on its + // own - only the sibling success (`then`) branch does; this branch's only content is + // the log message. + let failure_message = "remove-controller attempt failed, will retry if time remains"; + let failure_message_pos = command + .find(failure_message) + .expect("the failure message is present in the generated script"); + let up_to_failure_message = &command[..failure_message_pos + failure_message.len()]; + let else_branch_start = up_to_failure_message + .rfind("else") + .expect("the failure message is inside an `else` branch"); + let else_branch = &up_to_failure_message[else_branch_start..]; + assert!(!else_branch.contains("finished=true")); + assert!(!else_branch.contains("break")); + } + + /// If every retry within `DEADLINE` fails, the script must say so loudly (an `ERROR:` + /// prefixed line, consistent with the `quorum-manager` main loop's own degraded-mode + /// messages) rather than silently letting the pod terminate with the voter never removed — + /// see `pre_stop_deadline_seconds`'s and this function's doc comments for why a silent + /// failure here is the specific gap that can strand a later restart-from-zero. + #[test] + fn controller_remove_self_pre_stop_command_logs_loudly_when_every_attempt_fails() { + let command = controller_remove_self_pre_stop_command(None); + assert!(command.contains(r#"if [ "$finished" != true ]; then"#)); + let error_branch = command + .split(r#"if [ "$finished" != true ]; then"#) + .nth(1) + .expect("the failure branch follows the retry loop"); + assert!(error_branch.contains("echo \"ERROR:")); + } + + /// The retry `DEADLINE` must be derived from the pod's actual `gracefulShutdownTimeout` + /// (via [`pre_stop_deadline_seconds`]) rather than hardcoded, and must stay within the + /// documented floor/cap regardless of how short or long that timeout is. + #[test] + fn pre_stop_deadline_seconds_is_derived_from_graceful_shutdown_timeout_within_floor_and_cap() { + // No configured timeout (shouldn't happen in practice) falls back to the floor. + assert_eq!( + pre_stop_deadline_seconds(None), + PRE_STOP_MIN_DEADLINE_SECONDS + ); + + // A short timeout (shorter than the reserved buffer) still gets at least the floor, + // never less than the original fixed behavior. + assert_eq!( + pre_stop_deadline_seconds(Some(Duration::from_secs(10))), + PRE_STOP_MIN_DEADLINE_SECONDS + ); + + // A generous timeout (the operator's own 30-minute default) is capped, not handed the + // entire budget minus the reserve. + assert_eq!( + pre_stop_deadline_seconds(Some(Duration::from_minutes_unchecked(30))), + PRE_STOP_MAX_DEADLINE_SECONDS + ); + + // A timeout comfortably between the floor and the cap (once the reserve is subtracted) + // is used as-is. + assert_eq!( + pre_stop_deadline_seconds(Some(Duration::from_secs(90))), + 90 - PRE_STOP_RESERVED_FOR_KAFKA_SHUTDOWN_SECONDS + ); + } + + /// The generated script's own `DEADLINE` must actually use + /// [`pre_stop_deadline_seconds`]'s output, not a literal left over from before it existed. + #[test] + fn controller_remove_self_pre_stop_command_deadline_reflects_the_configured_timeout() { + let command = + controller_remove_self_pre_stop_command(Some(Duration::from_minutes_unchecked(30))); + assert!(command.contains(&format!( + "DEADLINE=$((SECONDS + {}))", + PRE_STOP_MAX_DEADLINE_SECONDS + ))); + } + /// The old majority-based guard (`majority=$(( total_voters / 2 + 1 ))`, /// `remaining_after_removal -ge majority`) always blocked the last safe removal of a /// 2-voter quorum (2 -> 1): `majority` was 2, `remaining_after_removal` was 1, and @@ -507,8 +583,8 @@ mod tests { /// The only invariant that actually matters is "never remove the last voter", so this /// asserts the generated script uses that condition instead. #[test] - fn quorum_manager_pre_stop_command_allows_removing_the_second_to_last_voter() { - let command = quorum_manager_pre_stop_command(); + fn controller_remove_self_pre_stop_command_allows_removing_the_second_to_last_voter() { + let command = controller_remove_self_pre_stop_command(None); assert!( command.contains(r#"remaining_after_removal" -ge 1 ]"#), "expected the guard to allow removal whenever at least one voter remains \ @@ -559,20 +635,27 @@ mod tests { /// pod while it's terminating, so the outcome can never change. The branch must `break` /// immediately instead of falling through to the loop's `sleep 2`. #[test] - fn quorum_manager_pre_stop_command_gives_up_immediately_on_the_last_voter() { - let command = quorum_manager_pre_stop_command(); - let zero_voters_branch = command + fn controller_remove_self_pre_stop_command_gives_up_immediately_on_the_last_voter() { + let command = controller_remove_self_pre_stop_command(None); + let after_zero_voters_message = command .split("Removing self would leave zero voters") .nth(1) .expect("the zero-voters message is present in the generated script"); - let next_fi = zero_voters_branch - .find("fi") - .expect("an `fi` closes this branch"); + let next_sleep = after_zero_voters_message + .find("sleep 2") + .expect("the loop's retry `sleep 2` follows somewhere after this branch"); + let until_next_retry = &after_zero_voters_message[..next_sleep]; + + assert!( + until_next_retry.contains("finished=true"), + "the zero-voters branch must mark the loop finished (no voter needs removing), \ + text was: {until_next_retry}" + ); assert!( - zero_voters_branch[..next_fi].contains("break"), + until_next_retry.contains("break"), "the zero-voters branch must break out of the retry loop immediately instead of \ - retrying until DEADLINE, branch was: {}", - &zero_voters_branch[..next_fi] + falling through to the loop's `sleep 2` and retrying until DEADLINE, text was: \ + {until_next_retry}" ); } @@ -632,16 +715,17 @@ mod tests { /// promptly, the whole call can run far longer than `N` seconds. Confirmed directly, /// independent of Kafka: `timeout 3 bash -c 'trap "" TERM; sleep 30'` takes the full 30s, /// not 3s, while `timeout --kill-after=2 3 bash -c 'trap "" TERM; sleep 30'` is correctly - /// bounded to ~5s. This matters most for `quorum_manager_pre_stop_command`, which runs + /// bounded to ~5s. This matters most for `controller_remove_self_pre_stop_command`, which runs /// exactly when peers may be mid-termination (a blackholed, not actively-refused, /// connection is exactly the kind of thing a JVM AdminClient can hang on past its own - /// `timeout` wrapper) — confirmed live: during a full namespace deletion, a controller's - /// sidecar kept running (past `preStop`, so its `SIGTERM` hadn't even been delivered to - /// the main loop yet) for 100+ seconds, far past the script's own ~25-40s design budget. + /// `timeout` wrapper) — confirmed live (back when this ran as the `quorum-manager` + /// sidecar's own `preStop`, before it moved to the `kafka` container): during a full + /// namespace deletion, the `preStop` kept running for 100+ seconds, far past the script's + /// own ~25-40s design budget at the time. #[test] fn every_cli_call_has_a_kill_after_so_timeout_is_actually_enforced() { let container_command = quorum_manager_container_command(); - let pre_stop_command = quorum_manager_pre_stop_command(); + let pre_stop_command = controller_remove_self_pre_stop_command(None); for command in [&container_command, &pre_stop_command] { for line in command diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 16f2f4b9..915999cb 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -52,8 +52,8 @@ use crate::{ build::{ command::{ broker_kafka_container_commands, controller_kafka_container_command, - kafka_log_opts, kafka_log_opts_env_var, quorum_manager_container_command, - quorum_manager_pre_stop_command, + controller_remove_self_pre_stop_command, kafka_log_opts, kafka_log_opts_env_var, + quorum_manager_container_command, }, graceful_shutdown::add_graceful_shutdown_config, kerberos::add_kerberos_pod_config, @@ -567,12 +567,14 @@ pub fn build_controller_rolegroup_statefulset( /* period_seconds */ 5, /* failure_threshold */ 60, )) - // Liveness intentionally stays a plain TCP check, same as startupProbe - .liveness_probe(controller_tcp_probe( + // See `controller_stuck_unattached_liveness_probe`'s doc comment for why this is no + // longer a plain TCP check. + .liveness_probe(controller_stuck_unattached_liveness_probe( kafka_security.client_port(), + METRICS_PORT, /* timeout_seconds */ 10, - /* period_seconds */ 10, - /* failure_threshold */ 6, + /* period_seconds */ 30, + /* failure_threshold */ 20, )) .readiness_probe(controller_raft_state_probe( METRICS_PORT, @@ -580,6 +582,23 @@ pub fn build_controller_rolegroup_statefulset( /* period_seconds */ 10, /* failure_threshold */ 6, )); + // Skipped when Kerberos is enabled, matching `build_quorum_manager_container`'s own + // gating — `admin-client.properties` (the file this removal call relies on) only covers + // the TLS/SSL case. + if !kafka_security.has_kerberos_enabled() { + cb_kafka.lifecycle_pre_stop(LifecycleHandler { + exec: Some(ExecAction { + command: Some(vec![ + "/bin/bash".to_string(), + "-c".to_string(), + controller_remove_self_pre_stop_command( + merged_config.graceful_shutdown_timeout, + ), + ]), + }), + ..LifecycleHandler::default() + }); + } add_log_config_volume( &mut pod_builder, @@ -681,10 +700,9 @@ pub fn build_controller_rolegroup_statefulset( /// A `Probe` that dials the controller's KRaft listener socket via a plain TCP connect. /// /// This only proves the socket is open, not that the node has a healthy Raft state (leader, -/// follower, or voted). It is intentionally still used for `startupProbe` (there is no -/// meaningful Raft state to check yet while the process is still starting) and for -/// `livenessProbe` (an unhealthy Raft state, e.g. `candidate`/`unattached`, means the node -/// cannot currently reach its peers, which restarting this pod cannot fix on its own). +/// follower, or voted). Used for `startupProbe`: there is no meaningful Raft state to check +/// yet while the process is still starting, so a bare TCP check is all that's meaningful this +/// early. fn controller_tcp_probe( port: Port, timeout_seconds: i32, @@ -729,6 +747,40 @@ fn controller_raft_state_probe( } } +/// A `Probe` combining a plain TCP check of the controller's KRaft listener with a check that +/// its local Raft state isn't stuck in `unattached`. +/// +/// This is needed to work around a bug in KRaft where a new controller is stuck in a loop +/// trying to fetch Raft metadata from its self. +/// +/// This can happen when the headless service used to point to the bootstrap controllers +/// happens to resolve to this exact pod. +fn controller_stuck_unattached_liveness_probe( + client_port: Port, + metrics_port: Port, + timeout_seconds: i32, + period_seconds: i32, + failure_threshold: i32, +) -> Probe { + Probe { + exec: Some(ExecAction { + command: Some(vec![ + "bash".to_string(), + "-c".to_string(), + format!( + "timeout 2 bash -c 'cat < /dev/null > /dev/tcp/localhost/{client_port}' || exit 1\n\ + state=$(curl -s --max-time 2 localhost:{metrics_port}/metrics | grep -oE 'kafka_server_raft_metrics_current_state\\{{state=\"[a-z]+\"\\}}' | grep -oE '\"[a-z]+\"' | tr -d '\"')\n\ + [ \"$state\" != \"unattached\" ]" + ), + ]), + }), + timeout_seconds: Some(timeout_seconds), + period_seconds: Some(period_seconds), + failure_threshold: Some(failure_threshold), + ..Probe::default() + } +} + /// We only expose client HTTP / HTTPS and Metrics ports. fn container_ports(kafka_security: &ValidatedKafkaSecurity) -> Vec { let mut ports = vec![ @@ -761,8 +813,8 @@ fn container_ports(kafka_security: &ValidatedKafkaSecurity) -> Vec Date: Thu, 20 Aug 2026 10:04:21 +0200 Subject: [PATCH 10/13] Cleanup CHANGELOG and KRaft docs. --- CHANGELOG.md | 35 ++++++------------- .../pages/usage-guide/kraft-controller.adoc | 17 ++++----- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b44c913f..b7aa1695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,27 +4,19 @@ All notable changes to this project will be documented in this file. ## [Unreleased] -### Added - -- A new `quorum-manager` sidecar container on each controller pod admits itself into the KRaft - voter set on startup. Exactly one controller bootstraps the quorum standalone at format time - (`kafka-storage.sh format --standalone`); every other controller, whether present from - the start or added later, formats with `--no-initial-controllers` and joins purely - through the sidecar ([#1010]). -- A new `readinessProbe` on the controller's `kafka` container that fails when new pods cannot - join the quorum ([#1010]). -- A new `livenessProbe` on the controller's `kafka` container that fails when the state has - been stuck `unattached` for an extended period ([#1010]). -- A new `preStop` hook on the `kafka` container that removes the controller from the voters list. - This is the oposite step to what the `quorum-manager` does ([#1010]). -- The `quorum-manager` sidecar now kills an in-flight `add-controller` attempt as soon as its - pod starts terminating, instead of letting it run to completion. Without this, a call already - in flight could succeed after the `kafka` container's `preStop` had already checked the voter - list and found nothing to remove, re-adding a pod that was simultaneously being removed and - leaving it stuck in the on-disk voter list ([#1010]). - ### Changed +- The dynamic KRaft quorum created by the operator is now scaled automatically. Previously, + manual intervention was needed after every scale operation. + This change introduces a new side-car container (`quorum-manager`) to all controller pods + that adds the new controller to the voter list. + On termination, a new `preStop` hook on the controller container (`kafka`) removes the pod from + the voter list before shutdown. + The property `controller.quorum.bootstrap.servers` now contains the headless service names + of all controller role groups instead of individual peer host names. This prevevents the + restart controller from restarting all pods in the quorum when a new one is added/deleted. + The controller `StatefulSet` is now scaled using `OrderedBy` instead of the `Parallel` strategy + to ensure only one voter is added/removed at a time and thus keep the quorum healthy ([#1010]). - Internal operator refactoring: introduce a build() step in the reconciler that assembles all relevant Kubernetes resources before anything is applied ([#985]). - Bump stackable-operator to 0.116.0 ([#994], [#1011]). @@ -35,11 +27,6 @@ All notable changes to this project will be documented in this file. - All product containers now run with `securityContext.runAsNonRoot` set to `true` to improve security ([#998]). - The reconciler now applies resources and derives the cluster status in discrete apply and update_status steps ([#1000]). -- `controller.quorum.bootstrap.servers` now points at each controller role group's - headless Service DNS name instead of individual pod addresses, so neither the container - commands nor that ConfigMap value change with the replica count anymore ([#1010]). -- The controller's StatefulSet now scales sequentially (`OrderedBy`) instead of parallel. - This ensures that one voter joins the quorum at a time ([#1010]). - Environment variable overrides (`envOverrides`) are now merged into the operator-set environment variables by name, so an override replaces the operator's value instead of producing a duplicated entry whose precedence depended on Kubernetes' duplicate-name diff --git a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc index 27ae3494..0254bc87 100644 --- a/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc +++ b/docs/modules/kafka/pages/usage-guide/kraft-controller.adoc @@ -88,9 +88,7 @@ KRaft mode requires major configuration changes compared to ZooKeeper: * Each controller pod runs an additional `quorum-manager` sidecar container that, on startup, admits the pod into the KRaft voter set (`kafka-metadata-quorum.sh add-controller`). Removing the pod from the voter set again (`remove-controller`) on termination runs as the `kafka` container's - *own* `preStop` hook instead, not the sidecar's: `preStop` only delays that same container's `SIGTERM`, and it's - the `kafka` container's own Raft process — the thing actually leaving the voter set — that needs to stay alive - while removal is attempted, which matters most when the departing pod is the current leader. + *own* `preStop` hook. * Controller pods have a `startupProbe` (a plain TCP check on the KRaft listener port) and a `livenessProbe` that combines that same TCP check with a check that the controller's local Raft state hasn't been stuck `unattached` for an extended period — a symptom of a dynamically-joining controller resolving @@ -98,13 +96,10 @@ KRaft mode requires major configuration changes compared to ZooKeeper: forces a fresh DNS resolution attempt. A `readinessProbe` separately checks that the Raft state is one of `leader`, `follower`, or `voted` via the controller's metrics endpoint, so a controller that cannot join or rejoin the quorum is correctly reported as not ready instead of appearing healthy. -* Admitting a controller into the KRaft voter set is *solely* the concern of the `quorum-manager` sidecar container - — the format step never asserts a voter list. Exactly one controller (the one with the numerically - lowest `node.id` among all controller pod descriptors) formats with `kafka-storage.sh format --standalone`, - bootstrapping a single-node quorum by itself. Every other controller — whether it is part of the cluster's initial - desired replica count or added later on scale-up — formats with `--no-initial-controllers` and joins purely - through the sidecar's `add-controller` call. Brokers always format with `--no-initial-controllers` too; they are - never voters. Because no voter list is baked into any container's command, the command is identical regardless of +* Exactly one controller (the one with the numerically lowest `node.id` among all controller pod descriptors) formats + with `kafka-storage.sh format --standalone`, bootstrapping a single-node quorum by itself. + Every other controller formats with `--no-initial-controllers` and joins purely through the sidecar's `add-controller` call. + Brokers always format with `--no-initial-controllers` too; they are never voters. the current replica count. * `controller.quorum.bootstrap.servers` (used by the `kafka` process itself to find the controller quorum, by the `quorum-manager` sidecar for its own `add-controller` calls, and by the `kafka` container's own `preStop` hook @@ -142,7 +137,7 @@ Controller replicas can be scaled up and down on a running cluster. A per-pod `q pod into the KRaft voter set on startup, and the `kafka` container's own `preStop` hook removes it again on termination, as described under "Internal operator details" above. -Scaling more than one controller down at a time is processed one pod at a time (`OrderedReady` pod management), not +Scaling more than one controller at a time is processed one pod at a time (`OrderedReady` pod management), not in parallel, so that each pod's removal from the voter set can complete before the next one is terminated. == Kraft migration guide From 4b42def44319c0c1a07232cc1652615e3e7d854a Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:04:07 +0200 Subject: [PATCH 11/13] Remove the kcat-prober container and consolidate container probes --- CHANGELOG.md | 8 + extra/crds.yaml | 160 -------- .../src/controller/build/kerberos.rs | 12 +- .../src/controller/build/resource/mod.rs | 1 + .../src/controller/build/resource/probes.rs | 165 ++++++++ .../controller/build/resource/statefulset.rs | 369 +++++++++--------- .../src/controller/build/security.rs | 6 +- rust/operator-binary/src/crd/role/broker.rs | 1 - .../kuttl/configuration/10-assert.yaml.j2 | 1 - tests/templates/kuttl/smoke/33-assert.yaml.j2 | 8 - 10 files changed, 375 insertions(+), 356 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/resource/probes.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index b7aa1695..a794642f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,14 @@ All notable changes to this project will be documented in this file. which could cause problems with GitOps tools (e.g. ArgoCD) reporting a diff in the custom resources. See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#998]). +### Removed + +- BREAKING: The broker pod's separate `kcat-prober` sidecar container has been removed; its + `kcat`-based readiness probe now runs directly on the `kafka` container instead (`kcat` has + shipped in the same product image as `kafka` since #527, so the dedicated container/image was + no longer needed). The `kcat-prober` value is no longer accepted in a broker's + `logging.containers` CRD field ([#1010]). + [#985]: https://github.com/stackabletech/kafka-operator/pull/985 [#990]: https://github.com/stackabletech/kafka-operator/pull/990 [#994]: https://github.com/stackabletech/kafka-operator/pull/994 diff --git a/extra/crds.yaml b/extra/crds.yaml index d355b6cf..910e2698 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -181,86 +181,6 @@ spec: description: Configuration per logger type: object type: object - kcat-prober: - anyOf: - - required: - - custom - - {} - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object vector: anyOf: - required: @@ -717,86 +637,6 @@ spec: description: Configuration per logger type: object type: object - kcat-prober: - anyOf: - - required: - - custom - - {} - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object vector: anyOf: - required: diff --git a/rust/operator-binary/src/controller/build/kerberos.rs b/rust/operator-binary/src/controller/build/kerberos.rs index 624525f7..bc54c368 100644 --- a/rust/operator-binary/src/controller/build/kerberos.rs +++ b/rust/operator-binary/src/controller/build/kerberos.rs @@ -45,7 +45,6 @@ pub enum Error { pub fn add_kerberos_pod_config( kafka_security: &ValidatedKafkaSecurity, role: &KafkaRole, - cb_kcat_prober: &mut ContainerBuilder, cb_kafka: &mut ContainerBuilder, pb: &mut PodBuilder, ) -> Result<(), Error> { @@ -68,10 +67,9 @@ pub fn add_kerberos_pod_config( ) .context(AddVolumeSnafu)?; - for cb in [cb_kafka, cb_kcat_prober] { - cb.add_volume_mount("kerberos", STACKABLE_KERBEROS_DIR) - .context(AddVolumeMountSnafu)?; - } + cb_kafka + .add_volume_mount("kerberos", STACKABLE_KERBEROS_DIR) + .context(AddVolumeMountSnafu)?; } Ok(()) @@ -80,8 +78,8 @@ pub fn add_kerberos_pod_config( constant!(KRB5_CONFIG: EnvVarName = "KRB5_CONFIG"); constant!(KAFKA_OPTS: EnvVarName = "KAFKA_OPTS"); -/// The environment variables the Kerberos configuration requires on the Kafka and kcat-prober -/// containers, or an empty set when Kerberos is disabled. +/// The environment variables the Kerberos configuration requires on the Kafka container, or an +/// empty set when Kerberos is disabled. /// /// Returned as an [`EnvVarSet`] (rather than added to the containers directly) so the callers /// can merge the user's `envOverrides` on top, letting an override win on a name collision. diff --git a/rust/operator-binary/src/controller/build/resource/mod.rs b/rust/operator-binary/src/controller/build/resource/mod.rs index 2d0977ad..7f458714 100644 --- a/rust/operator-binary/src/controller/build/resource/mod.rs +++ b/rust/operator-binary/src/controller/build/resource/mod.rs @@ -4,6 +4,7 @@ pub mod config_map; pub mod discovery; pub mod listener; pub mod pdb; +pub mod probes; pub mod rbac; pub mod service; pub mod statefulset; diff --git a/rust/operator-binary/src/controller/build/resource/probes.rs b/rust/operator-binary/src/controller/build/resource/probes.rs new file mode 100644 index 00000000..8b8a5ad6 --- /dev/null +++ b/rust/operator-binary/src/controller/build/resource/probes.rs @@ -0,0 +1,165 @@ +//! Container probes for the Kafka `kafka` container (broker and controller roles). + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::pod::probe::{self, ProbeBuilder}, + k8s_openapi::{ + api::core::v1::{Probe, TCPSocketAction}, + apimachinery::pkg::util::intstr::IntOrString, + }, + shared::time::Duration, + v2::types::common::Port, +}; + +use crate::controller::{ + build::security::kcat_prober_container_commands, security::ValidatedKafkaSecurity, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to build the {name} probe"))] + BuildProbe { + source: probe::Error, + name: &'static str, + }, +} + +/// The broker `kafka` container's readiness probe. +/// +/// Uses `kcat` rather than the official Kafka tools, since they incur a lot of unacceptable perf +/// overhead when run repeatedly as a probe. Only allow the global load balancing service to send +/// traffic to pods that are members of the quorum. This also acts as a hint to the StatefulSet +/// controller to wait for each pod to enter quorum before taking down the next. +pub fn broker_kcat_readiness_probe( + kafka_security: &ValidatedKafkaSecurity, +) -> Result { + ProbeBuilder::exec_command( + // If the broker is able to get its fellow cluster members then it has at least + // completed basic registration at some point + kcat_prober_container_commands(kafka_security), + ) + .with_period(Duration::from_secs(2)) + .with_timeout(Duration::from_secs(5)) + // `ProbeBuilder` otherwise defaults this to 1; kept at Kubernetes' own default (3) to match + // the pre-`ProbeBuilder` behaviour, which left this field unset. + .with_failure_threshold(3) + .build() + .context(BuildProbeSnafu { + name: "kcat readiness", + }) +} + +/// A `Probe` combining a plain TCP check of the broker's client listener with a check that the +/// broker's own JMX `BrokerState` metric reports `RUNNING` (state `3`). +/// +/// Used for both the `startupProbe` (so the `livenessProbe` doesn't start counting failures +/// until the broker has actually finished starting - the client port can accept connections +/// before the broker reaches `RUNNING`, e.g. while still replaying its log) and the +/// `livenessProbe` (restarting a broker stuck in some other state, e.g. `RECOVERY` after a +/// crash); only the timing parameters differ between the two uses. +pub fn broker_running_probe( + client_port: Port, + metrics_port: Port, + timeout_seconds: u64, + period_seconds: u64, + failure_threshold: i32, +) -> Result { + ProbeBuilder::exec_command([ + "bash".to_string(), + "-c".to_string(), + format!( + "timeout 2 bash -c 'cat < /dev/null > /dev/tcp/localhost/{client_port}' || exit 1\n\ + curl -s --max-time 2 localhost:{metrics_port}/metrics | grep -qE 'kafka_server_kafkaserver_brokerstate 3(\\.0)?$'" + ), + ]) + .with_period(Duration::from_secs(period_seconds)) + .with_timeout(Duration::from_secs(timeout_seconds)) + .with_failure_threshold(failure_threshold) + .build() + .context(BuildProbeSnafu { + name: "broker running", + }) +} + +/// A `Probe` that dials the controller's KRaft listener socket via a plain TCP connect. +/// +/// This only proves the socket is open, not that the node has a healthy Raft state (leader, +/// follower, or voted). Used for `startupProbe`: there is no meaningful Raft state to check +/// yet while the process is still starting, so a bare TCP check is all that's meaningful this +/// early. +pub fn controller_tcp_probe( + port: Port, + timeout_seconds: u64, + period_seconds: u64, + failure_threshold: i32, +) -> Result { + ProbeBuilder::tcp_socket(TCPSocketAction { + port: IntOrString::Int(port.into()), + ..Default::default() + }) + .with_period(Duration::from_secs(period_seconds)) + .with_timeout(Duration::from_secs(timeout_seconds)) + .with_failure_threshold(failure_threshold) + .build() + .context(BuildProbeSnafu { + name: "controller startup", + }) +} + +/// A `Probe` that curls the JMX Prometheus exporter's `/metrics` endpoint and checks that the +/// controller's Raft state is one of the healthy states (`leader`, `follower`, or `voted`) +/// rather than stuck in `unattached` or `candidate`. +pub fn controller_raft_state_probe( + metrics_port: Port, + timeout_seconds: u64, + period_seconds: u64, + failure_threshold: i32, +) -> Result { + ProbeBuilder::exec_command([ + "bash".to_string(), + "-c".to_string(), + format!( + "curl -s localhost:{metrics_port}/metrics | grep -E 'kafka_server_raft_metrics_current_state\\{{state=\"(leader|follower|voted)\",?\\}}'" + ), + ]) + .with_period(Duration::from_secs(period_seconds)) + .with_timeout(Duration::from_secs(timeout_seconds)) + .with_failure_threshold(failure_threshold) + .build() + .context(BuildProbeSnafu { + name: "controller raft state", + }) +} + +/// A `Probe` combining a plain TCP check of the controller's KRaft listener with a check that +/// its local Raft state isn't stuck in `unattached`. +/// +/// This is needed to work around a bug in KRaft where a new controller is stuck in a loop +/// trying to fetch Raft metadata from its self. +/// +/// This can happen when the headless service used to point to the bootstrap controllers +/// happens to resolve to this exact pod. +pub fn controller_stuck_unattached_liveness_probe( + client_port: Port, + metrics_port: Port, + timeout_seconds: u64, + period_seconds: u64, + failure_threshold: i32, +) -> Result { + ProbeBuilder::exec_command([ + "bash".to_string(), + "-c".to_string(), + format!( + "timeout 2 bash -c 'cat < /dev/null > /dev/tcp/localhost/{client_port}' || exit 1\n\ + state=$(curl -s --max-time 2 localhost:{metrics_port}/metrics | grep -oE 'kafka_server_raft_metrics_current_state\\{{state=\"[a-z]+\"\\}}' | grep -oE '\"[a-z]+\"' | tr -d '\"')\n\ + [ \"$state\" != \"unattached\" ]" + ), + ]) + .with_period(Duration::from_secs(period_seconds)) + .with_timeout(Duration::from_secs(timeout_seconds)) + .with_failure_threshold(failure_threshold) + .build() + .context(BuildProbeSnafu { + name: "controller stuck-unattached liveness", + }) +} diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 33f33263..bf3c2701 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -19,11 +19,10 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec, StatefulSetUpdateStrategy}, core::v1::{ - ConfigMapVolumeSource, ContainerPort, EnvVar, ExecAction, LifecycleHandler, - PodSpec, Probe, TCPSocketAction, Volume, + ConfigMapVolumeSource, ContainerPort, EnvVar, ExecAction, LifecycleHandler, Volume, }, }, - apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, + apimachinery::pkg::apis::meta::v1::LabelSelector, }, product_logging, v2::{ @@ -39,13 +38,11 @@ use stackable_operator::{ STACKABLE_LOG_DIR, ValidatedContainerLogConfigChoice, vector_container, }, role_group_utils::ResourceNames, - types::{ - common::Port, - kubernetes::{ConfigMapKey, ContainerName, PersistentVolumeClaimName, VolumeName}, - }, + types::kubernetes::{ConfigMapKey, ContainerName, PersistentVolumeClaimName, VolumeName}, }, }; +use super::probes; use crate::{ controller::{ RoleGroupName, ValidatedCluster, ValidatedRoleGroupConfig, @@ -63,7 +60,6 @@ use crate::{ security::{ STACKABLE_TLS_KAFKA_INTERNAL_DIR, STACKABLE_TLS_KAFKA_INTERNAL_VOLUME_NAME, add_broker_volume_and_volume_mounts, add_controller_volume_and_volume_mounts, - kcat_prober_container_commands, }, }, node_id_hasher::node_id_hash32_offset, @@ -191,6 +187,9 @@ pub enum Error { source: crate::controller::PodDescriptorsError, }, + #[snafu(display("failed to build container probe"))] + BuildProbe { source: probes::Error }, + #[snafu(display("failed to construct JVM arguments"))] ConstructJvmArguments { source: crate::controller::build::jvm::Error, @@ -235,12 +234,6 @@ pub fn build_broker_rolegroup_statefulset( role_group_name, ); - let kcat_prober_container_name = BrokerContainer::KcatProber.to_string(); - let mut cb_kcat_prober = - ContainerBuilder::new(&kcat_prober_container_name).context(InvalidContainerNameSnafu { - name: kcat_prober_container_name.clone(), - })?; - let kafka_container_name = BrokerContainer::Kafka.to_string(); let mut cb_kafka = ContainerBuilder::new(&kafka_container_name).context(InvalidContainerNameSnafu { @@ -257,7 +250,6 @@ pub fn build_broker_rolegroup_statefulset( add_broker_volume_and_volume_mounts( kafka_security, &mut pod_builder, - &mut cb_kcat_prober, &mut cb_kafka, &requested_secret_lifetime, ) @@ -278,14 +270,8 @@ pub fn build_broker_rolegroup_statefulset( )); if kafka_security.has_kerberos_enabled() { - add_kerberos_pod_config( - kafka_security, - kafka_role, - &mut cb_kcat_prober, - &mut cb_kafka, - &mut pod_builder, - ) - .context(AddKerberosConfigSnafu)?; + add_kerberos_pod_config(kafka_security, kafka_role, &mut cb_kafka, &mut pod_builder) + .context(AddKerberosConfigSnafu)?; } // Operator-set env vars first; the user's `envOverrides` are merged on top last and win. @@ -303,6 +289,28 @@ pub fn build_broker_rolegroup_statefulset( .merge(validated_rg.env_overrides.clone()) .into(); + // The client port can accept connections before the broker has replayed its log and + // reached the JMX `RUNNING` state, so the startupProbe waits for both, giving it up to + // 5 minutes (60 * 5s) before the livenessProbe is allowed to start counting failures. + let broker_startup_probe = probes::broker_running_probe( + kafka_security.client_port(), + METRICS_PORT, + /* timeout_seconds */ 5, + /* period_seconds */ 5, + /* failure_threshold */ 60, + ) + .context(BuildProbeSnafu)?; + let broker_liveness_probe = probes::broker_running_probe( + kafka_security.client_port(), + METRICS_PORT, + /* timeout_seconds */ 10, + /* period_seconds */ 30, + /* failure_threshold */ 20, + ) + .context(BuildProbeSnafu)?; + let broker_readiness_probe = + probes::broker_kcat_readiness_probe(kafka_security).context(BuildProbeSnafu)?; + cb_kafka .image_from_product_image(resolved_product_image) .command(vec![ @@ -335,44 +343,10 @@ pub fn build_broker_rolegroup_statefulset( .context(AddVolumeMountSnafu)? .add_volume_mount(STACKABLE_LOG_DIR_NAME, STACKABLE_LOG_DIR) .context(AddVolumeMountSnafu)? - .resources(merged_config.resources().clone().into()); - - // Use kcat sidecar for probing container status rather than the official Kafka tools, since they incur a lot of - // unacceptable perf overhead - cb_kcat_prober - .image_from_product_image(resolved_product_image) - .command(vec!["sleep".to_string(), "infinity".to_string()]) - .add_env_vars(Vec::::from( - EnvVarSet::new() - .with_field_path(&POD_NAME, &FieldPathEnvVar::Name) - .merge(kerberos_env_vars(kafka_security)), - )) - .resources( - ResourceRequirementsBuilder::new() - .with_cpu_request("100m") - .with_cpu_limit("200m") - .with_memory_request("128Mi") - .with_memory_limit("128Mi") - .build(), - ) - .add_volume_mount( - LISTENER_BOOTSTRAP_VOLUME_NAME, - STACKABLE_LISTENER_BOOTSTRAP_DIR, - ) - .context(AddVolumeMountSnafu)? - .add_volume_mount(LISTENER_BROKER_VOLUME_NAME, STACKABLE_LISTENER_BROKER_DIR) - .context(AddVolumeMountSnafu)? - // Only allow the global load balancing service to send traffic to pods that are members of the quorum - // This also acts as a hint to the StatefulSet controller to wait for each pod to enter quorum before taking down the next - .readiness_probe(Probe { - exec: Some(ExecAction { - // If the broker is able to get its fellow cluster members then it has at least completed basic registration at some point - command: Some(kcat_prober_container_commands(kafka_security)), - }), - timeout_seconds: Some(5), - period_seconds: Some(2), - ..Probe::default() - }); + .resources(merged_config.resources().clone().into()) + .startup_probe(broker_startup_probe) + .liveness_probe(broker_liveness_probe) + .readiness_probe(broker_readiness_probe); add_log_config_volume( &mut pod_builder, @@ -414,7 +388,6 @@ pub fn build_broker_rolegroup_statefulset( .metadata(metadata) .image_pull_secrets_from_product_image(resolved_product_image) .add_container(cb_kafka.build()) - .add_container(cb_kcat_prober.build()) .affinity(&merged_config.affinity); add_common_pod_config( @@ -438,10 +411,6 @@ pub fn build_broker_rolegroup_statefulset( let mut pod_template = pod_builder.build_template(); - let pod_template_spec = pod_template.spec.get_or_insert_with(PodSpec::default); - // Don't run kcat pod as PID 1, to ensure that default signal handlers apply - pod_template_spec.share_process_namespace = Some(true); - // Pod overrides were already merged (role <- role group) during validation. pod_template.merge_from(validated_rg.pod_overrides.clone()); @@ -529,6 +498,36 @@ pub fn build_controller_rolegroup_statefulset( .pod_descriptors(Some(kafka_role)) .context(BuildPodDescriptorsSnafu)?; + // The controller listener socket only opens once the KRaft node has finished replaying + // its metadata log, which can take a while on a slow first boot or after a long outage. + // The startupProbe gives it up to 5 minutes (60 * 5s) before the liveness probe is + // allowed to start counting failures at all, so a slow (but progressing) boot is never + // mistaken for a stuck process. + let controller_startup_probe = probes::controller_tcp_probe( + kafka_security.client_port(), + /* timeout_seconds */ 5, + /* period_seconds */ 5, + /* failure_threshold */ 60, + ) + .context(BuildProbeSnafu)?; + // See `probes::controller_stuck_unattached_liveness_probe`'s doc comment for why this is no + // longer a plain TCP check. + let controller_liveness_probe = probes::controller_stuck_unattached_liveness_probe( + kafka_security.client_port(), + METRICS_PORT, + /* timeout_seconds */ 10, + /* period_seconds */ 30, + /* failure_threshold */ 20, + ) + .context(BuildProbeSnafu)?; + let controller_readiness_probe = probes::controller_raft_state_probe( + METRICS_PORT, + /* timeout_seconds */ 10, + /* period_seconds */ 10, + /* failure_threshold */ 6, + ) + .context(BuildProbeSnafu)?; + cb_kafka .image_from_product_image(resolved_product_image) .command(vec![ @@ -554,32 +553,9 @@ pub fn build_controller_rolegroup_statefulset( .add_volume_mount(STACKABLE_LOG_DIR_NAME, STACKABLE_LOG_DIR) .context(AddVolumeMountSnafu)? .resources(merged_config.resources().clone().into()) - // The controller listener socket only opens once the KRaft node has finished replaying - // its metadata log, which can take a while on a slow first boot or after a long outage. - // The startupProbe gives it up to 5 minutes (60 * 5s) before the liveness probe is - // allowed to start counting failures at all, so a slow (but progressing) boot is never - // mistaken for a stuck process. - .startup_probe(controller_tcp_probe( - kafka_security.client_port(), - /* timeout_seconds */ 5, - /* period_seconds */ 5, - /* failure_threshold */ 60, - )) - // See `controller_stuck_unattached_liveness_probe`'s doc comment for why this is no - // longer a plain TCP check. - .liveness_probe(controller_stuck_unattached_liveness_probe( - kafka_security.client_port(), - METRICS_PORT, - /* timeout_seconds */ 10, - /* period_seconds */ 30, - /* failure_threshold */ 20, - )) - .readiness_probe(controller_raft_state_probe( - METRICS_PORT, - /* timeout_seconds */ 10, - /* period_seconds */ 10, - /* failure_threshold */ 6, - )); + .startup_probe(controller_startup_probe) + .liveness_probe(controller_liveness_probe) + .readiness_probe(controller_readiness_probe); // Skipped when Kerberos is enabled, matching `build_quorum_manager_container`'s own // gating — `admin-client.properties` (the file this removal call relies on) only covers // the TLS/SSL case. @@ -693,90 +669,6 @@ pub fn build_controller_rolegroup_statefulset( }) } -/// A `Probe` that dials the controller's KRaft listener socket via a plain TCP connect. -/// -/// This only proves the socket is open, not that the node has a healthy Raft state (leader, -/// follower, or voted). Used for `startupProbe`: there is no meaningful Raft state to check -/// yet while the process is still starting, so a bare TCP check is all that's meaningful this -/// early. -fn controller_tcp_probe( - port: Port, - timeout_seconds: i32, - period_seconds: i32, - failure_threshold: i32, -) -> Probe { - Probe { - tcp_socket: Some(TCPSocketAction { - port: IntOrString::Int(port.into()), - ..Default::default() - }), - timeout_seconds: Some(timeout_seconds), - period_seconds: Some(period_seconds), - failure_threshold: Some(failure_threshold), - ..Probe::default() - } -} - -/// A `Probe` that curls the JMX Prometheus exporter's `/metrics` endpoint and checks that the -/// controller's Raft state is one of the healthy states (`leader`, `follower`, or `voted`) -/// rather than stuck in `unattached` or `candidate`. -fn controller_raft_state_probe( - metrics_port: Port, - timeout_seconds: i32, - period_seconds: i32, - failure_threshold: i32, -) -> Probe { - Probe { - exec: Some(ExecAction { - command: Some(vec![ - "bash".to_string(), - "-c".to_string(), - format!( - "curl -s localhost:{metrics_port}/metrics | grep -E 'kafka_server_raft_metrics_current_state\\{{state=\"(leader|follower|voted)\",?\\}}'" - ), - ]), - }), - timeout_seconds: Some(timeout_seconds), - period_seconds: Some(period_seconds), - failure_threshold: Some(failure_threshold), - ..Probe::default() - } -} - -/// A `Probe` combining a plain TCP check of the controller's KRaft listener with a check that -/// its local Raft state isn't stuck in `unattached`. -/// -/// This is needed to work around a bug in KRaft where a new controller is stuck in a loop -/// trying to fetch Raft metadata from its self. -/// -/// This can happen when the headless service used to point to the bootstrap controllers -/// happens to resolve to this exact pod. -fn controller_stuck_unattached_liveness_probe( - client_port: Port, - metrics_port: Port, - timeout_seconds: i32, - period_seconds: i32, - failure_threshold: i32, -) -> Probe { - Probe { - exec: Some(ExecAction { - command: Some(vec![ - "bash".to_string(), - "-c".to_string(), - format!( - "timeout 2 bash -c 'cat < /dev/null > /dev/tcp/localhost/{client_port}' || exit 1\n\ - state=$(curl -s --max-time 2 localhost:{metrics_port}/metrics | grep -oE 'kafka_server_raft_metrics_current_state\\{{state=\"[a-z]+\"\\}}' | grep -oE '\"[a-z]+\"' | tr -d '\"')\n\ - [ \"$state\" != \"unattached\" ]" - ), - ]), - }), - timeout_seconds: Some(timeout_seconds), - period_seconds: Some(period_seconds), - failure_threshold: Some(failure_threshold), - ..Probe::default() - } -} - /// We only expose client HTTP / HTTPS and Metrics ports. fn container_ports(kafka_security: &ValidatedKafkaSecurity) -> Vec { let mut ports = vec![ @@ -1506,6 +1398,131 @@ mod tests { .expect("the kafka container is built") } + fn broker_kafka_container( + cluster: &crate::controller::ValidatedCluster, + ) -> stackable_operator::k8s_openapi::api::core::v1::Container { + let resources = crate::controller::build::build(cluster).expect("build succeeds"); + let sts = resources + .stateful_sets + .into_iter() + .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-broker-default")) + .expect("the broker StatefulSet is built"); + sts.spec + .expect("the StatefulSet has a spec") + .template + .spec + .expect("the pod template has a spec") + .containers + .into_iter() + .find(|c| c.name == "kafka") + .expect("the kafka container is built") + } + + #[test] + fn broker_kafka_container_has_a_startup_probe() { + let cluster = kraft_mode_cluster(); + let container = broker_kafka_container(&cluster); + let client_port = cluster.cluster_config.kafka_security.client_port(); + + let startup_probe = container + .startup_probe + .expect("the broker kafka container must have a startupProbe"); + let exec = startup_probe + .exec + .expect("the startupProbe must be an exec check, not a bare tcpSocket check"); + let command = exec.command.expect("exec has a command"); + let script = command.last().expect("the exec command has a script arg"); + + assert!( + script.contains(&format!("/dev/tcp/localhost/{client_port}")), + "expected a TCP reachability check against the broker's own client port, script was: {script}" + ); + assert!( + script.contains("kafka_server_kafkaserver_brokerstate 3"), + "expected a check for the broker's JMX BrokerState metric being RUNNING (3), \ + script was: {script}" + ); + assert_eq!(startup_probe.timeout_seconds, Some(5)); + assert_eq!(startup_probe.period_seconds, Some(5)); + assert_eq!(startup_probe.failure_threshold, Some(60)); + } + + /// The liveness probe must check both TCP reachability (a genuinely dead/hung process must + /// still be restarted) and that the broker's JMX `BrokerState` metric reports `RUNNING` + /// (state `3`) - see `probes::broker_running_probe`'s doc comment for why the same check + /// backs both the startup and liveness probes. + #[test] + fn broker_kafka_container_liveness_probe_checks_tcp_and_running_state() { + let cluster = kraft_mode_cluster(); + let container = broker_kafka_container(&cluster); + let client_port = cluster.cluster_config.kafka_security.client_port(); + + let liveness_probe = container + .liveness_probe + .expect("the broker kafka container must have a livenessProbe"); + let exec = liveness_probe + .exec + .expect("the livenessProbe must be an exec check, not a bare tcpSocket check"); + let command = exec.command.expect("exec has a command"); + let script = command.last().expect("the exec command has a script arg"); + + assert!( + script.contains(&format!("/dev/tcp/localhost/{client_port}")), + "expected a TCP reachability check against the broker's own client port, script was: {script}" + ); + assert!( + script.contains("kafka_server_kafkaserver_brokerstate 3"), + "expected a check for the broker's JMX BrokerState metric being RUNNING (3), \ + script was: {script}" + ); + + assert_eq!(liveness_probe.timeout_seconds, Some(10)); + assert_eq!(liveness_probe.period_seconds, Some(30)); + assert_eq!(liveness_probe.failure_threshold, Some(20)); + } + + /// The `kcat`-based readiness probe runs directly on the `kafka` container - there is no + /// separate `kcat-prober` sidecar (removed since `kcat` ships in the same product image the + /// `kafka` container already uses, so a dedicated container was no longer needed). + #[test] + fn broker_kafka_container_readiness_probe_uses_kcat() { + let cluster = kraft_mode_cluster(); + let container = broker_kafka_container(&cluster); + + let readiness_probe = container + .readiness_probe + .expect("the broker kafka container must have a readinessProbe"); + let exec = readiness_probe + .exec + .expect("the readinessProbe must be an exec check"); + let command = exec.command.expect("exec has a command"); + assert_eq!(command[0], "/stackable/kcat"); + } + + #[test] + fn broker_pods_have_no_kcat_prober_sidecar() { + let cluster = kraft_mode_cluster(); + let resources = crate::controller::build::build(&cluster).expect("build succeeds"); + let sts = resources + .stateful_sets + .into_iter() + .find(|sts| sts.metadata.name.as_deref() == Some("simple-kafka-broker-default")) + .expect("the broker StatefulSet is built"); + let containers = sts + .spec + .expect("the StatefulSet has a spec") + .template + .spec + .expect("the pod template has a spec") + .containers; + + assert!( + !containers.iter().any(|c| c.name == "kcat-prober"), + "expected no separate kcat-prober container, got: {:?}", + containers.iter().map(|c| &c.name).collect::>() + ); + } + #[test] fn controller_kafka_container_has_a_startup_probe() { let cluster = kraft_mode_cluster(); diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index db3b9f28..46592856 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -254,13 +254,13 @@ pub fn controller_admin_client_properties( pub fn add_broker_volume_and_volume_mounts( security: &ValidatedKafkaSecurity, pod_builder: &mut PodBuilder, - cb_kcat_prober: &mut ContainerBuilder, cb_kafka: &mut ContainerBuilder, requested_secret_lifetime: &Duration, ) -> Result<(), Error> { // add tls (server or client authentication volumes) if required if let Some(tls_server_secret_class) = tls_secret_class(security) { - // We have to mount tls pem files for kcat (the mount can be used directly) + // We have to mount tls pem files for kcat's readiness-probe command (the mount can be + // used directly) pod_builder .add_volume(create_kcat_tls_volume( STACKABLE_TLS_KCAT_VOLUME_NAME, @@ -268,7 +268,7 @@ pub fn add_broker_volume_and_volume_mounts( requested_secret_lifetime, )?) .context(AddVolumeSnafu)?; - cb_kcat_prober + cb_kafka .add_volume_mount(STACKABLE_TLS_KCAT_VOLUME_NAME, STACKABLE_TLS_KCAT_DIR) .context(AddVolumeMountSnafu)?; // Keystores fore the kafka container diff --git a/rust/operator-binary/src/crd/role/broker.rs b/rust/operator-binary/src/crd/role/broker.rs index c1eafa34..3c9b70a1 100644 --- a/rust/operator-binary/src/crd/role/broker.rs +++ b/rust/operator-binary/src/crd/role/broker.rs @@ -31,7 +31,6 @@ use crate::crd::role::commons::{CommonConfig, Storage, StorageFragment}; #[strum(serialize_all = "kebab-case")] pub enum BrokerContainer { Vector, - KcatProber, Kafka, } diff --git a/tests/templates/kuttl/configuration/10-assert.yaml.j2 b/tests/templates/kuttl/configuration/10-assert.yaml.j2 index 3de5ea66..3d75ac23 100644 --- a/tests/templates/kuttl/configuration/10-assert.yaml.j2 +++ b/tests/templates/kuttl/configuration/10-assert.yaml.j2 @@ -23,7 +23,6 @@ spec: cpu: 250m # value set in the rolegroup configuration memory: 3Gi - - name: kcat-prober {% if lookup('env', 'VECTOR_AGGREGATOR') %} - name: vector {% endif %} diff --git a/tests/templates/kuttl/smoke/33-assert.yaml.j2 b/tests/templates/kuttl/smoke/33-assert.yaml.j2 index 70678faa..01d3817f 100644 --- a/tests/templates/kuttl/smoke/33-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/33-assert.yaml.j2 @@ -64,14 +64,6 @@ spec: requests: cpu: 300m # From podOverrides memory: 2Gi - - name: kcat-prober - resources: - limits: - cpu: 200m - memory: 128Mi - requests: - cpu: 100m - memory: 128Mi {% if vector_enabled %} - name: vector env: From 0596a0a305104d367d159791568a61a09862cfd8 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:30:48 +0200 Subject: [PATCH 12/13] Cleanups --- .../kafka/partials/supported-versions.adoc | 5 +- .../controller/build/resource/statefulset.rs | 47 ++++++------------- 2 files changed, 15 insertions(+), 37 deletions(-) diff --git a/docs/modules/kafka/partials/supported-versions.adoc b/docs/modules/kafka/partials/supported-versions.adoc index 1a57dce0..5a962cb4 100644 --- a/docs/modules/kafka/partials/supported-versions.adoc +++ b/docs/modules/kafka/partials/supported-versions.adoc @@ -7,10 +7,7 @@ * 3.9.2 (LTS) * 3.9.1 (deprecated) -Support for clusters running in Kraft mode (which includes Apache Kafka 4.x.x) is experimental because it has not been thoroughly tested in production environments yet. +Support for clusters running in Kraft mode (which includes Apache Kafka >= 4.x) is experimental due to the following known issues: -Also there are some known issues such as: - -* Controller scaling is not reliable. * Kerberos authentication is not tested yet. * Service exposition is not definitive. diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index bf3c2701..00adff25 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -292,22 +292,12 @@ pub fn build_broker_rolegroup_statefulset( // The client port can accept connections before the broker has replayed its log and // reached the JMX `RUNNING` state, so the startupProbe waits for both, giving it up to // 5 minutes (60 * 5s) before the livenessProbe is allowed to start counting failures. - let broker_startup_probe = probes::broker_running_probe( - kafka_security.client_port(), - METRICS_PORT, - /* timeout_seconds */ 5, - /* period_seconds */ 5, - /* failure_threshold */ 60, - ) - .context(BuildProbeSnafu)?; - let broker_liveness_probe = probes::broker_running_probe( - kafka_security.client_port(), - METRICS_PORT, - /* timeout_seconds */ 10, - /* period_seconds */ 30, - /* failure_threshold */ 20, - ) - .context(BuildProbeSnafu)?; + let broker_startup_probe = + probes::broker_running_probe(kafka_security.client_port(), METRICS_PORT, 5, 5, 60) + .context(BuildProbeSnafu)?; + let broker_liveness_probe = + probes::broker_running_probe(kafka_security.client_port(), METRICS_PORT, 10, 30, 20) + .context(BuildProbeSnafu)?; let broker_readiness_probe = probes::broker_kcat_readiness_probe(kafka_security).context(BuildProbeSnafu)?; @@ -503,30 +493,21 @@ pub fn build_controller_rolegroup_statefulset( // The startupProbe gives it up to 5 minutes (60 * 5s) before the liveness probe is // allowed to start counting failures at all, so a slow (but progressing) boot is never // mistaken for a stuck process. - let controller_startup_probe = probes::controller_tcp_probe( - kafka_security.client_port(), - /* timeout_seconds */ 5, - /* period_seconds */ 5, - /* failure_threshold */ 60, - ) - .context(BuildProbeSnafu)?; + let controller_startup_probe = + probes::controller_tcp_probe(kafka_security.client_port(), 5, 5, 60) + .context(BuildProbeSnafu)?; // See `probes::controller_stuck_unattached_liveness_probe`'s doc comment for why this is no // longer a plain TCP check. let controller_liveness_probe = probes::controller_stuck_unattached_liveness_probe( kafka_security.client_port(), METRICS_PORT, - /* timeout_seconds */ 10, - /* period_seconds */ 30, - /* failure_threshold */ 20, - ) - .context(BuildProbeSnafu)?; - let controller_readiness_probe = probes::controller_raft_state_probe( - METRICS_PORT, - /* timeout_seconds */ 10, - /* period_seconds */ 10, - /* failure_threshold */ 6, + 10, + 30, + 20, ) .context(BuildProbeSnafu)?; + let controller_readiness_probe = + probes::controller_raft_state_probe(METRICS_PORT, 10, 10, 6).context(BuildProbeSnafu)?; cb_kafka .image_from_product_image(resolved_product_image) From ed3180b81531b59d10dcae0b76d7b546c5b33ec3 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:54:29 +0200 Subject: [PATCH 13/13] Stop Kafka to prevent kuttl namespace deletion timeouts --- .../kuttl/smoke-kraft/90-assert.yaml.j2 | 20 ++++++++++ .../kuttl/smoke-kraft/90-stop-kafka.yaml.j2 | 40 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 tests/templates/kuttl/smoke-kraft/90-assert.yaml.j2 create mode 100644 tests/templates/kuttl/smoke-kraft/90-stop-kafka.yaml.j2 diff --git a/tests/templates/kuttl/smoke-kraft/90-assert.yaml.j2 b/tests/templates/kuttl/smoke-kraft/90-assert.yaml.j2 new file mode 100644 index 00000000..c7a90a43 --- /dev/null +++ b/tests/templates/kuttl/smoke-kraft/90-assert.yaml.j2 @@ -0,0 +1,20 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=stopped kafkaclusters.kafka.stackable.tech/test-kafka --timeout 601s +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-broker-default +status: + replicas: 0 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-kafka-controller-default +status: + replicas: 0 diff --git a/tests/templates/kuttl/smoke-kraft/90-stop-kafka.yaml.j2 b/tests/templates/kuttl/smoke-kraft/90-stop-kafka.yaml.j2 new file mode 100644 index 00000000..cafaf9ba --- /dev/null +++ b/tests/templates/kuttl/smoke-kraft/90-stop-kafka.yaml.j2 @@ -0,0 +1,40 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 600 +--- +apiVersion: kafka.stackable.tech/v1alpha1 +kind: KafkaCluster +metadata: + name: test-kafka +spec: + image: +{% if test_scenario['values']['kafka-kraft'].find(",") > 0 %} + custom: "{{ test_scenario['values']['kafka-kraft'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['kafka-kraft'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['kafka-kraft'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + metadataManager: kraft +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + brokers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 3 + controllers: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 3 + clusterOperation: + stopped: true + reconciliationPaused: false