diff --git a/CHANGELOG.md b/CHANGELOG.md index 8df6c07e..7320cf72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,11 @@ All notable changes to this project will be documented in this file. operator upgrade, delete each broker, coordinator and router StatefulSet so that the operator immediately recreates it with the new labels ([#865]). - Make operations infallible where dependent on static inputs ([#869], [#874]). +- Bump stackable-operator to 0.119.0 ([#880]). +- test: Bump vector-aggregator to 0.58.0 ([#885]). +- Internal operator refactoring: build each role group through per-role modules (brokers, + coordinators, historicals, middle managers, routers) and a shared role group builder, with + each role's configuration held in typed fields of the validated cluster ([#887]). - Support Vector 0.58.0 ([#885], [#889]). - Replace the unmaintained `serde_yaml` with `yaml_serde` ([#888]). - All pods now set `securityContext.seccompProfile.type` to `RuntimeDefault` ([#888]). @@ -65,6 +70,7 @@ All notable changes to this project will be documented in this file. [#879]: https://github.com/stackabletech/druid-operator/pull/879 [#880]: https://github.com/stackabletech/druid-operator/pull/880 [#885]: https://github.com/stackabletech/druid-operator/pull/885 +[#887]: https://github.com/stackabletech/druid-operator/pull/887 [#888]: https://github.com/stackabletech/druid-operator/pull/888 [#889]: https://github.com/stackabletech/druid-operator/pull/889 diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index c160f5f3..c795f84f 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -190,7 +190,7 @@ mod test { use super::{CONTROLLER_NAME, OPERATOR_NAME, PRODUCT_NAME, *}; use crate::{ controller::build::{ - properties::ConfigFileName, resource::config_map::build_rolegroup_config_map, + build, properties::ConfigFileName, test_support::role_group_config_map, }, crd::{DruidRole, PROP_SEGMENT_CACHE_LOCATIONS}, }; @@ -225,31 +225,22 @@ mod test { let druid = crate::controller::validate::test_support::druid_from_yaml(&yaml); let cluster = crate::controller::validate::test_support::validated_cluster(&druid); + let resources = build(&cluster).expect("build succeeds"); - // The segment cache property is injected dynamically by the config_map builder from the - // merged resources of the validated role group config. - let rg = cluster - .role_group_configs - .get(&DruidRole::Historical) - .expect("historical role groups") - .get(&RoleGroupName::from_str(tested_rolegroup_name).unwrap()) - .expect("tested rolegroup") - .clone(); - - let rg_configmap = build_rolegroup_config_map( + // The segment cache property is derived by the Historical role module from the merged + // resources of the role group and rendered into runtime.properties. + let druid_segment_cache_property = role_group_config_map( + &resources, &cluster, &DruidRole::Historical, - &RoleGroupName::from_str(tested_rolegroup_name).unwrap(), - &rg, + &RoleGroupName::from_str(tested_rolegroup_name).expect("valid role group name"), ) - .expect("build rolegroup config map"); - - let druid_segment_cache_property = rg_configmap - .data - .unwrap() - .get(&ConfigFileName::RuntimeProperties.to_string()) - .unwrap() - .to_string(); + .data + .as_ref() + .expect("the ConfigMap has data") + .get(&ConfigFileName::RuntimeProperties.to_string()) + .expect("runtime.properties is rendered") + .to_string(); let escaped_segment_cache_property = stackable_operator::v2::config_file_writer::to_java_properties_string( diff --git a/rust/operator-binary/src/controller/build/authentication/ldap.rs b/rust/operator-binary/src/controller/build/authentication/ldap.rs index a5ba23a4..3d2bd2cc 100644 --- a/rust/operator-binary/src/controller/build/authentication/ldap.rs +++ b/rust/operator-binary/src/controller/build/authentication/ldap.rs @@ -2,12 +2,12 @@ use std::collections::BTreeMap; use snafu::ResultExt; use stackable_operator::{ - builder::pod::{PodBuilder, container::ContainerBuilder}, crd::authentication::ldap, + k8s_openapi::api::core::v1::{Volume, VolumeMount}, }; use super::{ - AddLdapVolumesSnafu, ConstructLdapEndpointUrlSnafu, Error, MissingLdapBindCredentialsSnafu, + ConstructLdapEndpointUrlSnafu, Error, LdapVolumesSnafu, MissingLdapBindCredentialsSnafu, }; use crate::crd::{ STACKABLE_TRUST_STORE_PASSWORD, file_reference, @@ -97,12 +97,11 @@ pub(super) fn prepare_container_commands( } } -pub(super) fn add_volumes_and_mounts( +/// The volumes of the LDAP bind credentials and the provider's TLS CA certificate, and their +/// mounts. +pub(super) fn volumes_and_mounts( provider: &ldap::v1alpha1::AuthenticationProvider, - pb: &mut PodBuilder, - cb_druid: &mut ContainerBuilder, - cb_prepare: &mut ContainerBuilder, -) -> Result<(), Error> { +) -> Result<(Vec, Vec), Error> { // TODO: Connecting to an LDAP server without bind credentials does not seem to be configurable in Druid at the moment // see https://github.com/stackabletech/druid-operator/issues/383 for future work. // Expect bind credentials to be provided for now, and throw return a useful error if there are none. @@ -110,7 +109,5 @@ pub(super) fn add_volumes_and_mounts( return MissingLdapBindCredentialsSnafu.fail(); } - provider - .add_volumes_and_mounts(pb, vec![cb_druid, cb_prepare]) - .context(AddLdapVolumesSnafu) + provider.volumes_and_mounts().context(LdapVolumesSnafu) } diff --git a/rust/operator-binary/src/controller/build/authentication/mod.rs b/rust/operator-binary/src/controller/build/authentication/mod.rs index 2a050b59..1515d53a 100644 --- a/rust/operator-binary/src/controller/build/authentication/mod.rs +++ b/rust/operator-binary/src/controller/build/authentication/mod.rs @@ -9,7 +9,7 @@ use std::collections::BTreeMap; use snafu::Snafu; use stackable_operator::{ - builder::pod::{PodBuilder, container::ContainerBuilder}, + k8s_openapi::api::core::v1::{Volume, VolumeMount}, v2::builder::pod::container::EnvVarSet, }; @@ -17,7 +17,7 @@ use crate::{ authentication::DruidAuthenticationConfig, controller::validate::ValidatedCluster, crd::{ - DruidRole, env_var_reference, + env_var_reference, security::{ INTERNAL_INITIAL_CLIENT_PASSWORD_ENV, INTERNAL_INITIAL_CLIENT_PASSWORD_SECRET_KEY, }, @@ -51,13 +51,13 @@ pub enum Error { source: stackable_operator::crd::authentication::oidc::v1alpha1::Error, }, - #[snafu(display("failed to add LDAP Volumes and VolumeMounts to the Pod and containers"))] - AddLdapVolumes { + #[snafu(display("failed to build the LDAP volumes and volume mounts"))] + LdapVolumes { source: stackable_operator::crd::authentication::ldap::v1alpha1::Error, }, - #[snafu(display("failed to add OIDC Volumes and VolumeMounts to the Pod and containers"))] - AddOidcVolumes { + #[snafu(display("failed to build the OIDC volumes and volume mounts"))] + OidcVolumes { source: stackable_operator::commons::tls_verification::TlsClientDetailsError, }, @@ -98,9 +98,12 @@ fn set_authenticator_chain( /// Creates the authentication and authorization parts of the runtime.properties config file. /// Configuration related to TLS authentication is added in `controller::build::security`. +/// +/// `oidc_enabled` says whether the role authenticates its users with OIDC when the cluster uses +/// OIDC; a role that does not gets the system authenticator only. pub fn generate_runtime_properties_config( auth: &DruidAuthenticationConfig, - role: &DruidRole, + oidc_enabled: bool, ) -> Result> { let mut config: BTreeMap = BTreeMap::new(); @@ -111,7 +114,7 @@ pub fn generate_runtime_properties_config( } DruidAuthenticationConfig::Oidc { provider, oidc } => { generate_common_runtime_properties_config(&mut config); - oidc::generate_runtime_properties_config(provider, oidc, role, &mut config)? + oidc::generate_runtime_properties_config(provider, oidc, oidc_enabled, &mut config)? } DruidAuthenticationConfig::Tls {} => (), } @@ -145,10 +148,12 @@ pub fn prepare_container_commands(auth: &DruidAuthenticationConfig) -> Vec EnvVarSet { let internal_secret_name = build_shared_internal_secret_name(&cluster.name); let mut env_vars = EnvVarSet::new().with_secret_key_ref( @@ -158,25 +163,24 @@ pub fn get_env_var_mounts( ); if let DruidAuthenticationConfig::Oidc { oidc, .. } = auth { - env_vars = env_vars.merge(oidc::get_env_var_mounts(role, oidc, &internal_secret_name)); + env_vars = env_vars.merge(oidc::get_env_var_mounts( + oidc_enabled, + oidc, + &internal_secret_name, + )); } env_vars } -pub fn add_volumes_and_mounts( +/// The pod volumes the authentication provider needs and their mounts on the `druid` and +/// `prepare` containers. +pub fn volumes_and_mounts( auth: &DruidAuthenticationConfig, - pb: &mut PodBuilder, - cb_druid: &mut ContainerBuilder, - cb_prepare: &mut ContainerBuilder, -) -> Result<()> { +) -> Result<(Vec, Vec)> { match auth { - DruidAuthenticationConfig::Ldap { provider } => { - ldap::add_volumes_and_mounts(provider, pb, cb_druid, cb_prepare) - } - DruidAuthenticationConfig::Oidc { provider, .. } => { - oidc::add_volumes_and_mounts(provider, pb, cb_druid, cb_prepare) - } - DruidAuthenticationConfig::Tls {} => Ok(()), + DruidAuthenticationConfig::Ldap { provider } => ldap::volumes_and_mounts(provider), + DruidAuthenticationConfig::Oidc { provider, .. } => oidc::volumes_and_mounts(provider), + DruidAuthenticationConfig::Tls {} => Ok((Vec::new(), Vec::new())), } } @@ -258,8 +262,7 @@ mod test { #[test] fn test_ldap_config_is_added() { - let got = generate_runtime_properties_config(&ldap_auth_config(), &DruidRole::Coordinator) - .unwrap(); + let got = generate_runtime_properties_config(&ldap_auth_config(), true).unwrap(); assert!(got.contains_key("druid.auth.authenticator.Ldap.type")); } @@ -268,8 +271,7 @@ mod test { /// alongside the provider-specific (LDAP/OIDC) config. #[test] fn common_system_auth_config_is_added() { - let got = generate_runtime_properties_config(&ldap_auth_config(), &DruidRole::Coordinator) - .unwrap(); + let got = generate_runtime_properties_config(&ldap_auth_config(), true).unwrap(); assert_eq!( got.get("druid.auth.authenticator.DruidSystemAuthenticator.type"), @@ -290,11 +292,8 @@ mod test { /// contributes nothing to runtime.properties. #[test] fn tls_only_produces_no_runtime_properties() { - let got = generate_runtime_properties_config( - &DruidAuthenticationConfig::Tls {}, - &DruidRole::Broker, - ) - .unwrap(); + let got = + generate_runtime_properties_config(&DruidAuthenticationConfig::Tls {}, true).unwrap(); assert!(got.is_empty()); } diff --git a/rust/operator-binary/src/controller/build/authentication/oidc.rs b/rust/operator-binary/src/controller/build/authentication/oidc.rs index c233abcd..8a1d828a 100644 --- a/rust/operator-binary/src/controller/build/authentication/oidc.rs +++ b/rust/operator-binary/src/controller/build/authentication/oidc.rs @@ -2,16 +2,16 @@ use std::collections::BTreeMap; use snafu::ResultExt; use stackable_operator::{ - builder::pod::{PodBuilder, container::ContainerBuilder}, crd::authentication::oidc, + k8s_openapi::api::core::v1::{Volume, VolumeMount}, v2::{builder::pod::container::EnvVarSet, types::kubernetes::SecretName}, }; -use super::{AddOidcVolumesSnafu, ConstructOidcWellKnownUrlSnafu, Error}; +use super::{ConstructOidcWellKnownUrlSnafu, Error, OidcVolumesSnafu}; use crate::{ authentication::DruidClientAuthenticationOptions, crd::{ - COOKIE_PASSPHRASE_ENV, COOKIE_PASSPHRASE_SECRET_KEY, DruidRole, env_var_reference, + COOKIE_PASSPHRASE_ENV, COOKIE_PASSPHRASE_SECRET_KEY, env_var_reference, security::add_cert_to_jvm_trust_store_cmd, }, }; @@ -80,23 +80,19 @@ fn add_authenticator_config( Ok(()) } -/// Creates the OIDC parts of the runtime.properties config file. -/// OIDC authentication is not configured on middlemanagers, because end users don't interact with them directly using the web console and -/// turning on OIDC will lead to problems with the communication with coordinators during data ingest. +/// Creates the OIDC parts of the runtime.properties config file. A role that does not +/// authenticate its users with OIDC (`oidc_enabled` false) gets the system authenticator only. pub(super) fn generate_runtime_properties_config( provider: &oidc::v1alpha1::AuthenticationProvider, oidc: &DruidClientAuthenticationOptions, - role: &DruidRole, + oidc_enabled: bool, config: &mut BTreeMap, ) -> Result<(), Error> { - match role { - DruidRole::MiddleManager => { - super::set_authenticator_chain(config, &[]); - } - _ => { - add_authenticator_config(provider, oidc, config)?; - super::add_authorizer_config(config, OIDC_AUTHORIZER); - } + if oidc_enabled { + add_authenticator_config(provider, oidc, config)?; + super::add_authorizer_config(config, OIDC_AUTHORIZER); + } else { + super::set_authenticator_chain(config, &[]); } Ok(()) } @@ -110,45 +106,38 @@ pub(super) fn main_container_commands( } } -/// Mounts the OIDC credentials secret and the auto-generated internal secret containing the cookie passphrase. -/// Not necessary on middlemanagers, because OIDC is not configured on them. +/// Mounts the OIDC credentials secret and the auto-generated internal secret containing the +/// cookie passphrase. Nothing for a role that does not authenticate its users with OIDC +/// (`oidc_enabled` false). pub(super) fn get_env_var_mounts( - role: &DruidRole, + oidc_enabled: bool, oidc: &DruidClientAuthenticationOptions, internal_secret_name: &SecretName, ) -> EnvVarSet { let mut env_vars = EnvVarSet::new(); - match role { - DruidRole::MiddleManager => (), - _ => { - for env_var in oidc::v1alpha1::AuthenticationProvider::client_credentials_env_var_mounts( - oidc.client_credentials_secret_ref.to_owned(), - ) { - env_vars = env_vars.with_env_var(env_var).expect( - "env_var name is valid because it is either OIDC_<16-hex-characters>_CLIENT_ID \ - or OIDC_<16-hex-characters>_CLIENT_SECRET", - ); - } - env_vars = env_vars.with_secret_key_ref( - &COOKIE_PASSPHRASE_ENV, - internal_secret_name, - &COOKIE_PASSPHRASE_SECRET_KEY, + if oidc_enabled { + for env_var in oidc::v1alpha1::AuthenticationProvider::client_credentials_env_var_mounts( + oidc.client_credentials_secret_ref.to_owned(), + ) { + env_vars = env_vars.with_env_var(env_var).expect( + "env_var name is valid because it is either OIDC_<16-hex-characters>_CLIENT_ID \ + or OIDC_<16-hex-characters>_CLIENT_SECRET", ); } + env_vars = env_vars.with_secret_key_ref( + &COOKIE_PASSPHRASE_ENV, + internal_secret_name, + &COOKIE_PASSPHRASE_SECRET_KEY, + ); } env_vars } -pub(super) fn add_volumes_and_mounts( +/// The volumes of the OIDC provider's TLS CA certificate, if it has one, and their mounts. +pub(super) fn volumes_and_mounts( provider: &oidc::v1alpha1::AuthenticationProvider, - pb: &mut PodBuilder, - cb_druid: &mut ContainerBuilder, - cb_prepare: &mut ContainerBuilder, -) -> Result<(), Error> { - provider - .tls - .add_volumes_and_mounts(pb, vec![cb_druid, cb_prepare]) - .context(AddOidcVolumesSnafu) +) -> Result<(Vec, Vec), Error> { + provider.tls.volumes_and_mounts().context(OidcVolumesSnafu) } #[cfg(test)] @@ -246,13 +235,8 @@ mod tests { let oidc = test_options(); let mut config = BTreeMap::new(); - generate_runtime_properties_config( - &provider, - &oidc, - &DruidRole::MiddleManager, - &mut config, - ) - .expect("generate runtime properties"); + generate_runtime_properties_config(&provider, &oidc, false, &mut config) + .expect("generate runtime properties"); assert_eq!( config.get("druid.auth.authenticatorChain"), @@ -269,7 +253,7 @@ mod tests { let oidc = test_options(); let mut config = BTreeMap::new(); - generate_runtime_properties_config(&provider, &oidc, &DruidRole::Broker, &mut config) + generate_runtime_properties_config(&provider, &oidc, true, &mut config) .expect("generate runtime properties"); assert_eq!( @@ -292,7 +276,7 @@ mod tests { fn middlemanager_gets_no_oidc_env_vars() { let oidc = test_options(); - let envs = get_env_var_mounts(&DruidRole::MiddleManager, &oidc, &internal_secret_name()); + let envs = get_env_var_mounts(false, &oidc, &internal_secret_name()); assert!(envs.iter().next().is_none()); } @@ -302,7 +286,7 @@ mod tests { fn non_middlemanager_gets_oidc_env_vars() { let oidc = test_options(); - let envs = get_env_var_mounts(&DruidRole::Broker, &oidc, &internal_secret_name()); + let envs = get_env_var_mounts(true, &oidc, &internal_secret_name()); assert!( envs.get(&COOKIE_PASSPHRASE_ENV).is_some(), diff --git a/rust/operator-binary/src/controller/build/container.rs b/rust/operator-binary/src/controller/build/container.rs new file mode 100644 index 00000000..bd4de28b --- /dev/null +++ b/rust/operator-binary/src/controller/build/container.rs @@ -0,0 +1,479 @@ +//! Builds the containers of a role group's pod and collects the volumes they mount. +//! +//! Every Druid role runs the same containers: the `prepare` init container, the `druid` container +//! and, when the Vector agent is enabled, the Vector sidecar. What differs between roles arrives +//! resolved in the [`RoleGroupInputs`]; the role itself is read only for the values that are +//! functions of it (ports, process name, container commands) and for the object names. + +use std::str::FromStr; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::pod::{ + container::ContainerBuilder, resources::ResourceRequirementsBuilder, volume::VolumeBuilder, + }, + k8s_openapi::api::core::v1::{ + Container as PodContainer, EnvVar, PersistentVolumeClaim, Volume, + }, + product_logging, + v2::{ + builder::pod::container::{EnvVarName, EnvVarSet, new_container_builder}, + product_logging::framework::{ + STACKABLE_LOG_DIR, ValidatedContainerLogConfigChoice, vector_container, + }, + role_group_utils::ResourceNames, + types::kubernetes::VolumeName, + }, +}; + +use crate::{ + controller::build::{ + authentication, + properties::product_logging::MAX_DRUID_LOG_FILES_SIZE, + recommended_labels_for_unversioned_role_group_resources, + resource::listener::{LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener_pvc}, + role_group::RoleGroupInputs, + security::{ + self, add_tls_volume_and_volume_mounts, build_tls_key_stores_cmd, container_ports, + get_tcp_socket_probe, + }, + }, + crd::{ + Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, HDFS_CONFIG_DIRECTORY, + LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, RW_CONFIG_DIRECTORY, + }, +}; + +// volume names +stackable_operator::constant!(pub(crate) DRUID_CONFIG_VOLUME_NAME: VolumeName = "config"); +stackable_operator::constant!(pub(crate) HDFS_CONFIG_VOLUME_NAME: VolumeName = "hdfs"); +stackable_operator::constant!(pub(crate) LOG_CONFIG_VOLUME_NAME: VolumeName = "log-config"); +stackable_operator::constant!(pub(crate) LOG_VOLUME_NAME: VolumeName = "log"); +stackable_operator::constant!(pub(crate) RW_CONFIG_VOLUME_NAME: VolumeName = "rwconfig"); + +// Needed for the `containerdebug` process to log its tracing information to. +stackable_operator::constant!(pub(crate) CONTAINERDEBUG_LOG_DIRECTORY: EnvVarName = "CONTAINERDEBUG_LOG_DIRECTORY"); + +// volume mount directory (not a volume name) +const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to build the authentication volumes and volume mounts"))] + AuthenticationVolumes { source: authentication::Error }, + + #[snafu(display("failed to build the TLS volumes and volume mounts"))] + TlsVolumes { source: security::Error }, + + #[snafu(display( + "Druid does not support skipping the verification of the tls enabled S3 server" + ))] + S3TlsNoVerificationNotSupported, + + #[snafu(display("failed to configure S3 connection"))] + ConfigureS3 { + source: stackable_operator::crd::s3::v1alpha1::ConnectionError, + }, + + #[snafu(display("failed to add needed volumeMount"))] + AddVolumeMount { + source: stackable_operator::builder::pod::container::Error, + }, +} + +type Result = std::result::Result; + +/// The containers of a role group's pod and the volumes they mount, in pod order. +pub(crate) struct ContainersAndVolumes { + /// The `prepare` init container. + pub(crate) init_containers: Vec, + /// The `druid` container, then the Vector sidecar when the Vector agent is enabled. + pub(crate) containers: Vec, + /// The pod volumes in the order the pod lists them; see [`containers_and_volumes`]. + pub(crate) pod_volumes: Vec, + /// The listener claim template of a role with a Listener. `None` for the other roles, which + /// keeps their StatefulSets without a `volumeClaimTemplates` field. + pub(crate) volume_claim_templates: Option>, +} + +/// Builds the containers every role runs and collects the volumes they mount. +/// +/// The pod volumes are collected in the order they are mounted: the operator-managed volumes +/// (TLS, config, log config, log, HDFS), then the volumes only the role has (the Historical +/// segment cache), then the volumes derived from user input (authentication, S3, `extraVolumes`). +/// This order is part of the pod template, so changing it restarts every pod; it is pinned by +/// `pod_volumes_keep_their_order` in `build/mod.rs`. A volume whose name collides with another's +/// is rejected when the StatefulSet builder adds the volumes to the pod. +pub(crate) fn containers_and_volumes(inputs: &RoleGroupInputs) -> Result { + let cluster = inputs.cluster; + let role = &inputs.role; + let resource_names = cluster.role_group_resource_names(role, &inputs.role_group_name); + let resolved_product_image = &cluster.image; + let s3_conn = cluster.cluster_config.s3_connection.as_ref(); + let druid_tls_security = &cluster.cluster_config.druid_tls_security; + let druid_auth_config = &cluster.cluster_config.druid_auth_config; + let metadata_database_connection_details = &cluster.cluster_config.metadata_db_connection; + + let mut cb_prepare = new_container_builder(Container::Prepare.name()); + let mut cb_druid = new_container_builder(Container::Druid.name()); + let mut pod_volumes: Vec = Vec::new(); + + if let Some(pre_stop) = &inputs.druid_pre_stop { + cb_druid.lifecycle_pre_stop(pre_stop.clone()); + } + + let mut main_container_commands = role.main_container_prepare_commands(s3_conn); + let mut prepare_container_commands = vec![]; + if let ValidatedContainerLogConfigChoice::Automatic(log_config) = + &inputs.logging.prepare_container + { + // This command needs to be added at the beginning of the shell commands, + // otherwise the output of the following commands will not be captured! + prepare_container_commands.push(product_logging::framework::capture_shell_output( + STACKABLE_LOG_DIR, + Container::Prepare.name().as_ref(), + log_config, + )); + } + prepare_container_commands.extend(build_tls_key_stores_cmd(druid_tls_security)); + + // Operator-managed volumes and volume mounts first. Their mount paths are constants, so the + // mounts cannot collide with each other and adding them is infallible. Volumes and mounts + // derived from user input (authentication, S3, `extraVolumes`) come afterwards; their mounts + // stay fallible, as they can collide with the operator-managed ones. + // + // The TLS certificate of a role with a Listener also carries the Listener's address, so its + // secret volume is scoped to the listener volume. + let listener_volume_scope = inputs + .listener + .is_some() + .then(|| LISTENER_VOLUME_NAME.to_string()); + add_tls_volume_and_volume_mounts( + druid_tls_security, + &mut cb_prepare, + &mut cb_druid, + &mut pod_volumes, + &inputs.requested_secret_lifetime, + listener_volume_scope, + ) + .context(TlsVolumesSnafu)?; + add_config_volume_and_volume_mounts(&resource_names, &mut cb_druid, &mut pod_volumes); + add_log_config_volume_and_volume_mounts( + &resource_names, + &inputs.logging.druid_container, + &mut cb_druid, + &mut pod_volumes, + ); + add_log_volume_and_volume_mounts(&mut cb_druid, &mut cb_prepare, &mut pod_volumes); + add_hdfs_cm_volume_and_volume_mounts( + &cluster.cluster_config.deep_storage, + &mut cb_druid, + &mut pod_volumes, + ); + + // The volumes only this role has (the Historical segment cache) and their mounts come after + // the operator-managed volumes: the pod's `volumes` are an ordered list and this position is + // pinned by `pod_volumes_keep_their_order` in `build/mod.rs`. Their mount paths are constants + // of the role module, distinct from the operator-managed mounts above. + cb_druid + .add_volume_mounts(inputs.role_druid_mounts.iter().cloned()) + .expect("The mount paths are statically defined and there should be no duplicates."); + pod_volumes.extend(inputs.role_pod_volumes.iter().cloned()); + + // The listener volume mount is static as well, so it belongs here, before the derived volumes + // and mounts below. The listener volume itself is a claim template. + let mut volume_claim_templates = None; + if let Some(listener) = &inputs.listener { + cb_druid + .add_volume_mount(&*LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .expect("The mount paths are statically defined and there should be no duplicates."); + + // Used for PVC templates, which cannot be modified once they are deployed. The version + // label is omitted so the labels stay stable across version upgrades. + let unversioned_recommended_labels = + recommended_labels_for_unversioned_role_group_resources( + cluster, + role, + &inputs.role_group_name, + ); + + volume_claim_templates = Some(vec![build_group_listener_pvc( + listener, + &unversioned_recommended_labels, + )]); + } + + if let Some(auth_config) = druid_auth_config { + let (volumes, mounts) = + authentication::volumes_and_mounts(auth_config).context(AuthenticationVolumesSnafu)?; + pod_volumes.extend(volumes); + cb_druid + .add_volume_mounts(mounts.clone()) + .context(AddVolumeMountSnafu)?; + cb_prepare + .add_volume_mounts(mounts) + .context(AddVolumeMountSnafu)?; + prepare_container_commands.extend(authentication::prepare_container_commands(auth_config)); + main_container_commands.extend(authentication::main_container_commands(auth_config)) + } + + if let Some(s3) = s3_conn { + if s3.tls.uses_tls() && !s3.tls.uses_tls_verification() { + S3TlsNoVerificationNotSupportedSnafu.fail()?; + } + let (volumes, mounts) = s3.volumes_and_mounts().context(ConfigureS3Snafu)?; + pod_volumes.extend(volumes); + cb_druid + .add_volume_mounts(mounts) + .context(AddVolumeMountSnafu)?; + } + + cb_prepare + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![prepare_container_commands.join("\n")]) + .resources( + ResourceRequirementsBuilder::new() + .with_cpu_request("100m") + .with_cpu_limit("400m") + .with_memory_request("512Mi") + .with_memory_limit("512Mi") + .build(), + ); + + // All operator-set environment variables of the druid container, collected into an + // `EnvVarSet` so that every name occurs only once. + let mut env_vars = EnvVarSet::new(); + + for env_var in metadata_database_connection_details + .username_env + .iter() + .chain(metadata_database_connection_details.password_env.iter()) + { + env_vars = env_vars.with_env_var(env_var.clone()).expect( + "the database credential env var names are generated by operator-rs from the \ + unique database name and are therefore valid", + ); + } + + if let Some(auth_config) = druid_auth_config { + env_vars = env_vars.merge(authentication::get_env_var_mounts( + auth_config, + cluster, + inputs.oidc_enabled, + )); + } + + env_vars = env_vars.with_value( + &CONTAINERDEBUG_LOG_DIRECTORY, + format!("{STACKABLE_LOG_DIR}/containerdebug"), + ); + + // Environment variable overrides (highest precedence), merged from role and role group. + // They are merged in last so that they override any operator-set environment variable. + let env_vars = env_vars.merge(inputs.env_overrides.clone()); + + main_container_commands.push(role.main_container_start_command()); + cb_druid + .image_from_product_image(resolved_product_image) + .command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![main_container_commands.join("\n")]) + .add_env_vars(Vec::::from(env_vars)) + .add_container_ports(container_ports(druid_tls_security, role)) + .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) + // 10s * 30 = 300s to come up + .startup_probe(get_tcp_socket_probe(druid_tls_security, 30, 10, 30, 3)) + // 10s * 1 = 10s to get removed from service + .readiness_probe(get_tcp_socket_probe(druid_tls_security, 10, 10, 1, 3)) + // 10s * 3 = 30s to be restarted + .liveness_probe(get_tcp_socket_probe(druid_tls_security, 10, 10, 3, 3)) + .resources(inputs.resources.clone()); + + // Add extra mounts if any are specified and the current role is MiddleManager + // Extra mounts may be needed for ingestion to add required certificates, truststores or similar + // files. + // Mounts are added to all roles, as we are currently unsure where they may be needed + // Known roles are MiddleManagers for ingestion and Historicals for deep storage (GCS plugin) + // We may at some time in the future revisit this and limit it again to avoid needlessly + // propagating potentially confidential files throughout the cluster + for volume in &cluster.cluster_config.extra_volumes { + // Extract values into vars so we make it impossible to log something other than + // what we actually use to create the mounts - maybe paranoid, but hey .. + let volume_name = &volume.name; + let mount_point = format!("{USERDATA_MOUNTPOINT}/{}", volume.name); + + tracing::info!( + ?volume_name, + ?mount_point, + ?role, + "Adding user specified extra volume", + ); + pod_volumes.push(volume.clone()); + cb_druid + .add_volume_mount(volume_name, mount_point) + .context(AddVolumeMountSnafu)?; + } + + let mut containers = vec![cb_druid.build()]; + + // The Vector agent reads the static `vector.yaml` (added to the rolegroup ConfigMap) from the + // config volume; the validated aggregator address comes from the up-front `ValidatedLogging`. + if let Some(vector_log_config) = &inputs.logging.vector_container { + containers.push(vector_container( + Container::Vector.name(), + resolved_product_image, + vector_log_config, + &resource_names, + &DRUID_CONFIG_VOLUME_NAME, + &LOG_VOLUME_NAME, + EnvVarSet::new(), + )); + } + + Ok(ContainersAndVolumes { + init_containers: vec![cb_prepare.build()], + containers, + pod_volumes, + volume_claim_templates, + }) +} + +/// Adds the HDFS discovery ConfigMap volume and its mount if HDFS deep storage is configured. +/// +/// # Panics +/// +/// Panics if the volume mounts cannot be added to the container builder. Only call this on a +/// container builder whose mount paths are still distinct from the ones added here. +fn add_hdfs_cm_volume_and_volume_mounts( + deep_storage_spec: &DeepStorageSpec, + cb_druid: &mut ContainerBuilder, + pod_volumes: &mut Vec, +) { + // hdfs deep storage mount + if let DeepStorageSpec::Hdfs(hdfs) = deep_storage_spec { + cb_druid + .add_volume_mount(&*HDFS_CONFIG_VOLUME_NAME, HDFS_CONFIG_DIRECTORY) + .expect("The mount paths are statically defined and there should be no duplicates."); + pod_volumes.push( + VolumeBuilder::new(&*HDFS_CONFIG_VOLUME_NAME) + .with_config_map(hdfs.config_map_name.to_string()) + .build(), + ); + } +} + +/// Adds the config and rwconfig volumes and their mounts. +/// +/// # Panics +/// +/// Panics if the volume mounts cannot be added to the container builder. Only call this on a +/// container builder whose mount paths are still distinct from the ones added here. +fn add_config_volume_and_volume_mounts( + resource_names: &ResourceNames, + cb_druid: &mut ContainerBuilder, + pod_volumes: &mut Vec, +) { + cb_druid + .add_volume_mount(&*DRUID_CONFIG_VOLUME_NAME, DRUID_CONFIG_DIRECTORY) + .expect("The mount paths are statically defined and there should be no duplicates."); + pod_volumes.push( + VolumeBuilder::new(&*DRUID_CONFIG_VOLUME_NAME) + .with_config_map(resource_names.role_group_config_map().to_string()) + .build(), + ); + cb_druid + .add_volume_mount(&*RW_CONFIG_VOLUME_NAME, RW_CONFIG_DIRECTORY) + .expect("The mount paths are statically defined and there should be no duplicates."); + pod_volumes.push( + VolumeBuilder::new(&*RW_CONFIG_VOLUME_NAME) + .with_empty_dir(Some(""), None) + .build(), + ); +} + +/// Adds the log config ConfigMap volume and its mount. +/// +/// # Panics +/// +/// Panics if the volume mounts cannot be added to the container builder. Only call this on a +/// container builder whose mount paths are still distinct from the ones added here. +fn add_log_config_volume_and_volume_mounts( + resource_names: &ResourceNames, + druid_log_config: &ValidatedContainerLogConfigChoice, + cb_druid: &mut ContainerBuilder, + pod_volumes: &mut Vec, +) { + cb_druid + .add_volume_mount(&*LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIRECTORY) + .expect("The mount paths are statically defined and there should be no duplicates."); + + let config_map = match druid_log_config { + ValidatedContainerLogConfigChoice::Custom(config_map_name) => config_map_name.to_string(), + ValidatedContainerLogConfigChoice::Automatic(_) => { + resource_names.role_group_config_map().to_string() + } + }; + + pod_volumes.push( + VolumeBuilder::new(&*LOG_CONFIG_VOLUME_NAME) + .with_config_map(config_map) + .build(), + ); +} + +/// Adds the log volume and its mounts on the druid and prepare containers. +/// +/// # Panics +/// +/// Panics if the volume mounts cannot be added to the container builders. Only call this on +/// container builders whose mount paths are still distinct from the ones added here. +fn add_log_volume_and_volume_mounts( + cb_druid: &mut ContainerBuilder, + cb_prepare: &mut ContainerBuilder, + pod_volumes: &mut Vec, +) { + cb_druid + .add_volume_mount(&*LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .expect("The mount paths are statically defined and there should be no duplicates."); + cb_prepare + .add_volume_mount(&*LOG_VOLUME_NAME, STACKABLE_LOG_DIR) + .expect("The mount paths are statically defined and there should be no duplicates."); + pod_volumes.push( + VolumeBuilder::new(&*LOG_VOLUME_NAME) + .with_empty_dir( + Some(""), + Some(product_logging::framework::calculate_log_volume_size_limit( + &[MAX_DRUID_LOG_FILES_SIZE], + )), + ) + .build(), + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_constants() { + // Test that dereferencing the constants does not panic. + let _ = *DRUID_CONFIG_VOLUME_NAME; + let _ = *HDFS_CONFIG_VOLUME_NAME; + let _ = *LOG_CONFIG_VOLUME_NAME; + let _ = *LOG_VOLUME_NAME; + let _ = *RW_CONFIG_VOLUME_NAME; + let _ = *CONTAINERDEBUG_LOG_DIRECTORY; + } +} diff --git a/rust/operator-binary/src/controller/build/graceful_shutdown.rs b/rust/operator-binary/src/controller/build/graceful_shutdown.rs index bca1f69b..0f6d3660 100644 --- a/rust/operator-binary/src/controller/build/graceful_shutdown.rs +++ b/rust/operator-binary/src/controller/build/graceful_shutdown.rs @@ -1,102 +1,80 @@ use indoc::formatdoc; -use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::pod::{PodBuilder, container::ContainerBuilder}, k8s_openapi::api::core::v1::{ExecAction, LifecycleHandler}, shared::time::Duration, }; use crate::crd::{DruidRole, security::DruidTlsSecurity}; -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("Failed to set terminationGracePeriod"))] - SetTerminationGracePeriod { - source: stackable_operator::builder::pod::Error, - }, -} - -pub fn add_graceful_shutdown_config( +/// The `druid` container's pre-stop hook that drains a MiddleManager before it stops: the hook +/// disables the MiddleManager so the Overlord sends it no new tasks, then waits until the running +/// tasks finish or the termination grace period elapses. +/// +/// See: +/// +/// The role is read for the port the hook talks to. The `DRUID_PID` file is written by +/// [`DruidRole::main_container_start_command`]. +pub fn drain_pre_stop_hook( role: &DruidRole, tls_security: &DruidTlsSecurity, - graceful_shutdown_timeout: Option, - pod_builder: &mut PodBuilder, - druid_builder: &mut ContainerBuilder, -) -> Result<(), Error> { - // This must be always set by the merge mechanism, as we provide a default value, - // users can not disable graceful shutdown. - if let Some(termination_grace_period) = graceful_shutdown_timeout { - // Every role gets the termination grace period; only the MiddleManager additionally needs a - // pre-stop hook that drains running tasks before the period elapses. - pod_builder - .termination_grace_period(&termination_grace_period) - .context(SetTerminationGracePeriodSnafu)?; + termination_grace_period: &Duration, +) -> LifecycleHandler { + let (protocol, port) = if tls_security.tls_enabled() { + ("https", role.get_https_port()) + } else { + ("http", role.get_http_port()) + }; - if *role == DruidRole::MiddleManager { - let (protocol, port) = if tls_security.tls_enabled() { - ("https", role.get_https_port()) - } else { - ("http", role.get_http_port()) - }; + let middle_manager_host = format!("{protocol}://127.0.0.1:{port}"); + let debug_message = "$(date --utc +%FT%T,%3N) INFO [stackable_lifecycle_pre_stop] -"; + let sleep_interval = 2; - let middle_manager_host = format!("{protocol}://127.0.0.1:{port}"); - let debug_message = "$(date --utc +%FT%T,%3N) INFO [stackable_lifecycle_pre_stop] -"; - let sleep_interval = 2; + LifecycleHandler { + exec: Some(ExecAction { + command: Some(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + formatdoc!( + r#" + log() {{ + echo "{debug_message} $1" >> /proc/$(cat /tmp/DRUID_PID)/fd/1 2>&1 + }} - // The middle manager can be terminated gracefully by disabling it, meaning - // the overlord will not send any new tasks and it will terminate after - // all tasks are finished or the termination grace period is exceeded. - // See: https://druid.apache.org/docs/latest/operations/rolling-updates/#rolling-restart-graceful-termination-based - // The DRUID_PID is set in the crd/src/lib.rs `main_container_start_command` method. - druid_builder.lifecycle_pre_stop(LifecycleHandler { - exec: Some(ExecAction { - command: Some(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - formatdoc!(r#" - log() {{ - echo "{debug_message} $1" >> /proc/$(cat /tmp/DRUID_PID)/fd/1 2>&1 - }} + response=$(curl -v --fail --insecure -X POST {middle_manager_host}/druid/worker/v1/disable) + log "Disable middle manager to stop overlord from sending tasks: $response" - response=$(curl -v --fail --insecure -X POST {middle_manager_host}/druid/worker/v1/disable) - log "Disable middle manager to stop overlord from sending tasks: $response" + end_time_seconds=$(date --date="+{termination_grace_period_seconds} seconds" '+%s') + while : + do + current_time_seconds=$(date '+%s') + log "Check if termination grace period ({termination_grace_period_seconds} seconds) is reached..." + if [ $current_time_seconds -gt $end_time_seconds ] + then + log "The termination grace period is reached!" + break + fi - end_time_seconds=$(date --date="+{termination_grace_period_seconds} seconds" '+%s') - while : - do - current_time_seconds=$(date '+%s') - log "Check if termination grace period ({termination_grace_period_seconds} seconds) is reached..." - if [ $current_time_seconds -gt $end_time_seconds ] - then - log "The termination grace period is reached!" - break - fi + tasks=$(curl -v --fail --insecure -X GET {middle_manager_host}/druid/worker/v1/tasks) + log "Check if all tasks are finished... Running: $tasks" + if [ $tasks = "[]" ] + then + log "All tasks finished!" + break + fi - tasks=$(curl -v --fail --insecure -X GET {middle_manager_host}/druid/worker/v1/tasks) - log "Check if all tasks are finished... Running: $tasks" - if [ $tasks = "[]" ] - then - log "All tasks finished!" - break - fi - - log "Sleeping {sleep_interval} seconds..." - log "" - sleep {sleep_interval} - done - log "All done!" - "#, - termination_grace_period_seconds = termination_grace_period.as_secs() - ), - ]), - }), - ..Default::default() - }); - } + log "Sleeping {sleep_interval} seconds..." + log "" + sleep {sleep_interval} + done + log "All done!" + "#, + termination_grace_period_seconds = termination_grace_period.as_secs() + ), + ]), + }), + ..Default::default() } - - Ok(()) } diff --git a/rust/operator-binary/src/controller/build/jvm.rs b/rust/operator-binary/src/controller/build/jvm.rs index 390356d8..6543090f 100644 --- a/rust/operator-binary/src/controller/build/jvm.rs +++ b/rust/operator-binary/src/controller/build/jvm.rs @@ -7,13 +7,10 @@ use stackable_operator::{ use super::properties::ConfigFileName; use crate::crd::{ - DruidRole, RW_CONFIG_DIRECTORY, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, + RW_CONFIG_DIRECTORY, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, STACKABLE_TRUST_STORE_TYPE, }; -/// The Derby error log file, written by the Coordinator's embedded Derby (default metadata store). -const DERBY_LOG_FILE: &str = "/stackable/var/druid/derby.log"; - /// AWS Region JVM system property, passed to the AWS SDK v2 (used by Druid >= 37.0.0). /// Rendered both into `jvm.config` and into the MiddleManager Peon JVM opts. /// See: @@ -30,11 +27,14 @@ pub enum Error { /// Please note that this function is slightly different than all other operators, because memory /// management is far more advanced in this operator. +/// +/// `role_jvm_args` are the arguments only the role has (the Coordinator's Derby log file); they +/// follow the arguments every role has. pub fn construct_jvm_args( - druid_role: &DruidRole, jvm_argument_overrides: &JvmArgumentOverrides, heap: MemoryQuantity, direct_memory: Option, + role_jvm_args: &[String], s3_conn: Option<&ConnectionSpec>, // TODO (@NickLarsenNZ): Remove this once we don't support Druid less than 37.0.0 druid_version: Result, @@ -74,9 +74,7 @@ pub fn construct_jvm_args( format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), format!("-Djavax.net.ssl.trustStoreType={STACKABLE_TRUST_STORE_TYPE}"), ]); - if druid_role == &DruidRole::Coordinator { - jvm_args.push(format!("-Dderby.stream.error.file={DERBY_LOG_FILE}")); - } + jvm_args.extend(role_jvm_args.iter().cloned()); // TODO (@NickLarsenNZ): Remove the condition (keep the body) once we no longer support Druid // less than 37.0.0 // Druid >= 37.0.0 uses the AWS SDK v2, which requires a region to be set via the JVM system @@ -101,6 +99,7 @@ mod tests { use stackable_operator::v2::types::operator::RoleGroupName; use super::*; + use crate::crd::DruidRole; #[test] fn test_construct_jvm_arguments_defaults() { @@ -158,6 +157,8 @@ mod tests { kind: DruidCluster metadata: name: simple-druid + namespace: default + uid: c27b3971-ca72-42c1-80a4-abdfc1db0ddd spec: image: productVersion: 30.0.0 @@ -265,26 +266,27 @@ mod tests { ); } + /// The rendered `jvm.config` of the role's `default` role group: the heap and direct memory + /// sizes come from the role module, the arguments from [`construct_jvm_args`]. No S3 + /// connection is configured, so the region (and the version gating it) is never rendered. fn construct_jvm_config_for_test(druid_cluster: &str, druid_role: &DruidRole) -> String { - use crate::controller::validate::test_support::druid_from_yaml; + use crate::controller::{ + build::{build, test_support::role_group_config_map}, + validate::test_support::{druid_from_yaml, validated_cluster}, + }; let druid = druid_from_yaml(druid_cluster); - let merged_role = druid.merged_role(druid_role).unwrap(); - let rg = merged_role - .get(&RoleGroupName::from_str("default").unwrap()) - .unwrap(); - let (heap, direct) = rg.config.resources.get_memory_sizes(druid_role).unwrap(); - - construct_jvm_args( + let cluster = validated_cluster(&druid); + let resources = build(&cluster).expect("build succeeds"); + role_group_config_map( + &resources, + &cluster, druid_role, - &rg.product_specific_common_config.jvm_argument_overrides, - heap, - direct, - // No S3 connection in these tests, so the region (and the version gating it) is never - // rendered into the JVM args. - None, - Version::parse("37.0.0"), + &RoleGroupName::from_str("default").expect("valid role group name"), ) - .unwrap() + .data + .as_ref() + .expect("the ConfigMap has data")[&ConfigFileName::JvmConfig.to_string()] + .clone() } } diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs index b93ed917..b4ddb8bf 100644 --- a/rust/operator-binary/src/controller/build/mod.rs +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -8,7 +8,10 @@ use std::{ use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::meta::ObjectMetaBuilder, - k8s_openapi::api::core::v1::Secret, + k8s_openapi::api::{ + apps::v1::StatefulSet, + core::v1::{ConfigMap, Secret, Service}, + }, kube::ResourceExt, kvp::Labels, v2::{ @@ -24,26 +27,33 @@ use stackable_operator::{ use crate::{ controller::{ CONTROLLER_NAME, KubernetesResources, OPERATOR_NAME, PRODUCT_NAME, Prepared, - build::resource::{ - config_map::build_rolegroup_config_map, - discovery::build_discovery_configmap, - listener::{build_group_listener, group_listener_name}, - pdb::build_pdb, - rbac::{build_role_binding, build_service_account}, - service::{build_rolegroup_headless_service, build_rolegroup_metrics_service}, - statefulset::build_rolegroup_statefulset, + build::{ + resource::{ + discovery::build_discovery_configmap, + listener::{build_group_listener, group_listener_name}, + pdb::build_pdb, + rbac::{build_role_binding, build_service_account}, + }, + role_group::{ + RoleGroupBuilder, broker, coordinator, historical, middle_manager, router, + }, }, validate::ValidatedCluster, }, - crd::{COOKIE_PASSPHRASE_SECRET_KEY, security::INTERNAL_INITIAL_CLIENT_PASSWORD_SECRET_KEY}, + crd::{ + COOKIE_PASSPHRASE_SECRET_KEY, DruidRole, + security::INTERNAL_INITIAL_CLIENT_PASSWORD_SECRET_KEY, + }, internal_secret::build_shared_internal_secret_name, }; pub mod authentication; +pub mod container; pub mod graceful_shutdown; pub mod jvm; pub mod properties; pub mod resource; +pub mod role_group; pub mod security; /// Returns an [`ObjectMetaBuilder`] pre-filled with the cluster's namespace, an owner @@ -131,15 +141,10 @@ pub(crate) fn role_group_selector( #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("failed to build ConfigMap for role group {role_group}"))] - ConfigMap { - source: resource::config_map::Error, - role_group: RoleGroupName, - }, - - #[snafu(display("failed to build StatefulSet for role group {role_group}"))] - StatefulSet { - source: resource::statefulset::Error, + #[snafu(display("failed to build the resources of {role} role group {role_group}"))] + RoleGroup { + source: role_group::Error, + role: DruidRole, role_group: RoleGroupName, }, @@ -153,58 +158,99 @@ pub enum Error { /// dereferenced and validated by this point. /// The remaining errors are resource-assembly failures only. pub fn build(cluster: &ValidatedCluster) -> Result, Error> { - let mut stateful_sets = vec![]; - let mut services = vec![]; let mut listeners = vec![]; - let mut config_maps = vec![]; let mut pod_disruption_budgets = vec![]; + let mut built = RoleGroupResources::default(); - for (druid_role, role_group_configs) in &cluster.role_group_configs { - let role_config = cluster.role_config(druid_role); + // Role-level resources first, then each role group's, in `DruidRole` variant order. + if let Some(pdb) = build_pdb(&cluster.coordinator_config.pdb, cluster, &coordinator::ROLE) { + pod_disruption_budgets.push(pdb); + } + listeners.push(build_group_listener( + cluster, + &cluster.coordinator_config.listener_class, + group_listener_name(&cluster.name, &coordinator::ROLE), + &coordinator::ROLE, + )); + for (role_group_name, rg_config) in &cluster.coordinator_role_group_configs { + coordinator::build(cluster, role_group_name, rg_config) + .and_then(|builder| built.add(&builder)) + .with_context(|_| RoleGroupSnafu { + role: coordinator::ROLE, + role_group: role_group_name.clone(), + })?; + } - if let Some(pdb) = build_pdb(&role_config.pdb, cluster, druid_role) { - pod_disruption_budgets.push(pdb); - } + if let Some(pdb) = build_pdb(&cluster.broker_config.pdb, cluster, &broker::ROLE) { + pod_disruption_budgets.push(pdb); + } + listeners.push(build_group_listener( + cluster, + &cluster.broker_config.listener_class, + group_listener_name(&cluster.name, &broker::ROLE), + &broker::ROLE, + )); + for (role_group_name, rg_config) in &cluster.broker_role_group_configs { + broker::build(cluster, role_group_name, rg_config) + .and_then(|builder| built.add(&builder)) + .with_context(|_| RoleGroupSnafu { + role: broker::ROLE, + role_group: role_group_name.clone(), + })?; + } - if let Some(listener_class) = &role_config.listener_class - && let Some(listener_group_name) = group_listener_name(&cluster.name, druid_role) - { - listeners.push(build_group_listener( - cluster, - listener_class, - listener_group_name, - druid_role, - )); - } + if let Some(pdb) = build_pdb(&cluster.historical_config.pdb, cluster, &historical::ROLE) { + pod_disruption_budgets.push(pdb); + } + for (role_group_name, rg_config) in &cluster.historical_role_group_configs { + historical::build(cluster, role_group_name, rg_config) + .and_then(|builder| built.add(&builder)) + .with_context(|_| RoleGroupSnafu { + role: historical::ROLE, + role_group: role_group_name.clone(), + })?; + } - for (role_group_name, rg) in role_group_configs { - services.push(build_rolegroup_headless_service( - cluster, - druid_role, - role_group_name, - )); - services.push(build_rolegroup_metrics_service( - cluster, - druid_role, - role_group_name, - )); - config_maps.push( - build_rolegroup_config_map(cluster, druid_role, role_group_name, rg).context( - ConfigMapSnafu { - role_group: role_group_name.clone(), - }, - )?, - ); - stateful_sets.push( - build_rolegroup_statefulset(cluster, druid_role, role_group_name, rg).context( - StatefulSetSnafu { - role_group: role_group_name.clone(), - }, - )?, - ); - } + if let Some(pdb) = build_pdb( + &cluster.middle_manager_config.pdb, + cluster, + &middle_manager::ROLE, + ) { + pod_disruption_budgets.push(pdb); + } + for (role_group_name, rg_config) in &cluster.middle_manager_role_group_configs { + middle_manager::build(cluster, role_group_name, rg_config) + .and_then(|builder| built.add(&builder)) + .with_context(|_| RoleGroupSnafu { + role: middle_manager::ROLE, + role_group: role_group_name.clone(), + })?; } + if let Some(pdb) = build_pdb(&cluster.router_config.pdb, cluster, &router::ROLE) { + pod_disruption_budgets.push(pdb); + } + listeners.push(build_group_listener( + cluster, + &cluster.router_config.listener_class, + group_listener_name(&cluster.name, &router::ROLE), + &router::ROLE, + )); + for (role_group_name, rg_config) in &cluster.router_role_group_configs { + router::build(cluster, role_group_name, rg_config) + .and_then(|builder| built.add(&builder)) + .with_context(|_| RoleGroupSnafu { + role: router::ROLE, + role_group: role_group_name.clone(), + })?; + } + + let RoleGroupResources { + services, + mut config_maps, + stateful_sets, + } = built; + if let Some(discovery_config_map) = build_discovery_configmap(cluster).context(DiscoveryConfigMapSnafu)? { @@ -224,6 +270,25 @@ pub fn build(cluster: &ValidatedCluster) -> Result }) } +/// The resources of the role groups, in the order the role-group loops of [`build`] add them. +#[derive(Default)] +struct RoleGroupResources { + services: Vec, + config_maps: Vec, + stateful_sets: Vec, +} + +impl RoleGroupResources { + /// Adds the headless and metrics Services, the ConfigMap and the StatefulSet of one role + /// group. + fn add(&mut self, builder: &RoleGroupBuilder) -> Result<(), role_group::Error> { + self.services.extend(builder.build_services()); + self.config_maps.push(builder.build_config_map()?); + self.stateful_sets.push(builder.build_statefulset()?); + Ok(()) + } +} + /// The shared internal Secret, emitted on *every* run so the apply step can track it in /// `ClusterResources` like every other resource (a tracked resource that is not re-added in a /// run would be deleted as an orphan). @@ -335,24 +400,445 @@ fn get_random_base64() -> String { openssl::base64::encode_block(&buf) } +#[cfg(test)] +pub(crate) mod test_support { + use stackable_operator::{ + k8s_openapi::api::{apps::v1::StatefulSet, core::v1::ConfigMap}, + v2::types::operator::RoleGroupName, + }; + + use super::{KubernetesResources, Prepared}; + use crate::{controller::validate::ValidatedCluster, crd::DruidRole}; + + /// The built ConfigMap of the role group, found by the name the builder derives from the + /// cluster. + pub(crate) fn role_group_config_map<'a>( + resources: &'a KubernetesResources, + cluster: &ValidatedCluster, + role: &DruidRole, + role_group_name: &RoleGroupName, + ) -> &'a ConfigMap { + let name = cluster + .role_group_resource_names(role, role_group_name) + .role_group_config_map() + .to_string(); + resources + .config_maps + .iter() + .find(|config_map| config_map.metadata.name.as_deref() == Some(name.as_str())) + .expect("the role group's ConfigMap is built") + } + + /// The built StatefulSet of the role group, found by the name the builder derives from the + /// cluster. + pub(crate) fn role_group_stateful_set<'a>( + resources: &'a KubernetesResources, + cluster: &ValidatedCluster, + role: &DruidRole, + role_group_name: &RoleGroupName, + ) -> &'a StatefulSet { + let name = cluster + .role_group_resource_names(role, role_group_name) + .stateful_set_name() + .to_string(); + resources + .stateful_sets + .iter() + .find(|stateful_set| stateful_set.metadata.name.as_deref() == Some(name.as_str())) + .expect("the role group's StatefulSet is built") + } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; use stackable_operator::{ crd::listener::{self, v1alpha1::Listener}, - k8s_openapi::{ByteString, api::core::v1::Secret}, + k8s_openapi::{ + ByteString, + api::{ + apps::v1::StatefulSetSpec, + core::v1::{Container as PodContainer, PodSpec, Secret}, + }, + }, kube::{Resource, api::ObjectMeta}, + utils::yaml_from_str_singleton_map, + v2::types::operator::RoleGroupName, }; + use strum::IntoEnumIterator; - use super::{KubernetesResources, Prepared, SecretKey, build, internal_secret_keys}; + use super::{ + KubernetesResources, Prepared, SecretKey, build, + container::{ + DRUID_CONFIG_VOLUME_NAME, HDFS_CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, + LOG_VOLUME_NAME, RW_CONFIG_VOLUME_NAME, + }, + internal_secret_keys, + properties::ConfigFileName, + resource::{config_map::INDEXER_JAVA_OPTS, listener::LISTENER_VOLUME_NAME}, + role_group::coordinator::derby_log_jvm_argument, + security::{TLS_MOUNT_VOLUME_NAME, TLS_VOLUME_NAME}, + test_support::{role_group_config_map, role_group_stateful_set}, + }; use crate::{ - controller::validate::test_support::{ - MINIMAL_DRUID_YAML, druid_from_yaml, validated_cluster, + controller::validate::{ + ValidatedCluster, + test_support::{ + MINIMAL_DRUID_YAML, dereferenced_objects, druid_from_yaml, validated_cluster, + validated_cluster_from, + }, + }, + crd::{ + Container, DruidRole, PROP_SEGMENT_CACHE_LOCATIONS, + resource::SEGMENT_CACHE_VOLUME_NAME, security::TLS_PORT_NAME, }, - crd::security::TLS_PORT_NAME, }; + /// The user-supplied extra volume of [`rich_druid_yaml`]. + const EXTRA_VOLUME_NAME: &str = "user-data"; + + /// A cluster that takes every role-specific build path at once: TLS between the roles, OIDC + /// authentication (which skips MiddleManagers), OPA authorization, the Vector agent on every + /// role, a custom log ConfigMap on the Brokers, overrides for the three rendered files and + /// the environment, an extra volume, a Historical segment cache, and two Broker role groups. + /// + /// S3 ingestion is not part of it: the MiddleManager indexer options and the JVM arguments + /// it adds are the same for every role group of the role and are covered by the + /// `ingestion-s3-ext` kuttl test. + fn rich_druid_yaml() -> String { + format!( + r#" +apiVersion: druid.stackable.tech/v1alpha1 +kind: DruidCluster +metadata: + name: simple-druid + namespace: default + uid: c27b3971-ca72-42c1-80a4-abdfc1db0ddd +spec: + image: + productVersion: 30.0.0 + clusterConfig: + authentication: + - authenticationClass: oidc + oidc: + clientCredentialsSecret: oidc-client-credentials + authorization: + opa: + configMapName: opa + package: druid + deepStorage: + hdfs: + configMapName: simple-hdfs + directory: /druid + extraVolumes: + - name: {EXTRA_VOLUME_NAME} + configMap: + name: {EXTRA_VOLUME_NAME} + metadataDatabase: + postgresql: + host: druid-postgresql + database: druid + credentialsSecretName: mySecret + tls: + serverAndInternalSecretClass: tls + vectorAggregatorConfigMapName: vector-aggregator-discovery + zookeeperConfigMapName: simple-druid-znode + brokers: + config: + logging: + enableVectorAgent: true + containers: + druid: + custom: + configMap: druid-log-config + configOverrides: + runtime.properties: + druid.broker.http.numConnections: "10" + security.properties: + networkaddress.cache.ttl: "10" + envOverrides: + BROKER_ENV: broker + roleGroups: + default: + replicas: 1 + secondary: + replicas: 2 + coordinators: + config: + logging: + enableVectorAgent: true + roleGroups: + default: + replicas: 1 + historicals: + config: + logging: + enableVectorAgent: true + resources: + memory: + limit: 4Gi + storage: + segmentCache: + freePercentage: 3 + emptyDir: + capacity: 2G + roleGroups: + default: + replicas: 1 + middleManagers: + config: + logging: + enableVectorAgent: true + roleGroups: + default: + replicas: 1 + routers: + config: + logging: + enableVectorAgent: true + roleGroups: + default: + replicas: 1 +"# + ) + } + + /// The OIDC `AuthenticationClass` referenced by [`rich_druid_yaml`]. + const OIDC_AUTHENTICATION_CLASS_YAML: &str = r#" +metadata: + name: oidc +spec: + provider: + oidc: + hostname: my.oidc.server + principalClaim: preferred_username + scopes: [] +"#; + + /// Builds [`rich_druid_yaml`] through the real validate step with the OIDC + /// `AuthenticationClass` and an OPA document URL dereferenced. + fn rich_validated_cluster() -> ValidatedCluster { + let druid = druid_from_yaml(&rich_druid_yaml()); + let oidc_authentication_class = yaml_from_str_singleton_map(OIDC_AUTHENTICATION_CLASS_YAML) + .expect("the test AuthenticationClass YAML is valid"); + let dereferenced_objects = dereferenced_objects(); + validated_cluster_from( + &druid, + &crate::controller::dereference::DereferencedObjects { + opa_base_document_url: Some( + "http://opa.default.svc.cluster.local:8081/v1/data/druid".to_string(), + ), + authentication_classes: vec![oidc_authentication_class], + ..dereferenced_objects + }, + ) + } + + /// The `default` role group, which every role of [`rich_druid_yaml`] has. + fn default_role_group() -> RoleGroupName { + "default".parse().expect("valid role group name") + } + + fn stateful_set_spec<'a>( + resources: &'a KubernetesResources, + cluster: &ValidatedCluster, + role: &DruidRole, + ) -> &'a StatefulSetSpec { + role_group_stateful_set(resources, cluster, role, &default_role_group()) + .spec + .as_ref() + .expect("the StatefulSet has a spec") + } + + fn pod_spec(spec: &StatefulSetSpec) -> &PodSpec { + spec.template + .spec + .as_ref() + .expect("the pod template has a spec") + } + + fn container<'a>(spec: &'a StatefulSetSpec, name: &str) -> &'a PodContainer { + pod_spec(spec) + .containers + .iter() + .find(|container| container.name == name) + .expect("the container exists") + } + + fn container_mounts(spec: &StatefulSetSpec, name: &str) -> Vec { + container(spec, name) + .volume_mounts + .iter() + .flatten() + .map(|mount| mount.name.clone()) + .collect() + } + + fn has_pod_volume(spec: &StatefulSetSpec, name: &str) -> bool { + pod_spec(spec) + .volumes + .iter() + .flatten() + .any(|volume| volume.name == name) + } + + fn has_claim_template(spec: &StatefulSetSpec, name: &str) -> bool { + spec.volume_claim_templates + .iter() + .flatten() + .any(|pvc| pvc.metadata.name.as_deref() == Some(name)) + } + + fn config_map_data<'a>( + resources: &'a KubernetesResources, + cluster: &ValidatedCluster, + role: &DruidRole, + ) -> &'a BTreeMap { + role_group_config_map(resources, cluster, role, &default_role_group()) + .data + .as_ref() + .expect("the ConfigMap has data") + } + + /// A pod's `volumes` are an ordered list: reordering them changes the pod template of every + /// running StatefulSet and so restarts its pods. Pinned on the fixture that has every + /// optional volume: the operator-managed volumes first, the Historical segment cache after + /// them, the user's extra volumes last. + #[test] + fn pod_volumes_keep_their_order() { + let cluster = rich_validated_cluster(); + let resources = build(&cluster).expect("build succeeds"); + for role in DruidRole::iter() { + let mut expected: Vec = vec![ + TLS_MOUNT_VOLUME_NAME.to_string(), + TLS_VOLUME_NAME.to_string(), + DRUID_CONFIG_VOLUME_NAME.to_string(), + RW_CONFIG_VOLUME_NAME.to_string(), + LOG_CONFIG_VOLUME_NAME.to_string(), + LOG_VOLUME_NAME.to_string(), + HDFS_CONFIG_VOLUME_NAME.to_string(), + ]; + if matches!(role, DruidRole::Historical) { + expected.push(SEGMENT_CACHE_VOLUME_NAME.to_string()); + } + expected.push(EXTRA_VOLUME_NAME.to_string()); + + let actual: Vec = pod_spec(stateful_set_spec(&resources, &cluster, &role)) + .volumes + .iter() + .flatten() + .map(|volume| volume.name.clone()) + .collect(); + assert_eq!(actual, expected, "{role:?} pod volumes"); + } + } + + /// Only Brokers, Coordinators and Routers get a listener claim template. + #[test] + fn only_broker_coordinator_and_router_have_listener_claim_templates() { + let cluster = rich_validated_cluster(); + let resources = build(&cluster).expect("build succeeds"); + let listener_volume_name = LISTENER_VOLUME_NAME.to_string(); + for role in DruidRole::iter() { + let expected = matches!( + role, + DruidRole::Broker | DruidRole::Coordinator | DruidRole::Router + ); + assert_eq!( + has_claim_template( + stateful_set_spec(&resources, &cluster, &role), + &listener_volume_name + ), + expected, + "{role:?} listener claim template" + ); + } + } + + /// Only Historicals get the segment cache: its pod volume, its mount on the `druid` + /// container, and `druid.segmentCache.locations` in runtime.properties. + #[test] + fn only_historicals_have_the_segment_cache() { + let cluster = rich_validated_cluster(); + let resources = build(&cluster).expect("build succeeds"); + let segment_cache_volume_name = SEGMENT_CACHE_VOLUME_NAME.to_string(); + let druid_container_name = Container::Druid.name().to_string(); + for role in DruidRole::iter() { + let expected = matches!(role, DruidRole::Historical); + let spec = stateful_set_spec(&resources, &cluster, &role); + assert_eq!( + has_pod_volume(spec, &segment_cache_volume_name), + expected, + "{role:?} segment cache volume" + ); + assert_eq!( + container_mounts(spec, &druid_container_name).contains(&segment_cache_volume_name), + expected, + "{role:?} segment cache mount" + ); + assert_eq!( + config_map_data(&resources, &cluster, &role) + [&ConfigFileName::RuntimeProperties.to_string()] + .contains(PROP_SEGMENT_CACHE_LOCATIONS), + expected, + "{role:?} segment cache locations" + ); + } + } + + /// Only MiddleManagers set `druid.indexer.runner.javaOptsArray`, in both runtime.properties + /// and security.properties, and only they have a pre-stop hook on the `druid` container. + #[test] + fn only_middle_managers_have_indexer_opts_and_the_drain_hook() { + let cluster = rich_validated_cluster(); + let resources = build(&cluster).expect("build succeeds"); + let druid_container_name = Container::Druid.name().to_string(); + for role in DruidRole::iter() { + let expected = matches!(role, DruidRole::MiddleManager); + let data = config_map_data(&resources, &cluster, &role); + assert_eq!( + data[&ConfigFileName::RuntimeProperties.to_string()].contains(INDEXER_JAVA_OPTS), + expected, + "{role:?} indexer opts in runtime.properties" + ); + assert_eq!( + data[&ConfigFileName::SecurityProperties.to_string()].contains(INDEXER_JAVA_OPTS), + expected, + "{role:?} indexer opts in security.properties" + ); + assert_eq!( + container( + stateful_set_spec(&resources, &cluster, &role), + &druid_container_name + ) + .lifecycle + .as_ref() + .and_then(|lifecycle| lifecycle.pre_stop.as_ref()) + .is_some(), + expected, + "{role:?} pre-stop hook" + ); + } + } + + /// Only the Coordinator's jvm.config carries the Derby error-log argument. + #[test] + fn only_coordinators_set_the_derby_log() { + let cluster = rich_validated_cluster(); + let resources = build(&cluster).expect("build succeeds"); + let derby_log_argument = derby_log_jvm_argument(); + for role in DruidRole::iter() { + assert_eq!( + config_map_data(&resources, &cluster, &role) + [&ConfigFileName::JvmConfig.to_string()] + .contains(&derby_log_argument), + matches!(role, DruidRole::Coordinator), + "{role:?} derby log" + ); + } + } + /// The expected `app.kubernetes.io/version` label value for the given product version. /// /// The `-stackable` suffix carries the operator's own version, which is `0.0.0-dev` on main diff --git a/rust/operator-binary/src/controller/build/resource/config_map.rs b/rust/operator-binary/src/controller/build/resource/config_map.rs index 406f69b1..2cbcb1b2 100644 --- a/rust/operator-binary/src/controller/build/resource/config_map.rs +++ b/rust/operator-binary/src/controller/build/resource/config_map.rs @@ -1,15 +1,19 @@ -//! Builds the rolegroup [`ConfigMap`] from a [`ValidatedCluster`]. +//! Renders a role group's config files from its [`RoleGroupInputs`] and builds the rolegroup +//! [`ConfigMap`] from the [`RoleGroupBuilder`] that carries them. //! -//! The per-file configs (runtime.properties / security.properties / jvm.config) are rendered here -//! from the merged [`DruidRoleGroupConfig`] (config plus the merged config overrides, including the -//! merged JVM argument overrides used for `jvm.config`); the recommended cluster-level runtime -//! properties are carried on `ValidatedCluster`. +//! The per-file configs (runtime.properties / security.properties / jvm.config) are rendered from +//! the inputs (the resolved config plus the merged config overrides, including the merged JVM +//! argument overrides used for `jvm.config`); the recommended cluster-level runtime properties +//! are carried on the inputs' `ValidatedCluster`. What only one role writes (the Historical's +//! resource-derived properties, the MiddleManager's indexer options, the Coordinator's Derby log +//! argument, the role's static defaults) arrives resolved on the inputs and enters at the layer +//! it occupies in the file. //! //! Metadata, owner reference and recommended labels are derived entirely from `ValidatedCluster` //! (which carries the validated name/namespace/uid and implements `Resource`). //! -//! The builder does not read the raw [`crate::crd::v1alpha1::DruidCluster`] at all: everything it -//! needs is carried on `ValidatedCluster` (resolved during the validate step). +//! Nothing here reads the raw [`crate::crd::v1alpha1::DruidCluster`]: everything is carried on +//! the inputs (resolved during the validate step and by the role module). use std::collections::BTreeMap; @@ -20,30 +24,25 @@ use stackable_operator::{ crd::s3, k8s_openapi::api::core::v1::{ConfigMap, EnvVar}, product_logging::framework::VECTOR_CONFIG_FILE, - v2::{config_file_writer::to_java_properties_string, types::operator::RoleGroupName}, + v2::config_file_writer::to_java_properties_string, }; use crate::{ - controller::{ - build::{ - authentication::generate_runtime_properties_config as generate_auth_runtime_properties, - jvm::{AWS_REGION, construct_jvm_args}, - object_meta, - properties::{ - ConfigFileName, - extensions::get_extension_list, - product_logging::{build_log4j2, vector_config_file_content}, - runtime_properties, security_properties, - }, - recommended_labels_for_role_group_resources, - security::add_tls_config_properties, + controller::build::{ + authentication::generate_runtime_properties_config as generate_auth_runtime_properties, + jvm::construct_jvm_args, + object_meta, + properties::{ + ConfigFileName, + extensions::get_extension_list, + product_logging::{build_log4j2, vector_config_file_content}, + runtime_properties, security_properties, }, - validate::{DruidRoleGroupConfig, ValidatedCluster}, - }, - crd::{ - DruidConfigOverrides, DruidRole, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, - STACKABLE_TRUST_STORE_TYPE, build_string_list, env_var_reference, file_reference, + recommended_labels_for_role_group_resources, + role_group::{RoleGroupBuilder, RoleGroupInputs}, + security::add_tls_config_properties, }, + crd::{DruidConfigOverrides, build_string_list, env_var_reference, file_reference}, }; // Druid `runtime.properties` config-property keys assembled into the rolegroup ConfigMap here. @@ -60,10 +59,9 @@ const AUTH_AUTHORIZER_OPA_URI: &str = "druid.auth.authorizer.OpaAuthorizer.opaUr #[derive(Snafu, Debug)] #[allow(clippy::enum_variant_names)] pub enum Error { - #[snafu(display("failed to build ConfigMap for role group {role_group}"))] + #[snafu(display("failed to build ConfigMap"))] BuildRoleGroupConfig { source: stackable_operator::builder::configmap::Error, - role_group: String, }, #[snafu(display("failed to configure S3 connection"))] @@ -76,23 +74,16 @@ pub enum Error { source: stackable_operator::v2::config_file_writer::PropertiesWriterError, }, - #[snafu(display("failed to serialize [security.properties] for {rolegroup}"))] + #[snafu(display("failed to serialize [security.properties]"))] JvmSecurityProperties { source: stackable_operator::v2::config_file_writer::PropertiesWriterError, - rolegroup: String, }, - #[snafu(display("failed to update Druid config from resources"))] - UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, - #[snafu(display("there was an error generating the authentication runtime settings"))] GenerateAuthenticationRuntimeSettings { source: crate::controller::build::authentication::Error, }, - #[snafu(display("failed to derive Druid memory settings from resources"))] - DeriveMemorySettings { source: crate::crd::resource::Error }, - #[snafu(display("failed to construct the jvm.config"))] GetJvmConfig { source: crate::controller::build::jvm::Error, @@ -101,35 +92,10 @@ pub enum Error { type Result = std::result::Result; -const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; - -/// The `druid.indexer.runner.javaOptsArray` entry that must be present in *every* rendered file -/// (runtime.properties and security.properties) for MiddleManagers. -fn middlemanager_indexer_java_opts( - s3_conn: Option<&s3::v1alpha1::ConnectionSpec>, - druid_version: &Result, -) -> (String, String) { - let mut java_opts = vec![ - format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), - format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), - format!("-Djavax.net.ssl.trustStoreType={STACKABLE_TRUST_STORE_TYPE}"), - ]; - - // TODO (@NickLarsenNZ): Remove the condition (keep the body) once we no longer support Druid - // less than 37.0.0. - // Druid >= 37.0.0 uses the AWS SDK v2, which requires a region to be set via the JVM system - // property `aws.region` for the Peon processes spawned by the MiddleManager. - if matches!(druid_version, Ok(v) if *v >= Version::new(37, 0, 0)) - && let Some(s3) = s3_conn - { - java_opts.push(format!( - "-D{AWS_REGION}={region_name}", - region_name = s3.region.name - )); - } - - (INDEXER_JAVA_OPTS.to_string(), build_string_list(&java_opts)) -} +/// The property carrying the JVM options of the tasks a MiddleManager spawns. Its value comes +/// from the MiddleManager role module and is written to both runtime.properties and +/// security.properties. +pub(crate) const INDEXER_JAVA_OPTS: &str = "druid.indexer.runner.javaOptsArray"; /// Returns the user-supplied key/value overrides for the given config file from a /// [`DruidConfigOverrides`]. @@ -146,14 +112,13 @@ fn key_value_overrides( } } -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -pub fn build_rolegroup_config_map( - cluster: &ValidatedCluster, - role: &DruidRole, - role_group_name: &RoleGroupName, - rg: &DruidRoleGroupConfig, -) -> Result { - let resource_names = cluster.role_group_resource_names(role, role_group_name); +/// The role group's rendered config files, keyed by file name: runtime.properties, jvm.config and +/// security.properties, plus log4j2.properties when the operator renders the `druid` container's +/// log config and `vector.yaml` when the Vector agent is enabled. +pub(crate) fn config_map_data(inputs: &RoleGroupInputs) -> Result> { + let cluster = inputs.cluster; + // Read for the TLS port properties. + let role = &inputs.role; let cluster_config = &cluster.cluster_config; let druid_tls_security = &cluster_config.druid_tls_security; let druid_auth_config = &cluster_config.druid_auth_config; @@ -171,14 +136,9 @@ pub fn build_rolegroup_config_map( { let mut conf: BTreeMap = Default::default(); - // Add any properties derived from storage manifests, such as segment cache locations. - // This has to be done here since there is no other suitable place for it. - // Previously such properties were added in the compute_files() function, - // but that code path is now incompatible with the design of fragment merging. - rg.config - .resources - .update_druid_config_file(&mut conf) - .context(UpdateDruidConfigFromResourcesSnafu)?; + // The properties derived from the resources (the Historical segment cache location and + // processing settings) are the lowest layer: every layer below overrides them. + conf.extend(inputs.resource_runtime_properties.clone()); // NOTE: druid.host can be set manually - if it isn't, the canonical host name of // the local host is used. This should work with the agent and k8s host networking // but might need to be revisited in the future @@ -303,25 +263,24 @@ pub fn build_rolegroup_config_map( if let Some(auth_config) = druid_auth_config { conf.extend( - generate_auth_runtime_properties(auth_config, role) + generate_auth_runtime_properties(auth_config, inputs.oidc_enabled) .context(GenerateAuthenticationRuntimeSettingsSnafu)?, ); }; // Role/rolegroup runtime.properties: the recommended cluster-config-derived properties, - // the MiddleManager indexer opts, the per-role defaults and finally the user overrides + // the MiddleManager indexer opts, the role's defaults and finally the user overrides // (each layer wins over the previous, and over the cluster-level properties above). conf.extend(runtime_properties::cluster_runtime_properties( &cluster_config.deep_storage, cluster_config.opa_connection_string.is_some(), )); - if *role == DruidRole::MiddleManager { - let (k, v) = middlemanager_indexer_java_opts(s3_conn, &druid_version); - conf.insert(k, v); + if let Some(indexer_java_opts) = &inputs.indexer_java_opts { + conf.insert(INDEXER_JAVA_OPTS.to_string(), indexer_java_opts.clone()); } - conf.extend(runtime_properties::defaults(role)); + conf.extend(inputs.role_runtime_defaults.clone()); conf.extend(key_value_overrides( - &rg.config_overrides, + &inputs.config_overrides, ConfigFileName::RuntimeProperties, )); @@ -335,16 +294,11 @@ pub fn build_rolegroup_config_map( // ----- jvm.config ----- { - let (heap, direct) = rg - .config - .resources - .get_memory_sizes(role) - .context(DeriveMemorySettingsSnafu)?; let jvm_config = construct_jvm_args( - role, - &rg.product_specific_common_config.jvm_argument_overrides, - heap, - direct, + &inputs.jvm_argument_overrides, + inputs.heap, + inputs.direct_memory, + &inputs.role_jvm_args, s3_conn, // `construct_jvm_args` takes the version by value and `semver::Error` is not `Clone`, // so re-parse here rather than cloning the `druid_version` used by reference above. @@ -357,24 +311,41 @@ pub fn build_rolegroup_config_map( // ----- security.properties ----- { let mut security_config: BTreeMap = BTreeMap::new(); - if *role == DruidRole::MiddleManager { - let (k, v) = middlemanager_indexer_java_opts(s3_conn, &druid_version); - security_config.insert(k, v); + if let Some(indexer_java_opts) = &inputs.indexer_java_opts { + security_config.insert(INDEXER_JAVA_OPTS.to_string(), indexer_java_opts.clone()); } let security_overrides = - key_value_overrides(&rg.config_overrides, ConfigFileName::SecurityProperties); + key_value_overrides(&inputs.config_overrides, ConfigFileName::SecurityProperties); security_config.extend(security_properties::build(security_overrides)); cm_conf_data.insert( ConfigFileName::SecurityProperties.to_string(), - to_java_properties_string(security_config.iter()).with_context(|_| { - JvmSecurityPropertiesSnafu { - rolegroup: role_group_name.to_string(), - } - })?, + to_java_properties_string(security_config.iter()) + .context(JvmSecurityPropertiesSnafu)?, ); } + if let Some(log4j2_config) = build_log4j2(&inputs.logging.druid_container) { + cm_conf_data.insert(ConfigFileName::Log4j2Properties.to_string(), log4j2_config); + } + + // The Vector agent config (`vector.yaml`) is a static, env-var-parameterized file (mirroring + // the hive-/opensearch-operator). It is only added when the Vector agent is enabled. + if inputs.logging.enable_vector_agent { + cm_conf_data.insert(VECTOR_CONFIG_FILE.to_string(), vector_config_file_content()); + } + + Ok(cm_conf_data) +} + +/// The rolegroup [`ConfigMap`]: the rendered config files under the role group's name and +/// recommended labels. +pub(crate) fn build_config_map(builder: &RoleGroupBuilder) -> Result { + let cluster = builder.cluster; + let role = &builder.role; + let role_group_name = &builder.role_group_name; + let resource_names = cluster.role_group_resource_names(role, role_group_name); + let mut config_map_builder = ConfigMapBuilder::new(); config_map_builder.metadata( object_meta( @@ -384,26 +355,13 @@ pub fn build_rolegroup_config_map( ) .build(), ); - - for (filename, file_content) in cm_conf_data.iter() { + for (filename, file_content) in &builder.config_map_data { config_map_builder.add_data(filename, file_content); } - if let Some(log4j2_config) = build_log4j2(&rg.config.logging.druid_container) { - config_map_builder.add_data(ConfigFileName::Log4j2Properties.to_string(), log4j2_config); - } - - // The Vector agent config (`vector.yaml`) is a static, env-var-parameterized file (mirroring - // the hive-/opensearch-operator). It is only added when the Vector agent is enabled. - if rg.config.logging.enable_vector_agent { - config_map_builder.add_data(VECTOR_CONFIG_FILE, vector_config_file_content()); - } - config_map_builder .build() - .with_context(|_| BuildRoleGroupConfigSnafu { - role_group: role_group_name.to_string(), - }) + .context(BuildRoleGroupConfigSnafu) } #[cfg(test)] @@ -415,23 +373,21 @@ mod tests { }; use super::*; - use crate::controller::validate::test_support::{ - MINIMAL_DRUID_YAML, druid_from_yaml, validated_cluster, + use crate::{ + controller::{ + build::{build, test_support::role_group_config_map}, + validate::test_support::{MINIMAL_DRUID_YAML, druid_from_yaml, validated_cluster}, + }, + crd::DruidRole, }; - /// Builds the rolegroup `ConfigMap` for `/` from the minimal test fixture. + /// The rolegroup `ConfigMap` of `/`, built from the minimal test fixture. fn build_cm(role: DruidRole, role_group: &str) -> ConfigMap { let druid = druid_from_yaml(MINIMAL_DRUID_YAML); let cluster = validated_cluster(&druid); - let role_group_name = RoleGroupName::from_str(role_group).unwrap(); - let rg = cluster - .role_group_configs - .get(&role) - .expect("role present") - .get(&role_group_name) - .expect("role group present") - .clone(); - build_rolegroup_config_map(&cluster, &role, &role_group_name, &rg).expect("config map") + let role_group_name = RoleGroupName::from_str(role_group).expect("valid role group name"); + let resources = build(&cluster).expect("build succeeds"); + role_group_config_map(&resources, &cluster, &role, &role_group_name).clone() } #[test] diff --git a/rust/operator-binary/src/controller/build/resource/listener.rs b/rust/operator-binary/src/controller/build/resource/listener.rs index d8d50cdc..7340b4fc 100644 --- a/rust/operator-binary/src/controller/build/resource/listener.rs +++ b/rust/operator-binary/src/controller/build/resource/listener.rs @@ -70,26 +70,9 @@ pub fn build_group_listener_pvc( ) } -/// The name of the group [`Listener`] of the given role, or `None` for roles that expose no -/// Listener. -pub fn group_listener_name( - cluster_name: &ClusterName, - druid_role: &DruidRole, -) -> Option { - match druid_role { - DruidRole::Coordinator | DruidRole::Broker | DruidRole::Router => { - Some(general_group_listener_name(cluster_name, druid_role)) - } - DruidRole::Historical | DruidRole::MiddleManager => None, - } -} - -/// Returns the name of the group listener for a specific role, without -/// checking if the role actually exposes one. -pub fn general_group_listener_name( - cluster_name: &ClusterName, - druid_role: &DruidRole, -) -> ListenerName { +/// The name of the group [`Listener`] of the given role. Only Brokers, Coordinators and Routers +/// expose one; their role modules and `build` ask for it, the dereference step for the Router's. +pub fn group_listener_name(cluster_name: &ClusterName, druid_role: &DruidRole) -> ListenerName { const _: () = assert!( ClusterName::MAX_LENGTH + 1 /* dash */ + RoleName::MAX_LENGTH <= ListenerName::MAX_LENGTH, "The string `-` must not exceed the limit of Listener names." @@ -123,16 +106,6 @@ pub fn build_listener_connection_string( )) } -/// The listener volume name depending on the role -pub fn secret_volume_listener_scope(role: &DruidRole) -> Option { - match role { - DruidRole::Broker | DruidRole::Coordinator | DruidRole::Router => { - Some(LISTENER_VOLUME_NAME.to_string()) - } - DruidRole::Historical | DruidRole::MiddleManager => None, - } -} - #[cfg(test)] mod tests { use std::{collections::BTreeMap, str::FromStr}; @@ -154,33 +127,12 @@ mod tests { let _ = *LISTENER_VOLUME_NAME; } - #[test] - fn group_listener_name_only_for_externally_reachable_roles() { - let cluster_name = cluster().name; - assert!(group_listener_name(&cluster_name, &DruidRole::Broker).is_some()); - assert!(group_listener_name(&cluster_name, &DruidRole::Coordinator).is_some()); - assert!(group_listener_name(&cluster_name, &DruidRole::Router).is_some()); - assert!(group_listener_name(&cluster_name, &DruidRole::Historical).is_none()); - assert!(group_listener_name(&cluster_name, &DruidRole::MiddleManager).is_none()); - } - #[test] fn group_listener_name_is_cluster_and_role_scoped() { - let name = group_listener_name(&cluster().name, &DruidRole::Broker) - .expect("broker has a listener"); + let name = group_listener_name(&cluster().name, &DruidRole::Broker); assert_eq!(name.to_string(), "simple-druid-broker"); } - #[test] - fn secret_volume_listener_scope_only_for_externally_reachable_roles() { - assert_eq!( - secret_volume_listener_scope(&DruidRole::Broker), - Some("listener".to_string()) - ); - assert!(secret_volume_listener_scope(&DruidRole::Historical).is_none()); - assert!(secret_volume_listener_scope(&DruidRole::MiddleManager).is_none()); - } - /// An ingress address exposing both the plaintext and TLS ports, so the connection-string builder's /// port selection can be exercised. fn ingress_with_both_ports() -> ListenerIngress { diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 4b9a5b40..520f7d84 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -1,336 +1,63 @@ -//! Builds the rolegroup [`StatefulSet`] from a [`ValidatedCluster`]. - -use std::str::FromStr; +//! Builds the rolegroup [`StatefulSet`] from a role group's [`RoleGroupBuilder`]. +//! +//! Everything this needs is already resolved on the builder, so it reads straight through: open +//! the pod, add the containers and volumes gathered for the role group, close it. use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{ meta::ObjectMetaBuilder, - pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, volume::VolumeBuilder, - }, + pod::{PodBuilder, security::PodSecurityContextBuilder}, }, constants::RESTART_CONTROLLER_ENABLED_LABEL, k8s_openapi::{ DeepMerge, - api::{ - apps::v1::{StatefulSet, StatefulSetSpec}, - core::v1::{EnvVar, PersistentVolumeClaim}, - }, + api::apps::v1::{StatefulSet, StatefulSetSpec}, apimachinery::pkg::apis::meta::v1::LabelSelector, }, - product_logging, - v2::{ - builder::pod::container::{EnvVarName, EnvVarSet, new_container_builder}, - product_logging::framework::{ - STACKABLE_LOG_DIR, ValidatedContainerLogConfigChoice, vector_container, - }, - role_group_utils::ResourceNames, - types::{kubernetes::VolumeName, operator::RoleGroupName}, - }, }; -use crate::{ - controller::{ - build::{ - authentication, - graceful_shutdown::add_graceful_shutdown_config, - object_meta, - properties::product_logging::MAX_DRUID_LOG_FILES_SIZE, - recommended_labels_for_role_group_resources, - recommended_labels_for_unversioned_role_group_resources, - resource::listener::{ - LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, build_group_listener_pvc, - group_listener_name, secret_volume_listener_scope, - }, - role_group_selector, - security::{ - add_tls_volume_and_volume_mounts, build_tls_key_stores_cmd, container_ports, - get_tcp_socket_probe, - }, - }, - validate::{DruidRoleGroupConfig, ValidatedCluster}, - }, - crd::{ - Container, DRUID_CONFIG_DIRECTORY, DeepStorageSpec, DruidRole, HDFS_CONFIG_DIRECTORY, - LOG_CONFIG_DIRECTORY, METRICS_PORT, METRICS_PORT_NAME, RW_CONFIG_DIRECTORY, - ValidatedDruidConfig, - }, +use crate::controller::build::{ + object_meta, recommended_labels_for_role_group_resources, role_group::RoleGroupBuilder, + role_group_selector, }; -// volume names -stackable_operator::constant!(DRUID_CONFIG_VOLUME_NAME: VolumeName = "config"); -stackable_operator::constant!(HDFS_CONFIG_VOLUME_NAME: VolumeName = "hdfs"); -stackable_operator::constant!(LOG_CONFIG_VOLUME_NAME: VolumeName = "log-config"); -stackable_operator::constant!(LOG_VOLUME_NAME: VolumeName = "log"); -stackable_operator::constant!(RW_CONFIG_VOLUME_NAME: VolumeName = "rwconfig"); - -// Needed for the `containerdebug` process to log its tracing information to. -stackable_operator::constant!(CONTAINERDEBUG_LOG_DIRECTORY: EnvVarName = "CONTAINERDEBUG_LOG_DIRECTORY"); - -// volume mount directory (not a volume name) -const USERDATA_MOUNTPOINT: &str = "/stackable/userdata"; - #[derive(Snafu, Debug)] pub enum Error { - #[snafu(display("failed to configure graceful shutdown"))] - GracefulShutdown { - source: crate::controller::build::graceful_shutdown::Error, - }, - - #[snafu(display("failed to add OIDC Volumes and VolumeMounts to the Pod and containers"))] - AuthVolumesBuild { - source: crate::controller::build::authentication::Error, - }, - - #[snafu(display("failed to initialize security context"))] - FailedToInitializeSecurityContext { - source: crate::controller::build::security::Error, - }, - - #[snafu(display( - "Druid does not support skipping the verification of the tls enabled S3 server" - ))] - S3TlsNoVerificationNotSupported, - - #[snafu(display("failed to configure S3 connection"))] - ConfigureS3 { - source: stackable_operator::crd::s3::v1alpha1::ConnectionError, + #[snafu(display("failed to set the termination grace period"))] + TerminationGracePeriod { + source: stackable_operator::builder::pod::Error, }, - #[snafu(display("failed to update Druid config from resources"))] - UpdateDruidConfigFromResources { source: crate::crd::resource::Error }, - #[snafu(display("failed to add needed volume"))] AddVolume { source: stackable_operator::builder::pod::Error, }, - - #[snafu(display("failed to add needed volumeMount"))] - AddVolumeMount { - source: stackable_operator::builder::pod::container::Error, - }, } type Result = std::result::Result; -pub fn build_rolegroup_statefulset( - cluster: &ValidatedCluster, - role: &DruidRole, - role_group_name: &RoleGroupName, - rg: &DruidRoleGroupConfig, -) -> Result { - let merged_rolegroup_config = &rg.config; +/// The role group's [`StatefulSet`]. +pub(crate) fn build_statefulset(builder: &RoleGroupBuilder) -> Result { + let cluster = builder.cluster; + let role = &builder.role; + let role_group_name = &builder.role_group_name; let resource_names = cluster.role_group_resource_names(role, role_group_name); - // Everything below used to be threaded in as separate parameters; it all lives on the - // `ValidatedCluster` now. - let resolved_product_image = &cluster.image; - let s3_conn = cluster.cluster_config.s3_connection.as_ref(); - let druid_tls_security = &cluster.cluster_config.druid_tls_security; - let druid_auth_config = &cluster.cluster_config.druid_auth_config; - // prepare container builder - let mut cb_prepare = new_container_builder(Container::Prepare.name()); - // druid container builder - let mut cb_druid = new_container_builder(Container::Druid.name()); - // init pod builder - let mut pb = PodBuilder::new(); - pb.affinity(&merged_rolegroup_config.affinity); - add_graceful_shutdown_config( - role, - druid_tls_security, - merged_rolegroup_config.graceful_shutdown_timeout, - &mut pb, - &mut cb_druid, - ) - .context(GracefulShutdownSnafu)?; - - let metadata_database_connection_details = &cluster.cluster_config.metadata_db_connection; - - let mut main_container_commands = role.main_container_prepare_commands(s3_conn); - let mut prepare_container_commands = vec![]; - if let ValidatedContainerLogConfigChoice::Automatic(log_config) = - &merged_rolegroup_config.logging.prepare_container - { - // This command needs to be added at the beginning of the shell commands, - // otherwise the output of the following commands will not be captured! - prepare_container_commands.push(product_logging::framework::capture_shell_output( - STACKABLE_LOG_DIR, - Container::Prepare.name().as_ref(), - log_config, - )); - } - prepare_container_commands.extend(build_tls_key_stores_cmd(druid_tls_security)); - - // Operator-managed volumes and volume mounts first. Their mount paths are constants, so the - // mounts cannot collide with each other and adding them is infallible. Adding the volumes - // stays fallible, because the volumes are built from computed arguments. Volumes and mounts - // derived from user input (authentication, S3, `extraVolumes`) are added afterwards and stay - // fallible, as they can collide with the operator-managed ones. - add_tls_volume_and_volume_mounts( - druid_tls_security, - &mut cb_prepare, - &mut cb_druid, - &mut pb, - &merged_rolegroup_config.requested_secret_lifetime, - // add listener - secret_volume_listener_scope(role), - ) - .context(FailedToInitializeSecurityContextSnafu)?; - add_config_volume_and_volume_mounts(&resource_names, &mut cb_druid, &mut pb)?; - add_log_config_volume_and_volume_mounts( - &resource_names, - merged_rolegroup_config, - &mut cb_druid, - &mut pb, - )?; - add_log_volume_and_volume_mounts(&mut cb_druid, &mut cb_prepare, &mut pb)?; - add_hdfs_cm_volume_and_volume_mounts( - &cluster.cluster_config.deep_storage, - &mut cb_druid, - &mut pb, - )?; - merged_rolegroup_config - .resources - .update_volumes_and_volume_mounts(&mut cb_druid, &mut pb) - .context(UpdateDruidConfigFromResourcesSnafu)?; - - // The listener volume mount is static as well, so it belongs here, before the derived volumes - // and mounts below. The listener volume itself is a PVC template, see `pvcs`. - let mut pvcs: Option> = None; - if let Some(group_listener_name) = group_listener_name(&cluster.name, role) { - cb_druid - .add_volume_mount(&*LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) - .expect("The mount paths are statically defined and there should be no duplicates."); - - // Used for PVC templates, which cannot be modified once they are deployed. The version - // label is omitted so the labels stay stable across version upgrades. - let unversioned_recommended_labels = - recommended_labels_for_unversioned_role_group_resources(cluster, role, role_group_name); - - pvcs = Some(vec![build_group_listener_pvc( - &group_listener_name, - &unversioned_recommended_labels, - )]); - } - - if let Some(auth_config) = druid_auth_config { - authentication::add_volumes_and_mounts( - auth_config, - &mut pb, - &mut cb_druid, - &mut cb_prepare, - ) - .context(AuthVolumesBuildSnafu)?; - prepare_container_commands.extend(authentication::prepare_container_commands(auth_config)); - main_container_commands.extend(authentication::main_container_commands(auth_config)) - } - - if let Some(s3) = s3_conn { - if s3.tls.uses_tls() && !s3.tls.uses_tls_verification() { - S3TlsNoVerificationNotSupportedSnafu.fail()?; - } - s3.add_volumes_and_mounts(&mut pb, vec![&mut cb_druid]) - .context(ConfigureS3Snafu)?; - } - - cb_prepare - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![prepare_container_commands.join("\n")]) - .resources( - ResourceRequirementsBuilder::new() - .with_cpu_request("100m") - .with_cpu_limit("400m") - .with_memory_request("512Mi") - .with_memory_limit("512Mi") - .build(), - ); - - // All operator-set environment variables of the druid container, collected into an - // `EnvVarSet` so that every name occurs only once. - let mut env_vars = EnvVarSet::new(); - - for env_var in metadata_database_connection_details - .username_env - .iter() - .chain(metadata_database_connection_details.password_env.iter()) - { - env_vars = env_vars.with_env_var(env_var.clone()).expect( - "the database credential env var names are generated by operator-rs from the \ - unique database name and are therefore valid", - ); - } - if let Some(auth_config) = druid_auth_config { - env_vars = env_vars.merge(authentication::get_env_var_mounts( - auth_config, - cluster, - role, - )); + let mut pb = PodBuilder::new(); + pb.affinity(&builder.affinity); + // This must be always set by the merge mechanism, as we provide a default value, + // users can not disable graceful shutdown. + if let Some(termination_grace_period) = &builder.graceful_shutdown_timeout { + pb.termination_grace_period(termination_grace_period) + .context(TerminationGracePeriodSnafu)?; } - env_vars = env_vars.with_value( - &CONTAINERDEBUG_LOG_DIRECTORY, - format!("{STACKABLE_LOG_DIR}/containerdebug"), - ); - - // Environment variable overrides (highest precedence), merged from role and role group. - // They are merged in last so that they override any operator-set environment variable. - let env_vars = env_vars.merge(rg.env_overrides.clone()); - - main_container_commands.push(role.main_container_start_command()); - cb_druid - .image_from_product_image(resolved_product_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![main_container_commands.join("\n")]) - .add_env_vars(Vec::::from(env_vars)) - .add_container_ports(container_ports(druid_tls_security, role)) - .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) - // 10s * 30 = 300s to come up - .startup_probe(get_tcp_socket_probe(druid_tls_security, 30, 10, 30, 3)) - // 10s * 1 = 10s to get removed from service - .readiness_probe(get_tcp_socket_probe(druid_tls_security, 10, 10, 1, 3)) - // 10s * 3 = 30s to be restarted - .liveness_probe(get_tcp_socket_probe(druid_tls_security, 10, 10, 3, 3)) - .resources(merged_rolegroup_config.resources.as_resource_requirements()); - - // Add extra mounts if any are specified and the current role is MiddleManager - // Extra mounts may be needed for ingestion to add required certificates, truststores or similar - // files. - // Mounts are added to all roles, as we are currently unsure where they may be needed - // Known roles are MiddleManagers for ingestion and Historicals for deep storage (GCS plugin) - // We may at some time in the future revisit this and limit it again to avoid needlessly - // propagating potentially confidential files throughout the cluster - for volume in &cluster.cluster_config.extra_volumes { - // Extract values into vars so we make it impossible to log something other than - // what we actually use to create the mounts - maybe paranoid, but hey .. - let volume_name = &volume.name; - let mount_point = format!("{USERDATA_MOUNTPOINT}/{}", volume.name); - - tracing::info!( - ?volume_name, - ?mount_point, - ?role, - "Adding user specified extra volume", - ); - pb.add_volume(volume.clone()).context(AddVolumeSnafu)?; - cb_druid - .add_volume_mount(volume_name, mount_point) - .context(AddVolumeMountSnafu)?; - } + // The volumes in the order the builder gathered them, which is part of the pod template + // (pinned by `pod_volumes_keep_their_order` in `build/mod.rs`). A volume whose name collides + // with another volume's is rejected here. + pb.add_volumes(builder.pod_volumes.clone()) + .context(AddVolumeSnafu)?; let metadata = ObjectMetaBuilder::new() .with_labels(recommended_labels_for_role_group_resources( @@ -340,9 +67,7 @@ pub fn build_rolegroup_statefulset( )) .build(); - pb.image_pull_secrets_from_product_image(resolved_product_image) - .add_init_container(cb_prepare.build()) - .add_container(cb_druid.build()) + pb.image_pull_secrets_from_product_image(&cluster.image) .metadata(metadata) .service_account_name( cluster @@ -355,24 +80,16 @@ pub fn build_rolegroup_statefulset( .fs_group(1000) .build(), ); - - // The Vector agent reads the static `vector.yaml` (added to the rolegroup ConfigMap) from the - // config volume; the validated aggregator address comes from the up-front `ValidatedLogging`. - if let Some(vector_log_config) = &merged_rolegroup_config.logging.vector_container { - pb.add_container(vector_container( - Container::Vector.name(), - resolved_product_image, - vector_log_config, - &resource_names, - &DRUID_CONFIG_VOLUME_NAME, - &LOG_VOLUME_NAME, - EnvVarSet::new(), - )); + for init_container in &builder.init_containers { + pb.add_init_container(init_container.clone()); + } + for container in &builder.containers { + pb.add_container(container.clone()); } let mut pod_template = pb.build_template(); // The role and rolegroup pod overrides were already merged (rolegroup wins) during validation. - pod_template.merge_from(rg.pod_overrides.clone()); + pod_template.merge_from(builder.pod_overrides.clone()); Ok(StatefulSet { metadata: object_meta( @@ -386,208 +103,79 @@ pub fn build_rolegroup_statefulset( pod_management_policy: Some("Parallel".to_string()), // Leave `replicas` unset when the role group does not specify a count, so a // HorizontalPodAutoscaler can own the replica count without the operator fighting it. - replicas: rg.replicas.map(i32::from), + replicas: builder.replicas.map(i32::from), selector: LabelSelector { match_labels: Some(role_group_selector(cluster, role, role_group_name).into()), ..LabelSelector::default() }, service_name: Some(resource_names.headless_service_name().to_string()), template: pod_template, - volume_claim_templates: pvcs, + volume_claim_templates: builder.volume_claim_templates.clone(), ..StatefulSetSpec::default() }), status: None, }) } -/// Adds the HDFS discovery ConfigMap volume and its mount if HDFS deep storage is configured. -/// -/// # Panics -/// -/// Panics if the volume mounts cannot be added to the container builder. Only call this on a -/// container builder whose mount paths are still distinct from the ones added here. -fn add_hdfs_cm_volume_and_volume_mounts( - deep_storage_spec: &DeepStorageSpec, - cb_druid: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - // hdfs deep storage mount - if let DeepStorageSpec::Hdfs(hdfs) = deep_storage_spec { - cb_druid - .add_volume_mount(&*HDFS_CONFIG_VOLUME_NAME, HDFS_CONFIG_DIRECTORY) - .expect("The mount paths are statically defined and there should be no duplicates."); - pb.add_volume( - VolumeBuilder::new(&*HDFS_CONFIG_VOLUME_NAME) - .with_config_map(hdfs.config_map_name.to_string()) - .build(), - ) - .context(AddVolumeSnafu)?; - } - - Ok(()) -} - -/// Adds the role group ConfigMap volume, the writable config volume and their mounts. -/// -/// # Panics -/// -/// Panics if the volume mounts cannot be added to the container builder. Only call this on a -/// container builder whose mount paths are still distinct from the ones added here. -fn add_config_volume_and_volume_mounts( - resource_names: &ResourceNames, - cb_druid: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - cb_druid - .add_volume_mount(&*DRUID_CONFIG_VOLUME_NAME, DRUID_CONFIG_DIRECTORY) - .expect("The mount paths are statically defined and there should be no duplicates."); - pb.add_volume( - VolumeBuilder::new(&*DRUID_CONFIG_VOLUME_NAME) - .with_config_map(resource_names.role_group_config_map().to_string()) - .build(), - ) - .context(AddVolumeSnafu)?; - cb_druid - .add_volume_mount(&*RW_CONFIG_VOLUME_NAME, RW_CONFIG_DIRECTORY) - .expect("The mount paths are statically defined and there should be no duplicates."); - pb.add_volume( - VolumeBuilder::new(&*RW_CONFIG_VOLUME_NAME) - .with_empty_dir(Some(""), None) - .build(), - ) - .context(AddVolumeSnafu)?; - - Ok(()) -} - -/// Adds the log config ConfigMap volume and its mount. -/// -/// # Panics -/// -/// Panics if the volume mounts cannot be added to the container builder. Only call this on a -/// container builder whose mount paths are still distinct from the ones added here. -fn add_log_config_volume_and_volume_mounts( - resource_names: &ResourceNames, - merged_rolegroup_config: &ValidatedDruidConfig, - cb_druid: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - cb_druid - .add_volume_mount(&*LOG_CONFIG_VOLUME_NAME, LOG_CONFIG_DIRECTORY) - .expect("The mount paths are statically defined and there should be no duplicates."); - - let config_map = match &merged_rolegroup_config.logging.druid_container { - ValidatedContainerLogConfigChoice::Custom(config_map_name) => config_map_name.to_string(), - ValidatedContainerLogConfigChoice::Automatic(_) => { - resource_names.role_group_config_map().to_string() - } - }; - - pb.add_volume( - VolumeBuilder::new(&*LOG_CONFIG_VOLUME_NAME) - .with_config_map(config_map) - .build(), - ) - .context(AddVolumeSnafu)?; - - Ok(()) -} - -/// Adds the log volume and its mounts on the druid and prepare containers. -/// -/// # Panics -/// -/// Panics if the volume mounts cannot be added to the container builders. Only call this on -/// container builders whose mount paths are still distinct from the ones added here. -fn add_log_volume_and_volume_mounts( - cb_druid: &mut ContainerBuilder, - cb_prepare: &mut ContainerBuilder, - pb: &mut PodBuilder, -) -> Result<()> { - cb_druid - .add_volume_mount(&*LOG_VOLUME_NAME, STACKABLE_LOG_DIR) - .expect("The mount paths are statically defined and there should be no duplicates."); - cb_prepare - .add_volume_mount(&*LOG_VOLUME_NAME, STACKABLE_LOG_DIR) - .expect("The mount paths are statically defined and there should be no duplicates."); - pb.add_volume( - VolumeBuilder::new(&*LOG_VOLUME_NAME) - .with_empty_dir( - Some(""), - Some(product_logging::framework::calculate_log_volume_size_limit( - &[MAX_DRUID_LOG_FILES_SIZE], - )), - ) - .build(), - ) - .context(AddVolumeSnafu)?; - - Ok(()) -} - #[cfg(test)] mod tests { + use std::str::FromStr; + use stackable_operator::{ k8s_openapi::api::core::v1::{ConfigMapVolumeSource, Volume}, v2::types::operator::RoleGroupName, }; use super::*; - use crate::controller::validate::test_support::{ - MINIMAL_DRUID_YAML, druid_from_yaml, validated_cluster, + use crate::{ + controller::{ + build::{ + Error as BuildError, build, + container::{CONTAINERDEBUG_LOG_DIRECTORY, LOG_VOLUME_NAME}, + role_group, + test_support::role_group_stateful_set, + }, + validate::test_support::{MINIMAL_DRUID_YAML, druid_from_yaml, validated_cluster}, + }, + crd::{Container, DruidRole}, }; - #[test] - fn test_constants() { - // Test that dereferencing the constants does not panic. - let _ = *DRUID_CONFIG_VOLUME_NAME; - let _ = *HDFS_CONFIG_VOLUME_NAME; - let _ = *LOG_CONFIG_VOLUME_NAME; - let _ = *LOG_VOLUME_NAME; - let _ = *RW_CONFIG_VOLUME_NAME; - let _ = *CONTAINERDEBUG_LOG_DIRECTORY; - } - /// The user-supplied `envOverrides` must be merged in after all operator-set environment /// variables, so that they can override any of them. `CONTAINERDEBUG_LOG_DIRECTORY` is used /// as the example here because it is set unconditionally by the operator. #[test] fn env_overrides_override_operator_set_env_vars() { - let druid = druid_from_yaml(MINIMAL_DRUID_YAML); + let druid = druid_from_yaml(&MINIMAL_DRUID_YAML.replace( + " brokers:\n", + " brokers:\n envOverrides:\n CONTAINERDEBUG_LOG_DIRECTORY: /custom/log/dir\n", + )); let cluster = validated_cluster(&druid); - let role_group_name = RoleGroupName::from_str("default").expect("valid role group name"); - let mut rg = cluster - .role_group_configs - .get(&DruidRole::Broker) - .expect("broker role groups") - .get(&role_group_name) - .expect("default role group") - .clone(); - rg.env_overrides = EnvVarSet::new().with_value( - &EnvVarName::from_str("CONTAINERDEBUG_LOG_DIRECTORY").expect("valid env var name"), - "/custom/log/dir", - ); + let resources = build(&cluster).expect("build succeeds"); - let stateful_set = - build_rolegroup_statefulset(&cluster, &DruidRole::Broker, &role_group_name, &rg) - .expect("the StatefulSet builds"); - - let env = stateful_set - .spec - .expect("the StatefulSet has a spec") - .template - .spec - .expect("the pod template has a spec") - .containers - .into_iter() - .find(|container| container.name == "druid") - .expect("the druid container exists") - .env - .expect("the druid container has env vars"); + let env = role_group_stateful_set( + &resources, + &cluster, + &DruidRole::Broker, + &RoleGroupName::from_str("default").expect("valid role group name"), + ) + .spec + .as_ref() + .expect("the StatefulSet has a spec") + .template + .spec + .as_ref() + .expect("the pod template has a spec") + .containers + .iter() + .find(|container| container.name == Container::Druid.name().as_ref()) + .expect("the druid container exists") + .env + .as_ref() + .expect("the druid container has env vars"); let containerdebug: Vec<_> = env .iter() - .filter(|env_var| env_var.name == "CONTAINERDEBUG_LOG_DIRECTORY") + .filter(|env_var| env_var.name == CONTAINERDEBUG_LOG_DIRECTORY.as_ref()) .collect(); assert_eq!( containerdebug.len(), @@ -598,23 +186,28 @@ mod tests { } /// A user-supplied extra volume whose name collides with an operator-managed volume must be - /// reported as an error (the operator's own volumes are added first and are infallible, so - /// the collision must surface on the user-supplied side, never as a panic). + /// reported as an error (the operator's own volumes are gathered first, so the collision + /// must surface on the user-supplied side when the volumes are added to the pod, never as a + /// panic). #[test] fn extra_volume_colliding_with_operator_volume_is_an_error() { let mut druid = druid_from_yaml(MINIMAL_DRUID_YAML); druid.spec.cluster_config.extra_volumes = vec![user_volume(LOG_VOLUME_NAME.as_ref())]; let cluster = validated_cluster(&druid); - let role_group_name = RoleGroupName::from_str("default").expect("valid role group name"); - let rg = broker_default_role_group(&cluster, &role_group_name); - let Err(error) = - build_rolegroup_statefulset(&cluster, &DruidRole::Broker, &role_group_name, &rg) - else { + let Err(error) = build(&cluster) else { panic!("the colliding extra volume must be rejected"); }; assert!( - matches!(error, Error::AddVolume { .. }), + matches!( + error, + BuildError::RoleGroup { + source: role_group::Error::StatefulSet { + source: Error::AddVolume { .. }, + }, + .. + } + ), "unexpected error: {error:?}" ); } @@ -629,17 +222,4 @@ mod tests { ..Volume::default() } } - - fn broker_default_role_group( - cluster: &ValidatedCluster, - role_group_name: &RoleGroupName, - ) -> DruidRoleGroupConfig { - cluster - .role_group_configs - .get(&DruidRole::Broker) - .expect("broker role groups") - .get(role_group_name) - .expect("default role group") - .clone() - } } diff --git a/rust/operator-binary/src/controller/build/role_group/broker.rs b/rust/operator-binary/src/controller/build/role_group/broker.rs new file mode 100644 index 00000000..e61bf4bc --- /dev/null +++ b/rust/operator-binary/src/controller/build/role_group/broker.rs @@ -0,0 +1,69 @@ +//! Broker role groups. + +use std::collections::BTreeMap; + +use snafu::{OptionExt, ResultExt}; +use stackable_operator::{memory::MemoryQuantity, v2::types::operator::RoleGroupName}; + +use super::{ + Error, HeapMemorySnafu, MissingSecretLifetimeSnafu, RoleGroupBuilder, RoleGroupInputs, +}; +use crate::{ + controller::{ + build::{properties::runtime_properties, resource::listener::group_listener_name}, + validate::{BrokerRoleGroupConfig, ValidatedCluster}, + }, + crd::{BrokerConfig, DruidRole, resource::heap_memory}, +}; + +pub(crate) const ROLE: DruidRole = DruidRole::Broker; + +/// Resolves one Broker role group into its [`RoleGroupBuilder`]. +pub(crate) fn build<'a>( + cluster: &'a ValidatedCluster, + role_group_name: &RoleGroupName, + rg_config: &BrokerRoleGroupConfig, +) -> Result, Error> { + let BrokerConfig { + resources, + // Validated up front into `rg_config.config.logging`. + logging: _, + affinity, + graceful_shutdown_timeout, + requested_secret_lifetime, + } = &rg_config.config.config; + + // Brokers reserve 400 MiB of direct memory for their query processing buffers. + let direct_memory = MemoryQuantity::from_mebi(400.); + let heap = heap_memory(resources, Some(direct_memory)).context(HeapMemorySnafu)?; + + RoleGroupBuilder::new(RoleGroupInputs { + cluster, + role: ROLE, + role_group_name: role_group_name.clone(), + affinity: affinity.clone(), + graceful_shutdown_timeout: *graceful_shutdown_timeout, + requested_secret_lifetime: requested_secret_lifetime.context(MissingSecretLifetimeSnafu)?, + resources: resources.clone().into(), + heap, + direct_memory: Some(direct_memory), + logging: rg_config.config.logging.clone(), + listener: Some(group_listener_name(&cluster.name, &ROLE)), + role_pod_volumes: Vec::new(), + role_druid_mounts: Vec::new(), + resource_runtime_properties: BTreeMap::new(), + indexer_java_opts: None, + role_jvm_args: Vec::new(), + druid_pre_stop: None, + oidc_enabled: true, + role_runtime_defaults: runtime_properties::defaults(&ROLE), + replicas: rg_config.replicas, + config_overrides: rg_config.config_overrides.clone(), + env_overrides: rg_config.env_overrides.clone(), + pod_overrides: rg_config.pod_overrides.clone(), + jvm_argument_overrides: rg_config + .product_specific_common_config + .jvm_argument_overrides + .clone(), + }) +} diff --git a/rust/operator-binary/src/controller/build/role_group/coordinator.rs b/rust/operator-binary/src/controller/build/role_group/coordinator.rs new file mode 100644 index 00000000..b38e0ed3 --- /dev/null +++ b/rust/operator-binary/src/controller/build/role_group/coordinator.rs @@ -0,0 +1,77 @@ +//! Coordinator role groups. + +use std::collections::BTreeMap; + +use snafu::{OptionExt, ResultExt}; +use stackable_operator::v2::types::operator::RoleGroupName; + +use super::{ + Error, HeapMemorySnafu, MissingSecretLifetimeSnafu, RoleGroupBuilder, RoleGroupInputs, +}; +use crate::{ + controller::{ + build::{properties::runtime_properties, resource::listener::group_listener_name}, + validate::{CoordinatorRoleGroupConfig, ValidatedCluster}, + }, + crd::{CoordinatorConfig, DruidRole, resource::heap_memory}, +}; + +pub(crate) const ROLE: DruidRole = DruidRole::Coordinator; + +/// The error log file of the Coordinator's embedded Derby, the default metadata store. +const DERBY_LOG_FILE: &str = "/stackable/var/druid/derby.log"; + +/// The JVM argument that points Derby's error log at [`DERBY_LOG_FILE`]. +pub(crate) fn derby_log_jvm_argument() -> String { + format!("-Dderby.stream.error.file={DERBY_LOG_FILE}") +} + +/// Resolves one Coordinator role group into its [`RoleGroupBuilder`]. +pub(crate) fn build<'a>( + cluster: &'a ValidatedCluster, + role_group_name: &RoleGroupName, + rg_config: &CoordinatorRoleGroupConfig, +) -> Result, Error> { + let CoordinatorConfig { + resources, + // Validated up front into `rg_config.config.logging`. + logging: _, + affinity, + graceful_shutdown_timeout, + requested_secret_lifetime, + } = &rg_config.config.config; + + // The Coordinator needs no direct memory, so the whole allocatable memory is heap. + let heap = heap_memory(resources, None).context(HeapMemorySnafu)?; + + RoleGroupBuilder::new(RoleGroupInputs { + cluster, + role: ROLE, + role_group_name: role_group_name.clone(), + affinity: affinity.clone(), + graceful_shutdown_timeout: *graceful_shutdown_timeout, + requested_secret_lifetime: requested_secret_lifetime.context(MissingSecretLifetimeSnafu)?, + resources: resources.clone().into(), + heap, + direct_memory: None, + logging: rg_config.config.logging.clone(), + listener: Some(group_listener_name(&cluster.name, &ROLE)), + role_pod_volumes: Vec::new(), + role_druid_mounts: Vec::new(), + resource_runtime_properties: BTreeMap::new(), + indexer_java_opts: None, + // The Coordinator's embedded Derby (the default metadata store) logs its errors to a file. + role_jvm_args: vec![derby_log_jvm_argument()], + druid_pre_stop: None, + oidc_enabled: true, + role_runtime_defaults: runtime_properties::defaults(&ROLE), + replicas: rg_config.replicas, + config_overrides: rg_config.config_overrides.clone(), + env_overrides: rg_config.env_overrides.clone(), + pod_overrides: rg_config.pod_overrides.clone(), + jvm_argument_overrides: rg_config + .product_specific_common_config + .jvm_argument_overrides + .clone(), + }) +} diff --git a/rust/operator-binary/src/controller/build/role_group/historical.rs b/rust/operator-binary/src/controller/build/role_group/historical.rs new file mode 100644 index 00000000..31dad47e --- /dev/null +++ b/rust/operator-binary/src/controller/build/role_group/historical.rs @@ -0,0 +1,83 @@ +//! Historical role groups. + +use std::collections::BTreeMap; + +use snafu::{OptionExt, ResultExt}; +use stackable_operator::v2::types::operator::RoleGroupName; + +use super::{ + DeriveMemorySettingsSnafu, Error, MissingSecretLifetimeSnafu, RoleGroupBuilder, RoleGroupInputs, +}; +use crate::{ + controller::{ + build::properties::runtime_properties, + validate::{HistoricalRoleGroupConfig, ValidatedCluster}, + }, + crd::{ + DruidRole, HistoricalConfig, PROP_SEGMENT_CACHE_LOCATIONS, + memory::HistoricalDerivedSettings, + resource::{segment_cache_locations, segment_cache_volume, segment_cache_volume_mount}, + }, +}; + +pub(crate) const ROLE: DruidRole = DruidRole::Historical; + +/// Resolves one Historical role group into its [`RoleGroupBuilder`]. +/// +/// Historicals are the one role whose JVM memory and `runtime.properties` processing settings +/// are derived from the resources, and the one role with a segment cache volume. +pub(crate) fn build<'a>( + cluster: &'a ValidatedCluster, + role_group_name: &RoleGroupName, + rg_config: &HistoricalRoleGroupConfig, +) -> Result, Error> { + let HistoricalConfig { + resources, + // Validated up front into `rg_config.config.logging`. + logging: _, + affinity, + graceful_shutdown_timeout, + requested_secret_lifetime, + } = &rg_config.config.config; + + let derived_settings = + HistoricalDerivedSettings::try_from(resources).context(DeriveMemorySettingsSnafu)?; + + let segment_cache = &resources.storage.segment_cache; + let mut resource_runtime_properties = BTreeMap::new(); + resource_runtime_properties.insert( + PROP_SEGMENT_CACHE_LOCATIONS.to_string(), + segment_cache_locations(segment_cache), + ); + derived_settings.add_settings(&mut resource_runtime_properties); + + RoleGroupBuilder::new(RoleGroupInputs { + cluster, + role: ROLE, + role_group_name: role_group_name.clone(), + affinity: affinity.clone(), + graceful_shutdown_timeout: *graceful_shutdown_timeout, + requested_secret_lifetime: requested_secret_lifetime.context(MissingSecretLifetimeSnafu)?, + resources: resources.clone().into(), + heap: derived_settings.heap_memory(), + direct_memory: Some(derived_settings.direct_access_memory()), + logging: rg_config.config.logging.clone(), + listener: None, + role_pod_volumes: vec![segment_cache_volume(segment_cache)], + role_druid_mounts: vec![segment_cache_volume_mount()], + resource_runtime_properties, + indexer_java_opts: None, + role_jvm_args: Vec::new(), + druid_pre_stop: None, + oidc_enabled: true, + role_runtime_defaults: runtime_properties::defaults(&ROLE), + replicas: rg_config.replicas, + config_overrides: rg_config.config_overrides.clone(), + env_overrides: rg_config.env_overrides.clone(), + pod_overrides: rg_config.pod_overrides.clone(), + jvm_argument_overrides: rg_config + .product_specific_common_config + .jvm_argument_overrides + .clone(), + }) +} diff --git a/rust/operator-binary/src/controller/build/role_group/middle_manager.rs b/rust/operator-binary/src/controller/build/role_group/middle_manager.rs new file mode 100644 index 00000000..f840f41f --- /dev/null +++ b/rust/operator-binary/src/controller/build/role_group/middle_manager.rs @@ -0,0 +1,117 @@ +//! MiddleManager role groups. + +use std::collections::BTreeMap; + +use semver::Version; +use snafu::{OptionExt, ResultExt}; +use stackable_operator::{crd::s3, v2::types::operator::RoleGroupName}; + +use super::{ + Error, HeapMemorySnafu, MissingSecretLifetimeSnafu, RoleGroupBuilder, RoleGroupInputs, +}; +use crate::{ + controller::{ + build::{ + graceful_shutdown::drain_pre_stop_hook, jvm::AWS_REGION, properties::runtime_properties, + }, + validate::{MiddleManagerRoleGroupConfig, ValidatedCluster}, + }, + crd::{ + DruidRole, MiddleManagerConfig, STACKABLE_TRUST_STORE, STACKABLE_TRUST_STORE_PASSWORD, + STACKABLE_TRUST_STORE_TYPE, build_string_list, resource::heap_memory, + }, +}; + +pub(crate) const ROLE: DruidRole = DruidRole::MiddleManager; + +/// The value of `druid.indexer.runner.javaOptsArray`: the JVM options of the tasks a +/// MiddleManager spawns, which run in their own JVMs. +fn indexer_java_opts( + s3_conn: Option<&s3::v1alpha1::ConnectionSpec>, + druid_version: &Result, +) -> String { + let mut java_opts = vec![ + format!("-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}"), + format!("-Djavax.net.ssl.trustStorePassword={STACKABLE_TRUST_STORE_PASSWORD}"), + format!("-Djavax.net.ssl.trustStoreType={STACKABLE_TRUST_STORE_TYPE}"), + ]; + + // TODO (@NickLarsenNZ): Remove the condition (keep the body) once we no longer support Druid + // less than 37.0.0. + // Druid >= 37.0.0 uses the AWS SDK v2, which requires a region to be set via the JVM system + // property `aws.region` for the Peon processes spawned by the MiddleManager. + if matches!(druid_version, Ok(v) if *v >= Version::new(37, 0, 0)) + && let Some(s3) = s3_conn + { + java_opts.push(format!( + "-D{AWS_REGION}={region_name}", + region_name = s3.region.name + )); + } + + build_string_list(&java_opts) +} + +/// Resolves one MiddleManager role group into its [`RoleGroupBuilder`]. +pub(crate) fn build<'a>( + cluster: &'a ValidatedCluster, + role_group_name: &RoleGroupName, + rg_config: &MiddleManagerRoleGroupConfig, +) -> Result, Error> { + let MiddleManagerConfig { + resources, + // Validated up front into `rg_config.config.logging`. + logging: _, + affinity, + graceful_shutdown_timeout, + requested_secret_lifetime, + } = &rg_config.config.config; + + // The MiddleManager needs no direct memory, so the whole allocatable memory is heap. + let heap = heap_memory(resources, None).context(HeapMemorySnafu)?; + + RoleGroupBuilder::new(RoleGroupInputs { + cluster, + role: ROLE, + role_group_name: role_group_name.clone(), + affinity: affinity.clone(), + graceful_shutdown_timeout: *graceful_shutdown_timeout, + requested_secret_lifetime: requested_secret_lifetime.context(MissingSecretLifetimeSnafu)?, + resources: resources.clone().into(), + heap, + direct_memory: None, + logging: rg_config.config.logging.clone(), + listener: None, + role_pod_volumes: Vec::new(), + role_druid_mounts: Vec::new(), + resource_runtime_properties: BTreeMap::new(), + // The tasks a MiddleManager spawns run in their own JVMs, which need the trust store (and + // the S3 region for the AWS SDK v2). + indexer_java_opts: Some(indexer_java_opts( + cluster.cluster_config.s3_connection.as_ref(), + &Version::parse(&cluster.image.product_version), + )), + role_jvm_args: Vec::new(), + // MiddleManagers drain their running tasks before they stop. The hook needs the + // termination grace period, which the config merge always sets. + druid_pre_stop: graceful_shutdown_timeout.map(|termination_grace_period| { + drain_pre_stop_hook( + &ROLE, + &cluster.cluster_config.druid_tls_security, + &termination_grace_period, + ) + }), + // End users never talk to MiddleManagers directly, and OIDC on them breaks their + // communication with the Coordinators during ingestion. + oidc_enabled: false, + role_runtime_defaults: runtime_properties::defaults(&ROLE), + replicas: rg_config.replicas, + config_overrides: rg_config.config_overrides.clone(), + env_overrides: rg_config.env_overrides.clone(), + pod_overrides: rg_config.pod_overrides.clone(), + jvm_argument_overrides: rg_config + .product_specific_common_config + .jvm_argument_overrides + .clone(), + }) +} diff --git a/rust/operator-binary/src/controller/build/role_group/mod.rs b/rust/operator-binary/src/controller/build/role_group/mod.rs new file mode 100644 index 00000000..693b54ad --- /dev/null +++ b/rust/operator-binary/src/controller/build/role_group/mod.rs @@ -0,0 +1,212 @@ +//! Building the Kubernetes resources of one role group. +//! +//! Two phases, so neither has to reach into the other. First each role's module gathers its +//! [`RoleGroupInputs`]: the values every role has, resolved from its typed role-group config, plus +//! the values only that role has (its group Listener, the Historical segment cache and +//! resource-derived `runtime.properties` entries, the MiddleManager indexer options and drain +//! hook, the Coordinator's Derby log argument). [`RoleGroupBuilder::new`] turns those into +//! finished containers, pod volumes and rendered `ConfigMap` entries. Then [`RoleGroupBuilder`]'s +//! `build_*` methods emit the Kubernetes objects, reading straight through fields that are already +//! resolved. +//! +//! Every Druid role runs the same containers (the `prepare` init container, the `druid` container +//! and the Vector sidecar), so the resolve and emit phases live here once. Everything that differs +//! between roles happens in the five role modules' `build` functions. + +use std::collections::BTreeMap; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + commons::affinity::StackableAffinity, + k8s_openapi::api::{ + apps::v1::StatefulSet, + core::v1::{ + ConfigMap, Container, LifecycleHandler, PersistentVolumeClaim, PodTemplateSpec, + ResourceRequirements, Service, Volume, VolumeMount, + }, + }, + memory::MemoryQuantity, + shared::time::Duration, + v2::{ + builder::pod::container::EnvVarSet, + jvm_argument_overrides::JvmArgumentOverrides, + types::{kubernetes::ListenerName, operator::RoleGroupName}, + }, +}; + +use super::{container, resource}; +use crate::{ + controller::validate::{ValidatedCluster, ValidatedLogging}, + crd::{DruidConfigOverrides, DruidRole}, +}; + +pub(crate) mod broker; +pub(crate) mod coordinator; +pub(crate) mod historical; +pub(crate) mod middle_manager; +pub(crate) mod router; + +/// Why building one role group failed. The caller adds which role and role group it was. +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to derive the Druid memory settings from the resources"))] + DeriveMemorySettings { source: crate::crd::memory::Error }, + + #[snafu(display("failed to derive the heap size from the resources"))] + HeapMemory { source: crate::crd::resource::Error }, + + #[snafu(display("missing secret lifetime"))] + MissingSecretLifetime, + + #[snafu(display("failed to render the config files"))] + ConfigMapData { source: resource::config_map::Error }, + + #[snafu(display("failed to build the containers and volumes"))] + ContainersAndVolumes { source: container::Error }, + + #[snafu(display("failed to build ConfigMap"))] + ConfigMap { source: resource::config_map::Error }, + + #[snafu(display("failed to build StatefulSet"))] + StatefulSet { + source: resource::statefulset::Error, + }, +} + +/// What [`RoleGroupBuilder::new`] resolves, gathered by one role module from its typed +/// role-group config. +pub(crate) struct RoleGroupInputs<'a> { + pub(crate) cluster: &'a ValidatedCluster, + /// Read for the role group's object names and labels, and for the values that are functions + /// of the role: its ports, its process name and its container commands. + pub(crate) role: DruidRole, + pub(crate) role_group_name: RoleGroupName, + pub(crate) affinity: StackableAffinity, + pub(crate) graceful_shutdown_timeout: Option, + pub(crate) requested_secret_lifetime: Duration, + /// The `druid` container's resource requirements. + pub(crate) resources: ResourceRequirements, + /// The JVM heap size. + pub(crate) heap: MemoryQuantity, + /// The JVM direct memory size of the roles that use direct memory: Broker, Historical and + /// Router. + pub(crate) direct_memory: Option, + pub(crate) logging: ValidatedLogging, + /// The role's group Listener name; `Some` for Brokers, Coordinators and Routers. Drives the + /// listener claim template, its mount on the `druid` container and the listener scope of the + /// TLS volume. + pub(crate) listener: Option, + /// Pod volumes only this role has (the Historical segment cache); empty for the other roles. + pub(crate) role_pod_volumes: Vec, + /// The `druid` container's mounts of `role_pod_volumes`. + pub(crate) role_druid_mounts: Vec, + /// `runtime.properties` entries derived from the resources (the Historical segment cache + /// location and processing settings), the lowest layer; empty for the other roles. + pub(crate) resource_runtime_properties: BTreeMap, + /// The value of `druid.indexer.runner.javaOptsArray`, the JVM options of the tasks a + /// MiddleManager spawns; `Some` for MiddleManagers only. Written to both runtime.properties + /// and security.properties. + pub(crate) indexer_java_opts: Option, + /// The role's static `runtime.properties` defaults, the layer right below the user's + /// overrides. + pub(crate) role_runtime_defaults: BTreeMap, + /// JVM arguments only this role has (the Coordinator's Derby log file), rendered after the + /// arguments every role has; empty for the other roles. + pub(crate) role_jvm_args: Vec, + /// The `druid` container's pre-stop hook; `Some` for MiddleManagers, which drain their tasks + /// before they stop. + pub(crate) druid_pre_stop: Option, + /// Whether the role authenticates its users with OIDC when the cluster uses OIDC; false for + /// MiddleManagers, which then run the system authenticator only. + pub(crate) oidc_enabled: bool, + pub(crate) replicas: Option, + pub(crate) config_overrides: DruidConfigOverrides, + pub(crate) env_overrides: EnvVarSet, + pub(crate) pod_overrides: PodTemplateSpec, + pub(crate) jvm_argument_overrides: JvmArgumentOverrides, +} + +/// Everything one role group's Kubernetes objects are built from, already resolved. +/// +/// These fields are outputs rather than configuration: the containers are built and the +/// `ConfigMap` entries rendered. The `build_*` methods therefore read straight through, and +/// nothing about which role this is reaches them beyond the role's name. +pub(crate) struct RoleGroupBuilder<'a> { + pub(crate) cluster: &'a ValidatedCluster, + /// Read for the role group's object names and labels. + pub(crate) role: DruidRole, + pub(crate) role_group_name: RoleGroupName, + /// Read for the pod's affinity. + pub(crate) affinity: StackableAffinity, + /// Read for the pod's termination grace period. + pub(crate) graceful_shutdown_timeout: Option, + /// The role group's replica count, passed to the StatefulSet as is; `None` makes Kubernetes + /// run one pod. + pub(crate) replicas: Option, + pub(crate) pod_overrides: PodTemplateSpec, + /// The listener claim template of a role with a Listener. `None` for the other roles, which + /// keeps their StatefulSets without a `volumeClaimTemplates` field. + pub(crate) volume_claim_templates: Option>, + /// The `prepare` init container. + pub(crate) init_containers: Vec, + /// The `druid` container, then the Vector sidecar when the Vector agent is enabled. + pub(crate) containers: Vec, + /// Every pod-level volume the containers mount, in the order the pod lists them. + pub(crate) pod_volumes: Vec, + /// The rendered `ConfigMap` entries, keyed by file name. + pub(crate) config_map_data: BTreeMap, +} + +impl<'a> RoleGroupBuilder<'a> { + /// Resolves one role group: renders its config files, builds the containers every role runs + /// and collects their pod volumes. + pub(crate) fn new(inputs: RoleGroupInputs<'a>) -> Result { + let config_map_data = + resource::config_map::config_map_data(&inputs).context(ConfigMapDataSnafu)?; + let container::ContainersAndVolumes { + init_containers, + containers, + pod_volumes, + volume_claim_templates, + } = container::containers_and_volumes(&inputs).context(ContainersAndVolumesSnafu)?; + + Ok(Self { + cluster: inputs.cluster, + role: inputs.role, + role_group_name: inputs.role_group_name, + affinity: inputs.affinity, + graceful_shutdown_timeout: inputs.graceful_shutdown_timeout, + replicas: inputs.replicas, + pod_overrides: inputs.pod_overrides, + volume_claim_templates, + init_containers, + containers, + pod_volumes, + config_map_data, + }) + } + + /// The headless and metrics `Service`s. + pub(crate) fn build_services(&self) -> Vec { + vec![ + resource::service::build_rolegroup_headless_service( + self.cluster, + &self.role, + &self.role_group_name, + ), + resource::service::build_rolegroup_metrics_service( + self.cluster, + &self.role, + &self.role_group_name, + ), + ] + } + + pub(crate) fn build_config_map(&self) -> Result { + resource::config_map::build_config_map(self).context(ConfigMapSnafu) + } + + pub(crate) fn build_statefulset(&self) -> Result { + resource::statefulset::build_statefulset(self).context(StatefulSetSnafu) + } +} diff --git a/rust/operator-binary/src/controller/build/role_group/router.rs b/rust/operator-binary/src/controller/build/role_group/router.rs new file mode 100644 index 00000000..2c68909d --- /dev/null +++ b/rust/operator-binary/src/controller/build/role_group/router.rs @@ -0,0 +1,69 @@ +//! Router role groups. + +use std::collections::BTreeMap; + +use snafu::{OptionExt, ResultExt}; +use stackable_operator::{memory::MemoryQuantity, v2::types::operator::RoleGroupName}; + +use super::{ + Error, HeapMemorySnafu, MissingSecretLifetimeSnafu, RoleGroupBuilder, RoleGroupInputs, +}; +use crate::{ + controller::{ + build::{properties::runtime_properties, resource::listener::group_listener_name}, + validate::{RouterRoleGroupConfig, ValidatedCluster}, + }, + crd::{DruidRole, RouterConfig, resource::heap_memory}, +}; + +pub(crate) const ROLE: DruidRole = DruidRole::Router; + +/// Resolves one Router role group into its [`RoleGroupBuilder`]. +pub(crate) fn build<'a>( + cluster: &'a ValidatedCluster, + role_group_name: &RoleGroupName, + rg_config: &RouterRoleGroupConfig, +) -> Result, Error> { + let RouterConfig { + resources, + // Validated up front into `rg_config.config.logging`. + logging: _, + affinity, + graceful_shutdown_timeout, + requested_secret_lifetime, + } = &rg_config.config.config; + + // Routers reserve 128 MiB of direct memory for their query processing buffers. + let direct_memory = MemoryQuantity::from_mebi(128.); + let heap = heap_memory(resources, Some(direct_memory)).context(HeapMemorySnafu)?; + + RoleGroupBuilder::new(RoleGroupInputs { + cluster, + role: ROLE, + role_group_name: role_group_name.clone(), + affinity: affinity.clone(), + graceful_shutdown_timeout: *graceful_shutdown_timeout, + requested_secret_lifetime: requested_secret_lifetime.context(MissingSecretLifetimeSnafu)?, + resources: resources.clone().into(), + heap, + direct_memory: Some(direct_memory), + logging: rg_config.config.logging.clone(), + listener: Some(group_listener_name(&cluster.name, &ROLE)), + role_pod_volumes: Vec::new(), + role_druid_mounts: Vec::new(), + resource_runtime_properties: BTreeMap::new(), + indexer_java_opts: None, + role_jvm_args: Vec::new(), + druid_pre_stop: None, + oidc_enabled: true, + role_runtime_defaults: runtime_properties::defaults(&ROLE), + replicas: rg_config.replicas, + config_overrides: rg_config.config_overrides.clone(), + env_overrides: rg_config.env_overrides.clone(), + pod_overrides: rg_config.pod_overrides.clone(), + jvm_argument_overrides: rg_config + .product_specific_common_config + .jvm_argument_overrides + .clone(), + }) +} diff --git a/rust/operator-binary/src/controller/build/security.rs b/rust/operator-binary/src/controller/build/security.rs index 5ab3ba8a..5eba208d 100644 --- a/rust/operator-binary/src/controller/build/security.rs +++ b/rust/operator-binary/src/controller/build/security.rs @@ -8,21 +8,17 @@ use std::{collections::BTreeMap, str::FromStr}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ - builder::{ - self, - pod::{ - PodBuilder, - container::ContainerBuilder, - volume::{ - SecretFormat, SecretOperatorVolumeSourceBuilder, - SecretOperatorVolumeSourceBuilderError, VolumeBuilder, - }, + builder::pod::{ + container::ContainerBuilder, + volume::{ + SecretFormat, SecretOperatorVolumeSourceBuilder, + SecretOperatorVolumeSourceBuilderError, VolumeBuilder, }, }, commons::secret_class::SecretClassVolumeProvisionParts, crd::listener, k8s_openapi::{ - api::core::v1::{ContainerPort, Probe, ServicePort, TCPSocketAction}, + api::core::v1::{ContainerPort, Probe, ServicePort, TCPSocketAction, Volume}, apimachinery::pkg::util::intstr::IntOrString, }, shared::time::Duration, @@ -41,9 +37,6 @@ pub enum Error { SecretVolumeBuild { source: SecretOperatorVolumeSourceBuilderError, }, - - #[snafu(display("failed to add needed volume"))] - AddVolume { source: builder::pod::Error }, } // Ports @@ -86,8 +79,8 @@ const TCP_PROTOCOL: &str = "TCP"; const STACKABLE_MOUNT_TLS_DIR: &str = "/stackable/mount_tls"; // volume names -stackable_operator::constant!(TLS_VOLUME_NAME: VolumeName = "tls"); -stackable_operator::constant!(TLS_MOUNT_VOLUME_NAME: VolumeName = "tls-mount"); +stackable_operator::constant!(pub(crate) TLS_VOLUME_NAME: VolumeName = "tls"); +stackable_operator::constant!(pub(crate) TLS_MOUNT_VOLUME_NAME: VolumeName = "tls-mount"); pub fn container_ports(tls: &DruidTlsSecurity, role: &DruidRole) -> Vec { let (name, port) = exposed_port(tls, role); @@ -131,7 +124,7 @@ fn exposed_port(tls: &DruidTlsSecurity, role: &DruidRole) -> (&'static str, Port } /// Adds required tls volume mounts to image and product container builders -/// Adds required tls volumes to pod builder +/// Adds the TLS volumes to `pod_volumes` and their mounts to the containers. /// /// # Panics /// @@ -141,7 +134,7 @@ pub fn add_tls_volume_and_volume_mounts( tls: &DruidTlsSecurity, prepare: &mut ContainerBuilder, druid: &mut ContainerBuilder, - pod: &mut PodBuilder, + pod_volumes: &mut Vec, requested_secret_lifetime: &Duration, listener_scope: Option, ) -> Result<(), Error> { @@ -163,7 +156,7 @@ pub fn add_tls_volume_and_volume_mounts( secret_volume_source_builder.with_listener_volume_scope(listener_scope); } - pod.add_volume( + pod_volumes.push( VolumeBuilder::new(&*TLS_MOUNT_VOLUME_NAME) .ephemeral( secret_volume_source_builder @@ -171,8 +164,7 @@ pub fn add_tls_volume_and_volume_mounts( .context(SecretVolumeBuildSnafu)?, ) .build(), - ) - .context(AddVolumeSnafu)?; + ); prepare .add_volume_mount(&*TLS_MOUNT_VOLUME_NAME, STACKABLE_MOUNT_TLS_DIR) .expect("The mount paths are statically defined and there should be no duplicates."); @@ -180,12 +172,11 @@ pub fn add_tls_volume_and_volume_mounts( .add_volume_mount(&*TLS_MOUNT_VOLUME_NAME, STACKABLE_MOUNT_TLS_DIR) .expect("The mount paths are statically defined and there should be no duplicates."); - pod.add_volume( + pod_volumes.push( VolumeBuilder::new(&*TLS_VOLUME_NAME) .with_empty_dir(Option::<&str>::None, None) .build(), - ) - .context(AddVolumeSnafu)?; + ); prepare .add_volume_mount(&*TLS_VOLUME_NAME, STACKABLE_TLS_DIR) diff --git a/rust/operator-binary/src/controller/dereference.rs b/rust/operator-binary/src/controller/dereference.rs index 447ad4f3..c1f239d8 100644 --- a/rust/operator-binary/src/controller/dereference.rs +++ b/rust/operator-binary/src/controller/dereference.rs @@ -10,7 +10,7 @@ use stackable_operator::{ }; use crate::{ - controller::build::resource::listener::general_group_listener_name, + controller::build::resource::listener::group_listener_name, crd::{ DeepStorageSpec, DruidRole, authentication::fetch_authentication_classes, authorization::DruidAuthorization, v1alpha1, @@ -209,7 +209,7 @@ pub async fn dereference( .context(AuthenticationClassRetrievalSnafu)?; let cluster_name = get_cluster_name(druid).context(ClusterIdentitySnafu)?; - let listener_name = general_group_listener_name(&cluster_name, &DruidRole::Router); + let listener_name = group_listener_name(&cluster_name, &DruidRole::Router); let router_listener = client .get_opt::(listener_name.as_ref(), namespace) diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 61ccd8b4..d2d37507 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -9,36 +9,53 @@ use std::{ str::FromStr, }; -use snafu::{ResultExt, Snafu, ensure}; +use serde::Serialize; +use snafu::{OptionExt, ResultExt, Snafu, ensure}; use stackable_operator::{ cli::OperatorEnvironmentOptions, commons::{ pdb::PdbConfig, product_image_selection::{self, ResolvedProductImage}, }, + config::{ + fragment::{FromFragment, ValidationError}, + merge::Merge, + }, crd::{listener::v1alpha1::Listener, s3}, database_connections::drivers::jdbc::{JdbcDatabaseConnection, JdbcDatabaseConnectionDetails}, k8s_openapi::api::core::v1::{ConfigMap, Secret, Volume}, - kube::{Resource, api::ObjectMeta}, + kube::{Resource, ResourceExt, api::ObjectMeta}, + product_logging::spec::Logging, + role_utils::GenericRoleConfig, + schemars::JsonSchema, v2::{ HasName, HasUid, NameIsValidLabelValue, controller_utils::{get_cluster_name, get_namespace, get_uid}, + product_logging::framework::{ + ValidatedContainerLogConfigChoice, VectorContainerLogConfig, + validate_logging_configuration_for_container, + }, role_group_utils::ResourceNames, - role_utils, + role_utils::{ + self, JavaCommonConfig, Role, RoleGroup, RoleGroupConfig, with_validated_config, + }, types::{ - kubernetes::{ListenerClassName, NamespaceName, Uid}, + kubernetes::{ConfigMapName, ListenerClassName, NamespaceName, Uid}, operator::{ClusterName, ProductVersion, RoleGroupName}, }, }, }; -use strum::IntoEnumIterator; use crate::{ authentication::DruidAuthenticationConfig, controller::{PRODUCT_NAME, dereference::DereferencedObjects}, crd::{ - DeepStorageSpec, DruidRole, authentication::AuthenticationClassesResolved, - database::MetadataDatabaseConnection, security::DruidTlsSecurity, v1alpha1, + BrokerConfig, BrokerConfigFragment, Container, CoordinatorConfig, + CoordinatorConfigFragment, DeepStorageSpec, DruidConfigOverrides, DruidRole, + HistoricalConfig, HistoricalConfigFragment, MiddleManagerConfig, + MiddleManagerConfigFragment, RouterConfig, RouterConfigFragment, + authentication::AuthenticationClassesResolved, database::MetadataDatabaseConnection, + security::DruidTlsSecurity, v1alpha1, }, }; @@ -50,8 +67,31 @@ pub enum Error { source: product_image_selection::Error, }, - #[snafu(display("failed to resolve and merge config for role and role group"))] - FailedToResolveConfig { source: crate::crd::Error }, + #[snafu(display( + "failed to merge and validate the config of {role} role group {role_group:?}" + ))] + ValidateRoleGroupConfig { + source: ValidationError, + role: DruidRole, + role_group: String, + }, + + #[snafu(display("invalid {role} role group name {role_group:?}"))] + ParseRoleGroupName { + source: stackable_operator::v2::macros::attributed_string_type::Error, + role: DruidRole, + role_group: String, + }, + + #[snafu(display("failed to validate container log configuration"))] + ValidateLoggingConfig { + source: stackable_operator::v2::product_logging::framework::Error, + }, + + #[snafu(display( + "the Vector agent is enabled but the Vector aggregator ConfigMap name is missing" + ))] + MissingVectorAggregatorConfigMapName, #[snafu(display("failed to determine the cluster's name, namespace, or uid"))] ClusterIdentity { @@ -76,17 +116,78 @@ pub enum Error { type Result = std::result::Result; -/// A validated, merged role-group config. +/// Validated logging configuration for the Druid, Prepare (init) and (optional) Vector containers. /// -/// This is the upstream [`stackable_operator::v2::role_utils::RoleGroupConfig`] (config plus the -/// four merged override categories), with the typed per-role config erased to -/// [`crate::crd::ValidatedDruidConfig`] so that all roles share a single type. The rendered -/// per-file configs (runtime.properties / security.properties / jvm.config) are produced later, in -/// the config-map build step. -/// -/// Defined in [`crate::crd`] (where it has access to the private typed config fields) and -/// re-exported here for the build step. -pub use crate::crd::DruidRoleGroupConfig; +/// Produced up-front by [`validate_logging`] so that an invalid custom log ConfigMap name or a +/// missing Vector aggregator discovery ConfigMap name fails reconciliation during validation +/// rather than at resource-build time. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ValidatedLogging { + pub druid_container: ValidatedContainerLogConfigChoice, + pub prepare_container: ValidatedContainerLogConfigChoice, + pub vector_container: Option, + pub enable_vector_agent: bool, +} + +/// A Coordinator role group's validated, merged config together with its validated logging. +#[derive(Clone, Debug, PartialEq)] +pub struct ValidatedCoordinatorConfig { + pub config: CoordinatorConfig, + /// Validated logging configuration (derived from `config.logging` during validation). + pub logging: ValidatedLogging, +} + +/// A Broker role group's validated, merged config together with its validated logging. +#[derive(Clone, Debug, PartialEq)] +pub struct ValidatedBrokerConfig { + pub config: BrokerConfig, + /// Validated logging configuration (derived from `config.logging` during validation). + pub logging: ValidatedLogging, +} + +/// A Historical role group's validated, merged config together with its validated logging. +#[derive(Clone, Debug, PartialEq)] +pub struct ValidatedHistoricalConfig { + pub config: HistoricalConfig, + /// Validated logging configuration (derived from `config.logging` during validation). + pub logging: ValidatedLogging, +} + +/// A MiddleManager role group's validated, merged config together with its validated logging. +#[derive(Clone, Debug, PartialEq)] +pub struct ValidatedMiddleManagerConfig { + pub config: MiddleManagerConfig, + /// Validated logging configuration (derived from `config.logging` during validation). + pub logging: ValidatedLogging, +} + +/// A Router role group's validated, merged config together with its validated logging. +#[derive(Clone, Debug, PartialEq)] +pub struct ValidatedRouterConfig { + pub config: RouterConfig, + /// Validated logging configuration (derived from `config.logging` during validation). + pub logging: ValidatedLogging, +} + +/// The [`RoleGroupConfig`] of one Coordinator role group. +pub type CoordinatorRoleGroupConfig = + RoleGroupConfig; + +/// The [`RoleGroupConfig`] of one Broker role group. +pub type BrokerRoleGroupConfig = + RoleGroupConfig; + +/// The [`RoleGroupConfig`] of one Historical role group. +pub type HistoricalRoleGroupConfig = + RoleGroupConfig; + +/// The [`RoleGroupConfig`] of one MiddleManager role group. +pub type MiddleManagerRoleGroupConfig = + RoleGroupConfig; + +/// The [`RoleGroupConfig`] of one Router role group. +pub type RouterRoleGroupConfig = + RoleGroupConfig; /// Cluster-wide resolved fields that are not role/rolegroup specific. pub struct ValidatedClusterConfig { @@ -120,17 +221,25 @@ impl ValidatedClusterConfig { } } -/// Per-role configuration extracted during validation, so the reconcile/build steps consume this -/// instead of re-reading the raw [`v1alpha1::DruidCluster`]. +/// The role-level config of a role that exposes a group Listener: Broker, Coordinator and Router +/// ([`v1alpha1::DruidRoleConfig`] in the CRD). +#[derive(Clone, Debug)] +pub struct ValidatedListenerRoleConfig { + pub pdb: PdbConfig, + pub listener_class: ListenerClassName, +} + +/// The role-level config of a role without a Listener: Historical and MiddleManager +/// ([`GenericRoleConfig`] in the CRD). #[derive(Clone, Debug)] pub struct ValidatedRoleConfig { pub pdb: PdbConfig, - /// The role's listener class, or `None` for roles without a listener - /// ([`DruidRole::Historical`]/[`DruidRole::MiddleManager`]). - pub listener_class: Option, } /// Synchronous inputs the rest of `reconcile_druid` needs after dereferencing. +/// +/// Every Druid role is required by the CRD, so every role has its role-level config and its map +/// of role groups; the fields are in [`DruidRole`] variant order. pub struct ValidatedCluster { /// Mirrors `name`/`namespace`/`uid` below so that `ValidatedCluster` can implement /// [`Resource`] and be passed directly to the metadata/owner-reference builders. @@ -144,59 +253,53 @@ pub struct ValidatedCluster { /// value, so the build steps don't re-parse it per resource. pub product_version: ProductVersion, pub cluster_config: ValidatedClusterConfig, - /// The per-role config (PDB and listener class) for every [`DruidRole`]. - pub role_configs: BTreeMap, - pub role_group_configs: BTreeMap>, + /// The Coordinator role-level config. + pub coordinator_config: ValidatedListenerRoleConfig, + /// The validated config of every Coordinator role group, keyed by role group name. + pub coordinator_role_group_configs: BTreeMap, + /// The Broker role-level config. + pub broker_config: ValidatedListenerRoleConfig, + /// The validated config of every Broker role group, keyed by role group name. + pub broker_role_group_configs: BTreeMap, + /// The Historical role-level config. + pub historical_config: ValidatedRoleConfig, + /// The validated config of every Historical role group, keyed by role group name. + pub historical_role_group_configs: BTreeMap, + /// The MiddleManager role-level config. + pub middle_manager_config: ValidatedRoleConfig, + /// The validated config of every MiddleManager role group, keyed by role group name. + pub middle_manager_role_group_configs: BTreeMap, + /// The Router role-level config. + pub router_config: ValidatedListenerRoleConfig, + /// The validated config of every Router role group, keyed by role group name. + pub router_role_group_configs: BTreeMap, pub router_listener: Option, pub internal_secret: Option, pub discovery_config_map: Option, } impl ValidatedCluster { - #[allow(clippy::too_many_arguments)] - pub(crate) fn new( - name: ClusterName, - namespace: NamespaceName, - uid: Uid, - image: ResolvedProductImage, - cluster_config: ValidatedClusterConfig, - role_configs: BTreeMap, - role_group_configs: BTreeMap>, - router_listener: Option, - internal_secret: Option, - discovery_config_map: Option, - ) -> Self { - let metadata = ObjectMeta { + /// The metadata that mirrors the cluster's name, namespace and uid, so that + /// `ValidatedCluster` can implement [`Resource`] and stand in for the raw `DruidCluster` + /// when building child-object metadata and owner references. + pub(crate) fn object_meta( + name: &ClusterName, + namespace: &NamespaceName, + uid: &Uid, + ) -> ObjectMeta { + ObjectMeta { name: Some(name.to_string()), namespace: Some(namespace.to_string()), uid: Some(uid.to_string()), ..ObjectMeta::default() - }; - // `app_version_label_value` is constructed to be a valid label value, so it is also a valid - // `ProductVersion`. - let product_version = ProductVersion::from_str(&image.app_version_label_value) - .expect("the app version label value is a valid product version"); - Self { - metadata, - name, - namespace, - uid, - image, - product_version, - cluster_config, - role_configs, - role_group_configs, - router_listener, - internal_secret, - discovery_config_map, } } - /// The validated per-role config (PDB and listener class) for the given role. - pub(crate) fn role_config(&self, role: &DruidRole) -> &ValidatedRoleConfig { - self.role_configs - .get(role) - .expect("every DruidRole has a validated role config") + /// The product version as a valid label value. `app_version_label_value` is constructed to + /// be a valid label value, so it is also a valid `ProductVersion`. + pub(crate) fn product_version(image: &ResolvedProductImage) -> ProductVersion { + ProductVersion::from_str(&image.app_version_label_value) + .expect("the app version label value is a valid product version") } /// Type-safe names for the per-cluster RBAC resources: the ServiceAccount shared by all @@ -224,8 +327,9 @@ impl ValidatedCluster { // Implementing `Resource` (plus `HasName`/`HasUid`) lets `ValidatedCluster` stand in for the raw // `DruidCluster` when building child-object metadata and owner references. The identity-bearing -// methods are backed by the `metadata` built in `new`, while kind/group/version/plural delegate to -// the CRD so produced owner references are byte-identical to the previous raw-cluster ones. +// methods are backed by the `metadata` built by [`ValidatedCluster::object_meta`], while +// kind/group/version/plural delegate to the CRD so produced owner references are byte-identical +// to the raw-cluster ones. impl Resource for ValidatedCluster { type DynamicType = ::DynamicType; type Scope = ::Scope; @@ -301,27 +405,62 @@ pub fn validate( let druid_auth_config = DruidAuthenticationConfig::from_auth_classes(resolved_authentication_classes); - let mut role_group_configs: BTreeMap> = - BTreeMap::new(); - let mut role_configs: BTreeMap = BTreeMap::new(); - - for druid_role in DruidRole::iter() { - let group_map = druid - .merged_role(&druid_role) - .context(FailedToResolveConfigSnafu)?; - role_group_configs.insert(druid_role.clone(), group_map); - - role_configs.insert( - druid_role.clone(), - ValidatedRoleConfig { - pdb: druid - .generic_role_config(&druid_role) - .pod_disruption_budget - .clone(), - listener_class: druid_role.listener_class_name(druid), - }, - ); - } + let cluster_name = druid.name_any(); + let deep_storage = &druid.spec.cluster_config.deep_storage; + // The Vector aggregator discovery ConfigMap name (a typed `ConfigMapName`, so an invalid name + // is already rejected at deserialization). It is only required when the Vector agent is + // enabled for a role group; see [`validate_logging`]. + let vector_aggregator_config_map_name = druid + .spec + .cluster_config + .vector_aggregator_config_map_name + .clone(); + + // The first failure propagates, so this order decides which misconfiguration the user is told + // about when more than one role is wrong. It is `DruidRole`'s variant order. + let coordinator_role = &druid.spec.coordinators; + let coordinator_role_group_configs = validate_coordinator_role_group_configs( + coordinator_role, + &CoordinatorConfig::default_config(&cluster_name, &DruidRole::Coordinator, deep_storage), + &vector_aggregator_config_map_name, + )?; + let coordinator_config = validated_listener_role_config(&coordinator_role.role_config); + + let broker_role = &druid.spec.brokers; + let broker_role_group_configs = validate_broker_role_group_configs( + broker_role, + &BrokerConfig::default_config(&cluster_name, &DruidRole::Broker, deep_storage), + &vector_aggregator_config_map_name, + )?; + let broker_config = validated_listener_role_config(&broker_role.role_config); + + let historical_role = &druid.spec.historicals; + let historical_role_group_configs = validate_historical_role_group_configs( + historical_role, + &HistoricalConfig::default_config(&cluster_name, &DruidRole::Historical, deep_storage), + &vector_aggregator_config_map_name, + )?; + let historical_config = validated_role_config(&historical_role.role_config); + + let middle_manager_role = &druid.spec.middle_managers; + let middle_manager_role_group_configs = validate_middle_manager_role_group_configs( + middle_manager_role, + &MiddleManagerConfig::default_config( + &cluster_name, + &DruidRole::MiddleManager, + deep_storage, + ), + &vector_aggregator_config_map_name, + )?; + let middle_manager_config = validated_role_config(&middle_manager_role.role_config); + + let router_role = &druid.spec.routers; + let router_role_group_configs = validate_router_role_group_configs( + router_role, + &RouterConfig::default_config(&cluster_name, &DruidRole::Router, deep_storage), + &vector_aggregator_config_map_name, + )?; + let router_config = validated_listener_role_config(&router_role.role_config); let name = get_cluster_name(druid).context(ClusterIdentitySnafu)?; let namespace = get_namespace(druid).context(ClusterIdentitySnafu)?; @@ -364,12 +503,14 @@ pub fn validate( .as_ref() .map(|base_url| format!("{base_url}/allow")); - Ok(ValidatedCluster::new( + Ok(ValidatedCluster { + metadata: ValidatedCluster::object_meta(&name, &namespace, &uid), name, namespace, uid, + product_version: ValidatedCluster::product_version(&image), image, - ValidatedClusterConfig { + cluster_config: ValidatedClusterConfig { zookeeper_connection_string: dereferenced_objects.zookeeper_connection_string.clone(), opa_connection_string, s3_connection, @@ -382,35 +523,403 @@ pub fn validate( deep_storage: druid.spec.cluster_config.deep_storage.clone(), extra_volumes: druid.spec.cluster_config.extra_volumes.clone(), }, - role_configs, - role_group_configs, - dereferenced_objects.router_listener.clone(), - dereferenced_objects.internal_secret.clone(), - dereferenced_objects.discovery_config_map.clone(), - )) + coordinator_config, + coordinator_role_group_configs, + broker_config, + broker_role_group_configs, + historical_config, + historical_role_group_configs, + middle_manager_config, + middle_manager_role_group_configs, + router_config, + router_role_group_configs, + router_listener: dereferenced_objects.router_listener.clone(), + internal_secret: dereferenced_objects.internal_secret.clone(), + discovery_config_map: dereferenced_objects.discovery_config_map.clone(), + }) } -#[cfg(test)] -pub(crate) mod test_support { - use std::{collections::BTreeMap, str::FromStr}; - - use stackable_operator::{ - database_connections::drivers::jdbc::JdbcDatabaseConnection, - kube::ResourceExt, - utils::yaml_from_str_singleton_map, - v2::types::{ - kubernetes::{NamespaceName, SecretClassName, Uid}, - operator::ClusterName, +/// The validated role-level config of a role with a group Listener. +/// +/// [`v1alpha1::DruidRoleConfig`] and [`GenericRoleConfig`] are destructured without `..`, so a +/// field added upstream fails to compile here instead of being silently dropped. +fn validated_listener_role_config( + role_config: &v1alpha1::DruidRoleConfig, +) -> ValidatedListenerRoleConfig { + let v1alpha1::DruidRoleConfig { + generic: GenericRoleConfig { + pod_disruption_budget, }, - }; - use strum::IntoEnumIterator; + listener_class, + } = role_config; + ValidatedListenerRoleConfig { + pdb: pod_disruption_budget.clone(), + listener_class: listener_class.clone(), + } +} + +/// The validated role-level config of a role without a Listener. +/// +/// [`GenericRoleConfig`] is destructured without `..`, so a field added upstream fails to compile +/// here instead of being silently dropped. +fn validated_role_config(role_config: &GenericRoleConfig) -> ValidatedRoleConfig { + let GenericRoleConfig { + pod_disruption_budget, + } = role_config; + ValidatedRoleConfig { + pdb: pod_disruption_budget.clone(), + } +} + +/// Merges and validates every Coordinator role group (via [`validate_role_group_config`]), then +/// its logging, then its name. +fn validate_coordinator_role_group_configs( + role: &Role< + CoordinatorConfigFragment, + DruidConfigOverrides, + v1alpha1::DruidRoleConfig, + JavaCommonConfig, + >, + default_config: &CoordinatorConfigFragment, + vector_aggregator_config_map_name: &Option, +) -> Result> { + let druid_role = DruidRole::Coordinator; + role.role_groups + .iter() + .map(|(role_group_name, role_group)| { + let merged = validate_role_group_config::<_, CoordinatorConfig, _>( + &druid_role, + role, + role_group_name, + role_group, + default_config, + )?; + let logging = + validate_logging(&merged.config.logging, vector_aggregator_config_map_name)?; + let validated = CoordinatorRoleGroupConfig { + replicas: merged.replicas, + config: ValidatedCoordinatorConfig { + config: merged.config, + logging, + }, + config_overrides: merged.config_overrides, + env_overrides: merged.env_overrides, + cli_overrides: merged.cli_overrides, + pod_overrides: merged.pod_overrides, + product_specific_common_config: merged.product_specific_common_config, + }; + Ok(( + parse_role_group_name(&druid_role, role_group_name)?, + validated, + )) + }) + .collect() +} + +/// Merges and validates every Broker role group (via [`validate_role_group_config`]), then its +/// logging, then its name. +fn validate_broker_role_group_configs( + role: &Role< + BrokerConfigFragment, + DruidConfigOverrides, + v1alpha1::DruidRoleConfig, + JavaCommonConfig, + >, + default_config: &BrokerConfigFragment, + vector_aggregator_config_map_name: &Option, +) -> Result> { + let druid_role = DruidRole::Broker; + role.role_groups + .iter() + .map(|(role_group_name, role_group)| { + let merged = validate_role_group_config::<_, BrokerConfig, _>( + &druid_role, + role, + role_group_name, + role_group, + default_config, + )?; + let logging = + validate_logging(&merged.config.logging, vector_aggregator_config_map_name)?; + let validated = BrokerRoleGroupConfig { + replicas: merged.replicas, + config: ValidatedBrokerConfig { + config: merged.config, + logging, + }, + config_overrides: merged.config_overrides, + env_overrides: merged.env_overrides, + cli_overrides: merged.cli_overrides, + pod_overrides: merged.pod_overrides, + product_specific_common_config: merged.product_specific_common_config, + }; + Ok(( + parse_role_group_name(&druid_role, role_group_name)?, + validated, + )) + }) + .collect() +} + +/// Merges and validates every Historical role group (via [`validate_role_group_config`]), then +/// its logging, then its name. +fn validate_historical_role_group_configs( + role: &Role< + HistoricalConfigFragment, + DruidConfigOverrides, + GenericRoleConfig, + JavaCommonConfig, + >, + default_config: &HistoricalConfigFragment, + vector_aggregator_config_map_name: &Option, +) -> Result> { + let druid_role = DruidRole::Historical; + role.role_groups + .iter() + .map(|(role_group_name, role_group)| { + let merged = validate_role_group_config::<_, HistoricalConfig, _>( + &druid_role, + role, + role_group_name, + role_group, + default_config, + )?; + let logging = + validate_logging(&merged.config.logging, vector_aggregator_config_map_name)?; + let validated = HistoricalRoleGroupConfig { + replicas: merged.replicas, + config: ValidatedHistoricalConfig { + config: merged.config, + logging, + }, + config_overrides: merged.config_overrides, + env_overrides: merged.env_overrides, + cli_overrides: merged.cli_overrides, + pod_overrides: merged.pod_overrides, + product_specific_common_config: merged.product_specific_common_config, + }; + Ok(( + parse_role_group_name(&druid_role, role_group_name)?, + validated, + )) + }) + .collect() +} + +/// Merges and validates every MiddleManager role group (via [`validate_role_group_config`]), then +/// its logging, then its name. +fn validate_middle_manager_role_group_configs( + role: &Role< + MiddleManagerConfigFragment, + DruidConfigOverrides, + GenericRoleConfig, + JavaCommonConfig, + >, + default_config: &MiddleManagerConfigFragment, + vector_aggregator_config_map_name: &Option, +) -> Result> { + let druid_role = DruidRole::MiddleManager; + role.role_groups + .iter() + .map(|(role_group_name, role_group)| { + let merged = validate_role_group_config::<_, MiddleManagerConfig, _>( + &druid_role, + role, + role_group_name, + role_group, + default_config, + )?; + let logging = + validate_logging(&merged.config.logging, vector_aggregator_config_map_name)?; + let validated = MiddleManagerRoleGroupConfig { + replicas: merged.replicas, + config: ValidatedMiddleManagerConfig { + config: merged.config, + logging, + }, + config_overrides: merged.config_overrides, + env_overrides: merged.env_overrides, + cli_overrides: merged.cli_overrides, + pod_overrides: merged.pod_overrides, + product_specific_common_config: merged.product_specific_common_config, + }; + Ok(( + parse_role_group_name(&druid_role, role_group_name)?, + validated, + )) + }) + .collect() +} + +/// Merges and validates every Router role group (via [`validate_role_group_config`]), then its +/// logging, then its name. +fn validate_router_role_group_configs( + role: &Role< + RouterConfigFragment, + DruidConfigOverrides, + v1alpha1::DruidRoleConfig, + JavaCommonConfig, + >, + default_config: &RouterConfigFragment, + vector_aggregator_config_map_name: &Option, +) -> Result> { + let druid_role = DruidRole::Router; + role.role_groups + .iter() + .map(|(role_group_name, role_group)| { + let merged = validate_role_group_config::<_, RouterConfig, _>( + &druid_role, + role, + role_group_name, + role_group, + default_config, + )?; + let logging = + validate_logging(&merged.config.logging, vector_aggregator_config_map_name)?; + let validated = RouterRoleGroupConfig { + replicas: merged.replicas, + config: ValidatedRouterConfig { + config: merged.config, + logging, + }, + config_overrides: merged.config_overrides, + env_overrides: merged.env_overrides, + cli_overrides: merged.cli_overrides, + pod_overrides: merged.pod_overrides, + product_specific_common_config: merged.product_specific_common_config, + }; + Ok(( + parse_role_group_name(&druid_role, role_group_name)?, + validated, + )) + }) + .collect() +} + +/// Merges and validates one role group via [`with_validated_config`], which folds the config +/// fragment (default <- role <- role group) plus the `configOverrides`, `envOverrides`, +/// `cliOverrides`, `podOverrides` and `jvmArgumentOverrides` (role group wins) into a single +/// [`RoleGroupConfig`]. +fn validate_role_group_config( + druid_role: &DruidRole, + role: &Role, + role_group_name: &str, + role_group: &RoleGroup, + default_config: &Config, +) -> Result> +where + Config: Clone + Merge, + ValidatedConfig: FromFragment, + RoleConfig: Default + JsonSchema + Serialize, +{ + let merged = with_validated_config::< + ValidatedConfig, + JavaCommonConfig, + Config, + RoleConfig, + DruidConfigOverrides, + >(role_group, role, default_config) + .with_context(|_| ValidateRoleGroupConfigSnafu { + role: druid_role.clone(), + role_group: role_group_name.to_owned(), + })?; + + Ok(RoleGroupConfig { + // Passed through as is, including `None`: the StatefulSet then leaves `.spec.replicas` + // unset. + replicas: merged.replicas, + config: merged.config.config, + config_overrides: merged.config.config_overrides, + env_overrides: merged.config.env_overrides.into(), + cli_overrides: merged.config.cli_overrides, + pod_overrides: merged.config.pod_overrides, + product_specific_common_config: merged.config.product_specific_common_config, + }) +} + +/// Parses a role group name from the CRD into its type-safe form. +fn parse_role_group_name(druid_role: &DruidRole, role_group_name: &str) -> Result { + RoleGroupName::from_str(role_group_name).with_context(|_| ParseRoleGroupNameSnafu { + role: druid_role.clone(), + role_group: role_group_name.to_owned(), + }) +} - use super::{RoleGroupName, ValidatedCluster, ValidatedClusterConfig, ValidatedRoleConfig}; - use crate::{ - controller::CONTAINER_IMAGE_BASE_NAME, - crd::{DruidRole, security::DruidTlsSecurity, v1alpha1}, +/// Validates the logging configuration for the Druid, Prepare and (optional) Vector containers. +/// +/// `vector_aggregator_config_map_name` is the discovery ConfigMap name of the Vector aggregator; +/// it is required only when the Vector agent is enabled. +fn validate_logging( + logging: &Logging, + vector_aggregator_config_map_name: &Option, +) -> Result { + let druid_container = validate_logging_configuration_for_container(logging, &Container::Druid) + .context(ValidateLoggingConfigSnafu)?; + let prepare_container = + validate_logging_configuration_for_container(logging, &Container::Prepare) + .context(ValidateLoggingConfigSnafu)?; + + let vector_container = if logging.enable_vector_agent { + let vector_aggregator_config_map_name = vector_aggregator_config_map_name + .clone() + .context(MissingVectorAggregatorConfigMapNameSnafu)?; + Some(VectorContainerLogConfig { + log_config: validate_logging_configuration_for_container(logging, &Container::Vector) + .context(ValidateLoggingConfigSnafu)?, + vector_aggregator_config_map_name, + }) + } else { + None }; + Ok(ValidatedLogging { + druid_container, + prepare_container, + vector_container, + enable_vector_agent: logging.enable_vector_agent, + }) +} + +#[cfg(test)] +pub(crate) mod test_support { + use stackable_operator::{cli::OperatorEnvironmentOptions, utils::yaml_from_str_singleton_map}; + + use super::{ValidatedCluster, validate}; + use crate::{controller::dereference::DereferencedObjects, crd::v1alpha1}; + + /// Dereferenced inputs with test defaults: no S3, no OPA, no auth, a fixed ZooKeeper string. + pub fn dereferenced_objects() -> DereferencedObjects { + DereferencedObjects { + zookeeper_connection_string: "zookeeper-connection-string".to_string(), + opa_base_document_url: None, + s3_ingestion_connection: None, + s3_deep_storage_bucket: None, + authentication_classes: Vec::new(), + router_listener: None, + internal_secret: None, + discovery_config_map: None, + } + } + + pub fn operator_environment() -> OperatorEnvironmentOptions { + OperatorEnvironmentOptions { + operator_namespace: "stackable-operators".to_string(), + operator_service_name: "druid-operator".to_string(), + image_repository: "oci.example.org".to_string(), + } + } + + /// Builds a [`ValidatedCluster`] through the real [`validate`] step, so that everything the + /// dereferenced inputs switch on (authentication, OPA, S3, the Router Listener) takes the same + /// path as in the operator. + pub fn validated_cluster_from( + druid: &v1alpha1::DruidCluster, + dereferenced_objects: &DereferencedObjects, + ) -> ValidatedCluster { + validate(druid, dereferenced_objects, &operator_environment()) + .expect("test: the cluster validates") + } + /// A minimal but fully-valid `DruidCluster` with one role group per role. pub const MINIMAL_DRUID_YAML: &str = r#" apiVersion: druid.stackable.tech/v1alpha1 @@ -460,112 +969,27 @@ spec: yaml_from_str_singleton_map(yaml).expect("invalid test DruidCluster YAML") } - /// Builds a [`ValidatedCluster`] from a `DruidCluster` using test defaults for the - /// dereferenced/cluster-wide fields (no S3, no auth, fixed zookeeper string). Runs the real - /// `merged_role` pipeline for every role. + /// Builds a [`ValidatedCluster`] from a `DruidCluster` with the test defaults of + /// [`dereferenced_objects`] (no S3, no OPA, no auth, a fixed ZooKeeper string). pub fn validated_cluster(druid: &v1alpha1::DruidCluster) -> ValidatedCluster { - let image = druid - .spec - .image - .resolve( - CONTAINER_IMAGE_BASE_NAME, - "oci.example.org", - &crate::built_info::PKG_VERSION_SEMVER, - ) - .expect("test: resolvable product image"); - - let druid_tls_security = DruidTlsSecurity::new( - false, - Some(SecretClassName::from_str("tls").expect("test: valid secret class")), - ); - - let mut role_group_configs: BTreeMap> = - BTreeMap::new(); - let mut role_configs: BTreeMap = BTreeMap::new(); - for role in DruidRole::iter() { - role_group_configs.insert( - role.clone(), - druid.merged_role(&role).expect("test: merged role"), - ); - role_configs.insert( - role.clone(), - ValidatedRoleConfig { - pdb: druid - .generic_role_config(&role) - .pod_disruption_budget - .clone(), - listener_class: role.listener_class_name(druid), - }, - ); - } - - let metadata_db_connection = druid - .spec - .cluster_config - .metadata_database - .jdbc_connection_details("metadata") - .expect("test: valid metadata db connection"); - - ValidatedCluster::new( - ClusterName::from_str(&druid.name_any()).expect("test: valid cluster name"), - NamespaceName::from_str("default").expect("test: valid namespace"), - Uid::from_str("c27b3971-ca72-42c1-80a4-abdfc1db0ddd").expect("test: valid uid"), - image, - ValidatedClusterConfig { - zookeeper_connection_string: "zookeeper-connection-string".to_string(), - opa_connection_string: None, - s3_connection: None, - deep_storage_bucket_name: None, - druid_tls_security, - druid_auth_config: None, - metadata_database: druid.spec.cluster_config.metadata_database.clone(), - additional_extensions: druid.spec.cluster_config.additional_extensions.clone(), - metadata_db_connection, - deep_storage: druid.spec.cluster_config.deep_storage.clone(), - extra_volumes: druid.spec.cluster_config.extra_volumes.clone(), - }, - role_configs, - role_group_configs, - None, - None, - None, - ) + validated_cluster_from(druid, &dereferenced_objects()) } } #[cfg(test)] mod tests { - use stackable_operator::cli::OperatorEnvironmentOptions; - use strum::IntoEnumIterator; + use std::str::FromStr; + + use stackable_operator::v2::types::operator::RoleGroupName; use super::{ Error, - test_support::{MINIMAL_DRUID_YAML, druid_from_yaml}, + test_support::{ + MINIMAL_DRUID_YAML, dereferenced_objects, druid_from_yaml, operator_environment, + }, validate, }; - use crate::{controller::dereference::DereferencedObjects, crd::DruidRole}; - - /// Dereferenced inputs with test defaults: no S3, no OPA, no auth, a fixed ZooKeeper string. - fn dereferenced_objects() -> DereferencedObjects { - DereferencedObjects { - zookeeper_connection_string: "zookeeper-connection-string".to_string(), - opa_base_document_url: None, - s3_ingestion_connection: None, - s3_deep_storage_bucket: None, - authentication_classes: Vec::new(), - router_listener: None, - internal_secret: None, - discovery_config_map: None, - } - } - - fn operator_environment() -> OperatorEnvironmentOptions { - OperatorEnvironmentOptions { - operator_namespace: "stackable-operators".to_string(), - operator_service_name: "druid-operator".to_string(), - image_repository: "oci.example.org".to_string(), - } - } + use crate::crd::DRUID_DEFAULT_LISTENER_CLASS; #[test] fn test_validate_ok() { @@ -581,48 +1005,39 @@ mod tests { "c27b3971-ca72-42c1-80a4-abdfc1db0ddd" ); - // Every role is merged and gets a role config. - for role in DruidRole::iter() { - assert!( - validated.role_group_configs.contains_key(&role), - "missing role-group configs for {role:?}" - ); - assert!( - validated.role_configs.contains_key(&role), - "missing role config for {role:?}" - ); - } - - // Only Broker/Coordinator/Router carry a listener class; Historical/MiddleManager don't. + // Every role is merged: each has its one `default` role group. + let default = RoleGroupName::from_str("default").expect("valid role group name"); assert!( validated - .role_config(&DruidRole::Broker) - .listener_class - .is_some() + .coordinator_role_group_configs + .contains_key(&default) ); + assert!(validated.broker_role_group_configs.contains_key(&default)); assert!( validated - .role_config(&DruidRole::Coordinator) - .listener_class - .is_some() + .historical_role_group_configs + .contains_key(&default) ); assert!( validated - .role_config(&DruidRole::Router) - .listener_class - .is_some() + .middle_manager_role_group_configs + .contains_key(&default) ); - assert!( - validated - .role_config(&DruidRole::Historical) - .listener_class - .is_none() + assert!(validated.router_role_group_configs.contains_key(&default)); + + // The externally reachable roles (Broker, Coordinator, Router) carry the default listener + // class. + assert_eq!( + validated.coordinator_config.listener_class, + *DRUID_DEFAULT_LISTENER_CLASS ); - assert!( - validated - .role_config(&DruidRole::MiddleManager) - .listener_class - .is_none() + assert_eq!( + validated.broker_config.listener_class, + *DRUID_DEFAULT_LISTENER_CLASS + ); + assert_eq!( + validated.router_config.listener_class, + *DRUID_DEFAULT_LISTENER_CLASS ); // Defaults from the dereferenced inputs flow through. diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index b340e57a..844b2c5f 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -91,7 +91,7 @@ mod tests { }; use super::*; - use crate::crd::v1alpha1; + use crate::controller::validate::test_support::{druid_from_yaml, validated_cluster}; #[rstest] #[case(DruidRole::Coordinator)] @@ -100,13 +100,13 @@ mod tests { #[case(DruidRole::MiddleManager)] #[case(DruidRole::Router)] fn test_affinity_defaults(#[case] role: DruidRole) { - use stackable_operator::utils::yaml_from_str_singleton_map; - let input = r#" apiVersion: druid.stackable.tech/v1alpha1 kind: DruidCluster metadata: name: simple-druid + namespace: default + uid: c27b3971-ca72-42c1-80a4-abdfc1db0ddd spec: image: productVersion: 30.0.0 @@ -142,11 +142,40 @@ mod tests { default: replicas: 1 "#; - let druid: v1alpha1::DruidCluster = yaml_from_str_singleton_map(input).unwrap(); - let merged_role = druid.merged_role(&role).unwrap(); - let merged_config = merged_role - .get(&RoleGroupName::from_str("default").unwrap()) - .unwrap(); + let cluster = validated_cluster(&druid_from_yaml(input)); + let role_group_name = RoleGroupName::from_str("default").expect("valid role group name"); + let affinity = match role { + DruidRole::Coordinator => { + &cluster.coordinator_role_group_configs[&role_group_name] + .config + .config + .affinity + } + DruidRole::Broker => { + &cluster.broker_role_group_configs[&role_group_name] + .config + .config + .affinity + } + DruidRole::Historical => { + &cluster.historical_role_group_configs[&role_group_name] + .config + .config + .affinity + } + DruidRole::MiddleManager => { + &cluster.middle_manager_role_group_configs[&role_group_name] + .config + .config + .affinity + } + DruidRole::Router => { + &cluster.router_role_group_configs[&role_group_name] + .config + .config + .affinity + } + }; let mut expected_affinities = vec![]; @@ -257,7 +286,7 @@ mod tests { }; assert_eq!( - merged_config.config.affinity, + affinity.clone(), StackableAffinity { pod_affinity: Some(PodAffinity { preferred_during_scheduling_ignored_during_execution: Some(expected_affinities), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 30832298..0a6697f9 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,13 +1,8 @@ -use std::{ - collections::{BTreeMap, HashSet}, - ops::Deref, - str::FromStr, -}; +use std::{collections::HashSet, ops::Deref, str::FromStr}; use indoc::formatdoc; use security::add_cert_to_jvm_trust_store_cmd; use serde::{Deserialize, Serialize}; -use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ commons::{ affinity::StackableAffinity, @@ -15,10 +10,7 @@ use stackable_operator::{ product_image_selection::ProductImage, resources::{NoRuntimeLimits, Resources}, }, - config::{ - fragment::{Fragment, ValidationError}, - merge::Merge, - }, + config::{fragment::Fragment, merge::Merge}, constant, crd::{ authentication::{core, oidc}, @@ -26,7 +18,7 @@ use stackable_operator::{ }, deep_merger::ObjectOverrides, k8s_openapi::api::core::v1::Volume, - kube::{CustomResource, ResourceExt}, + kube::CustomResource, product_logging::{ self, framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, @@ -40,15 +32,12 @@ use stackable_operator::{ v2::{ builder::pod::container::EnvVarName, config_overrides::KeyValueConfigOverrides, - product_logging::framework::{ - STACKABLE_LOG_DIR, ValidatedContainerLogConfigChoice, VectorContainerLogConfig, - validate_logging_configuration_for_container, - }, - role_utils::{JavaCommonConfig, Role, RoleGroupConfig, with_validated_config}, + product_logging::framework::STACKABLE_LOG_DIR, + role_utils::{JavaCommonConfig, Role}, types::{ common::Port, kubernetes::{ConfigMapName, ContainerName, ListenerClassName, SecretKey}, - operator::{RoleGroupName, RoleName}, + operator::RoleName, }, }, versioned::versioned, @@ -59,7 +48,6 @@ use crate::crd::{ affinity::get_affinity, authorization::DruidAuthorization, database::MetadataDatabaseConnection, - resource::RoleResource, tls::{DruidTls, default_druid_tls}, }; @@ -150,35 +138,6 @@ pub struct DruidConfigOverrides { pub security_properties: KeyValueConfigOverrides, } -#[derive(Snafu, Debug)] -#[allow(clippy::enum_variant_names)] -pub enum Error { - #[snafu(display("missing secret lifetime"))] - MissingSecretLifetime, - - #[snafu(display("failed to merge and validate config for role group {role_group:?}"))] - FailedToMergeRoleGroupConfig { - source: ValidationError, - role_group: String, - }, - - #[snafu(display("invalid role group name {role_group:?}"))] - ParseRoleGroupName { - source: stackable_operator::v2::macros::attributed_string_type::Error, - role_group: String, - }, - - #[snafu(display("failed to validate container log configuration"))] - ValidateLoggingConfig { - source: stackable_operator::v2::product_logging::framework::Error, - }, - - #[snafu(display( - "the Vector agent is enabled but the Vector aggregator ConfigMap name is missing" - ))] - MissingVectorAggregatorConfigMapName, -} - #[versioned( version(name = "v1alpha1"), crates( @@ -339,143 +298,6 @@ impl HasStatusCondition for v1alpha1::DruidCluster { } } -impl v1alpha1::DruidCluster { - /// Merges and validates all role groups of the given role. Invoked from the validate step - /// (`controller::validate`). - /// - /// All four override categories (config / env / cli / pod) are merged by - /// [`with_validated_config`] (role group wins over role); the typed per-role config is then - /// erased to the shared [`ValidatedDruidConfig`] view consumed by the build step. - /// - /// This lives in `crate::crd` and is a `macro_rules!` rather than a generic function on purpose: the erasure - /// reads the per-role configs' private `resources` field and calls their private - /// `default_config`, and the five roles wrap *different* `resources` types into different - /// [`RoleResource`] variants. - pub fn merged_role( - &self, - role: &DruidRole, - ) -> Result, Error> { - let deep_storage = &self.spec.cluster_config.deep_storage; - let name = self.name_any(); - - // The Vector aggregator discovery ConfigMap name (a typed `ConfigMapName`, so an invalid - // name is already rejected at deserialization). It is only required when the Vector agent - // is enabled for a role group. - let vector_aggregator_config_map_name = self - .spec - .cluster_config - .vector_aggregator_config_map_name - .clone(); - - // All roles erase to an identical `ValidatedDruidConfig`; only the typed config, the role - // config type and the `RoleResource` variant (historicals carry typed storage) differ. - macro_rules! merged_role { - ($field:ident, $config:ty, $role_config:ty, $resource:path) => {{ - let typed_role = &self.spec.$field; - let default_config = <$config>::default_config(&name, role, deep_storage); - let mut groups = BTreeMap::new(); - for (rg_name, rg) in &typed_role.role_groups { - let validated = with_validated_config::< - $config, - JavaCommonConfig, - _, - $role_config, - DruidConfigOverrides, - >(rg, typed_role, &default_config) - .with_context(|_| FailedToMergeRoleGroupConfigSnafu { - role_group: rg_name.clone(), - })?; - let common = ValidatedDruidConfig { - resources: $resource(validated.config.config.resources), - logging: validate_logging( - &validated.config.config.logging, - &vector_aggregator_config_map_name, - )?, - affinity: validated.config.config.affinity, - graceful_shutdown_timeout: validated - .config - .config - .graceful_shutdown_timeout, - requested_secret_lifetime: validated - .config - .config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?, - }; - let role_group_name = RoleGroupName::from_str(rg_name).with_context(|_| { - ParseRoleGroupNameSnafu { - role_group: rg_name.clone(), - } - })?; - groups.insert( - role_group_name, - DruidRoleGroupConfig { - replicas: validated.replicas, - config: common, - config_overrides: validated.config.config_overrides, - env_overrides: validated.config.env_overrides.into(), - cli_overrides: validated.config.cli_overrides, - pod_overrides: validated.config.pod_overrides, - product_specific_common_config: validated - .config - .product_specific_common_config, - }, - ); - } - groups - }}; - } - - let groups = match role { - DruidRole::Broker => { - merged_role!( - brokers, - BrokerConfig, - v1alpha1::DruidRoleConfig, - RoleResource::Druid - ) - } - DruidRole::Coordinator => merged_role!( - coordinators, - CoordinatorConfig, - v1alpha1::DruidRoleConfig, - RoleResource::Druid - ), - DruidRole::Historical => merged_role!( - historicals, - HistoricalConfig, - GenericRoleConfig, - RoleResource::Historical - ), - DruidRole::MiddleManager => merged_role!( - middle_managers, - MiddleManagerConfig, - GenericRoleConfig, - RoleResource::Druid - ), - DruidRole::Router => { - merged_role!( - routers, - RouterConfig, - v1alpha1::DruidRoleConfig, - RoleResource::Druid - ) - } - }; - Ok(groups) - } - - pub fn generic_role_config(&self, role: &DruidRole) -> &GenericRoleConfig { - match role { - DruidRole::Broker => &self.spec.brokers.role_config.generic, - DruidRole::Coordinator => &self.spec.coordinators.role_config.generic, - DruidRole::Historical => &self.spec.historicals.role_config, - DruidRole::MiddleManager => &self.spec.middle_managers.role_config, - DruidRole::Router => &self.spec.routers.role_config.generic, - } - } -} - #[derive( Clone, Debug, @@ -514,69 +336,6 @@ impl Container { } } -/// The validated, merged Druid config shared by all role groups (the typed per-role config erased -/// to a single view). Mirrors the opensearch-operator's `ValidatedOpenSearchConfig`. -#[derive(Clone)] -pub struct ValidatedDruidConfig { - pub affinity: StackableAffinity, - pub graceful_shutdown_timeout: Option, - pub logging: ValidatedLogging, - pub requested_secret_lifetime: Duration, - pub resources: RoleResource, -} - -/// Validated logging configuration for the Druid, Prepare (init) and (optional) Vector containers. -/// -/// Produced up-front by [`validate_logging`] (mirroring the hive-/opensearch-operator) so that an -/// invalid custom log ConfigMap name or a missing Vector aggregator discovery ConfigMap name fails -/// reconciliation during validation rather than at resource-build time. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ValidatedLogging { - pub druid_container: ValidatedContainerLogConfigChoice, - pub prepare_container: ValidatedContainerLogConfigChoice, - pub vector_container: Option, - pub enable_vector_agent: bool, -} - -/// Validates the logging configuration for the Druid, Prepare and (optional) Vector containers. -/// -/// `vector_aggregator_config_map_name` is the discovery ConfigMap name of the Vector aggregator; -/// it is required (and was validated up-front) only when the Vector agent is enabled. -fn validate_logging( - logging: &Logging, - vector_aggregator_config_map_name: &Option, -) -> Result { - let druid_container = validate_logging_configuration_for_container(logging, &Container::Druid) - .context(ValidateLoggingConfigSnafu)?; - let prepare_container = - validate_logging_configuration_for_container(logging, &Container::Prepare) - .context(ValidateLoggingConfigSnafu)?; - - let vector_container = if logging.enable_vector_agent { - let vector_aggregator_config_map_name = vector_aggregator_config_map_name - .clone() - .context(MissingVectorAggregatorConfigMapNameSnafu)?; - Some(VectorContainerLogConfig { - log_config: validate_logging_configuration_for_container(logging, &Container::Vector) - .context(ValidateLoggingConfigSnafu)?, - vector_aggregator_config_map_name, - }) - } else { - None - }; - - Ok(ValidatedLogging { - druid_container, - prepare_container, - vector_container, - enable_vector_agent: logging.enable_vector_agent, - }) -} - -/// A validated, merged role-group config. -pub type DruidRoleGroupConfig = - RoleGroupConfig; - impl Default for v1alpha1::DruidRoleConfig { fn default() -> Self { v1alpha1::DruidRoleConfig { @@ -586,7 +345,7 @@ impl Default for v1alpha1::DruidRoleConfig { } } -constant!(DRUID_DEFAULT_LISTENER_CLASS: ListenerClassName = "cluster-internal"); +constant!(pub(crate) DRUID_DEFAULT_LISTENER_CLASS: ListenerClassName = "cluster-internal"); /// Serde default for `listenerClass`. Kept as a function because `#[serde(default = "...")]` /// requires a function path. @@ -623,6 +382,15 @@ impl Deref for DruidRole { } } +/// Formats the role as its role name. `DruidRole` already derefs to [`RoleName`], but snafu +/// `display` strings format a field by its own type, so the role needs `Display` itself. +impl std::fmt::Display for DruidRole { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let role_name: &RoleName = self; + role_name.fmt(f) + } +} + impl DruidRole { /// Returns the name of the internal druid process name associated with the role. /// These strings are used by druid internally to identify processes. @@ -730,22 +498,6 @@ impl DruidRole { create_vector_shutdown_file_command(STACKABLE_LOG_DIR), } } - - pub fn listener_class_name(&self, druid: &v1alpha1::DruidCluster) -> Option { - match self { - Self::Broker => Some(druid.spec.brokers.role_config.listener_class.to_owned()), - Self::Coordinator => Some( - druid - .spec - .coordinators - .role_config - .listener_class - .to_owned(), - ), - Self::Router => Some(druid.spec.routers.role_config.listener_class.to_owned()), - Self::Historical | Self::MiddleManager => None, - } - } } #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize, Display)] @@ -817,7 +569,7 @@ macro_rules! role_group_config { )] pub struct $name { #[fragment_attrs(serde(default))] - resources: Resources<$storage, NoRuntimeLimits>, + pub(crate) resources: Resources<$storage, NoRuntimeLimits>, #[fragment_attrs(serde(default))] pub logging: Logging, #[fragment_attrs(serde(default))] @@ -835,7 +587,7 @@ macro_rules! role_group_config { } impl $name { - fn default_config( + pub(crate) fn default_config( cluster_name: &str, role: &DruidRole, deep_storage: &DeepStorageSpec, diff --git a/rust/operator-binary/src/crd/resource.rs b/rust/operator-binary/src/crd/resource.rs index 9512afc5..55500fe8 100644 --- a/rust/operator-binary/src/crd/resource.rs +++ b/rust/operator-binary/src/crd/resource.rs @@ -1,16 +1,15 @@ -use std::{collections::BTreeMap, str::FromStr, sync::LazyLock}; +use std::{str::FromStr, sync::LazyLock}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ - builder, - builder::pod::{PodBuilder, container::ContainerBuilder, volume::VolumeBuilder}, + builder::pod::volume::VolumeBuilder, commons::resources::{ CpuLimitsFragment, MemoryLimitsFragment, NoRuntimeLimits, NoRuntimeLimitsFragment, Resources, ResourcesFragment, }, constant, k8s_openapi::{ - api::core::v1::{EmptyDirVolumeSource, ResourceRequirements}, + api::core::v1::{EmptyDirVolumeSource, Volume, VolumeMount}, apimachinery::pkg::api::resource::Quantity, }, memory::MemoryQuantity, @@ -18,23 +17,17 @@ use stackable_operator::{ }; use crate::crd::{ - DruidRole, PROP_SEGMENT_CACHE_LOCATIONS, - memory::{HistoricalDerivedSettings, RESERVED_OS_MEMORY}, - storage::{self, default_free_percentage_empty_dir_fragment}, + memory::RESERVED_OS_MEMORY, + storage::{self, FreePercentageEmptyDir, default_free_percentage_empty_dir_fragment}, }; const PATH_SEGMENT_CACHE: &str = "/stackable/var/druid/segment-cache"; // volume names -constant!(SEGMENT_CACHE_VOLUME_NAME: VolumeName = "segment-cache"); +constant!(pub(crate) SEGMENT_CACHE_VOLUME_NAME: VolumeName = "segment-cache"); -/// This Error cannot derive PartialEq because fragment::ValidationError doesn't derive it #[derive(Snafu, Debug)] -#[allow(clippy::enum_variant_names)] pub enum Error { - #[snafu(display("failed to derive Druid settings from resources"))] - DeriveMemorySettings { source: crate::crd::memory::Error }, - #[snafu(display("failed to get memory limits"))] GetMemoryLimit, @@ -42,131 +35,58 @@ pub enum Error { ParseMemoryQuantity { source: stackable_operator::memory::Error, }, - - #[snafu(display("the operator produced an internally inconsistent state"))] - InconsistentConfiguration, - - #[snafu(display("failed to add needed volume"))] - AddVolume { source: builder::pod::Error }, } -#[derive(Debug, Clone, PartialEq)] -pub enum RoleResource { - Druid(Resources), - Historical(Resources), +/// The heap size of a role whose JVM settings are not derived from its CPU: the memory limit +/// minus the OS reserve and minus the role's direct memory, if it has any. +pub fn heap_memory( + resources: &Resources, + direct_memory: Option, +) -> Result { + let total_memory = MemoryQuantity::try_from( + resources + .memory + .limit + .as_ref() + .context(GetMemoryLimitSnafu)?, + ) + .context(ParseMemoryQuantitySnafu)?; + let heap_memory = total_memory - *RESERVED_OS_MEMORY; + Ok(match direct_memory { + Some(direct_memory) => heap_memory - direct_memory, + None => heap_memory, + }) } -impl RoleResource { - pub fn as_resource_requirements(&self) -> ResourceRequirements { - match self { - Self::Druid(r) => r.clone().into(), - Self::Historical(r) => r.clone().into(), - } - } - - /// Update the given configuration file with resource properties. - /// Currently it only adds historical-specific configs for direct memory buffers, thread counts and segment cache. - pub fn update_druid_config_file( - &self, - config: &mut BTreeMap, - ) -> Result<(), Error> { - match self { - Self::Historical(r) => { - let free_percentage = r.storage.segment_cache.free_percentage.unwrap_or(5u16); - let capacity = &r.storage.segment_cache.empty_dir.capacity; - config - .entry(PROP_SEGMENT_CACHE_LOCATIONS.to_string()) - .or_insert_with(|| { - format!( - r#"[{{"path":"{}","maxSize":"{}","freeSpacePercent":"{}"}}]"#, - PATH_SEGMENT_CACHE, capacity.0, free_percentage - ) - }); - - let settings = - HistoricalDerivedSettings::try_from(r).context(DeriveMemorySettingsSnafu)?; - settings.add_settings(config); - } - Self::Druid(_) => (), - } - Ok(()) - } - - /// Adds the segment cache volume and its mount for the Historical role. - /// - /// # Panics - /// - /// Panics if the volume mount cannot be added to the container builder. Only call this on a - /// container builder whose mount paths are still distinct from the one added here. - pub fn update_volumes_and_volume_mounts( - &self, - cb: &mut ContainerBuilder, - pb: &mut PodBuilder, - ) -> Result<(), Error> { - if let Self::Historical(r) = self { - cb.add_volume_mount(&*SEGMENT_CACHE_VOLUME_NAME, PATH_SEGMENT_CACHE) - .expect( - "The mount paths are statically defined and there should be no duplicates.", - ); - pb.add_volume( - VolumeBuilder::new(&*SEGMENT_CACHE_VOLUME_NAME) - .empty_dir(EmptyDirVolumeSource { - medium: r.storage.segment_cache.empty_dir.medium.clone(), - size_limit: Some(r.storage.segment_cache.empty_dir.capacity.clone()), - }) - .build(), - ) - .context(AddVolumeSnafu)?; - } +/// The value of `druid.segmentCache.locations` for a Historical: its one segment cache location, +/// sized from the segment cache storage. +pub fn segment_cache_locations(segment_cache: &FreePercentageEmptyDir) -> String { + let free_percentage = segment_cache.free_percentage.unwrap_or(5u16); + let capacity = &segment_cache.empty_dir.capacity; + format!( + r#"[{{"path":"{path}","maxSize":"{max_size}","freeSpacePercent":"{free_space_percent}"}}]"#, + path = PATH_SEGMENT_CACHE, + max_size = capacity.0, + free_space_percent = free_percentage, + ) +} - Ok(()) - } +/// The segment cache volume of a Historical: an `emptyDir` sized from the segment cache storage. +pub fn segment_cache_volume(segment_cache: &FreePercentageEmptyDir) -> Volume { + VolumeBuilder::new(&*SEGMENT_CACHE_VOLUME_NAME) + .empty_dir(EmptyDirVolumeSource { + medium: segment_cache.empty_dir.medium.clone(), + size_limit: Some(segment_cache.empty_dir.capacity.clone()), + }) + .build() +} - /// Computes the heap and direct access memory sizes per role. The settings can be used to configure - /// the JVM accordingly. The direct memory size is an [`Option`] because not all roles require - /// direct access memory. - pub fn get_memory_sizes( - &self, - role: &DruidRole, - ) -> Result<(MemoryQuantity, Option), Error> { - match self { - Self::Historical(r) => { - let settings = - HistoricalDerivedSettings::try_from(r).context(DeriveMemorySettingsSnafu)?; - Ok(( - settings.heap_memory(), - Some(settings.direct_access_memory()), - )) - } - Self::Druid(r) => { - let total_memory = - MemoryQuantity::try_from(r.memory.limit.as_ref().context(GetMemoryLimitSnafu)?) - .context(ParseMemoryQuantitySnafu)?; - match role { - DruidRole::Historical => Err(Error::InconsistentConfiguration), - DruidRole::Coordinator => { - // The coordinator needs no direct memory - let heap_memory = total_memory - *RESERVED_OS_MEMORY; - Ok((heap_memory, None)) - } - DruidRole::Broker => { - let direct_memory = MemoryQuantity::from_mebi(400.); - let heap_memory = total_memory - *RESERVED_OS_MEMORY - direct_memory; - Ok((heap_memory, Some(direct_memory))) - } - DruidRole::MiddleManager => { - // The middle manager needs no direct memory - let heap_memory = total_memory - *RESERVED_OS_MEMORY; - Ok((heap_memory, None)) - } - DruidRole::Router => { - let direct_memory = MemoryQuantity::from_mebi(128.); - let heap_memory = total_memory - *RESERVED_OS_MEMORY - direct_memory; - Ok((heap_memory, Some(direct_memory))) - } - } - } - } +/// The `druid` container's mount of the segment cache volume. +pub fn segment_cache_volume_mount() -> VolumeMount { + VolumeMount { + name: SEGMENT_CACHE_VOLUME_NAME.to_string(), + mount_path: PATH_SEGMENT_CACHE.to_string(), + ..VolumeMount::default() } } @@ -259,10 +179,12 @@ mod test { }; use super::*; - use crate::crd::{ - DruidRole, - storage::{HistoricalStorage, default_free_percentage_empty_dir}, - v1alpha1, + use crate::{ + controller::validate::test_support::validated_cluster, + crd::{ + storage::{HistoricalStorage, default_free_percentage_empty_dir}, + v1alpha1, + }, }; #[test] @@ -416,19 +338,23 @@ mod test { } #[test] - fn test_resources() -> Result<(), Error> { + fn test_resources() { let cluster: v1alpha1::DruidCluster = yaml_from_str_singleton_map(include_str!( "../../test/resources/crd/resource_merge/druid_cluster.yaml" )) .expect("failed to parse YAML"); - - let middle_managers = cluster.merged_role(&DruidRole::MiddleManager).unwrap(); - - if let Some(RoleResource::Druid(middlemanager_resources_from_rg)) = middle_managers - .get(&RoleGroupName::from_str("resources-from-role-group").unwrap()) - .map(|rg| &rg.config.resources) - { - let expected = Resources { + let validated = validated_cluster(&cluster); + let middle_manager_resources = |role_group: &str| { + &validated.middle_manager_role_group_configs + [&RoleGroupName::from_str(role_group).expect("valid role group name")] + .config + .config + .resources + }; + + assert_eq!( + middle_manager_resources("resources-from-role-group"), + &Resources { cpu: CpuLimits { min: Some(Quantity("300m".to_owned())), max: Some(Quantity("3".to_owned())), @@ -438,21 +364,13 @@ mod test { runtime_limits: NoRuntimeLimits {}, }, storage: storage::DruidStorage {}, - }; - - assert_eq!( - middlemanager_resources_from_rg, &expected, - "middlemanager resources from role group" - ); - } else { - panic!("No role group named [resources-from-role-group] found"); - } + }, + "middlemanager resources from role group" + ); - if let Some(RoleResource::Druid(middlemanager_resources_from_rg)) = middle_managers - .get(&RoleGroupName::from_str("resources-from-role").unwrap()) - .map(|rg| &rg.config.resources) - { - let expected = Resources { + assert_eq!( + middle_manager_resources("resources-from-role"), + &Resources { cpu: CpuLimits { min: Some(Quantity("100m".to_owned())), max: Some(Quantity("1".to_owned())), @@ -462,58 +380,37 @@ mod test { runtime_limits: NoRuntimeLimits {}, }, storage: storage::DruidStorage {}, - }; - - assert_eq!( - middlemanager_resources_from_rg, &expected, - "resources from role" - ); - } else { - panic!("No role group named [resources-from-role] found"); - } - - Ok(()) + }, + "resources from role" + ); } #[test] - fn test_segment_cache() -> Result<(), Error> { + fn test_segment_cache() { let cluster: v1alpha1::DruidCluster = yaml_from_str_singleton_map(include_str!( "../../test/resources/crd/resource_merge/segment_cache.yaml" )) .expect("failed to parse YAML"); - - let historicals = cluster.merged_role(&DruidRole::Historical).unwrap(); - - // ---------- default role group - let res = &historicals - .get(&RoleGroupName::from_str("default").unwrap()) - .unwrap() - .config - .resources; - let mut got = BTreeMap::new(); - - assert!(res.update_druid_config_file(&mut got).is_ok()); - assert!(got.contains_key(PROP_SEGMENT_CACHE_LOCATIONS)); - - let value = got.get(PROP_SEGMENT_CACHE_LOCATIONS).unwrap(); - let expected = r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"5g","freeSpacePercent":"3"}]"#.to_string(); - assert_eq!(value, &expected, "primary"); - - // ---------- secondary role group - let res = &historicals - .get(&RoleGroupName::from_str("secondary").unwrap()) - .unwrap() - .config - .resources; - let mut got = BTreeMap::new(); - - assert!(res.update_druid_config_file(&mut got).is_ok()); - assert!(got.contains_key(PROP_SEGMENT_CACHE_LOCATIONS)); - - let value = got.get(PROP_SEGMENT_CACHE_LOCATIONS).unwrap(); - let expected = r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"2g","freeSpacePercent":"7"}]"#.to_string(); - assert_eq!(value, &expected, "secondary"); - - Ok(()) + let validated = validated_cluster(&cluster); + let historical_segment_cache = |role_group: &str| { + &validated.historical_role_group_configs + [&RoleGroupName::from_str(role_group).expect("valid role group name")] + .config + .config + .resources + .storage + .segment_cache + }; + + assert_eq!( + segment_cache_locations(historical_segment_cache("default")), + r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"5g","freeSpacePercent":"3"}]"#, + "primary" + ); + assert_eq!( + segment_cache_locations(historical_segment_cache("secondary")), + r#"[{"path":"/stackable/var/druid/segment-cache","maxSize":"2g","freeSpacePercent":"7"}]"#, + "secondary" + ); } } diff --git a/rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml b/rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml index a2a13549..d6aa2b0d 100644 --- a/rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml +++ b/rust/operator-binary/test/resources/crd/resource_merge/druid_cluster.yaml @@ -4,7 +4,7 @@ kind: DruidCluster metadata: name: test-resource-merge namespace: default - uid: test-resource-merge + uid: 7d1c3f0a-5b2e-4c8d-9e6f-0a1b2c3d4e5f spec: image: productVersion: 37.0.0 diff --git a/rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml b/rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml index 8e95cf26..d40364f2 100644 --- a/rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml +++ b/rust/operator-binary/test/resources/crd/resource_merge/segment_cache.yaml @@ -4,7 +4,7 @@ kind: DruidCluster metadata: name: test-resource-merge namespace: default - uid: test-resource-merge + uid: 7d1c3f0a-5b2e-4c8d-9e6f-0a1b2c3d4e5f spec: image: productVersion: 37.0.0 diff --git a/rust/operator-binary/test/resources/druid_controller/segment_cache.yaml b/rust/operator-binary/test/resources/druid_controller/segment_cache.yaml index e36f24df..d79dfb27 100644 --- a/rust/operator-binary/test/resources/druid_controller/segment_cache.yaml +++ b/rust/operator-binary/test/resources/druid_controller/segment_cache.yaml @@ -4,7 +4,7 @@ kind: DruidCluster metadata: name: test-resource-merge namespace: default - uid: test-resource-merge + uid: 7d1c3f0a-5b2e-4c8d-9e6f-0a1b2c3d4e5f spec: image: productVersion: 37.0.0