fetch kubelet config when initializing operators

Author: razvan

crates/stackable-operator/src/client.rs

@@ -21,7 +21,10 @@ use tracing::trace;
 
 use crate::{
     kvp::LabelSelectorExt,
-    utils::cluster_info::{KubernetesClusterInfo, KubernetesClusterInfoOpts},
+    utils::{
+        cluster_info::{KubernetesClusterInfo, KubernetesClusterInfoOpts},
+        kubelet,
+    },
 };
 
 pub type Result<T, E = Error> = std::result::Result<T, E>;
@@ -84,6 +87,9 @@ pub enum Error {
 
     #[snafu(display("unable to create kubernetes client"))]
     CreateKubeClient { source: kube::Error },
+
+    #[snafu(display("unable to fetch kubelet config"))]
+    KubeletConfig { source: kubelet::Error },
 }
 
 /// This `Client` can be used to access Kubernetes.
@@ -651,7 +657,10 @@ pub async fn initialize_operator(
         .context(InferKubeConfigSnafu)?;
     let default_namespace = kubeconfig.default_namespace.clone();
     let client = kube::Client::try_from(kubeconfig).context(CreateKubeClientSnafu)?;
-    let cluster_info = KubernetesClusterInfo::new(cluster_info_opts);
+    let kubelet_config = kubelet::KubeletConfig::fetch(&client)
+        .await
+        .context(KubeletConfigSnafu)?;
+    let cluster_info = KubernetesClusterInfo::new(&kubelet_config, cluster_info_opts);
 
     Ok(Client::new(
         client,

crates/stackable-operator/src/utils/cluster_info.rs

@@ -1,9 +1,6 @@
-use std::str::FromStr;
-
+use super::kubelet::KubeletConfig;
 use crate::commons::networking::DomainName;
 
-const KUBERNETES_CLUSTER_DOMAIN_DEFAULT: &str = "cluster.local";
-
 /// Some information that we know about the Kubernetes cluster.
 #[derive(Debug, Clone)]
 pub struct KubernetesClusterInfo {
@@ -21,20 +18,19 @@ pub struct KubernetesClusterInfoOpts {
 }
 
 impl KubernetesClusterInfo {
-    pub fn new(cluster_info_opts: &KubernetesClusterInfoOpts) -> Self {
+    pub fn new(
+        kubelet_config: &KubeletConfig,
+        cluster_info_opts: &KubernetesClusterInfoOpts,
+    ) -> Self {
         let cluster_domain = match &cluster_info_opts.kubernetes_cluster_domain {
             Some(cluster_domain) => {
                 tracing::info!(%cluster_domain, "Using configured Kubernetes cluster domain");
 
                 cluster_domain.clone()
             }
             None => {
-                // TODO(sbernauer): Do some sort of advanced auto-detection, see https://github.com/stackabletech/issues/issues/436.
-                // There have been attempts of parsing the `/etc/resolv.conf`, but they have been
-                // reverted. Please read on the linked issue for details.
-                let cluster_domain = DomainName::from_str(KUBERNETES_CLUSTER_DOMAIN_DEFAULT)
-                    .expect("KUBERNETES_CLUSTER_DOMAIN_DEFAULT constant must a valid domain");
-                tracing::info!(%cluster_domain, "Defaulting Kubernetes cluster domain as it has not been configured");
+                let cluster_domain = kubelet_config.cluster_domain.clone();
+                tracing::info!(%cluster_domain, "Using kubelet config cluster domain");
 
                 cluster_domain
             }

crates/stackable-operator/src/utils/kubelet.rs

@@ -1,75 +1,70 @@
+use http;
 use k8s_openapi::api::core::v1::Node;
 use kube::{
     Api,
     api::{ListParams, ResourceExt},
     client::Client,
 };
 use serde::Deserialize;
-use http;
-use snafu::{Snafu, ResultExt, OptionExt};
+use snafu::{OptionExt, ResultExt, Snafu};
+
+use crate::commons::networking::DomainName;
 
 #[derive(Debug, Snafu)]
 pub enum Error {
-
     #[snafu(display("failed to list nodes"))]
-    ListNodes {
-        source: kube::Error,
-    },
+    ListNodes { source: kube::Error },
     #[snafu(display("failed to build proxy/configz request"))]
-    ConfigzRequest {
-        source: http::Error,
-    },
+    ConfigzRequest { source: http::Error },
 
     #[snafu(display("failed to fetch kubelet config from node {node}"))]
-    FetchNodeKubeletConfig {
-        source: kube::Error,
-        node: String,
-    },
+    FetchNodeKubeletConfig { source: kube::Error, node: String },
 
     #[snafu(display("failed to fetch `kubeletconfig` JSON key from configz response"))]
     KubeletConfigJsonKey,
 
     #[snafu(display("failed to deserialize kubelet config JSON"))]
-    KubeletConfigJson {
-        source: serde_json::Error,
-    },
+    KubeletConfigJson { source: serde_json::Error },
 
     #[snafu(display("empty Kubernetes nodes list"))]
     EmptyKubernetesNodesList,
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
-struct KubeletConfig {
-    cluster_domain: String,
+pub struct KubeletConfig {
+    pub cluster_domain: DomainName,
 }
 
 impl KubeletConfig {
     pub async fn fetch(client: &Client) -> Result<Self, Error> {
         let api: Api<Node> = Api::all(client.clone());
-        let nodes = api.list(&ListParams::default()).await.context(ListNodesSnafu)?;
+        let nodes = api
+            .list(&ListParams::default())
+            .await
+            .context(ListNodesSnafu)?;
         let node = nodes.iter().next().context(EmptyKubernetesNodesListSnafu)?;
 
         let name = node.name_any();
 
-        // Query node stats by issuing a request to the admin endpoint.
-        // See https://kubernetes.io/docs/reference/instrumentation/node-metrics/
+        // Query kukbelet config
         let url = format!("/api/v1/nodes/{}/proxy/configz", name);
-        let req = http::Request::get(url).body(Default::default()).context(ConfigzRequestSnafu)?;
+        let req = http::Request::get(url)
+            .body(Default::default())
+            .context(ConfigzRequestSnafu)?;
 
         // Deserialize JSON response as a JSON value. Alternatively, a type that
         // implements `Deserialize` can be used.
-        let resp = client.request::<serde_json::Value>(req).await.context(FetchNodeKubeletConfigSnafu { node: name })?;
+        let resp = client
+            .request::<serde_json::Value>(req)
+            .await
+            .context(FetchNodeKubeletConfigSnafu { node: name })?;
 
         // Our JSON value is an object so we can treat it like a dictionary.
         let summary = resp
             .get("kubeletconfig")
             .context(KubeletConfigJsonKeySnafu)?;
 
-        // The base JSON representation includes a lot of metrics, including
-        // container metrics. Use a `NodeMetrics` type to deserialize only the
-        // values we care about.
         serde_json::from_value::<KubeletConfig>(summary.to_owned()).context(KubeletConfigJsonSnafu)
     }
-
 }

crates/stackable-operator/src/utils/mod.rs

@@ -1,7 +1,7 @@
 pub mod bash;
 pub mod cluster_info;
 pub mod crds;
-mod kubelet;
+pub mod kubelet;
 pub mod logging;
 mod option;
 mod url;