Refactor how RSS constructs the initial blueprint

Author: jgallagher

sled-agent/src/rack_setup/plan/service.rs

@@ -4,8 +4,12 @@
 
 //! Plan generation for "where should services be initialized".
 
+use anyhow::anyhow;
+use anyhow::bail;
+use chrono::Utc;
 use iddqd::IdOrdMap;
 use iddqd::errors::DuplicateItem;
+use iddqd::id_upcast;
 use illumos_utils::zpool::ZpoolName;
 use internal_dns_types::config::{
     DnsConfigBuilder, DnsConfigParams, Host, Zone,
@@ -15,12 +19,16 @@ use nexus_sled_agent_shared::inventory::{
     Inventory, OmicronZoneDataset, SledRole,
 };
 use nexus_types::deployment::{
-    BlueprintPhysicalDiskConfig, BlueprintPhysicalDiskDisposition,
+    Blueprint, BlueprintDatasetConfig, BlueprintDatasetDisposition,
+    BlueprintHostPhase2DesiredSlots, BlueprintPhysicalDiskConfig,
+    BlueprintPhysicalDiskDisposition, BlueprintSledConfig, BlueprintSource,
     BlueprintZoneConfig, BlueprintZoneDisposition, BlueprintZoneImageSource,
-    BlueprintZoneType, OmicronZoneExternalFloatingAddr,
-    OmicronZoneExternalFloatingIp, OmicronZoneExternalSnatIp,
+    BlueprintZoneType, CockroachDbPreserveDowngrade,
+    OmicronZoneExternalFloatingAddr, OmicronZoneExternalFloatingIp,
+    OmicronZoneExternalSnatIp, OximeterReadMode, PendingMgsUpdates,
     blueprint_zone_type,
 };
+use nexus_types::external_api::views::SledState;
 use omicron_common::address::{
     DENDRITE_PORT, DNS_HTTP_PORT, DNS_PORT, Ipv6Subnet, MGD_PORT, MGS_PORT,
     NEXUS_INTERNAL_PORT, NEXUS_LOCKSTEP_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS,
@@ -46,7 +54,7 @@ use omicron_common::policy::{
     SINGLE_NODE_CLICKHOUSE_REDUNDANCY,
 };
 use omicron_uuid_kinds::{
-    DatasetUuid, ExternalIpUuid, GenericUuid, OmicronZoneUuid,
+    BlueprintUuid, DatasetUuid, ExternalIpUuid, GenericUuid, OmicronZoneUuid,
     PhysicalDiskUuid, SledUuid, ZpoolUuid,
 };
 use rand::seq::IndexedRandom;
@@ -58,6 +66,7 @@ use sled_agent_client::{
 use sled_agent_types::rack_init::RackInitializeRequest as Config;
 use sled_agent_types::sled::StartSledAgentRequest;
 use slog::Logger;
+use slog_error_chain::InlineErrorChain;
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV6};
 use std::num::Wrapping;
@@ -158,9 +167,26 @@ impl SledConfig {
     }
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
+#[derive(Clone, Debug)]
+pub(crate) struct PlannedSledDescription {
+    pub(crate) underlay_address: SocketAddrV6,
+    pub(crate) sled_id: SledUuid,
+    pub(crate) config: SledConfig,
+}
+
+impl iddqd::IdOrdItem for PlannedSledDescription {
+    type Key<'a> = SocketAddrV6;
+
+    fn key(&self) -> Self::Key<'_> {
+        self.underlay_address
+    }
+
+    id_upcast!();
+}
+
+#[derive(Clone, Debug)]
 pub struct Plan {
-    pub services: HashMap<SocketAddrV6, SledConfig>,
+    pub all_sleds: IdOrdMap<PlannedSledDescription>,
     pub dns_config: DnsConfigParams,
 }
 
@@ -210,29 +236,6 @@ pub fn from_source_nat_config_to_external_snat_ip(
 }
 
 impl Plan {
-    async fn is_sled_scrimlet(
-        log: &Logger,
-        address: SocketAddrV6,
-    ) -> Result<bool, PlanError> {
-        let dur = std::time::Duration::from_secs(60);
-        let client = reqwest::ClientBuilder::new()
-            .connect_timeout(dur)
-            .timeout(dur)
-            .build()
-            .map_err(PlanError::HttpClient)?;
-        let client = SledAgentClient::new_with_client(
-            &format!("http://{}", address),
-            client,
-            log.new(o!("SledAgentClient" => address.to_string())),
-        );
-
-        let role = client.sled_role_get().await?.into_inner();
-        match role {
-            SledRole::Gimlet => Ok(false),
-            SledRole::Scrimlet => Ok(true),
-        }
-    }
-
     async fn get_inventory(
         log: &Logger,
         address: SocketAddrV6,
@@ -804,13 +807,17 @@ impl Plan {
                 .expect("freshly generated zone IDs are unique");
         }
 
-        let services: HashMap<_, _> = sled_info
+        let all_sleds = sled_info
             .into_iter()
-            .map(|sled_info| (sled_info.sled_address, sled_info.request))
+            .map(|sled_info| PlannedSledDescription {
+                underlay_address: sled_info.sled_address,
+                sled_id: sled_info.sled_id,
+                config: sled_info.request,
+            })
             .collect();
 
         let dns_config = dns_builder.build_full_config_for_initial_generation();
-        Ok(Self { services, dns_config })
+        Ok(Self { all_sleds, dns_config })
     }
 
     pub async fn create(
@@ -828,8 +835,10 @@ impl Plan {
                         let sled_address = get_sled_address(subnet);
                         let inventory =
                             Self::get_inventory(log, sled_address).await?;
-                        let is_scrimlet =
-                            Self::is_sled_scrimlet(log, sled_address).await?;
+                        let is_scrimlet = match inventory.sled_role {
+                            SledRole::Gimlet => false,
+                            SledRole::Scrimlet => true,
+                        };
                         Ok(SledInfo::new(
                             sled_request.body.id,
                             subnet,
@@ -846,6 +855,102 @@ impl Plan {
         let plan = Self::create_transient(config, sled_info)?;
         Ok(plan)
     }
+
+    pub(crate) fn to_blueprint(
+        &self,
+        sled_agent_config_generation: Generation,
+    ) -> anyhow::Result<Blueprint> {
+        let mut blueprint_sleds = BTreeMap::new();
+        for sled_description in &self.all_sleds {
+            let sled_config = &sled_description.config;
+            let mut datasets = IdOrdMap::new();
+            for d in sled_config.datasets.values() {
+                // Only the "Crucible" dataset needs to know the address
+                let address = if *d.name.kind() == DatasetKind::Crucible {
+                    let address = sled_config.zones.iter().find_map(|z| {
+                        if let BlueprintZoneType::Crucible(
+                            blueprint_zone_type::Crucible { address, dataset },
+                        ) = &z.zone_type
+                        {
+                            if &dataset.pool_name == d.name.pool() {
+                                return Some(*address);
+                            }
+                        };
+                        None
+                    });
+                    if address.is_some() {
+                        address
+                    } else {
+                        bail!(
+                            "could not find Crucible zone for zpool {}",
+                            d.name.pool()
+                        )
+                    }
+                } else {
+                    None
+                };
+
+                datasets
+                    .insert_unique(BlueprintDatasetConfig {
+                        disposition: BlueprintDatasetDisposition::InService,
+                        id: d.id,
+                        pool: *d.name.pool(),
+                        kind: d.name.kind().clone(),
+                        address,
+                        compression: d.inner.compression,
+                        quota: d.inner.quota,
+                        reservation: d.inner.reservation,
+                    })
+                    .map_err(|e| {
+                        anyhow!(InlineErrorChain::new(&e).to_string())
+                    })?;
+            }
+
+            blueprint_sleds.insert(
+                sled_description.sled_id,
+                BlueprintSledConfig {
+                    state: SledState::Active,
+                    sled_agent_generation: sled_agent_config_generation,
+                    disks: sled_config.disks.clone(),
+                    datasets,
+                    zones: sled_config.zones.clone(),
+                    host_phase_2:
+                        BlueprintHostPhase2DesiredSlots::current_contents(),
+                    remove_mupdate_override: None,
+                },
+            );
+        }
+
+        let id = BlueprintUuid::new_v4();
+        Ok(Blueprint {
+            id,
+            sleds: blueprint_sleds,
+            pending_mgs_updates: PendingMgsUpdates::new(),
+            parent_blueprint_id: None,
+            internal_dns_version: self.dns_config.generation,
+            // We don't configure external DNS during RSS, so set it to an
+            // initial generation of 1. Nexus will bump this up when it updates
+            // external DNS (including creating the recovery silo).
+            external_dns_version: Generation::new(),
+            target_release_minimum_generation: Generation::new(),
+            nexus_generation: Generation::new(),
+            // Nexus will fill in the CockroachDB values during initialization.
+            cockroachdb_fingerprint: String::new(),
+            cockroachdb_setting_preserve_downgrade:
+                CockroachDbPreserveDowngrade::DoNotModify,
+            // We do not create clickhouse clusters in RSS. We create them via
+            // reconfigurator only.
+            clickhouse_cluster_config: None,
+            // The oximeter read policy always defaults to single node. The
+            // initial generation of this policy in the DB is 1
+            oximeter_read_mode: OximeterReadMode::SingleNode,
+            oximeter_read_version: Generation::new(),
+            time_created: Utc::now(),
+            creator: "RSS".to_string(),
+            comment: "initial blueprint from rack setup".to_string(),
+            source: BlueprintSource::Rss,
+        })
+    }
 }
 
 struct AddressBumpAllocator {
@@ -1429,9 +1534,9 @@ mod tests {
         let plan = Plan::create_transient(&config, sleds)
             .expect("Should have created plan");
 
-        assert_eq!(plan.services.len(), 1);
+        assert_eq!(plan.all_sleds.len(), 1);
 
-        let sled_config = plan.services.iter().next().unwrap().1;
+        let sled_config = &plan.all_sleds.iter().next().unwrap().config;
         assert_eq!(sled_config.disks.len(), DISK_COUNT);
 
         let zone_count = sled_config.zones.len();

sled-agent/src/rack_setup/plan/sled.rs

@@ -6,8 +6,6 @@
 
 use crate::bootstrap::config::BOOTSTRAP_AGENT_RACK_INIT_PORT;
 use omicron_uuid_kinds::SledUuid;
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
 use sled_agent_types::rack_init::RackInitializeRequest as Config;
 use sled_agent_types::sled::StartSledAgentRequest;
 use sled_agent_types::sled::StartSledAgentRequestBody;
@@ -16,7 +14,7 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::net::{Ipv6Addr, SocketAddrV6};
 use uuid::Uuid;
 
-#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)]
+#[derive(Clone, Debug)]
 pub struct Plan {
     pub rack_id: Uuid,
     pub sleds: BTreeMap<SocketAddrV6, StartSledAgentRequest>,
@@ -63,8 +61,6 @@ impl Plan {
             )
         });
 
-        info!(log, "Serializing plan");
-
         let mut sleds = BTreeMap::new();
         for (addr, allocation) in allocations {
             sleds.insert(addr, allocation);