From 5b26fc45acf59b7ba186e88f9799d2e24d4fbfb6 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 11:11:27 -0500 Subject: [PATCH 01/16] builder: add method to add boundary NTP directly instead of always promoting internal NTP --- .../planning/src/blueprint_builder/builder.rs | 180 +++++++++++++----- 1 file changed, 132 insertions(+), 48 deletions(-) diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 06e438fb3e1..5455805d804 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -1813,30 +1813,45 @@ impl<'a> BlueprintBuilder<'a> { self.sled_add_zone(sled_id, zone) } - pub fn sled_promote_internal_ntp_to_boundary_ntp( - &mut self, - sled_id: SledUuid, - image_source: BlueprintZoneImageSource, - external_ip: ExternalSnatNetworkingChoice, - ) -> Result<(), Error> { - // The upstream NTP/DNS servers and domain _should_ come from Nexus and - // be modifiable by the operator, but currently can only be set at RSS. - // We can only promote a new boundary NTP zone by copying these settings - // from an existing one. - let (ntp_servers, dns_servers, domain) = self - .parent_blueprint + // The upstream NTP/DNS servers and domain _should_ come from Nexus and be + // modifiable by the operator, but currently can only be set at RSS. We can + // only promote a new boundary NTP zone by copying these settings from an + // existing one. + fn infer_boundary_ntp_config_from_parent_blueprint( + &self, + ) -> Result { + self.parent_blueprint .all_omicron_zones(BlueprintZoneDisposition::any) .find_map(|(_, z)| match &z.zone_type { - BlueprintZoneType::BoundaryNtp(zone_config) => Some(( - zone_config.ntp_servers.clone(), - zone_config.dns_servers.clone(), - zone_config.domain.clone(), - )), + BlueprintZoneType::BoundaryNtp(zone_config) => { + Some(BoundaryNtpConfig { + ntp_servers: zone_config.ntp_servers.clone(), + dns_servers: zone_config.dns_servers.clone(), + domain: zone_config.domain.clone(), + }) + } _ => None, }) - .ok_or(Error::NoBoundaryNtpZonesInParentBlueprint)?; + .ok_or(Error::NoBoundaryNtpZonesInParentBlueprint) + } - self.sled_promote_internal_ntp_to_boundary_ntp_with_config( + /// Add a new boundary NTP server to a sled. + /// + /// This is unusual: typically during planning we promote internal NTP + /// servers to boundary NTP servers via + /// `sled_promote_internal_ntp_to_boundary_ntp()`, because adding a new + /// boundary NTP zone to a sled is only valid if the sled doesn't currently + /// have any NTP zone at all. Only tests and possibly RSS can really make + /// use of this. + pub fn sled_add_zone_boundary_ntp( + &mut self, + sled_id: SledUuid, + image_source: BlueprintZoneImageSource, + external_ip: ExternalSnatNetworkingChoice, + ) -> Result<(), Error> { + let BoundaryNtpConfig { ntp_servers, dns_servers, domain } = + self.infer_boundary_ntp_config_from_parent_blueprint()?; + self.sled_add_zone_boundary_ntp_with_config( sled_id, ntp_servers, dns_servers, @@ -1846,7 +1861,15 @@ impl<'a> BlueprintBuilder<'a> { ) } - pub fn sled_promote_internal_ntp_to_boundary_ntp_with_config( + /// Add a new boundary NTP server to a sled. + /// + /// This is unusual: typically during planning we promote internal NTP + /// servers to boundary NTP servers via + /// `sled_promote_internal_ntp_to_boundary_ntp()`, because adding a new + /// boundary NTP zone to a sled is only valid if the sled doesn't currently + /// have any NTP zone at all. Only tests and possibly RSS can really make + /// use of this. + pub fn sled_add_zone_boundary_ntp_with_config( &mut self, sled_id: SledUuid, ntp_servers: Vec, @@ -1861,38 +1884,16 @@ impl<'a> BlueprintBuilder<'a> { )) })?; - // Find the internal NTP zone and expunge it. - let mut internal_ntp_zone_id_iter = editor + // Ensure we have no other in-service NTP zones. + if let Some(in_service_ntp_zone) = editor .zones(BlueprintZoneDisposition::is_in_service) - .filter_map(|zone| { - if matches!(zone.zone_type, BlueprintZoneType::InternalNtp(_)) { - Some(zone.id) - } else { - None - } - }); - - // We should have exactly one internal NTP zone. - let internal_ntp_zone_id = - internal_ntp_zone_id_iter.next().ok_or_else(|| { - Error::Planner(anyhow!( - "cannot promote internal NTP zone on sled {sled_id}: \ - no internal NTP zone found" - )) - })?; - if internal_ntp_zone_id_iter.next().is_some() { + .find(|zone| zone.zone_type.is_ntp()) + { return Err(Error::Planner(anyhow!( - "sled {sled_id} has multiple internal NTP zones" + "attempted to add boundary NTP zone to sled {sled_id} which \ + already has an in-service NTP zone: {in_service_ntp_zone:?}" ))); } - std::mem::drop(internal_ntp_zone_id_iter); - - // Expunge the internal NTP zone. - editor.expunge_zone(&internal_ntp_zone_id).map_err(|error| { - Error::Planner(anyhow!(error).context(format!( - "error expunging internal NTP zone from sled {sled_id}" - ))) - })?; // Add the new boundary NTP zone. let new_zone_id = self.rng.sled_rng(sled_id).next_zone(); @@ -1947,6 +1948,83 @@ impl<'a> BlueprintBuilder<'a> { ) } + pub fn sled_promote_internal_ntp_to_boundary_ntp( + &mut self, + sled_id: SledUuid, + image_source: BlueprintZoneImageSource, + external_ip: ExternalSnatNetworkingChoice, + ) -> Result<(), Error> { + let BoundaryNtpConfig { ntp_servers, dns_servers, domain } = + self.infer_boundary_ntp_config_from_parent_blueprint()?; + self.sled_promote_internal_ntp_to_boundary_ntp_with_config( + sled_id, + ntp_servers, + dns_servers, + domain, + image_source, + external_ip, + ) + } + + pub fn sled_promote_internal_ntp_to_boundary_ntp_with_config( + &mut self, + sled_id: SledUuid, + ntp_servers: Vec, + dns_servers: Vec, + domain: Option, + image_source: BlueprintZoneImageSource, + external_ip: ExternalSnatNetworkingChoice, + ) -> Result<(), Error> { + let editor = self.sled_editors.get_mut(&sled_id).ok_or_else(|| { + Error::Planner(anyhow!( + "tried to promote NTP zone on unknown sled {sled_id}" + )) + })?; + + // Find the internal NTP zone and expunge it. + let mut internal_ntp_zone_id_iter = editor + .zones(BlueprintZoneDisposition::is_in_service) + .filter_map(|zone| { + if matches!(zone.zone_type, BlueprintZoneType::InternalNtp(_)) { + Some(zone.id) + } else { + None + } + }); + + // We should have exactly one internal NTP zone. + let internal_ntp_zone_id = + internal_ntp_zone_id_iter.next().ok_or_else(|| { + Error::Planner(anyhow!( + "cannot promote internal NTP zone on sled {sled_id}: \ + no internal NTP zone found" + )) + })?; + if internal_ntp_zone_id_iter.next().is_some() { + return Err(Error::Planner(anyhow!( + "sled {sled_id} has multiple internal NTP zones" + ))); + } + std::mem::drop(internal_ntp_zone_id_iter); + + // Expunge the internal NTP zone. + editor.expunge_zone(&internal_ntp_zone_id).map_err(|error| { + Error::Planner(anyhow!(error).context(format!( + "error expunging internal NTP zone from sled {sled_id}" + ))) + })?; + + // Add the new boundary NTP zone. + self.sled_add_zone_boundary_ntp_with_config( + sled_id, + ntp_servers, + dns_servers, + domain, + image_source, + external_ip, + ) + } + pub fn sled_expunge_zone( &mut self, sled_id: SledUuid, @@ -2542,6 +2620,12 @@ impl fmt::Display for BpMupdateOverrideNotClearedReason { } } +struct BoundaryNtpConfig { + ntp_servers: Vec, + dns_servers: Vec, + domain: Option, +} + #[cfg(test)] pub mod test { use super::*; From 5cef8ac337db00e08ea0ec114ea1a795158f37b2 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 11:15:17 -0500 Subject: [PATCH 02/16] use builder in rack_set_initialized_with_services() --- nexus/db-queries/src/db/datastore/rack.rs | 381 ++++++++---------- .../allocators/external_networking.rs | 3 +- 2 files changed, 163 insertions(+), 221 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 92854041b6b..ce8eef59a43 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -1067,6 +1067,9 @@ mod test { use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; use nexus_db_model::{DnsGroup, Generation, InitialDnsGroup}; use nexus_inventory::now_db_precision; + use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; + use nexus_reconfigurator_planning::blueprint_editor::ExternalNetworkingAllocator; + use nexus_reconfigurator_planning::planner::PlannerRng; use nexus_reconfigurator_planning::system::{ SledBuilder, SystemDescription, }; @@ -1077,13 +1080,13 @@ mod test { use nexus_types::deployment::CockroachDbPreserveDowngrade; use nexus_types::deployment::ExternalIpPolicy; use nexus_types::deployment::PendingMgsUpdates; + use nexus_types::deployment::SledFilter; use nexus_types::deployment::{ BlueprintZoneConfig, OmicronZoneExternalFloatingAddr, OmicronZoneExternalFloatingIp, }; use nexus_types::deployment::{ - BlueprintZoneDisposition, BlueprintZoneImageSource, - OmicronZoneExternalSnatIp, OximeterReadMode, + BlueprintZoneDisposition, BlueprintZoneImageSource, OximeterReadMode, }; use nexus_types::external_api::shared::SiloIdentityMode; use nexus_types::external_api::views::SledState; @@ -1093,13 +1096,12 @@ mod test { use nexus_types::inventory::NetworkInterfaceKind; use omicron_common::address::NEXUS_OPTE_IPV6_SUBNET; use omicron_common::address::{ - DNS_OPTE_IPV4_SUBNET, NEXUS_OPTE_IPV4_SUBNET, NTP_OPTE_IPV4_SUBNET, + DNS_OPTE_IPV4_SUBNET, NEXUS_OPTE_IPV4_SUBNET, }; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::{ IdentityMetadataCreateParams, MacAddr, Vni, }; - use omicron_common::api::internal::shared::SourceNatConfig; use omicron_common::zpool_name::ZpoolName; use omicron_test_utils::dev; use omicron_uuid_kinds::BlueprintUuid; @@ -1417,8 +1419,16 @@ mod test { Ipv4Addr::new(1, 2, 3, 6), )) .unwrap(); - let external_ip_policy = - ExternalIpPolicy::single_pool_no_external_dns(service_ip_pool); + let external_ip_policy = { + let mut builder = ExternalIpPolicy::builder(); + builder + .push_service_pool_range(service_ip_pool) + .expect("valid pool"); + builder + .add_external_dns_ip("1.2.3.4".parse().unwrap()) + .expect("valid IP"); + builder.build() + }; let mut system = SystemDescription::new(); system @@ -1429,217 +1439,150 @@ mod test { .expect("failed to add sled2") .sled(SledBuilder::new().id(sled3.id())) .expect("failed to add sled3"); + let planning_input = system + .to_planning_input_builder() + .expect("created planning input builder") + .build(); + let empty_starting_blueprint = BlueprintBuilder::build_empty_with_sleds( + std::iter::empty(), + test_name, + ); + let mut external_networking_alloc = + ExternalNetworkingAllocator::from_blueprint( + &empty_starting_blueprint, + planning_input.external_ip_policy(), + ) + .expect("constructed allocator"); + + let mut builder = BlueprintBuilder::new_based_on( + &opctx.log, + &empty_starting_blueprint, + &planning_input, + test_name, + PlannerRng::from_entropy(), + ) + .expect("created blueprint builder"); + for (sled_id, sled_resources) in + planning_input.all_sled_resources(SledFilter::InService) + { + builder + .sled_add_disks(sled_id, &sled_resources) + .expect("added disks"); + } - let external_dns_ip = IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4)); - let external_dns_pip = DNS_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) - .unwrap(); - let external_dns_id = OmicronZoneUuid::new_v4(); - let nexus_ip = IpAddr::V4(Ipv4Addr::new(1, 2, 3, 6)); - let nexus_pip = NEXUS_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) - .unwrap(); - let nexus_id = OmicronZoneUuid::new_v4(); - let ntp1_ip = IpAddr::V4(Ipv4Addr::new(1, 2, 3, 5)); - let ntp1_pip = NTP_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) - .unwrap(); - let ntp1_id = OmicronZoneUuid::new_v4(); - let ntp2_ip = IpAddr::V4(Ipv4Addr::new(1, 2, 3, 5)); - let ntp2_pip = NTP_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 2) - .unwrap(); - let ntp2_id = OmicronZoneUuid::new_v4(); - let ntp3_id = OmicronZoneUuid::new_v4(); - let mut macs = MacAddr::iter_system(); + let external_dns_networking = external_networking_alloc + .for_new_external_dns() + .expect("got IP for external DNS"); + let external_dns_ip = external_dns_networking.external_ip; + + let nexus_networking = external_networking_alloc + .for_new_nexus() + .expect("got IP for Nexus"); + let nexus_ip = nexus_networking.external_ip; + + let ntp1_networking = external_networking_alloc + .for_new_boundary_ntp() + .expect("got IP for boundary NTP"); + let ntp2_networking = external_networking_alloc + .for_new_boundary_ntp() + .expect("got IP for boundary NTP"); + let ntp1_ip = ntp1_networking.snat_cfg.ip; + let ntp2_ip = ntp2_networking.snat_cfg.ip; + + // Add specific zones: + // + // * Sled 1 gets external DNS and boundary NTP + // * Sled 2 gets Nexus and boundary NTP + // * Sled 3 gets internal NTP + builder + .sled_add_zone_external_dns( + sled1.id(), + BlueprintZoneImageSource::InstallDataset, + external_dns_networking, + ) + .expect("added zone"); + builder + .sled_add_zone_nexus_with_config( + sled2.id(), + false, + Vec::new(), + BlueprintZoneImageSource::InstallDataset, + nexus_networking, + *Generation::new(), + ) + .expect("added zone"); - let mut blueprint_zones = BTreeMap::new(); - let dataset = random_dataset(); - blueprint_zones.insert( - sled1.id(), - [ - BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: external_dns_id, - filesystem_pool: dataset.pool_name, - zone_type: BlueprintZoneType::ExternalDns( - blueprint_zone_type::ExternalDns { - dataset, - http_address: "[::1]:80".parse().unwrap(), - dns_address: OmicronZoneExternalFloatingAddr { - id: ExternalIpUuid::new_v4(), - addr: SocketAddr::new(external_dns_ip, 53), - }, - nic: NetworkInterface { - id: Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: external_dns_id.into_untyped_uuid(), - }, - name: "external-dns".parse().unwrap(), - ip: external_dns_pip.into(), - mac: macs.next().unwrap(), - subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET), - vni: Vni::SERVICES_VNI, - primary: true, - slot: 0, - transit_ips: vec![], - }, - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }, - BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: ntp1_id, - filesystem_pool: random_zpool(), - zone_type: BlueprintZoneType::BoundaryNtp( - blueprint_zone_type::BoundaryNtp { - address: "[::1]:80".parse().unwrap(), - ntp_servers: vec![], - dns_servers: vec![], - domain: None, - nic: NetworkInterface { - id: Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: ntp1_id.into_untyped_uuid(), - }, - name: "ntp1".parse().unwrap(), - ip: ntp1_pip.into(), - mac: macs.next().unwrap(), - subnet: IpNet::from(*NTP_OPTE_IPV4_SUBNET), - vni: Vni::SERVICES_VNI, - primary: true, - slot: 0, - transit_ips: vec![], - }, - external_ip: OmicronZoneExternalSnatIp { - id: ExternalIpUuid::new_v4(), - snat_cfg: SourceNatConfig::new( - ntp1_ip, 16384, 32767, - ) - .unwrap(), - }, - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }, - ] - .into_iter() - .collect::>(), - ); - blueprint_zones.insert( - sled2.id(), - [ - BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: nexus_id, - filesystem_pool: random_zpool(), - zone_type: BlueprintZoneType::Nexus( - blueprint_zone_type::Nexus { - internal_address: "[::1]:80".parse().unwrap(), - lockstep_port: - omicron_common::address::NEXUS_LOCKSTEP_PORT, - external_ip: OmicronZoneExternalFloatingIp { - id: ExternalIpUuid::new_v4(), - ip: nexus_ip, - }, - external_tls: false, - external_dns_servers: vec![], - nic: NetworkInterface { - id: Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: nexus_id.into_untyped_uuid(), - }, - name: "nexus".parse().unwrap(), - ip: nexus_pip.into(), - mac: macs.next().unwrap(), - subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET), - vni: Vni::SERVICES_VNI, - primary: true, - slot: 0, - transit_ips: vec![], - }, - nexus_generation: *Generation::new(), - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }, - BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: ntp2_id, - filesystem_pool: random_zpool(), - zone_type: BlueprintZoneType::BoundaryNtp( - blueprint_zone_type::BoundaryNtp { - address: "[::1]:80".parse().unwrap(), - ntp_servers: vec![], - dns_servers: vec![], - domain: None, - nic: NetworkInterface { - id: Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: ntp2_id.into_untyped_uuid(), - }, - name: "ntp2".parse().unwrap(), - ip: ntp2_pip.into(), - mac: macs.next().unwrap(), - subnet: IpNet::from(*NTP_OPTE_IPV4_SUBNET), - vni: Vni::SERVICES_VNI, - primary: true, - slot: 0, - transit_ips: vec![], - }, - external_ip: OmicronZoneExternalSnatIp { - id: ExternalIpUuid::new_v4(), - snat_cfg: SourceNatConfig::new( - ntp2_ip, 0, 16383, - ) - .unwrap(), - }, - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }, - ] - .into_iter() - .collect(), - ); - blueprint_zones.insert( - sled3.id(), - [BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: ntp3_id, - filesystem_pool: random_zpool(), - zone_type: BlueprintZoneType::InternalNtp( - blueprint_zone_type::InternalNtp { - address: "[::1]:80".parse().unwrap(), - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }] - .into_iter() - .collect(), - ); - let blueprint_id = BlueprintUuid::new_v4(); - let blueprint = Blueprint { - id: blueprint_id, - sleds: make_sled_config_only_zones(blueprint_zones), - pending_mgs_updates: PendingMgsUpdates::new(), - cockroachdb_setting_preserve_downgrade: - CockroachDbPreserveDowngrade::DoNotModify, - parent_blueprint_id: None, - internal_dns_version: *Generation::new(), - external_dns_version: *Generation::new(), - target_release_minimum_generation: *Generation::new(), - nexus_generation: *Generation::new(), - cockroachdb_fingerprint: String::new(), - clickhouse_cluster_config: None, - oximeter_read_version: *Generation::new(), - oximeter_read_mode: OximeterReadMode::SingleNode, - time_created: now_db_precision(), - creator: "test suite".to_string(), - comment: "test blueprint".to_string(), - source: BlueprintSource::Test, - }; + for (sled_id, external_ip) in + [(sled1.id(), ntp1_networking), (sled2.id(), ntp2_networking)] + { + builder + .sled_add_zone_boundary_ntp_with_config( + sled_id, + Vec::new(), + Vec::new(), + None, + BlueprintZoneImageSource::InstallDataset, + external_ip, + ) + .expect("added boundary NTP"); + } + builder + .sled_ensure_zone_ntp( + sled3.id(), + BlueprintZoneImageSource::InstallDataset, + ) + .expect("ensured internal NTP"); + + let mut blueprint = builder.build(BlueprintSource::Test); + + // We're emulating RSS, which inserts the initial blueprint. Clear out + // the link back to the empty parent we started with. + blueprint.parent_blueprint_id = None; + + // Find the zone IDs of the services we added above. + let mut nexus_id = None; + let mut external_dns_id = None; + let mut ntp1_id = None; + let mut ntp2_id = None; + let mut ntp3_id = None; + for (sled_id, zone) in + blueprint.all_omicron_zones(BlueprintZoneDisposition::is_in_service) + { + match &zone.zone_type { + BlueprintZoneType::BoundaryNtp(_) => { + let which = if sled_id == sled1.id() { + &mut ntp1_id + } else if sled_id == sled2.id() { + &mut ntp2_id + } else { + panic!("unexpected boundary NTP zone") + }; + assert!(which.is_none(), "just 1 NTP zone per sled"); + *which = Some(zone.id); + } + BlueprintZoneType::InternalNtp(_) => { + assert_eq!(sled_id, sled3.id()); + assert!(ntp3_id.is_none(), "just 1 internal NTP zone"); + ntp3_id = Some(zone.id); + } + BlueprintZoneType::ExternalDns(_) => { + assert_eq!(sled_id, sled1.id()); + assert!(external_dns_id.is_none(), "just 1 DNS zone"); + external_dns_id = Some(zone.id); + } + BlueprintZoneType::Nexus(_) => { + assert_eq!(sled_id, sled2.id()); + assert!(nexus_id.is_none(), "just 1 Nexus zone"); + nexus_id = Some(zone.id); + } + _ => (), + } + } + let nexus_id = nexus_id.expect("found Nexus zone"); + let external_dns_id = external_dns_id.expect("found DNS zone"); + let ntp1_id = ntp1_id.expect("found NTP zone 1"); + let ntp2_id = ntp2_id.expect("found NTP zone 2"); + let ntp3_id = ntp3_id.expect("found NTP zone 3"); let rack = datastore .rack_set_initialized( @@ -1697,13 +1640,13 @@ mod test { assert!(ntp1_external_ip.is_service); assert_eq!(ntp1_external_ip.kind, IpKind::SNat); - assert_eq!(ntp1_external_ip.first_port.0, 16384); - assert_eq!(ntp1_external_ip.last_port.0, 32767); + assert_eq!(ntp1_external_ip.first_port.0, 0); + assert_eq!(ntp1_external_ip.last_port.0, 16383); assert!(ntp2_external_ip.is_service); assert_eq!(ntp2_external_ip.kind, IpKind::SNat); - assert_eq!(ntp2_external_ip.first_port.0, 0); - assert_eq!(ntp2_external_ip.last_port.0, 16383); + assert_eq!(ntp2_external_ip.first_port.0, 16384); + assert_eq!(ntp2_external_ip.last_port.0, 32767); // Furthermore, we should be able to see that these IP addresses have // been allocated as a part of a service IP pool. diff --git a/nexus/reconfigurator/planning/src/blueprint_editor/allocators/external_networking.rs b/nexus/reconfigurator/planning/src/blueprint_editor/allocators/external_networking.rs index 53703f6a7ab..979cd1bd453 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor/allocators/external_networking.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor/allocators/external_networking.rs @@ -73,8 +73,7 @@ impl ExternalNetworkingAllocator { /// Construct an `ExternalNetworkingAllocator` that hands out IPs based on /// `external_ip_policy`, treating any IPs used by in-service zones /// in `blueprint` as already-in-use. - #[cfg(test)] - pub(crate) fn from_blueprint( + pub fn from_blueprint( blueprint: &nexus_types::deployment::Blueprint, external_ip_policy: &ExternalIpPolicy, ) -> anyhow::Result { From 6baa021df71f9537f241f024421941bcd805de1e Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 11:28:02 -0500 Subject: [PATCH 03/16] rack_set_initialized_missing_service_pool_ip_throws_error() --- nexus/db-queries/src/db/datastore/rack.rs | 168 ++++++++---------- .../planning/src/blueprint_builder/builder.rs | 2 +- 2 files changed, 79 insertions(+), 91 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index ce8eef59a43..81d07cd328e 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -1069,6 +1069,7 @@ mod test { use nexus_inventory::now_db_precision; use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; use nexus_reconfigurator_planning::blueprint_editor::ExternalNetworkingAllocator; + use nexus_reconfigurator_planning::blueprint_editor::ExternalNetworkingChoice; use nexus_reconfigurator_planning::planner::PlannerRng; use nexus_reconfigurator_planning::system::{ SledBuilder, SystemDescription, @@ -1111,10 +1112,12 @@ mod test { use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use oxnet::IpNet; + use slog::Logger; use std::collections::{BTreeMap, HashMap}; use std::net::Ipv6Addr; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::num::NonZeroU32; + use std::sync::LazyLock; // Default impl is for tests only, and really just so that tests can more // easily specify just the parts that they want. @@ -1201,6 +1204,49 @@ mod test { Uuid::parse_str(nexus_test_utils::RACK_UUID).unwrap() } + // Return a `BlueprintBuilder` configured from `system` and based on an + // empty parent blueprint. + // + // If used for RSS tests (most of the time!), the blueprint built by this + // builder will need to have its `parent_blueprint_id` manually set to + // `None` to erase the link to the empty parent. + fn blueprint_builder_with_empty_parent( + log: &Logger, + system: &SystemDescription, + test_name: &str, + ) -> BlueprintBuilder<'static> { + static EMPTY_BLUEPRINT: LazyLock = LazyLock::new(|| { + BlueprintBuilder::build_empty_with_sleds( + std::iter::empty(), + "EMPTY_BLUEPRINT static", + ) + }); + + let planning_input = system + .to_planning_input_builder() + .expect("created planning input builder") + .build(); + + let mut builder = BlueprintBuilder::new_based_on( + log, + &*EMPTY_BLUEPRINT, + &planning_input, + test_name, + PlannerRng::from_entropy(), + ) + .expect("created blueprint builder"); + + for (sled_id, sled_resources) in + planning_input.all_sled_resources(SledFilter::InService) + { + builder + .sled_add_disks(sled_id, &sled_resources) + .expect("added disks"); + } + + builder + } + #[tokio::test] async fn rack_set_initialized_empty() { let logctx = dev::test_setup_log("rack_set_initialized_empty"); @@ -1432,44 +1478,23 @@ mod test { let mut system = SystemDescription::new(); system - .set_external_ip_policy(external_ip_policy) + .set_external_ip_policy(external_ip_policy.clone()) .sled(SledBuilder::new().id(sled1.id())) .expect("failed to add sled1") .sled(SledBuilder::new().id(sled2.id())) .expect("failed to add sled2") .sled(SledBuilder::new().id(sled3.id())) .expect("failed to add sled3"); - let planning_input = system - .to_planning_input_builder() - .expect("created planning input builder") - .build(); - let empty_starting_blueprint = BlueprintBuilder::build_empty_with_sleds( - std::iter::empty(), - test_name, - ); + let mut builder = + blueprint_builder_with_empty_parent(&opctx.log, &system, test_name); + let mut external_networking_alloc = - ExternalNetworkingAllocator::from_blueprint( - &empty_starting_blueprint, - planning_input.external_ip_policy(), + ExternalNetworkingAllocator::from_current_zones( + &builder, + &external_ip_policy, ) .expect("constructed allocator"); - let mut builder = BlueprintBuilder::new_based_on( - &opctx.log, - &empty_starting_blueprint, - &planning_input, - test_name, - PlannerRng::from_entropy(), - ) - .expect("created blueprint builder"); - for (sled_id, sled_resources) in - planning_input.all_sled_resources(SledFilter::InService) - { - builder - .sled_add_disks(sled_id, &sled_resources) - .expect("added disks"); - } - let external_dns_networking = external_networking_alloc .for_new_external_dns() .expect("got IP for external DNS"); @@ -2256,74 +2281,37 @@ mod test { system .sled(SledBuilder::new().id(sled.id())) .expect("failed to add sled"); + let mut builder = + blueprint_builder_with_empty_parent(&opctx.log, &system, test_name); + // We didn't add anything to the `system` IP pool, but pick an IP + // anyway. This should fail below. let nexus_ip = IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4)); let nexus_pip = NEXUS_OPTE_IPV4_SUBNET .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) .unwrap(); - let nexus_id = OmicronZoneUuid::new_v4(); let mut macs = MacAddr::iter_system(); - let mut blueprint_zones = BTreeMap::new(); - blueprint_zones.insert( - sled.id(), - [BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: nexus_id, - filesystem_pool: random_zpool(), - zone_type: BlueprintZoneType::Nexus( - blueprint_zone_type::Nexus { - internal_address: "[::1]:80".parse().unwrap(), - lockstep_port: - omicron_common::address::NEXUS_LOCKSTEP_PORT, - external_ip: OmicronZoneExternalFloatingIp { - id: ExternalIpUuid::new_v4(), - ip: nexus_ip, - }, - external_tls: false, - external_dns_servers: vec![], - nic: NetworkInterface { - id: Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: nexus_id.into_untyped_uuid(), - }, - name: "nexus".parse().unwrap(), - ip: nexus_pip.into(), - mac: macs.next().unwrap(), - subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET), - vni: Vni::SERVICES_VNI, - primary: true, - slot: 0, - transit_ips: vec![], - }, - nexus_generation: *Generation::new(), - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }] - .into_iter() - .collect::>(), - ); - let blueprint_id = BlueprintUuid::new_v4(); - let blueprint = Blueprint { - id: blueprint_id, - sleds: make_sled_config_only_zones(blueprint_zones), - pending_mgs_updates: PendingMgsUpdates::new(), - cockroachdb_setting_preserve_downgrade: - CockroachDbPreserveDowngrade::DoNotModify, - parent_blueprint_id: None, - internal_dns_version: *Generation::new(), - external_dns_version: *Generation::new(), - target_release_minimum_generation: *Generation::new(), - nexus_generation: *Generation::new(), - cockroachdb_fingerprint: String::new(), - clickhouse_cluster_config: None, - oximeter_read_version: *Generation::new(), - oximeter_read_mode: OximeterReadMode::SingleNode, - time_created: now_db_precision(), - creator: "test suite".to_string(), - comment: "test blueprint".to_string(), - source: BlueprintSource::Test, - }; + builder + .sled_add_zone_nexus_with_config( + sled.id(), + false, + Vec::new(), + BlueprintZoneImageSource::InstallDataset, + ExternalNetworkingChoice { + external_ip: nexus_ip, + nic_ip: nexus_pip.into(), + nic_subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET), + nic_mac: macs.next().unwrap(), + }, + *Generation::new(), + ) + .expect("added Nexus"); + + let mut blueprint = builder.build(BlueprintSource::Test); + + // We're emulating RSS, which inserts the initial blueprint. Clear out + // the link back to the empty parent we started with. + blueprint.parent_blueprint_id = None; let result = datastore .rack_set_initialized( diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 5455805d804..c106bbfa0e8 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -606,7 +606,7 @@ impl<'a> BlueprintBuilder<'a> { pub fn new_based_on( log: &Logger, parent_blueprint: &'a Blueprint, - input: &'a PlanningInput, + input: &PlanningInput, creator: &str, mut rng: PlannerRng, ) -> anyhow::Result> { From d00b536eff1fc60862ea26f4bf19d8f7237d8884 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 13:13:05 -0500 Subject: [PATCH 04/16] rack_set_initialized_with_many_nexus_services() --- nexus/db-queries/src/db/datastore/rack.rs | 140 ++++------------------ 1 file changed, 26 insertions(+), 114 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 81d07cd328e..53ef7fd1242 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -1740,100 +1740,34 @@ mod test { let mut system = SystemDescription::new(); system - .set_external_ip_policy(external_ip_policy) + .set_external_ip_policy(external_ip_policy.clone()) .sled(SledBuilder::new().id(sled.id())) .expect("failed to add sled"); + let mut builder = + blueprint_builder_with_empty_parent(&opctx.log, &system, test_name); - let nexus_id1 = OmicronZoneUuid::new_v4(); - let nexus_id2 = OmicronZoneUuid::new_v4(); - let nexus_pip1 = NEXUS_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) - .unwrap(); - let nexus_pip2 = NEXUS_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 2) - .unwrap(); - let mut macs = MacAddr::iter_system(); - - let mut blueprint_zones = BTreeMap::new(); - blueprint_zones.insert( - sled.id(), - [ - BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: nexus_id1, - filesystem_pool: random_zpool(), - zone_type: BlueprintZoneType::Nexus( - blueprint_zone_type::Nexus { - internal_address: "[::1]:80".parse().unwrap(), - lockstep_port: - omicron_common::address::NEXUS_LOCKSTEP_PORT, - external_ip: OmicronZoneExternalFloatingIp { - id: ExternalIpUuid::new_v4(), - ip: nexus_ip_start.into(), - }, - external_tls: false, - external_dns_servers: vec![], - nic: NetworkInterface { - id: Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: nexus_id1.into_untyped_uuid(), - }, - name: "nexus1".parse().unwrap(), - ip: nexus_pip1.into(), - mac: macs.next().unwrap(), - subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET), - vni: Vni::SERVICES_VNI, - primary: true, - slot: 0, - transit_ips: vec![], - }, - nexus_generation: *Generation::new(), - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }, - BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: nexus_id2, - filesystem_pool: random_zpool(), - zone_type: BlueprintZoneType::Nexus( - blueprint_zone_type::Nexus { - internal_address: "[::1]:80".parse().unwrap(), - lockstep_port: - omicron_common::address::NEXUS_LOCKSTEP_PORT, - external_ip: OmicronZoneExternalFloatingIp { - id: ExternalIpUuid::new_v4(), - ip: nexus_ip_end.into(), - }, - external_tls: false, - external_dns_servers: vec![], - nic: NetworkInterface { - id: Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: nexus_id2.into_untyped_uuid(), - }, - name: "nexus2".parse().unwrap(), - ip: nexus_pip2.into(), - mac: macs.next().unwrap(), - subnet: oxnet::IpNet::from( - *NEXUS_OPTE_IPV4_SUBNET, - ), - vni: Vni::SERVICES_VNI, - primary: true, - slot: 0, - transit_ips: vec![], - }, - nexus_generation: *Generation::new(), - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }, - ] - .into_iter() - .collect::>(), - ); - - let datasets = vec![]; + let mut external_networking_alloc = + ExternalNetworkingAllocator::from_current_zones( + &builder, + &external_ip_policy, + ) + .expect("constructed allocator"); + for _ in 0..2 { + builder + .sled_add_zone_nexus_with_config( + sled.id(), + false, + Vec::new(), + BlueprintZoneImageSource::InstallDataset, + external_networking_alloc + .for_new_nexus() + .expect("got Nexus IP"), + *Generation::new(), + ) + .expect("added Nexus"); + } + let mut blueprint = builder.build(BlueprintSource::Test); + blueprint.parent_blueprint_id = None; // treat this as the initial bp let internal_records = vec![ DnsRecord::Aaaa("fe80::1:2:3:4".parse().unwrap()), @@ -1857,34 +1791,12 @@ mod test { HashMap::from([("api.sys".to_string(), external_records.clone())]), ); - let blueprint_id = BlueprintUuid::new_v4(); - let blueprint = Blueprint { - id: blueprint_id, - sleds: make_sled_config_only_zones(blueprint_zones), - pending_mgs_updates: PendingMgsUpdates::new(), - cockroachdb_setting_preserve_downgrade: - CockroachDbPreserveDowngrade::DoNotModify, - parent_blueprint_id: None, - internal_dns_version: *Generation::new(), - external_dns_version: *Generation::new(), - target_release_minimum_generation: *Generation::new(), - nexus_generation: *Generation::new(), - cockroachdb_fingerprint: String::new(), - clickhouse_cluster_config: None, - oximeter_read_version: *Generation::new(), - oximeter_read_mode: OximeterReadMode::SingleNode, - time_created: now_db_precision(), - creator: "test suite".to_string(), - comment: "test blueprint".to_string(), - source: BlueprintSource::Test, - }; - let rack = datastore .rack_set_initialized( &opctx, RackInit { blueprint: blueprint.clone(), - datasets: datasets.clone(), + datasets: vec![], service_ip_pool_ranges: vec![service_ip_pool], internal_dns, external_dns, From 8302b01d5265b1cd77add60619036a405da742fe Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 13:16:43 -0500 Subject: [PATCH 05/16] rack_set_initialized_with_ipv6_public_addresses() --- nexus/db-queries/src/db/datastore/rack.rs | 91 ++++++----------------- 1 file changed, 23 insertions(+), 68 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 53ef7fd1242..540ec7bdef9 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -1945,56 +1945,33 @@ mod test { let mut system = SystemDescription::new(); system - .set_external_ip_policy(external_ip_policy) + .set_external_ip_policy(external_ip_policy.clone()) .sled(SledBuilder::new().id(sled.id())) .expect("failed to add sled"); - let nexus_id = OmicronZoneUuid::new_v4(); - let nexus_pip = NEXUS_OPTE_IPV6_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES as u128 + 1) - .unwrap(); - let mut macs = MacAddr::iter_system(); + let mut builder = + blueprint_builder_with_empty_parent(&opctx.log, &system, test_name); - let mut blueprint_zones = BTreeMap::new(); - blueprint_zones.insert( - sled.id(), - [BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: nexus_id, - filesystem_pool: random_zpool(), - zone_type: BlueprintZoneType::Nexus( - blueprint_zone_type::Nexus { - internal_address: "[::1]:80".parse().unwrap(), - lockstep_port: - omicron_common::address::NEXUS_LOCKSTEP_PORT, - external_ip: OmicronZoneExternalFloatingIp { - id: ExternalIpUuid::new_v4(), - ip: nexus_ip_start.into(), - }, - external_tls: false, - external_dns_servers: vec![], - nic: NetworkInterface { - id: Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: nexus_id.into_untyped_uuid(), - }, - name: "nexus1".parse().unwrap(), - ip: nexus_pip.into(), - mac: macs.next().unwrap(), - subnet: IpNet::from(*NEXUS_OPTE_IPV6_SUBNET), - vni: Vni::SERVICES_VNI, - primary: true, - slot: 0, - transit_ips: vec![], - }, - nexus_generation: *Generation::new(), - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }] - .into_iter() - .collect::>(), - ); + let mut external_networking_alloc = + ExternalNetworkingAllocator::from_current_zones( + &builder, + &external_ip_policy, + ) + .expect("constructed allocator"); + builder + .sled_add_zone_nexus_with_config( + sled.id(), + false, + Vec::new(), + BlueprintZoneImageSource::InstallDataset, + external_networking_alloc + .for_new_nexus() + .expect("got Nexus IP"), + *Generation::new(), + ) + .expect("added Nexus"); + let mut blueprint = builder.build(BlueprintSource::Test); + blueprint.parent_blueprint_id = None; // treat this as the initial bp let datasets = vec![]; @@ -2020,28 +1997,6 @@ mod test { HashMap::from([("api.sys".to_string(), external_records.clone())]), ); - let blueprint_id = BlueprintUuid::new_v4(); - let blueprint = Blueprint { - id: blueprint_id, - sleds: make_sled_config_only_zones(blueprint_zones), - pending_mgs_updates: PendingMgsUpdates::new(), - cockroachdb_setting_preserve_downgrade: - CockroachDbPreserveDowngrade::DoNotModify, - parent_blueprint_id: None, - internal_dns_version: *Generation::new(), - external_dns_version: *Generation::new(), - target_release_minimum_generation: *Generation::new(), - cockroachdb_fingerprint: String::new(), - clickhouse_cluster_config: None, - oximeter_read_version: *Generation::new(), - oximeter_read_mode: OximeterReadMode::SingleNode, - time_created: now_db_precision(), - creator: "test suite".to_string(), - comment: "test blueprint".to_string(), - source: BlueprintSource::Test, - nexus_generation: *Generation::new(), - }; - let rack = datastore .rack_set_initialized( &opctx, From ce1c5dc1aef5d16307418a563b79a6b4e5aeda8f Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 14:07:17 -0500 Subject: [PATCH 06/16] rack_set_initialized_overlapping_ips_throws_error() --- nexus/db-queries/src/db/datastore/rack.rs | 197 ++++------------------ 1 file changed, 31 insertions(+), 166 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 540ec7bdef9..7deb85a0a0a 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -1062,11 +1062,9 @@ mod test { use crate::db::pub_test_utils::TestDatabase; use crate::db::pub_test_utils::helpers::SledUpdateBuilder; use async_bb8_diesel::AsyncSimpleConnection; - use iddqd::IdOrdMap; use internal_dns_types::names::DNS_ZONE; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; use nexus_db_model::{DnsGroup, Generation, InitialDnsGroup}; - use nexus_inventory::now_db_precision; use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; use nexus_reconfigurator_planning::blueprint_editor::ExternalNetworkingAllocator; use nexus_reconfigurator_planning::blueprint_editor::ExternalNetworkingChoice; @@ -1074,48 +1072,31 @@ mod test { use nexus_reconfigurator_planning::system::{ SledBuilder, SystemDescription, }; - use nexus_sled_agent_shared::inventory::OmicronZoneDataset; - use nexus_types::deployment::BlueprintHostPhase2DesiredSlots; - use nexus_types::deployment::BlueprintSledConfig; use nexus_types::deployment::BlueprintSource; use nexus_types::deployment::CockroachDbPreserveDowngrade; use nexus_types::deployment::ExternalIpPolicy; use nexus_types::deployment::PendingMgsUpdates; use nexus_types::deployment::SledFilter; - use nexus_types::deployment::{ - BlueprintZoneConfig, OmicronZoneExternalFloatingAddr, - OmicronZoneExternalFloatingIp, - }; use nexus_types::deployment::{ BlueprintZoneDisposition, BlueprintZoneImageSource, OximeterReadMode, }; use nexus_types::external_api::shared::SiloIdentityMode; - use nexus_types::external_api::views::SledState; use nexus_types::identity::Asset; use nexus_types::internal_api::params::DnsRecord; - use nexus_types::inventory::NetworkInterface; - use nexus_types::inventory::NetworkInterfaceKind; - use omicron_common::address::NEXUS_OPTE_IPV6_SUBNET; - use omicron_common::address::{ - DNS_OPTE_IPV4_SUBNET, NEXUS_OPTE_IPV4_SUBNET, - }; + use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::{ - IdentityMetadataCreateParams, MacAddr, Vni, + IdentityMetadataCreateParams, MacAddr, }; - use omicron_common::zpool_name::ZpoolName; use omicron_test_utils::dev; use omicron_uuid_kinds::BlueprintUuid; - use omicron_uuid_kinds::ExternalIpUuid; use omicron_uuid_kinds::GenericUuid; - use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; - use omicron_uuid_kinds::ZpoolUuid; use oxnet::IpNet; use slog::Logger; use std::collections::{BTreeMap, HashMap}; use std::net::Ipv6Addr; - use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + use std::net::{IpAddr, Ipv4Addr}; use std::num::NonZeroU32; use std::sync::LazyLock; @@ -1229,7 +1210,7 @@ mod test { let mut builder = BlueprintBuilder::new_based_on( log, - &*EMPTY_BLUEPRINT, + &EMPTY_BLUEPRINT, &planning_input, test_name, PlannerRng::from_entropy(), @@ -1411,44 +1392,6 @@ mod test { fn_to_get_all!(ip_pool_range, IpPoolRange); fn_to_get_all!(crucible_dataset, CrucibleDataset); - fn random_zpool() -> ZpoolName { - ZpoolName::new_external(ZpoolUuid::new_v4()) - } - - fn random_dataset() -> OmicronZoneDataset { - OmicronZoneDataset { - pool_name: illumos_utils::zpool::ZpoolName::new_external( - ZpoolUuid::new_v4(), - ) - .to_string() - .parse() - .unwrap(), - } - } - - fn make_sled_config_only_zones( - blueprint_zones: BTreeMap>, - ) -> BTreeMap { - blueprint_zones - .into_iter() - .map(|(sled_id, zones)| { - ( - sled_id, - BlueprintSledConfig { - state: SledState::Active, - sled_agent_generation: Generation::new().next(), - disks: IdOrdMap::new(), - datasets: IdOrdMap::new(), - zones, - remove_mupdate_override: None, - host_phase_2: - BlueprintHostPhase2DesiredSlots::current_contents(), - }, - ) - }) - .collect() - } - #[tokio::test] async fn rack_set_initialized_with_services() { let test_name = "rack_set_initialized_with_services"; @@ -2215,116 +2158,38 @@ mod test { let mut system = SystemDescription::new(); system - .set_external_ip_policy(external_ip_policy) + .set_external_ip_policy(external_ip_policy.clone()) .sled(SledBuilder::new().id(sled.id())) .expect("failed to add sled"); - // Request two services which happen to be using the same IP address. - let external_dns_id = OmicronZoneUuid::new_v4(); - let external_dns_pip = DNS_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) - .unwrap(); - let nexus_id = OmicronZoneUuid::new_v4(); - let nexus_pip = NEXUS_OPTE_IPV4_SUBNET - .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES + 1) - .unwrap(); - let mut macs = MacAddr::iter_system(); + let mut builder = + blueprint_builder_with_empty_parent(&opctx.log, &system, test_name); - let mut blueprint_zones = BTreeMap::new(); - let dataset = random_dataset(); - blueprint_zones.insert( - sled.id(), - [ - BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: external_dns_id, - filesystem_pool: dataset.pool_name, - zone_type: BlueprintZoneType::ExternalDns( - blueprint_zone_type::ExternalDns { - dataset, - http_address: "[::1]:80".parse().unwrap(), - dns_address: OmicronZoneExternalFloatingAddr { - id: ExternalIpUuid::new_v4(), - addr: SocketAddr::new(ip, 53), - }, - nic: NetworkInterface { - id: Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: external_dns_id.into_untyped_uuid(), - }, - name: "external-dns".parse().unwrap(), - ip: external_dns_pip.into(), - mac: macs.next().unwrap(), - subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET), - vni: Vni::SERVICES_VNI, - primary: true, - slot: 0, - transit_ips: vec![], - }, - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }, - BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: nexus_id, - filesystem_pool: random_zpool(), - zone_type: BlueprintZoneType::Nexus( - blueprint_zone_type::Nexus { - internal_address: "[::1]:80".parse().unwrap(), - lockstep_port: - omicron_common::address::NEXUS_LOCKSTEP_PORT, - external_ip: OmicronZoneExternalFloatingIp { - id: ExternalIpUuid::new_v4(), - ip, - }, - external_tls: false, - external_dns_servers: vec![], - nic: NetworkInterface { - id: Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: nexus_id.into_untyped_uuid(), - }, - name: "nexus".parse().unwrap(), - ip: nexus_pip.into(), - mac: macs.next().unwrap(), - subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET), - vni: Vni::SERVICES_VNI, - primary: true, - slot: 0, - transit_ips: vec![], - }, - nexus_generation: *Generation::new(), - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }, - ] - .into_iter() - .collect::>(), - ); + let mut external_networking_alloc = + ExternalNetworkingAllocator::from_current_zones( + &builder, + &external_ip_policy, + ) + .expect("constructed allocator"); - let blueprint_id = BlueprintUuid::new_v4(); - let blueprint = Blueprint { - id: blueprint_id, - sleds: make_sled_config_only_zones(blueprint_zones), - pending_mgs_updates: PendingMgsUpdates::new(), - cockroachdb_setting_preserve_downgrade: - CockroachDbPreserveDowngrade::DoNotModify, - parent_blueprint_id: None, - internal_dns_version: *Generation::new(), - external_dns_version: *Generation::new(), - target_release_minimum_generation: *Generation::new(), - nexus_generation: *Generation::new(), - cockroachdb_fingerprint: String::new(), - clickhouse_cluster_config: None, - oximeter_read_version: *Generation::new(), - oximeter_read_mode: OximeterReadMode::SingleNode, - time_created: now_db_precision(), - creator: "test suite".to_string(), - comment: "test blueprint".to_string(), - source: BlueprintSource::Test, - }; + // Add two Nexus zones, but reuse the same external IP for both. + let nexus_external_ip = + external_networking_alloc.for_new_nexus().expect("got Nexus IP"); + for _ in 0..2 { + builder + .sled_add_zone_nexus_with_config( + sled.id(), + false, + Vec::new(), + BlueprintZoneImageSource::InstallDataset, + nexus_external_ip, + *Generation::new(), + ) + .expect("added Nexus"); + } + + let mut blueprint = builder.build(BlueprintSource::Test); + blueprint.parent_blueprint_id = None; // treat this as the initial bp let result = datastore .rack_set_initialized( From fc4658d75e9657aa1ba1c107c4800dfa1d7f80b7 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 14:11:52 -0500 Subject: [PATCH 07/16] revert now-unneccesary visibility change --- .../src/blueprint_editor/allocators/external_networking.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nexus/reconfigurator/planning/src/blueprint_editor/allocators/external_networking.rs b/nexus/reconfigurator/planning/src/blueprint_editor/allocators/external_networking.rs index 979cd1bd453..53703f6a7ab 100644 --- a/nexus/reconfigurator/planning/src/blueprint_editor/allocators/external_networking.rs +++ b/nexus/reconfigurator/planning/src/blueprint_editor/allocators/external_networking.rs @@ -73,7 +73,8 @@ impl ExternalNetworkingAllocator { /// Construct an `ExternalNetworkingAllocator` that hands out IPs based on /// `external_ip_policy`, treating any IPs used by in-service zones /// in `blueprint` as already-in-use. - pub fn from_blueprint( + #[cfg(test)] + pub(crate) fn from_blueprint( blueprint: &nexus_types::deployment::Blueprint, external_ip_policy: &ExternalIpPolicy, ) -> anyhow::Result { From 3373282973474a72ccbfa2cbcf45bb728955925c Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 14:20:22 -0500 Subject: [PATCH 08/16] add BlueprintBuilder::build_empty() --- nexus/db-queries/src/db/datastore/deployment.rs | 15 +++------------ nexus/db-queries/src/db/datastore/rack.rs | 5 +---- nexus/reconfigurator/execution/src/dns.rs | 5 +---- .../planning/src/blueprint_builder/builder.rs | 6 ++++++ .../src/app/background/tasks/blueprint_planner.rs | 14 ++++++-------- 5 files changed, 17 insertions(+), 28 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index 6bfb26db881..9a669cbde84 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -3311,10 +3311,7 @@ mod tests { let (opctx, datastore) = (db.opctx(), db.datastore()); // Create an empty blueprint from it - let blueprint1 = BlueprintBuilder::build_empty_with_sleds( - std::iter::empty(), - "test", - ); + let blueprint1 = BlueprintBuilder::build_empty("test"); let authz_blueprint = authz_blueprint_from_id(blueprint1.id); // Trying to read it from the database should fail with the relevant @@ -4039,10 +4036,7 @@ mod tests { // Create three blueprints: // * `blueprint1` has no parent // * `blueprint2` and `blueprint3` both have `blueprint1` as parent - let blueprint1 = BlueprintBuilder::build_empty_with_sleds( - std::iter::empty(), - "test1", - ); + let blueprint1 = BlueprintBuilder::build_empty("test1"); let blueprint2 = BlueprintBuilder::new_based_on( &logctx.log, &blueprint1, @@ -4190,10 +4184,7 @@ mod tests { let (opctx, datastore) = (db.opctx(), db.datastore()); // Create an initial blueprint and a child. - let blueprint1 = BlueprintBuilder::build_empty_with_sleds( - std::iter::empty(), - "test1", - ); + let blueprint1 = BlueprintBuilder::build_empty("test1"); let blueprint2 = BlueprintBuilder::new_based_on( &logctx.log, &blueprint1, diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 7deb85a0a0a..274a26cb3b5 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -1197,10 +1197,7 @@ mod test { test_name: &str, ) -> BlueprintBuilder<'static> { static EMPTY_BLUEPRINT: LazyLock = LazyLock::new(|| { - BlueprintBuilder::build_empty_with_sleds( - std::iter::empty(), - "EMPTY_BLUEPRINT static", - ) + BlueprintBuilder::build_empty("EMPTY_BLUEPRINT static") }); let planning_input = system diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index e92a82cefe9..fb932f75792 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -650,10 +650,7 @@ mod test { /// test blueprint_internal_dns_config(): trivial case of an empty blueprint #[test] fn test_blueprint_internal_dns_empty() { - let blueprint = BlueprintBuilder::build_empty_with_sleds( - std::iter::empty(), - "test-suite", - ); + let blueprint = BlueprintBuilder::build_empty("test-suite"); let blueprint_dns = blueprint_internal_dns_config( &blueprint, &IdOrdMap::new(), diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index c106bbfa0e8..824081e01ff 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -530,6 +530,12 @@ pub struct BlueprintBuilder<'a> { } impl<'a> BlueprintBuilder<'a> { + /// Directly construct an empty blueprint: no sleds; default values for all + /// other fields. + pub fn build_empty(creator: &str) -> Blueprint { + Self::build_empty_with_sleds(iter::empty(), creator) + } + /// Directly construct a `Blueprint` that contains an empty zone config for /// the given sleds. pub fn build_empty_with_sleds( diff --git a/nexus/src/app/background/tasks/blueprint_planner.rs b/nexus/src/app/background/tasks/blueprint_planner.rs index 4d9d87c0212..cd037aab251 100644 --- a/nexus/src/app/background/tasks/blueprint_planner.rs +++ b/nexus/src/app/background/tasks/blueprint_planner.rs @@ -638,10 +638,10 @@ mod test { // Create a large number of blueprints (49), which we'll use to test the // limit (see below). for i in 0..49 { - let blueprint = BlueprintBuilder::build_empty_with_sleds( - std::iter::empty(), - &format!("test_blueprint_planner_limit blueprint {}", i), - ); + let blueprint = BlueprintBuilder::build_empty(&format!( + "test_blueprint_planner_limit blueprint {}", + i + )); datastore .blueprint_insert(&opctx, &blueprint) .await @@ -691,8 +691,7 @@ mod test { // Insert one more blueprint, pushing the number of blueprints to the // limit (50). - let blueprint = BlueprintBuilder::build_empty_with_sleds( - std::iter::empty(), + let blueprint = BlueprintBuilder::build_empty( "test_blueprint_planner_limit 50th blueprint", ); datastore.blueprint_insert(&opctx, &blueprint).await.unwrap_or_else( @@ -715,8 +714,7 @@ mod test { ); // But manual planning should continue to work. - let blueprint = BlueprintBuilder::build_empty_with_sleds( - std::iter::empty(), + let blueprint = BlueprintBuilder::build_empty( "test_blueprint_planner_limit 51st blueprint", ); datastore.blueprint_insert(&opctx, &blueprint).await.unwrap_or_else( From efd74fad8adac2be1936a27558f31aa92b7d8f30 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 14:37:29 -0500 Subject: [PATCH 09/16] example system: start with truly empty blueprint --- .../output/cmds-blueprint-history-stdout | 28 +++++++++---------- .../tests/output/cmds-example-stdout | 16 +++++------ .../planning/src/blueprint_builder/builder.rs | 6 ++++ nexus/reconfigurator/planning/src/example.rs | 5 ++-- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-blueprint-history-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-blueprint-history-stdout index 2488810e455..72ec7dd7aab 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-blueprint-history-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-blueprint-history-stdout @@ -26,7 +26,7 @@ blueprint df06bb57-ad42-4431-9206-abff322896c7 created from blueprint af934083-5 > # which does not include any of those blueprints. > blueprint-history TIME BLUEPRINT - 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint with 3 empty sleds + 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint with 0 empty sleds dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 @@ -34,7 +34,7 @@ TIME BLUEPRINT > # You can give it a specific blueprint. > blueprint-history 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 TIME BLUEPRINT - 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint with 3 empty sleds + 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint with 0 empty sleds dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 @@ -43,7 +43,7 @@ TIME BLUEPRINT > # Running it from the latest blueprint should report all of them. > blueprint-history latest TIME BLUEPRINT - 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint with 3 empty sleds + 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint with 0 empty sleds dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 58d5e830-0884-47d8-a7cd-b2b3751adeb4 @@ -64,21 +64,21 @@ TIME BLUEPRINT > # Show diffs, too. > blueprint-history --diff latest TIME BLUEPRINT - 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint with 3 empty sleds + 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint with 0 empty sleds dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 from: blueprint 184f10b3-61cb-41ef-9b93-3489b2bac559 to: blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 - MODIFIED SLEDS: + ADDED SLEDS: - sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 1 -> 2): + sled 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c (active, config generation 2): host phase 2 contents: ------------------------ slot boot image source ------------------------ - A current contents - B current contents ++ A current contents ++ B current contents physical disks: @@ -184,14 +184,14 @@ to: blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 + nexus 466a9f29-62bf-4e63-924a-b9efdb86afec install dataset in service fd00:1122:3344:102::22 - sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 1 -> 2): + sled 98e6b7c2-2efa-41ca-b20a-0a4d61102fe6 (active, config generation 2): host phase 2 contents: ------------------------ slot boot image source ------------------------ - A current contents - B current contents ++ A current contents ++ B current contents physical disks: @@ -294,14 +294,14 @@ to: blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 + nexus 0c71b3b2-6ceb-4e8f-b020-b08675e83038 install dataset in service fd00:1122:3344:101::22 - sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 1 -> 2): + sled d81c6a84-79b8-4958-ae41-ea46c9b19763 (active, config generation 2): host phase 2 contents: ------------------------ slot boot image source ------------------------ - A current contents - B current contents ++ A current contents ++ B current contents physical disks: diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout index 8477b992a1e..a3028f9b689 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout @@ -694,16 +694,16 @@ external DNS: from: blueprint 02697f74-b14a-4418-90f0-c28b2a3a6aa9 to: blueprint 86db3308-f817-4626-8838-4085949a6a41 - MODIFIED SLEDS: + ADDED SLEDS: - sled 89d02b1b-478c-401a-8e28-7a26f74fa41b (active, config generation 1 -> 2): + sled 89d02b1b-478c-401a-8e28-7a26f74fa41b (active, config generation 2): host phase 2 contents: ------------------------ slot boot image source ------------------------ - A current contents - B current contents ++ A current contents ++ B current contents physical disks: @@ -914,16 +914,16 @@ external DNS: from: blueprint 86db3308-f817-4626-8838-4085949a6a41 to: blueprint 02697f74-b14a-4418-90f0-c28b2a3a6aa9 - MODIFIED SLEDS: + REMOVED SLEDS: - sled 89d02b1b-478c-401a-8e28-7a26f74fa41b (active, config generation 2 -> 1): + sled 89d02b1b-478c-401a-8e28-7a26f74fa41b (was active, config generation 2): host phase 2 contents: ------------------------ slot boot image source ------------------------ - A current contents - B current contents +- A current contents +- B current contents physical disks: diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 824081e01ff..8f189f66673 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -536,6 +536,12 @@ impl<'a> BlueprintBuilder<'a> { Self::build_empty_with_sleds(iter::empty(), creator) } + /// A version of [`Self::build_empty`] that allows the + /// blueprint ID to be generated from a deterministic RNG. + pub fn build_empty_seeded(creator: &str, rng: PlannerRng) -> Blueprint { + Self::build_empty_with_sleds_seeded(iter::empty(), creator, rng) + } + /// Directly construct a `Blueprint` that contains an empty zone config for /// the given sleds. pub fn build_empty_with_sleds( diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index c2291538cb8..4f1b1a10c67 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -535,9 +535,8 @@ impl ExampleSystemBuilder { let base_input = input_builder.clone().build(); - // Start with an empty blueprint containing only our sleds, no zones. - let initial_blueprint = BlueprintBuilder::build_empty_with_sleds_seeded( - base_input.all_sled_ids(SledFilter::Commissioned), + // Start with an empty blueprint. + let initial_blueprint = BlueprintBuilder::build_empty_seeded( "test suite", rng.blueprint1_rng, ); From 4b9eeaf08e0b16e30f409361f67034a58bfdb17a Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 14:55:47 -0500 Subject: [PATCH 10/16] use build_empty() where we can --- nexus/db-queries/src/db/datastore/vpc.rs | 5 +---- .../src/app/background/tasks/crdb_node_id_collector.rs | 10 ++-------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 8fb61894efa..682da75e243 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -3312,10 +3312,7 @@ mod tests { }; // Create an initial, empty blueprint, and make it the target. - let bp0 = BlueprintBuilder::build_empty_with_sleds( - sled_ids.iter().copied(), - "test", - ); + let bp0 = BlueprintBuilder::build_empty("test"); bp_insert_and_make_target(&opctx, &datastore, &bp0).await; // Our blueprint doesn't describe any services, so we shouldn't find any diff --git a/nexus/src/app/background/tasks/crdb_node_id_collector.rs b/nexus/src/app/background/tasks/crdb_node_id_collector.rs index e7923db4be8..51b655b818a 100644 --- a/nexus/src/app/background/tasks/crdb_node_id_collector.rs +++ b/nexus/src/app/background/tasks/crdb_node_id_collector.rs @@ -401,10 +401,7 @@ mod tests { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); - let blueprint = BlueprintBuilder::build_empty_with_sleds( - iter::once(SledUuid::new_v4()), - "test", - ); + let blueprint = BlueprintBuilder::build_empty("test"); let blueprint_target = BlueprintTarget { target_id: blueprint.id, enabled: true, @@ -464,10 +461,7 @@ mod tests { let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); - let blueprint = BlueprintBuilder::build_empty_with_sleds( - iter::once(SledUuid::new_v4()), - "test", - ); + let blueprint = BlueprintBuilder::build_empty("test"); let blueprint_target = BlueprintTarget { target_id: blueprint.id, enabled: true, From ff1b86a116f05483d64cc6fc2bab22c5eee094b9 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 15:56:16 -0500 Subject: [PATCH 11/16] use example system in test_ensure_external_networking_works_with_good_target() --- .../db-queries/src/db/datastore/deployment.rs | 142 +++++------------- nexus/types/src/deployment/planning_input.rs | 13 ++ 2 files changed, 47 insertions(+), 108 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index 9a669cbde84..ffb44a69c7a 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -3146,12 +3146,9 @@ mod tests { use nexus_types::deployment::BlueprintHostPhase2DesiredContents; use nexus_types::deployment::BlueprintHostPhase2DesiredSlots; use nexus_types::deployment::BlueprintPhysicalDiskDisposition; - use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneDisposition; use nexus_types::deployment::BlueprintZoneImageSource; - use nexus_types::deployment::BlueprintZoneType; use nexus_types::deployment::ExpectedActiveRotSlot; - use nexus_types::deployment::OmicronZoneExternalFloatingIp; use nexus_types::deployment::PendingMgsUpdate; use nexus_types::deployment::PlanningInput; use nexus_types::deployment::PlanningInputBuilder; @@ -3159,7 +3156,6 @@ mod tests { use nexus_types::deployment::SledDisk; use nexus_types::deployment::SledFilter; use nexus_types::deployment::SledResources; - use nexus_types::deployment::blueprint_zone_type; use nexus_types::external_api::views::PhysicalDiskPolicy; use nexus_types::external_api::views::PhysicalDiskState; use nexus_types::external_api::views::SledPolicy; @@ -3168,36 +3164,24 @@ mod tests { use nexus_types::inventory::Collection; use omicron_common::address::IpRange; use omicron_common::address::Ipv6Subnet; - use omicron_common::api::external::MacAddr; - use omicron_common::api::external::Name; use omicron_common::api::external::TufArtifactMeta; use omicron_common::api::external::TufRepoDescription; use omicron_common::api::external::TufRepoMeta; - use omicron_common::api::external::Vni; - use omicron_common::api::internal::shared::NetworkInterface; - use omicron_common::api::internal::shared::NetworkInterfaceKind; use omicron_common::disk::DiskIdentity; use omicron_common::disk::M2Slot; use omicron_common::update::ArtifactId; - use omicron_common::zpool_name::ZpoolName; use omicron_test_utils::dev; use omicron_test_utils::dev::poll::CondCheckError; use omicron_test_utils::dev::poll::wait_for_condition; - use omicron_uuid_kinds::ExternalIpUuid; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; - use oxnet::IpNet; use pretty_assertions::assert_eq; use rand::Rng; use std::collections::BTreeSet; use std::mem; - use std::net::IpAddr; - use std::net::Ipv4Addr; use std::net::Ipv6Addr; - use std::net::SocketAddrV6; - use std::str::FromStr; use std::sync::Arc; use std::sync::LazyLock; use std::sync::atomic::AtomicBool; @@ -4283,104 +4267,46 @@ mod tests { logctx.cleanup_successful(); } - async fn create_blueprint_with_external_ip( - datastore: &DataStore, - opctx: &OpContext, - ) -> Blueprint { - // Create an initial blueprint and a child. - let sled_id = SledUuid::new_v4(); - let mut blueprint = BlueprintBuilder::build_empty_with_sleds( - [sled_id].into_iter(), - "test1", - ); - - // To observe realistic database behavior, we need the invocation of - // "blueprint_ensure_external_networking_resources" to actually write something - // back to the database. - // - // While this is *mostly* made-up blueprint contents, the part that matters - // is that it's provisioning a zone (Nexus) which does have resources - // to be allocated. - let ip_range = IpRange::try_from(( - Ipv4Addr::new(10, 0, 0, 1), - Ipv4Addr::new(10, 0, 0, 10), - )) - .unwrap(); - let (service_authz_ip_pool, service_ip_pool) = datastore - .ip_pools_service_lookup(&opctx, IpVersion::V4) - .await - .expect("lookup service ip pool"); - datastore - .ip_pool_add_range( - &opctx, - &service_authz_ip_pool, - &service_ip_pool, - &ip_range, - ) - .await - .expect("add range to service ip pool"); - let zone_id = OmicronZoneUuid::new_v4(); - blueprint - .sleds - .get_mut(&sled_id) - .unwrap() - .zones - .insert_unique(BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: zone_id, - filesystem_pool: ZpoolName::new_external(ZpoolUuid::new_v4()), - zone_type: BlueprintZoneType::Nexus( - blueprint_zone_type::Nexus { - internal_address: SocketAddrV6::new( - Ipv6Addr::LOCALHOST, - 0, - 0, - 0, - ), - lockstep_port: 0, - external_ip: OmicronZoneExternalFloatingIp { - id: ExternalIpUuid::new_v4(), - ip: "10.0.0.1".parse().unwrap(), - }, - nic: NetworkInterface { - id: Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: *zone_id.as_untyped_uuid(), - }, - name: Name::from_str("mynic").unwrap(), - ip: "172.30.2.6".parse().unwrap(), - mac: MacAddr::random_system(), - subnet: IpNet::host_net(IpAddr::V6( - Ipv6Addr::LOCALHOST, - )), - vni: Vni::random(), - primary: true, - slot: 1, - transit_ips: vec![], - }, - external_tls: false, - external_dns_servers: vec![], - nexus_generation: Generation::new(), - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }) - .expect("freshly generated zone IDs are unique"); - - blueprint - } - #[tokio::test] async fn test_ensure_external_networking_works_with_good_target() { + const TEST_NAME: &str = + "test_ensure_external_networking_works_with_good_target"; // Setup - let logctx = dev::test_setup_log( - "test_ensure_external_networking_works_with_good_target", - ); + let logctx = dev::test_setup_log(TEST_NAME); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); - let blueprint = - create_blueprint_with_external_ip(&datastore, &opctx).await; + let (example, mut blueprint) = + ExampleSystemBuilder::new(&opctx.log, TEST_NAME).build(); + + // Insert the IP pool ranges used by our example system. + for pool_range in + example.system.external_ip_policy().clone().into_raw_ranges() + { + // This looks up the pool again for each range; we only need at most + // two (one V4, one V6), but our example system doesn't have many + // ranges so this should be fine. + let (service_authz_ip_pool, service_ip_pool) = datastore + .ip_pools_service_lookup(&opctx, pool_range.version().into()) + .await + .expect("lookup service ip pool"); + datastore + .ip_pool_add_range( + &opctx, + &service_authz_ip_pool, + &service_ip_pool, + &pool_range, + ) + .await + .expect("add range to service IP pool"); + } + + // `ExampleSystemBuilder` returns a blueprint that has an empty parent. + // To make `blueprint` the target, we have to either insert that parent + // and make it the target first, or modify `blueprint` to make it look + // like it's the original. The latter is shorter. + blueprint.parent_blueprint_id = None; + datastore.blueprint_insert(&opctx, &blueprint).await.unwrap(); let bp_target = BlueprintTarget { diff --git a/nexus/types/src/deployment/planning_input.rs b/nexus/types/src/deployment/planning_input.rs index b9a7ce27777..89cd8800248 100644 --- a/nexus/types/src/deployment/planning_input.rs +++ b/nexus/types/src/deployment/planning_input.rs @@ -1189,6 +1189,19 @@ impl ExternalIpPolicy { pub fn external_dns_ips(&self) -> &BTreeSet { &self.external_dns_ips } + + /// Consume this `ExternalIpPolicy`, returning an iterator of all of the + /// ranges contained in it. + /// + /// This destroys all meaningful information (e.g., v4 vs v6 pool; which IPs + /// are reserved for external DNS) and should only be used by tests. + pub fn into_raw_ranges(self) -> impl Iterator { + let v4_ranges = + self.service_pool_ipv4_ranges.into_iter().map(IpRange::from); + let v6_ranges = + self.service_pool_ipv6_ranges.into_iter().map(IpRange::from); + v4_ranges.chain(v6_ranges) + } } #[derive(Debug, Clone, Default)] From 8d968c524a805fb88244553537b7e682d8892705 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 16:31:14 -0500 Subject: [PATCH 12/16] test_external_dns_external_ips_specified_by_rack_setup() --- .../deployment/external_networking.rs | 223 +++++++++--------- 1 file changed, 117 insertions(+), 106 deletions(-) diff --git a/nexus/db-queries/src/db/datastore/deployment/external_networking.rs b/nexus/db-queries/src/db/datastore/deployment/external_networking.rs index 3ae7e4b478d..0723873433c 100644 --- a/nexus/db-queries/src/db/datastore/deployment/external_networking.rs +++ b/nexus/db-queries/src/db/datastore/deployment/external_networking.rs @@ -513,7 +513,11 @@ mod tests { use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; use nexus_db_model::SqlU16; use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; + use nexus_reconfigurator_planning::blueprint_editor::ExternalNetworkingAllocator; + use nexus_reconfigurator_planning::example::ExampleSystemBuilder; + use nexus_reconfigurator_planning::planner::PlannerRng; use nexus_sled_agent_shared::inventory::OmicronZoneDataset; + use nexus_types::deployment::BlueprintSource; use nexus_types::deployment::BlueprintTarget; use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneImageSource; @@ -527,6 +531,7 @@ mod tests { use omicron_common::address::DNS_OPTE_IPV4_SUBNET; use omicron_common::address::IpRange; use omicron_common::address::IpRangeIter; + use omicron_common::address::Ipv4Range; use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET; use omicron_common::address::NTP_OPTE_IPV4_SUBNET; use omicron_common::address::NUM_SOURCE_NAT_PORTS; @@ -535,10 +540,7 @@ mod tests { use omicron_common::api::external::Vni; use omicron_common::zpool_name::ZpoolName; use omicron_test_utils::dev; - use omicron_uuid_kinds::BlueprintUuid; use omicron_uuid_kinds::ExternalIpUuid; - use omicron_uuid_kinds::ExternalZpoolUuid; - use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use oxnet::IpNet; use std::net::IpAddr; @@ -1348,149 +1350,158 @@ mod tests { const TEST_NAME: &str = "test_external_dns_external_ips_specified_by_rack_setup"; - // Helper closures to reduce boilerplate below. + // Helper closure to reduce boilerplate below. let make_bp_target = |blueprint_id| BlueprintTarget { target_id: blueprint_id, enabled: false, time_made_target: Utc::now(), }; - let mut opte_ip_iter = DNS_OPTE_IPV4_SUBNET.addr_iter(); - let mut mac_iter = MacAddr::iter_system(); - let mut make_external_dns_zone = |ip, disposition| { - let zone_id = OmicronZoneUuid::new_v4(); - let pool = ZpoolName::External(ExternalZpoolUuid::new_v4()); - BlueprintZoneConfig { - disposition, - id: zone_id, - filesystem_pool: pool, - zone_type: BlueprintZoneType::ExternalDns( - blueprint_zone_type::ExternalDns { - dataset: OmicronZoneDataset { pool_name: pool }, - http_address: "[::1]:0".parse().unwrap(), - dns_address: OmicronZoneExternalFloatingAddr { - id: ExternalIpUuid::new_v4(), - addr: SocketAddr::new(ip, 0), - }, - nic: NetworkInterface { - id: Uuid::new_v4(), - kind: NetworkInterfaceKind::Service { - id: zone_id.into_untyped_uuid(), - }, - name: "test-external-dns".parse().unwrap(), - ip: opte_ip_iter.next().unwrap().into(), - mac: mac_iter.next().unwrap(), - subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET), - vni: Vni::SERVICES_VNI, - primary: true, - slot: 0, - transit_ips: vec![], - }, - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - } - }; // Set up. let logctx = dev::test_setup_log(TEST_NAME); let db = TestDatabase::new_with_datastore(&logctx.log).await; let (opctx, datastore) = (db.opctx(), db.datastore()); - // Create a blueprint with one sled and no zones. Insert it and make it - // the target. - let sled_id = SledUuid::new_v4(); - let bp0 = BlueprintBuilder::build_empty_with_sleds( - std::iter::once(sled_id), - TEST_NAME, - ); - datastore.blueprint_insert(opctx, &bp0).await.expect("inserted bp0"); - datastore - .blueprint_target_set_current(opctx, make_bp_target(bp0.id)) - .await - .expect("made bp0 the target"); + // Create a blueprint with no external DNS zones. + let (example_system, bp1) = + ExampleSystemBuilder::new(&opctx.log, TEST_NAME) + .external_dns_count(0) + .expect("external DNS count can be 0") + .build(); + + // Insert the example system's initial empty blueprint and the one we + // want to test. Advance the target to point to `blueprint`. + let bp0 = &example_system.initial_blueprint; + for bp in [bp0, &bp1] { + datastore.blueprint_insert(opctx, bp).await.expect("inserted bp"); + datastore + .blueprint_target_set_current(opctx, make_bp_target(bp.id)) + .await + .expect("made bp the target"); + } // No external DNS zones => no external DNS IPs. + assert!( + example_system + .input + .external_ip_policy() + .external_dns_ips() + .is_empty() + ); let external_dns_ips = datastore .external_dns_external_ips_specified_by_rack_setup(opctx) .await .expect("got external DNS IPs"); assert_eq!(external_dns_ips, BTreeSet::new()); - // Create a blueprint with three in-service external DNS zones. We - // should get their IPs back. - let expected_ips = ["192.168.1.1", "192.168.1.2", "192.168.1.3"] - .into_iter() - .map(|ip| ip.parse::().unwrap()) - .collect::>(); - let mut bp1 = bp0.clone(); - bp1.id = BlueprintUuid::new_v4(); - bp1.parent_blueprint_id = Some(bp0.id); - for &ip in &expected_ips { - bp1.sleds - .get_mut(&sled_id) - .unwrap() - .zones - .insert_unique(make_external_dns_zone( - ip, - BlueprintZoneDisposition::InService, - )) - .expect("freshly generated zone IDs are unique"); + // Extend the external IP policy to allow for external DNS. + let all_external_dns_ips = IpRange::V4(Ipv4Range { + first: "192.168.1.1".parse().unwrap(), + last: "192.168.1.5".parse().unwrap(), + }); + let input = { + let mut policy_builder = example_system + .input + .external_ip_policy() + .clone() + .into_builder(); + policy_builder + .push_service_pool_range(all_external_dns_ips) + .expect("valid range"); + for ip in all_external_dns_ips.iter() { + policy_builder.add_external_dns_ip(ip).expect("valid IP"); + } + + let mut input_builder = example_system.input.into_builder(); + input_builder.policy_mut().external_ips = policy_builder.build(); + input_builder.build() + }; + + // Add an in-service external DNS zone for each IP. + let mut builder = BlueprintBuilder::new_based_on( + &opctx.log, + &bp1, + &input, + TEST_NAME, + PlannerRng::from_entropy(), + ) + .expect("created builder"); + + let mut external_networking_alloc = + ExternalNetworkingAllocator::from_current_zones( + &builder, + input.external_ip_policy(), + ) + .expect("created allocator"); + + let sled_id = bp1.sleds().next().expect("at least 1 sled exists"); + for _ in all_external_dns_ips.iter() { + builder + .sled_add_zone_external_dns( + sled_id, + BlueprintZoneImageSource::InstallDataset, + external_networking_alloc + .for_new_external_dns() + .expect("got IP for external DNS"), + ) + .expect("added external DNS"); } - // Insert bp1 and make it the target. Confirm we get back the expected + // Insert bp2 and make it the target. Confirm we get back the expected // external DNS IPs. - datastore.blueprint_insert(opctx, &bp1).await.expect("inserted bp1"); + let bp2 = builder.build(BlueprintSource::Test); + let expected_ips = all_external_dns_ips.iter().collect::>(); + datastore.blueprint_insert(opctx, &bp2).await.expect("inserted bp2"); datastore - .blueprint_target_set_current(opctx, make_bp_target(bp1.id)) + .blueprint_target_set_current(opctx, make_bp_target(bp2.id)) .await - .expect("made bp1 the target"); + .expect("made bp2 the target"); let external_dns_ips = datastore .external_dns_external_ips_specified_by_rack_setup(opctx) .await .expect("got external DNS IPs"); assert_eq!(external_dns_ips, expected_ips); - // Create a third blueprint with multiple expunged external DNS zones - // covering a couple additional IPs. Those should also be returned. - let extra_ips = ["192.168.1.4", "192.168.1.5"] - .into_iter() - .map(|ip| ip.parse::().unwrap()) + // Create a new blueprint that expunges two of those zones. + let mut builder = BlueprintBuilder::new_based_on( + &opctx.log, + &bp2, + &input, + TEST_NAME, + PlannerRng::from_entropy(), + ) + .expect("created builder"); + + let to_expunge = builder + .current_zones(BlueprintZoneDisposition::is_in_service) + .filter_map(|(sled_id, zone)| { + if zone.zone_type.is_external_dns() { + Some((sled_id, zone.id)) + } else { + None + } + }) + .take(2) .collect::>(); - assert_eq!(expected_ips.intersection(&extra_ips).count(), 0); - - let mut bp2 = bp1.clone(); - bp2.id = BlueprintUuid::new_v4(); - bp2.parent_blueprint_id = Some(bp1.id); - for &ip in &extra_ips { - for i in 0..4 { - bp2.sleds - .get_mut(&sled_id) - .unwrap() - .zones - .insert_unique(make_external_dns_zone( - ip, - BlueprintZoneDisposition::Expunged { - as_of_generation: Generation::new(), - ready_for_cleanup: i % 2 == 0, - }, - )) - .expect("freshly generated zone IDs are unique"); - } + assert_eq!(to_expunge.len(), 2); + + for (sled_id, zone_id) in to_expunge { + builder.sled_expunge_zone(sled_id, zone_id).expect("expunged zone"); } - // Insert bp1 and make it the target. Confirm we get back the expected + // Insert bp3 and make it the target. Confirm we still get back all five // external DNS IPs. - datastore.blueprint_insert(opctx, &bp2).await.expect("inserted bp2"); + let bp3 = builder.build(BlueprintSource::Test); + let expected_ips = all_external_dns_ips.iter().collect::>(); + datastore.blueprint_insert(opctx, &bp3).await.expect("inserted bp3"); datastore - .blueprint_target_set_current(opctx, make_bp_target(bp2.id)) + .blueprint_target_set_current(opctx, make_bp_target(bp3.id)) .await - .expect("made bp2 the target"); + .expect("made bp3 the target"); let external_dns_ips = datastore .external_dns_external_ips_specified_by_rack_setup(opctx) .await .expect("got external DNS IPs"); - let expected_ips = - expected_ips.union(&extra_ips).copied().collect::>(); assert_eq!(external_dns_ips, expected_ips); // Clean up. From 3efe2c0b071d3786a2534572cd3ac845464e732c Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 16:48:29 -0500 Subject: [PATCH 13/16] test_default_cockroach_admin_addrs_from_blueprint() --- .../tasks/crdb_node_id_collector.rs | 180 ++++++++---------- 1 file changed, 76 insertions(+), 104 deletions(-) diff --git a/nexus/src/app/background/tasks/crdb_node_id_collector.rs b/nexus/src/app/background/tasks/crdb_node_id_collector.rs index 51b655b818a..f9563a17299 100644 --- a/nexus/src/app/background/tasks/crdb_node_id_collector.rs +++ b/nexus/src/app/background/tasks/crdb_node_id_collector.rs @@ -241,17 +241,15 @@ mod tests { use httptest::responders::status_code; use nexus_db_queries::db::pub_test_utils::TestDatabase; use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; - use nexus_sled_agent_shared::inventory::OmicronZoneDataset; - use nexus_types::deployment::BlueprintZoneConfig; + use nexus_reconfigurator_planning::example::ExampleSystemBuilder; + use nexus_reconfigurator_planning::planner::PlannerRng; + use nexus_types::deployment::BlueprintSource; use nexus_types::deployment::BlueprintZoneDisposition; use nexus_types::deployment::BlueprintZoneImageSource; use omicron_common::api::external::Generation; - use omicron_common::zpool_name::ZpoolName; use omicron_test_utils::dev; - use omicron_uuid_kinds::SledUuid; - use omicron_uuid_kinds::ZpoolUuid; use std::collections::BTreeMap; - use std::iter; + use std::collections::BTreeSet; use std::net::SocketAddr; // The `CockroachAdminFromBlueprintViaFixedPort` type above is the standard @@ -260,109 +258,83 @@ mod tests { // can _write_ that test), so test it in isolation here. #[test] fn test_default_cockroach_admin_addrs_from_blueprint() { - // Construct an empty blueprint with one sled. - let sled_id = SledUuid::new_v4(); - let mut blueprint = BlueprintBuilder::build_empty_with_sleds( - iter::once(sled_id), - "test", - ); - let bp_sled = &mut blueprint - .sleds - .get_mut(&sled_id) - .expect("found entry for test sled"); - - let zpool_id = ZpoolUuid::new_v4(); - let make_crdb_zone_config = - |disposition, id, addr: SocketAddrV6| BlueprintZoneConfig { - disposition, - id, - filesystem_pool: ZpoolName::new_external(zpool_id), - zone_type: BlueprintZoneType::CockroachDb( - blueprint_zone_type::CockroachDb { - address: addr, - dataset: OmicronZoneDataset { - pool_name: format!("oxp_{}", zpool_id) - .parse() - .unwrap(), - }, - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }; - - // Add three CRDB zones with known addresses; the first and third are - // in service, and the second is expunged. Only the first and third - // should show up when we ask for addresses below. - let crdb_id1 = OmicronZoneUuid::new_v4(); - let crdb_id2 = OmicronZoneUuid::new_v4(); - let crdb_id3 = OmicronZoneUuid::new_v4(); - let crdb_addr1: SocketAddrV6 = "[2001:db8::1]:1111".parse().unwrap(); - let crdb_addr2: SocketAddrV6 = "[2001:db8::2]:1234".parse().unwrap(); - let crdb_addr3: SocketAddrV6 = "[2001:db8::3]:1234".parse().unwrap(); - bp_sled - .zones - .insert_unique(make_crdb_zone_config( - BlueprintZoneDisposition::InService, - crdb_id1, - crdb_addr1, - )) - .expect("freshly generated zone IDs are unique"); - bp_sled - .zones - .insert_unique(make_crdb_zone_config( - BlueprintZoneDisposition::Expunged { + const TEST_NAME: &str = + "test_default_cockroach_admin_addrs_from_blueprint"; + let logctx = dev::test_setup_log(TEST_NAME); + let log = &logctx.log; + + // Build an example system with one sled. + let (example_system, bp0) = + ExampleSystemBuilder::new(log, TEST_NAME).nsleds(1).build(); + let input = example_system.input; + + // `ExampleSystemBuilder` doesn't place any cockroach nodes; assert so + // we bail out early if that changes. + let ncockroach = bp0 + .all_omicron_zones(BlueprintZoneDisposition::is_in_service) + .filter(|(_, z)| z.zone_type.is_cockroach()) + .count(); + assert_eq!(ncockroach, 0); + + // This blueprint has no cockroach zones, so should have no admin addrs + // either. + let admin_addrs = CockroachAdminFromBlueprintViaFixedPort + .cockroach_admin_addrs(&bp0) + .collect::>(); + assert_eq!(admin_addrs, Vec::new()); + + // Add 5 cockroach zones to our sled. + let mut builder = BlueprintBuilder::new_based_on( + log, + &bp0, + &input, + TEST_NAME, + PlannerRng::from_entropy(), + ) + .expect("constructed builder"); + let sled_id = bp0.sleds().next().expect("1 sled"); + for _ in 0..5 { + builder + .sled_add_zone_cockroachdb( + sled_id, + BlueprintZoneImageSource::InstallDataset, + ) + .expect("added cockroach"); + } + let mut bp1 = builder.build(BlueprintSource::Test); + + // Mutate this blueprint: expunge 2 of the 5 zones. Record the expected + // admin addrs from the other three. + let mut expected = BTreeSet::new(); + let sled_config = bp1.sleds.get_mut(&sled_id).unwrap(); + let mut seen = 0; + for mut zone in sled_config.zones.iter_mut() { + if !zone.zone_type.is_cockroach() { + continue; + } + // Keep even; expunge odd. + if seen % 2 == 0 { + let ip = zone.underlay_ip(); + let addr = SocketAddrV6::new(ip, COCKROACH_ADMIN_PORT, 0, 0); + expected.insert((zone.id, addr)); + } else { + zone.disposition = BlueprintZoneDisposition::Expunged { as_of_generation: Generation::new(), ready_for_cleanup: false, - }, - crdb_id2, - crdb_addr2, - )) - .expect("freshly generated zone IDs are unique"); - bp_sled - .zones - .insert_unique(make_crdb_zone_config( - BlueprintZoneDisposition::InService, - crdb_id3, - crdb_addr3, - )) - .expect("freshly generated zone IDs are unique"); - - // Also add a non-CRDB zone to ensure it's filtered out. - bp_sled - .zones - .insert_unique(BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id: OmicronZoneUuid::new_v4(), - filesystem_pool: ZpoolName::new_external(ZpoolUuid::new_v4()), - zone_type: BlueprintZoneType::CruciblePantry( - blueprint_zone_type::CruciblePantry { - address: "[::1]:0".parse().unwrap(), - }, - ), - image_source: BlueprintZoneImageSource::InstallDataset, - }) - .expect("freshly generated zone IDs are unique"); - - // We expect to see CRDB zones 1 and 3 with their IPs but the ports - // changed to `COCKROACH_ADMIN_PORT`. - let mut expected = vec![ - ( - crdb_id1, - SocketAddrV6::new(*crdb_addr1.ip(), COCKROACH_ADMIN_PORT, 0, 0), - ), - ( - crdb_id3, - SocketAddrV6::new(*crdb_addr3.ip(), COCKROACH_ADMIN_PORT, 0, 0), - ), - ]; - // We sort starting with zone id, since the original zones are sorted - // that way in a map. - expected.sort_unstable(); + }; + } + seen += 1; + } + assert_eq!(seen, 5); + assert_eq!(expected.len(), 3); + // Confirm we only see the three expected addrs. let admin_addrs = CockroachAdminFromBlueprintViaFixedPort - .cockroach_admin_addrs(&blueprint) - .collect::>(); + .cockroach_admin_addrs(&bp1) + .collect::>(); assert_eq!(expected, admin_addrs); + + logctx.cleanup_successful(); } #[tokio::test] From 9c30cd7f73597e8a5a94fad87868123754fbbc15 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 16:50:47 -0500 Subject: [PATCH 14/16] remove build_empty_with_sleds() --- .../planning/src/blueprint_builder/builder.rs | 62 ++----------------- 1 file changed, 6 insertions(+), 56 deletions(-) diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 8f189f66673..44bb29207ac 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -33,7 +33,6 @@ use nexus_types::deployment::BlueprintHostPhase2DesiredContents; use nexus_types::deployment::BlueprintHostPhase2DesiredSlots; use nexus_types::deployment::BlueprintPhysicalDiskConfig; use nexus_types::deployment::BlueprintPhysicalDiskDisposition; -use nexus_types::deployment::BlueprintSledConfig; use nexus_types::deployment::BlueprintSource; use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneDisposition; @@ -486,9 +485,8 @@ struct OximeterReadPolicy { /// /// 1. Build one directly. This would generally only be used once in the /// lifetime of a rack, to assemble the first blueprint during rack setup. -/// It is also common in tests. To start with a blueprint that contains an -/// empty zone config for some number of sleds, use -/// [`BlueprintBuilder::build_empty_with_sleds`]. +/// It is also common in tests. To start with an empty initial blueprint, +/// use [`BlueprintBuilder::build_empty`]. /// /// 2. Build one _from_ another blueprint, called the "parent", making changes /// as desired. Use [`BlueprintBuilder::new_based_on`] for this. Once the @@ -533,64 +531,16 @@ impl<'a> BlueprintBuilder<'a> { /// Directly construct an empty blueprint: no sleds; default values for all /// other fields. pub fn build_empty(creator: &str) -> Blueprint { - Self::build_empty_with_sleds(iter::empty(), creator) + Self::build_empty_seeded(creator, PlannerRng::from_entropy()) } /// A version of [`Self::build_empty`] that allows the /// blueprint ID to be generated from a deterministic RNG. - pub fn build_empty_seeded(creator: &str, rng: PlannerRng) -> Blueprint { - Self::build_empty_with_sleds_seeded(iter::empty(), creator, rng) - } - - /// Directly construct a `Blueprint` that contains an empty zone config for - /// the given sleds. - pub fn build_empty_with_sleds( - sled_ids: impl Iterator, - creator: &str, - ) -> Blueprint { - Self::build_empty_with_sleds_impl( - sled_ids, - creator, - PlannerRng::from_entropy(), - ) - } - - /// A version of [`Self::build_empty_with_sleds`] that allows the - /// blueprint ID to be generated from a deterministic RNG. - pub fn build_empty_with_sleds_seeded( - sled_ids: impl Iterator, - creator: &str, - rng: PlannerRng, - ) -> Blueprint { - Self::build_empty_with_sleds_impl(sled_ids, creator, rng) - } - - fn build_empty_with_sleds_impl( - sled_ids: impl Iterator, - creator: &str, - mut rng: PlannerRng, - ) -> Blueprint { - let sleds = sled_ids - .map(|sled_id| { - let config = BlueprintSledConfig { - state: SledState::Active, - sled_agent_generation: Generation::new(), - disks: IdOrdMap::default(), - datasets: IdOrdMap::default(), - zones: IdOrdMap::default(), - remove_mupdate_override: None, - host_phase_2: - BlueprintHostPhase2DesiredSlots::current_contents(), - }; - (sled_id, config) - }) - .collect::>(); - let num_sleds = sleds.len(); - + pub fn build_empty_seeded(creator: &str, mut rng: PlannerRng) -> Blueprint { let id = rng.next_blueprint(); Blueprint { id, - sleds, + sleds: BTreeMap::new(), pending_mgs_updates: PendingMgsUpdates::new(), parent_blueprint_id: None, internal_dns_version: Generation::new(), @@ -605,7 +555,7 @@ impl<'a> BlueprintBuilder<'a> { oximeter_read_mode: OximeterReadMode::SingleNode, time_created: now_db_precision(), creator: creator.to_owned(), - comment: format!("starting blueprint with {num_sleds} empty sleds"), + comment: format!("starting blueprint (empty)"), // The only reason to create empty blueprints is tests. If that // changes (e.g., if RSS starts using this builder to generate its // blueprints), we could take a `source` argument instead. From de048ec9af32f54ca8c2f28ae4c62f89f02ab6a4 Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 16:55:01 -0500 Subject: [PATCH 15/16] expectorate --- .../tests/output/cmds-blueprint-history-stdout | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev-tools/reconfigurator-cli/tests/output/cmds-blueprint-history-stdout b/dev-tools/reconfigurator-cli/tests/output/cmds-blueprint-history-stdout index 72ec7dd7aab..ea19fbdcd47 100644 --- a/dev-tools/reconfigurator-cli/tests/output/cmds-blueprint-history-stdout +++ b/dev-tools/reconfigurator-cli/tests/output/cmds-blueprint-history-stdout @@ -26,7 +26,7 @@ blueprint df06bb57-ad42-4431-9206-abff322896c7 created from blueprint af934083-5 > # which does not include any of those blueprints. > blueprint-history TIME BLUEPRINT - 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint with 0 empty sleds + 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint (empty) dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 @@ -34,7 +34,7 @@ TIME BLUEPRINT > # You can give it a specific blueprint. > blueprint-history 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 TIME BLUEPRINT - 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint with 0 empty sleds + 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint (empty) dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 @@ -43,7 +43,7 @@ TIME BLUEPRINT > # Running it from the latest blueprint should report all of them. > blueprint-history latest TIME BLUEPRINT - 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint with 0 empty sleds + 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint (empty) dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 8da82a8e-bf97-4fbd-8ddd-9f6462732cf1 58d5e830-0884-47d8-a7cd-b2b3751adeb4 @@ -64,7 +64,7 @@ TIME BLUEPRINT > # Show diffs, too. > blueprint-history --diff latest TIME BLUEPRINT - 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint with 0 empty sleds + 184f10b3-61cb-41ef-9b93-3489b2bac559 starting blueprint (empty) dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 from: blueprint 184f10b3-61cb-41ef-9b93-3489b2bac559 to: blueprint dbcbd3d6-41ff-48ae-ac0b-1becc9b2fd21 From 263cd1f4026cbd1400153a01098f1cf3ee88167c Mon Sep 17 00:00:00 2001 From: John Gallagher Date: Wed, 12 Nov 2025 17:06:53 -0500 Subject: [PATCH 16/16] clippy --- nexus/reconfigurator/planning/src/blueprint_builder/builder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 44bb29207ac..ca9c8c67480 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -555,7 +555,7 @@ impl<'a> BlueprintBuilder<'a> { oximeter_read_mode: OximeterReadMode::SingleNode, time_created: now_db_precision(), creator: creator.to_owned(), - comment: format!("starting blueprint (empty)"), + comment: "starting blueprint (empty)".to_string(), // The only reason to create empty blueprints is tests. If that // changes (e.g., if RSS starts using this builder to generate its // blueprints), we could take a `source` argument instead.