Add two features for local storage support (#9386)

Bump the sled-agent API to add two new features for local storage
support:

- Add an endpoint for Nexus to create and destroy local storage
datasets. These will be allocated and deallocated as part of the higher
level Disk lifecycle for the forthcoming local storage disk type.

- Add the ability to delegate a specific zvol to a Propolis zone. This
required accepting a new `DelegatedZvol` parameter during vmm
registration.

Author: jmpesp

common/src/api/internal/shared.rs

@@ -7,8 +7,12 @@
 use crate::{
     address::NUM_SOURCE_NAT_PORTS,
     api::external::{self, BfdMode, ImportExportPolicy, Name, Vni},
+    disk::DatasetName,
+    zpool_name::ZpoolName,
 };
 use daft::Diffable;
+use omicron_uuid_kinds::DatasetUuid;
+use omicron_uuid_kinds::ExternalZpoolUuid;
 use oxnet::{IpNet, Ipv4Net, Ipv6Net};
 use schemars::JsonSchema;
 use serde::{Deserialize, Deserializer, Serialize, Serializer, de};
@@ -944,7 +948,7 @@ pub enum DatasetKind {
     // Other datasets
     Debug,
 
-    /// Used for transient storage, contains volumes delegated to VMMs
+    /// Used for local storage disk types, contains volumes delegated to VMMs
     LocalStorage,
 }
 
@@ -1114,6 +1118,71 @@ pub struct SledIdentifiers {
     pub serial: String,
 }
 
+/// Delegate a ZFS volume to a zone
+#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum DelegatedZvol {
+    /// Delegate a slice of the local storage dataset present on this pool into
+    /// the zone.
+    LocalStorage { zpool_id: ExternalZpoolUuid, dataset_id: DatasetUuid },
+}
+
+impl DelegatedZvol {
+    /// Return the fully qualified dataset name that the volume is in.
+    pub fn parent_dataset_name(&self) -> String {
+        match &self {
+            DelegatedZvol::LocalStorage { zpool_id, dataset_id } => {
+                // The local storage dataset is the parent for an allocation
+                let local_storage_parent = DatasetName::new(
+                    ZpoolName::External(*zpool_id),
+                    DatasetKind::LocalStorage,
+                );
+
+                format!("{}/{}", local_storage_parent.full_name(), dataset_id)
+            }
+        }
+    }
+
+    /// Return the mountpoint for the parent dataset in the zone
+    pub fn parent_dataset_mountpoint(&self) -> String {
+        match &self {
+            DelegatedZvol::LocalStorage { dataset_id, .. } => {
+                format!("/{}", dataset_id)
+            }
+        }
+    }
+
+    /// Return the fully qualified volume name
+    pub fn volume_name(&self) -> String {
+        match &self {
+            DelegatedZvol::LocalStorage { .. } => {
+                // For now, all local storage zvols use the same name
+                format!("{}/vol", self.parent_dataset_name())
+            }
+        }
+    }
+
+    /// Return the device that should be delegated into the zone
+    pub fn zvol_device(&self) -> String {
+        match &self {
+            DelegatedZvol::LocalStorage { .. } => {
+                // Use the `rdsk` device to avoid interacting with an additional
+                // buffer cache that would be used if we used `dsk`.
+                format!("/dev/zvol/rdsk/{}", self.volume_name())
+            }
+        }
+    }
+
+    pub fn volblocksize(&self) -> u32 {
+        match &self {
+            DelegatedZvol::LocalStorage { .. } => {
+                // all Local storage zvols use 4096 byte blocks
+                4096
+            }
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1207,4 +1276,23 @@ mod tests {
             );
         }
     }
+
+    #[test]
+    fn test_delegated_zvol_device_name() {
+        let delegated_zvol = DelegatedZvol::LocalStorage {
+            zpool_id: "cb832c2e-fa94-4911-89a9-895ac8b1e8f3".parse().unwrap(),
+            dataset_id: "2bbf0908-21da-4bc3-882b-1a1e715c54bd".parse().unwrap(),
+        };
+
+        assert_eq!(
+            delegated_zvol.zvol_device(),
+            [
+                String::from("/dev/zvol/rdsk"),
+                String::from("oxp_cb832c2e-fa94-4911-89a9-895ac8b1e8f3/crypt"),
+                String::from("local_storage"),
+                String::from("2bbf0908-21da-4bc3-882b-1a1e715c54bd/vol"),
+            ]
+            .join("/"),
+        );
+    }
 }

illumos-utils/src/zfs.rs

@@ -220,6 +220,76 @@ pub struct DestroySnapshotError {
     err: crate::ExecutionError,
 }
 
+#[derive(thiserror::Error, Debug)]
+pub enum EnsureDatasetVolumeErrorInner {
+    #[error(transparent)]
+    Execution(#[from] crate::ExecutionError),
+
+    #[error(transparent)]
+    GetValue(#[from] GetValueError),
+
+    #[error("value {value_name} parse error: {value} not a number!")]
+    ValueParseError { value_name: String, value: String },
+
+    #[error("expected {value_name} to be {expected}, but saw {actual}")]
+    ValueMismatch { value_name: String, expected: u64, actual: u64 },
+}
+
+/// Error returned by [`Zfs::ensure_dataset_volume`].
+#[derive(thiserror::Error, Debug)]
+#[error("Failed to ensure volume '{name}': {err}")]
+pub struct EnsureDatasetVolumeError {
+    name: String,
+    #[source]
+    err: EnsureDatasetVolumeErrorInner,
+}
+
+impl EnsureDatasetVolumeError {
+    pub fn execution(name: String, err: crate::ExecutionError) -> Self {
+        EnsureDatasetVolumeError {
+            name,
+            err: EnsureDatasetVolumeErrorInner::Execution(err),
+        }
+    }
+
+    pub fn get_value(name: String, err: GetValueError) -> Self {
+        EnsureDatasetVolumeError {
+            name,
+            err: EnsureDatasetVolumeErrorInner::GetValue(err),
+        }
+    }
+
+    pub fn value_parse(
+        name: String,
+        value_name: String,
+        value: String,
+    ) -> Self {
+        EnsureDatasetVolumeError {
+            name,
+            err: EnsureDatasetVolumeErrorInner::ValueParseError {
+                value_name,
+                value,
+            },
+        }
+    }
+
+    pub fn value_mismatch(
+        name: String,
+        value_name: String,
+        expected: u64,
+        actual: u64,
+    ) -> Self {
+        EnsureDatasetVolumeError {
+            name,
+            err: EnsureDatasetVolumeErrorInner::ValueMismatch {
+                value_name,
+                expected,
+                actual,
+            },
+        }
+    }
+}
+
 /// Wraps commands for interacting with ZFS.
 pub struct Zfs {}
 
@@ -1339,6 +1409,88 @@ impl Zfs {
         }
         Ok(result)
     }
+
+    pub async fn ensure_dataset_volume(
+        name: String,
+        size: ByteCount,
+        block_size: u32,
+    ) -> Result<(), EnsureDatasetVolumeError> {
+        let mut command = Command::new(PFEXEC);
+        let cmd = command.args(&[ZFS, "create"]);
+
+        cmd.args(&[
+            "-V",
+            &size.to_bytes().to_string(),
+            "-o",
+            &format!("volblocksize={}", block_size),
+            &name,
+        ]);
+
+        // The command to create a dataset is not idempotent and will fail with
+        // "dataset already exists" if the volume is created already. Eat this
+        // and return Ok instead.
+
+        match execute_async(cmd).await {
+            Ok(_) => Ok(()),
+
+            Err(crate::ExecutionError::CommandFailure(info))
+                if info.stderr.contains("dataset already exists") =>
+            {
+                // Validate that the total size and volblocksize are what is
+                // being requested: these cannot be changed once the volume is
+                // created.
+
+                let [actual_size, actual_block_size] =
+                    Self::get_values(&name, &["volsize", "volblocksize"], None)
+                        .await
+                        .map_err(|err| {
+                            EnsureDatasetVolumeError::get_value(
+                                name.clone(),
+                                err,
+                            )
+                        })?;
+
+                let actual_size: u64 = actual_size.parse().map_err(|_| {
+                    EnsureDatasetVolumeError::value_parse(
+                        name.clone(),
+                        String::from("volsize"),
+                        actual_size,
+                    )
+                })?;
+
+                let actual_block_size: u32 =
+                    actual_block_size.parse().map_err(|_| {
+                        EnsureDatasetVolumeError::value_parse(
+                            name.clone(),
+                            String::from("volblocksize"),
+                            actual_block_size,
+                        )
+                    })?;
+
+                if actual_size != size.to_bytes() {
+                    return Err(EnsureDatasetVolumeError::value_mismatch(
+                        name.clone(),
+                        String::from("volsize"),
+                        size.to_bytes(),
+                        actual_size,
+                    ));
+                }
+
+                if actual_block_size != block_size {
+                    return Err(EnsureDatasetVolumeError::value_mismatch(
+                        name.clone(),
+                        String::from("volblocksize"),
+                        u64::from(block_size),
+                        u64::from(actual_block_size),
+                    ));
+                }
+
+                Ok(())
+            }
+
+            Err(err) => Err(EnsureDatasetVolumeError::execution(name, err)),
+        }
+    }
 }
 
 /// A read-only snapshot of a ZFS filesystem.

nexus/mgs-updates/src/test_util/host_phase_2_test_state.rs

@@ -890,5 +890,20 @@ mod api_impl {
         ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
             unimplemented!()
         }
+
+        async fn local_storage_dataset_ensure(
+            _request_context: RequestContext<Self::Context>,
+            _path_params: Path<LocalStoragePathParam>,
+            _body: TypedBody<LocalStorageDatasetEnsureRequest>,
+        ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+            unimplemented!()
+        }
+
+        async fn local_storage_dataset_delete(
+            _request_context: RequestContext<Self::Context>,
+            _path_params: Path<LocalStoragePathParam>,
+        ) -> Result<HttpResponseUpdatedNoContent, HttpError> {
+            unimplemented!()
+        }
     }
 }

nexus/src/app/instance.rs

@@ -1511,6 +1511,7 @@ impl super::Nexus {
                 host_domain: None,
                 search_domains: Vec::new(),
             },
+            delegated_zvols: vec![],
         };
 
         let instance_id = InstanceUuid::from_untyped_uuid(db_instance.id());