From 6c97cc32d6a3313230164c279fb5eb6c06fada8c Mon Sep 17 00:00:00 2001 From: jlothe Date: Mon, 3 Nov 2025 16:26:24 +0530 Subject: [PATCH 1/9] CON-3876 - NVMEoTCP support for csi-driver and host-libs Signed-off-by: jlothe --- pkg/driver/controller_server.go | 35 +++- pkg/driver/node_server.go | 44 ++++ pkg/driver/utils.go | 9 + .../common-host-libs/linux/device.go | 44 +++- .../common-host-libs/linux/initiator.go | 29 ++- .../common-host-libs/linux/multipath.go | 48 +++++ .../common-host-libs/linux/nvme.go | 197 ++++++++++++++++++ .../common-host-libs/model/types.go | 27 +++ .../common-host-libs/tunelinux/nvme.go | 2 + 9 files changed, 426 insertions(+), 9 deletions(-) create mode 100644 vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go create mode 100644 vendor/github.com/hpe-storage/common-host-libs/tunelinux/nvme.go diff --git a/pkg/driver/controller_server.go b/pkg/driver/controller_server.go index d94ffb59..1358d005 100644 --- a/pkg/driver/controller_server.go +++ b/pkg/driver/controller_server.go @@ -924,7 +924,9 @@ func (driver *Driver) controllerPublishVolume( requestedAccessProtocol = iscsi } else if requestedAccessProtocol == "fc" { requestedAccessProtocol = fc - } + } else if requestedAccessProtocol == "nvmetcp" { + requestedAccessProtocol = "nvmetcp" + } if existingNode != nil { log.Tracef("CSP has already been notified about the node with ID %s and UUID %s", existingNode.ID, existingNode.UUID) @@ -957,12 +959,33 @@ func (driver *Driver) controllerPublishVolume( // TODO: add any additional info necessary to mount the device publishContext := map[string]string{} - publishContext[serialNumberKey] = publishInfo.SerialNumber - publishContext[accessProtocolKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol - publishContext[targetNamesKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames, ",") - publishContext[targetScopeKey] = requestedTargetScope - publishContext[lunIDKey] = strconv.Itoa(int(publishInfo.AccessInfo.BlockDeviceAccessInfo.LunID)) + + // NVMe over TCP support + if strings.EqualFold(publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol, "nvmetcp") { + publishContext[serialNumberKey] = publishInfo.SerialNumber + publishContext[accessProtocolKey] = "nvmetcp" + // NQN, target address, and port for NVMe/TCP + if len(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames) > 0 { + publishContext[targetNamesKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames[0] + } + if len(publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.NvmeDiscoveryIPs) > 0 { + publishContext[discoveryIPsKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.NvmeDiscoveryIPs[0] + } + if publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.TargetPort != "" { + publishContext[targetPortKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.TargetPort + } else { + publishContext[targetPortKey] = "4420" // default NVMe/TCP port + } + + }else{ + publishContext[serialNumberKey] = publishInfo.SerialNumber + publishContext[accessProtocolKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol + publishContext[targetNamesKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames, ",") + publishContext[targetScopeKey] = requestedTargetScope + publishContext[lunIDKey] = strconv.Itoa(int(publishInfo.AccessInfo.BlockDeviceAccessInfo.LunID)) + } + // Start of population of target array details if publishInfo.AccessInfo.BlockDeviceAccessInfo.SecondaryBackendDetails.PeerArrayDetails != nil { secondaryArrayMarshalledStr, err := json.Marshal(&publishInfo.AccessInfo.BlockDeviceAccessInfo.SecondaryBackendDetails) diff --git a/pkg/driver/node_server.go b/pkg/driver/node_server.go index 0f69493d..e9104cbc 100644 --- a/pkg/driver/node_server.go +++ b/pkg/driver/node_server.go @@ -41,6 +41,8 @@ var ( const ( fileHostIPKey = "hostIP" + nvmetcp = "nvmetcp" + targetPortKey = "targetPort" ) var isWatcherEnabled = false @@ -572,6 +574,41 @@ func (driver *Driver) setupDevice( log.Tracef(">>>>> setupDevice, volumeID: %s, publishContext: %v", volumeID, log.MapScrubber(publishContext)) defer log.Trace("<<<<< setupDevice") + // Handle NVMe over TCP volumes + if publishContext[accessProtocolKey] == "nvmetcp" { + volume := &model.Volume{ + SerialNumber: publishContext[serialNumberKey], + AccessProtocol: publishContext[accessProtocolKey], + Nqn: publishContext[targetNamesKey], // NQN for NVMe + TargetAddress: publishContext[discoveryIPsKey], // Target IP + TargetPort: publishContext[targetPortKey], // Target port (default 4420) + ConnectionMode: defaultConnectionMode, + } + + // Cleanup any stale device existing before stage + device, _ := driver.chapiDriver.GetDevice(volume) + if device != nil { + device.TargetScope = volume.TargetScope + err := driver.chapiDriver.DeleteDevice(device) + if err != nil { + log.Warnf("Failed to cleanup stale NVMe device %s before staging, err %s", device.AltFullPathName, err.Error()) + } + } + + // Create NVMe Device + devices, err := driver.chapiDriver.CreateDevices([]*model.Volume{volume}) + if err != nil { + log.Errorf("Failed to create NVMe device from publish info. Error: %s", err.Error()) + return nil, err + } + if len(devices) == 0 { + log.Errorf("Failed to get the NVMe device just created using the volume %+v", volume) + return nil, fmt.Errorf("unable to find the NVMe device for volume %+v", volume) + } + + return devices[0], nil + } + // TODO: Enhance CHAPI to work with a PublishInfo object rather than a volume discoveryIps := strings.Split(publishContext[discoveryIPsKey], ",") @@ -2203,6 +2240,7 @@ func (driver *Driver) nodeGetInfo() (string, error) { var iqns []*string var wwpns []*string + var nqn *string for _, initiator := range initiators { if initiator.Type == iscsi { for i := 0; i < len(initiator.Init); i++ { @@ -2215,6 +2253,11 @@ func (driver *Driver) nodeGetInfo() (string, error) { } } + nvmeNqn, err := GetNvmeInitiator() + if err == nil && nvmeNqn != "" { + nqn = &nvmeNqn + } + var cidrNetworks []*string for _, network := range networks { log.Infof("Processing network named %s with IpV4 CIDR %s", network.Name, network.CidrNetwork) @@ -2230,6 +2273,7 @@ func (driver *Driver) nodeGetInfo() (string, error) { Iqns: iqns, Networks: cidrNetworks, Wwpns: wwpns, + Nqn: nqn, } nodeID, err := driver.flavor.LoadNodeInfo(node) diff --git a/pkg/driver/utils.go b/pkg/driver/utils.go index 6268ecc1..6b1d3f00 100644 --- a/pkg/driver/utils.go +++ b/pkg/driver/utils.go @@ -126,3 +126,12 @@ func removeDataFile(dirPath string, fileName string) error { func isValidIP(ip string) bool { return ip != "" && net.ParseIP(ip) != nil } +// GetNvmeInitiator returns the NVMe host NQN as a string +func GetNvmeInitiator() (string, error) { + data, err := ioutil.ReadFile("/etc/nvme/hostnqn") + if err != nil { + return "", err + } + nqn := strings.TrimSpace(string(data)) + return nqn, nil +} \ No newline at end of file diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/device.go b/vendor/github.com/hpe-storage/common-host-libs/linux/device.go index 175a8874..3c0eab25 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/device.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/device.go @@ -49,6 +49,7 @@ const ( // lrwxrwxrwx 1 root root 0 Mar 8 16:51 sdg -> ../devices/platform/host4/session2/target4:0:0/4:0:0:2/block/sdg deviceByHctlPatternFmt = ".*%s:%s:%s:%s.*block/(?P.*)" lunNotSupportedErr = "LOGICAL UNIT NOT SUPPORTED" + nvmeDevicePattern = "nvme[0-9]+n[0-9]+" ) var ( @@ -450,7 +451,13 @@ func rescanLoginVolumeForBackend(volObj *model.Volume) error { if err != nil { return err } - } else { + } else if strings.EqualFold(volObj.AccessProtocol, "nvmetcp") { + // NVMe over TCP volume + err = HandleNvmeTcpDiscovery(volObj) + if err != nil { + return err + } + }else { // Check if client intends us to specifically login using multiple IP addresses(cloud volumes) if len(volObj.Networks) > 0 { // check if ifaces are created and enable port binding @@ -1094,6 +1101,17 @@ func GetDeviceFromVolume(vol *model.Volume) (*model.Device, error) { if err != nil { return nil, err } + + if len(devices) > 0 { + return devices[0], nil + } + + // Try NVMe device lookup if not found above + nvmeDev, err := GetNvmeDeviceFromNamespace(vol.SerialNumber) + if err == nil && nvmeDev != nil { + log.Debugf("Found NVMe device: %+v", nvmeDev) + return nvmeDev, nil + } if len(devices) == 0 { return nil, fmt.Errorf("unable to find device matching volume serial number %s", vol.SerialNumber) } @@ -1426,3 +1444,27 @@ func isMappedLuksDevice(devPath string) (bool, error) { return false, err } } + +// GetNvmeDeviceFromNamespace returns NVMe device path for given namespace or serial +func GetNvmeDeviceFromNamespace(serialOrNamespace string) (*model.Device, error) { + nvmeRoot := "/dev/" + files, err := ioutil.ReadDir(nvmeRoot) + if err != nil { + return nil, err + } + nvmeRegex := regexp.MustCompile(nvmeDevicePattern) + for _, f := range files { + if nvmeRegex.MatchString(f.Name()) { + // Optionally, check namespace/serial via sysfs + sysfsSerialPath := fmt.Sprintf("/sys/class/block/%s/serial", f.Name()) + serial, _ := util.FileReadFirstLine(sysfsSerialPath) + if serialOrNamespace == f.Name() || serialOrNamespace == serial { + return &model.Device{ + Pathname: nvmeRoot + f.Name(), + SerialNumber: serial, + }, nil + } + } + } + return nil, fmt.Errorf("NVMe device not found for %s", serialOrNamespace) +} \ No newline at end of file diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go b/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go index e2f1e773..1858b32c 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go @@ -4,6 +4,7 @@ package linux import ( "errors" + log "github.com/hpe-storage/common-host-libs/logger" "github.com/hpe-storage/common-host-libs/model" "github.com/hpe-storage/common-host-libs/util" @@ -32,15 +33,24 @@ func GetInitiators() ([]*model.Initiator, error) { if err != nil { log.Debug("Error getting FcInitiator: ", err) } + // Add NVMe initiator discovery + nvmeInits, err := getNvmeInitiators() + if err != nil { + log.Debug("Error getting NvmeInitiator: ", err) + } + if fcInits != nil { inits = append(inits, fcInits) } if iscsiInits != nil { inits = append(inits, iscsiInits) } + if nvmeInits != nil { + inits = append(inits, nvmeInits) + } - if fcInits == nil && iscsiInits == nil { - return nil, errors.New("iscsi and fc initiators not found") + if fcInits == nil && iscsiInits == nil && nvmeInits == nil { + return nil, errors.New("iscsi, fc, and nvme initiators not found") } log.Debug("initiators ", inits) @@ -94,3 +104,18 @@ func getFcInitiators() (fcInit *model.Initiator, err error) { } return fcInit, nil } + +func getNvmeInitiators() (init *model.Initiator, err error) { + log.Trace(">>>>> getNvmeInitiators") + defer log.Trace("<<<<< getNvmeInitiators") + + hostnqn, err := GetNvmeInitiator() + if err != nil { + log.Debugf("NVMe host NQN not found, assuming not an NVMe host") + return nil, nil + } + + initiators := []string{hostnqn} + init = &model.Initiator{Type: "nvmeotcp", Init: initiators} + return init, nil +} diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/multipath.go b/vendor/github.com/hpe-storage/common-host-libs/linux/multipath.go index b3fd78fe..4bbaaf68 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/multipath.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/multipath.go @@ -4,6 +4,8 @@ package linux import ( "fmt" + "io/ioutil" + "path/filepath" "regexp" "strings" "sync" @@ -219,6 +221,15 @@ func cleanupDeviceAndSlaves(dev *model.Device) (err error) { log.Tracef(">>>>> cleanupDeviceAndSlaves called for %+v", dev) defer log.Trace("<<<<< cleanupDeviceAndSlaves") + + // --- NVMe multipath handling --- + if dev != nil && dev.Pathname != "" && IsNvmeDevice(dev.Pathname) { + if err := HandleMultipathForDevice(dev); err != nil { + return err + } + } + // --- End NVMe multipath handling --- + isFC := isFibreChannelDevice(dev.Slaves) // disable queuing on multipath @@ -498,3 +509,40 @@ func multipathGetPathsOfDevice(dev *model.Device, needActivePath bool) (paths [] } return paths, nil } +// IsNvmeDevice returns true if the device path is an NVMe device (e.g., /dev/nvmeXnY) +func IsNvmeDevice(devPath string) bool { + base := filepath.Base(devPath) + matched, _ := regexp.MatchString(`^nvme\d+n\d+$`, base) + return matched +} + +// IsNvmeMultipathEnabled checks if NVMe native multipath is enabled for a device +func IsNvmeMultipathEnabled(devPath string) bool { + base := filepath.Base(devPath) + nvmeMpPath := filepath.Join("/sys/class/block", base, "nvme_multipath") + data, err := ioutil.ReadFile(nvmeMpPath) + if err != nil { + return false + } + return strings.TrimSpace(string(data)) == "Y" +} +// HandleMultipathForDevice handles multipath for both SCSI and NVMe devices +func HandleMultipathForDevice(dev *model.Device) error { + + if IsNvmeDevice(dev.Pathname) { + if IsNvmeMultipathEnabled(dev.Pathname) { + log.Debugf("NVMe native multipath is enabled for %s, skipping dm-multipath logic", dev.Pathname) + // All path management is handled by the NVMe subsystem + return nil + } + log.Debugf("NVMe device %s does not have native multipath enabled", dev.Pathname) + // Optional: fallback to dm-multipath for NVMe if required by your environment + // If not required, just return nil here + // Otherwise, call your existing dm-multipath logic below + // return handleDmMultipathForNvme(dev) + return nil + } + // Existing dm-multipath logic for SCSI/iSCSI devices + // return handleDmMultipathForScsi(dev) + return nil +} \ No newline at end of file diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go b/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go new file mode 100644 index 00000000..2ad1a044 --- /dev/null +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go @@ -0,0 +1,197 @@ +// Copyright 2025 Hewlett Packard Enterprise Development LP + +package linux + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + log "github.com/hpe-storage/common-host-libs/logger" + "github.com/hpe-storage/common-host-libs/model" + "github.com/hpe-storage/common-host-libs/util" +) + +const ( + nvmecmd = "nvme" + nvmeConnectCmd = "nvme connect" + nvmeDisconnectCmd = "nvme disconnect" + nvmeListCmd = "nvme list" + nvmeListSubsysCmd = "nvme list-subsys" + defaultNvmePort = "4420" + nvmeHostPathFormat = "/sys/class/nvme/" + nvmeNamespacePattern = "nvme[0-9]+n[0-9]+" +) + +// GetNvmeInitiator gets the NVMe host NQN +func GetNvmeInitiator() (string, error) { + // Read from /etc/nvme/hostnqn or generate one + hostnqnPath := "/etc/nvme/hostnqn" + hostnqn, err := util.FileReadFirstLine(hostnqnPath) + if err != nil { + log.Debugf("Could not read hostnqn from %s, generating one", hostnqnPath) + // Generate hostnqn using nvme gen-hostnqn + args := []string{"gen-hostnqn"} + hostnqn, _, err = util.ExecCommandOutput(nvmecmd, args) + if err != nil { + return "", err + } + hostnqn = strings.TrimSpace(hostnqn) + } + return hostnqn, nil +} + +// ApplyNvmeTcpTuning applies recommended sysctl and module settings for NVMe over TCP +func ApplyNvmeTcpTuning() error { + var tuningErrors []string + + // Example: Increase network buffer sizes for high throughput + if err := setSysctl("net.core.rmem_max", "16777216"); err != nil { + tuningErrors = append(tuningErrors, err.Error()) + } + if err := setSysctl("net.core.wmem_max", "16777216"); err != nil { + tuningErrors = append(tuningErrors, err.Error()) + } + + // Example: Set NVMe core parameters (if needed) + if err := setKernelParam("/sys/module/nvme_core/parameters/multipath", "Y"); err != nil { + tuningErrors = append(tuningErrors, err.Error()) + } + + // Add more NVMe/TCP-specific tuning as needed... + + if len(tuningErrors) > 0 { + return fmt.Errorf("NVMe TCP tuning errors: %s", strings.Join(tuningErrors, "; ")) + } + return nil +} + +func setSysctl(key, value string) error { + cmd := fmt.Sprintf("sysctl -w %s=%s", key, value) + out, _, err := util.ExecCommandOutput("sh", []string{"-c", cmd}) + if err != nil { + return fmt.Errorf("failed to set %s: %v (%s)", key, err, out) + } + return nil +} + +func setKernelParam(path, value string) error { + f, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + return fmt.Errorf("failed to open %s: %v", path, err) + } + defer f.Close() + if _, err := f.WriteString(value); err != nil { + return fmt.Errorf("failed to write %s to %s: %v", value, path, err) + } + return nil +} + +// ConnectNvmeTarget connects to an NVMe over TCP target +func ConnectNvmeTarget(target *model.NvmeTarget) error { + + args := []string{ + "connect", + "-t", "tcp", + "-n", target.NQN, + "-a", target.Address, + "-s", target.Port, + } + + _, _, err := util.ExecCommandOutput(nvmecmd, args) + return err +} + +// DisconnectNvmeTarget disconnects from an NVMe target +func DisconnectNvmeTarget(target *model.NvmeTarget) error { + args := []string{ + "disconnect", + "-n", target.NQN, + } + + _, _, err := util.ExecCommandOutput(nvmecmd, args) + return err +} + +// RescanNvme performs NVMe namespace rescan +func RescanNvme() error { + // NVMe typically doesn't require explicit rescanning like SCSI + // The kernel automatically detects new namespaces + return nil +} +// HandleNvmeTcpDiscovery performs NVMe/TCP connection and device verification for a volume. +func HandleNvmeTcpDiscovery(volume *model.Volume) error { + log.Tracef(">>>>> HandleNvmeTcpDiscovery for volume %s", volume.SerialNumber) + defer log.Trace("<<<<< HandleNvmeTcpDiscovery") + + // 1. Apply NVMe/TCP tuning recommendations + if err := ApplyNvmeTcpTuning(); err != nil { + log.Warnf("Failed to apply NVMe TCP tuning: %v", err) + // Continue even if tuning fails + } + + // 2. Prepare NVMe target info + target := &model.NvmeTarget{ + NQN: volume.Nqn, + Address: volume.TargetAddress, + Port: volume.TargetPort, + } + + // 3. Connect to NVMe target + if err := ConnectNvmeTarget(target); err != nil { + return fmt.Errorf("failed to connect to NVMe target: %v", err) + } + + // 4. Optionally, verify device presence (wait for /dev/nvmeXnY) + found := false + for i := 0; i < 10; i++ { + devices, _ := FindNvmeDevices(volume.SerialNumber) + if len(devices) > 0 { + found = true + break + } + time.Sleep(1 * time.Second) + } + if !found { + return fmt.Errorf("NVMe device for serial %s not found after connect", volume.SerialNumber) + } + + return nil +} + +// FindNvmeDevices searches for NVMe devices matching the given serial number +func FindNvmeDevices(serialNumber string) ([]string, error) { + var devices []string + + // Scan /dev for nvme devices + files, err := ioutil.ReadDir("/dev") + if err != nil { + return nil, err + } + + nvmeRegex := regexp.MustCompile(`^nvme\d+n\d+$`) + for _, f := range files { + if nvmeRegex.MatchString(f.Name()) { + devicePath := filepath.Join("/dev", f.Name()) + + // Check serial number via sysfs + sysfsSerialPath := fmt.Sprintf("/sys/class/block/%s/serial", f.Name()) + if serial, err := util.FileReadFirstLine(sysfsSerialPath); err == nil { + if strings.TrimSpace(serial) == serialNumber { + devices = append(devices, devicePath) + } + } + + // Also check if the device name itself matches (for namespace matching) + if f.Name() == serialNumber { + devices = append(devices, devicePath) + } + } + } + + return devices, nil +} \ No newline at end of file diff --git a/vendor/github.com/hpe-storage/common-host-libs/model/types.go b/vendor/github.com/hpe-storage/common-host-libs/model/types.go index f049ccdd..b0d62487 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/model/types.go +++ b/vendor/github.com/hpe-storage/common-host-libs/model/types.go @@ -145,6 +145,14 @@ type IscsiTarget struct { Scope string // GST or VST } +// NvmeTarget struct +type NvmeTarget struct { + NQN string + Address string + Port string // 4420 (default NVMe over TCP port) + Scope string // GST or VST +} + // Device struct type Device struct { VolumeID string `json:"volume_id,omitempty"` @@ -159,6 +167,7 @@ type Device struct { Size int64 `json:"size,omitempty"` // size in MiB Slaves []string `json:"slaves,omitempty"` IscsiTargets []*IscsiTarget `json:"iscsi_target,omitempty"` + NvmeTargets []*NvmeTarget `json:"nvme_target,omitempty"` Hcils []string `json:"-"` // export it if needed TargetScope string `json:"target_scope,omitempty"` //GST="group", VST="volume" or empty(older array fiji etc), and no-op for FC State string `json:"state,omitempty"` // state of the device needed to verify the device is active @@ -190,6 +199,9 @@ type Volume struct { AccessProtocol string `json:"access_protocol,omitempty"` Iqn string `json:"iqn,omitempty"` // deprecated Iqns []string `json:"iqns,omitempty"` + Nqn string `json:"nqn,omitempty"` + TargetAddress string `json:"target_address,omitempty"` + TargetPort string `json:"target_port,omitempty"` DiscoveryIP string `json:"discovery_ip,omitempty"` // deprecated DiscoveryIPs []string `json:"discovery_ips,omitempty"` MountPoint string `json:"Mountpoint,omitempty"` @@ -201,6 +213,7 @@ type Volume struct { TargetScope string `json:"target_scope,omitempty"` //GST="group", VST="volume" or empty(older array fiji etc), and no-op for FC IscsiSessions []*IscsiSession `json:"iscsi_sessions,omitempty"` FcSessions []*FcSession `json:"fc_sessions,omitempty"` + NvmeSessions []*NvmeSession `json:"nvme_sessions,omitempty"` VolumeGroupId string `json:"volume_group_id"` SecondaryArrayDetails string `json:"secondary_array_details,omitempty"` UsedBytes int64 `json:"used_bytes,omitempty"` @@ -228,6 +241,11 @@ type IscsiSession struct { InitiatorNameLegacy string `json:"initiatorName,omitempty"` InitiatorIP string `json:"initiator_ip_addr,omitempty"` } +// NvmeSession info +type NvmeSession struct { + InitiatorNQN string `json:"initiator_nqn,omitempty"` + InitiatorIP string `json:"initiator_ip_addr,omitempty"` +} func (s FcSession) InitiatorWwpnStr() string { if s.InitiatorWwpnLegacy != "" { @@ -300,6 +318,7 @@ type BlockDeviceAccessInfo struct { LunID int32 `json:"lun_id,omitempty"` SecondaryBackendDetails IscsiAccessInfo + NvmetcpAccessInfo } // Information of LUN id, IQN, discovery IP's the secondary array @@ -312,6 +331,7 @@ type SecondaryLunInfo struct { LunID int32 `json:"lun_id,omitempty""` TargetNames []string `json:"target_names,omitempty"` IscsiAccessInfo + NvmetcpAccessInfo } // IscsiAccessInfo contains the fields necessary for iSCSI access @@ -321,6 +341,12 @@ type IscsiAccessInfo struct { ChapPassword string `json:"chap_password,omitempty"` } +// NvmetcpAccessInfo contains the fields necessary for NVMe/TCP access +type NvmetcpAccessInfo struct { + NvmeDiscoveryIPs []string `json:"nvme_discovery_ips,omitempty"` + TargetNames []string `json:"target_names,omitempty"` // NQN(s) + TargetPort string `json:"target_port,omitempty"` // e.g., 4420 +} // VirtualDeviceAccessInfo contains the required data to access a virtual device type VirtualDeviceAccessInfo struct { } @@ -398,6 +424,7 @@ type Node struct { Iqns []*string `json:"iqns,omitempty"` Networks []*string `json:"networks,omitempty"` Wwpns []*string `json:"wwpns,omitempty"` + Nqn *string `json:"nqn,omitempty"` ChapUser string `json:"chap_user,omitempty"` ChapPassword string `json:"chap_password,omitempty"` AccessProtocol string `json:"access_protocol,omitempty"` diff --git a/vendor/github.com/hpe-storage/common-host-libs/tunelinux/nvme.go b/vendor/github.com/hpe-storage/common-host-libs/tunelinux/nvme.go new file mode 100644 index 00000000..8a8e7bae --- /dev/null +++ b/vendor/github.com/hpe-storage/common-host-libs/tunelinux/nvme.go @@ -0,0 +1,2 @@ +package tunelinux + From c8a69070bb481bf36e38015d81f60b91b7241bf8 Mon Sep 17 00:00:00 2001 From: jlothe Date: Wed, 5 Nov 2025 10:22:39 +0530 Subject: [PATCH 2/9] CON-3876 Updated the nvme discoveryIPs to keep it common for both iscsi and nvme Signed-off-by: jlothe --- vendor/github.com/hpe-storage/common-host-libs/model/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/github.com/hpe-storage/common-host-libs/model/types.go b/vendor/github.com/hpe-storage/common-host-libs/model/types.go index b0d62487..b0d4e913 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/model/types.go +++ b/vendor/github.com/hpe-storage/common-host-libs/model/types.go @@ -343,7 +343,7 @@ type IscsiAccessInfo struct { // NvmetcpAccessInfo contains the fields necessary for NVMe/TCP access type NvmetcpAccessInfo struct { - NvmeDiscoveryIPs []string `json:"nvme_discovery_ips,omitempty"` + DiscoveryIPs []string `json:"discovery_ips,omitempty"` TargetNames []string `json:"target_names,omitempty"` // NQN(s) TargetPort string `json:"target_port,omitempty"` // e.g., 4420 } From ad7a318a709e6715369ef851c9c177c085d030d6 Mon Sep 17 00:00:00 2001 From: jlothe Date: Wed, 5 Nov 2025 10:47:12 +0530 Subject: [PATCH 3/9] CON 3876 - Fixed build issue Signed-off-by: jlothe --- vendor/github.com/hpe-storage/common-host-libs/util/volume.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vendor/github.com/hpe-storage/common-host-libs/util/volume.go b/vendor/github.com/hpe-storage/common-host-libs/util/volume.go index eb6ab8fc..f8a57616 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/util/volume.go +++ b/vendor/github.com/hpe-storage/common-host-libs/util/volume.go @@ -4,6 +4,7 @@ package util import ( "encoding/json" + "github.com/hpe-storage/common-host-libs/logger" "github.com/hpe-storage/common-host-libs/model" ) @@ -70,7 +71,7 @@ func GetSecondaryArrayDiscoveryIps(details string) []string { numberOfSecondaryBackends := len(secondaryArrayDetails.PeerArrayDetails) var secondaryDiscoverIps []string for i := 0; i < numberOfSecondaryBackends; i++ { - for _, discoveryIpRetrieved := range secondaryArrayDetails.PeerArrayDetails[i].DiscoveryIPs { + for _, discoveryIpRetrieved := range secondaryArrayDetails.PeerArrayDetails[i].IscsiAccessInfo.DiscoveryIPs { secondaryDiscoverIps = append(secondaryDiscoverIps, discoveryIpRetrieved) } } From c25a979cae36598de0433e80e406bb2607adf546 Mon Sep 17 00:00:00 2001 From: jlothe Date: Wed, 5 Nov 2025 13:46:38 +0530 Subject: [PATCH 4/9] CON-3876 Added changes to update nodeinfo for nvme discovery details Signed-off-by: jlothe --- pkg/driver/controller_server.go | 4 ++-- pkg/driver/node_server.go | 13 +++++++---- pkg/flavor/kubernetes/flavor.go | 23 +++++++++++++++++-- .../common-host-libs/model/types.go | 4 ++-- .../pkg/apis/hpestorage/v1/types.go | 1 + 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/pkg/driver/controller_server.go b/pkg/driver/controller_server.go index 1358d005..a4750017 100644 --- a/pkg/driver/controller_server.go +++ b/pkg/driver/controller_server.go @@ -968,8 +968,8 @@ func (driver *Driver) controllerPublishVolume( if len(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames) > 0 { publishContext[targetNamesKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames[0] } - if len(publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.NvmeDiscoveryIPs) > 0 { - publishContext[discoveryIPsKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.NvmeDiscoveryIPs[0] + if len(publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.DiscoveryIPs) > 0 { + publishContext[discoveryIPsKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.DiscoveryIPs[0] } if publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.TargetPort != "" { publishContext[targetPortKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.TargetPort diff --git a/pkg/driver/node_server.go b/pkg/driver/node_server.go index e9104cbc..f262768e 100644 --- a/pkg/driver/node_server.go +++ b/pkg/driver/node_server.go @@ -2254,9 +2254,14 @@ func (driver *Driver) nodeGetInfo() (string, error) { } nvmeNqn, err := GetNvmeInitiator() - if err == nil && nvmeNqn != "" { - nqn = &nvmeNqn - } + if err == nil && nvmeNqn != "" { + nqn = &nvmeNqn + } + + var nqns []*string + if nqn != nil { + nqns = append(nqns, nqn) + } var cidrNetworks []*string for _, network := range networks { @@ -2273,7 +2278,7 @@ func (driver *Driver) nodeGetInfo() (string, error) { Iqns: iqns, Networks: cidrNetworks, Wwpns: wwpns, - Nqn: nqn, + Nqns: nqns, } nodeID, err := driver.flavor.LoadNodeInfo(node) diff --git a/pkg/flavor/kubernetes/flavor.go b/pkg/flavor/kubernetes/flavor.go index 5e6dafe0..b9007203 100644 --- a/pkg/flavor/kubernetes/flavor.go +++ b/pkg/flavor/kubernetes/flavor.go @@ -239,13 +239,18 @@ func (flavor *Flavor) LoadNodeInfo(node *model.Node) (string, error) { nodeInfo.Spec.WWPNs = wwpnsFromNode updateNodeRequired = true } + nqnsFromNode := getNqnsFromNode(node) + if !reflect.DeepEqual(nodeInfo.Spec.NQNs, nqnsFromNode) { + nodeInfo.Spec.NQNs = nqnsFromNode + updateNodeRequired = true + } if !updateNodeRequired { // no update needed to existing CRD return node.UUID, nil } - log.Infof("updating Node %s with iqns %v wwpns %v networks %v", - nodeInfo.Name, nodeInfo.Spec.IQNs, nodeInfo.Spec.WWPNs, nodeInfo.Spec.Networks) + log.Infof("updating Node %s with iqns %v wwpns %v networks %v nqns %v", + nodeInfo.Name, nodeInfo.Spec.IQNs, nodeInfo.Spec.WWPNs, nodeInfo.Spec.Networks, nodeInfo.Spec.NQNs) _, err := flavor.crdClient.StorageV1().HPENodeInfos().Update(nodeInfo) if err != nil { log.Errorf("Error updating the node %s - %s\n", nodeInfo.Name, err.Error()) @@ -262,6 +267,7 @@ func (flavor *Flavor) LoadNodeInfo(node *model.Node) (string, error) { IQNs: getIqnsFromNode(node), Networks: getNetworksFromNode(node), WWPNs: getWwpnsFromNode(node), + NQNs: getNqnsFromNode(node), }, } @@ -303,6 +309,14 @@ func getNetworksFromNode(node *model.Node) []string { return networks } +func getNqnsFromNode(node *model.Node) []string { + var nqns []string + for i := 0; i < len(node.Nqns); i++ { + nqns = append(nqns, *node.Nqns[i]) + } + return nqns +} + // UnloadNodeInfo remove the HPENodeInfo from the list of CRDs func (flavor *Flavor) UnloadNodeInfo() { log.Tracef(">>>>>> UnloadNodeInfo with name %s", flavor.nodeName) @@ -353,12 +367,17 @@ func (flavor *Flavor) GetNodeInfo(nodeID string) (*model.Node, error) { for i := range wwpns { wwpns[i] = &nodeInfo.Spec.WWPNs[i] } + nqns := make([]*string, len(nodeInfo.Spec.NQNs)) + for i := range nqns { + nqns[i] = &nodeInfo.Spec.NQNs[i] + } node := &model.Node{ Name: nodeInfo.ObjectMeta.Name, UUID: nodeInfo.Spec.UUID, Iqns: iqns, Networks: networks, Wwpns: wwpns, + Nqns: nqns, } return node, nil diff --git a/vendor/github.com/hpe-storage/common-host-libs/model/types.go b/vendor/github.com/hpe-storage/common-host-libs/model/types.go index b0d4e913..d79a82b4 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/model/types.go +++ b/vendor/github.com/hpe-storage/common-host-libs/model/types.go @@ -328,7 +328,7 @@ type SecondaryBackendDetails struct { // Information of the each secondary array type SecondaryLunInfo struct { - LunID int32 `json:"lun_id,omitempty""` + LunID int32 `json:"lun_id,omitempty"` TargetNames []string `json:"target_names,omitempty"` IscsiAccessInfo NvmetcpAccessInfo @@ -424,7 +424,7 @@ type Node struct { Iqns []*string `json:"iqns,omitempty"` Networks []*string `json:"networks,omitempty"` Wwpns []*string `json:"wwpns,omitempty"` - Nqn *string `json:"nqn,omitempty"` + Nqns []*string `json:"nqns,omitempty"` ChapUser string `json:"chap_user,omitempty"` ChapPassword string `json:"chap_password,omitempty"` AccessProtocol string `json:"access_protocol,omitempty"` diff --git a/vendor/github.com/hpe-storage/k8s-custom-resources/pkg/apis/hpestorage/v1/types.go b/vendor/github.com/hpe-storage/k8s-custom-resources/pkg/apis/hpestorage/v1/types.go index 98112a44..4d0e0bc0 100644 --- a/vendor/github.com/hpe-storage/k8s-custom-resources/pkg/apis/hpestorage/v1/types.go +++ b/vendor/github.com/hpe-storage/k8s-custom-resources/pkg/apis/hpestorage/v1/types.go @@ -32,6 +32,7 @@ type HPENodeInfoSpec struct { IQNs []string `json:"iqns,omitempty"` Networks []string `json:"networks,omitempty"` WWPNs []string `json:"wwpns,omitempty"` + NQNs []string `json:"nqns,omitempty"` ChapUser string `json:"chapUser,omitempty"` ChapPassword string `json:"chapPassword,omitempty"` } From e4894f06d8a7249756e2475ee4676122db473818 Mon Sep 17 00:00:00 2001 From: jlothe Date: Wed, 5 Nov 2025 17:11:21 +0530 Subject: [PATCH 5/9] CON - 3876 Fixed issue for discovery IP for nvme Signed-off-by: jlothe --- pkg/driver/node_server.go | 2 +- .../github.com/hpe-storage/common-host-libs/linux/device.go | 2 +- .../github.com/hpe-storage/common-host-libs/linux/iscsi.go | 2 +- .../github.com/hpe-storage/common-host-libs/linux/nvme.go | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/driver/node_server.go b/pkg/driver/node_server.go index f262768e..0d1a1de1 100644 --- a/pkg/driver/node_server.go +++ b/pkg/driver/node_server.go @@ -2181,7 +2181,7 @@ func (driver *Driver) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoReque watcher, _ := util.InitializeWatcher(getNodeInfoFunc) // Add list of files /and directories to watch. The list contains // iSCSI , FC and CHAP Info and Networking config directories - list := []string{"/etc/sysconfig/network-scripts/", "/etc/sysconfig/network/", "/etc/iscsi/initiatorname.iscsi", "/etc/networks", "/etc/iscsi/iscsid.conf"} + list := []string{"/etc/sysconfig/network-scripts/", "/etc/sysconfig/network/", "/etc/iscsi/initiatorname.iscsi", "/etc/networks", "/etc/iscsi/iscsid.conf", "/etc/nvme/hostnqn"} watcher.AddWatchList(list) // Start event the watcher in a separate thread. go watcher.StartWatcher() diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/device.go b/vendor/github.com/hpe-storage/common-host-libs/linux/device.go index 3c0eab25..4cddcf4c 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/device.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/device.go @@ -426,7 +426,7 @@ func rescanLoginVolume(volume *model.Volume) error { secondaryVolObj.LunID = strconv.Itoa(int(secondaryLunInfo.LunID)) secondaryVolObj.Iqns = secondaryLunInfo.TargetNames secondaryVolObj.TargetScope = volume.TargetScope - secondaryVolObj.DiscoveryIPs = secondaryLunInfo.DiscoveryIPs + secondaryVolObj.DiscoveryIPs = secondaryLunInfo.IscsiAccessInfo.DiscoveryIPs secondaryVolObj.Chap = volume.Chap secondaryVolObj.ConnectionMode = volume.ConnectionMode secondaryVolObj.SerialNumber = volume.SerialNumber diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/iscsi.go b/vendor/github.com/hpe-storage/common-host-libs/linux/iscsi.go index d79f1da6..69506bc5 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/iscsi.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/iscsi.go @@ -251,7 +251,7 @@ func HandleIscsiDiscovery(volume *model.Volume) (err error) { secondaryVolObj.LunID = strconv.Itoa(int(secondaryLunInfo.LunID)) secondaryVolObj.Iqns = secondaryLunInfo.TargetNames secondaryVolObj.TargetScope = volume.TargetScope - secondaryVolObj.DiscoveryIPs = secondaryLunInfo.DiscoveryIPs + secondaryVolObj.DiscoveryIPs = secondaryLunInfo.IscsiAccessInfo.DiscoveryIPs secondaryVolObj.Chap = volume.Chap secondaryVolObj.ConnectionMode = volume.ConnectionMode secondaryVolObj.SerialNumber = volume.SerialNumber diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go b/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go index 2ad1a044..87266abe 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go @@ -25,15 +25,15 @@ const ( defaultNvmePort = "4420" nvmeHostPathFormat = "/sys/class/nvme/" nvmeNamespacePattern = "nvme[0-9]+n[0-9]+" + nvmeHostPath = "/etc/nvme/hostnqn" ) // GetNvmeInitiator gets the NVMe host NQN func GetNvmeInitiator() (string, error) { // Read from /etc/nvme/hostnqn or generate one - hostnqnPath := "/etc/nvme/hostnqn" - hostnqn, err := util.FileReadFirstLine(hostnqnPath) + hostnqn, err := util.FileReadFirstLine(nvmeHostPath) if err != nil { - log.Debugf("Could not read hostnqn from %s, generating one", hostnqnPath) + log.Debugf("Could not read hostnqn from %s, generating one", nvmeHostPath) // Generate hostnqn using nvme gen-hostnqn args := []string{"gen-hostnqn"} hostnqn, _, err = util.ExecCommandOutput(nvmecmd, args) From 26f2f350a929f65a7be0807627ae7221f9b15a11 Mon Sep 17 00:00:00 2001 From: jlothe Date: Thu, 6 Nov 2025 09:46:39 +0530 Subject: [PATCH 6/9] CON-3876 Added node sh changes for nvme Signed-off-by: jlothe --- cmd/csi-driver/conform/hpe-storage-node.sh | 47 ++++++++++++++++++- .../common-host-libs/linux/initiator.go | 30 +++++++++++- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/cmd/csi-driver/conform/hpe-storage-node.sh b/cmd/csi-driver/conform/hpe-storage-node.sh index 833d2e52..8546ae75 100644 --- a/cmd/csi-driver/conform/hpe-storage-node.sh +++ b/cmd/csi-driver/conform/hpe-storage-node.sh @@ -74,6 +74,12 @@ if [ "$CONFORM_TO" = "ubuntu" ]; then apt-get -qq install -y sg3-utils exit_on_error $? fi + # Install NVMe CLI tools + if [ ! -f /usr/sbin/nvme ]; then + apt-get -qq update + apt-get -qq install -y nvme-cli + exit_on_error $? + fi elif [ "$CONFORM_TO" = "redhat" ]; then # Install device-mapper-multipath @@ -101,6 +107,12 @@ elif [ "$CONFORM_TO" = "redhat" ]; then exit_on_error $? fi + # Install NVMe CLI tools + if [ ! -f /usr/sbin/nvme ]; then + yum -y install nvme-cli + exit_on_error $? + fi + elif [ "$CONFORM_TO" = "sles" ]; then # Install device-mapper-multipath if [ ! -f /sbin/multipathd ]; then @@ -128,13 +140,19 @@ elif [ "$CONFORM_TO" = "sles" ]; then exit_on_error $? fi + # Install NVMe CLI tools + if [ ! -f /usr/sbin/nvme ]; then + zypper -n install nvme-cli + exit_on_error $? + fi + elif [ "$CONFORM_TO" = "slem" ]; then # SLE Micro echo -n "Ensuring critical binaries are in the SLEM image: " - for bin in /usr/bin/sg_inq /sbin/mount.nfs4 /sbin/iscsid /sbin/multipathd; do + for bin in /usr/bin/sg_inq /sbin/mount.nfs4 /sbin/iscsid /sbin/multipathd /usr/sbin/nvme; do echo -n "$bin " if [ ! -f $bin ]; then - echo "$bin is missing. Run 'transactional-update -n pkg install multipath-tools open-iscsi nfs-client sg3_utils' on the worker nodes and reboot." + echo "$bin is missing. Run 'transactional-update -n pkg install multipath-tools open-iscsi nfs-client sg3_utils nvme-cli' on the worker nodes and reboot." exit_on_error 1 fi done @@ -147,6 +165,12 @@ elif [ "$CONFORM_TO" = "coreos" ]; then echo "Generating first-boot IQN" echo "InitiatorName=$(iscsi-iname)" > /etc/iscsi/initiatorname.iscsi fi + # Generate NVMe host NQN if it doesn't exist + if ! [[ -e /etc/nvme/hostnqn ]]; then + echo "Generating first-boot Host NQN" + mkdir -p /etc/nvme + echo "$(nvme gen-hostnqn)" > /etc/nvme/hostnqn + fi else echo "unsupported configuration for node package checks. os $os_name" exit 1 @@ -161,6 +185,25 @@ fi # Load iscsi_tcp modules, its a no-op if its already loaded modprobe iscsi_tcp +# Load NVMe-oTCP modules +modprobe nvme-core +modprobe nvme-tcp + + +# Generate NVMe host NQN if it doesn't exist (for non-CoreOS systems) +if [ "$CONFORM_TO" != "coreos" ]; then + if ! [[ -e /etc/nvme/hostnqn ]]; then + echo "Generating Host NQN" + mkdir -p /etc/nvme + nvme gen-hostnqn > /etc/nvme/hostnqn + exit_on_error $? + fi +fi + +# Ensure hostnqn file has proper permissions +if [[ -e /etc/nvme/hostnqn ]]; then + chmod 644 /etc/nvme/hostnqn +fi # Don't let udev automatically scan targets(all luns) on Unit Attention. # This will prevent udev scanning devices which we are attempting to remove if [ -f /lib/udev/rules.d/90-scsi-ua.rules ]; then diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go b/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go index 1858b32c..d1cf3fb9 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go @@ -15,6 +15,9 @@ var ( initiatorNamePattern = "^InitiatorName=(?P.*)$" iscsi = "iscsi" fc = "fc" + nvmeHostnqnPath = "/etc/nvme/hostnqn" // Add this + nvmeHostnqnPattern = "^(?Pnqn\\..*)$" // Add this + nvmeotcp = "nvmeotcp" ) //GetInitiators : get the host initiators @@ -105,7 +108,7 @@ func getFcInitiators() (fcInit *model.Initiator, err error) { return fcInit, nil } -func getNvmeInitiators() (init *model.Initiator, err error) { +/*func getNvmeInitiators() (init *model.Initiator, err error) { log.Trace(">>>>> getNvmeInitiators") defer log.Trace("<<<<< getNvmeInitiators") @@ -118,4 +121,29 @@ func getNvmeInitiators() (init *model.Initiator, err error) { initiators := []string{hostnqn} init = &model.Initiator{Type: "nvmeotcp", Init: initiators} return init, nil +}*/ +func getNvmeInitiators() (init *model.Initiator, err error) { + log.Trace(">>>>> getNvmeInitiators") + defer log.Trace("<<<<< getNvmeInitiators") + + exists, _, err := util.FileExists(nvmeHostnqnPath) + if !exists { + log.Debugf("%s not found, assuming not an NVMe host", nvmeHostnqnPath) + return nil, nil + } + + initiators, err := util.FileGetStringsWithPattern(nvmeHostnqnPath, nvmeHostnqnPattern) + if err != nil { + log.Errorf("failed to get nqn from %s error %s", nvmeHostnqnPath, err.Error()) + return nil, err + } + + if len(initiators) == 0 { + log.Errorf("empty nqn found from %s", nvmeHostnqnPath) + return nil, errors.New("empty nqn found") + } + + log.Debugf("got nvme initiator name as %s", initiators[0]) + init = &model.Initiator{Type: nvmeotcp, Init: initiators} + return init, nil } From 51887ecfe1c4d5daeedbbc56dad163f354f67425 Mon Sep 17 00:00:00 2001 From: jlothe Date: Thu, 6 Nov 2025 17:02:58 +0530 Subject: [PATCH 7/9] CON-3876- Added changes in entry so loadNodeinfo will read the nqn from etc Signed-off-by: jlothe --- cmd/csi-driver/conform/entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/csi-driver/conform/entrypoint.sh b/cmd/csi-driver/conform/entrypoint.sh index b534117d..64bdcb51 100644 --- a/cmd/csi-driver/conform/entrypoint.sh +++ b/cmd/csi-driver/conform/entrypoint.sh @@ -28,6 +28,7 @@ if [ "$nodeService" = true ] || [ "$nodeInit" = true ]; then ln -s /host/etc/multipath.conf /etc/multipath.conf ln -s /host/etc/multipath /etc/multipath ln -s /host/etc/iscsi /etc/iscsi + ln -s /host/etc/nvme /etc/nvme # symlink to host os release files for parsing if [ -f /host/etc/redhat-release ]; then From c6c03962f03a13a44328d6891c433f55550500db Mon Sep 17 00:00:00 2001 From: jlothe Date: Fri, 7 Nov 2025 15:00:20 +0530 Subject: [PATCH 8/9] CON-3876 Fixed wwpns population in node info Signed-off-by: jlothe --- pkg/driver/constants.go | 1 + pkg/driver/node_server.go | 17 +++++------------ pkg/flavor/kubernetes/flavor.go | 2 +- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/pkg/driver/constants.go b/pkg/driver/constants.go index 52b3de41..5380cc35 100644 --- a/pkg/driver/constants.go +++ b/pkg/driver/constants.go @@ -6,6 +6,7 @@ const ( // Protocol types iscsi = "iscsi" fc = "fc" + nvmeotcp = "nvmeotcp" // defaultFileSystem is the implemenation-specific default value defaultFileSystem = "xfs" diff --git a/pkg/driver/node_server.go b/pkg/driver/node_server.go index 0d1a1de1..abc249f2 100644 --- a/pkg/driver/node_server.go +++ b/pkg/driver/node_server.go @@ -2240,29 +2240,22 @@ func (driver *Driver) nodeGetInfo() (string, error) { var iqns []*string var wwpns []*string - var nqn *string + var nqns []*string for _, initiator := range initiators { if initiator.Type == iscsi { for i := 0; i < len(initiator.Init); i++ { iqns = append(iqns, &initiator.Init[i]) } + } else if initiator.Type == nvmeotcp { + for i := 0; i < len(initiator.Init); i++ { + nqns = append(nqns, &initiator.Init[i]) + } } else { for i := 0; i < len(initiator.Init); i++ { wwpns = append(wwpns, &initiator.Init[i]) } } } - - nvmeNqn, err := GetNvmeInitiator() - if err == nil && nvmeNqn != "" { - nqn = &nvmeNqn - } - - var nqns []*string - if nqn != nil { - nqns = append(nqns, nqn) - } - var cidrNetworks []*string for _, network := range networks { log.Infof("Processing network named %s with IpV4 CIDR %s", network.Name, network.CidrNetwork) diff --git a/pkg/flavor/kubernetes/flavor.go b/pkg/flavor/kubernetes/flavor.go index b9007203..bf854e9c 100644 --- a/pkg/flavor/kubernetes/flavor.go +++ b/pkg/flavor/kubernetes/flavor.go @@ -379,7 +379,7 @@ func (flavor *Flavor) GetNodeInfo(nodeID string) (*model.Node, error) { Wwpns: wwpns, Nqns: nqns, } - + log.Tracef("HPE Node Info sent to CSP: %v", node) return node, nil } } From 9194ea6bb3321594827348a6bb458c1101eebe44 Mon Sep 17 00:00:00 2001 From: jlothe Date: Fri, 14 Nov 2025 17:43:06 +0530 Subject: [PATCH 9/9] CON-3876 & 3882 Fixed discovery IP, device creation and filesystem issues Signed-off-by: jlothe --- Dockerfile | 3 +- pkg/driver/controller_server.go | 30 ++----- pkg/driver/node_server.go | 51 ++++------- .../common-host-libs/linux/device.go | 84 ++++++++++++++----- .../common-host-libs/linux/initiator.go | 4 +- .../common-host-libs/linux/nvme.go | 49 +++++++++-- .../common-host-libs/model/types.go | 2 +- 7 files changed, 133 insertions(+), 90 deletions(-) diff --git a/Dockerfile b/Dockerfile index b603cfee..ddb7e1fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,7 +76,8 @@ RUN ln -s /chroot/chroot-host-wrapper.sh /chroot/blkid \ && ln -s /chroot/chroot-host-wrapper.sh /chroot/ip \ && ln -s /chroot/chroot-host-wrapper.sh /chroot/dnsdomainname \ && ln -s /chroot/chroot-host-wrapper.sh /chroot/sg_inq \ - && ln -s /chroot/chroot-host-wrapper.sh /chroot/find + && ln -s /chroot/chroot-host-wrapper.sh /chroot/find \ + && ln -s /chroot/chroot-host-wrapper.sh /chroot/nvme ENV PATH="/chroot:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" diff --git a/pkg/driver/controller_server.go b/pkg/driver/controller_server.go index a4750017..05f8c0cd 100644 --- a/pkg/driver/controller_server.go +++ b/pkg/driver/controller_server.go @@ -960,31 +960,13 @@ func (driver *Driver) controllerPublishVolume( // TODO: add any additional info necessary to mount the device publishContext := map[string]string{} - // NVMe over TCP support - if strings.EqualFold(publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol, "nvmetcp") { - publishContext[serialNumberKey] = publishInfo.SerialNumber - publishContext[accessProtocolKey] = "nvmetcp" - // NQN, target address, and port for NVMe/TCP - if len(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames) > 0 { - publishContext[targetNamesKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames[0] - } - if len(publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.DiscoveryIPs) > 0 { - publishContext[discoveryIPsKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.DiscoveryIPs[0] - } - if publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.TargetPort != "" { - publishContext[targetPortKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.NvmetcpAccessInfo.TargetPort - } else { - publishContext[targetPortKey] = "4420" // default NVMe/TCP port - } - - }else{ + publishContext[serialNumberKey] = publishInfo.SerialNumber publishContext[accessProtocolKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol publishContext[targetNamesKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames, ",") publishContext[targetScopeKey] = requestedTargetScope publishContext[lunIDKey] = strconv.Itoa(int(publishInfo.AccessInfo.BlockDeviceAccessInfo.LunID)) - - } + publishContext[discoveryIPsKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.DiscoveryIPs, ",") // Start of population of target array details if publishInfo.AccessInfo.BlockDeviceAccessInfo.SecondaryBackendDetails.PeerArrayDetails != nil { @@ -1000,9 +982,15 @@ func (driver *Driver) controllerPublishVolume( } if strings.EqualFold(publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol, iscsi) { - publishContext[discoveryIPsKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.IscsiAccessInfo.DiscoveryIPs, ",") + publishContext[discoveryIPsKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.DiscoveryIPs, ",") } + if strings.EqualFold(publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol, nvmetcp) { + publishContext[targetPortKey] = "4420" // default NVMe/TCP port + publishContext[discoveryIPsKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.DiscoveryIPs, ",") + } + + if readOnlyAccessMode { publishContext[readOnlyKey] = strconv.FormatBool(readOnlyAccessMode) } else { // Default case, we stick to old behavior diff --git a/pkg/driver/node_server.go b/pkg/driver/node_server.go index 01563835..b15eb5a0 100644 --- a/pkg/driver/node_server.go +++ b/pkg/driver/node_server.go @@ -575,50 +575,27 @@ func (driver *Driver) setupDevice( log.Tracef(">>>>> setupDevice, volumeID: %s, publishContext: %v", volumeID, log.MapScrubber(publishContext)) defer log.Trace("<<<<< setupDevice") - // Handle NVMe over TCP volumes - if publishContext[accessProtocolKey] == "nvmetcp" { - volume := &model.Volume{ - SerialNumber: publishContext[serialNumberKey], - AccessProtocol: publishContext[accessProtocolKey], - Nqn: publishContext[targetNamesKey], // NQN for NVMe - TargetAddress: publishContext[discoveryIPsKey], // Target IP - TargetPort: publishContext[targetPortKey], // Target port (default 4420) - ConnectionMode: defaultConnectionMode, - } - - // Cleanup any stale device existing before stage - device, _ := driver.chapiDriver.GetDevice(volume) - if device != nil { - device.TargetScope = volume.TargetScope - err := driver.chapiDriver.DeleteDevice(device) - if err != nil { - log.Warnf("Failed to cleanup stale NVMe device %s before staging, err %s", device.AltFullPathName, err.Error()) - } - } - - // Create NVMe Device - devices, err := driver.chapiDriver.CreateDevices([]*model.Volume{volume}) - if err != nil { - log.Errorf("Failed to create NVMe device from publish info. Error: %s", err.Error()) - return nil, err - } - if len(devices) == 0 { - log.Errorf("Failed to get the NVMe device just created using the volume %+v", volume) - return nil, fmt.Errorf("unable to find the NVMe device for volume %+v", volume) - } - - return devices[0], nil - } - // TODO: Enhance CHAPI to work with a PublishInfo object rather than a volume discoveryIps := strings.Split(publishContext[discoveryIPsKey], ",") - iqns := strings.Split(publishContext[targetNamesKey], ",") - + + // For NVMe/TCP, set Nqn field in the model.Volume and update iqns accordingly + var nqn string + var iqns = []string{} + if publishContext[accessProtocolKey] == nvmetcp { + nqn = publishContext[targetNamesKey] + iqns = []string{} + }else if publishContext[accessProtocolKey] == iscsi{ + nqn = "" + iqns = strings.Split(publishContext[targetNamesKey], ",") + } volume := &model.Volume{ SerialNumber: publishContext[serialNumberKey], AccessProtocol: publishContext[accessProtocolKey], Iqns: iqns, + Nqn: nqn, + TargetAddress: publishContext[targetNamesKey], + TargetPort: publishContext[targetPortKey], TargetScope: publishContext[targetScopeKey], LunID: publishContext[lunIDKey], DiscoveryIPs: discoveryIps, diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/device.go b/vendor/github.com/hpe-storage/common-host-libs/linux/device.go index 4cddcf4c..55a2b539 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/device.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/device.go @@ -394,6 +394,7 @@ func setAltFullPathName(dev *model.Device) (err error) { // Helper function to perform rescan and detect the newly attached volume (FC/iSCSI volume) func rescanLoginVolume(volume *model.Volume) error { log.Traceln(">>>>> rescanLoginVolume", volume, "and accessProtocol", volume.AccessProtocol) + log.Traceln(">>>>> rescanLoginVolume Nqn", volume.Nqn, "and TargetAddress", volume.TargetAddress) defer log.Traceln("<<<< rescanLoginVolume") var err error var primaryVolObj *model.Volume @@ -408,6 +409,9 @@ func rescanLoginVolume(volume *model.Volume) error { primaryVolObj.ConnectionMode = volume.ConnectionMode primaryVolObj.SerialNumber = volume.SerialNumber primaryVolObj.Networks = volume.Networks + primaryVolObj.Nqn = volume.Nqn + primaryVolObj.TargetAddress = strings.Join(volume.DiscoveryIPs, ",") + primaryVolObj.TargetPort = volume.TargetPort err = rescanLoginVolumeForBackend(primaryVolObj) @@ -499,6 +503,30 @@ func isLuksDevice(devPath string) (bool, error) { func createLinuxDevice(volume *model.Volume) (dev *model.Device, err error) { log.Debugf(">>>> createLinuxDevice called with volume %s serialNumber %s and lunID %s", volume.Name, volume.SerialNumber, volume.LunID) defer log.Debug("<<<<< createLinuxDevice") + // Handle NVMe/TCP device creation + if strings.EqualFold(volume.AccessProtocol, "nvmetcp") { + log.Tracef("NVMe/TCP requested, NQN: %s, Serial: %s", volume.Nqn, volume.SerialNumber) + // Initiate NVMe/TCP discovery and connection + if err := rescanLoginVolume(volume); err != nil { + return nil, fmt.Errorf("NVMe/TCP discovery failed: %v", err) + } + // Allow time for device to register + time.Sleep(time.Second * 2) + // Attempt to locate NVMe device by NQN or Serial + nvmeDev, lookupErr := GetNvmeDeviceFromNamespace(volume.Nqn) + if lookupErr == nil && nvmeDev != nil { + log.Infof("NVMe/TCP device found: %+v", nvmeDev) + return nvmeDev, nil + } + // Fallback to SerialNumber if NQN lookup fails + nvmeDev, lookupErr = GetNvmeDeviceFromNamespace(volume.SerialNumber) + if lookupErr == nil && nvmeDev != nil { + log.Infof("NVMe/TCP device found by serial: %+v", nvmeDev) + return nvmeDev, nil + } + return nil, fmt.Errorf("unable to locate NVMe/TCP device for NQN %s or serial %s", volume.Nqn, volume.SerialNumber) + } + // Rescan and detect the newly attached volume err = rescanLoginVolume(volume) if err != nil { @@ -1097,32 +1125,35 @@ func GetDeviceFromVolume(vol *model.Volume) (*model.Device, error) { log.Tracef(">>>>> GetDeviceFromVolume for serial %s", vol.SerialNumber) defer log.Trace("<<<<< GetDeviceFromVolume") - devices, err := GetLinuxDmDevices(false, vol) - if err != nil { - return nil, err - } - - if len(devices) > 0 { - return devices[0], nil - } + // Try NVMe device lookup first + if strings.EqualFold(vol.AccessProtocol, "nvmetcp") { + nvmeDev, err := GetNvmeDeviceFromNamespace(vol.SerialNumber) + log.Tracef("NVMe device: %+v", nvmeDev) + if err == nil && nvmeDev != nil { + log.Debugf("Found NVMe device: %+v", nvmeDev) + return nvmeDev, nil + } + }else{ + devices, err := GetLinuxDmDevices(false, vol) + if err != nil { + return nil, err + } - // Try NVMe device lookup if not found above - nvmeDev, err := GetNvmeDeviceFromNamespace(vol.SerialNumber) - if err == nil && nvmeDev != nil { - log.Debugf("Found NVMe device: %+v", nvmeDev) - return nvmeDev, nil - } - if len(devices) == 0 { - return nil, fmt.Errorf("unable to find device matching volume serial number %s", vol.SerialNumber) + if len(devices) > 0 { + return devices[0], nil + } + if len(devices) == 0 { + return nil, fmt.Errorf("unable to find device matching volume serial number %s", vol.SerialNumber) + } + return devices[0], nil } - return devices[0], nil + return nil, fmt.Errorf("device not found for volume %s with access protocol %s", vol.Name, vol.AccessProtocol) } //CreateLinuxDevices : attached and creates linux devices to host func CreateLinuxDevices(vols []*model.Volume) (devs []*model.Device, err error) { log.Tracef(">>>>> CreateLinuxDevices") defer log.Trace("<<<<< CreateLinuxDevices") - var devices []*model.Device for _, vol := range vols { log.Tracef("create request with serialnumber :%s, accessprotocol %s discoveryip %s, iqn %s ", vol.SerialNumber, vol.AccessProtocol, vol.DiscoveryIP, vol.Iqn) @@ -1456,14 +1487,23 @@ func GetNvmeDeviceFromNamespace(serialOrNamespace string) (*model.Device, error) for _, f := range files { if nvmeRegex.MatchString(f.Name()) { // Optionally, check namespace/serial via sysfs - sysfsSerialPath := fmt.Sprintf("/sys/class/block/%s/serial", f.Name()) + sysfsSerialPath := fmt.Sprintf("/sys/class/block/%s/subsystem/%s/nguid", f.Name(), f.Name()) + log.Tracef("serial path=%s", sysfsSerialPath) serial, _ := util.FileReadFirstLine(sysfsSerialPath) - if serialOrNamespace == f.Name() || serialOrNamespace == serial { + log.Tracef("serial path=%s", sysfsSerialPath) + // Normalize the serial from sysfs by removing dashes and whitespace + normalizedSerial := strings.ReplaceAll(strings.TrimSpace(serial), "-", "") + log.Tracef("found serial number %s, normalized: %s", serial, normalizedSerial) + if serialOrNamespace == f.Name() || serialOrNamespace == normalizedSerial { + log.Tracef("serial number %s matched=%s", serialOrNamespace, normalizedSerial) return &model.Device{ Pathname: nvmeRoot + f.Name(), - SerialNumber: serial, + AltFullPathName: nvmeRoot + f.Name(), + SerialNumber: normalizedSerial, }, nil - } + }else{ + log.Tracef("serial number %s did not match=%s", serialOrNamespace, normalizedSerial) + } } } return nil, fmt.Errorf("NVMe device not found for %s", serialOrNamespace) diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go b/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go index d1cf3fb9..8d3ebac6 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go @@ -15,8 +15,8 @@ var ( initiatorNamePattern = "^InitiatorName=(?P.*)$" iscsi = "iscsi" fc = "fc" - nvmeHostnqnPath = "/etc/nvme/hostnqn" // Add this - nvmeHostnqnPattern = "^(?Pnqn\\..*)$" // Add this + nvmeHostnqnPath = "/etc/nvme/hostnqn" + nvmeHostnqnPattern = "^(?Pnqn\\..*)$" nvmeotcp = "nvmeotcp" ) diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go b/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go index 87266abe..bda7eb26 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go @@ -94,6 +94,14 @@ func setKernelParam(path, value string) error { // ConnectNvmeTarget connects to an NVMe over TCP target func ConnectNvmeTarget(target *model.NvmeTarget) error { + var discoveryIP = strings.Split(target.Address, ",") + if len(discoveryIP) == 0 { + return fmt.Errorf("no discovery IPs provided for NVMe target") + } + + // Use the first discovery IP for connection + target.Address = discoveryIP[0] + args := []string{ "connect", "-t", "tcp", @@ -102,8 +110,33 @@ func ConnectNvmeTarget(target *model.NvmeTarget) error { "-s", target.Port, } - _, _, err := util.ExecCommandOutput(nvmecmd, args) - return err + _, rc, err := util.ExecCommandOutput(nvmecmd, args) + if err != nil && rc != 114 { + log.Warnf("NVMe connect failed for discovery IP %s, rc=%d, error: %s", discoveryIP[0], rc, err) + //try discovery with another IP if multiple are provided + for i := 1; i < len(discoveryIP); i++ { + log.Warnf("NVMe connect failed, trying next discovery IP %s", discoveryIP[i]) + args := []string{ + "connect", + "-t", "tcp", + "-n", target.NQN, + "-a", discoveryIP[i], + "-s", target.Port, + } + _, rc, err = util.ExecCommandOutput(nvmecmd, args) + if err != nil && rc != 114 { + log.Warnf("NVMe connect failed for discovery IP %s, rc=%d, error: %s", discoveryIP[i], rc, err) + continue + }else{ + log.Infof("Successfully connected to NVMe target using discovery IP %s", discoveryIP[i]) + return nil + } + } + return fmt.Errorf("failed to connect to NVMe target: %v", err) + }else{ + log.Infof("Successfully connected to NVMe target using discovery IP %s", discoveryIP[0]) + return nil + } } // DisconnectNvmeTarget disconnects from an NVMe target @@ -137,7 +170,7 @@ func HandleNvmeTcpDiscovery(volume *model.Volume) error { // 2. Prepare NVMe target info target := &model.NvmeTarget{ NQN: volume.Nqn, - Address: volume.TargetAddress, + Address: strings.Join(volume.DiscoveryIPs, ","), Port: volume.TargetPort, } @@ -178,10 +211,14 @@ func FindNvmeDevices(serialNumber string) ([]string, error) { if nvmeRegex.MatchString(f.Name()) { devicePath := filepath.Join("/dev", f.Name()) - // Check serial number via sysfs - sysfsSerialPath := fmt.Sprintf("/sys/class/block/%s/serial", f.Name()) + // Check serial number via sysfs + sysfsSerialPath := fmt.Sprintf("/sys/class/block/%s/subsystem/%s/nguid", f.Name(), f.Name()) + log.Tracef("serial path=%s", sysfsSerialPath) if serial, err := util.FileReadFirstLine(sysfsSerialPath); err == nil { - if strings.TrimSpace(serial) == serialNumber { + // Normalize the serial from sysfs by removing dashes and whitespace + normalizedSerial := strings.ReplaceAll(strings.TrimSpace(serial), "-", "") + log.Tracef("found serial number %s, normalized: %s", serial, normalizedSerial) + if strings.TrimSpace(normalizedSerial) == serialNumber { devices = append(devices, devicePath) } } diff --git a/vendor/github.com/hpe-storage/common-host-libs/model/types.go b/vendor/github.com/hpe-storage/common-host-libs/model/types.go index 98f070ab..456b7f35 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/model/types.go +++ b/vendor/github.com/hpe-storage/common-host-libs/model/types.go @@ -315,6 +315,7 @@ type AccessInfo struct { type BlockDeviceAccessInfo struct { AccessProtocol string `json:"access_protocol,omitempty"` TargetNames []string `json:"target_names,omitempty"` + DiscoveryIPs []string `json:"discovery_ips,omitempty"` LunID int32 `json:"lun_id,omitempty"` SecondaryBackendDetails IscsiAccessInfo @@ -343,7 +344,6 @@ type IscsiAccessInfo struct { // NvmetcpAccessInfo contains the fields necessary for NVMe/TCP access type NvmetcpAccessInfo struct { - DiscoveryIPs []string `json:"discovery_ips,omitempty"` TargetNames []string `json:"target_names,omitempty"` // NQN(s) TargetPort string `json:"target_port,omitempty"` // e.g., 4420 }