diff --git a/Dockerfile b/Dockerfile index b603cfee..ddb7e1fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,7 +76,8 @@ RUN ln -s /chroot/chroot-host-wrapper.sh /chroot/blkid \ && ln -s /chroot/chroot-host-wrapper.sh /chroot/ip \ && ln -s /chroot/chroot-host-wrapper.sh /chroot/dnsdomainname \ && ln -s /chroot/chroot-host-wrapper.sh /chroot/sg_inq \ - && ln -s /chroot/chroot-host-wrapper.sh /chroot/find + && ln -s /chroot/chroot-host-wrapper.sh /chroot/find \ + && ln -s /chroot/chroot-host-wrapper.sh /chroot/nvme ENV PATH="/chroot:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" diff --git a/cmd/csi-driver/conform/entrypoint.sh b/cmd/csi-driver/conform/entrypoint.sh index b534117d..64bdcb51 100644 --- a/cmd/csi-driver/conform/entrypoint.sh +++ b/cmd/csi-driver/conform/entrypoint.sh @@ -28,6 +28,7 @@ if [ "$nodeService" = true ] || [ "$nodeInit" = true ]; then ln -s /host/etc/multipath.conf /etc/multipath.conf ln -s /host/etc/multipath /etc/multipath ln -s /host/etc/iscsi /etc/iscsi + ln -s /host/etc/nvme /etc/nvme # symlink to host os release files for parsing if [ -f /host/etc/redhat-release ]; then diff --git a/cmd/csi-driver/conform/hpe-storage-node.sh b/cmd/csi-driver/conform/hpe-storage-node.sh index 833d2e52..8546ae75 100644 --- a/cmd/csi-driver/conform/hpe-storage-node.sh +++ b/cmd/csi-driver/conform/hpe-storage-node.sh @@ -74,6 +74,12 @@ if [ "$CONFORM_TO" = "ubuntu" ]; then apt-get -qq install -y sg3-utils exit_on_error $? fi + # Install NVMe CLI tools + if [ ! -f /usr/sbin/nvme ]; then + apt-get -qq update + apt-get -qq install -y nvme-cli + exit_on_error $? + fi elif [ "$CONFORM_TO" = "redhat" ]; then # Install device-mapper-multipath @@ -101,6 +107,12 @@ elif [ "$CONFORM_TO" = "redhat" ]; then exit_on_error $? fi + # Install NVMe CLI tools + if [ ! -f /usr/sbin/nvme ]; then + yum -y install nvme-cli + exit_on_error $? + fi + elif [ "$CONFORM_TO" = "sles" ]; then # Install device-mapper-multipath if [ ! -f /sbin/multipathd ]; then @@ -128,13 +140,19 @@ elif [ "$CONFORM_TO" = "sles" ]; then exit_on_error $? fi + # Install NVMe CLI tools + if [ ! -f /usr/sbin/nvme ]; then + zypper -n install nvme-cli + exit_on_error $? + fi + elif [ "$CONFORM_TO" = "slem" ]; then # SLE Micro echo -n "Ensuring critical binaries are in the SLEM image: " - for bin in /usr/bin/sg_inq /sbin/mount.nfs4 /sbin/iscsid /sbin/multipathd; do + for bin in /usr/bin/sg_inq /sbin/mount.nfs4 /sbin/iscsid /sbin/multipathd /usr/sbin/nvme; do echo -n "$bin " if [ ! -f $bin ]; then - echo "$bin is missing. Run 'transactional-update -n pkg install multipath-tools open-iscsi nfs-client sg3_utils' on the worker nodes and reboot." + echo "$bin is missing. Run 'transactional-update -n pkg install multipath-tools open-iscsi nfs-client sg3_utils nvme-cli' on the worker nodes and reboot." exit_on_error 1 fi done @@ -147,6 +165,12 @@ elif [ "$CONFORM_TO" = "coreos" ]; then echo "Generating first-boot IQN" echo "InitiatorName=$(iscsi-iname)" > /etc/iscsi/initiatorname.iscsi fi + # Generate NVMe host NQN if it doesn't exist + if ! [[ -e /etc/nvme/hostnqn ]]; then + echo "Generating first-boot Host NQN" + mkdir -p /etc/nvme + echo "$(nvme gen-hostnqn)" > /etc/nvme/hostnqn + fi else echo "unsupported configuration for node package checks. os $os_name" exit 1 @@ -161,6 +185,25 @@ fi # Load iscsi_tcp modules, its a no-op if its already loaded modprobe iscsi_tcp +# Load NVMe-oTCP modules +modprobe nvme-core +modprobe nvme-tcp + + +# Generate NVMe host NQN if it doesn't exist (for non-CoreOS systems) +if [ "$CONFORM_TO" != "coreos" ]; then + if ! [[ -e /etc/nvme/hostnqn ]]; then + echo "Generating Host NQN" + mkdir -p /etc/nvme + nvme gen-hostnqn > /etc/nvme/hostnqn + exit_on_error $? + fi +fi + +# Ensure hostnqn file has proper permissions +if [[ -e /etc/nvme/hostnqn ]]; then + chmod 644 /etc/nvme/hostnqn +fi # Don't let udev automatically scan targets(all luns) on Unit Attention. # This will prevent udev scanning devices which we are attempting to remove if [ -f /lib/udev/rules.d/90-scsi-ua.rules ]; then diff --git a/pkg/driver/constants.go b/pkg/driver/constants.go index 52b3de41..5380cc35 100644 --- a/pkg/driver/constants.go +++ b/pkg/driver/constants.go @@ -6,6 +6,7 @@ const ( // Protocol types iscsi = "iscsi" fc = "fc" + nvmeotcp = "nvmeotcp" // defaultFileSystem is the implemenation-specific default value defaultFileSystem = "xfs" diff --git a/pkg/driver/controller_server.go b/pkg/driver/controller_server.go index d94ffb59..05f8c0cd 100644 --- a/pkg/driver/controller_server.go +++ b/pkg/driver/controller_server.go @@ -924,7 +924,9 @@ func (driver *Driver) controllerPublishVolume( requestedAccessProtocol = iscsi } else if requestedAccessProtocol == "fc" { requestedAccessProtocol = fc - } + } else if requestedAccessProtocol == "nvmetcp" { + requestedAccessProtocol = "nvmetcp" + } if existingNode != nil { log.Tracef("CSP has already been notified about the node with ID %s and UUID %s", existingNode.ID, existingNode.UUID) @@ -957,12 +959,15 @@ func (driver *Driver) controllerPublishVolume( // TODO: add any additional info necessary to mount the device publishContext := map[string]string{} - publishContext[serialNumberKey] = publishInfo.SerialNumber - publishContext[accessProtocolKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol - publishContext[targetNamesKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames, ",") - publishContext[targetScopeKey] = requestedTargetScope - publishContext[lunIDKey] = strconv.Itoa(int(publishInfo.AccessInfo.BlockDeviceAccessInfo.LunID)) - + + + publishContext[serialNumberKey] = publishInfo.SerialNumber + publishContext[accessProtocolKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol + publishContext[targetNamesKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames, ",") + publishContext[targetScopeKey] = requestedTargetScope + publishContext[lunIDKey] = strconv.Itoa(int(publishInfo.AccessInfo.BlockDeviceAccessInfo.LunID)) + publishContext[discoveryIPsKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.DiscoveryIPs, ",") + // Start of population of target array details if publishInfo.AccessInfo.BlockDeviceAccessInfo.SecondaryBackendDetails.PeerArrayDetails != nil { secondaryArrayMarshalledStr, err := json.Marshal(&publishInfo.AccessInfo.BlockDeviceAccessInfo.SecondaryBackendDetails) @@ -977,9 +982,15 @@ func (driver *Driver) controllerPublishVolume( } if strings.EqualFold(publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol, iscsi) { - publishContext[discoveryIPsKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.IscsiAccessInfo.DiscoveryIPs, ",") + publishContext[discoveryIPsKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.DiscoveryIPs, ",") } + if strings.EqualFold(publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol, nvmetcp) { + publishContext[targetPortKey] = "4420" // default NVMe/TCP port + publishContext[discoveryIPsKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.DiscoveryIPs, ",") + } + + if readOnlyAccessMode { publishContext[readOnlyKey] = strconv.FormatBool(readOnlyAccessMode) } else { // Default case, we stick to old behavior diff --git a/pkg/driver/node_server.go b/pkg/driver/node_server.go index 1ca8876c..b15eb5a0 100644 --- a/pkg/driver/node_server.go +++ b/pkg/driver/node_server.go @@ -42,6 +42,8 @@ var ( const ( fileHostIPKey = "hostIP" + nvmetcp = "nvmetcp" + targetPortKey = "targetPort" ) var isWatcherEnabled = false @@ -576,12 +578,24 @@ func (driver *Driver) setupDevice( // TODO: Enhance CHAPI to work with a PublishInfo object rather than a volume discoveryIps := strings.Split(publishContext[discoveryIPsKey], ",") - iqns := strings.Split(publishContext[targetNamesKey], ",") - + + // For NVMe/TCP, set Nqn field in the model.Volume and update iqns accordingly + var nqn string + var iqns = []string{} + if publishContext[accessProtocolKey] == nvmetcp { + nqn = publishContext[targetNamesKey] + iqns = []string{} + }else if publishContext[accessProtocolKey] == iscsi{ + nqn = "" + iqns = strings.Split(publishContext[targetNamesKey], ",") + } volume := &model.Volume{ SerialNumber: publishContext[serialNumberKey], AccessProtocol: publishContext[accessProtocolKey], Iqns: iqns, + Nqn: nqn, + TargetAddress: publishContext[targetNamesKey], + TargetPort: publishContext[targetPortKey], TargetScope: publishContext[targetScopeKey], LunID: publishContext[lunIDKey], DiscoveryIPs: discoveryIps, @@ -2163,7 +2177,7 @@ func (driver *Driver) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoReque watcher, _ := util.InitializeWatcher(getNodeInfoFunc) // Add list of files /and directories to watch. The list contains // iSCSI , FC and CHAP Info and Networking config directories - list := []string{"/etc/sysconfig/network-scripts/", "/etc/sysconfig/network/", "/etc/iscsi/initiatorname.iscsi", "/etc/networks", "/etc/iscsi/iscsid.conf"} + list := []string{"/etc/sysconfig/network-scripts/", "/etc/sysconfig/network/", "/etc/iscsi/initiatorname.iscsi", "/etc/networks", "/etc/iscsi/iscsid.conf", "/etc/nvme/hostnqn"} watcher.AddWatchList(list) // Start event the watcher in a separate thread. go watcher.StartWatcher() @@ -2222,18 +2236,22 @@ func (driver *Driver) nodeGetInfo() (string, error) { var iqns []*string var wwpns []*string + var nqns []*string for _, initiator := range initiators { if initiator.Type == iscsi { for i := 0; i < len(initiator.Init); i++ { iqns = append(iqns, &initiator.Init[i]) } + } else if initiator.Type == nvmeotcp { + for i := 0; i < len(initiator.Init); i++ { + nqns = append(nqns, &initiator.Init[i]) + } } else { for i := 0; i < len(initiator.Init); i++ { wwpns = append(wwpns, &initiator.Init[i]) } } } - var cidrNetworks []*string for _, network := range networks { log.Infof("Processing network named %s with IpV4 CIDR %s", network.Name, network.CidrNetwork) @@ -2249,6 +2267,7 @@ func (driver *Driver) nodeGetInfo() (string, error) { Iqns: iqns, Networks: cidrNetworks, Wwpns: wwpns, + Nqns: nqns, } nodeID, err := driver.flavor.LoadNodeInfo(node) diff --git a/pkg/driver/utils.go b/pkg/driver/utils.go index 6268ecc1..6b1d3f00 100644 --- a/pkg/driver/utils.go +++ b/pkg/driver/utils.go @@ -126,3 +126,12 @@ func removeDataFile(dirPath string, fileName string) error { func isValidIP(ip string) bool { return ip != "" && net.ParseIP(ip) != nil } +// GetNvmeInitiator returns the NVMe host NQN as a string +func GetNvmeInitiator() (string, error) { + data, err := ioutil.ReadFile("/etc/nvme/hostnqn") + if err != nil { + return "", err + } + nqn := strings.TrimSpace(string(data)) + return nqn, nil +} \ No newline at end of file diff --git a/pkg/flavor/kubernetes/flavor.go b/pkg/flavor/kubernetes/flavor.go index 5e6dafe0..bf854e9c 100644 --- a/pkg/flavor/kubernetes/flavor.go +++ b/pkg/flavor/kubernetes/flavor.go @@ -239,13 +239,18 @@ func (flavor *Flavor) LoadNodeInfo(node *model.Node) (string, error) { nodeInfo.Spec.WWPNs = wwpnsFromNode updateNodeRequired = true } + nqnsFromNode := getNqnsFromNode(node) + if !reflect.DeepEqual(nodeInfo.Spec.NQNs, nqnsFromNode) { + nodeInfo.Spec.NQNs = nqnsFromNode + updateNodeRequired = true + } if !updateNodeRequired { // no update needed to existing CRD return node.UUID, nil } - log.Infof("updating Node %s with iqns %v wwpns %v networks %v", - nodeInfo.Name, nodeInfo.Spec.IQNs, nodeInfo.Spec.WWPNs, nodeInfo.Spec.Networks) + log.Infof("updating Node %s with iqns %v wwpns %v networks %v nqns %v", + nodeInfo.Name, nodeInfo.Spec.IQNs, nodeInfo.Spec.WWPNs, nodeInfo.Spec.Networks, nodeInfo.Spec.NQNs) _, err := flavor.crdClient.StorageV1().HPENodeInfos().Update(nodeInfo) if err != nil { log.Errorf("Error updating the node %s - %s\n", nodeInfo.Name, err.Error()) @@ -262,6 +267,7 @@ func (flavor *Flavor) LoadNodeInfo(node *model.Node) (string, error) { IQNs: getIqnsFromNode(node), Networks: getNetworksFromNode(node), WWPNs: getWwpnsFromNode(node), + NQNs: getNqnsFromNode(node), }, } @@ -303,6 +309,14 @@ func getNetworksFromNode(node *model.Node) []string { return networks } +func getNqnsFromNode(node *model.Node) []string { + var nqns []string + for i := 0; i < len(node.Nqns); i++ { + nqns = append(nqns, *node.Nqns[i]) + } + return nqns +} + // UnloadNodeInfo remove the HPENodeInfo from the list of CRDs func (flavor *Flavor) UnloadNodeInfo() { log.Tracef(">>>>>> UnloadNodeInfo with name %s", flavor.nodeName) @@ -353,14 +367,19 @@ func (flavor *Flavor) GetNodeInfo(nodeID string) (*model.Node, error) { for i := range wwpns { wwpns[i] = &nodeInfo.Spec.WWPNs[i] } + nqns := make([]*string, len(nodeInfo.Spec.NQNs)) + for i := range nqns { + nqns[i] = &nodeInfo.Spec.NQNs[i] + } node := &model.Node{ Name: nodeInfo.ObjectMeta.Name, UUID: nodeInfo.Spec.UUID, Iqns: iqns, Networks: networks, Wwpns: wwpns, + Nqns: nqns, } - + log.Tracef("HPE Node Info sent to CSP: %v", node) return node, nil } } diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/device.go b/vendor/github.com/hpe-storage/common-host-libs/linux/device.go index 175a8874..55a2b539 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/device.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/device.go @@ -49,6 +49,7 @@ const ( // lrwxrwxrwx 1 root root 0 Mar 8 16:51 sdg -> ../devices/platform/host4/session2/target4:0:0/4:0:0:2/block/sdg deviceByHctlPatternFmt = ".*%s:%s:%s:%s.*block/(?P.*)" lunNotSupportedErr = "LOGICAL UNIT NOT SUPPORTED" + nvmeDevicePattern = "nvme[0-9]+n[0-9]+" ) var ( @@ -393,6 +394,7 @@ func setAltFullPathName(dev *model.Device) (err error) { // Helper function to perform rescan and detect the newly attached volume (FC/iSCSI volume) func rescanLoginVolume(volume *model.Volume) error { log.Traceln(">>>>> rescanLoginVolume", volume, "and accessProtocol", volume.AccessProtocol) + log.Traceln(">>>>> rescanLoginVolume Nqn", volume.Nqn, "and TargetAddress", volume.TargetAddress) defer log.Traceln("<<<< rescanLoginVolume") var err error var primaryVolObj *model.Volume @@ -407,6 +409,9 @@ func rescanLoginVolume(volume *model.Volume) error { primaryVolObj.ConnectionMode = volume.ConnectionMode primaryVolObj.SerialNumber = volume.SerialNumber primaryVolObj.Networks = volume.Networks + primaryVolObj.Nqn = volume.Nqn + primaryVolObj.TargetAddress = strings.Join(volume.DiscoveryIPs, ",") + primaryVolObj.TargetPort = volume.TargetPort err = rescanLoginVolumeForBackend(primaryVolObj) @@ -425,7 +430,7 @@ func rescanLoginVolume(volume *model.Volume) error { secondaryVolObj.LunID = strconv.Itoa(int(secondaryLunInfo.LunID)) secondaryVolObj.Iqns = secondaryLunInfo.TargetNames secondaryVolObj.TargetScope = volume.TargetScope - secondaryVolObj.DiscoveryIPs = secondaryLunInfo.DiscoveryIPs + secondaryVolObj.DiscoveryIPs = secondaryLunInfo.IscsiAccessInfo.DiscoveryIPs secondaryVolObj.Chap = volume.Chap secondaryVolObj.ConnectionMode = volume.ConnectionMode secondaryVolObj.SerialNumber = volume.SerialNumber @@ -450,7 +455,13 @@ func rescanLoginVolumeForBackend(volObj *model.Volume) error { if err != nil { return err } - } else { + } else if strings.EqualFold(volObj.AccessProtocol, "nvmetcp") { + // NVMe over TCP volume + err = HandleNvmeTcpDiscovery(volObj) + if err != nil { + return err + } + }else { // Check if client intends us to specifically login using multiple IP addresses(cloud volumes) if len(volObj.Networks) > 0 { // check if ifaces are created and enable port binding @@ -492,6 +503,30 @@ func isLuksDevice(devPath string) (bool, error) { func createLinuxDevice(volume *model.Volume) (dev *model.Device, err error) { log.Debugf(">>>> createLinuxDevice called with volume %s serialNumber %s and lunID %s", volume.Name, volume.SerialNumber, volume.LunID) defer log.Debug("<<<<< createLinuxDevice") + // Handle NVMe/TCP device creation + if strings.EqualFold(volume.AccessProtocol, "nvmetcp") { + log.Tracef("NVMe/TCP requested, NQN: %s, Serial: %s", volume.Nqn, volume.SerialNumber) + // Initiate NVMe/TCP discovery and connection + if err := rescanLoginVolume(volume); err != nil { + return nil, fmt.Errorf("NVMe/TCP discovery failed: %v", err) + } + // Allow time for device to register + time.Sleep(time.Second * 2) + // Attempt to locate NVMe device by NQN or Serial + nvmeDev, lookupErr := GetNvmeDeviceFromNamespace(volume.Nqn) + if lookupErr == nil && nvmeDev != nil { + log.Infof("NVMe/TCP device found: %+v", nvmeDev) + return nvmeDev, nil + } + // Fallback to SerialNumber if NQN lookup fails + nvmeDev, lookupErr = GetNvmeDeviceFromNamespace(volume.SerialNumber) + if lookupErr == nil && nvmeDev != nil { + log.Infof("NVMe/TCP device found by serial: %+v", nvmeDev) + return nvmeDev, nil + } + return nil, fmt.Errorf("unable to locate NVMe/TCP device for NQN %s or serial %s", volume.Nqn, volume.SerialNumber) + } + // Rescan and detect the newly attached volume err = rescanLoginVolume(volume) if err != nil { @@ -1090,21 +1125,35 @@ func GetDeviceFromVolume(vol *model.Volume) (*model.Device, error) { log.Tracef(">>>>> GetDeviceFromVolume for serial %s", vol.SerialNumber) defer log.Trace("<<<<< GetDeviceFromVolume") - devices, err := GetLinuxDmDevices(false, vol) - if err != nil { - return nil, err - } - if len(devices) == 0 { - return nil, fmt.Errorf("unable to find device matching volume serial number %s", vol.SerialNumber) + // Try NVMe device lookup first + if strings.EqualFold(vol.AccessProtocol, "nvmetcp") { + nvmeDev, err := GetNvmeDeviceFromNamespace(vol.SerialNumber) + log.Tracef("NVMe device: %+v", nvmeDev) + if err == nil && nvmeDev != nil { + log.Debugf("Found NVMe device: %+v", nvmeDev) + return nvmeDev, nil + } + }else{ + devices, err := GetLinuxDmDevices(false, vol) + if err != nil { + return nil, err + } + + if len(devices) > 0 { + return devices[0], nil + } + if len(devices) == 0 { + return nil, fmt.Errorf("unable to find device matching volume serial number %s", vol.SerialNumber) + } + return devices[0], nil } - return devices[0], nil + return nil, fmt.Errorf("device not found for volume %s with access protocol %s", vol.Name, vol.AccessProtocol) } //CreateLinuxDevices : attached and creates linux devices to host func CreateLinuxDevices(vols []*model.Volume) (devs []*model.Device, err error) { log.Tracef(">>>>> CreateLinuxDevices") defer log.Trace("<<<<< CreateLinuxDevices") - var devices []*model.Device for _, vol := range vols { log.Tracef("create request with serialnumber :%s, accessprotocol %s discoveryip %s, iqn %s ", vol.SerialNumber, vol.AccessProtocol, vol.DiscoveryIP, vol.Iqn) @@ -1426,3 +1475,36 @@ func isMappedLuksDevice(devPath string) (bool, error) { return false, err } } + +// GetNvmeDeviceFromNamespace returns NVMe device path for given namespace or serial +func GetNvmeDeviceFromNamespace(serialOrNamespace string) (*model.Device, error) { + nvmeRoot := "/dev/" + files, err := ioutil.ReadDir(nvmeRoot) + if err != nil { + return nil, err + } + nvmeRegex := regexp.MustCompile(nvmeDevicePattern) + for _, f := range files { + if nvmeRegex.MatchString(f.Name()) { + // Optionally, check namespace/serial via sysfs + sysfsSerialPath := fmt.Sprintf("/sys/class/block/%s/subsystem/%s/nguid", f.Name(), f.Name()) + log.Tracef("serial path=%s", sysfsSerialPath) + serial, _ := util.FileReadFirstLine(sysfsSerialPath) + log.Tracef("serial path=%s", sysfsSerialPath) + // Normalize the serial from sysfs by removing dashes and whitespace + normalizedSerial := strings.ReplaceAll(strings.TrimSpace(serial), "-", "") + log.Tracef("found serial number %s, normalized: %s", serial, normalizedSerial) + if serialOrNamespace == f.Name() || serialOrNamespace == normalizedSerial { + log.Tracef("serial number %s matched=%s", serialOrNamespace, normalizedSerial) + return &model.Device{ + Pathname: nvmeRoot + f.Name(), + AltFullPathName: nvmeRoot + f.Name(), + SerialNumber: normalizedSerial, + }, nil + }else{ + log.Tracef("serial number %s did not match=%s", serialOrNamespace, normalizedSerial) + } + } + } + return nil, fmt.Errorf("NVMe device not found for %s", serialOrNamespace) +} \ No newline at end of file diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go b/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go index e2f1e773..8d3ebac6 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/initiator.go @@ -4,6 +4,7 @@ package linux import ( "errors" + log "github.com/hpe-storage/common-host-libs/logger" "github.com/hpe-storage/common-host-libs/model" "github.com/hpe-storage/common-host-libs/util" @@ -14,6 +15,9 @@ var ( initiatorNamePattern = "^InitiatorName=(?P.*)$" iscsi = "iscsi" fc = "fc" + nvmeHostnqnPath = "/etc/nvme/hostnqn" + nvmeHostnqnPattern = "^(?Pnqn\\..*)$" + nvmeotcp = "nvmeotcp" ) //GetInitiators : get the host initiators @@ -32,15 +36,24 @@ func GetInitiators() ([]*model.Initiator, error) { if err != nil { log.Debug("Error getting FcInitiator: ", err) } + // Add NVMe initiator discovery + nvmeInits, err := getNvmeInitiators() + if err != nil { + log.Debug("Error getting NvmeInitiator: ", err) + } + if fcInits != nil { inits = append(inits, fcInits) } if iscsiInits != nil { inits = append(inits, iscsiInits) } + if nvmeInits != nil { + inits = append(inits, nvmeInits) + } - if fcInits == nil && iscsiInits == nil { - return nil, errors.New("iscsi and fc initiators not found") + if fcInits == nil && iscsiInits == nil && nvmeInits == nil { + return nil, errors.New("iscsi, fc, and nvme initiators not found") } log.Debug("initiators ", inits) @@ -94,3 +107,43 @@ func getFcInitiators() (fcInit *model.Initiator, err error) { } return fcInit, nil } + +/*func getNvmeInitiators() (init *model.Initiator, err error) { + log.Trace(">>>>> getNvmeInitiators") + defer log.Trace("<<<<< getNvmeInitiators") + + hostnqn, err := GetNvmeInitiator() + if err != nil { + log.Debugf("NVMe host NQN not found, assuming not an NVMe host") + return nil, nil + } + + initiators := []string{hostnqn} + init = &model.Initiator{Type: "nvmeotcp", Init: initiators} + return init, nil +}*/ +func getNvmeInitiators() (init *model.Initiator, err error) { + log.Trace(">>>>> getNvmeInitiators") + defer log.Trace("<<<<< getNvmeInitiators") + + exists, _, err := util.FileExists(nvmeHostnqnPath) + if !exists { + log.Debugf("%s not found, assuming not an NVMe host", nvmeHostnqnPath) + return nil, nil + } + + initiators, err := util.FileGetStringsWithPattern(nvmeHostnqnPath, nvmeHostnqnPattern) + if err != nil { + log.Errorf("failed to get nqn from %s error %s", nvmeHostnqnPath, err.Error()) + return nil, err + } + + if len(initiators) == 0 { + log.Errorf("empty nqn found from %s", nvmeHostnqnPath) + return nil, errors.New("empty nqn found") + } + + log.Debugf("got nvme initiator name as %s", initiators[0]) + init = &model.Initiator{Type: nvmeotcp, Init: initiators} + return init, nil +} diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/iscsi.go b/vendor/github.com/hpe-storage/common-host-libs/linux/iscsi.go index d79f1da6..69506bc5 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/iscsi.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/iscsi.go @@ -251,7 +251,7 @@ func HandleIscsiDiscovery(volume *model.Volume) (err error) { secondaryVolObj.LunID = strconv.Itoa(int(secondaryLunInfo.LunID)) secondaryVolObj.Iqns = secondaryLunInfo.TargetNames secondaryVolObj.TargetScope = volume.TargetScope - secondaryVolObj.DiscoveryIPs = secondaryLunInfo.DiscoveryIPs + secondaryVolObj.DiscoveryIPs = secondaryLunInfo.IscsiAccessInfo.DiscoveryIPs secondaryVolObj.Chap = volume.Chap secondaryVolObj.ConnectionMode = volume.ConnectionMode secondaryVolObj.SerialNumber = volume.SerialNumber diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/multipath.go b/vendor/github.com/hpe-storage/common-host-libs/linux/multipath.go index b3fd78fe..4bbaaf68 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/linux/multipath.go +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/multipath.go @@ -4,6 +4,8 @@ package linux import ( "fmt" + "io/ioutil" + "path/filepath" "regexp" "strings" "sync" @@ -219,6 +221,15 @@ func cleanupDeviceAndSlaves(dev *model.Device) (err error) { log.Tracef(">>>>> cleanupDeviceAndSlaves called for %+v", dev) defer log.Trace("<<<<< cleanupDeviceAndSlaves") + + // --- NVMe multipath handling --- + if dev != nil && dev.Pathname != "" && IsNvmeDevice(dev.Pathname) { + if err := HandleMultipathForDevice(dev); err != nil { + return err + } + } + // --- End NVMe multipath handling --- + isFC := isFibreChannelDevice(dev.Slaves) // disable queuing on multipath @@ -498,3 +509,40 @@ func multipathGetPathsOfDevice(dev *model.Device, needActivePath bool) (paths [] } return paths, nil } +// IsNvmeDevice returns true if the device path is an NVMe device (e.g., /dev/nvmeXnY) +func IsNvmeDevice(devPath string) bool { + base := filepath.Base(devPath) + matched, _ := regexp.MatchString(`^nvme\d+n\d+$`, base) + return matched +} + +// IsNvmeMultipathEnabled checks if NVMe native multipath is enabled for a device +func IsNvmeMultipathEnabled(devPath string) bool { + base := filepath.Base(devPath) + nvmeMpPath := filepath.Join("/sys/class/block", base, "nvme_multipath") + data, err := ioutil.ReadFile(nvmeMpPath) + if err != nil { + return false + } + return strings.TrimSpace(string(data)) == "Y" +} +// HandleMultipathForDevice handles multipath for both SCSI and NVMe devices +func HandleMultipathForDevice(dev *model.Device) error { + + if IsNvmeDevice(dev.Pathname) { + if IsNvmeMultipathEnabled(dev.Pathname) { + log.Debugf("NVMe native multipath is enabled for %s, skipping dm-multipath logic", dev.Pathname) + // All path management is handled by the NVMe subsystem + return nil + } + log.Debugf("NVMe device %s does not have native multipath enabled", dev.Pathname) + // Optional: fallback to dm-multipath for NVMe if required by your environment + // If not required, just return nil here + // Otherwise, call your existing dm-multipath logic below + // return handleDmMultipathForNvme(dev) + return nil + } + // Existing dm-multipath logic for SCSI/iSCSI devices + // return handleDmMultipathForScsi(dev) + return nil +} \ No newline at end of file diff --git a/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go b/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go new file mode 100644 index 00000000..bda7eb26 --- /dev/null +++ b/vendor/github.com/hpe-storage/common-host-libs/linux/nvme.go @@ -0,0 +1,234 @@ +// Copyright 2025 Hewlett Packard Enterprise Development LP + +package linux + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + log "github.com/hpe-storage/common-host-libs/logger" + "github.com/hpe-storage/common-host-libs/model" + "github.com/hpe-storage/common-host-libs/util" +) + +const ( + nvmecmd = "nvme" + nvmeConnectCmd = "nvme connect" + nvmeDisconnectCmd = "nvme disconnect" + nvmeListCmd = "nvme list" + nvmeListSubsysCmd = "nvme list-subsys" + defaultNvmePort = "4420" + nvmeHostPathFormat = "/sys/class/nvme/" + nvmeNamespacePattern = "nvme[0-9]+n[0-9]+" + nvmeHostPath = "/etc/nvme/hostnqn" +) + +// GetNvmeInitiator gets the NVMe host NQN +func GetNvmeInitiator() (string, error) { + // Read from /etc/nvme/hostnqn or generate one + hostnqn, err := util.FileReadFirstLine(nvmeHostPath) + if err != nil { + log.Debugf("Could not read hostnqn from %s, generating one", nvmeHostPath) + // Generate hostnqn using nvme gen-hostnqn + args := []string{"gen-hostnqn"} + hostnqn, _, err = util.ExecCommandOutput(nvmecmd, args) + if err != nil { + return "", err + } + hostnqn = strings.TrimSpace(hostnqn) + } + return hostnqn, nil +} + +// ApplyNvmeTcpTuning applies recommended sysctl and module settings for NVMe over TCP +func ApplyNvmeTcpTuning() error { + var tuningErrors []string + + // Example: Increase network buffer sizes for high throughput + if err := setSysctl("net.core.rmem_max", "16777216"); err != nil { + tuningErrors = append(tuningErrors, err.Error()) + } + if err := setSysctl("net.core.wmem_max", "16777216"); err != nil { + tuningErrors = append(tuningErrors, err.Error()) + } + + // Example: Set NVMe core parameters (if needed) + if err := setKernelParam("/sys/module/nvme_core/parameters/multipath", "Y"); err != nil { + tuningErrors = append(tuningErrors, err.Error()) + } + + // Add more NVMe/TCP-specific tuning as needed... + + if len(tuningErrors) > 0 { + return fmt.Errorf("NVMe TCP tuning errors: %s", strings.Join(tuningErrors, "; ")) + } + return nil +} + +func setSysctl(key, value string) error { + cmd := fmt.Sprintf("sysctl -w %s=%s", key, value) + out, _, err := util.ExecCommandOutput("sh", []string{"-c", cmd}) + if err != nil { + return fmt.Errorf("failed to set %s: %v (%s)", key, err, out) + } + return nil +} + +func setKernelParam(path, value string) error { + f, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + return fmt.Errorf("failed to open %s: %v", path, err) + } + defer f.Close() + if _, err := f.WriteString(value); err != nil { + return fmt.Errorf("failed to write %s to %s: %v", value, path, err) + } + return nil +} + +// ConnectNvmeTarget connects to an NVMe over TCP target +func ConnectNvmeTarget(target *model.NvmeTarget) error { + + var discoveryIP = strings.Split(target.Address, ",") + if len(discoveryIP) == 0 { + return fmt.Errorf("no discovery IPs provided for NVMe target") + } + + // Use the first discovery IP for connection + target.Address = discoveryIP[0] + + args := []string{ + "connect", + "-t", "tcp", + "-n", target.NQN, + "-a", target.Address, + "-s", target.Port, + } + + _, rc, err := util.ExecCommandOutput(nvmecmd, args) + if err != nil && rc != 114 { + log.Warnf("NVMe connect failed for discovery IP %s, rc=%d, error: %s", discoveryIP[0], rc, err) + //try discovery with another IP if multiple are provided + for i := 1; i < len(discoveryIP); i++ { + log.Warnf("NVMe connect failed, trying next discovery IP %s", discoveryIP[i]) + args := []string{ + "connect", + "-t", "tcp", + "-n", target.NQN, + "-a", discoveryIP[i], + "-s", target.Port, + } + _, rc, err = util.ExecCommandOutput(nvmecmd, args) + if err != nil && rc != 114 { + log.Warnf("NVMe connect failed for discovery IP %s, rc=%d, error: %s", discoveryIP[i], rc, err) + continue + }else{ + log.Infof("Successfully connected to NVMe target using discovery IP %s", discoveryIP[i]) + return nil + } + } + return fmt.Errorf("failed to connect to NVMe target: %v", err) + }else{ + log.Infof("Successfully connected to NVMe target using discovery IP %s", discoveryIP[0]) + return nil + } +} + +// DisconnectNvmeTarget disconnects from an NVMe target +func DisconnectNvmeTarget(target *model.NvmeTarget) error { + args := []string{ + "disconnect", + "-n", target.NQN, + } + + _, _, err := util.ExecCommandOutput(nvmecmd, args) + return err +} + +// RescanNvme performs NVMe namespace rescan +func RescanNvme() error { + // NVMe typically doesn't require explicit rescanning like SCSI + // The kernel automatically detects new namespaces + return nil +} +// HandleNvmeTcpDiscovery performs NVMe/TCP connection and device verification for a volume. +func HandleNvmeTcpDiscovery(volume *model.Volume) error { + log.Tracef(">>>>> HandleNvmeTcpDiscovery for volume %s", volume.SerialNumber) + defer log.Trace("<<<<< HandleNvmeTcpDiscovery") + + // 1. Apply NVMe/TCP tuning recommendations + if err := ApplyNvmeTcpTuning(); err != nil { + log.Warnf("Failed to apply NVMe TCP tuning: %v", err) + // Continue even if tuning fails + } + + // 2. Prepare NVMe target info + target := &model.NvmeTarget{ + NQN: volume.Nqn, + Address: strings.Join(volume.DiscoveryIPs, ","), + Port: volume.TargetPort, + } + + // 3. Connect to NVMe target + if err := ConnectNvmeTarget(target); err != nil { + return fmt.Errorf("failed to connect to NVMe target: %v", err) + } + + // 4. Optionally, verify device presence (wait for /dev/nvmeXnY) + found := false + for i := 0; i < 10; i++ { + devices, _ := FindNvmeDevices(volume.SerialNumber) + if len(devices) > 0 { + found = true + break + } + time.Sleep(1 * time.Second) + } + if !found { + return fmt.Errorf("NVMe device for serial %s not found after connect", volume.SerialNumber) + } + + return nil +} + +// FindNvmeDevices searches for NVMe devices matching the given serial number +func FindNvmeDevices(serialNumber string) ([]string, error) { + var devices []string + + // Scan /dev for nvme devices + files, err := ioutil.ReadDir("/dev") + if err != nil { + return nil, err + } + + nvmeRegex := regexp.MustCompile(`^nvme\d+n\d+$`) + for _, f := range files { + if nvmeRegex.MatchString(f.Name()) { + devicePath := filepath.Join("/dev", f.Name()) + + // Check serial number via sysfs + sysfsSerialPath := fmt.Sprintf("/sys/class/block/%s/subsystem/%s/nguid", f.Name(), f.Name()) + log.Tracef("serial path=%s", sysfsSerialPath) + if serial, err := util.FileReadFirstLine(sysfsSerialPath); err == nil { + // Normalize the serial from sysfs by removing dashes and whitespace + normalizedSerial := strings.ReplaceAll(strings.TrimSpace(serial), "-", "") + log.Tracef("found serial number %s, normalized: %s", serial, normalizedSerial) + if strings.TrimSpace(normalizedSerial) == serialNumber { + devices = append(devices, devicePath) + } + } + + // Also check if the device name itself matches (for namespace matching) + if f.Name() == serialNumber { + devices = append(devices, devicePath) + } + } + } + + return devices, nil +} \ No newline at end of file diff --git a/vendor/github.com/hpe-storage/common-host-libs/model/types.go b/vendor/github.com/hpe-storage/common-host-libs/model/types.go index b38c31bd..456b7f35 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/model/types.go +++ b/vendor/github.com/hpe-storage/common-host-libs/model/types.go @@ -145,6 +145,14 @@ type IscsiTarget struct { Scope string // GST or VST } +// NvmeTarget struct +type NvmeTarget struct { + NQN string + Address string + Port string // 4420 (default NVMe over TCP port) + Scope string // GST or VST +} + // Device struct type Device struct { VolumeID string `json:"volume_id,omitempty"` @@ -159,6 +167,7 @@ type Device struct { Size int64 `json:"size,omitempty"` // size in MiB Slaves []string `json:"slaves,omitempty"` IscsiTargets []*IscsiTarget `json:"iscsi_target,omitempty"` + NvmeTargets []*NvmeTarget `json:"nvme_target,omitempty"` Hcils []string `json:"-"` // export it if needed TargetScope string `json:"target_scope,omitempty"` //GST="group", VST="volume" or empty(older array fiji etc), and no-op for FC State string `json:"state,omitempty"` // state of the device needed to verify the device is active @@ -190,6 +199,9 @@ type Volume struct { AccessProtocol string `json:"access_protocol,omitempty"` Iqn string `json:"iqn,omitempty"` // deprecated Iqns []string `json:"iqns,omitempty"` + Nqn string `json:"nqn,omitempty"` + TargetAddress string `json:"target_address,omitempty"` + TargetPort string `json:"target_port,omitempty"` DiscoveryIP string `json:"discovery_ip,omitempty"` // deprecated DiscoveryIPs []string `json:"discovery_ips,omitempty"` MountPoint string `json:"Mountpoint,omitempty"` @@ -201,6 +213,7 @@ type Volume struct { TargetScope string `json:"target_scope,omitempty"` //GST="group", VST="volume" or empty(older array fiji etc), and no-op for FC IscsiSessions []*IscsiSession `json:"iscsi_sessions,omitempty"` FcSessions []*FcSession `json:"fc_sessions,omitempty"` + NvmeSessions []*NvmeSession `json:"nvme_sessions,omitempty"` VolumeGroupId string `json:"volume_group_id"` SecondaryArrayDetails string `json:"secondary_array_details,omitempty"` UsedBytes int64 `json:"used_bytes,omitempty"` @@ -228,6 +241,11 @@ type IscsiSession struct { InitiatorNameLegacy string `json:"initiatorName,omitempty"` InitiatorIP string `json:"initiator_ip_addr,omitempty"` } +// NvmeSession info +type NvmeSession struct { + InitiatorNQN string `json:"initiator_nqn,omitempty"` + InitiatorIP string `json:"initiator_ip_addr,omitempty"` +} func (s FcSession) InitiatorWwpnStr() string { if s.InitiatorWwpnLegacy != "" { @@ -297,9 +315,11 @@ type AccessInfo struct { type BlockDeviceAccessInfo struct { AccessProtocol string `json:"access_protocol,omitempty"` TargetNames []string `json:"target_names,omitempty"` + DiscoveryIPs []string `json:"discovery_ips,omitempty"` LunID int32 `json:"lun_id,omitempty"` SecondaryBackendDetails IscsiAccessInfo + NvmetcpAccessInfo } // Information of LUN id, IQN, discovery IP's the secondary array @@ -309,9 +329,10 @@ type SecondaryBackendDetails struct { // Information of the each secondary array type SecondaryLunInfo struct { - LunID int32 `json:"lun_id,omitempty""` + LunID int32 `json:"lun_id,omitempty"` TargetNames []string `json:"target_names,omitempty"` IscsiAccessInfo + NvmetcpAccessInfo } // IscsiAccessInfo contains the fields necessary for iSCSI access @@ -321,6 +342,11 @@ type IscsiAccessInfo struct { ChapPassword string `json:"chap_password,omitempty"` } +// NvmetcpAccessInfo contains the fields necessary for NVMe/TCP access +type NvmetcpAccessInfo struct { + TargetNames []string `json:"target_names,omitempty"` // NQN(s) + TargetPort string `json:"target_port,omitempty"` // e.g., 4420 +} // VirtualDeviceAccessInfo contains the required data to access a virtual device type VirtualDeviceAccessInfo struct { } @@ -392,15 +418,16 @@ type Token struct { // Node represents a host that would access volumes through the CSP type Node struct { - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - UUID string `json:"uuid,omitempty"` - Iqns []*string `json:"iqns,omitempty"` - Networks []*string `json:"networks,omitempty"` - Wwpns []*string `json:"wwpns,omitempty"` - ChapUser string `json:"chap_user,omitempty"` - ChapPassword string `json:"chap_password,omitempty"` - AccessProtocol string `json:"access_protocol,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + UUID string `json:"uuid,omitempty"` + Iqns []*string `json:"iqns,omitempty"` + Networks []*string `json:"networks,omitempty"` + Wwpns []*string `json:"wwpns,omitempty"` + Nqns []*string `json:"nqns,omitempty"` + ChapUser string `json:"chap_user,omitempty"` + ChapPassword string `json:"chap_password,omitempty"` + AccessProtocol string `json:"access_protocol,omitempty"` } // KeyValue is a store of key-value pairs diff --git a/vendor/github.com/hpe-storage/common-host-libs/tunelinux/nvme.go b/vendor/github.com/hpe-storage/common-host-libs/tunelinux/nvme.go new file mode 100644 index 00000000..8a8e7bae --- /dev/null +++ b/vendor/github.com/hpe-storage/common-host-libs/tunelinux/nvme.go @@ -0,0 +1,2 @@ +package tunelinux + diff --git a/vendor/github.com/hpe-storage/common-host-libs/util/volume.go b/vendor/github.com/hpe-storage/common-host-libs/util/volume.go index eb6ab8fc..f8a57616 100644 --- a/vendor/github.com/hpe-storage/common-host-libs/util/volume.go +++ b/vendor/github.com/hpe-storage/common-host-libs/util/volume.go @@ -4,6 +4,7 @@ package util import ( "encoding/json" + "github.com/hpe-storage/common-host-libs/logger" "github.com/hpe-storage/common-host-libs/model" ) @@ -70,7 +71,7 @@ func GetSecondaryArrayDiscoveryIps(details string) []string { numberOfSecondaryBackends := len(secondaryArrayDetails.PeerArrayDetails) var secondaryDiscoverIps []string for i := 0; i < numberOfSecondaryBackends; i++ { - for _, discoveryIpRetrieved := range secondaryArrayDetails.PeerArrayDetails[i].DiscoveryIPs { + for _, discoveryIpRetrieved := range secondaryArrayDetails.PeerArrayDetails[i].IscsiAccessInfo.DiscoveryIPs { secondaryDiscoverIps = append(secondaryDiscoverIps, discoveryIpRetrieved) } } diff --git a/vendor/github.com/hpe-storage/k8s-custom-resources/pkg/apis/hpestorage/v1/types.go b/vendor/github.com/hpe-storage/k8s-custom-resources/pkg/apis/hpestorage/v1/types.go index 98112a44..4d0e0bc0 100644 --- a/vendor/github.com/hpe-storage/k8s-custom-resources/pkg/apis/hpestorage/v1/types.go +++ b/vendor/github.com/hpe-storage/k8s-custom-resources/pkg/apis/hpestorage/v1/types.go @@ -32,6 +32,7 @@ type HPENodeInfoSpec struct { IQNs []string `json:"iqns,omitempty"` Networks []string `json:"networks,omitempty"` WWPNs []string `json:"wwpns,omitempty"` + NQNs []string `json:"nqns,omitempty"` ChapUser string `json:"chapUser,omitempty"` ChapPassword string `json:"chapPassword,omitempty"` }