CON-3876 - NVMEoTCP support for csi-driver and host-libs

Dockerfile

@@ -76,7 +76,8 @@ RUN ln -s /chroot/chroot-host-wrapper.sh /chroot/blkid \
     && ln -s /chroot/chroot-host-wrapper.sh /chroot/ip \
     && ln -s /chroot/chroot-host-wrapper.sh /chroot/dnsdomainname \
     && ln -s /chroot/chroot-host-wrapper.sh /chroot/sg_inq \
-    && ln -s /chroot/chroot-host-wrapper.sh /chroot/find
+    && ln -s /chroot/chroot-host-wrapper.sh /chroot/find \
+    && ln -s /chroot/chroot-host-wrapper.sh /chroot/nvme
 
 ENV PATH="/chroot:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 

cmd/csi-driver/conform/entrypoint.sh

@@ -28,6 +28,7 @@ if [ "$nodeService" = true ] || [ "$nodeInit" = true ]; then
     ln -s /host/etc/multipath.conf /etc/multipath.conf
     ln -s /host/etc/multipath /etc/multipath
     ln -s /host/etc/iscsi /etc/iscsi
+    ln -s /host/etc/nvme /etc/nvme
 
     # symlink to host os release files for parsing
     if [ -f /host/etc/redhat-release ]; then

cmd/csi-driver/conform/hpe-storage-node.sh

@@ -74,6 +74,12 @@ if [ "$CONFORM_TO" = "ubuntu" ]; then
         apt-get -qq install -y sg3-utils
         exit_on_error $?
     fi
+    # Install NVMe CLI tools
+    if [ ! -f /usr/sbin/nvme ]; then
+        apt-get -qq update
+        apt-get -qq install -y nvme-cli
+        exit_on_error $?
+    fi
 
 elif [ "$CONFORM_TO" = "redhat" ]; then
     # Install device-mapper-multipath
@@ -101,6 +107,12 @@ elif [ "$CONFORM_TO" = "redhat" ]; then
         exit_on_error $?
     fi
 
+    # Install NVMe CLI tools
+    if [ ! -f /usr/sbin/nvme ]; then
+        yum -y install nvme-cli
+        exit_on_error $?
+    fi
+
 elif [ "$CONFORM_TO" = "sles" ]; then
     # Install device-mapper-multipath
     if [ ! -f /sbin/multipathd ]; then
@@ -128,13 +140,19 @@ elif [ "$CONFORM_TO" = "sles" ]; then
         exit_on_error $?
     fi
 
+    # Install NVMe CLI tools
+    if [ ! -f /usr/sbin/nvme ]; then
+        zypper -n install nvme-cli
+        exit_on_error $?
+    fi
+
 elif [ "$CONFORM_TO" = "slem" ]; then
     # SLE Micro
     echo -n "Ensuring critical binaries are in the SLEM image: "
-    for bin in /usr/bin/sg_inq /sbin/mount.nfs4 /sbin/iscsid /sbin/multipathd; do
+    for bin in /usr/bin/sg_inq /sbin/mount.nfs4 /sbin/iscsid /sbin/multipathd /usr/sbin/nvme; do
         echo -n "$bin "
         if [ ! -f $bin ]; then
-            echo "$bin is missing. Run 'transactional-update -n pkg install multipath-tools open-iscsi nfs-client sg3_utils' on the worker nodes and reboot."
+            echo "$bin is missing. Run 'transactional-update -n pkg install multipath-tools open-iscsi nfs-client sg3_utils nvme-cli' on the worker nodes and reboot."
             exit_on_error 1
         fi
     done
@@ -147,6 +165,12 @@ elif [ "$CONFORM_TO" = "coreos" ]; then
         echo "Generating first-boot IQN"
         echo "InitiatorName=$(iscsi-iname)" > /etc/iscsi/initiatorname.iscsi
     fi
+    # Generate NVMe host NQN if it doesn't exist
+    if ! [[ -e /etc/nvme/hostnqn ]]; then
+        echo "Generating first-boot Host NQN"
+        mkdir -p /etc/nvme
+        echo "$(nvme gen-hostnqn)" > /etc/nvme/hostnqn
+    fi
 else
     echo "unsupported configuration for node package checks. os $os_name"
     exit 1
@@ -161,6 +185,25 @@ fi
 # Load iscsi_tcp modules, its a no-op if its already loaded
 modprobe iscsi_tcp
 
+# Load NVMe-oTCP modules
+modprobe nvme-core
+modprobe nvme-tcp
+
+
+# Generate NVMe host NQN if it doesn't exist (for non-CoreOS systems)
+if [ "$CONFORM_TO" != "coreos" ]; then
+    if ! [[ -e /etc/nvme/hostnqn ]]; then
+        echo "Generating Host NQN"
+        mkdir -p /etc/nvme
+        nvme gen-hostnqn > /etc/nvme/hostnqn
+        exit_on_error $?
+    fi
+fi
+
+# Ensure hostnqn file has proper permissions
+if [[ -e /etc/nvme/hostnqn ]]; then
+    chmod 644 /etc/nvme/hostnqn
+fi
 # Don't let udev automatically scan targets(all luns) on Unit Attention.
 # This will prevent udev scanning devices which we are attempting to remove
 if [ -f /lib/udev/rules.d/90-scsi-ua.rules ]; then

pkg/driver/constants.go

@@ -6,6 +6,7 @@ const (
 	// Protocol types
 	iscsi = "iscsi"
 	fc    = "fc"
+	nvmeotcp = "nvmeotcp"
 
 	// defaultFileSystem is the implemenation-specific default value
 	defaultFileSystem = "xfs"

pkg/driver/controller_server.go

@@ -924,7 +924,9 @@ func (driver *Driver) controllerPublishVolume(
                 requestedAccessProtocol = iscsi
         } else if requestedAccessProtocol == "fc" {
                 requestedAccessProtocol = fc
-        }
+        } else if requestedAccessProtocol == "nvmetcp" {
+		requestedAccessProtocol = "nvmetcp"
+	}
 
 	if existingNode != nil {
 		log.Tracef("CSP has already been notified about the node with ID %s and UUID %s", existingNode.ID, existingNode.UUID)
@@ -957,12 +959,15 @@ func (driver *Driver) controllerPublishVolume(
 
 	// TODO: add any additional info necessary to mount the device
 	publishContext := map[string]string{}
-	publishContext[serialNumberKey] = publishInfo.SerialNumber
-	publishContext[accessProtocolKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol
-	publishContext[targetNamesKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames, ",")
-	publishContext[targetScopeKey] = requestedTargetScope
-	publishContext[lunIDKey] = strconv.Itoa(int(publishInfo.AccessInfo.BlockDeviceAccessInfo.LunID))
-
+
+
+		publishContext[serialNumberKey] = publishInfo.SerialNumber
+		publishContext[accessProtocolKey] = publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol
+		publishContext[targetNamesKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.TargetNames, ",")
+		publishContext[targetScopeKey] = requestedTargetScope
+		publishContext[lunIDKey] = strconv.Itoa(int(publishInfo.AccessInfo.BlockDeviceAccessInfo.LunID))
+		publishContext[discoveryIPsKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.DiscoveryIPs, ",")
+
 	// Start of population of target array details
 	if publishInfo.AccessInfo.BlockDeviceAccessInfo.SecondaryBackendDetails.PeerArrayDetails != nil {
 		secondaryArrayMarshalledStr, err := json.Marshal(&publishInfo.AccessInfo.BlockDeviceAccessInfo.SecondaryBackendDetails)
@@ -977,9 +982,15 @@ func (driver *Driver) controllerPublishVolume(
 	}
 
 	if strings.EqualFold(publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol, iscsi) {
-		publishContext[discoveryIPsKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.IscsiAccessInfo.DiscoveryIPs, ",")
+		publishContext[discoveryIPsKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.DiscoveryIPs, ",")
 	}
 
+	if strings.EqualFold(publishInfo.AccessInfo.BlockDeviceAccessInfo.AccessProtocol, nvmetcp) {
+		publishContext[targetPortKey] = "4420" // default NVMe/TCP port
+		publishContext[discoveryIPsKey] = strings.Join(publishInfo.AccessInfo.BlockDeviceAccessInfo.DiscoveryIPs, ",")
+	}
+
+
 	if readOnlyAccessMode {
 		publishContext[readOnlyKey] = strconv.FormatBool(readOnlyAccessMode)
 	} else { // Default case, we stick to old behavior

pkg/driver/node_server.go

@@ -42,6 +42,8 @@ var (
 
 const (
 	fileHostIPKey = "hostIP"
+	nvmetcp = "nvmetcp"
+    	targetPortKey = "targetPort"
 )
 
 var isWatcherEnabled = false
@@ -576,12 +578,24 @@ func (driver *Driver) setupDevice(
 	// TODO: Enhance CHAPI to work with a PublishInfo object rather than a volume
 
 	discoveryIps := strings.Split(publishContext[discoveryIPsKey], ",")
-	iqns := strings.Split(publishContext[targetNamesKey], ",")
-
+
+	// For NVMe/TCP, set Nqn field in the model.Volume and update iqns accordingly
+	var nqn string
+	var iqns = []string{}
+	if publishContext[accessProtocolKey] == nvmetcp {
+		nqn = publishContext[targetNamesKey]
+		iqns = []string{}
+	}else if publishContext[accessProtocolKey] == iscsi{
+		nqn = ""
+		iqns = strings.Split(publishContext[targetNamesKey], ",")
+	}
 	volume := &model.Volume{
 		SerialNumber:          publishContext[serialNumberKey],
 		AccessProtocol:        publishContext[accessProtocolKey],
 		Iqns:                  iqns,
+		Nqn:                   nqn,
+		TargetAddress: 	       publishContext[targetNamesKey],
+		TargetPort:            publishContext[targetPortKey],
 		TargetScope:           publishContext[targetScopeKey],
 		LunID:                 publishContext[lunIDKey],
 		DiscoveryIPs:          discoveryIps,
@@ -2163,7 +2177,7 @@ func (driver *Driver) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoReque
 		watcher, _ := util.InitializeWatcher(getNodeInfoFunc)
 		// Add list of files /and directories to watch. The list contains
 		// iSCSI , FC and CHAP Info and Networking config directories
-		list := []string{"/etc/sysconfig/network-scripts/", "/etc/sysconfig/network/", "/etc/iscsi/initiatorname.iscsi", "/etc/networks", "/etc/iscsi/iscsid.conf"}
+		list := []string{"/etc/sysconfig/network-scripts/", "/etc/sysconfig/network/", "/etc/iscsi/initiatorname.iscsi", "/etc/networks", "/etc/iscsi/iscsid.conf", "/etc/nvme/hostnqn"}
 		watcher.AddWatchList(list)
 		// Start event the watcher in a separate thread.
 		go watcher.StartWatcher()
@@ -2222,18 +2236,22 @@ func (driver *Driver) nodeGetInfo() (string, error) {
 
 	var iqns []*string
 	var wwpns []*string
+	var nqns []*string
 	for _, initiator := range initiators {
 		if initiator.Type == iscsi {
 			for i := 0; i < len(initiator.Init); i++ {
 				iqns = append(iqns, &initiator.Init[i])
 			}
+		} else if initiator.Type == nvmeotcp {
+			for i := 0; i < len(initiator.Init); i++ {
+				nqns = append(nqns, &initiator.Init[i])
+			}
 		} else {
 			for i := 0; i < len(initiator.Init); i++ {
 				wwpns = append(wwpns, &initiator.Init[i])
 			}
 		}
 	}
-
 	var cidrNetworks []*string
 	for _, network := range networks {
 		log.Infof("Processing network named %s with IpV4 CIDR %s", network.Name, network.CidrNetwork)
@@ -2249,6 +2267,7 @@ func (driver *Driver) nodeGetInfo() (string, error) {
 		Iqns:     iqns,
 		Networks: cidrNetworks,
 		Wwpns:    wwpns,
+		Nqns:     nqns,
 	}
 
 	nodeID, err := driver.flavor.LoadNodeInfo(node)

pkg/driver/utils.go

@@ -126,3 +126,12 @@ func removeDataFile(dirPath string, fileName string) error {
 func isValidIP(ip string) bool {
 	return ip != "" && net.ParseIP(ip) != nil
 }
+// GetNvmeInitiator returns the NVMe host NQN as a string
+func GetNvmeInitiator() (string, error) {
+    data, err := ioutil.ReadFile("/etc/nvme/hostnqn")
+    if err != nil {
+        return "", err
+    }
+    nqn := strings.TrimSpace(string(data))
+    return nqn, nil
+}

pkg/flavor/kubernetes/flavor.go

@@ -239,13 +239,18 @@ func (flavor *Flavor) LoadNodeInfo(node *model.Node) (string, error) {
 			nodeInfo.Spec.WWPNs = wwpnsFromNode
 			updateNodeRequired = true
 		}
+		nqnsFromNode := getNqnsFromNode(node)
+		if !reflect.DeepEqual(nodeInfo.Spec.NQNs, nqnsFromNode) {
+			nodeInfo.Spec.NQNs = nqnsFromNode
+			updateNodeRequired = true
+		}
 
 		if !updateNodeRequired {
 			// no update needed to existing CRD
 			return node.UUID, nil
 		}
-		log.Infof("updating Node %s with iqns %v wwpns %v networks %v",
-			nodeInfo.Name, nodeInfo.Spec.IQNs, nodeInfo.Spec.WWPNs, nodeInfo.Spec.Networks)
+		log.Infof("updating Node %s with iqns %v wwpns %v networks %v nqns %v",
+			nodeInfo.Name, nodeInfo.Spec.IQNs, nodeInfo.Spec.WWPNs, nodeInfo.Spec.Networks, nodeInfo.Spec.NQNs)
 		_, err := flavor.crdClient.StorageV1().HPENodeInfos().Update(nodeInfo)
 		if err != nil {
 			log.Errorf("Error updating the node %s - %s\n", nodeInfo.Name, err.Error())
@@ -262,6 +267,7 @@ func (flavor *Flavor) LoadNodeInfo(node *model.Node) (string, error) {
 				IQNs:     getIqnsFromNode(node),
 				Networks: getNetworksFromNode(node),
 				WWPNs:    getWwpnsFromNode(node),
+				NQNs:     getNqnsFromNode(node),
 			},
 		}
 
@@ -303,6 +309,14 @@ func getNetworksFromNode(node *model.Node) []string {
 	return networks
 }
 
+func getNqnsFromNode(node *model.Node) []string {
+    var nqns []string
+    for i := 0; i < len(node.Nqns); i++ {
+        nqns = append(nqns, *node.Nqns[i])
+    }
+    return nqns
+}
+
 // UnloadNodeInfo remove the HPENodeInfo from the list of CRDs
 func (flavor *Flavor) UnloadNodeInfo() {
 	log.Tracef(">>>>>> UnloadNodeInfo with name %s", flavor.nodeName)
@@ -353,14 +367,19 @@ func (flavor *Flavor) GetNodeInfo(nodeID string) (*model.Node, error) {
 			for i := range wwpns {
 				wwpns[i] = &nodeInfo.Spec.WWPNs[i]
 			}
+			nqns := make([]*string, len(nodeInfo.Spec.NQNs))
+			for i := range nqns {
+				nqns[i] = &nodeInfo.Spec.NQNs[i]
+			}
 			node := &model.Node{
 				Name:     nodeInfo.ObjectMeta.Name,
 				UUID:     nodeInfo.Spec.UUID,
 				Iqns:     iqns,
 				Networks: networks,
 				Wwpns:    wwpns,
+				Nqns:     nqns,
 			}
-
+			log.Tracef("HPE Node Info sent to CSP: %v", node)
 			return node, nil
 		}
 	}