Adds multi-shmem region and affinity shmem support

Author: hunhoffe

doc/src/development/Testing.md

@@ -105,6 +105,16 @@ Benchmarks are named as such:
 * ```s10_*```: User-space applications benchmarks
 * ```s11_*```: Rackscale (distributed) benchmarks
 
+The ```s11_*``` benchmarks may be configured with two features:
+* ```baseline```: Runs NrOS configured similarly to rackscale, for comparison
+* ```affinity-shmem```: Runs the ```ivshmem-server``` using shmem with NUMA affinity.
+  This option requires preconfiguring hugetlbfs with
+  ```sudo hugeadm --create-global-mounts```,
+  having a kernel with 2MB huge pages enabled, and then also adding 1024 2MB pages per
+  node, with a command like:
+  ```echo <page-num> | sudo numactl -m <node-num> tee -a /proc/sys/vm/nr_hugepages_mempolicy```
+  The number of huge pages per node may be verified with ```numastat -m```.
+ 
 ## Network
 
 nrk has support for three network interfaces at the moment: virtio, e1000 and

doc/src/environment/Environment.md

@@ -15,47 +15,25 @@ already uncommented. Then, run the following commands:
 
 ```bash
 sudo apt update
-sudo apt install build-essential libpmem-dev libdaxctl-dev ninja-build
+sudo apt install build-essential libpmem-dev libdaxctl-dev ninja-build flex bison
 apt source qemu
 sudo apt build-dep qemu
+
+For non-rackscale mode, execute the following:
+```
 wget https://download.qemu.org/qemu-6.0.0.tar.xz
 tar xvJf qemu-6.0.0.tar.xz
 cd qemu-6.0.0
 ```
 
-If you are planning on running the rackscale NrOS build, you'll need to modify
-the ivshmem server code. Open ```contrib/ivshmem-server/ivshmem-server.c```.
-Go to the function```ivshmem_server_ftruncate```. Replace it with:
-```c
-static int
-ivshmem_server_ftruncate(int fd, uint64_t shmsize)
-{
-    int ret;
-    struct stat mapstat;
-
-    /* align shmsize to next power of 2 */
-    shmsize = pow2ceil(shmsize);
-
-    if (fstat(fd, &mapstat) != -1 && mapstat.st_size == shmsize) {
-        return 0;
-    }
-
-    /*
-     * This is a do-while loop in case
-     * shmsize > IVSHMEM_SERVER_MAX_HUGEPAGE_SIZE
-     */
-    do {
-        ret = ftruncate64(fd, shmsize);
-        if (ret == 0) {
-            return ret;
-        }
-        shmsize *= 2;
-    } while (shmsize <= IVSHMEM_SERVER_MAX_HUGEPAGE_SIZE);
-
-    return -1;
-}
+For rackscale mode, instead run:
+```
+git clone https://github.com/hunhoffe/qemu.git qemu-6.0.0.tar.xz
+cd qemu-6.0.0
+git checkout --track origin/dev/ivshmem-numa
 ```
 
+For either option, build and install with:
 ```bash
 ./configure --enable-rdma --enable-libpmem
 make -j 28

kernel/run.py

@@ -38,7 +38,7 @@ def exception_handler(exception_type, exception, traceback):
                           "-Z", "build-std-features=compiler-builtins-mem"]
 ARCH = "x86_64"
 
-IVSHMEM_DEVICE_ADDR = 6
+IVSHMEM_DEVICE_ADDR = 10
 
 def get_network_config(workers):
     """
@@ -137,10 +137,10 @@ def get_network_config(workers):
 parser.add_argument('--kgdb', action="store_true",
                     help="Use the GDB remote debugger to connect to the kernel")
 parser.add_argument('--qemu-ivshmem',
-                    type=int,
+                    type=str,
                     help="Enable the ivshmem device with the size in MiB.",
                     required=False,
-                    default=0)
+                    default="")
 parser.add_argument('--qemu-shmem-path',
                     type=str,
                     help="Provide shared memory file path.",
@@ -407,12 +407,17 @@ def run_qemu(args):
                           'isa-debug-exit,iobase=0xf4,iosize=0x04']
 
     if args.qemu_ivshmem:
-        # If you change the the device addr, you must change it in vibrio::rumprt::dev
-        qemu_default_args += ['-device', 'ivshmem-doorbell,vectors=3,chardev=id,addr={}'.format(IVSHMEM_DEVICE_ADDR)]
-        qemu_default_args += [
-            '-chardev',
-            'socket,path={},id=id'.format(args.qemu_shmem_path)
-        ]
+        sizes = [int(s.strip()) for s in args.qemu_ivshmem.split(",")]
+        names = [s.strip() for s in args.qemu_shmem_path.split(",")]
+        assert len(sizes) == len(names)
+
+        for i in range(len(sizes)):
+            # TODO: Only device IVSHMEM_DEVICE_ADDR will handle interrupts?
+            qemu_default_args += ['-device', 'ivshmem-doorbell,vectors=3,chardev=id{},addr={}'.format(i, IVSHMEM_DEVICE_ADDR + i)]
+            qemu_default_args += [
+                '-chardev',
+                'socket,path={},id=id{}'.format(names[i], i)
+            ]
 
     # Enable networking:
     mac, tap = NETWORK_CONFIG[args.tap]['mac'], args.tap

kernel/src/arch/x86_64/mod.rs

@@ -307,6 +307,9 @@ fn _start(argc: isize, _argv: *const *const u8) -> isize {
     // form by klogger::init above, but now we do it for more ports)
     debug::init();
 
+    use crate::transport::shmem::SHMEM_INITIALIZED;
+    lazy_static::initialize(&SHMEM_INITIALIZED);
+
     // Parse memory map provided by UEFI, create an initial emergency memory
     // manager with a little bit of memory so we can do some early allocations.
     let (emanager, memory_regions) = memory::process_uefi_memory_regions();
@@ -396,8 +399,8 @@ fn _start(argc: isize, _argv: *const *const u8) -> isize {
     #[cfg(feature = "rackscale")]
     {
         {
-            use crate::transport::shmem::SHMEM_DEVICE;
-            lazy_static::initialize(&SHMEM_DEVICE);
+            use crate::transport::shmem::SHMEM;
+            lazy_static::initialize(&SHMEM);
 
             if crate::CMDLINE
                 .get()
@@ -412,7 +415,7 @@ fn _start(argc: isize, _argv: *const *const u8) -> isize {
                 };
 
                 // Setup to receive interrupts
-                SHMEM_DEVICE.enable_msix_vector(
+                SHMEM.devices[0].enable_msix_vector(
                     REMOTE_TLB_WORK_PENDING_SHMEM_VECTOR as usize,
                     0,
                     REMOTE_TLB_WORK_PENDING_VECTOR,

kernel/src/arch/x86_64/process.rs

@@ -88,7 +88,7 @@ lazy_static! {
             let pcm = per_core_mem();
             pcm.set_mem_affinity(SHARED_AFFINITY).expect("Can't change affinity");
         } else {
-            // Get location of the logs from the controller, who will created them in shared memory
+            // Get location of the logs from the controller, who will have created them in shared memory
             use crate::arch::rackscale::get_shmem_structure::{rpc_get_shmem_structure, ShmemStructure};
 
             let mut log_ptrs = [0u64; MAX_PROCESSES];

kernel/src/arch/x86_64/rackscale/get_shmem_frames.rs

@@ -20,7 +20,7 @@ use crate::error::{KError, KResult};
 use crate::memory::backends::PhysicalPageProvider;
 use crate::memory::Frame;
 use crate::process::Pid;
-use crate::transport::shmem::{ShmemRegion, SHMEM_DEVICE};
+use crate::transport::shmem::ShmemRegion;
 
 use crate::memory::backends::AllocatorStatistics;
 

kernel/src/arch/x86_64/rackscale/processops/allocate_physical.rs

@@ -20,7 +20,7 @@ use crate::memory::backends::PhysicalPageProvider;
 use crate::memory::{Frame, PAddr, BASE_PAGE_SIZE};
 use crate::nrproc::NrProcess;
 use crate::process::Pid;
-use crate::transport::shmem::{is_shmem_frame, ShmemRegion};
+use crate::transport::shmem::ShmemRegion;
 
 #[derive(Debug)]
 pub(crate) struct AllocatePhysicalReq {
@@ -65,10 +65,6 @@ pub(crate) fn rpc_allocate_physical(pid: Pid, size: u64, affinity: u64) -> KResu
                 size,
             };
             let frame = shmem_region.get_frame(0);
-
-            // TODO(rackscale performance): should be debug assert
-            assert!(is_shmem_frame(frame, false, false));
-
             let fid = NrProcess::<Ring3Process>::allocate_frame_to_process(pid, frame)?;
 
             // Add frame mapping to client map
@@ -128,9 +124,6 @@ pub(crate) fn handle_allocate_physical(
             .expect("DCM should ensure we have a frame to allocate here.")
     };
 
-    // TODO(rackscale performance): should be debug assert
-    assert!(is_shmem_frame(frame, false, false));
-
     construct_ret(hdr, payload, Ok((dcm_node_id, frame.base.as_u64())));
     Ok(state)
 }

kernel/src/arch/x86_64/rackscale/syscalls.rs

@@ -16,7 +16,6 @@ use crate::process::{KernArcBuffer, UserSlice};
 use crate::syscalls::{
     FsDispatch, ProcessDispatch, SystemCallDispatch, SystemDispatch, VSpaceDispatch,
 };
-use crate::transport::shmem::is_shmem_frame;
 
 use super::super::syscall::{Arch86SystemCall, Arch86SystemDispatch, Arch86VSpaceDispatch};
 use super::fileops::close::rpc_close;
@@ -81,9 +80,6 @@ impl VSpaceDispatch<u64> for Arch86LwkSystemCall {
                     .allocate_base_page()
                     .expect("We ensure there is capabity in the FrameCacheBase above");
 
-                // TODO(rackscale performance): should be debug assert
-                assert!(is_shmem_frame(frame, false, false));
-
                 initial_base_frames
                     .try_push(frame)
                     .expect("Can't fail see `try_with_capacity`");
@@ -102,8 +98,6 @@ impl VSpaceDispatch<u64> for Arch86LwkSystemCall {
             let mut allocated_frames = rpc_get_shmem_frames(Some(pid), total_needed_large_pages)?;
 
             for i in 0..lp {
-                // TODO(rackscale performance): should be debug assert
-                assert!(is_shmem_frame(allocated_frames[i], false, false));
                 total_len += allocated_frames[i].size;
                 unsafe { allocated_frames[i].zero() };
                 frames
@@ -116,18 +110,12 @@ impl VSpaceDispatch<u64> for Arch86LwkSystemCall {
 
             // Grow base pages
             if total_needed_base_pages > 0 {
-                // TODO(rackscale performance): should be debug assert
-                assert!(is_shmem_frame(allocated_frames[lp], false, false));
-
                 let mut base_page_iter = allocated_frames[lp].into_iter();
                 for _i in 0..total_needed_base_pages {
                     let mut frame = base_page_iter
                         .next()
                         .expect("needed base frames should all fit within one large frame");
 
-                    // TODO(rackscale performance): should be debug assert
-                    assert!(is_shmem_frame(frame, false, false));
-
                     total_len += frame.size;
                     unsafe { frame.zero() };
                     if paddr.is_none() {
@@ -151,9 +139,6 @@ impl VSpaceDispatch<u64> for Arch86LwkSystemCall {
                         .next()
                         .expect("needed base frames should all fit within one large frame");
 
-                    // TODO(rackscale performance): should be debug assert
-                    assert!(is_shmem_frame(frame, false, false));
-
                     per_process_bp_cache
                         .grow_base_pages(&[frame])
                         .expect("We ensure not to overfill the FrameCacheBase above.");

kernel/src/arch/x86_64/tlb.rs

@@ -419,7 +419,7 @@ pub(crate) fn remote_shootdown(handles: Vec<TlbFlushHandle>) {
     use crate::arch::irq::REMOTE_TLB_WORK_PENDING_SHMEM_VECTOR;
     use crate::arch::kcb::per_core_mem;
     use crate::memory::SHARED_AFFINITY;
-    use crate::transport::shmem::SHMEM_DEVICE;
+    use crate::transport::shmem::SHMEM;
 
     let my_mtid = kpi::system::mtid_from_gtid(*crate::environment::CORE_ID);
     let my_mid = kpi::system::mid_from_gtid(*crate::environment::CORE_ID);
@@ -468,7 +468,8 @@ pub(crate) fn remote_shootdown(handles: Vec<TlbFlushHandle>) {
                 i,
                 handles[i].core_map.is_empty()
             );
-            SHMEM_DEVICE.set_doorbell(REMOTE_TLB_WORK_PENDING_SHMEM_VECTOR, i.try_into().unwrap());
+            SHMEM.devices[0]
+                .set_doorbell(REMOTE_TLB_WORK_PENDING_SHMEM_VECTOR, i.try_into().unwrap());
         }
     }
 

kernel/src/integration_tests.rs

@@ -765,37 +765,41 @@ fn vmxnet_smoltcp() {
     shutdown(ExitReason::Ok);
 }
 
-/// Write and test the content on a shared-mem device.
-pub(crate) const BUFFER_CONTENT: u8 = 0xb;
-
 /// Test cxl device in the kernel.
 #[cfg(all(feature = "integration-test", target_arch = "x86_64"))]
 pub(crate) fn cxl_write() {
     use crate::memory::KERNEL_BASE;
-    use crate::transport::shmem::SHMEM_DEVICE;
+    use crate::transport::shmem::SHMEM;
 
-    lazy_static::initialize(&SHMEM_DEVICE);
+    lazy_static::initialize(&SHMEM);
 
-    for i in 0..SHMEM_DEVICE.region.size {
-        let region = (SHMEM_DEVICE.region.base + KERNEL_BASE + i as u64) as *mut u8;
-        unsafe { core::ptr::write(region, BUFFER_CONTENT) };
+    let mut buffer_content: u8 = 0x0;
+    for device in SHMEM.devices.iter() {
+        buffer_content += 1;
+        for i in 0..(device.region.size / 1024) {
+            let region = (device.region.base + KERNEL_BASE + (i * 1024) as u64) as *mut u8;
+            unsafe { core::ptr::write(region, buffer_content) };
+        }
     }
-
     shutdown(ExitReason::Ok);
 }
 
 /// Test cxl device in the kernel.
 #[cfg(all(feature = "integration-test", target_arch = "x86_64"))]
 pub(crate) fn cxl_read() {
     use crate::memory::KERNEL_BASE;
-    use crate::transport::shmem::SHMEM_DEVICE;
+    use crate::transport::shmem::SHMEM;
 
-    lazy_static::initialize(&SHMEM_DEVICE);
+    lazy_static::initialize(&SHMEM);
 
-    for i in 0..SHMEM_DEVICE.region.size {
-        let region = (SHMEM_DEVICE.region.base + KERNEL_BASE + i as u64) as *mut u8;
-        let read = unsafe { core::ptr::read(region) };
-        assert_eq!(read, BUFFER_CONTENT);
+    let mut buffer_content: u8 = 0x0;
+    for device in SHMEM.devices.iter() {
+        buffer_content += 1;
+        for i in 0..(device.region.size / 1024) {
+            let region = (device.region.base + KERNEL_BASE + (i * 1024) as u64) as *mut u8;
+            let read = unsafe { core::ptr::read(region) };
+            assert_eq!(read, buffer_content);
+        }
     }
 
     shutdown(ExitReason::Ok);
@@ -805,18 +809,21 @@ pub(crate) fn cxl_read() {
 #[cfg(all(feature = "integration-test", target_arch = "x86_64"))]
 pub(crate) fn shmem_interruptor() {
     use crate::arch::irq::REMOTE_TLB_WORK_PENDING_SHMEM_VECTOR;
-    use crate::transport::shmem::SHMEM_DEVICE;
+    use crate::transport::shmem::SHMEM;
 
-    lazy_static::initialize(&SHMEM_DEVICE);
+    lazy_static::initialize(&SHMEM);
     {
         // The ivshmem server allocates IDs consecutively, so we'll assume interruptee is
         // current_id - 1
         log::info!(
             "Sending shmem interrupt to: {:?} on vector {:?}",
-            SHMEM_DEVICE.id - 1,
+            SHMEM.devices[0].id - 1,
             1
         );
-        SHMEM_DEVICE.set_doorbell(REMOTE_TLB_WORK_PENDING_SHMEM_VECTOR, SHMEM_DEVICE.id - 1);
+        SHMEM.devices[0].set_doorbell(
+            REMOTE_TLB_WORK_PENDING_SHMEM_VECTOR,
+            SHMEM.devices[0].id - 1,
+        );
     }
     shutdown(ExitReason::Ok);
 }
@@ -828,11 +835,11 @@ pub(crate) fn shmem_interruptee() {
     use core::time::Duration;
 
     use crate::arch::irq::{REMOTE_TLB_WORK_PENDING_SHMEM_VECTOR, REMOTE_TLB_WORK_PENDING_VECTOR};
-    use crate::transport::shmem::SHMEM_DEVICE;
+    use crate::transport::shmem::SHMEM;
 
-    lazy_static::initialize(&SHMEM_DEVICE);
+    lazy_static::initialize(&SHMEM);
 
-    SHMEM_DEVICE.enable_msix_vector(
+    SHMEM.get_interrupt_device().unwrap().enable_msix_vector(
         REMOTE_TLB_WORK_PENDING_SHMEM_VECTOR as usize,
         0,
         REMOTE_TLB_WORK_PENDING_VECTOR,