Add comments about memory size; fix bug in controller readiness check

Author: hunhoffe

kernel/src/arch/x86_64/rackscale/controller.rs

@@ -64,12 +64,11 @@ pub(crate) fn run() {
         unreachable!("No supported transport layer specified in kernel argument");
     };
 
+    ClientReadyCount.fetch_add(1, Ordering::SeqCst);
+
     // Wait for all clients to connect before fulfilling any RPCs.
     while !DCMServerReady.load(Ordering::SeqCst) {}
 
-    // Don't modify this line without modifying testutils/rackscale_runner.rs
-    log::warn!("CONTROLLER READY");
-
     server
         .add_client(&CLIENT_REGISTRAR)
         .expect("Failed to accept client");
@@ -136,6 +135,15 @@ pub(crate) fn poll_interface() {
     let _ = server.add_client(&DCM_CLIENT_REGISTRAR).unwrap();
     log::info!("Added DCM server RPC client!");
 
+    // Wait for all clients to initialize before signaling the all-clear
+    while ClientReadyCount.load(Ordering::SeqCst) != (*crate::environment::NUM_MACHINES - 1) as u64
+    {
+    }
+    ClientReadyCount.store(0, Ordering::SeqCst);
+
+    // Don't modify this line without modifying testutils/rackscale_runner.rs
+    log::warn!("CONTROLLER READY");
+
     // Now that server is ready, the clients may be accepted.
     DCMServerReady.store(true, Ordering::SeqCst);
 

kernel/tests/s11_rackscale_benchmarks.rs

@@ -26,7 +26,6 @@ fn s11_rackscale_shmem_fxmark_benchmark() {
 }
 
 #[test]
-#[ignore]
 #[cfg(not(feature = "baremetal"))]
 fn s11_rackscale_ethernet_fxmark_benchmark() {
     rackscale_fxmark_benchmark(RackscaleTransport::Ethernet);
@@ -137,6 +136,7 @@ fn rackscale_fxmark_benchmark(transport: RackscaleTransport) {
         if is_smoke {
             8192
         } else {
+            // Memory must also be divisible by number of nodes, which could be 1, 2, 3, or 4
             2048 * (((num_cores + 3 - 1) / 3) * 3)
         }
     }
@@ -438,6 +438,7 @@ fn s11_rackscale_shmem_leveldb_benchmark() {
         if is_smoke {
             8192
         } else {
+            // Memory must also be divisible by number of nodes, which could be 1, 2, 3, or 4
             2048 * (((((num_cores + 1) / 2) + 3 - 1) / 3) * 3)
         }
     }
@@ -632,6 +633,7 @@ fn rackscale_memcached_benchmark(transport: RackscaleTransport) {
         if is_smoke {
             8192
         } else {
+            // Memory must also be divisible by number of nodes, which could be 1, 2, 3, or 4
             2048 * (((((num_cores + 1) / 2) + 3 - 1) / 3) * 3)
         }
     }
@@ -715,6 +717,7 @@ fn rackscale_monetdb_benchmark(transport: RackscaleTransport) {
         if is_smoke {
             8192
         } else {
+            // Memory must also be divisible by number of nodes, which could be 1, 2, 3, or 4
             2048 * (((((num_cores + 1) / 2) + 3 - 1) / 3) * 3)
         }
     }

kernel/testutils/src/rackscale_runner.rs

@@ -539,10 +539,12 @@ impl<T: Clone + Send + 'static> RackscaleBench<T> {
             let cores_per_client = total_cores / num_clients;
 
             // Break if not enough total cores for the controller, or if we would have to split controller across nodes to make it fit
-            if !is_baseline
-                && ((total_cores + num_clients + 1 > machine.max_cores())
-                    || (num_clients == machine.max_numa_nodes()
-                        && cores_per_client + num_clients + 1 > total_cores_per_node))
+            // We want controller to have it's own socket, so if it's not a 1 socket machine, break when there's equal number of clients
+            // to numa nodes.
+            if total_cores + num_clients + 1 > machine.max_cores()
+                || num_clients == machine.max_numa_nodes()
+                    && cores_per_client + num_clients + 1 > total_cores_per_node
+                || num_clients == max_numa && max_numa > 1
             {
                 break;
             }