adjust for 2P run.

Author: davidz-ampere

benchmarks/run.py

@@ -11,18 +11,18 @@ def get_file_dir():
     return os.path.dirname(os.path.realpath(__file__))
 
 
-def docker_init():
+def docker_init(node):
     tag = "amperecomputingai/llama.cpp:2.0.0"
     if subprocess.run(
             ["docker", "pull", tag]).returncode != 0:
         print("Docker pull process failed!")
         sys.exit(1)
-    container_name = "llama_benchmark"
+    container_name = f"llama_benchmark_n{node}"
     subprocess.run(["docker", "rm", "-f", container_name])
     memory = (psutil.virtual_memory().total >> 30) - 30  # leave 30GB for OS
     assert memory > 10, "less than 10GB of memory available on the system for llama.cpp"
     if subprocess.run(
-            ["docker", "run", "--privileged=true", "--name", container_name, "-d", "-m", f"{str(memory)}g", "-v",
+            ["docker", "run", "--privileged=true", "--cpuset-mems", f"{str(node)}", "--name", container_name, "-d", "-m", f"{str(memory)}g", "-v",
              f"{get_file_dir()}:/runner", "--entrypoint", "/bin/bash", "-it", tag]).returncode != 0:
         print("Docker run process failed!")
         sys.exit(1)
@@ -106,12 +106,16 @@ def parse_args():
     parser.add_argument("--timeout",
                         type=float, default=900,
                         help="timeout to apply per single benchmark case")
+    parser.add_argument("-n", "--numa",
+                        type=int, default=0,
+                        help="numa mode of the docker container")
+
     return parser.parse_args()
 
 
 def main():
     args = parse_args()
-    benchmark(docker_init(), args)
+    benchmark(docker_init(args.numa), args)
 
 
 if __name__ == "__main__":

benchmarks/run_2p.sh

@@ -0,0 +1,24 @@
+set -e
+
+sync
+echo 3 | sudo tee /proc/sys/vm/drop_caches
+echo 1 | sudo tee /proc/sys/vm/swappiness
+echo 8 | sudo tee /proc/sys/vm/dirty_ratio
+echo 1 | sudo tee /proc/sys/vm/zone_reclaim_mode
+echo 0 | sudo tee /proc/sys/kernel/numa_balancing
+
+VAR_PAGESIZE=$(getconf PAGESIZE)
+if [ $VAR_PAGESIZE = 4096 ]; then
+  echo always | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
+elif [ $VAR_PAGESIZE = 65536 ]; then
+  echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
+fi
+
+# Warm up
+python3 run.py -m DeepSeek-R1-Distill-Qwen-7B-Q8R16_n0.gguf -t 80 -b 1 -p 512 -r 0-79 -n 0
+python3 run.py -m DeepSeek-R1-Distill-Qwen-7B-Q8R16_n1.gguf -t 80 -b 1 -p 512 -r 80-159 -n 1
+
+# Run  
+python3 run.py -m DeepSeek-R1-Distill-Qwen-7B-Q8R16_n0.gguf -t 80 64 48 40 32 24 20 16 12 10 8 -b 1 2 4 8 -p 512 -r 0-79 -n 0 &
+python3 run.py -m DeepSeek-R1-Distill-Qwen-7B-Q8R16_n1.gguf -t 80 64 48 40 32 24 20 16 12 10 8 -b 1 2 4 8 -p 512 -r 80-159 -n 1 &
+wait

benchmarks/utils/benchmark.py

@@ -93,8 +93,9 @@ def summarize_results(logs_dir, args, start, finish):
                 ["n_proc", "n_threads", "batch_size", "prompt_size", "output_tokens", "pp_throughput_tps",
                  "pp_avg_latency_sec", "tg_throughput_tps", "tg_avg_latency_sec", "pp+tg_throughput_tps", "concurrency", "start", "finish"])
         writer.writerow(
-            [args.num_processes, args.num_threads, args.batch_size, args.prompt_size, TOKENS, pp_throughput,
-             avg_pp_latency, tg_throughput, avg_tg_latency, avg_total_speed, args.batch_size * args.num_processes, start, finish])
+            [args.num_processes, args.num_threads, args.batch_size, args.prompt_size, TOKENS, f"{pp_throughput:.3f}",
+             f"{avg_pp_latency:.3f}", f"{tg_throughput:.3f}", f"{avg_tg_latency:.3f}", f"{avg_total_speed:.3f}", args.batch_size * args.num_processes, f"{start:.3f}", f"{finish:.3f}"])
+
     print(f"Result saved in {results_filename}")
 
 
@@ -118,12 +119,12 @@ def main():
         logfile = f"{logs_dir}/log_{n}"
         if os.path.exists("/llm/batched-bench"):
             # command-line for v1
-            cmd = ["numactl", f"--physcpubind={gen_threads_config(args.num_threads, n)}",
+            cmd = ["numactl", f"--physcpubind={gen_threads_config(args.num_threads, n)}", "--localalloc",
                    "/llm/batched-bench", args.model, str(args.kv_cache), "2048", "512", "0", "0", "0", str(args.prompt_size), str(TOKENS),
                    str(args.batch_size), str(args.num_threads)]
         elif os.path.exists("/llm/llama-batched-bench"):
             # command-line for v2
-            cmd = ["numactl", f"--physcpubind={gen_threads_config(args.num_threads, n)}",
+            cmd = ["numactl", f"--physcpubind={gen_threads_config(args.num_threads, n)}", "--localalloc",
                    "/llm/llama-batched-bench", "-m", args.model, "-c", str(args.kv_cache), "-b", "2048", "-ub", "512", "-npp", str(args.prompt_size), "-ntg", str(TOKENS),
                    "-npl", str(args.batch_size), "-t", str(args.num_threads), "-tb", str(args.num_threads), "-td", str(args.num_threads)]
         else: