Enforcing CPU and memory limits

lib/scenario_runner.py

@@ -18,6 +18,7 @@
 from pathlib import Path
 import random
 import shutil
+import math
 import yaml
 from collections import OrderedDict
 from datetime import datetime
@@ -1060,6 +1061,15 @@ def _setup_services(self):
         # This use case is when you have running containers on your host and want to benchmark some code running in them
         services = self._usage_scenario.get('services', {})
 
+        # total available memory
+        DOCKER_AVAILABLE_MEMORY = int(subprocess.check_output(['docker', 'info', '--format', '{{.MemTotal}}'], encoding='UTF-8', errors='replace').strip())
+        unassigned_memory_services = len(services)
+        unassigned_memory = DOCKER_AVAILABLE_MEMORY-1024**3 # we want to leave 1 GB free on the host / docker VM to avoid OOM situations
+
+        SYSTEM_ASSIGNABLE_CPU_COUNT = int(subprocess.check_output(['docker', 'info', '--format', '{{.NCPU}}'], encoding='UTF-8', errors='replace').strip()) -1
+        if SYSTEM_ASSIGNABLE_CPU_COUNT <= 0:
+            raise RuntimeError(f"Cannot assign docker containers to any CPU as no CPUs are available to Docker. Available CPU count: {SYSTEM_ASSIGNABLE_CPU_COUNT}")
+
         # Check if there are service dependencies defined with 'depends_on'.
         # If so, change the order of the services accordingly.
         services_ordered = self._order_services(services)
@@ -1275,15 +1285,38 @@ def _setup_services(self):
             if 'pause-after-phase' in service:
                 self.__services_to_pause_phase[service['pause-after-phase']] = self.__services_to_pause_phase.get(service['pause-after-phase'], []) + [container_name]
 
+            # apply cpuset but keep one core for GMT and metric providers free
+            # This cannot be configured via user as no knowledge of machine shall be required
+            docker_run_string.append('--cpuset-cpus')
+            docker_run_string.append(','.join(map(str, range(1,SYSTEM_ASSIGNABLE_CPU_COUNT+1)))) # range inclusive as we do not assign to 0
+
+            docker_run_string.append('--memory-swap=0') # GMT should never swap as it gives hard to interpret / non-linear performance results
+            docker_run_string.append('--memory-swappiness=0') # GMT should never swap as it gives hard to interpret / non-linear performance results
+            docker_run_string.append('--oom-score-adj=1000') # containers will be killed first so host does not OOM
+
             # wildly the docker compose spec allows deploy to be None ... thus we need to check and cannot .get()
             if 'deploy' in service and service['deploy'] is not None and (memory := service['deploy'].get('resources', {}).get('limits', {}).get('memory', None)):
-                docker_run_string.append('--memory') # value in bytes
-                docker_run_string.append(str(memory))
+                memory_bytes = utils.docker_memory_to_bytes(memory)
+                docker_run_string.append('--memory')
+                docker_run_string.append(str(memory_bytes))
                 print('Applying Memory Limit from deploy')
             elif memory := service.get('mem_limit', None): # we only need to get resources or cpus. they must align anyway
+                memory_bytes = utils.docker_memory_to_bytes(memory)
                 docker_run_string.append('--memory')
-                docker_run_string.append(str(memory))  # value in bytes e.g. "10M"
+                docker_run_string.append(str(memory_bytes))
                 print('Applying Memory Limit from services')
+            else:
+                memory_bytes = math.floor(unassigned_memory/unassigned_memory_services)
+                docker_run_string.append('--memory')
+                docker_run_string.append(str(memory_bytes))
+                if memory_bytes < 1024**3:
+                    self.__warnings.append(f"Container '{container_name}' was auto-assigned less memory than 1 GB because no more memory was available to the host. If you feel that this is too low please set memory limits manually or upgrade to a bigger host.")
+
+            if memory_bytes > unassigned_memory:
+                raise ValueError(f"You are trying to assign more memory to container {container_name} than is left available on host system and already assigned containers. Requested memory: {memory_bytes} Bytes. Left unassigned memory: {unassigned_memory} Bytes")
+            unassigned_memory -= memory_bytes
+            unassigned_memory_services -= 1
+
 
             if 'deploy' in service and service['deploy'] is not None and (cpus := service['deploy'].get('resources', {}).get('limits', {}).get('cpus', None)):
                 docker_run_string.append('--cpus') # value in cores
@@ -1293,6 +1326,11 @@ def _setup_services(self):
                 docker_run_string.append('--cpus')
                 docker_run_string.append(str(cpus)) # value in (fractional) cores
                 print('Applying CPU Limit from services')
+            else:
+                print(f"Applying total system available CPUs: {SYSTEM_ASSIGNABLE_CPU_COUNT}")
+                docker_run_string.append('--cpus')
+                docker_run_string.append(str(SYSTEM_ASSIGNABLE_CPU_COUNT))
+
 
             if 'healthcheck' in service:  # must come last
                 if 'disable' in service['healthcheck'] and service['healthcheck']['disable'] is True:

lib/system_checks.py

@@ -28,7 +28,7 @@
 
 GMT_Resources = {
     'free_disk': 1024 ** 3, # 1GB in bytes
-    'free_memory':  1024 ** 3, # 1GB in bytes
+    'free_memory': 2 * 1024 ** 3, # 2GB in bytes
 }
 
 ######## CHECK FUNCTIONS ########
@@ -78,6 +78,16 @@ def check_free_disk(*_, **__):
     free_space_bytes = psutil.disk_usage(os.path.dirname(os.path.abspath(__file__))).free
     return free_space_bytes >= GMT_Resources['free_disk']
 
+def check_available_cpus(*_, **__):
+    docker_reported_cpus = int(subprocess.check_output(['docker', 'info', '--format', '{{.NCPU}}'], encoding='UTF-8', errors='replace').strip())
+    return os.cpu_count() >= 2 and docker_reported_cpus >= 2
+
+def check_docker_cpu_availability(*_, **__):
+    if platform.system() == 'Darwin':
+        return True # no checks on macOS as docker runs in VM here with custom CPU configuration
+    docker_reported_cpus = subprocess.check_output(['docker', 'info', '--format', '{{.NCPU}}'], encoding='UTF-8', errors='replace').strip()
+    return str(os.cpu_count()) == docker_reported_cpus # not casting to int to not get unexpected 0 or 1
+
 def check_free_memory(*_, **__):
     return psutil.virtual_memory().available >= GMT_Resources['free_memory']
 
@@ -154,7 +164,9 @@ def check_steal_time(*_, **__):
     (check_ntp, Status.WARN, 'ntp', 'You have NTP time syncing active. This can create noise in runs and should be deactivated.'),
     (check_cpu_utilization, Status.WARN, '< 5% CPU utilization', 'Your system seems to be busy. Utilization is above 5%. Consider terminating some processes for a more stable measurement.'),
     (check_largest_sampling_rate, Status.WARN, 'high sampling rate', 'You have chosen at least one provider with a sampling rate > 1000 ms. That is not recommended and might lead also to longer benchmarking times due to internal extra sleeps to adjust measurement frames.'),
-    (check_free_disk, Status.ERROR, '1 GiB free hdd space', 'We recommend to free up some disk space (< 1GiB available)'),
+    (check_available_cpus, Status.ERROR, '< 2 CPUs', 'You need at least 2 CPU cores on the system (and assigned to Docker in case of macOS) to run GMT'),
+    (check_docker_cpu_availability, Status.ERROR, 'Docker CPU reporting', 'Docker reports a different amount of available CPUs than the system itself - GMT cannot handle this currently'),
+    (check_free_disk, Status.ERROR, '1 GiB free hdd space', 'You need to free up some disk space to run GMT reliably (< 1GiB available)'),
     (check_free_memory, Status.ERROR, '1 GiB free memory', 'No free memory! Please kill some programs (< 1GiB available)'),
     (check_docker_daemon, Status.ERROR, 'docker daemon', 'The docker daemon could not be reached. Are you running in rootless mode or have added yourself to the docker group? See installation: [See https://docs.green-coding.io/docs/installation/]'),
     (check_docker_host_env, Status.ERROR, 'docker host env', 'You seem to be running a rootless docker and in this case you must set the DOCKER_HOST environment variable so that the docker library we use can find the docker agent. Typically this should be DOCKER_HOST=unix:///$XDG_RUNTIME_DIR/docker.sock'),

lib/utils.py

@@ -226,3 +226,35 @@ def find_own_cgroup_name():
         if found_cgroups != 1:
             raise RuntimeError(f"Could not find GMT\'s own cgroup or found too many. Amount: {found_cgroups}")
         return lines[0].split('/')[-1].strip()
+
+def docker_memory_to_bytes(memory_value):
+    """Convert memory string with units (e.g., '50M', '2G') to bytes."""
+    """Although GMT internally works with MiB this function is for converting for docker syntax"""
+    unit_multipliers = {
+        'B': 1,        # Byte
+        'K': 1_024,    # Kilobyte
+        'M': 1_024**2, # Megabyte
+        'G': 1_024**3, # Gigabyte
+        'T': 1_024**4, # Terabyte
+    }
+
+    if isinstance(memory_value, (float, int)) or memory_value[-1].isdigit():
+        # in case of float this will round down. but since float would be pure bytes anyway
+        # we must floor the value in any case as no less than a byte can be accounted
+        return int(memory_value)
+
+    # although not specified in the docker specification values like 10m and also 10MB are allowed.
+    # so if we see an additional B we remove it at the end
+    if memory_value[-1] == 'b' or memory_value[-1] == 'B':
+        memory_value = memory_value[:-1]
+
+    if memory_value[-1].isdigit():
+        unit = 'B'
+        num = memory_value
+    else:
+        num, unit = float(memory_value[:-1]), memory_value[-1].upper()
+
+    if unit in unit_multipliers:
+        return int(num * unit_multipliers[unit])
+
+    raise ValueError(f"Unrecognized memory unit: {unit}")

optimization_providers/resources/utilization.py

@@ -2,6 +2,7 @@
 
 from optimization_providers.base import Criticality, register_reporter
 from lib import error_helpers
+from lib import utils
 
 REPORTER_NAME = 'utilization'
 REPORTER_ICON = 'tachometer alternate'
@@ -10,33 +11,14 @@
 MAX_CPU_UTIL = 90 #%
 MIN_CPU_UTIL = 50 #%
 
-def memory_to_bytes(memory_str):
-    """Convert memory string with units (e.g., '50M', '2G') to bytes."""
-    unit_multipliers = {
-        'K': 1_000,         # Kilobyte
-        'M': 1_000_000,     # Megabyte
-        'G': 1_000_000_000, # Gigabyte
-        'T': 1_000_000_000, # Terabyte
-    }
-
-    if isinstance(memory_str, int) or memory_str[-1].isdigit():
-        return int(memory_str)
-
-    num, unit = float(memory_str[:-1]), memory_str[-1].upper()
-
-    if unit in unit_multipliers:
-        return int(num * unit_multipliers[unit])
-
-    raise ValueError(f"Unrecognized memory unit: {unit}")
-
 # pylint: disable=unused-argument
 @register_reporter('container_memory_utilization', Criticality.INFO, REPORTER_NAME, REPORTER_ICON, req_providers =['MemoryUsedCgroupContainerProvider'])
 def container_memory_utilization(self, run, measurements, repo_path, network, notes, phases):
 
     mem = {}
     for s, d in run.get('usage_scenario').get('services').items():
         if x := d.get('deploy', {}).get('resources', {}).get('limits', {}).get('memory', None):
-            mem[s] = memory_to_bytes(x)
+            mem[s] = utils.docker_memory_to_bytes(x)
 
     for service, measurement_stats in phases['data']['[RUNTIME]']['data']['memory_used_cgroup_container']['data'].items():
         if not service in mem: