Skip to content

Commit 7378f18

Browse files
fcostaoliveirafcostaoliveira
committed
Added caching and session-storage benchmarks
1 parent 85a3492 commit 7378f18

13 files changed

+1244
-42
lines changed

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "redis-benchmarks-specification"
3-
version = "0.1.302"
3+
version = "0.1.309"
44
description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
55
authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
66
readme = "Readme.md"

‎redis_benchmarks_specification/__compare__/args.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ def create_compare_arguments(parser):
7777
parser.add_argument("--comparison_deployment_name", type=str, default="")
7878
parser.add_argument("--metric_name", type=str, default="ALL_STATS.Totals.Ops/sec")
7979
parser.add_argument(
80-
"--running_platform", type=str, default="intel64-ubuntu22.04-redis-icx1"
80+
"--running_platform", type=str, default=None
8181
)
8282
parser.add_argument(
8383
"--running_platform_baseline",

‎redis_benchmarks_specification/__compare__/compare.py‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1182,7 +1182,7 @@ def from_rts_to_regression_table(
11821182
filters_baseline.append("deployment_name={}".format(baseline_deployment_name))
11831183
if baseline_github_org != "":
11841184
filters_baseline.append(f"github_org={baseline_github_org}")
1185-
if running_platform_baseline is not None:
1185+
if running_platform_baseline is not None and running_platform_baseline != "":
11861186
filters_baseline.append(
11871187
"running_platform={}".format(running_platform_baseline)
11881188
)
@@ -1202,7 +1202,7 @@ def from_rts_to_regression_table(
12021202
filters_baseline.append("hash==")
12031203
if "hash" not in by_str_comparison:
12041204
filters_comparison.append("hash==")
1205-
if running_platform_comparison is not None:
1205+
if running_platform_comparison is not None and running_platform_comparison != "":
12061206
filters_comparison.append(
12071207
"running_platform={}".format(running_platform_comparison)
12081208
)
@@ -1538,12 +1538,18 @@ def from_rts_to_regression_table(
15381538

15391539
def get_only_Totals(baseline_timeseries):
15401540
logging.warning("\t\tTime-series: {}".format(", ".join(baseline_timeseries)))
1541-
logging.info("Checking if Totals will reduce timeseries.")
1541+
logging.info(f"Checking if Totals will reduce timeseries. initial len={len(baseline_timeseries)}")
15421542
new_base = []
15431543
for ts_name in baseline_timeseries:
1544+
if "io-threads" in ts_name:
1545+
continue
1546+
if "oss-cluster" in ts_name:
1547+
continue
15441548
if "Totals" in ts_name:
15451549
new_base.append(ts_name)
15461550
baseline_timeseries = new_base
1551+
logging.info(f" final len={len(baseline_timeseries)}")
1552+
15471553
return baseline_timeseries
15481554

15491555

‎redis_benchmarks_specification/__runner__/runner.py‎

Lines changed: 54 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1393,17 +1393,27 @@ def delete_temporary_files(
13931393
ssl_cert_reqs = "required"
13941394
if tls_skip_verify:
13951395
ssl_cert_reqs = None
1396-
r = redis.StrictRedis(
1397-
host=host,
1398-
port=port,
1399-
password=password,
1400-
ssl=tls_enabled,
1401-
ssl_cert_reqs=ssl_cert_reqs,
1402-
ssl_keyfile=tls_key,
1403-
ssl_certfile=tls_cert,
1404-
ssl_ca_certs=tls_cacert,
1405-
ssl_check_hostname=False,
1406-
)
1396+
1397+
# Build Redis connection parameters
1398+
redis_params = {
1399+
"host": host,
1400+
"port": port,
1401+
"password": password,
1402+
"ssl": tls_enabled,
1403+
"ssl_cert_reqs": ssl_cert_reqs,
1404+
"ssl_check_hostname": False,
1405+
}
1406+
1407+
# Only add SSL certificate parameters if they are provided
1408+
if tls_enabled:
1409+
if tls_key is not None and tls_key != "":
1410+
redis_params["ssl_keyfile"] = tls_key
1411+
if tls_cert is not None and tls_cert != "":
1412+
redis_params["ssl_certfile"] = tls_cert
1413+
if tls_cacert is not None and tls_cacert != "":
1414+
redis_params["ssl_ca_certs"] = tls_cacert
1415+
1416+
r = redis.StrictRedis(**redis_params)
14071417
setup_name = "oss-standalone"
14081418
r.ping()
14091419

@@ -1456,17 +1466,26 @@ def delete_temporary_files(
14561466
slot_network_info = slot[2]
14571467
prefered_endpoint = slot_network_info[0]
14581468
prefered_port = slot_network_info[1]
1459-
shard_conn = redis.StrictRedis(
1460-
host=prefered_endpoint,
1461-
port=prefered_port,
1462-
password=password,
1463-
ssl=tls_enabled,
1464-
ssl_cert_reqs=ssl_cert_reqs,
1465-
ssl_keyfile=tls_key,
1466-
ssl_certfile=tls_cert,
1467-
ssl_ca_certs=tls_cacert,
1468-
ssl_check_hostname=False,
1469-
)
1469+
# Build shard connection parameters
1470+
shard_params = {
1471+
"host": prefered_endpoint,
1472+
"port": prefered_port,
1473+
"password": password,
1474+
"ssl": tls_enabled,
1475+
"ssl_cert_reqs": ssl_cert_reqs,
1476+
"ssl_check_hostname": False,
1477+
}
1478+
1479+
# Only add SSL certificate parameters if they are provided
1480+
if tls_enabled:
1481+
if tls_key is not None and tls_key != "":
1482+
shard_params["ssl_keyfile"] = tls_key
1483+
if tls_cert is not None and tls_cert != "":
1484+
shard_params["ssl_certfile"] = tls_cert
1485+
if tls_cacert is not None and tls_cacert != "":
1486+
shard_params["ssl_ca_certs"] = tls_cacert
1487+
1488+
shard_conn = redis.StrictRedis(**shard_params)
14701489
redis_conns.append(shard_conn)
14711490
logging.info(
14721491
"There are a total of {} shards".format(len(redis_conns))
@@ -1475,8 +1494,12 @@ def delete_temporary_files(
14751494

14761495
redis_pids = []
14771496
for conn in redis_conns:
1478-
redis_pid = conn.info()["process_id"]
1479-
redis_pids.append(redis_pid)
1497+
redis_info = conn.info()
1498+
redis_pid = redis_info.get("process_id")
1499+
if redis_pid is not None:
1500+
redis_pids.append(redis_pid)
1501+
else:
1502+
logging.warning("Redis process_id not found in INFO command, skipping PID collection for this connection")
14801503

14811504
# Check if all tested commands are supported by this Redis instance
14821505
supported_commands = get_supported_redis_commands(redis_conns)
@@ -2991,6 +3014,11 @@ def get_supported_redis_commands(redis_conns):
29913014
f"Retrieved {len(supported_commands)} supported Redis commands"
29923015
)
29933016

3017+
# Handle case where COMMAND returns 0 commands (likely not supported)
3018+
if len(supported_commands) == 0:
3019+
logging.warning("COMMAND returned 0 commands - likely not supported by this Redis instance")
3020+
return None
3021+
29943022
# Log some sample commands for debugging
29953023
if supported_commands:
29963024
sample_commands = sorted(list(supported_commands))[:10]
@@ -3013,8 +3041,8 @@ def get_supported_redis_commands(redis_conns):
30133041

30143042
def check_test_command_support(benchmark_config, supported_commands):
30153043
"""Check if all tested-commands in the benchmark config are supported"""
3016-
if supported_commands is None:
3017-
logging.warning("No supported commands list available, skipping command check")
3044+
if supported_commands is None or len(supported_commands) == 0:
3045+
logging.warning("No supported commands list available (COMMAND not supported or returned 0 commands), skipping command check")
30183046
return True, []
30193047

30203048
if "tested-commands" not in benchmark_config:

‎redis_benchmarks_specification/__self_contained_coordinator__/clients.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
1+
import logging
2+
3+
14
def prepare_memtier_benchmark_parameters(
25
clientconfig,
36
full_benchmark_path,
47
port,
58
server,
69
local_benchmark_output_filename,
710
oss_cluster_api_enabled,
11+
password=None,
812
):
913
benchmark_command = [
1014
full_benchmark_path,
@@ -15,6 +19,12 @@ def prepare_memtier_benchmark_parameters(
1519
"--server",
1620
"{}".format(server),
1721
]
22+
23+
# Add password authentication if provided
24+
if password is not None and password != "":
25+
benchmark_command.extend(["--authenticate", password])
26+
logging.info("Memtier benchmark will use password authentication")
27+
1828
if oss_cluster_api_enabled is True:
1929
benchmark_command.append("--cluster-mode")
2030
benchmark_command_str = " ".join(benchmark_command)

‎redis_benchmarks_specification/__self_contained_coordinator__/docker.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@
88

99

1010
def generate_standalone_redis_server_args(
11-
binary, port, dbdir, configuration_parameters=None, redis_arguments=""
11+
binary, port, dbdir, configuration_parameters=None, redis_arguments="", password=None
1212
):
13-
added_params = ["port", "protected-mode", "dir"]
13+
added_params = ["port", "protected-mode", "dir", "requirepass"]
1414
# start redis-server
1515
command = [
1616
binary,
@@ -19,6 +19,11 @@ def generate_standalone_redis_server_args(
1919
"--port",
2020
"{}".format(port),
2121
]
22+
23+
# Add password authentication if provided
24+
if password is not None and password != "":
25+
command.extend(["--requirepass", password])
26+
logging.info("Redis server will be started with password authentication")
2227
if dbdir != "":
2328
command.extend(["--dir", dbdir])
2429
if configuration_parameters is not None:
@@ -59,13 +64,16 @@ def spin_docker_standalone_redis(
5964
redis_proc_start_port,
6065
run_image,
6166
temporary_dir,
67+
password=None,
6268
):
6369
mnt_point = "/mnt/redis/"
6470
command = generate_standalone_redis_server_args(
6571
"{}redis-server".format(mnt_point),
6672
redis_proc_start_port,
6773
mnt_point,
6874
redis_configuration_parameters,
75+
"",
76+
password,
6977
)
7078
command_str = " ".join(command)
7179
db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus(

‎redis_benchmarks_specification/__self_contained_coordinator__/runners.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ def process_self_contained_coordinator_stream(
118118
verbose=False,
119119
run_tests_with_dataset=False,
120120
):
121+
# Use a default password for coordinator Redis instances
122+
redis_password = "redis_coordinator_password_2024"
121123
stream_id = "n/a"
122124
overall_result = False
123125
total_test_suite_runs = 0
@@ -276,6 +278,7 @@ def process_self_contained_coordinator_stream(
276278
redis_proc_start_port,
277279
run_image,
278280
temporary_dir,
281+
redis_password,
279282
)
280283
else:
281284
shard_count = 1
@@ -307,11 +310,15 @@ def process_self_contained_coordinator_stream(
307310
)
308311
)
309312

310-
r = redis.StrictRedis(port=redis_proc_start_port)
313+
r = redis.StrictRedis(port=redis_proc_start_port, password=redis_password)
311314
r.ping()
312315
redis_pids = []
313-
first_redis_pid = r.info()["process_id"]
314-
redis_pids.append(first_redis_pid)
316+
redis_info = r.info()
317+
first_redis_pid = redis_info.get("process_id")
318+
if first_redis_pid is not None:
319+
redis_pids.append(first_redis_pid)
320+
else:
321+
logging.warning("Redis process_id not found in INFO command")
315322
ceil_client_cpu_limit = extract_client_cpu_limit(
316323
benchmark_config
317324
)
@@ -385,6 +392,7 @@ def process_self_contained_coordinator_stream(
385392
"localhost",
386393
local_benchmark_output_filename,
387394
benchmark_tool_workdir,
395+
redis_password,
388396
)
389397
elif "vector_db_benchmark" in benchmark_tool:
390398
(

‎redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -496,6 +496,9 @@ def process_self_contained_coordinator_stream(
496496
restore_build_artifacts_default=True,
497497
args=None,
498498
):
499+
# Use a default password for coordinator Redis instances
500+
redis_password = "redis_coordinator_password_2024"
501+
499502
stream_id = "n/a"
500503
overall_result = False
501504
total_test_suite_runs = 0
@@ -861,6 +864,7 @@ def process_self_contained_coordinator_stream(
861864
mnt_point,
862865
redis_configuration_parameters,
863866
redis_arguments,
867+
redis_password,
864868
)
865869
command_str = " ".join(command)
866870
db_cpuset_cpus, current_cpu_pos = generate_cpuset_cpus(
@@ -876,13 +880,16 @@ def process_self_contained_coordinator_stream(
876880
temporary_dir,
877881
)
878882

879-
r = redis.StrictRedis(port=redis_proc_start_port)
883+
r = redis.StrictRedis(port=redis_proc_start_port, password=redis_password)
880884
r.ping()
881885
redis_conns = [r]
882886
reset_commandstats(redis_conns)
883887
redis_pids = []
884888
redis_info = r.info()
885-
first_redis_pid = redis_info["process_id"]
889+
first_redis_pid = redis_info.get("process_id")
890+
if first_redis_pid is None:
891+
logging.warning("Redis process_id not found in INFO command")
892+
first_redis_pid = "unknown"
886893
if git_hash is None and "redis_git_sha1" in redis_info:
887894
git_hash = redis_info["redis_git_sha1"]
888895
if (
@@ -972,7 +979,7 @@ def process_self_contained_coordinator_stream(
972979
full_benchmark_path,
973980
redis_proc_start_port,
974981
"localhost",
975-
None,
982+
redis_password,
976983
local_benchmark_output_filename,
977984
False,
978985
False,
@@ -1718,7 +1725,7 @@ def data_prepopulation_step(
17181725
full_benchmark_path,
17191726
port,
17201727
"localhost",
1721-
None,
1728+
redis_password,
17221729
local_benchmark_output_filename,
17231730
False,
17241731
)

0 commit comments

Comments
 (0)