tuxmain/tlsbench
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Reproduce setting

Browse source
This commit is contained in:
Pascal Engélibert 2026-05-21 10:27:03 +02:00
commit cf463794fb
2 changed files with 265 additions and 205 deletions
Download patch file Download diff file Expand all files Collapse all files

385
exp.py
View file

@ -72,24 +72,25 @@ CONFIGS = {
# i7-4790 -> pi3
"pi3": {
"experiments": [
#"impl-cipher-ver",
#"impl-cert-ver",
#"impl-kex-ver",
"impl-cipher-ver",
"impl-cert-ver",
"impl-kex-ver",
#"zrtt",
"realistic"
#"realistic"
],
"sides": [
"client",
"server",
],
"tls": [
False,
#False,
True,
],
"records": [
{ "filename": "wp2", "repeat": 10000, "time": 600 },
{ "filename": "yt2-ads", "repeat": 10000, "time": 600 },
{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 },
{ "filename": "wp2", "repeat": 10000, "time": 300 },
#{ "filename": "yt2-ads", "repeat": 10000, "time": 600 },
#{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 },
#{ "filename": "wp2", "repeat": 10000, "time": 300, "reproduce": 100 },
],
"repo_dir": "/home/tuxmain/reps/tlsbench",
"exp_dir": "/dev/shm/exp",
@ -112,22 +113,24 @@ CONFIGS = {
# i7-4790 -> core2
"core2": {
"experiments": [
"impl-cipher-ver",
#"impl-cipher-ver",
"impl-cert-ver",
"impl-kex-ver",
"zrtt",
#"impl-kex-ver",
#"zrtt",
#"realistic",
],
"sides": [
"client",
"server",
#"server",
],
"tls": [
False,
True,
#True,
],
"records": [
{ "filename": "wikipedia", "repeat": 2000 },
{ "filename": "wp2", "repeat": 10000, "time": 300 },
#{ "filename": "yt2-ads", "repeat": 10000, "time": 600 },
#{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 },
],
"repo_dir": "/home/tuxmain/reps/tlsbench",
"exp_dir": "/dev/shm/exp",
@ -142,7 +145,7 @@ CONFIGS = {
"rapl": False,
"sa": True,
"listen_port": 8080,
"idle": "idle - - - - - - - - - 1200.0000903606415 0 589 4764 3.478999999999999 0 -",
"idle": "idle - - - - - - - - - 1777360401.7482355 1777361601.7482355 1200.0000903606415 0 589 4764 3.478999999999999 0 -",
"notify_listen": ("0.0.0.0", 8090),
"notify_addr": "192.168.3.1:8090",
},
@ -181,22 +184,22 @@ CONFIGS = {
"experiments": [
#"impl-cipher-ver",
#"impl-cert-ver",
#"impl-kex-ver",
"impl-kex-ver",
#"zrtt",
"realistic",
#"realistic",
],
"sides": [
"client",
"server",
],
"tls": [
#False,
False,
True,
],
"records": [
{ "filename": "wp2", "repeat": 10000, "time": 600 },
{ "filename": "yt2-ads", "repeat": 10000, "time": 600 },
{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 },
{ "filename": "wp2", "repeat": 10000, "time": 300 },
#{ "filename": "yt2-ads", "repeat": 10000, "time": 600 },
#{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 },
],
"repo_dir": "/home/tuxmain/reps/tlsbench",
"exp_dir": "/dev/shm/exp",
@ -267,9 +270,10 @@ CONFIGS = {
"records": [
# Can't repeat more than 8000 times here
# TODO check if netreplay client frees ports correctly, or try to reuse ports
{ "filename": "wikipedia", "repeat": 8000, "time": 180 },
{ "filename": "wp2", "repeat": 8000, "time": 180 },
{ "filename": "yt2-ads", "repeat": 8000, "time": 180 },
{ "filename": "yt2-ublock", "repeat": 8000, "time": 180 },
#{ "filename": "wp2", "repeat": 10000, "time": 300, "reproduce": 100 },
],
"repo_dir": "/home/pengelib/tlsbench",
"exp_dir": "/dev/shm/exp",
@ -349,10 +353,10 @@ IMPLS = [
#"aws-lc", # Amazon's crypto widely used in Rust stuff
#"boring", # Google's fork of OpenSSL used in Chrome and Android
#"graviola", # New crypto in Rust
"openssl", # widely used
#"openssl", # widely used
#"openssl-static",
#"ring", # used in most Rust stuff
#"symcrypt", # Microsoft's crypto
"symcrypt", # Microsoft's crypto
#"wolfcrypt" # used in embedded (won't build with rpxy for now)
]
# Symmetric ciphers
@ -405,9 +409,9 @@ EXPERIMENTS = {
"kexes": ["X25519"],
"cert": [
"prime256v1",
#"secp384r1",
"secp384r1",
"rsa2048",
#"rsa3072",
"rsa3072",
"rsa4096",
],
"earlydata": ["0"],
@ -422,7 +426,7 @@ EXPERIMENTS = {
"kexes": [
"X25519",
"SECP256R1",
#"SECP384R1",
"SECP384R1",
"X25519MLKEM768",
"SECP256R1MLKEM768",
"MLKEM768",
@ -459,14 +463,14 @@ EXPERIMENTS = {
"AES_256_GCM_SHA384",
#"CHACHA20_POLY1305_SHA256",
#"ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,ECDHE_RSA_WITH_AES_128_GCM_SHA256",
"ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,ECDHE_RSA_WITH_AES_256_GCM_SHA384",
#"ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,ECDHE_RSA_WITH_AES_256_GCM_SHA384",
#"ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
],
"kexes": [
"X25519",
#"SECP256R1",
#"SECP384R1",
"X25519MLKEM768",
#"X25519MLKEM768",
#"SECP256R1MLKEM768",
#"MLKEM768",
],
@ -478,7 +482,7 @@ EXPERIMENTS = {
#"rsa4096",
"realistic",
],
"earlydata": ["0"],
"earlydata": ["1"],
},
}
@ -487,15 +491,15 @@ def alg_filter(kex, cert, cipher, impl):
if "MLKEM" in kex and "WITH" in cipher:
# WITH means TLS1.2
return False
if "MLKEM" in kex and "openssl" not in impl and impl != "aws-lc" and impl != "graviola":
if "MLKEM" in kex and "openssl" not in impl and impl != "aws-lc" and impl != "graviola" and impl != "boring":
return False
if kex == "SECP256R1MLKEM768" and "openssl" in impl:
return False
if cert == "secp384r1" and impl == "boring":
return False
#if cert == "secp384r1" and impl == "boring":
# return False
if kex == "SECP256R1MLKEM768" and impl == "graviola":
return False
if kex == "MLKEM768" and impl == "graviola":
if kex == "MLKEM768" and impl in ["graviola", "boring"]:
return False
if cipher == "ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,ECDHE_RSA_WITH_AES_256_GCM_SHA384" and "openssl" in impl:
return False
@ -846,165 +850,166 @@ def run_exp(config, only_record=None, idle=False, shutdown=False, debug=False):
continue
for side in config["sides"]:
for record in config["records"]:
print(f"EXPERIMENT {expname}: {impl} {alg} {kex} {cipher} ED={earlydata} {side} TLS={tls}")
run_id += 1
for reprod_i in range(record.get("reproduce", 1)):
print(f"EXPERIMENT {expname}: {impl} {alg} {kex} {cipher} ED={earlydata} {side} TLS={tls}")
run_id += 1
if side == "client":
netreplay = run_netreplay_server(
None,
config["exp_dir"],
config["repo_dir"],
record,
"0.0.0.0",
config["listen_port"],
tls,
None,
certs_dir,
only_record=only_record,
ciphers=cipher,
kexes=kex,
debug=debug,
ld_preload=config["ld_preload"] if "ld_preload" in config else None,
)
netreplay = run_netreplay_client(
ssh,
config["exp_dir"],
config["remote_repo_dir"],
record,
config["local_addr"],
config["listen_port"],
tls,
impl,
certs_dir,
only_record=only_record,
ciphers=cipher,
kexes=kex,
earlydata=earlydata,
debug=debug,
notify_addr=config["notify_addr"],
ld_preload=config["ld_preload"] if "ld_preload" in config else None,
concurrency=config.get("concurrency"),
)
else:
netreplay = run_netreplay_server(
ssh,
config["exp_dir"],
config["remote_repo_dir"],
record,
"0.0.0.0",
config["listen_port"],
tls,
impl,
certs_dir,
only_record=only_record,
ciphers=cipher,
kexes=kex,
debug=debug,
ld_preload=config["ld_preload"] if "ld_preload" in config else None,
)
netreplay = run_netreplay_client(
None,
config["exp_dir"],
config["repo_dir"],
record,
config["remote_addr"],
config["listen_port"],
tls,
None,
certs_dir,
only_record=only_record,
ciphers=cipher,
kexes=kex,
earlydata=earlydata,
debug=debug,
notify_addr=config["notify_addr"],
ld_preload=config["ld_preload"] if "ld_preload" in config else None,
skip_verif=True,
concurrency=config.get("concurrency"),
)
if side == "client":
netreplay = run_netreplay_server(
None,
config["exp_dir"],
config["repo_dir"],
record,
"0.0.0.0",
config["listen_port"],
tls,
None,
certs_dir,
only_record=only_record,
ciphers=cipher,
kexes=kex,
debug=debug,
ld_preload=config["ld_preload"] if "ld_preload" in config else None,
)
netreplay = run_netreplay_client(
ssh,
config["exp_dir"],
config["remote_repo_dir"],
record,
config["local_addr"],
config["listen_port"],
tls,
impl,
certs_dir,
only_record=only_record,
ciphers=cipher,
kexes=kex,
earlydata=earlydata,
debug=debug,
notify_addr=config["notify_addr"],
ld_preload=config["ld_preload"] if "ld_preload" in config else None,
concurrency=config.get("concurrency"),
)
else:
netreplay = run_netreplay_server(
ssh,
config["exp_dir"],
config["remote_repo_dir"],
record,
"0.0.0.0",
config["listen_port"],
tls,
impl,
certs_dir,
only_record=only_record,
ciphers=cipher,
kexes=kex,
debug=debug,
ld_preload=config["ld_preload"] if "ld_preload" in config else None,
)
netreplay = run_netreplay_client(
None,
config["exp_dir"],
config["repo_dir"],
record,
config["remote_addr"],
config["listen_port"],
tls,
None,
certs_dir,
only_record=only_record,
ciphers=cipher,
kexes=kex,
earlydata=earlydata,
debug=debug,
notify_addr=config["notify_addr"],
ld_preload=config["ld_preload"] if "ld_preload" in config else None,
skip_verif=True,
concurrency=config.get("concurrency"),
)
prof_filename = "-"
if config["perf"]:
prof_filename = f"{perf_dir}/perf-{timestr}-{run_id}.data"
process_pid = ssh_run(ssh, "pidof netreplay-"+impl).removesuffix("\n")
ssh_run_bg(ssh, f"perf record -F 997 --call-graph dwarf,64000 -g -o {prof_filename} -p {process_pid}")
# Measure
cpu = 0
if config["sa"]:
cpu = get_cpu_stat(ssh, ["netreplay-"+impl, "tokio-runtime-w", "tokio-rt-worker"])
remote_bytes_in, remote_bytes_out = get_net_stat(ssh)
energy = 0
energy_rapl = 0
if config["wattmeter"]:
energy = wattmeter.get_meter()
if config["rapl"]:
energy_rapl = get_rapl_energy(ssh, remote_path)
start = time.time()
# Wait for the client to terminate
signal.alarm(record["time"] if "time" in record else 3600)
session_count = 0
try:
while True:
rec_count = int.from_bytes(notify_socket.recv(4), "big")
if rec_count == 0xffffffff:
break
if rec_count > session_count:
if rec_count - session_count > 1:
print("count", session_count, "->", rec_count)
session_count = rec_count
except Timeout:
print("TIMEOUT: stop")
# Measure
end = time.time()
new_energy = 0
new_energy_rapl = 0
if config["wattmeter"]:
new_energy = wattmeter.get_meter()
if config["rapl"]:
new_energy_rapl = get_rapl_energy(ssh, remote_path)
new_remote_bytes_in, new_remote_bytes_out = get_net_stat(ssh)
# Kill netreplay
sh("killall netreplay")
try:
ssh_run(ssh, "killall netreplay-"+impl)
except invoke.exceptions.UnexpectedExit as e:
pass
# flush the UDP queue
signal.alarm(2)
try:
while True:
notify_socket.recv(4)
except Timeout:
pass
# Measure CPU after (as it may update only after the process is killed)
new_cpu = 0
if config["sa"]:
new_cpu = get_cpu_stat(ssh, ["netreplay-"+impl, "tokio-runtime-w", "tokio-rt-worker"])
record_filename = record["filename"]
cpu_diff = new_cpu - cpu
remote_bytes_in_diff = new_remote_bytes_in - remote_bytes_in
remote_bytes_out_diff = new_remote_bytes_out - remote_bytes_out
energy_diff = new_energy - energy
energy_rapl_diff = new_energy_rapl - energy_rapl
time_diff = end - start
while True:
prof_filename = "-"
if config["perf"]:
prof_filename = f"{perf_dir}/perf-{timestr}-{run_id}.data"
process_pid = ssh_run(ssh, "pidof netreplay-"+impl).removesuffix("\n")
ssh_run_bg(ssh, f"perf record -F 997 --call-graph dwarf,64000 -g -o {prof_filename} -p {process_pid}")
# Measure
cpu = 0
if config["sa"]:
cpu = get_cpu_stat(ssh, ["netreplay-"+impl, "tokio-runtime-w", "tokio-rt-worker"])
remote_bytes_in, remote_bytes_out = get_net_stat(ssh)
energy = 0
energy_rapl = 0
if config["wattmeter"]:
energy = wattmeter.get_meter()
if config["rapl"]:
energy_rapl = get_rapl_energy(ssh, remote_path)
start = time.time()
# Wait for the client to terminate
signal.alarm(record["time"] if "time" in record else 3600)
session_count = 0
try:
with open(logfile_path, "a") as logfile:
logfile.write(f"{expname} {impl} {alg} {kex} {cipher} {earlydata} {side} {tls_int} {record_filename} {session_count} {start} {end} {time_diff} {cpu_diff} {remote_bytes_in_diff} {remote_bytes_out_diff} {energy_diff} {energy_rapl_diff} {prof_filename}\n")
logfile.close()
break
except Exception as e:
print("Can't open log file:", e)
time.sleep(1)
sh(f"cp {logfile_path} {log_backup_dir}/{logfile_name}")
while True:
rec_count = int.from_bytes(notify_socket.recv(4), "big")
if rec_count == 0xffffffff:
break
if rec_count > session_count:
if rec_count - session_count > 1:
print("count", session_count, "->", rec_count)
session_count = rec_count
except Timeout:
print("TIMEOUT: stop")
# Measure
end = time.time()
new_energy = 0
new_energy_rapl = 0
if config["wattmeter"]:
new_energy = wattmeter.get_meter()
if config["rapl"]:
new_energy_rapl = get_rapl_energy(ssh, remote_path)
new_remote_bytes_in, new_remote_bytes_out = get_net_stat(ssh)
# Kill netreplay
sh("killall netreplay")
try:
ssh_run(ssh, "killall netreplay-"+impl)
except invoke.exceptions.UnexpectedExit as e:
pass
# flush the UDP queue
signal.alarm(2)
try:
while True:
notify_socket.recv(4)
except Timeout:
pass
# Measure CPU after (as it may update only after the process is killed)
new_cpu = 0
if config["sa"]:
new_cpu = get_cpu_stat(ssh, ["netreplay-"+impl, "tokio-runtime-w", "tokio-rt-worker"])
record_filename = record["filename"]
cpu_diff = new_cpu - cpu
remote_bytes_in_diff = new_remote_bytes_in - remote_bytes_in
remote_bytes_out_diff = new_remote_bytes_out - remote_bytes_out
energy_diff = new_energy - energy
energy_rapl_diff = new_energy_rapl - energy_rapl
time_diff = end - start
while True:
try:
with open(logfile_path, "a") as logfile:
logfile.write(f"{expname} {impl} {alg} {kex} {cipher} {earlydata} {side} {tls_int} {record_filename} {session_count} {start} {end} {time_diff} {cpu_diff} {remote_bytes_in_diff} {remote_bytes_out_diff} {energy_diff} {energy_rapl_diff} {prof_filename}\n")
logfile.close()
break
except Exception as e:
print("Can't open log file:", e)
time.sleep(1)
sh(f"cp {logfile_path} {log_backup_dir}/{logfile_name}")
if config["wattmeter"]:
YAPI.FreeAPI()
if shutdown and ssh:

91
plots.py
View file

@ -283,7 +283,7 @@ def make_log_plot(logs, exp, criterion, side, obj, record, machine=None, version
maketitle=maketitle
)
def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record):
def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record, version=None):
f = open(f"/dev/shm/plots/{obj}_by_{criterion}_{side}_{record}.dat", "w")
ciphers = {target: {} for target in logs_by_target}
impls = []
@ -305,9 +305,9 @@ def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record):
n = float(log.get("n", "1000"))
plain[target] = (float(log[COL[obj]]) * conv_factor - idle_val[target] * float(log["time"])) / n
for target in plain:
if plain[target] == None:
return
#for target in plain:
# if plain[target] == None:
# return
for target in logs_by_target:
logs = logs_by_target[target]
@ -334,17 +334,28 @@ def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record):
for impl in impls:
if impl not in ciphers[target][cipher]:
ciphers[target][cipher][impl] = 0
cipher_parts = cipher.split("/")
f.write("{} {}({}) {} {}\n".format(
target,
ALG_LABEL[cipher_parts[0]],
cipher_parts[1],
plain[target],
" ".join([
str(ciphers[target][cipher][impl])
for impl in impls
]),
))
if version == None:
cipher_parts = cipher.split("/")
f.write("{} {}({}) {} {}\n".format(
target,
ALG_LABEL[cipher_parts[0]],
cipher_parts[1],
plain[target],
" ".join([
str(ciphers[target][cipher][impl] if ciphers[target][cipher][impl] > 0 else "-")
for impl in impls
]),
))
else:
f.write("{} {} {} {}\n".format(
target,
ALG_LABEL[cipher],
plain[target],
" ".join([
str(ciphers[target][cipher][impl] if ciphers[target][cipher][impl] > 0 else "-")
for impl in impls
]),
))
f.close()
def make_profile_plot(logs, exp, criterion, side, record, no_flamegraph=False, machine=None, version=None, maketitle=False):
@ -822,6 +833,42 @@ def make_tx_summary(client_logs, server_logs):
])
print(tabulate(latex, separator=" & ", endline="\\\\ \\cline{2-8}", ruler=False))
def compute_box_plot(a):
a.sort()
median = a[len(a)//2] if len(a)%2 == 1 else (a[len(a)//2-1]+a[len(a)//2])/2
h1 = a[:len(a)//2]
h2 = a[(len(a)+1)//2:]
q1 = h1[len(h1)//2] if len(h1)%2 == 1 else (h1[len(h1)//2-1]+h1[len(h1)//2])/2
q3 = h2[len(h2)//2] if len(h2)%2 == 1 else (h2[len(h2)//2-1]+h2[len(h2)//2])/2
return [a[0], q1, median, q3, a[-1]]
def make_stability_plot(logs_by_target):
f = open(f"/dev/shm/plots/stability.dat", "w")
idle_val = {target: None for target in logs_by_target}
for target in logs_by_target:
logs = logs_by_target[target]
for log in logs:
if log["exp"] == "idle":
idle_val[target] = {
"energy": float(log[COL["energy"]]) / float(log["time"]) * 3600.0,
"io": float(log["bytes_in"])+float(log["bytes_out"]) / float(log["time"]),
}
for target in logs_by_target:
logs = logs_by_target[target]
for log in logs:
if log["exp"] == "idle":
continue
n = float(log.get("n", "1000"))
f.write("{}-{} {} {}\n".format(
target,
log["side"],
(float(log[COL["energy"]]) * 3600.0 - idle_val[target]["energy"] * float(log["time"])) / n,
(float(log["bytes_in"])+float(log["bytes_out"]) - idle_val[target]["io"] * float(log["time"])) / n / 1024**2
))
f.close()
def getargv(arg:str, default="", n:int=1, args:list=sys.argv):
if arg in args and len(args) > args.index(arg)+n:
return args[args.index(arg)+n]
@ -889,9 +936,9 @@ if __name__ == "__main__":
records.pop(record)
for side in ["client", "server"]:
for record in records:
make_log_plot_points(logs_by_target, "impl-cipher-ver", "cipher", side, "energy", record)
make_log_plot_points(logs_by_target, "impl-cert-ver", "cert", side, "energy", record)
make_log_plot_points(logs_by_target, "impl-kex-ver", "kex", side, "energy", record)
make_log_plot_points(logs_by_target, "impl-cipher-ver", "cipher", side, "energy", record, version="1.3")
make_log_plot_points(logs_by_target, "impl-cert-ver", "cert", side, "energy", record, version="1.3")
make_log_plot_points(logs_by_target, "impl-kex-ver", "kex", side, "energy", record, version="1.3")
cmp_versions(logs, ["impl-cipher-ver", "impl-cert-ver", "impl-kex-ver"], ["side", "cipher", "cert", "kex", "record"], ["cpu", "energy"])
elif cmd == "prof":
for side in ["client", "server"]:
@ -926,3 +973,11 @@ if __name__ == "__main__":
import matplotlib.pyplot as plt
import numpy as np
make_linear_regression(logs, "alg")
elif cmd == "stab":
# Args: stab <file1> <target1> [<file2> <target2>] [...]
logs_by_target = {sys.argv[3]: logs}
for i in range(4, len(sys.argv), 2):
logs_i, records_i = parse_logs(sys.argv[i])
target_i = sys.argv[i+1]
logs_by_target[target_i] = logs_i
make_stability_plot(logs_by_target)