tuxmain/tlsbench
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Reproduce setting

Browse source
This commit is contained in:
Pascal Engélibert 2026-05-21 10:27:03 +02:00
commit cf463794fb
2 changed files with 265 additions and 205 deletions
Download patch file Download diff file Expand all files Collapse all files

73
exp.py
View file

@ -72,24 +72,25 @@ CONFIGS = {
# i7-4790 -> pi3 # i7-4790 -> pi3
"pi3": { "pi3": {
"experiments": [ "experiments": [
#"impl-cipher-ver", "impl-cipher-ver",
#"impl-cert-ver", "impl-cert-ver",
#"impl-kex-ver", "impl-kex-ver",
#"zrtt", #"zrtt",
"realistic" #"realistic"
], ],
"sides": [ "sides": [
"client", "client",
"server", "server",
], ],
"tls": [ "tls": [
False, #False,
True, True,
], ],
"records": [ "records": [
{ "filename": "wp2", "repeat": 10000, "time": 600 }, { "filename": "wp2", "repeat": 10000, "time": 300 },
{ "filename": "yt2-ads", "repeat": 10000, "time": 600 }, #{ "filename": "yt2-ads", "repeat": 10000, "time": 600 },
{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 }, #{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 },
#{ "filename": "wp2", "repeat": 10000, "time": 300, "reproduce": 100 },
], ],
"repo_dir": "/home/tuxmain/reps/tlsbench", "repo_dir": "/home/tuxmain/reps/tlsbench",
"exp_dir": "/dev/shm/exp", "exp_dir": "/dev/shm/exp",
@ -112,22 +113,24 @@ CONFIGS = {
# i7-4790 -> core2 # i7-4790 -> core2
"core2": { "core2": {
"experiments": [ "experiments": [
"impl-cipher-ver", #"impl-cipher-ver",
"impl-cert-ver", "impl-cert-ver",
"impl-kex-ver", #"impl-kex-ver",
"zrtt", #"zrtt",
#"realistic", #"realistic",
], ],
"sides": [ "sides": [
"client", "client",
"server", #"server",
], ],
"tls": [ "tls": [
False, False,
True, #True,
], ],
"records": [ "records": [
{ "filename": "wikipedia", "repeat": 2000 }, { "filename": "wp2", "repeat": 10000, "time": 300 },
#{ "filename": "yt2-ads", "repeat": 10000, "time": 600 },
#{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 },
], ],
"repo_dir": "/home/tuxmain/reps/tlsbench", "repo_dir": "/home/tuxmain/reps/tlsbench",
"exp_dir": "/dev/shm/exp", "exp_dir": "/dev/shm/exp",
@ -142,7 +145,7 @@ CONFIGS = {
"rapl": False, "rapl": False,
"sa": True, "sa": True,
"listen_port": 8080, "listen_port": 8080,
"idle": "idle - - - - - - - - - 1200.0000903606415 0 589 4764 3.478999999999999 0 -", "idle": "idle - - - - - - - - - 1777360401.7482355 1777361601.7482355 1200.0000903606415 0 589 4764 3.478999999999999 0 -",
"notify_listen": ("0.0.0.0", 8090), "notify_listen": ("0.0.0.0", 8090),
"notify_addr": "192.168.3.1:8090", "notify_addr": "192.168.3.1:8090",
}, },
@ -181,22 +184,22 @@ CONFIGS = {
"experiments": [ "experiments": [
#"impl-cipher-ver", #"impl-cipher-ver",
#"impl-cert-ver", #"impl-cert-ver",
#"impl-kex-ver", "impl-kex-ver",
#"zrtt", #"zrtt",
"realistic", #"realistic",
], ],
"sides": [ "sides": [
"client", "client",
"server", "server",
], ],
"tls": [ "tls": [
#False, False,
True, True,
], ],
"records": [ "records": [
{ "filename": "wp2", "repeat": 10000, "time": 600 }, { "filename": "wp2", "repeat": 10000, "time": 300 },
{ "filename": "yt2-ads", "repeat": 10000, "time": 600 }, #{ "filename": "yt2-ads", "repeat": 10000, "time": 600 },
{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 }, #{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 },
], ],
"repo_dir": "/home/tuxmain/reps/tlsbench", "repo_dir": "/home/tuxmain/reps/tlsbench",
"exp_dir": "/dev/shm/exp", "exp_dir": "/dev/shm/exp",
@ -267,9 +270,10 @@ CONFIGS = {
"records": [ "records": [
# Can't repeat more than 8000 times here # Can't repeat more than 8000 times here
# TODO check if netreplay client frees ports correctly, or try to reuse ports # TODO check if netreplay client frees ports correctly, or try to reuse ports
{ "filename": "wikipedia", "repeat": 8000, "time": 180 }, { "filename": "wp2", "repeat": 8000, "time": 180 },
{ "filename": "yt2-ads", "repeat": 8000, "time": 180 }, { "filename": "yt2-ads", "repeat": 8000, "time": 180 },
{ "filename": "yt2-ublock", "repeat": 8000, "time": 180 }, { "filename": "yt2-ublock", "repeat": 8000, "time": 180 },
#{ "filename": "wp2", "repeat": 10000, "time": 300, "reproduce": 100 },
], ],
"repo_dir": "/home/pengelib/tlsbench", "repo_dir": "/home/pengelib/tlsbench",
"exp_dir": "/dev/shm/exp", "exp_dir": "/dev/shm/exp",
@ -349,10 +353,10 @@ IMPLS = [
#"aws-lc", # Amazon's crypto widely used in Rust stuff #"aws-lc", # Amazon's crypto widely used in Rust stuff
#"boring", # Google's fork of OpenSSL used in Chrome and Android #"boring", # Google's fork of OpenSSL used in Chrome and Android
#"graviola", # New crypto in Rust #"graviola", # New crypto in Rust
"openssl", # widely used #"openssl", # widely used
#"openssl-static", #"openssl-static",
#"ring", # used in most Rust stuff #"ring", # used in most Rust stuff
#"symcrypt", # Microsoft's crypto "symcrypt", # Microsoft's crypto
#"wolfcrypt" # used in embedded (won't build with rpxy for now) #"wolfcrypt" # used in embedded (won't build with rpxy for now)
] ]
# Symmetric ciphers # Symmetric ciphers
@ -405,9 +409,9 @@ EXPERIMENTS = {
"kexes": ["X25519"], "kexes": ["X25519"],
"cert": [ "cert": [
"prime256v1", "prime256v1",
#"secp384r1", "secp384r1",
"rsa2048", "rsa2048",
#"rsa3072", "rsa3072",
"rsa4096", "rsa4096",
], ],
"earlydata": ["0"], "earlydata": ["0"],
@ -422,7 +426,7 @@ EXPERIMENTS = {
"kexes": [ "kexes": [
"X25519", "X25519",
"SECP256R1", "SECP256R1",
#"SECP384R1", "SECP384R1",
"X25519MLKEM768", "X25519MLKEM768",
"SECP256R1MLKEM768", "SECP256R1MLKEM768",
"MLKEM768", "MLKEM768",
@ -459,14 +463,14 @@ EXPERIMENTS = {
"AES_256_GCM_SHA384", "AES_256_GCM_SHA384",
#"CHACHA20_POLY1305_SHA256", #"CHACHA20_POLY1305_SHA256",
#"ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,ECDHE_RSA_WITH_AES_128_GCM_SHA256", #"ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,ECDHE_RSA_WITH_AES_128_GCM_SHA256",
"ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,ECDHE_RSA_WITH_AES_256_GCM_SHA384", #"ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,ECDHE_RSA_WITH_AES_256_GCM_SHA384",
#"ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", #"ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
], ],
"kexes": [ "kexes": [
"X25519", "X25519",
#"SECP256R1", #"SECP256R1",
#"SECP384R1", #"SECP384R1",
"X25519MLKEM768", #"X25519MLKEM768",
#"SECP256R1MLKEM768", #"SECP256R1MLKEM768",
#"MLKEM768", #"MLKEM768",
], ],
@ -478,7 +482,7 @@ EXPERIMENTS = {
#"rsa4096", #"rsa4096",
"realistic", "realistic",
], ],
"earlydata": ["0"], "earlydata": ["1"],
}, },
} }
@ -487,15 +491,15 @@ def alg_filter(kex, cert, cipher, impl):
if "MLKEM" in kex and "WITH" in cipher: if "MLKEM" in kex and "WITH" in cipher:
# WITH means TLS1.2 # WITH means TLS1.2
return False return False
if "MLKEM" in kex and "openssl" not in impl and impl != "aws-lc" and impl != "graviola": if "MLKEM" in kex and "openssl" not in impl and impl != "aws-lc" and impl != "graviola" and impl != "boring":
return False return False
if kex == "SECP256R1MLKEM768" and "openssl" in impl: if kex == "SECP256R1MLKEM768" and "openssl" in impl:
return False return False
if cert == "secp384r1" and impl == "boring": #if cert == "secp384r1" and impl == "boring":
return False # return False
if kex == "SECP256R1MLKEM768" and impl == "graviola": if kex == "SECP256R1MLKEM768" and impl == "graviola":
return False return False
if kex == "MLKEM768" and impl == "graviola": if kex == "MLKEM768" and impl in ["graviola", "boring"]:
return False return False
if cipher == "ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,ECDHE_RSA_WITH_AES_256_GCM_SHA384" and "openssl" in impl: if cipher == "ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,ECDHE_RSA_WITH_AES_256_GCM_SHA384" and "openssl" in impl:
return False return False
@ -846,6 +850,7 @@ def run_exp(config, only_record=None, idle=False, shutdown=False, debug=False):
continue continue
for side in config["sides"]: for side in config["sides"]:
for record in config["records"]: for record in config["records"]:
for reprod_i in range(record.get("reproduce", 1)):
print(f"EXPERIMENT {expname}: {impl} {alg} {kex} {cipher} ED={earlydata} {side} TLS={tls}") print(f"EXPERIMENT {expname}: {impl} {alg} {kex} {cipher} ED={earlydata} {side} TLS={tls}")
run_id += 1 run_id += 1

71
plots.py
View file

@ -283,7 +283,7 @@ def make_log_plot(logs, exp, criterion, side, obj, record, machine=None, version
maketitle=maketitle maketitle=maketitle
) )
def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record): def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record, version=None):
f = open(f"/dev/shm/plots/{obj}_by_{criterion}_{side}_{record}.dat", "w") f = open(f"/dev/shm/plots/{obj}_by_{criterion}_{side}_{record}.dat", "w")
ciphers = {target: {} for target in logs_by_target} ciphers = {target: {} for target in logs_by_target}
impls = [] impls = []
@ -305,9 +305,9 @@ def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record):
n = float(log.get("n", "1000")) n = float(log.get("n", "1000"))
plain[target] = (float(log[COL[obj]]) * conv_factor - idle_val[target] * float(log["time"])) / n plain[target] = (float(log[COL[obj]]) * conv_factor - idle_val[target] * float(log["time"])) / n
for target in plain: #for target in plain:
if plain[target] == None: # if plain[target] == None:
return # return
for target in logs_by_target: for target in logs_by_target:
logs = logs_by_target[target] logs = logs_by_target[target]
@ -334,6 +334,7 @@ def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record):
for impl in impls: for impl in impls:
if impl not in ciphers[target][cipher]: if impl not in ciphers[target][cipher]:
ciphers[target][cipher][impl] = 0 ciphers[target][cipher][impl] = 0
if version == None:
cipher_parts = cipher.split("/") cipher_parts = cipher.split("/")
f.write("{} {}({}) {} {}\n".format( f.write("{} {}({}) {} {}\n".format(
target, target,
@ -341,7 +342,17 @@ def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record):
cipher_parts[1], cipher_parts[1],
plain[target], plain[target],
" ".join([ " ".join([
str(ciphers[target][cipher][impl]) str(ciphers[target][cipher][impl] if ciphers[target][cipher][impl] > 0 else "-")
for impl in impls
]),
))
else:
f.write("{} {} {} {}\n".format(
target,
ALG_LABEL[cipher],
plain[target],
" ".join([
str(ciphers[target][cipher][impl] if ciphers[target][cipher][impl] > 0 else "-")
for impl in impls for impl in impls
]), ]),
)) ))
@ -822,6 +833,42 @@ def make_tx_summary(client_logs, server_logs):
]) ])
print(tabulate(latex, separator=" & ", endline="\\\\ \\cline{2-8}", ruler=False)) print(tabulate(latex, separator=" & ", endline="\\\\ \\cline{2-8}", ruler=False))
def compute_box_plot(a):
a.sort()
median = a[len(a)//2] if len(a)%2 == 1 else (a[len(a)//2-1]+a[len(a)//2])/2
h1 = a[:len(a)//2]
h2 = a[(len(a)+1)//2:]
q1 = h1[len(h1)//2] if len(h1)%2 == 1 else (h1[len(h1)//2-1]+h1[len(h1)//2])/2
q3 = h2[len(h2)//2] if len(h2)%2 == 1 else (h2[len(h2)//2-1]+h2[len(h2)//2])/2
return [a[0], q1, median, q3, a[-1]]
def make_stability_plot(logs_by_target):
f = open(f"/dev/shm/plots/stability.dat", "w")
idle_val = {target: None for target in logs_by_target}
for target in logs_by_target:
logs = logs_by_target[target]
for log in logs:
if log["exp"] == "idle":
idle_val[target] = {
"energy": float(log[COL["energy"]]) / float(log["time"]) * 3600.0,
"io": float(log["bytes_in"])+float(log["bytes_out"]) / float(log["time"]),
}
for target in logs_by_target:
logs = logs_by_target[target]
for log in logs:
if log["exp"] == "idle":
continue
n = float(log.get("n", "1000"))
f.write("{}-{} {} {}\n".format(
target,
log["side"],
(float(log[COL["energy"]]) * 3600.0 - idle_val[target]["energy"] * float(log["time"])) / n,
(float(log["bytes_in"])+float(log["bytes_out"]) - idle_val[target]["io"] * float(log["time"])) / n / 1024**2
))
f.close()
def getargv(arg:str, default="", n:int=1, args:list=sys.argv): def getargv(arg:str, default="", n:int=1, args:list=sys.argv):
if arg in args and len(args) > args.index(arg)+n: if arg in args and len(args) > args.index(arg)+n:
return args[args.index(arg)+n] return args[args.index(arg)+n]
@ -889,9 +936,9 @@ if __name__ == "__main__":
records.pop(record) records.pop(record)
for side in ["client", "server"]: for side in ["client", "server"]:
for record in records: for record in records:
make_log_plot_points(logs_by_target, "impl-cipher-ver", "cipher", side, "energy", record) make_log_plot_points(logs_by_target, "impl-cipher-ver", "cipher", side, "energy", record, version="1.3")
make_log_plot_points(logs_by_target, "impl-cert-ver", "cert", side, "energy", record) make_log_plot_points(logs_by_target, "impl-cert-ver", "cert", side, "energy", record, version="1.3")
make_log_plot_points(logs_by_target, "impl-kex-ver", "kex", side, "energy", record) make_log_plot_points(logs_by_target, "impl-kex-ver", "kex", side, "energy", record, version="1.3")
cmp_versions(logs, ["impl-cipher-ver", "impl-cert-ver", "impl-kex-ver"], ["side", "cipher", "cert", "kex", "record"], ["cpu", "energy"]) cmp_versions(logs, ["impl-cipher-ver", "impl-cert-ver", "impl-kex-ver"], ["side", "cipher", "cert", "kex", "record"], ["cpu", "energy"])
elif cmd == "prof": elif cmd == "prof":
for side in ["client", "server"]: for side in ["client", "server"]:
@ -926,3 +973,11 @@ if __name__ == "__main__":
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import numpy as np import numpy as np
make_linear_regression(logs, "alg") make_linear_regression(logs, "alg")
elif cmd == "stab":
# Args: stab <file1> <target1> [<file2> <target2>] [...]
logs_by_target = {sys.argv[3]: logs}
for i in range(4, len(sys.argv), 2):
logs_i, records_i = parse_logs(sys.argv[i])
target_i = sys.argv[i+1]
logs_by_target[target_i] = logs_i
make_stability_plot(logs_by_target)