From cf463794fbc238a2554c819298c0f2c5adbffbc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pascal=20Eng=C3=A9libert?= Date: Thu, 21 May 2026 10:27:03 +0200 Subject: [PATCH] Reproduce setting --- exp.py | 385 ++++++++++++++++++++++++++++--------------------------- plots.py | 91 ++++++++++--- 2 files changed, 268 insertions(+), 208 deletions(-) diff --git a/exp.py b/exp.py index d8609fc..06176fa 100644 --- a/exp.py +++ b/exp.py @@ -72,24 +72,25 @@ CONFIGS = { # i7-4790 -> pi3 "pi3": { "experiments": [ - #"impl-cipher-ver", - #"impl-cert-ver", - #"impl-kex-ver", + "impl-cipher-ver", + "impl-cert-ver", + "impl-kex-ver", #"zrtt", - "realistic" + #"realistic" ], "sides": [ "client", "server", ], "tls": [ - False, + #False, True, ], "records": [ - { "filename": "wp2", "repeat": 10000, "time": 600 }, - { "filename": "yt2-ads", "repeat": 10000, "time": 600 }, - { "filename": "yt2-ublock", "repeat": 10000, "time": 600 }, + { "filename": "wp2", "repeat": 10000, "time": 300 }, + #{ "filename": "yt2-ads", "repeat": 10000, "time": 600 }, + #{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 }, + #{ "filename": "wp2", "repeat": 10000, "time": 300, "reproduce": 100 }, ], "repo_dir": "/home/tuxmain/reps/tlsbench", "exp_dir": "/dev/shm/exp", @@ -112,22 +113,24 @@ CONFIGS = { # i7-4790 -> core2 "core2": { "experiments": [ - "impl-cipher-ver", + #"impl-cipher-ver", "impl-cert-ver", - "impl-kex-ver", - "zrtt", + #"impl-kex-ver", + #"zrtt", #"realistic", ], "sides": [ "client", - "server", + #"server", ], "tls": [ False, - True, + #True, ], "records": [ - { "filename": "wikipedia", "repeat": 2000 }, + { "filename": "wp2", "repeat": 10000, "time": 300 }, + #{ "filename": "yt2-ads", "repeat": 10000, "time": 600 }, + #{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 }, ], "repo_dir": "/home/tuxmain/reps/tlsbench", "exp_dir": "/dev/shm/exp", @@ -142,7 +145,7 @@ CONFIGS = { "rapl": False, "sa": True, "listen_port": 8080, - "idle": "idle - - - - - - - - - 1200.0000903606415 0 589 4764 3.478999999999999 0 -", + "idle": "idle - - - - - - - - - 1777360401.7482355 1777361601.7482355 1200.0000903606415 0 589 4764 3.478999999999999 0 -", "notify_listen": ("0.0.0.0", 8090), "notify_addr": "192.168.3.1:8090", }, @@ -181,22 +184,22 @@ CONFIGS = { "experiments": [ #"impl-cipher-ver", #"impl-cert-ver", - #"impl-kex-ver", + "impl-kex-ver", #"zrtt", - "realistic", + #"realistic", ], "sides": [ "client", "server", ], "tls": [ - #False, + False, True, ], "records": [ - { "filename": "wp2", "repeat": 10000, "time": 600 }, - { "filename": "yt2-ads", "repeat": 10000, "time": 600 }, - { "filename": "yt2-ublock", "repeat": 10000, "time": 600 }, + { "filename": "wp2", "repeat": 10000, "time": 300 }, + #{ "filename": "yt2-ads", "repeat": 10000, "time": 600 }, + #{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 }, ], "repo_dir": "/home/tuxmain/reps/tlsbench", "exp_dir": "/dev/shm/exp", @@ -267,9 +270,10 @@ CONFIGS = { "records": [ # Can't repeat more than 8000 times here # TODO check if netreplay client frees ports correctly, or try to reuse ports - { "filename": "wikipedia", "repeat": 8000, "time": 180 }, + { "filename": "wp2", "repeat": 8000, "time": 180 }, { "filename": "yt2-ads", "repeat": 8000, "time": 180 }, { "filename": "yt2-ublock", "repeat": 8000, "time": 180 }, + #{ "filename": "wp2", "repeat": 10000, "time": 300, "reproduce": 100 }, ], "repo_dir": "/home/pengelib/tlsbench", "exp_dir": "/dev/shm/exp", @@ -349,10 +353,10 @@ IMPLS = [ #"aws-lc", # Amazon's crypto widely used in Rust stuff #"boring", # Google's fork of OpenSSL used in Chrome and Android #"graviola", # New crypto in Rust - "openssl", # widely used + #"openssl", # widely used #"openssl-static", #"ring", # used in most Rust stuff - #"symcrypt", # Microsoft's crypto + "symcrypt", # Microsoft's crypto #"wolfcrypt" # used in embedded (won't build with rpxy for now) ] # Symmetric ciphers @@ -405,9 +409,9 @@ EXPERIMENTS = { "kexes": ["X25519"], "cert": [ "prime256v1", - #"secp384r1", + "secp384r1", "rsa2048", - #"rsa3072", + "rsa3072", "rsa4096", ], "earlydata": ["0"], @@ -422,7 +426,7 @@ EXPERIMENTS = { "kexes": [ "X25519", "SECP256R1", - #"SECP384R1", + "SECP384R1", "X25519MLKEM768", "SECP256R1MLKEM768", "MLKEM768", @@ -459,14 +463,14 @@ EXPERIMENTS = { "AES_256_GCM_SHA384", #"CHACHA20_POLY1305_SHA256", #"ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,ECDHE_RSA_WITH_AES_256_GCM_SHA384", + #"ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,ECDHE_RSA_WITH_AES_256_GCM_SHA384", #"ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", ], "kexes": [ "X25519", #"SECP256R1", #"SECP384R1", - "X25519MLKEM768", + #"X25519MLKEM768", #"SECP256R1MLKEM768", #"MLKEM768", ], @@ -478,7 +482,7 @@ EXPERIMENTS = { #"rsa4096", "realistic", ], - "earlydata": ["0"], + "earlydata": ["1"], }, } @@ -487,15 +491,15 @@ def alg_filter(kex, cert, cipher, impl): if "MLKEM" in kex and "WITH" in cipher: # WITH means TLS1.2 return False - if "MLKEM" in kex and "openssl" not in impl and impl != "aws-lc" and impl != "graviola": + if "MLKEM" in kex and "openssl" not in impl and impl != "aws-lc" and impl != "graviola" and impl != "boring": return False if kex == "SECP256R1MLKEM768" and "openssl" in impl: return False - if cert == "secp384r1" and impl == "boring": - return False + #if cert == "secp384r1" and impl == "boring": + # return False if kex == "SECP256R1MLKEM768" and impl == "graviola": return False - if kex == "MLKEM768" and impl == "graviola": + if kex == "MLKEM768" and impl in ["graviola", "boring"]: return False if cipher == "ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,ECDHE_RSA_WITH_AES_256_GCM_SHA384" and "openssl" in impl: return False @@ -846,165 +850,166 @@ def run_exp(config, only_record=None, idle=False, shutdown=False, debug=False): continue for side in config["sides"]: for record in config["records"]: - print(f"EXPERIMENT {expname}: {impl} {alg} {kex} {cipher} ED={earlydata} {side} TLS={tls}") - - run_id += 1 + for reprod_i in range(record.get("reproduce", 1)): + print(f"EXPERIMENT {expname}: {impl} {alg} {kex} {cipher} ED={earlydata} {side} TLS={tls}") + + run_id += 1 - if side == "client": - netreplay = run_netreplay_server( - None, - config["exp_dir"], - config["repo_dir"], - record, - "0.0.0.0", - config["listen_port"], - tls, - None, - certs_dir, - only_record=only_record, - ciphers=cipher, - kexes=kex, - debug=debug, - ld_preload=config["ld_preload"] if "ld_preload" in config else None, - ) - netreplay = run_netreplay_client( - ssh, - config["exp_dir"], - config["remote_repo_dir"], - record, - config["local_addr"], - config["listen_port"], - tls, - impl, - certs_dir, - only_record=only_record, - ciphers=cipher, - kexes=kex, - earlydata=earlydata, - debug=debug, - notify_addr=config["notify_addr"], - ld_preload=config["ld_preload"] if "ld_preload" in config else None, - concurrency=config.get("concurrency"), - ) - else: - netreplay = run_netreplay_server( - ssh, - config["exp_dir"], - config["remote_repo_dir"], - record, - "0.0.0.0", - config["listen_port"], - tls, - impl, - certs_dir, - only_record=only_record, - ciphers=cipher, - kexes=kex, - debug=debug, - ld_preload=config["ld_preload"] if "ld_preload" in config else None, - ) - netreplay = run_netreplay_client( - None, - config["exp_dir"], - config["repo_dir"], - record, - config["remote_addr"], - config["listen_port"], - tls, - None, - certs_dir, - only_record=only_record, - ciphers=cipher, - kexes=kex, - earlydata=earlydata, - debug=debug, - notify_addr=config["notify_addr"], - ld_preload=config["ld_preload"] if "ld_preload" in config else None, - skip_verif=True, - concurrency=config.get("concurrency"), - ) + if side == "client": + netreplay = run_netreplay_server( + None, + config["exp_dir"], + config["repo_dir"], + record, + "0.0.0.0", + config["listen_port"], + tls, + None, + certs_dir, + only_record=only_record, + ciphers=cipher, + kexes=kex, + debug=debug, + ld_preload=config["ld_preload"] if "ld_preload" in config else None, + ) + netreplay = run_netreplay_client( + ssh, + config["exp_dir"], + config["remote_repo_dir"], + record, + config["local_addr"], + config["listen_port"], + tls, + impl, + certs_dir, + only_record=only_record, + ciphers=cipher, + kexes=kex, + earlydata=earlydata, + debug=debug, + notify_addr=config["notify_addr"], + ld_preload=config["ld_preload"] if "ld_preload" in config else None, + concurrency=config.get("concurrency"), + ) + else: + netreplay = run_netreplay_server( + ssh, + config["exp_dir"], + config["remote_repo_dir"], + record, + "0.0.0.0", + config["listen_port"], + tls, + impl, + certs_dir, + only_record=only_record, + ciphers=cipher, + kexes=kex, + debug=debug, + ld_preload=config["ld_preload"] if "ld_preload" in config else None, + ) + netreplay = run_netreplay_client( + None, + config["exp_dir"], + config["repo_dir"], + record, + config["remote_addr"], + config["listen_port"], + tls, + None, + certs_dir, + only_record=only_record, + ciphers=cipher, + kexes=kex, + earlydata=earlydata, + debug=debug, + notify_addr=config["notify_addr"], + ld_preload=config["ld_preload"] if "ld_preload" in config else None, + skip_verif=True, + concurrency=config.get("concurrency"), + ) - prof_filename = "-" - if config["perf"]: - prof_filename = f"{perf_dir}/perf-{timestr}-{run_id}.data" - process_pid = ssh_run(ssh, "pidof netreplay-"+impl).removesuffix("\n") - ssh_run_bg(ssh, f"perf record -F 997 --call-graph dwarf,64000 -g -o {prof_filename} -p {process_pid}") - - # Measure - cpu = 0 - if config["sa"]: - cpu = get_cpu_stat(ssh, ["netreplay-"+impl, "tokio-runtime-w", "tokio-rt-worker"]) - remote_bytes_in, remote_bytes_out = get_net_stat(ssh) - energy = 0 - energy_rapl = 0 - if config["wattmeter"]: - energy = wattmeter.get_meter() - if config["rapl"]: - energy_rapl = get_rapl_energy(ssh, remote_path) - start = time.time() - - # Wait for the client to terminate - signal.alarm(record["time"] if "time" in record else 3600) - session_count = 0 - try: - while True: - rec_count = int.from_bytes(notify_socket.recv(4), "big") - if rec_count == 0xffffffff: - break - if rec_count > session_count: - if rec_count - session_count > 1: - print("count", session_count, "->", rec_count) - session_count = rec_count - except Timeout: - print("TIMEOUT: stop") - - # Measure - end = time.time() - new_energy = 0 - new_energy_rapl = 0 - if config["wattmeter"]: - new_energy = wattmeter.get_meter() - if config["rapl"]: - new_energy_rapl = get_rapl_energy(ssh, remote_path) - new_remote_bytes_in, new_remote_bytes_out = get_net_stat(ssh) - - # Kill netreplay - sh("killall netreplay") - try: - ssh_run(ssh, "killall netreplay-"+impl) - except invoke.exceptions.UnexpectedExit as e: - pass - - # flush the UDP queue - signal.alarm(2) - try: - while True: - notify_socket.recv(4) - except Timeout: - pass - - # Measure CPU after (as it may update only after the process is killed) - new_cpu = 0 - if config["sa"]: - new_cpu = get_cpu_stat(ssh, ["netreplay-"+impl, "tokio-runtime-w", "tokio-rt-worker"]) - - record_filename = record["filename"] - cpu_diff = new_cpu - cpu - remote_bytes_in_diff = new_remote_bytes_in - remote_bytes_in - remote_bytes_out_diff = new_remote_bytes_out - remote_bytes_out - energy_diff = new_energy - energy - energy_rapl_diff = new_energy_rapl - energy_rapl - time_diff = end - start - while True: + prof_filename = "-" + if config["perf"]: + prof_filename = f"{perf_dir}/perf-{timestr}-{run_id}.data" + process_pid = ssh_run(ssh, "pidof netreplay-"+impl).removesuffix("\n") + ssh_run_bg(ssh, f"perf record -F 997 --call-graph dwarf,64000 -g -o {prof_filename} -p {process_pid}") + + # Measure + cpu = 0 + if config["sa"]: + cpu = get_cpu_stat(ssh, ["netreplay-"+impl, "tokio-runtime-w", "tokio-rt-worker"]) + remote_bytes_in, remote_bytes_out = get_net_stat(ssh) + energy = 0 + energy_rapl = 0 + if config["wattmeter"]: + energy = wattmeter.get_meter() + if config["rapl"]: + energy_rapl = get_rapl_energy(ssh, remote_path) + start = time.time() + + # Wait for the client to terminate + signal.alarm(record["time"] if "time" in record else 3600) + session_count = 0 try: - with open(logfile_path, "a") as logfile: - logfile.write(f"{expname} {impl} {alg} {kex} {cipher} {earlydata} {side} {tls_int} {record_filename} {session_count} {start} {end} {time_diff} {cpu_diff} {remote_bytes_in_diff} {remote_bytes_out_diff} {energy_diff} {energy_rapl_diff} {prof_filename}\n") - logfile.close() - break - except Exception as e: - print("Can't open log file:", e) - time.sleep(1) - sh(f"cp {logfile_path} {log_backup_dir}/{logfile_name}") + while True: + rec_count = int.from_bytes(notify_socket.recv(4), "big") + if rec_count == 0xffffffff: + break + if rec_count > session_count: + if rec_count - session_count > 1: + print("count", session_count, "->", rec_count) + session_count = rec_count + except Timeout: + print("TIMEOUT: stop") + + # Measure + end = time.time() + new_energy = 0 + new_energy_rapl = 0 + if config["wattmeter"]: + new_energy = wattmeter.get_meter() + if config["rapl"]: + new_energy_rapl = get_rapl_energy(ssh, remote_path) + new_remote_bytes_in, new_remote_bytes_out = get_net_stat(ssh) + + # Kill netreplay + sh("killall netreplay") + try: + ssh_run(ssh, "killall netreplay-"+impl) + except invoke.exceptions.UnexpectedExit as e: + pass + + # flush the UDP queue + signal.alarm(2) + try: + while True: + notify_socket.recv(4) + except Timeout: + pass + + # Measure CPU after (as it may update only after the process is killed) + new_cpu = 0 + if config["sa"]: + new_cpu = get_cpu_stat(ssh, ["netreplay-"+impl, "tokio-runtime-w", "tokio-rt-worker"]) + + record_filename = record["filename"] + cpu_diff = new_cpu - cpu + remote_bytes_in_diff = new_remote_bytes_in - remote_bytes_in + remote_bytes_out_diff = new_remote_bytes_out - remote_bytes_out + energy_diff = new_energy - energy + energy_rapl_diff = new_energy_rapl - energy_rapl + time_diff = end - start + while True: + try: + with open(logfile_path, "a") as logfile: + logfile.write(f"{expname} {impl} {alg} {kex} {cipher} {earlydata} {side} {tls_int} {record_filename} {session_count} {start} {end} {time_diff} {cpu_diff} {remote_bytes_in_diff} {remote_bytes_out_diff} {energy_diff} {energy_rapl_diff} {prof_filename}\n") + logfile.close() + break + except Exception as e: + print("Can't open log file:", e) + time.sleep(1) + sh(f"cp {logfile_path} {log_backup_dir}/{logfile_name}") if config["wattmeter"]: YAPI.FreeAPI() if shutdown and ssh: diff --git a/plots.py b/plots.py index 41b31ec..f170577 100644 --- a/plots.py +++ b/plots.py @@ -283,7 +283,7 @@ def make_log_plot(logs, exp, criterion, side, obj, record, machine=None, version maketitle=maketitle ) -def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record): +def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record, version=None): f = open(f"/dev/shm/plots/{obj}_by_{criterion}_{side}_{record}.dat", "w") ciphers = {target: {} for target in logs_by_target} impls = [] @@ -305,9 +305,9 @@ def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record): n = float(log.get("n", "1000")) plain[target] = (float(log[COL[obj]]) * conv_factor - idle_val[target] * float(log["time"])) / n - for target in plain: - if plain[target] == None: - return + #for target in plain: + # if plain[target] == None: + # return for target in logs_by_target: logs = logs_by_target[target] @@ -334,17 +334,28 @@ def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record): for impl in impls: if impl not in ciphers[target][cipher]: ciphers[target][cipher][impl] = 0 - cipher_parts = cipher.split("/") - f.write("{} {}({}) {} {}\n".format( - target, - ALG_LABEL[cipher_parts[0]], - cipher_parts[1], - plain[target], - " ".join([ - str(ciphers[target][cipher][impl]) - for impl in impls - ]), - )) + if version == None: + cipher_parts = cipher.split("/") + f.write("{} {}({}) {} {}\n".format( + target, + ALG_LABEL[cipher_parts[0]], + cipher_parts[1], + plain[target], + " ".join([ + str(ciphers[target][cipher][impl] if ciphers[target][cipher][impl] > 0 else "-") + for impl in impls + ]), + )) + else: + f.write("{} {} {} {}\n".format( + target, + ALG_LABEL[cipher], + plain[target], + " ".join([ + str(ciphers[target][cipher][impl] if ciphers[target][cipher][impl] > 0 else "-") + for impl in impls + ]), + )) f.close() def make_profile_plot(logs, exp, criterion, side, record, no_flamegraph=False, machine=None, version=None, maketitle=False): @@ -822,6 +833,42 @@ def make_tx_summary(client_logs, server_logs): ]) print(tabulate(latex, separator=" & ", endline="\\\\ \\cline{2-8}", ruler=False)) +def compute_box_plot(a): + a.sort() + median = a[len(a)//2] if len(a)%2 == 1 else (a[len(a)//2-1]+a[len(a)//2])/2 + h1 = a[:len(a)//2] + h2 = a[(len(a)+1)//2:] + q1 = h1[len(h1)//2] if len(h1)%2 == 1 else (h1[len(h1)//2-1]+h1[len(h1)//2])/2 + q3 = h2[len(h2)//2] if len(h2)%2 == 1 else (h2[len(h2)//2-1]+h2[len(h2)//2])/2 + return [a[0], q1, median, q3, a[-1]] + +def make_stability_plot(logs_by_target): + f = open(f"/dev/shm/plots/stability.dat", "w") + idle_val = {target: None for target in logs_by_target} + + for target in logs_by_target: + logs = logs_by_target[target] + for log in logs: + if log["exp"] == "idle": + idle_val[target] = { + "energy": float(log[COL["energy"]]) / float(log["time"]) * 3600.0, + "io": float(log["bytes_in"])+float(log["bytes_out"]) / float(log["time"]), + } + + for target in logs_by_target: + logs = logs_by_target[target] + for log in logs: + if log["exp"] == "idle": + continue + n = float(log.get("n", "1000")) + f.write("{}-{} {} {}\n".format( + target, + log["side"], + (float(log[COL["energy"]]) * 3600.0 - idle_val[target]["energy"] * float(log["time"])) / n, + (float(log["bytes_in"])+float(log["bytes_out"]) - idle_val[target]["io"] * float(log["time"])) / n / 1024**2 + )) + f.close() + def getargv(arg:str, default="", n:int=1, args:list=sys.argv): if arg in args and len(args) > args.index(arg)+n: return args[args.index(arg)+n] @@ -889,9 +936,9 @@ if __name__ == "__main__": records.pop(record) for side in ["client", "server"]: for record in records: - make_log_plot_points(logs_by_target, "impl-cipher-ver", "cipher", side, "energy", record) - make_log_plot_points(logs_by_target, "impl-cert-ver", "cert", side, "energy", record) - make_log_plot_points(logs_by_target, "impl-kex-ver", "kex", side, "energy", record) + make_log_plot_points(logs_by_target, "impl-cipher-ver", "cipher", side, "energy", record, version="1.3") + make_log_plot_points(logs_by_target, "impl-cert-ver", "cert", side, "energy", record, version="1.3") + make_log_plot_points(logs_by_target, "impl-kex-ver", "kex", side, "energy", record, version="1.3") cmp_versions(logs, ["impl-cipher-ver", "impl-cert-ver", "impl-kex-ver"], ["side", "cipher", "cert", "kex", "record"], ["cpu", "energy"]) elif cmd == "prof": for side in ["client", "server"]: @@ -926,3 +973,11 @@ if __name__ == "__main__": import matplotlib.pyplot as plt import numpy as np make_linear_regression(logs, "alg") + elif cmd == "stab": + # Args: stab [ ] [...] + logs_by_target = {sys.argv[3]: logs} + for i in range(4, len(sys.argv), 2): + logs_i, records_i = parse_logs(sys.argv[i]) + target_i = sys.argv[i+1] + logs_by_target[target_i] = logs_i + make_stability_plot(logs_by_target)