Profile

README.md

@@ -107,6 +107,11 @@ Most of the implementations can be used through RusTLS.
 
 However RusTLS clients won't enable to force TLS1.2 if 1.3 is available.
 
+* séparer conso crypto / trafic
+* mesure avec debugger : quelles fonctions consomment
+* mesure mémoire
+* mesure overhead
+
 ### WolfSSL
 
 ```bash
@@ -146,6 +151,13 @@ sudo make install
 	* https://github.com/MarcT0K/privacy-carbon-experiments
 * https://davidtnaylor.com/CostOfTheS.pdf
 	* 2014
+* https://www.researchgate.net/publication/359906722_Performance_Analysis_of_SSLTLS_Crypto_Libraries_Based_on_Operating_Platform
+	* 2022
+	* throughput & cpu
+	* OpenSSL, GnuTLS, Boring, S2n, NSS, Cryptlib
+* https://www.haproxy.com/blog/state-of-ssl-stacks
+	* 2025
+	* OpenSSL
 
 ## Sources
 
@@ -203,6 +215,8 @@ Just browse. Any traffic to and from the selected names will be recorded. Termin
 
 ### Measure
 
+Measure resource cost on a different machine.
+
 Add p2 the `/etc/hosts`:
 
 ```
@@ -217,14 +231,49 @@ sudo chmod +s /sbin/sa
 ```
 
 ```bash
-python exp.py make -c
-python exp.py send
+python exp.py make pi -c
+python exp.py send pi
 python exp.py update-certs # also do this command on p2
-python exp.py run --idle
+python exp.py run pi --idle
 ```
 
 On Debian, update-certs says 0 certs added even if it has actually updated some certs. This step is still needed.
 
+### Profile
+
+Profile code execution on the local machine.
+
+Add the domains the `/etc/hosts`:
+
+```
+127.0.0.1	google.com.localhost
+127.0.0.1	www.google.com.localhost
+# etc.
+```
+
+Install sa:
+
+```bash
+sudo apt install acct
+sudo chmod +s /sbin/sa
+```
+
+Install OpenSSL with debug symbols:
+
+```bash
+./Configure --release -g
+```
+
+Backup your system's `libcrypto.so` and `libssl.so` and replace them with the new ones.
+It would be simpler with `LD_PRELOAD` but Rust loads dynamic libraries in a particuliar way so it doesn't work.
+
+```bash
+python exp.py make local -c
+python exp.py send local
+python exp.py update-certs
+python exp.py run local
+```
+
 ## Problems
 
 ### Youtube gives 502 bad gateway.

exp.py

@@ -1,13 +1,55 @@
 #!/usr/bin/python3
 import os, sys, subprocess
 
+CONFIGS = {
+	"pi": {
+		"experiments": [
+			"impl-cipher-ver",
+			"impl-cert-ver",
+			"impl-kex-ver",
+		],
+		"setups": [
+			"none",
+			"client",
+			"server",
+		],
+		"p2_hostname": "p2",
+		"p2_addr": "192.168.3.14",
+		"p2_ssh": "exp@p2",
+		"p2_psw": "exp",
+		"p2_repodir": "/home/exp/exp",
+		"wattmeter": True,
+		"perf": False,
+		"p3_suffix": "",
+		"p3_port_plain": 80,
+		"p3_port_tls": 443,
+	},
+	"local": {
+		"experiments": [
+			#"impl-cipher-ver",
+			"impl-cert-ver",
+			#"impl-kex-ver",
+		],
+		"setups": [
+			"none-local",
+			"client-local",
+			#"server-local",
+		],
+		"p2_hostname": "localhost",
+		"p2_addr": "127.0.0.1",
+		"p2_repodir": "/home/tuxmain/reps/tlsbench",
+		"wattmeter": False,
+		"perf": True,
+		"p3_suffix": ".localhost",
+		"p3_port_plain": 8080,
+		"p3_port_tls": 8443,
+	}
+}
+
 REPODIR = "/home/tuxmain/reps/tlsbench"
-P2_SSH = "exp@p2"
-P2_PSW = "exp"
 P2_REPODIR = "/home/exp/exp"
 EXPDIR = "/dev/shm/exp"
 LOG_BACKUP_DIR = "/home/tuxmain"
-P2_ADDR = "192.168.3.14"
 DOMAINS_ = [
 	# Apple
 	"apple.com", "www.apple.com", "graffiti-tags.apple.com", "securemetrics.apple.com",
@@ -31,12 +73,11 @@ DOMAINS_ = [
 	# Peertube video
 	"videos.domainepublic.net",
 ]
-WATTMETER = True
 
 RECORDS = [
-	#{ "filename": "youtube", "repeat": 1000 },
+	{ "filename": "youtube", "repeat": 2 },
 	#{ "filename": "peertube", "repeat": 10 },
-	{ "filename": "wikipedia", "repeat": 100 },
+	#{ "filename": "wikipedia", "repeat": 1 },
 	#{ "filename": "apple", "repeat": 1000 },
 	#{ "filename": "google", "repeat": 1000 },
 ]
@@ -48,9 +89,9 @@ CERT_SIGN_ALGS = [
 ]
 IMPLS = [
 	"aws_lc_rs", # Amazon's Rust crypto widely used in Rust stuff
-	"boring", # Google's fork of OpenSSL used in Chrome and Android
+	#"boring", # Google's fork of OpenSSL used in Chrome and Android
 	"openssl", # widely used
-	#"ring", # used in most Rust stuff
+	"ring", # used in most Rust stuff
 	#"symcrypt", # Microsoft's crypto
 	#"wolfcrypt" # used in embedded (won't build with rpxy for now)
 ]
@@ -92,7 +133,7 @@ EXPERIMENTS = {
 		"kexes": ["X25519"],
 		"cert": ["prime256v1"],
 	},
-	## Compare signatures among implementations and TLS versions
+	# Compare signatures among implementations and TLS versions
 	"impl-cert-ver": {
 		"impls": IMPLS,
 		"records": RECORDS,
@@ -108,7 +149,7 @@ EXPERIMENTS = {
 			"rsa3072", "rsa4096"
 		],
 	},
-	## Compare key exchange groups among implementations and TLS versions
+	# Compare key exchange groups among implementations and TLS versions
 	"impl-kex-ver": {
 		"impls": IMPLS,
 		"records": RECORDS,
@@ -119,16 +160,16 @@ EXPERIMENTS = {
 		"kexes": ["X25519", "SECP256R1", "SECP384R1"],
 		"cert": ["prime256v1"],
 	},
-	#"debug": {
-	#	"impls": IMPLS,
-	#	"records": RECORDS,
-	#	"ciphers": [
-	#		"ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,ECDHE_RSA_WITH_AES_128_GCM_SHA256",
-	#		"ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
-	#	],
-	#	"kexes": ["X25519"],
-	#	"cert": ["prime256v1"],
-	#},
+	"debug": {
+		"impls": IMPLS,
+		"records": RECORDS,
+		"ciphers": [
+			"ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+			#"ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
+		],
+		"kexes": ["X25519"],
+		"cert": ["prime256v1"],
+	},
 }
 
 DOMAINS = []
@@ -251,35 +292,35 @@ RPXY_CONFIGS = {
 		"listen_http": 80,
 		"listen_https": 443,
 		"app": """[apps.{app}]
-	server_name = "{domain}"
-	tls = {{ tls_cert_path = "{cert}", tls_cert_key_path = "{key}", https_redirection = false }}
-	reverse_proxy = [{{ upstream = [{{ location = "{domain}" }}], set_host = "{domain}"}}]
-	[apps.{app}_localhost]
-	server_name = "{domain}.localhost"
-	tls = {{ tls_cert_path = "{cert}", tls_cert_key_path = "{key}", https_redirection = false }}
-	reverse_proxy = [{{ upstream = [{{ location = "{domain}" }}], set_host = "{domain}"}}]
-	[apps.{app}_p2]
-	server_name = "{domain}.p2"
-	tls = {{ tls_cert_path = "{cert}", tls_cert_key_path = "{key}", https_redirection = false }}
-	reverse_proxy = [{{ upstream = [{{ location = "{domain}" }}], set_host = "{domain}"}}]
-	"""
+server_name = "{domain}"
+tls = {{ tls_cert_path = "{cert}", tls_cert_key_path = "{key}", https_redirection = false }}
+reverse_proxy = [{{ upstream = [{{ location = "{domain}{p3_suffix}:{p3_port_plain}" }}], set_host = "{domain}"}}]
+[apps.{app}_localhost]
+server_name = "{domain}.localhost"
+tls = {{ tls_cert_path = "{cert}", tls_cert_key_path = "{key}", https_redirection = false }}
+reverse_proxy = [{{ upstream = [{{ location = "{domain}{p3_suffix}:{p3_port_plain}" }}], set_host = "{domain}"}}]
+[apps.{app}_p2]
+server_name = "{domain}.p2"
+tls = {{ tls_cert_path = "{cert}", tls_cert_key_path = "{key}", https_redirection = false }}
+reverse_proxy = [{{ upstream = [{{ location = "{domain}{p3_suffix}:{p3_port_plain}" }}], set_host = "{domain}"}}]
+"""
 	},
 	"tls": {
 		"listen_http": 80,
 		"listen_https": 443,
 		"app": """[apps.{app}]
-	server_name = "{domain}"
-	tls = {{ tls_cert_path = "{cert}", tls_cert_key_path = "{key}", https_redirection = false }}
-	reverse_proxy = [{{ upstream = [{{ location = "{domain}", tls = true }}], set_host = "{domain}"}}]
-	[apps.{app}_localhost]
-	server_name = "{domain}.localhost"
-	tls = {{ tls_cert_path = "{cert}", tls_cert_key_path = "{key}", https_redirection = false }}
-	reverse_proxy = [{{ upstream = [{{ location = "{domain}", tls = true }}], set_host = "{domain}"}}]
-	[apps.{app}_p2]
-	server_name = "{domain}.p2"
-	tls = {{ tls_cert_path = "{cert}", tls_cert_key_path = "{key}", https_redirection = false }}
-	reverse_proxy = [{{ upstream = [{{ location = "{domain}", tls = true }}], set_host = "{domain}"}}]
-	"""
+server_name = "{domain}"
+tls = {{ tls_cert_path = "{cert}", tls_cert_key_path = "{key}", https_redirection = false }}
+reverse_proxy = [{{ upstream = [{{ location = "{domain}{p3_suffix}:{p3_port_tls}", tls = true }}], set_host = "{domain}"}}]
+[apps.{app}_localhost]
+server_name = "{domain}.localhost"
+tls = {{ tls_cert_path = "{cert}", tls_cert_key_path = "{key}", https_redirection = false }}
+reverse_proxy = [{{ upstream = [{{ location = "{domain}{p3_suffix}:{p3_port_tls}", tls = true }}], set_host = "{domain}"}}]
+[apps.{app}_p2]
+server_name = "{domain}.p2"
+tls = {{ tls_cert_path = "{cert}", tls_cert_key_path = "{key}", https_redirection = false }}
+reverse_proxy = [{{ upstream = [{{ location = "{domain}{p3_suffix}:{p3_port_tls}", tls = true }}], set_host = "{domain}"}}]
+"""
 	},
 }
 
@@ -305,36 +346,67 @@ SETUPS = {
 		"listen_port": 80,
 		"tls_invariant": False,
 	},
-	#"both": {
-	#	"rpxy_config": "tls",
-	#	"netreplay_tls_mode": "both",
-	#	"p2_port": 443,
-	#	"listen_port": 443,
-	#	"tls_invariant": False,
-	#},
+	"both": {
+		"rpxy_config": "tls",
+		"netreplay_tls_mode": "both",
+		"p2_port": 443,
+		"listen_port": 443,
+		"tls_invariant": False,
+	},
+	"none-local": {
+		"rpxy_config": "plain",
+		"netreplay_tls_mode": "none",
+		"p2_port": 80,
+		"listen_port": 8080,
+		"tls_invariant": True,
+	},
+	"client-local": {
+		"rpxy_config": "tls",
+		"netreplay_tls_mode": "server",
+		"p2_port": 80,
+		"listen_port": 8443,
+		"tls_invariant": False,
+	},
+	"server-local": {
+		"rpxy_config": "plain",
+		"netreplay_tls_mode": "client",
+		"p2_port": 443,
+		"listen_port": 8080,
+		"tls_invariant": False,
+	},
+	"both-local": {
+		"rpxy_config": "tls",
+		"netreplay_tls_mode": "both",
+		"p2_port": 443,
+		"listen_port": 8443,
+		"tls_invariant": False,
+	},
 }
 
-def make_rpxy_config(outdir, domains, cryptodir, config_name):
+def make_rpxy_config(outdir, domains, cryptodir, config_name, p3_suffix, p3_port_plain, p3_port_tls):
 	if outdir[-1] != "/":
 		outdir += "/"
 	if cryptodir[-1] != "/":
 		cryptodir += "/"
 	
-	config = RPXY_CONFIGS[config_name]
+	rpxy_config = RPXY_CONFIGS[config_name]
 	f = open(outdir+config_name+".toml", "w")
-	f.write("listen_port = {}\nlisten_port_tls = {}\n".format(config["listen_http"], config["listen_https"]))
+	f.write("listen_port = {}\nlisten_port_tls = {}\n".format(rpxy_config["listen_http"], rpxy_config["listen_https"]))
 	for domain in domains:
 		app = domain.replace(".", "_")
 		root = get_domain_root(domain)
-		f.write(config["app"].format(
+		f.write(rpxy_config["app"].format(
 			app=app,
 			domain=domain,
 			cert=cryptodir+root+".crt",
 			key=cryptodir+root+".key",
+			p3_suffix=p3_suffix,
+			p3_port_plain=p3_port_plain,
+			p3_port_tls=p3_port_tls,
 		))
 	f.close()
 
-def make_everything(expdir, domains, make_ca):
+def make_everything(expdir, domains, make_ca, p3_suffix, p3_port_plain, p3_port_tls):
 	os.makedirs(expdir, exist_ok=True)
 	if expdir[-1] != "/":
 		expdir += "/"
@@ -347,7 +419,7 @@ def make_everything(expdir, domains, make_ca):
 	configdir = expdir+"configs/"
 	os.makedirs(configdir, exist_ok=True)
 	for config_name in RPXY_CONFIGS:
-		make_rpxy_config(configdir, domains, cryptodir, config_name)
+		make_rpxy_config(configdir, domains, cryptodir, config_name, p3_suffix, p3_port_plain, p3_port_tls)
 
 def choose_cert_alg(expdir, alg):
 	if expdir[-1] != "/":
@@ -368,7 +440,7 @@ def run_netreplay(expdir, repodir, record, p2_addr, p2_port, listen_port, tls_mo
 	if expdir[-1] != "/":
 		expdir += "/"
 	repodir = repodir.removesuffix("/")
-	env = {"RUST_LOG": "debug"}
+	env = {"RUST_LOG": "warning"}
 	if ciphers:
 		env["CIPHERS"] = ciphers
 	if kexes:
@@ -379,31 +451,54 @@ def run_netreplay(expdir, repodir, record, p2_addr, p2_port, listen_port, tls_mo
 	print(" ".join(cmd))
 	return subprocess.Popen(cmd, env=env)
 
-# https://stackoverflow.com/questions/8775598/how-to-start-a-background-process-with-nohup-using-fabric
-def runbg(ssh, cmd, vars={}):
-	print("[SSH]", cmd)
-	strvars = ""
-	for var in vars:
-		strvars += f"export {var}="+vars[var]+" && "
-	return ssh.run(f"{strvars}dtach -n `mktemp -u /tmp/dtach.XXXX` {cmd}")
+# Run with or without SSH
+def ssh_run(ssh, cmd, env={}, **kwargs):
+	if ssh == None:
+		# As long as there is no argument containing a space (escaped or quoted), we're fine
+		proc = subprocess.Popen(cmd.split(" "), stdout=subprocess.PIPE, env=env)
+		proc.wait()
+		return proc.stdout.read().decode()
+	else:
+		strvars = ""
+		for var in env:
+			strvars += var+"="+env[var]+" "
+		return ssh.run(strvars+cmd, **kwargs).stdout
+
+# Run with or without SSH
+def ssh_run_bg(ssh, cmd, env={}, **kwargs):
+	if ssh == None:
+		# As long as there is no argument containing a space (escaped or quoted), we're fine
+		print(cmd)
+		return subprocess.Popen(cmd.split(" "), env=env)
+	else:
+		# https://stackoverflow.com/questions/8775598/how-to-start-a-background-process-with-nohup-using-fabric
+		strvars = ""
+		for var in env:
+			strvars += "export "+var+"="+env[var].replace(" ", "\\ ")+" && "
+		return ssh.run(f"{strvars}dtach -n `mktemp -u /tmp/dtach.XXXX` {cmd}", **kwargs)
 
 def get_cpu_stat(ssh):
-	res = ssh.run("/sbin/sa --list-all-names", hide=True)
-	for line in res.stdout.split("\n"):
+	res = ssh_run(ssh, "/sbin/sa --list-all-names", hide=True)
+	for line in res.split("\n"):
 		if "rpxy" in line:
 			return float(re.finditer("\\s(\\d+\\.\\d+)cp\\s", line).__next__().group(1))
 	return 0.0
 
 def get_net_stat(ssh):
-	res = ssh.run("cat /proc/net/netstat", hide=True)
-	items = res.stdout.split("\n")[3].split(" ")
+	res = ssh_run(ssh, "cat /proc/net/netstat", hide=True)
+	items = res.split("\n")[3].split(" ")
 	bytes_in = int(items[7])
 	bytes_out = int(items[8])
 	return (bytes_in, bytes_out)
 
-def run_exp(ssh, expdir, p2_path, exps, only_record=None, idle=False):
+def run_exp(expdir, config, only_record=None, idle=False):
+	ssh = None
+	if "p2_ssh" in config:
+		ssh = connect_ssh(config)
+
+	p2_path = config["p2_repodir"]
 	wattmeter = None
-	if WATTMETER:
+	if config["wattmeter"]:
 		errmsg = YRefParam()
 		if YAPI.RegisterHub("usb", errmsg) != YAPI.SUCCESS:
 			sys.exit("init error" + errmsg.value)
@@ -413,14 +508,15 @@ def run_exp(ssh, expdir, p2_path, exps, only_record=None, idle=False):
 			exit(1)
 	
 	sh("killall netreplay")
-	for expname in exps:
-		exp = exps[expname]
+	for expname in config["experiments"]:
+		exp = EXPERIMENTS[expname]
 		for impl in exp["impls"]:
 			try:
-				ssh.run(f"killall rpxy_rustls_{impl}")
+				ssh_run(ssh, f"killall rpxy_rustls_{impl}")
 			except invoke.exceptions.UnexpectedExit as e:
 				pass
-	logfile_name = "log-"+str(int(time.time()))
+	timestr = str(int(time.time()))
+	logfile_name = "log-"+timestr
 	logfile_path = expdir+"/"+logfile_name
 	logfile = open(logfile_path, "w")
 	logfile.write("exp impl alg kex cipher setup record time cpu bytes_in bytes_out Wh\n")
@@ -431,7 +527,7 @@ def run_exp(ssh, expdir, p2_path, exps, only_record=None, idle=False):
 		rpxy_cpu = get_cpu_stat(ssh)
 		p2_bytes_in, p2_bytes_out = get_net_stat(ssh)
 		energy = 0
-		if WATTMETER:
+		if config["wattmeter"]:
 			energy = wattmeter.get_meter()
 		start = time.time()
 
@@ -439,7 +535,7 @@ def run_exp(ssh, expdir, p2_path, exps, only_record=None, idle=False):
 
 		end = time.time()
 		new_energy = 0
-		if WATTMETER:
+		if config["wattmeter"]:
 			new_energy = wattmeter.get_meter()
 		new_p2_bytes_in, new_p2_bytes_out = get_net_stat(ssh)
 		new_rpxy_cpu = get_cpu_stat(ssh)
@@ -469,35 +565,42 @@ def run_exp(ssh, expdir, p2_path, exps, only_record=None, idle=False):
 				time.sleep(1)
 	sh(f"cp {logfile_path} {LOG_BACKUP_DIR}/{logfile_name}")
 
-	for expname in exps:
-		exp = exps[expname]
+	run_id = 0
+	for expname in config["experiments"]:
+		exp = EXPERIMENTS[expname]
 		first_set = True
 		for impl in exp["impls"]:
 			for alg in exp["cert"]:
 				for kex in exp["kexes"]:
 					for cipher in exp["ciphers"]:
 						choose_cert_alg(expdir, alg)
-						ssh.run(f"python {p2_path}/exp.py cert {alg}")
-						for setup in SETUPS:
+						ssh_run(ssh, f"python {p2_path}/exp.py cert {alg}")
+						for setup in config["setups"]:
 							if SETUPS[setup]["tls_invariant"] and not first_set:
 								continue
 							setupdir = expdir+"setups/"+setup
 							for record in exp["records"]:
 								print(f"EXPERIMENT {expname}: {impl} {alg} {kex} {cipher} {setup}")
 								p2_rpxy_config = SETUPS[setup]["rpxy_config"]
-								vars = {"CIPHERS": cipher, "KEXES": kex, "RUST_LOG": "debug"}
-								
-								runbg(ssh, f"{p2_path}/rpxy_rustls_{impl} --config {expdir}/configs/{p2_rpxy_config}.toml --log-dir /dev/shm", vars)
+								vars = {"CIPHERS": cipher, "KEXES": kex, "RUST_LOG": "debug", "LD_PRELOAD": "/dev/shm/openssl-3.6.0/libssl.so.3 /dev/shm/openssl-3.6.0/libcrypto.so.3"}
+								cmd = f"{p2_path}/rpxy_rustls_{impl} --config {expdir}/configs/{p2_rpxy_config}.toml --log-dir /dev/shm"
+								ssh_run_bg(ssh, cmd, env=vars)
 								time.sleep(1)
+
+								if config["perf"]:
+									rpxy_pid = ssh_run(ssh, f"pidof rpxy_rustls_{impl}").removesuffix("\n")
+									ssh_run_bg(ssh, f"perf record -F 997 --call-graph dwarf,64000 -g -o {expdir}/perf-{timestr}-{run_id}.data -p {rpxy_pid}")
+								
+								run_id += 1
 								
 								rpxy_cpu = get_cpu_stat(ssh)
 								p2_bytes_in, p2_bytes_out = get_net_stat(ssh)
 								energy = 0
-								if WATTMETER:
+								if config["wattmeter"]:
 									energy = wattmeter.get_meter()
 								
 								start = time.time()
-								netreplay = run_netreplay(expdir, REPODIR, record, P2_ADDR, SETUPS[setup]["p2_port"], SETUPS[setup]["listen_port"], SETUPS[setup]["netreplay_tls_mode"], only_record=only_record, ciphers=cipher, kexes=kex)
+								netreplay = run_netreplay(expdir, REPODIR, record, config["p2_addr"], SETUPS[setup]["p2_port"], SETUPS[setup]["listen_port"], SETUPS[setup]["netreplay_tls_mode"], only_record=only_record, ciphers=cipher, kexes=kex)
 								
 								# TODO detect when netreplay has finished
 								try:
@@ -505,11 +608,11 @@ def run_exp(ssh, expdir, p2_path, exps, only_record=None, idle=False):
 								except KeyboardInterrupt:
 									netreplay.kill()
 									try:
-										ssh.run(f"killall rpxy_rustls_{impl}")
+										ssh_run(ssh, f"killall rpxy_rustls_{impl}")
 									except invoke.exceptions.UnexpectedExit as e:
 										pass
 									try:
-										ssh.run(f"killall dtach")
+										ssh_run(ssh, f"killall dtach")
 									except invoke.exceptions.UnexpectedExit as e:
 										pass
 									exit(0)
@@ -517,21 +620,22 @@ def run_exp(ssh, expdir, p2_path, exps, only_record=None, idle=False):
 								#time.sleep(30)
 								#sh("killall netreplay")
 								try:
-									ssh.run(f"rm /dev/shm/access.log /dev/shm/rpxy.log")
+									#ssh_run(ssh, f"rm /dev/shm/access.log /dev/shm/rpxy.log")
+									pass
 								except invoke.exceptions.UnexpectedExit as e:
 									pass
 								try:
-									ssh.run(f"killall rpxy_rustls_{impl}")
+									ssh_run(ssh, f"killall rpxy_rustls_{impl}")
 								except invoke.exceptions.UnexpectedExit as e:
 									pass
 								try:
-									ssh.run(f"killall dtach")
+									ssh_run(ssh, f"killall dtach")
 								except invoke.exceptions.UnexpectedExit as e:
 									pass
 								end = time.time()
 								
 								new_energy = 0
-								if WATTMETER:
+								if config["wattmeter"]:
 									new_energy = wattmeter.get_meter()
 								new_p2_bytes_in, new_p2_bytes_out = get_net_stat(ssh)
 								new_rpxy_cpu = get_cpu_stat(ssh)
@@ -552,7 +656,7 @@ def run_exp(ssh, expdir, p2_path, exps, only_record=None, idle=False):
 										time.sleep(1)
 								sh(f"cp {logfile_path} {LOG_BACKUP_DIR}/{logfile_name}")
 							first_set = False
-	if WATTMETER:
+	if config["wattmeter"]:
 		YAPI.FreeAPI()
 
 def update_certs():
@@ -583,15 +687,15 @@ def upload_dir(ssh, src, dst):
 	print(ssh.put(src+"/../tmp.tar.gz", dst))
 	print(ssh.run(f"cd {dst} && tar -xf tmp.tar.gz"))
 
-def connect_ssh():
+def connect_ssh(config):
 	ssh_passphrase = "--passphrase" in sys.argv
 	connect_kwargs = {}
 	if ssh_passphrase:
 		import getpass
 		connect_kwargs["passphrase"] = getpass.getpass("Enter passphrase for SSH key: ")
-	if P2_PSW != None:
-		connect_kwargs["password"] = P2_PSW
-	ssh = fabric.Connection(P2_SSH, connect_kwargs=connect_kwargs)
+	if "p2_psw" in config:
+		connect_kwargs["password"] = config["p2_psw"]
+	ssh = fabric.Connection(config["p2_ssh"], connect_kwargs=connect_kwargs)
 	return ssh
 
 if __name__ == "__main__":
@@ -599,35 +703,37 @@ if __name__ == "__main__":
 		print("""Options:
   make [-c]        Create everything
   cert <alg>       Select cert signature algorithm
-  send             Send configs and certs to p2
+  send <config>    Send configs and certs to p2
   update-certs     Update system's certs
-  run              Run experiment
+  run <config>     Run experiment
   script           Print Firefox script to override DNS
 
-Signature algorithms:
-  {sig_algs}
-
-Implementations:
-  {impls}
-
 Make options:
   -c  Make CA cert (otherwise use already existing one)
 
+Cert options:
+  <alg>            One of: {sig_algs}
+
+Send options:
+  <config>         One of: {configs}
+
 Run options:
-  --passphrase   Prompt SSH key decryption passphrase (when using pubkey login)
-  --count        Do not run experiments but display number of experiments
-  --record <id>  Only play this record
-  --idle         Also measure when idle
+  <config>         One of: {configs}
+  --passphrase     Prompt SSH key decryption passphrase (when using pubkey login)
+  --count          Do not run experiments but display number of experiments
+  --record <id>    Only play this record
+  --idle           Also measure when idle
 """.format(
-	sig_algs=" ".join(CERT_SIGN_ALGS),
-	impls=" ".join(IMPLS),
+	sig_algs = " ".join(CERT_SIGN_ALGS),
+	configs = " ".join([i for i in CONFIGS]),
 ))
 		exit()
 	
 	opt = sys.argv[1]
 	if opt == "make":
+		config = CONFIGS[sys.argv[2]]
 		make_ca = "-c" in sys.argv
-		make_everything(EXPDIR, DOMAINS, make_ca)
+		make_everything(EXPDIR, DOMAINS, make_ca, config["p3_suffix"], config["p3_port_plain"], config["p3_port_tls"])
 	elif opt == "cert":
 		alg = sys.argv[2]
 		if not alg in CERT_SIGN_ALGS:
@@ -635,19 +741,21 @@ Run options:
 			exit(1)
 		choose_cert_alg(EXPDIR, alg)
 	elif opt == "send":
+		config = CONFIGS[sys.argv[2]]
 		import fabric
-		ssh = connect_ssh()
+		ssh = connect_ssh(config)
 		upload_dir(ssh, EXPDIR, "/dev/shm")
 	elif opt == "update-certs":
 		import platform
 		update_certs()
 	elif opt == "run":
+		config = CONFIGS[sys.argv[2]]
 		if "--count" in sys.argv:
 			exps = 0
-			for expname in EXPERIMENTS:
-				exp = EXPERIMENTS[expname]
+			for expname in config["experiments"]:
+				exp = config["experiments"][expname]
 				exps += len(exp["impls"]) * len(exp["cert"]) * len(exp["kexes"]) * len(exp["ciphers"]) * len(exp["records"])
-			print("Experiments to make:", exps * len(SETUPS))
+			print("Experiments to make:", exps * len(config["setups"]))
 			exit(0)
 		
 		import time
@@ -655,13 +763,12 @@ Run options:
 		import re
 		import fabric
 		
-		if WATTMETER:
+		if config["wattmeter"]:
 			import yoctopuce
 			from yoctopuce.yocto_api import *
 			from yoctopuce.yocto_power import *
-		
-		ssh = connect_ssh()
-		run_exp(ssh, EXPDIR, P2_REPODIR, EXPERIMENTS, only_record=getargv("--record", None), idle="--idle" in sys.argv)
+
+		run_exp(EXPDIR, config, only_record=getargv("--record", None), idle="--idle" in sys.argv)
 	elif opt == "script":
 		print(SCRIPT_FIREFOX_HOSTS)
 	else:

plots.py

@@ -59,67 +59,71 @@ def gnuplot_histogram(**kwargs):
 	cluster = ""
 	for i in range(kwargs["nb_impls"]-1):
 		cluster += """, "" using {}:xticlabels(1) title col""".format(i+4)
-	f = open("{plots_dir}/{object}_by_{criterion}_{side}.gnuplot".format(plots_dir=PLOTS_DIR, **kwargs), "w")
+	f = open("{plots_dir}/{object}_by_{criterion}_{side}_{record}.gnuplot".format(plots_dir=PLOTS_DIR, **kwargs), "w")
 	f.write("""\
 set terminal pngcairo enhanced font "CMU Sans Serif,11" fontscale 1.0 size 800, 600
-set output "{plots_dir}/{object}_by_{criterion}_{side}.png"
+set output "{plots_dir}/{object}_by_{criterion}_{side}_{record}.png"
 set boxwidth 0.9 absolute
 set style fill solid 1.0 border lt -1
 set style histogram clustered gap 1 title textcolor lt -1
 set style data histograms
-set title font "CMU Sans Serif,12" "{object_title} by {criterion_title} ({side} side) ({unit})"
-set xtics border in scale 0,0 nomirror rotate by -45 autojustify
-set key fixed right top vertical Right noreverse noenhanced autotitle nobox
+set title font "CMU Sans Serif,12" "{object_title} by {criterion_title} ({record}, {side} side) ({unit})"
+#set xtics border in scale 0,0 nomirror rotate by -45 autojustify
+set xtics border in scale 0,0 nomirror autojustify
+#set key fixed right top vertical Right noreverse noenhanced autotitle nobox
+set key fixed left top vertical Left reverse noenhanced autotitle nobox
 set colorbox vertical origin screen 0.9, 0.2 size screen 0.05, 0.6 front noinvert bdefault
 set xrange [ * : * ] noreverse writeback
+set yrange [ 0 : * ]
+set grid y lt 1 lw .75 lc "gray"
 plot \
-	newhistogram "", "{plots_dir}/{object}_by_{criterion}_{side}.dat" using 2:xticlabels(1) notitle col, \
-	newhistogram "", "{plots_dir}/{object}_by_{criterion}_{side}.dat" using 3:xticlabels(1) title col{cluster}
+	newhistogram "", "{plots_dir}/{object}_by_{criterion}_{side}_{record}.dat" using 2:xticlabels(1) notitle col, \
+	newhistogram "", "{plots_dir}/{object}_by_{criterion}_{side}_{record}.dat" using 3:xticlabels(1) title col{cluster}
 """.format(plots_dir=PLOTS_DIR, cluster=cluster, **kwargs))
 	f.close()
-	os.system("gnuplot {plots_dir}/{object}_by_{criterion}_{side}.gnuplot".format(plots_dir=PLOTS_DIR, **kwargs))
+	os.system("gnuplot {plots_dir}/{object}_by_{criterion}_{side}_{record}.gnuplot".format(plots_dir=PLOTS_DIR, **kwargs))
 
-def make_plot(records, exp, criterion, side, obj):
-	f = open(f"/dev/shm/plots/{obj}_by_{criterion}_{side}.dat", "w")
+def make_plot(logs, exp, criterion, side, obj, record):
+	f = open(f"/dev/shm/plots/{obj}_by_{criterion}_{side}_{record}.dat", "w")
 	ciphers = {}
 	impls = []
 	plain_line = None
 	idle_val = None
 	
-	for record in records:
-		if record["exp"] == "idle":
-			idle_val = float(record[COL[obj]]) / float(record["time"])
-		if record["exp"] != exp:
+	for log in logs:
+		if log["exp"] == "idle":
+			idle_val = float(log[COL[obj]]) / float(log["time"])
+		if log["exp"] != exp or log["record"] != record:
 			continue
-		if record["setup"] == "none":
-			plain_line = "plain {}".format(float(record[COL[obj]]) - idle_val * float(record["time"]))
+		if log["setup"] == "none":
+			plain_line = "plain {}".format(float(log[COL[obj]]) - idle_val * float(log["time"]))
 	
 	if plain_line == None:
 		return
 	
-	for record in records:
-		if record["exp"] != exp:
+	for log in logs:
+		if log["exp"] != exp or log["record"] != record:
 			continue
-		elif record["setup"] == side:
-			if record[COL[criterion]] not in ciphers:
-				ciphers[record[COL[criterion]]] = {}
-			ciphers[record[COL[criterion]]][record["impl"]] = float(record[COL[obj]]) - idle_val * float(record["time"])
-			if record["impl"] not in impls:
-				impls.append(record["impl"])
+		elif log["setup"] == side:
+			if log[COL[criterion]] not in ciphers:
+				ciphers[log[COL[criterion]]] = {}
+			ciphers[log[COL[criterion]]][log["impl"]] = float(log[COL[obj]]) - idle_val * float(log["time"])
+			if log["impl"] not in impls:
+				impls.append(log["impl"])
 	impls.sort()
 	f.write("{} none {}\n".format(criterion, " ".join(impls)))
 	f.write(plain_line+" -"*len(impls)+"\n")
 	for cipher in ciphers:
 		f.write("{}({}) - {}\n".format(
 			ALG_LABEL[cipher],
-			VER_LABEL[record["cipher"]],
+			VER_LABEL[log["cipher"]],
 			" ".join([
 				str(ciphers[cipher][impl])
 				for impl in impls
 			]),
 		))
 	f.close()
-	gnuplot_histogram(object=obj, criterion=criterion, side=side, object_title=OBJ_TITLE[obj], criterion_title=CRITERION_TITLE[criterion], unit=UNIT[obj], nb_impls=len(impls))
+	gnuplot_histogram(object=obj, criterion=criterion, side=side, object_title=OBJ_TITLE[obj], criterion_title=CRITERION_TITLE[criterion], unit=UNIT[obj], nb_impls=len(impls), record=record)
 
 if __name__ == "__main__":
 	logfile_name = sys.argv[1]
@@ -129,20 +133,24 @@ if __name__ == "__main__":
 
 	colnames = lines[0].removesuffix("\n").split(" ")
 
-	records = []
+	logs = []
+	records = {}
 	for line in lines[1:]:
 		cols = line.removesuffix("\n").split(" ")
-		record = {}
+		log = {}
 		for col in range(len(cols)):
-			record[colnames[col]] = cols[col]
-		records.append(record)
+			log[colnames[col]] = cols[col]
+		if log["record"] != "-":
+			records[log["record"]] = ()
+		logs.append(log)
 	
 	os.makedirs("/dev/shm/plots", exist_ok=True)
 
 	for side in ["client", "server"]:
-		make_plot(records, "impl-cipher-ver", "cipher", side, "cpu")
-		make_plot(records, "impl-cipher-ver", "cipher", side, "energy")
-		make_plot(records, "impl-cert-ver", "cert", side, "cpu")
-		make_plot(records, "impl-cert-ver", "cert", side, "energy")
-		make_plot(records, "impl-kex-ver", "kex", side, "cpu")
-		make_plot(records, "impl-kex-ver", "kex", side, "energy")
+		for record in records:
+			make_plot(logs, "impl-cipher-ver", "cipher", side, "cpu", record)
+			make_plot(logs, "impl-cipher-ver", "cipher", side, "energy", record)
+			make_plot(logs, "impl-cert-ver", "cert", side, "cpu", record)
+			make_plot(logs, "impl-cert-ver", "cert", side, "energy", record)
+			make_plot(logs, "impl-kex-ver", "kex", side, "cpu", record)
+			make_plot(logs, "impl-kex-ver", "kex", side, "energy", record)

profile.py

@@ -0,0 +1,61 @@
+# Extract some data from profile flamegraphs produced by cargo flamegraph
+
+import re, sys
+
+FUNCTIONS = {
+	"rustls::record_layer::RecordLayer::decrypt_incoming": "decrypt",
+
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::cipher::MessageEncrypter>::encrypt": "encrypt",
+
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::tls13::Hkdf>::expander_for_okm": "hkdf",
+
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::SecureRandom>::fill": "rand",
+
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::SupportedKxGroup>::start": "kx",
+
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::hash::Hash>::start": "hash",
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::hash::Context>::finish": "hash",
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::hash::Context>::update": "hash",
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::hash::Context>::fork_finish": "hash",
+
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::tls13::Hkdf>::extract_from_secret": "hkdf",
+
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::ActiveKeyExchange>::complete": "kx",
+
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::tls13::HkdfExpander>::hash_len": "hkdf",
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::tls13::HkdfExpander>::expand_slice": "hkdf",
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::tls13::Hkdf>::extract_from_secret": "hkdf",
+	"<[a-zA-Z0-9_:]+ as rustls::crypto::tls13::Hkdf>::hmac_sign": "hkdf",
+
+	"ring::hkdf::fill_okm": "hkdf",
+	"aws_lc_0_32_2_HKDF": "hkdf",
+
+	"rustls_openssl::tls13::<impl rustls::crypto::cipher::Tls13AeadAlgorithm for rustls_openssl::aead::Algorithm>::encrypter": "encrypt",
+	"rustls::crypto::aws_lc_rs::tls13::AeadAlgorithm::encrypter": "encrypt",
+
+	"rustls::crypto::aws_lc_rs::tls13::AeadAlgorithm::decrypter": "decrypt",
+}
+
+def extract_function(data, name):
+	r = re.compile(name.replace("<", "&lt;").replace(">", "&gt;") + " \\(([0-9,]+) samples, ([0-9.]+)%\\)")
+	samples = 0
+	percents = 0.0
+	for match in r.finditer(data):
+		samples += int(match.group(1).replace(",", ""))
+		percents += float(match.group(2))
+	return (samples, percents)
+
+if __name__ == "__main__":
+	filename = sys.argv[1]
+	f = open(filename, "r")
+	c = f.read()
+	results = {}
+	for function in FUNCTIONS:
+		samples, percents = extract_function(c, function)
+		title = FUNCTIONS[function]
+		if title not in results:
+			results[title] = [0, 0.0]
+		results[title][0] += samples
+		results[title][1] += percents
+	for title in results:
+		print(title, results[title])