plots: fixes for multitarget

exp.py

@@ -72,14 +72,14 @@ CONFIGS = {
 	# i7-4790 -> pi3
 	"pi3": {
 		"experiments": [
-			"impl-cipher-ver",
+			#"impl-cipher-ver",
 			"impl-cert-ver",
-			"impl-kex-ver",
+			#"impl-kex-ver",
 			#"zrtt",
 			#"realistic"
 		],
 		"sides": [
-			"client",
+			#"client",
 			"server",
 		],
 		"tls": [
@@ -113,9 +113,9 @@ CONFIGS = {
 	# i7-4790 -> core2
 	"core2": {
 		"experiments": [
-			#"impl-cipher-ver",
+			"impl-cipher-ver",
 			"impl-cert-ver",
-			#"impl-kex-ver",
+			"impl-kex-ver",
 			#"zrtt",
 			#"realistic",
 		],
@@ -124,8 +124,8 @@ CONFIGS = {
 			#"server",
 		],
 		"tls": [
-			False,
+			#False,
-			#True,
+			True,
 		],
 		"records": [
 			{ "filename": "wp2", "repeat": 10000, "time": 300 },
@@ -182,9 +182,9 @@ CONFIGS = {
 	# i7-4790 -> i5-7300HQ
 	"i5": {
 		"experiments": [
-			#"impl-cipher-ver",
+			"impl-cipher-ver",
-			#"impl-cert-ver",
+			"impl-cert-ver",
-			"impl-kex-ver",
+			#"impl-kex-ver",
 			#"zrtt",
 			#"realistic",
 		],
@@ -193,13 +193,14 @@ CONFIGS = {
 			"server",
 		],
 		"tls": [
-			False,
+			#False,
 			True,
 		],
 		"records": [
 			{ "filename": "wp2", "repeat": 10000, "time": 300 },
 			#{ "filename": "yt2-ads", "repeat": 10000, "time": 600 },
 			#{ "filename": "yt2-ublock", "repeat": 10000, "time": 600 },
+			#{ "filename": "wp2", "repeat": 10000, "time": 300, "reproduce": 100 },
 		],
 		"repo_dir": "/home/tuxmain/reps/tlsbench",
 		"exp_dir": "/dev/shm/exp",
@@ -357,7 +358,6 @@ IMPLS = [
 	#"openssl-static",
 	#"ring", # used in most Rust stuff
 	"symcrypt", # Microsoft's crypto
-	#"wolfcrypt" # used in embedded (won't build with rpxy for now)
 ]
 # Symmetric ciphers
 # They also allow to choose the TLS version.
@@ -497,7 +497,7 @@ def alg_filter(kex, cert, cipher, impl):
 		return False
 	#if cert == "secp384r1" and impl == "boring":
 	#	return False
-	if kex == "SECP256R1MLKEM768" and impl == "graviola":
+	if kex == "SECP256R1MLKEM768" and impl in ["graviola", "boring"]:
 		return False
 	if kex == "MLKEM768" and impl in ["graviola", "boring"]:
 		return False
@@ -628,7 +628,10 @@ def run_netreplay_server(ssh, exp_dir, repo_dir, record, listen_addr, listen_por
 	if exp_dir[-1] != "/":
 		exp_dir += "/"
 	repo_dir = repo_dir.removesuffix("/")
-	env = {"RUST_LOG": "warning", "SSLKEYLOGFILE": "/dev/shm/netreplay.keys.txt"}
+	env = {
+		"RUST_LOG": "warning",
+		#"SSLKEYLOGFILE": "/dev/shm/netreplay.keys.txt" # exports keys for use with WireShark, but may fill memory if a lot of traffic is generated!
+	}
 	if ld_preload and impl in ld_preload:
 		env["LD_PRELOAD"] = ld_preload[impl]
 	if ciphers:
@@ -651,7 +654,11 @@ def run_netreplay_client(
 	if exp_dir[-1] != "/":
 		exp_dir += "/"
 	repo_dir = repo_dir.removesuffix("/")
-	env = {"RUST_LOG": "warning", "SSLKEYLOGFILE": "/dev/shm/netreplay.keys.txt", "EARLYDATA": earlydata}
+	env = {
+		"RUST_LOG": "warning",
+		#"SSLKEYLOGFILE": "/dev/shm/netreplay.keys.txt",
+		"EARLYDATA": earlydata,
+	}
 	if ld_preload and impl in ld_preload:
 		env["LD_PRELOAD"] = ld_preload[impl]
 	if ciphers:
@@ -872,6 +879,7 @@ def run_exp(config, only_record=None, idle=False, shutdown=False, debug=False):
 													debug=debug,
 													ld_preload=config["ld_preload"] if "ld_preload" in config else None,
 												)
+												time.sleep(4)
 												netreplay = run_netreplay_client(
 													ssh,
 													config["exp_dir"],
@@ -908,6 +916,8 @@ def run_exp(config, only_record=None, idle=False, shutdown=False, debug=False):
 													debug=debug,
 													ld_preload=config["ld_preload"] if "ld_preload" in config else None,
 												)
+												# Loading the certificates takes a few seconds on the RPi3
+												time.sleep(4)
 												netreplay = run_netreplay_client(
 													None,
 													config["exp_dir"],

plots.py

@@ -50,6 +50,8 @@ COL = {
 	"ed": "ed",
 	"side": "side",
 	"record": "record",
+	"io": "io",
+	"impl": "impl",
 }
 # Physical units by object
 UNIT = {
@@ -328,7 +330,7 @@ def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record, vers
 				if log["impl"] not in impls:
 					impls.append(log["impl"])
 	impls.sort()
-	f.write("target {} none {}\n".format(criterion, " ".join([impl_title(impl) for impl in impls])))
+	f.write("target fulltarget {} none {}\n".format(criterion, " ".join([impl_title(impl) for impl in impls])))
 	for target in plain:
 		for cipher in ciphers[target]:
 			for impl in impls:
@@ -336,8 +338,9 @@ def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record, vers
 					ciphers[target][cipher][impl] = 0
 			if version == None:
 				cipher_parts = cipher.split("/")
-				f.write("{} {}({}) {} {}\n".format(
+				f.write("{} {} {}({}) {} {}\n".format(
 					target,
+					ALG_LABEL[cipher_parts[0]]+"/"+target,
 					ALG_LABEL[cipher_parts[0]],
 					cipher_parts[1],
 					plain[target],
@@ -347,8 +350,9 @@ def make_log_plot_points(logs_by_target, exp, criterion, side, obj, record, vers
 					]),
 				))
 			else:
-				f.write("{} {} {} {}\n".format(
+				f.write("{} {} {} {} {}\n".format(
 					target,
+					ALG_LABEL[cipher]+"/"+target,
 					ALG_LABEL[cipher],
 					plain[target],
 					" ".join([
@@ -516,11 +520,72 @@ def make_linear_regression(logs, cat="impl"):
 	plt.savefig(f"{PLOTS_DIR}/correlation_energy_cpu.png")
 
 # Measure relative difference between TLS versions
-def cmp_versions(logs, exps, criteria, objs):
+def cmp_versions_multitarget(logs, exps, criteria, objs):
+	ciphers = {}
+	idle_val = {target: None for target in logs_by_target}
+	
+	for target in logs_by_target:
+		logs = logs_by_target[target]
+		for log in logs:
+			log["target"] = target
+			log["io"] = int(log["bytes_in"]) + int(log["bytes_out"])
+			if log["exp"] == "idle":
+				idle_val[target] = {obj:float(log[COL[obj]]) / float(log["time"]) for obj in objs}
+	
+	for target in logs_by_target:
+		logs = logs_by_target[target]
+		for log in logs:
+			if log["exp"] not in exps or log["tls"] == "0":
+				continue
+			ver = VER_LABEL[log["cipher"]]
+			if ver not in ciphers:
+				ciphers[ver] = {}
+			key = [target]
+			for criterion in criteria:
+				key.append(ALG_LABEL.get(log[COL[criterion]], log[COL[criterion]]))
+			key = "/".join(key)
+			if key not in ciphers[ver]:
+				ciphers[ver][key] = log
+	
+	diff_rel_max = {obj:0.0 for obj in objs}
+	diff_rel_sum = {obj:0.0 for obj in objs}
+	diff_rel_num = {obj:0 for obj in objs}
+	diff_rels = []
+	for key in ciphers["1.2"]:
+		if key not in ciphers["1.3"]:
+			continue
+		log12 = ciphers["1.2"][key]
+		log13 = ciphers["1.3"][key]
+		for obj in objs:
+			val12 = (float(log12[COL[obj]]) - idle_val[log12["target"]][obj] * float(log12["time"])) / float(log12["n"])
+			val13 = (float(log13[COL[obj]]) - idle_val[log13["target"]][obj] * float(log13["time"])) / float(log13["n"])
+			# Difference relative to the mean of the two values
+			try:
+				diff_rel = abs(val12 - val13) / ((val12 + val13) / 2)
+				#print(val12, val13, diff_rel, key)
+			except ZeroDivisionError:
+				continue
+			diff_rels.append(diff_rel)
+			diff_rel_max[obj] = max(diff_rel_max[obj], diff_rel)
+			diff_rel_sum[obj] += diff_rel
+			diff_rel_num[obj] += 1
+			if diff_rel > 0.05:
+				print(obj, diff_rel, key)
+	print("Diff rel max: ", diff_rel_max)
+	try:
+		diff_rel_avg = {obj:diff_rel_sum[obj]/diff_rel_num[obj] for obj in objs}
+		print("Diff rel avg: ", diff_rel_avg)
+	except ZeroDivisionError:
+		pass
+	print("Box plot:", compute_box_plot(diff_rels))
+
+# Measure relative difference between TLS versions
+def cmp_versions(logs_by_target, exps, criteria, objs):
 	ciphers = {}
 	idle_val = None
 	
 	for log in logs:
+		log["io"] = int(log["bytes_in"]) + int(log["bytes_out"])
 		if log["exp"] == "idle":
 			idle_val = {obj:float(log[COL[obj]]) / float(log["time"]) for obj in objs}
 	
@@ -546,8 +611,8 @@ def cmp_versions(logs, exps, criteria, objs):
 		log12 = ciphers["1.2"][key]
 		log13 = ciphers["1.3"][key]
 		for obj in objs:
-			val12 = float(log12[COL[obj]]) - idle_val[obj] * float(log12["time"])
+			val12 = (float(log12[COL[obj]]) - idle_val[obj] * float(log12["time"])) / float(log12["n"])
-			val13 = float(log13[COL[obj]]) - idle_val[obj] * float(log13["time"])
+			val13 = (float(log13[COL[obj]]) - idle_val[obj] * float(log13["time"])) / float(log13["n"])
 			# Difference relative to the mean of the two values
 			try:
 				diff_rel = abs(val12 - val13) / ((val12 + val13) / 2)
@@ -939,7 +1004,7 @@ if __name__ == "__main__":
 				make_log_plot_points(logs_by_target, "impl-cipher-ver", "cipher", side, "energy", record, version="1.3")
 				make_log_plot_points(logs_by_target, "impl-cert-ver", "cert", side, "energy", record, version="1.3")
 				make_log_plot_points(logs_by_target, "impl-kex-ver", "kex", side, "energy", record, version="1.3")
-		cmp_versions(logs, ["impl-cipher-ver", "impl-cert-ver", "impl-kex-ver"], ["side", "cipher", "cert", "kex", "record"], ["cpu", "energy"])
+		cmp_versions_multitarget(logs, ["impl-cipher-ver", "impl-cert-ver", "impl-kex-ver"], ["side", "cipher", "cert", "kex", "record", "ed", "impl"], ["energy", "io"])
 	elif cmd == "prof":
 		for side in ["client", "server"]:
 			for record in records: