netreplay generates certificates

README.md

@@ -25,13 +25,13 @@ In settings, disable DNS security.
 
 In `about:config`, set `network.dns.forceResolve` to `127.0.0.1`.
 
-Run the shell command:
+Run the shell commands:
 
 ```bash
 python exp.py make debug -c
 ```
 
-In Firefox, go to security settings, Certificates, import `/dev/shm/exp/certs/prime256v1/ca.crt` and trust it for identifying websites.
+In Firefox, go to security settings, Certificates, import all CA certificates in `/dev/shm/exp/certs/realistic` and trust them for identifying websites.
 
 #### After installation
 
@@ -40,11 +40,18 @@ Stop anything running on ports 80 or 443.
 Start the record proxy:
 
 ```bash
-./netreplay records/mynewrecord record
+mkdir -p /dev/shm/exp/fake/mynewrecord
+./netreplay records/mynewrecord record /dev/shm/exp/certs/realistic /dev/shm/exp/fake/mynewrecord
 ```
 
 Just browse. Any traffic will be recorded. Terminate netplayer with CTRL+C when finished.
 
+Between each recording, erase cookies and cache, close Firefox, then clean the certificate cache with the command:
+
+```bash
+rm .mozilla/firefox/*.tlsbench/SiteSecurityServiceState.bin
+```
+
 ### Measure
 
 Copy the tlsbench folder on the target.

exp.py

@@ -127,14 +127,16 @@ CONFIGS = {
 			"server",
 		],
 		"tls": [
-			#False,
+			False,
 			True,
 		],
 		"records": [
-			{ "filename": "wp2", "repeat": 10000, "time": 600, "reproduce": 1 },
+			#{ "filename": "wp2", "repeat": 10000, "time": 600, "reproduce": 1 },
-			{ "filename": "yt2-ads", "repeat": 10000, "time": 1200, "reproduce": 1 },
+			#{ "filename": "yt2-ads", "repeat": 10000, "time": 1200, "reproduce": 1 },
-			{ "filename": "yt2-ublock", "repeat": 10000, "time": 1200, "reproduce": 1 },
+			#{ "filename": "yt2-ublock", "repeat": 10000, "time": 1200, "reproduce": 1 },
 			#{ "filename": "wp2", "repeat": 10000, "time": 300, "reproduce": 100 },
+			#{ "filename": "guardian", "realistic": True, "repeat": 10000, "time": 60, "reproduce": 1 },
+			{ "filename": "wikipedia", "realistic": True, "repeat": 10000, "time": 120, "reproduce": 10 },
 		],
 		"repo_dir": "/home/tuxmain/reps/tlsbench",
 		"exp_dir": "/dev/shm/exp",
@@ -340,6 +342,7 @@ DOMAINS_ = [
 	# Firefox extensions
 	"addons.mozilla.org",
 ]
+DOMAINS_ = []
 
 CERT_SIGN_ALGS = [
 	"prime256v1", # widely used
@@ -348,8 +351,8 @@ CERT_SIGN_ALGS = [
 	"rsa2048", "rsa3072", "rsa4096", # widely used
 ]
 IMPLS = [
-	#"aws-lc", # Amazon's crypto widely used in Rust stuff
+	"aws-lc", # Amazon's crypto widely used in Rust stuff
-	#"boring", # Google's fork of OpenSSL used in Chrome and Android
+	"boring", # Google's fork of OpenSSL used in Chrome and Android
 	#"graviola", # New crypto in Rust
 	"openssl", # widely used
 	#"openssl-static",
@@ -615,11 +618,16 @@ def make_everything(exp_dir, domains, make_ca):
 	if exp_dir[-1] != "/":
 		exp_dir += "/"
 	os.makedirs(exp_dir, exist_ok=True)
+	realistic_dir = exp_dir+"certs/realistic/"
+	os.makedirs(realistic_dir, exist_ok=True)
 	for alg in CERT_SIGN_ALGS:
 		algdir = exp_dir+"certs/"+alg
 		os.makedirs(algdir, exist_ok=True)
 		make_certs(algdir, domains, alg, make_ca)
 
+		make_sk(realistic_dir+"ca_"+alg+".key", alg)
+		make_ca_cert(realistic_dir+"ca_"+alg+".crt", realistic_dir+"ca_"+alg+".key")
+
 def run_netreplay_server(ssh, exp_dir, repo_dir, record, listen_addr, listen_port, tls, impl, certs_dir, only_record=None, ciphers=None, kexes=None, debug=False, ld_preload=None):
 	if exp_dir[-1] != "/":
 		exp_dir += "/"
@@ -866,6 +874,9 @@ def run_exp(config, only_record=None, idle=False, shutdown=False, debug=False):
 										continue
 								for side in config["sides"]:
 									for record in config["records"]:
+										server_certs_dir = certs_dir
+										if record.get("realistic", False):
+											server_certs_dir = exp_dir+"/fake/"+record["filename"]
 										for reprod_i in range(record.get("reproduce", 1)):
 											print(f"EXPERIMENT {expname}: {impl} {alg} {kex} {cipher} ED={earlydata} {side} TLS={tls}")
 											
@@ -881,7 +892,7 @@ def run_exp(config, only_record=None, idle=False, shutdown=False, debug=False):
 													config["listen_port"],
 													tls,
 													None,
-													certs_dir,
+													server_certs_dir,
 													only_record=only_record,
 													ciphers=cipher,
 													kexes=kex,
@@ -918,7 +929,7 @@ def run_exp(config, only_record=None, idle=False, shutdown=False, debug=False):
 													config["listen_port"],
 													tls,
 													impl,
-													certs_dir,
+													server_certs_dir,
 													only_record=only_record,
 													ciphers=cipher,
 													kexes=kex,

makecerts.py

@@ -25,6 +25,62 @@ DOMAINS = [
 	"googleapis.com",
 	"i.ytimg.com",
 	"ad.doubleclick.net",
+	"theguardian.com",
+	"cdn.privacy-mgmt.com",
+	"bbc.com",
+	"cdn.optimizely.com",
+	"mybbc-analytics.files.bbci.co.uk",
+	"static.bbci.co.uk",
+	"emp.bbci.co.uk",
+	"ichef.bbci.co.uk",
+	"static.files.bbci.co.uk",
+	"gn-web-assets.api.bbc.com",
+	"a1.api.bbc.com",
+	"c2-eu.piano.io",
+	"buy-eu.piano.io",
+	"pub.doubleverify.com",
+	"direct-events-collector.spot.im",
+	"cdn.tinypass.com",
+	"cdn.cxense.com",
+	"prebid.the-ozone-project.com",
+	"scripts.webcontentassessor.com",
+	"cdn.permutive.com",
+	"cdn.speedcurve.com",
+	"sb.scorecardresearch.com",
+	"edigitalsurvey.com",
+	"vtrk.dv.tech",
+	"dotcom.bbc-reporting-api.app",
+	"flo.uri.sh",
+	"cdn.taboola.com",
+	"public.flourish.studio",
+	"cdnjs.cloudflare.com",
+	"js.stripe.com",
+	"s0.2mdn.net",
+	"m.stripe.network",
+	"q.stripe.com",
+	"m.stripe.com",
+	"cdn.brandmetrics.com",
+	"cdn.confiant-integrations.net",
+	"a.teads.tv",
+	"partage.insa-lyon.fr",
+	"docs.partage.renater.fr",
+	"instagram.com",
+	"static.cdninstagram.com",
+	"facebook.com",
+	"gateway.instagram.com",
+	"scontent-mrs2-1.cdninstagram.com",
+	"scontent-mrs2-2.cdninstagram.com",
+	"scontent-mrs2-3.cdninstagram.com",
+	"edge-chat.instagram.com",
+	"graph.instagram.com",
+	"static.xx.fbcdn.net",
+	"stackoverflow.com",
+	"cdn.cookielaw.org",
+	"gravatar.com",
+	"i.sstatic.net",
+	"geolocation.onetrust.com",
+	"googletagmanager.com",
+	"csp.withgoogle.com",
 ]
 
 def sh(cmds):