5@brief Aggregate the platform-dependent (wall-clock) benchmarks over N runs -> mean +/- 95% CI.
6@details Wall-clock timings vary run-to-run and across liboqs builds/CPUs; a single
7run is not a robust figure. This driver repeats each wall-clock measurement N times
8and reports mean +/- 95% CI (with per-run samples) so the paper can cite an interval
9rather than a single-run point. Deterministic experiments (masking chi^2, algebraic
10768-recovery, DCA correlation, footprint, NTT) reproduce exactly and are excluded.
12Two measurement paths, chosen for cost:
13 * liboqs baseline (keygen/encaps/decaps): subprocess the full benchmark N times.
14 * masking overhead + algebraic countermeasure overhead: call the timed routines
15 in-process N times (the slow deterministic recovery
is NOT repeated).
24import statistics as st
28HERE = os.path.dirname(os.path.abspath(__file__))
29RESULTS = os.path.join(os.path.dirname(os.path.dirname(HERE)), "results")
45 return TVAL.get(n, 1.96
if n > 30
else 2.10)
49 """Return a short environment tag: 'windows' or 'wsl' (or 'linux')."""
50 if platform.system() ==
"Windows":
53 with open(
"/proc/version")
as f:
54 return "wsl" if "microsoft" in f.read().lower()
else "linux"
70 sd = st.stdev(xs)
if len(xs) > 1
else 0.0
71 hw =
tval(len(xs)) * sd / math.sqrt(len(xs))
if len(xs) > 1
else 0.0
87 wr = importlib.import_module(
"whitebox_reference")
88 alg = importlib.import_module(
"algebraic_attack_analysis")
89 mask, a_mix, a_shuf, a_full = [], [], [], []
90 for _
in range(n_runs):
91 r = wr.measure_overhead()
if hasattr(wr,
"measure_overhead")
else None
95 [sys.executable, os.path.join(HERE,
"whitebox_reference.py")],
96 stdout=subprocess.DEVNULL,
97 stderr=subprocess.DEVNULL,
101 open(os.path.join(RESULTS,
"whitebox_reference.json"), encoding=
"utf-8")
102 )[
"overhead"][
"measured_overhead_factor"]
103 mask.append(float(r))
104 ob = alg.benchmark_countermeasure_overhead()
105 a_mix.append(ob[
"overhead_mixing"])
106 a_shuf.append(ob[
"overhead_shuffling"])
107 a_full.append(ob[
"overhead_full"])
108 metrics[
"masking_overhead_x"] = stats(mask)
109 metrics[
"alg_overhead_mixing_x"] = stats(a_mix)
110 metrics[
"alg_overhead_shuffling_x"] = stats(a_shuf)
111 metrics[
"alg_overhead_full_x"] = stats(a_full)
115 kg, en, de = [], [], []
116 for _
in range(n_runs):
118 [sys.executable, os.path.join(HERE,
"baseline_benchmark.py")],
119 stdout=subprocess.DEVNULL,
120 stderr=subprocess.DEVNULL,
124 open(os.path.join(RESULTS,
"baseline_benchmark.json"), encoding=
"utf-8")
126 kg.append(d[
"keygen"][
"mean_ms"])
127 en.append(d[
"encaps"][
"mean_ms"])
128 de.append(d[
"decaps"][
"mean_ms"])
129 metrics[
"liboqs_keygen_ms"] = stats(kg)
130 metrics[
"liboqs_encaps_ms"] = stats(en)
131 metrics[
"liboqs_decaps_ms"] = stats(de)
132 liboqs_note =
"liboqs measured"
135 "liboqs-python not available in this environment; liboqs baseline skipped"
139 py = platform.python_version()
143 "environment": f
"Intel Core i9-13980HX; {'Windows 11' if env=='windows' else 'WSL2 Ubuntu-20.04'}; Python {py}; {liboqs_note}",
144 "note":
"mean +/- 95% CI over N independent runs; wall-clock, platform/build-dependent; ratios are the build-invariant metric",
147 out = os.path.join(RESULTS, f
"wallclock_statistics_{env}.json")
148 json.dump(agg, open(out,
"w", encoding=
"utf-8"), indent=2)
149 print(f
"wrote {out} (N={n_runs}, env={env}, {liboqs_note})")
150 for lbl, s
in metrics.items():
151 cv = 100 * s[
"std"] / s[
"mean"]
if s[
"mean"]
else 0
153 f
" {lbl:24s} mean={s['mean']:.4f} +/-{s['ci95_halfwidth']:.4f} CV={cv:.1f}% [{s['min']:.4f},{s['max']:.4f}]"
157if __name__ ==
"__main__":
158 main(int(sys.argv[1])
if len(sys.argv) > 1
else 20)
detect_env()
Return a short environment tag: 'windows' or 'wsl' (or 'linux').