4@file arm64_aggregate.py
5@brief Aggregate N independent qemu-aarch64 liboqs runs into mean +/- 95% CI.
8@details A single qemu-aarch64 run of the ML-KEM-768 benchmark is not stable:
9functional emulation absolute times drift run-to-run with host load and the
10translation cache. Reporting one run would pin a moving number. This reads the N
11per-run JSONs produced by build_arm64_benchmark.sh and reduces each operation's
12per-run mean to a mean and a 95% confidence interval, the same interval statistic
13wallclock_ci.py uses (Student t, N runs), so the ARM64/QEMU baseline is an
14interval rather than a single transcribed value. mean_ms is retained per
15operation so consumers that read it (overhead_calculation.py) keep working.
17@note The absolute times are functional-emulation figures dependent on host load,
18not real ARM silicon; only the dimensionless ratio is comparable. Called by
19build_arm64_benchmark.sh under WSL; not part of the Python run_all suite because
20it needs the aarch64 cross-toolchain and qemu-aarch64.
25This software is provided for academic and research purposes.
26See LICENSE file for details.
31import statistics
as st
33from pathlib
import Path
51 """@brief Two-sided 95% t critical value for n samples."""
52 return TVAL.get(n, 1.96
if n > 30
else 2.10)
57 @brief Mean, standard deviation,
and 95% CI half-width
for a sample.
58 @param xs List of per-run means.
59 @return Dict
with mean_ms, stdev_ms, ci95_halfwidth_ms, min_ms, max_ms, n, samples_ms.
63 sd = st.stdev(xs) if n > 1
else 0.0
64 hw =
tval(n) * sd / math.sqrt(n)
if n > 1
else 0.0
66 "mean_ms": round(m, 4),
67 "stdev_ms": round(sd, 4),
68 "ci95_halfwidth_ms": round(hw, 4),
69 "min_ms": round(min(xs), 4),
70 "max_ms": round(max(xs), 4),
72 "samples_ms": [round(x, 4)
for x
in xs],
78 @brief Read per-run JSONs, aggregate the three operations, write the summary.
83 "usage: arm64_aggregate.py <run_dir> <output.json> [n_expected]"
85 run_dir = Path(sys.argv[1])
86 out_path = Path(sys.argv[2])
87 n_expected = int(sys.argv[3])
if len(sys.argv) > 3
else None
89 runs = sorted(run_dir.glob(
"run_*.json"))
92 f
"No run_*.json in {run_dir}; the benchmark loop produced nothing."
94 if n_expected
is not None and len(runs) != n_expected:
96 f
"Expected {n_expected} runs but found {len(runs)} in {run_dir}; refusing to "
97 f
"report an interval over a truncated sample rather than silently averaging fewer."
100 per_op = {
"keygen": [],
"encaps": [],
"decaps": []}
103 d = json.loads(r.read_text(encoding=
"utf-8"))
119 per_op[op].append(float(d[
"results"][op][
"mean_ms"]))
121 results = {op:
interval(xs)
for op, xs
in per_op.items()}
123 summary = dict(meta
or {})
124 summary[
"n_runs"] = len(runs)
125 summary[
"results"] = results
126 summary[
"provenance"] = (
127 f
"Native aarch64 liboqs ML-KEM-768 KeyGen/Encaps/Decaps benchmark "
128 f
"(bench_liboqs.c cross-compiled with the aarch64 GCC, run under qemu-aarch64 "
129 f
"user-mode emulation on the x86-64 host), reported as mean +/- 95% CI over "
130 f
"{len(runs)} independent runs. QEMU emulates the ARM64 instruction set while "
131 f
"the host CPU performs the work, so these are functional-emulation figures "
132 f
"dependent on host load, not any ARM device's wall-clock; only the "
133 f
"dimensionless ratio is the comparable metric. Regenerate with "
134 f
"code/c/build_arm64_benchmark.sh under WSL/Linux."
137 out_path.write_text(json.dumps(summary, indent=2), encoding=
"utf-8")
139 print(f
"wrote {out_path} (N={len(runs)} runs)")
140 for op, s
in results.items():
141 cv = 100 * s[
"stdev_ms"] / s[
"mean_ms"]
if s[
"mean_ms"]
else 0
143 f
" {op:8s} {s['mean_ms']:.3f} +/- {s['ci95_halfwidth_ms']:.3f} ms "
144 f
"CV={cv:.1f}% [{s['min_ms']:.3f}, {s['max_ms']:.3f}]"
149if __name__ ==
"__main__":
150 raise SystemExit(
main())
interval(xs)
Mean, standard deviation, and 95% CI half-width for a sample.
tval(n)
Two-sided 95% t critical value for n samples.
main()
Read per-run JSONs, aggregate the three operations, write the summary.