Supplementary File S3 — analysis notebook¶

from pathlib import Path
import statistics as st
from collections import Counter, defaultdict

import numpy as np
from openpyxl import load_workbook

S1 = Path("QAIS_Supplementary_File_S1_Criterion_Level_Appraisal_Data.xlsx")
S2 = Path("QAIS_Supplementary_File_S2_Study_Extraction_Record.xlsx")
for f in (S1, S2):
    assert f.exists(), f"{f} not found: place the notebook in the same folder as Files S1 and S2"

def table(path, sheet):
    ws = load_workbook(path, read_only=True, data_only=True)[sheet]
    rows = ws.iter_rows(values_only=True)
    head = next(rows)
    out = []
    for r in rows:
        if r[0] is None:
            break
        out.append(dict(zip(head, r)))
    return out

register   = table(S1, "Evidence register")
scores     = table(S1, "Criterion scores")
dup        = table(S1, "Duplicate appraisal")
extraction = table(S2, "Extraction record")
print(f"register {len(register)} | criterion vectors {len(scores)} | "
      f"duplicate subsample {len(dup)} | extraction rows {len(extraction)}")

# scoring rules, Section 2 of the manuscript
CRIT    = ["C1 design", "C2 fairness", "C3 statistics", "C4 reproducibility",
           "C5 task relevance", "C6 hardware realism", "C7 reporting"]
WEIGHTS = [20, 18, 12, 15, 12, 13, 10]
CUTS    = [(80, "Strong"), (65, "Good"), (50, "Moderate"), (35, "Weak"), (0, "Very weak")]

def weighted_total(r, weights=WEIGHTS):
    return round(sum(w * float(r[c]) / 4 for w, c in zip(weights, CRIT)), 6)

def band_of(score):
    return next(name for cut, name in CUTS if score >= cut)

reg_cap = {r["Study"]: r["Capability"] for r in register}
ext_cap = {r["Study"]: r["Capability"] for r in extraction}

assert set(reg_cap) == set(ext_cap), "Files S1 and S2 cover different studies"
checked  = [s for s in reg_cap if ext_cap[s]]
disagree = [s for s in checked if ext_cap[s] != reg_cap[s]]
no_record = sorted(s for s in reg_cap if not ext_cap[s])

print(f"same {len(reg_cap)} study identifiers in both files")
print(f"capability agrees for {len(checked) - len(disagree)} of {len(checked)} studies with an extraction row")
print(f"studies with no extraction row: {no_record}")
assert not disagree, disagree

PUBLISHED_TABLE2 = {
    # capability: (studies, on real hardware, Strong, Good, Moderate, Weak)
    "Quantum data encoding":                      (11, 5, 0, 5, 6, 0),
    "Quantum feature learning":                   (12, 5, 4, 3, 5, 0),
    "Variational quantum circuits":               (14, 6, 0, 9, 5, 0),
    "Hybrid quantum artificial intelligence":     (17, 6, 4, 8, 5, 0),
    "Quantum optimization":                       (12, 8, 3, 2, 6, 1),
    "Quantum reinforcement learning":             ( 9, 3, 2, 2, 4, 1),
    "Quantum neural networks":                    (13, 4, 4, 3, 6, 0),
    "Quantum generative artificial intelligence": ( 8, 2, 2, 3, 3, 0),
    "Quantum edge artificial intelligence":       ( 6, 1, 0, 3, 3, 0),
    "Fault-tolerant quantum intelligence":        ( 5, 0, 1, 2, 2, 0),
}
CAPS  = list(PUBLISHED_TABLE2)
BANDS = ["Strong", "Good", "Moderate", "Weak"]

computed = {}
for cap in CAPS:
    rows = [r for r in register if r["Capability"] == cap]
    band = Counter(r["Evidence band"] for r in rows)
    computed[cap] = (len(rows), sum(r["Real hardware"] == "Yes" for r in rows),
                     *[band[b] for b in BANDS])

print(f"{'capability':<44}{'computed':<24}{'published'}")
for cap in CAPS:
    flag = "ok" if computed[cap] == PUBLISHED_TABLE2[cap] else "MISMATCH"
    print(f"{cap:<44}{str(computed[cap]):<24}{str(PUBLISHED_TABLE2[cap]):<24}{flag}")
    assert computed[cap] == PUBLISHED_TABLE2[cap], cap

N   = sum(v[0] for v in computed.values())
NHW = sum(v[1] for v in computed.values())
bands_total = [sum(v[2 + i] for v in computed.values()) for i in range(4)]
print(f"\ntotals: {N} studies, {NHW} on real hardware ({100 * NHW / N:.0f}%), bands {bands_total}")
assert (N, NHW, bands_total) == (107, 40, [20, 40, 45, 2])

# "Highest-rated studies" column of Table 2: the three highest weighted totals in each capability.
# Where totals tie at the third place, any tied study may be listed; the check allows for that.
PUBLISHED_TOP3 = {
    "Quantum data encoding": [25, 26, 27],        "Quantum feature learning": [2, 28, 29],
    "Variational quantum circuits": [30, 31, 32], "Hybrid quantum artificial intelligence": [3, 4, 33],
    "Quantum optimization": [5, 6, 24],           "Quantum reinforcement learning": [19, 21, 34],
    "Quantum neural networks": [13, 35, 36],      "Quantum generative artificial intelligence": [22, 37, 38],
    "Quantum edge artificial intelligence": [23, 39, 40], "Fault-tolerant quantum intelligence": [7, 8, 16],
}
total_of = {r["Study"]: weighted_total(r) for r in scores}
for cap in CAPS:
    ranked = sorted(((total_of[r["Study"]], r["Study"]) for r in scores if r["Capability"] == cap), reverse=True)
    third  = ranked[2][0]
    listed = PUBLISHED_TOP3[cap]
    ok = (all(total_of[s] >= third for s in listed)
          and not [s for t, s in ranked if t > min(total_of[x] for x in listed) and s not in listed])
    ties = [s for t, s in ranked if t == third and s not in listed]
    print(f"{cap:<44}{str(listed):<14}{'ok' if ok else 'MISMATCH'}"
          + (f"   (tied at {third:.2f} with study {ties})" if ties else ""))
    assert ok, cap

SHORT = ["C1\nDesign", "C2\nFairness", "C3\nStatistics", "C4\nArtifacts",
         "C5\nRelevance", "C6\nHardware", "C7\nReporting"]

PUBLISHED_FIG4 = np.array([
    [3.5, 2.5, 1.5, 2.2, 2.6, 1.9, 3.8], [3.5, 3.3, 1.8, 2.4, 2.9, 2.1, 3.8],
    [3.2, 2.9, 1.8, 2.2, 3.1, 2.1, 3.5], [3.5, 3.1, 2.1, 2.1, 3.4, 2.0, 3.6],
    [3.2, 2.2, 2.2, 2.1, 3.2, 2.8, 3.5], [3.3, 2.6, 2.6, 2.9, 2.2, 1.9, 3.4],
    [3.5, 2.8, 2.3, 2.4, 3.3, 1.5, 3.6], [3.8, 3.2, 1.6, 2.5, 2.8, 1.5, 3.9],
    [3.5, 2.5, 2.2, 2.2, 2.8, 2.0, 3.7], [3.2, 2.6, 1.6, 2.2, 1.8, 3.8, 4.0]])

by_cap = defaultdict(list)
for r in scores:
    by_cap[r["Capability"]].append([float(r[c]) for c in CRIT])

MEANS = np.array([[st.mean(v[i] for v in by_cap[cap]) for i in range(7)] for cap in CAPS])
diff  = np.abs(np.round(MEANS, 1) - PUBLISHED_FIG4)
print(f"{diff.size} cells | {(diff < 1e-9).sum()} identical after rounding to one decimal")
assert diff.max() < 1e-9, [(CAPS[i], CRIT[j]) for i, j in zip(*np.where(diff > 1e-9))]

# Section 4.2 quotes each criterion's average across the ten capabilities
PUBLISHED_AVERAGES = {"C1 design": 3.42, "C3 statistics": 1.97, "C4 reproducibility": 2.31,
                      "C6 hardware realism": 2.16, "C7 reporting": 3.69}
print("\naverage of the ten capability means on each criterion:")
for name, v in zip(CRIT, MEANS.mean(axis=0)):
    want = PUBLISHED_AVERAGES.get(name)
    print(f"  {name:<22}{v:.2f}" + (f"   (Section 4.2: {want})" if want else ""))
    if want:
        assert round(v, 2) == want, name

SCHEMES = {
    "Baseline":         WEIGHTS,
    "Equal weights":    [100 / 7] * 7,
    "Fairness reduced": [10 if i == 1 else w + 8 / 6 for i, w in enumerate(WEIGHTS)],
    "Hardware raised":  [20 if i == 5 else w * 80 / 87 for i, w in enumerate(WEIGHTS)],
}
for name, w in SCHEMES.items():
    assert abs(sum(w) - 100) < 1e-9, name

TOT = {name: {r["Study"]: weighted_total(r, w) for r in scores} for name, w in SCHEMES.items()}
base = TOT["Baseline"]
ORDER5 = ["Very weak", "Weak", "Moderate", "Good", "Strong"]
below_base = sum(v < 50 for v in base.values())
mean_base = st.mean(base.values())

PUBLISHED_TABLE6 = {  # scheme: (studies moving one band, net change below Moderate, mean total)
    "Baseline":         (None, None, 69.0),
    "Equal weights":    (5, -1, 68.5),
    "Fairness reduced": (7, -1, 68.9),
    "Hardware raised":  (13, 2, 67.7),
}
print(f"{'scheme':<18}{'moved one band':<18}{'net below Moderate':<21}{'mean total'}")
shifts = []
for name, tot in TOT.items():
    moved = [s for s in tot if band_of(tot[s]) != band_of(base[s])]
    assert all(abs(ORDER5.index(band_of(tot[s])) - ORDER5.index(band_of(base[s]))) == 1 for s in moved)
    net   = sum(v < 50 for v in tot.values()) - below_base
    mean  = st.mean(tot.values())
    shifts.append(abs(mean - mean_base))
    if name == "Baseline":
        got = (None, None, round(mean, 1))
        print(f"{name:<18}{'-':<18}{'-':<21}{round(mean, 1)}")
    else:
        got = (len(moved), net, round(mean, 1))
        shown = f"{len(moved)} of {len(scores)} ({100 * len(moved) / len(scores):.0f}%)"
        print(f"{name:<18}{shown:<18}{net:<+21d}{round(mean, 1)}")
    assert got == PUBLISHED_TABLE6[name], (name, got)

print(f"\nlargest shift in the mean total: {max(shifts):.2f} points (Section 4.2: at most 1.4)")
assert round(max(shifts), 1) == 1.4
print("Table 6 reproduces exactly.")

# The mechanical part of "no capability changes level": no study cited in Table 5 as evidence for a
# capability's maturity assignment falls below the Moderate evidence-use threshold under any weighting.
TABLE5_EVIDENCE = {
    "Quantum neural networks": [13, 91, 35, 36],
    "Quantum optimization": [24, 80, 83, 5],
    "Quantum feature learning": [48, 49, 29, 2, 10],
    "Hybrid quantum artificial intelligence": [4, 68, 70, 72, 109],
    "Quantum reinforcement learning": [19, 20],
    "Quantum generative artificial intelligence": [37, 38, 101],
    "Quantum data encoding": [1, 41, 42, 43, 44],
    "Variational quantum circuits": [32, 56, 57, 58, 30, 31, 59],
    "Quantum edge artificial intelligence": [39, 23],
    "Fault-tolerant quantum intelligence": [7, 8, 9, 17],
}
for name, tot in TOT.items():
    below = sorted(s for s, v in tot.items() if v < 50)
    print(f"{name:<18}below Moderate: {below}")
cited = {s for v in TABLE5_EVIDENCE.values() for s in v}
unscored = sorted(cited - set(base))
fallen = sorted({s for tot in TOT.values() for s, v in tot.items() if v < 50} & cited)
print(f"\nTable 5 evidence studies: {len(cited)} ({len(cited) - len(unscored)} with criterion vectors; "
      f"no vector: {unscored})")
print(f"Table 5 evidence studies below Moderate under any weighting: {fallen or 'none'}")
assert not fallen

S1COLS = {"Baseline": "Weighted total", "Equal weights": "Total, equal weights",
          "Fairness reduced": "Total, C2 reduced to 10", "Hardware raised": "Total, C6 raised to 20"}
worst = max(abs(float(r[col]) - TOT[name][r["Study"]]) for name, col in S1COLS.items() for r in scores)
print(f"criterion scores: largest difference between a stored and a recomputed total {worst:.3f} "
      f"(values are stored to 2 dp)")
assert worst < 0.005 + 1e-9

reg_band = {r["Study"]: r["Evidence band"] for r in register}
band_mismatch = [s for s in base if reg_band[s] != band_of(base[s])
                 or next(r["Band"] for r in scores if r["Study"] == s) != band_of(base[s])]
print(f"evidence register and criterion sheet: band equals the band of the recomputed total for "
      f"{len(base) - len(band_mismatch)} of {len(base)} scored studies")
assert not band_mismatch, band_mismatch

dup_off = []
for r in dup:
    for p in ("Pass 1", "Pass 2"):
        t = round(sum(w * float(r[f"{p} C{k}"]) / 4 for k, w in enumerate(WEIGHTS, 1)), 6)
        if abs(t - float(r[f"{p} total"])) > 0.005 + 1e-9 or band_of(t) != r[f"{p} band"]:
            dup_off.append((r["Study"], p))
print(f"duplicate appraisal: total and band agree with the anchor levels for "
      f"{2 * len(dup) - len(dup_off)} of {2 * len(dup)} passes")
assert not dup_off, dup_off

ORDER = ["Very weak", "Weak", "Moderate", "Good", "Strong"]
b1 = [r["Pass 1 band"] for r in dup]
b2 = [r["Pass 2 band"] for r in dup]
n  = len(dup)

exact   = sum(a == b for a, b in zip(b1, b2))
within1 = sum(abs(ORDER.index(a) - ORDER.index(b)) <= 1 for a, b in zip(b1, b2))
mad     = st.mean(abs(float(r["Pass 1 total"]) - float(r["Pass 2 total"])) for r in dup)

c1, c2 = Counter(b1), Counter(b2)
cats = sorted(set(b1) | set(b2), key=ORDER.index)
po, pe = exact / n, sum(c1[c] * c2[c] for c in cats) / n ** 2
kappa = (po - pe) / (1 - pe)
w = lambda a, b: 1 - abs(ORDER.index(a) - ORDER.index(b)) / (len(ORDER) - 1)
po_w = sum(w(a, b) for a, b in zip(b1, b2)) / n
pe_w = sum(c1[a] * c2[b] * w(a, b) for a in cats for b in cats) / n ** 2
kappa_w = (po_w - pe_w) / (1 - pe_w)

for label, got, want in [("exact band agreement",           f"{exact} of {n}",   "9 of 20"),
                         ("agreement within one band",      f"{within1} of {n}", "17 of 20"),
                         ("unweighted Cohen's kappa",       f"{kappa:.2f}",      "0.19"),
                         ("linear-weighted kappa",          f"{kappa_w:.2f}",    "0.23"),
                         ("mean absolute score difference", f"{mad:.1f}",        "9.8")]:
    print(f"{'ok  ' if got == want else 'FAIL'} {label:<33}{got:<11}(manuscript {want})")
    assert got == want, label

import matplotlib.pyplot as plt
plt.rcParams["font.family"] = "DejaVu Sans"
LABELS = ["Data encoding", "Feature learning", "Variational circuits", "Hybrid quantum AI",
          "Optimization", "Reinforcement learning", "Neural networks", "Generative AI",
          "Edge intelligence", "Fault tolerance"]

fig, ax = plt.subplots(figsize=(7.4, 3.8), dpi=120)
im = ax.imshow(MEANS, cmap="Greys", vmin=0, vmax=4, aspect="auto")
ax.set_xticks(range(7), SHORT, fontsize=7)
ax.set_yticks(range(10), LABELS, fontsize=7.5)
ax.xaxis.set_ticks_position("top"); ax.tick_params(length=0)
for i in range(10):
    for j in range(7):
        ax.text(j, i, f"{MEANS[i, j]:.1f}", ha="center", va="center", fontsize=7,
                color="white" if MEANS[i, j] > 2.4 else "black")
ax.set_xticks(np.arange(-.5, 7), minor=True); ax.set_yticks(np.arange(-.5, 10), minor=True)
ax.grid(which="minor", color="white", lw=1); ax.tick_params(which="minor", length=0)
cb = fig.colorbar(im, ax=ax, orientation="horizontal", fraction=0.05, pad=0.06)
cb.set_label("Mean anchor level (0 = absent, 4 = exemplary)", fontsize=7); cb.ax.tick_params(labelsize=7)
fig.tight_layout(); fig.savefig("figure_criterion_heatmap.png", dpi=300, facecolor="white")
plt.show()

fig, (a1, a2) = plt.subplots(1, 2, figsize=(9, 3.4), dpi=120, gridspec_kw={"width_ratios": [2.1, 1]})
y, left = np.arange(10)[::-1], np.zeros(10)
for k, (band, colour) in enumerate(zip(BANDS, ["#1a1a1a", "#707070", "#b5b5b5", "#ececec"])):
    vals = np.array([computed[c][2 + k] for c in CAPS])
    a1.barh(y, vals, left=left, color=colour, edgecolor="black", lw=0.5, height=0.66, label=band)
    for i, v in enumerate(vals):
        if v:
            a1.text(left[i] + v / 2, y[i], str(v), ha="center", va="center", fontsize=6.5,
                    color="white" if k < 2 else "black")
    left = left + vals
a1.set_yticks(y, LABELS, fontsize=7.5); a1.set_xlabel("Number of studies", fontsize=8)
a1.set_xlim(0, 18.5); a1.tick_params(axis="x", labelsize=7)
a1.set_title("Evidence bands", fontsize=9, fontweight="bold")
a1.legend(fontsize=7, ncol=4, loc="lower right", frameon=False)
pct = [100 * computed[c][1] / computed[c][0] for c in CAPS]
a2.barh(y, pct, color="#8c8c8c", edgecolor="black", lw=0.5, height=0.66)
a2.axvline(100 * NHW / N, ls="--", lw=1, color="black")
for i, v in enumerate(pct):
    a2.text(104, y[i], f"{round(v)}%", va="center", fontsize=6.5)
a2.set_yticks(y, []); a2.set_xlim(0, 122); a2.set_xticks([0, 50, 100]); a2.tick_params(axis="x", labelsize=7)
a2.set_xlabel("On real hardware (%)", fontsize=8); a2.set_title("Hardware use", fontsize=9, fontweight="bold")
for a in (a1, a2):
    for s in ("top", "right"):
        a.spines[s].set_visible(False)
fig.tight_layout(); fig.savefig("figure_bands_and_hardware.png", dpi=300, facecolor="white")
plt.show()