#!/usr/bin/env python3
"""Adversarial tests for verify-sankey.py — both polarities.
Per ADR 0005, a geometric contract in this repo is a checker plus fixtures that
prove it fires when it should and stays quiet when it shouldn't. Each mutation
below is a defect that renders perfectly and that no other gate would catch:
a ribbon that narrows as it travels, a node that loses volume between its two
sides, a stage that quietly carries less than the one before it, a printed
number that no longer matches the bar beside it, and a dark variant edited by
hand until it no longer matches the light one.
The later cases attack the checker rather than the figure. A checker that cannot
see a shape reports success, and success is indistinguishable from a clean file —
so a path it cannot parse, a ribbon attached to nothing, node bars outside the
size window, a shipped variant that has gone missing, and a reference code sample
that no longer matches the example all have to be findings, and each is tested
here.
Every anchor below is a literal excerpt of the SHIPPED example-sankey.html —
the CI budget figure — so a fixture that fails to build means the example
moved, which is itself worth knowing.
Usage: python3 scripts/test-verify-sankey.py
Exit: 0 all pass, 1 a case failed.
"""
from __future__ import annotations
import os
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
CHECKER = ROOT / "scripts/verify-sankey.py"
ASSETS = ROOT / "skills/diagram-design/assets"
GOOD = ASSETS / "example-sankey.html"
REFERENCE = ROOT / "skills/diagram-design/references/type-sankey.md"
# The first ribbon of the shipped figure (Budget → Unit tests), in full and as
# the opening fragment mutations prepend attributes to.
RIBBON = ('')
RIBBON_ANCHOR = ' tuple[int, str]:
result = subprocess.run(
argv, capture_output=True, text=True,
encoding="utf-8", errors="replace", env=CHILD_ENV,
)
return result.returncode, (result.stdout or "") + (result.stderr or "")
def run(*paths: Path) -> tuple[int, str]:
return invoke([sys.executable, str(CHECKER), *(str(p) for p in paths)])
def write(directory: Path, name: str, source: str) -> Path:
path = directory / name
path.write_text(source, encoding="utf-8")
return path
def case(
failures: list[str],
directory: Path,
name: str,
source: str,
original: str,
expect: str,
describe: str,
) -> None:
"""Run one mutation: it must be a real edit, and it must be rejected."""
if source == original:
failures.append(f"could not build the {name} fixture (anchor moved)")
return
code, output = run(write(directory, f"{name}.html", source))
if code == 0:
failures.append(f"{describe} was accepted")
elif expect not in output:
failures.append(f"{describe} was reported without the expected reason: {output.strip()}")
else:
print(f"OK: {describe} is rejected")
def refill(source_text: str, colour: str) -> str:
"""Repaint the first ribbon. Its fill is the volume, so this is the value
every conservation sum is measured against."""
return source_text.replace(RIBBON_FILL, f'fill="{colour}"', 1)
def onribbon(source_text: str, extra: str) -> str:
"""Add an attribute to the first ribbon, leaving every coordinate alone."""
return source_text.replace(
RIBBON_ANCHOR, f' int:
source = GOOD.read_text(encoding="utf-8")
failures: list[str] = []
with tempfile.TemporaryDirectory() as raw:
directory = Path(raw)
# 1. The shipped example must pass untouched.
code, output = run(GOOD)
if code != 0:
failures.append(f"clean example was rejected: {output.strip()}")
else:
print("OK: the shipped sankey passes")
# 2. Narrow one end of a ribbon. The Unit-tests → Passed band arrives
# 10px thinner than it left — invisible by eye across a 368px span,
# because a reader has no reference width to compare it against. The
# return-curve controls move with the edge so only the width lies.
case(
failures, directory, "tapered",
source.replace(
"L880,236 C696,236",
"L880,226 C696,226",
),
source, "change in width",
"a ribbon that narrows along its run",
)
# 3. Break conservation at a node while keeping the ribbon's own width
# constant: the E2E node then takes in 80px and puts out 72px, and
# nothing about the picture looks wrong.
case(
failures, directory, "leaky-node",
source.replace(
"L880,312 C696,312 696,336 512,336 Z",
"L880,304 C696,304 696,328 512,328 Z",
),
source, "as tall as the flow",
"a node that puts out less than it takes in",
)
# 4. Shrink the last node so the final stage carries less than the two
# before it. This is the mutation that justifies the stage-total check
# existing separately from the per-node one.
case(
failures, directory, "stage",
source.replace(
'9,400 min<", ">7,400 min<"),
source, "off the scale",
"a node labelled with a value its height does not draw",
)
# 6. Move one Bezier control onto the target's x. Every endpoint stays
# exactly where it was, the width is unchanged, and conservation still
# balances — but the control now coincides with the endpoint, B'(1)
# vanishes, and the ribbon skews into its node bar. Nothing else here
# reads the control points, so nothing else would catch it.
case(
failures, directory, "slanted",
source.replace(
'd="M132,120 C316,120 316,120 500,120',
'd="M132,120 C316,120 500,120 500,120',
),
source, "meets its node bar at a slant",
"a ribbon whose control point collapses onto its endpoint",
)
# 7. A path this checker cannot read must be reported, never skipped.
case(
failures, directory, "unreadable",
source.replace(
"M132,120 C316,120 316,120 500,120 L500,224 C316,224 316,224 132,224 Z",
"M132,120 L500,120 L500,224 L132,224 Z",
),
source, "neither a readable ribbon",
"a path the checker cannot parse as a ribbon",
)
# 7. Detach a ribbon from the node it leaves. It still renders, and it is
# counted in no node's balance — so it has to be reported, not skipped.
case(
failures, directory, "unattached",
source.replace('d="M132,224 C316,224', 'd="M172,224 C316,224'),
source, "touches nothing",
"a ribbon that leaves from no node bar",
)
# 8. Delete every node bar. The checker then sees ribbons and no nodes,
# and must say it checked nothing rather than exit clean. (It used to
# be enough to WIDEN the bars past the parser window; the rect-shape
# rule now refuses that earlier, so reaching this guard takes an
# actual deletion — and the guard is worth keeping reachable, because
# a vacuous pass is the one failure a green checker cannot show you.)
case(
failures, directory, "no-bars",
re.sub(r'[ ]*\n', '', source),
source, "Nothing here could be checked",
"a file whose node bars the parser cannot see",
)
# 8a. A rect that is none of the three shapes a sankey draws — not a node
# bar, not a legend swatch, not the full-bleed paper. Painted last it
# covers whatever was measured, and widening the bars is how the
# parser was blinded before the rule existed.
case(
failures, directory, "fat-bars",
source.replace('width="12" height=', 'width="20" height='),
source, "neither a node bar, a legend swatch",
"a rect that is not any shape this figure draws",
)
# 8a-bis. A rect's SIZE is not enough — its position decides what it may
# claim to be. A swatch-sized 16x8 rect is a legend key below the
# band floor and ink over the figure inside it, where it is exactly
# big enough to cover a printed value. And a bar-width rect BELOW
# the band is a shape parse_bars will never read: unrefused, it is
# ink this checker certified without examining.
case(
failures, directory, "swatch-in-plot",
source.replace(ANNOT,
' \n'
+ ANNOT, 1),
source, "neither a node bar",
"a swatch-sized rect parked inside the plot band",
)
case(
failures, directory, "bar-below-band",
source.replace(ANNOT,
' \n'
+ ANNOT, 1),
source, "neither a node bar",
"a bar-width rect below the plot band",
)
# 8b. Strip every ribbon. Eight node bars remain, so the file still looks
# like a sankey — but there is no flow left at all, and every
# conservation check would pass on it vacuously.
case(
failures, directory, "no-ribbons",
"\n".join(l for l in source.splitlines() if '\n' + anchor, 1)
if dashed == source:
failures.append("could not build the dashed-relationship fixture (anchor moved)")
else:
code, output = run(write(directory, "dashed.html", dashed))
if code != 0:
failures.append(f"the documented dashed non-volume path was rejected: {output.strip()}")
else:
print("OK: a dashed unfilled path is allowed, as the reference promises")
case(
failures, directory, "filled-garbage",
source.replace(anchor,
' \n' + anchor, 1),
source, "neither a readable ribbon",
"a filled path that is not a readable ribbon",
)
# The exemption is narrow on purpose, and each clause needs its own
# case. "Unfilled" alone is not enough: a fill="none" path with a fat
# stroke renders as a band a reader reads as volume, and would skip
# every check below. Width cap and dash pattern are both load-bearing.
# Two routes reject these, and each case expects the one that fires. The
# width cap catches the fat pair first and diagnoses them more directly;
# the undashed hairline is thin enough to reach the exemption, which
# turns it down for want of a dash pattern.
for name, frag, expect, why in (
("fat-stroke",
'',
"paints", "an unfilled path drawn as a 40px band"),
("fat-dashed",
'',
"paints", "a dashed unfilled path too wide to be a hairline"),
("undashed",
'',
"neither a readable ribbon", "an unfilled hairline with no dash pattern"),
):
case(
failures, directory, name,
source.replace(anchor, " " + frag + "\n" + anchor, 1),
source, expect, why,
)
# 8e. Blank a node's printed value. Its drawn height is then compared
# against nothing, and the fidelity check quietly shrinks its sample
# rather than failing — which is how a label-format change once
# stopped every value from parsing without a single test noticing.
case(
failures, directory, "novalue",
source.replace(">12,000 min<", ">the whole budget<", 1),
source, "no readable value beside it",
"a node bar whose printed value cannot be read",
)
# 8f. Run a ribbon end past the bottom of the bar it attaches to. Its top
# edge is still inside, so a top-edge-only test counts it as attached
# and credits the volume to the wrong node.
case(
failures, directory, "overrun",
source.replace("C696,368 696,408 512,408 Z", "C696,368 696,428 512,428 Z", 1),
source, "touches nothing",
"a ribbon end overrunning the bar it attaches to",
)
# 8g. Set stroke-width from CSS instead of an attribute. The attributes
# this checker reads still say "hairline"; what renders is a band.
# A class does it from the page stylesheet, inline style does it on
# the element, and the same trick widens a real ribbon past the fill
# it is measured by — so geometry may not carry style or class at
# all. All three were live holes found in review.
styled = source.replace(
anchor,
' \n' + anchor, 1)
case(
failures, directory, "css-inline", styled, source,
"apparent volume", "a hairline widened by an inline style",
)
classed = source.replace(
anchor,
' \n' + anchor, 1
).replace("