feat(skill): pre-draw checkpoint, docs-sync gate, packaged self-check, ADRs
- SKILL.md: confirm type/pattern/size/cuts with the user before rendering; checklist hooks for the checkpoint and the packaged self-check - scripts/verify-docs-sync.py: CI now fails when the frontmatter description loses a type's lexical hook, the gallery can't reach a shipped example, or the README tree names a missing file - skills/diagram-design/scripts/self_check.py: distilled output checker that ships with the skill so installed agents can verify their own diagrams; adversarial suite in scripts/test-self-check.py - assets/index.html: six new tabs (OAuth sequence + five single-variant examples with a data-single variant lock) so every shipped example is browsable - docs/adr/0001-0004: settled policies recorded (pinned controller, taxonomy freeze, reveal-only autoplay, byte cap + trigger-rich description) - README: It's-working-if section, new gates documented; CONTRIBUTING: gate table, ADR guidance, type-addition invariants; CI: two new steps
This commit is contained in:
Cathryn Lavery
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Adversarial tests for the packaged skill self-check (self_check.py)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SELF_CHECK = ROOT / "skills/diagram-design/scripts/self_check.py"
|
||||
ASSET_DIR = ROOT / "skills/diagram-design/assets"
|
||||
TEMPLATE = ASSET_DIR / "template-motion.html"
|
||||
EXAMPLE = ASSET_DIR / "example-policy-trace-animated.html"
|
||||
STATIC_EXAMPLE = ASSET_DIR / "example-architecture.html"
|
||||
|
||||
|
||||
def load_self_check():
|
||||
spec = importlib.util.spec_from_file_location("self_check", SELF_CHECK)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def main() -> int:
|
||||
module = load_self_check()
|
||||
failures: list[str] = []
|
||||
|
||||
def check_pass(label: str, path: Path) -> None:
|
||||
errors = module.verify(path)
|
||||
if errors:
|
||||
failures.append(f"{label}: expected pass, got {errors}")
|
||||
else:
|
||||
print(f"OK: {label} passes")
|
||||
|
||||
def check_fail(label: str, source: str, needle: str) -> None:
|
||||
with tempfile.TemporaryDirectory() as scratch:
|
||||
candidate = Path(scratch) / "candidate.html"
|
||||
candidate.write_text(source, encoding="utf-8")
|
||||
errors = module.verify(candidate)
|
||||
if not any(needle in error for error in errors):
|
||||
failures.append(f"{label}: expected an error containing {needle!r}, got {errors}")
|
||||
else:
|
||||
print(f"OK: {label} rejected")
|
||||
|
||||
check_pass("shipped template", TEMPLATE)
|
||||
check_pass("shipped animated example", EXAMPLE)
|
||||
check_pass("shipped static example", STATIC_EXAMPLE)
|
||||
|
||||
static = STATIC_EXAMPLE.read_text(encoding="utf-8")
|
||||
animated = EXAMPLE.read_text(encoding="utf-8")
|
||||
|
||||
check_fail(
|
||||
"executable attribute",
|
||||
static.replace("<body>", '<body onload="fetch(1)">', 1),
|
||||
"executable attribute",
|
||||
)
|
||||
check_fail(
|
||||
"remote image",
|
||||
static.replace("<body>", '<body><img src="https://tracker.example/p.gif">', 1),
|
||||
"remote reference",
|
||||
)
|
||||
check_fail(
|
||||
"lookalike fonts host",
|
||||
static.replace(
|
||||
"https://fonts.googleapis.com/css2",
|
||||
"https://fonts.googleapis.com.evil.tld/css2",
|
||||
1,
|
||||
),
|
||||
"approved Google Fonts",
|
||||
)
|
||||
check_fail(
|
||||
"javascript URL",
|
||||
static.replace("<body>", '<body><a href="javascript:alert(1)">x</a>', 1),
|
||||
"executable URL",
|
||||
)
|
||||
check_fail(
|
||||
"data html URL",
|
||||
static.replace("<body>", '<body><a href="data:text/html,x">x</a>', 1),
|
||||
"executable URL",
|
||||
)
|
||||
check_fail(
|
||||
"iframe injection",
|
||||
static.replace("<body>", "<body><iframe></iframe>", 1),
|
||||
"not allowed",
|
||||
)
|
||||
check_fail(
|
||||
"arbitrary script",
|
||||
static.replace("</body>", "<script>alert(1)</script></body>", 1),
|
||||
"canonical data-diagram-controls attribute",
|
||||
)
|
||||
check_fail(
|
||||
"missing svg role",
|
||||
static.replace('role="img"', 'role="presentation"', 1),
|
||||
"role=img",
|
||||
)
|
||||
check_fail(
|
||||
"modified controller",
|
||||
animated.replace("'motion-ready'", "'motion-ready' /* patched */", 1),
|
||||
"exactly match the controller",
|
||||
)
|
||||
check_fail(
|
||||
"second script",
|
||||
animated.replace("</body>", "<script data-x></script></body>", 1),
|
||||
"at most one script",
|
||||
)
|
||||
check_fail(
|
||||
"hidden motion item",
|
||||
animated.replace(
|
||||
'<g data-motion-item data-step="1"',
|
||||
'<g style="opacity:0" data-motion-item data-step="1"',
|
||||
1,
|
||||
),
|
||||
"hidden in source",
|
||||
)
|
||||
check_fail(
|
||||
"missing noscript",
|
||||
animated.replace("<noscript>", "<div>", 1).replace("</noscript>", "</div>", 1),
|
||||
"<noscript>",
|
||||
)
|
||||
check_fail(
|
||||
"non-decimal step count",
|
||||
animated.replace('data-step-count="5"', 'data-step-count="٥"', 1),
|
||||
"ASCII decimal",
|
||||
)
|
||||
|
||||
if failures:
|
||||
for failure in failures:
|
||||
print(f"FAIL: {failure}")
|
||||
return 1
|
||||
print("All self-check tests passed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify that routing and browsing surfaces stay in sync with the skill.
|
||||
|
||||
Three drift classes, each of which has shipped before:
|
||||
|
||||
1. The SKILL.md frontmatter description is the only text an agent sees before
|
||||
deciding to load the skill — every visual type in the selection table must
|
||||
keep a lexical hook there.
|
||||
2. The gallery (assets/index.html) must reach every shipped example, and every
|
||||
gallery tab must point at a file that exists.
|
||||
3. Every concrete file named in README.md's architecture tree must exist.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SKILL = ROOT / "skills/diagram-design/SKILL.md"
|
||||
GALLERY = ROOT / "skills/diagram-design/assets/index.html"
|
||||
ASSET_DIR = ROOT / "skills/diagram-design/assets"
|
||||
README = ROOT / "README.md"
|
||||
VARIANTS = ("", "-dark", "-full")
|
||||
# Types whose selection-table name differs from its description vocabulary.
|
||||
DESCRIPTION_ALIASES = {
|
||||
"bar chart": "bar",
|
||||
"line chart": "line",
|
||||
"scatter plot": "scatter",
|
||||
}
|
||||
|
||||
|
||||
def normalized(text: str) -> str:
|
||||
text = text.casefold()
|
||||
text = re.sub(r"\s*/\s*", "/", text)
|
||||
return re.sub(r"\s+", " ", text)
|
||||
|
||||
|
||||
def frontmatter_description(markdown: str) -> str:
|
||||
parts = markdown.split("---")
|
||||
if len(parts) < 3:
|
||||
return ""
|
||||
match = re.search(r"^description:\s*(.+)$", parts[1], re.MULTILINE)
|
||||
return match.group(1).strip() if match else ""
|
||||
|
||||
|
||||
def selection_table_types(markdown: str) -> list[str]:
|
||||
start = markdown.find("### Visual-type guide")
|
||||
end = markdown.find("Rules of thumb", start)
|
||||
if start < 0 or end < 0:
|
||||
return []
|
||||
names = re.findall(r"^\|[^|]*\|\s*\*\*([^*]+)\*\*\s*\|", markdown[start:end], re.MULTILINE)
|
||||
return [name.strip() for name in names]
|
||||
|
||||
|
||||
def check_description(errors: list[str]) -> None:
|
||||
markdown = SKILL.read_text(encoding="utf-8")
|
||||
description = normalized(frontmatter_description(markdown))
|
||||
if not description:
|
||||
errors.append("SKILL.md frontmatter description is missing")
|
||||
return
|
||||
types = selection_table_types(markdown)
|
||||
if len(types) != 27:
|
||||
errors.append(f"expected 27 visual types in the selection table; found {len(types)}")
|
||||
for name in types:
|
||||
key = normalized(name)
|
||||
key = DESCRIPTION_ALIASES.get(key, key)
|
||||
if key not in description:
|
||||
errors.append(
|
||||
f"description lost the lexical hook for type {name!r} "
|
||||
f"(expected {key!r} in the SKILL.md frontmatter description)"
|
||||
)
|
||||
|
||||
|
||||
def gallery_types(source: str) -> list[str]:
|
||||
return re.findall(r'data-type="([^"]+)"', source)
|
||||
|
||||
|
||||
def check_gallery(errors: list[str]) -> None:
|
||||
source = GALLERY.read_text(encoding="utf-8")
|
||||
types = gallery_types(source)
|
||||
if not types:
|
||||
errors.append("gallery has no data-type tabs")
|
||||
return
|
||||
reachable = {f"example-{name}{variant}.html" for name in types for variant in VARIANTS}
|
||||
on_disk = {path.name for path in ASSET_DIR.glob("example-*.html")}
|
||||
for name in sorted(on_disk - reachable):
|
||||
errors.append(f"gallery cannot reach shipped example {name}; add a tab to assets/index.html")
|
||||
for name in sorted(types):
|
||||
if f"example-{name}.html" not in on_disk:
|
||||
errors.append(f"gallery tab {name!r} points at a missing example-{name}.html")
|
||||
|
||||
|
||||
def readme_tree_tokens(markdown: str) -> list[str]:
|
||||
blocks = re.findall(r"```\n(diagram-design/\n.*?)```", markdown, re.DOTALL)
|
||||
tokens: list[str] = []
|
||||
for block in blocks:
|
||||
tokens.extend(
|
||||
re.findall(r"([A-Za-z0-9][A-Za-z0-9_.*-]*\.(?:md|html|py|yml|yaml|json|txt|mmd|drawio|png))", block)
|
||||
)
|
||||
return tokens
|
||||
|
||||
|
||||
def check_readme_tree(errors: list[str]) -> None:
|
||||
markdown = README.read_text(encoding="utf-8")
|
||||
tokens = readme_tree_tokens(markdown)
|
||||
if not tokens:
|
||||
errors.append("README architecture tree not found or names no files")
|
||||
return
|
||||
for token in sorted(set(tokens)):
|
||||
matches = list(ROOT.rglob(token))
|
||||
if not matches:
|
||||
errors.append(f"README architecture tree names {token!r} but no such file exists")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
errors: list[str] = []
|
||||
check_description(errors)
|
||||
check_gallery(errors)
|
||||
check_readme_tree(errors)
|
||||
if errors:
|
||||
print("FAIL docs sync")
|
||||
for error in errors:
|
||||
print(f" - {error}")
|
||||
return 1
|
||||
print("OK docs sync: description hooks, gallery reachability, README tree")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user