From 4faae6696c2953b59dee2b89ad89c688f80c3a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AE=B8=E6=96=87=E6=B6=9B?= Date: Tue, 25 Aug 2026 14:57:59 +0800 Subject: [PATCH] Add adversarial tests for verify-semantic-motion.py and verify-sequence-oauth.py (#69) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add adversarial tests for verify-semantic-motion.py and verify-sequence-oauth.py Every other verify-*.py in scripts/ has a matching test-verify-*.py (verify-docs-sync, verify-geometry, verify-motion, verify-plugin-package), and CONTRIBUTING.md lists both verify-semantic-motion.py and verify-sequence-oauth.py as required PR gates, but neither had a dedicated test file exercising its own pass/fail behavior. - test-verify-semantic-motion.py: asserts the shipped SKILL.md / semantic-patterns.md / animation.md and the policy-trace animated example pass verify_markdown()/verify_example(), plus adversarial cases for a missing semantic-pattern router link, a missing pattern name, and a duplicate HTML/SVG id in the example. - test-verify-sequence-oauth.py: CLI smoke-tests the shipped repo (expects ALL GATES PASSED) and an adversarial case that removes the dashed-return marker from the OAuth 401 response in a temporary copy of the repo tree (verify-sequence-oauth.py resolves paths from its own __file__, so the adversarial run executes the copied script against the copied tree). Also wires both new tests into CONTRIBUTING.md's gate table and combined validation command, following the exact pattern used for the existing test-verify-*.py gates. (CI workflow wiring left out of this PR since it touches .github/workflows/ci.yml — happy to add in a follow-up if useful.) * Address review feedback on PR #69 - Wire test-verify-semantic-motion.py and test-verify-sequence-oauth.py into .github/workflows/ci.yml so CONTRIBUTING's new gates actually run in CI, with matching step-summary rows. - Bump .claude-plugin/plugin.json and .codex-plugin/plugin.json in sync to 2.5.7 so scripts/verify-plugin-package.py origin/main passes again. - Simplify test-verify-sequence-oauth.py's adversarial fixture to copy only the files verify-sequence-oauth.py's main() reads before the dashed-return check (type-sequence.md + cold-cache/oauth example trios), instead of shutil.copytree of the whole repository. * Re-bump plugin manifests above current main The prior 2.5.7 bump was superseded by subsequent merges of main into this branch (main itself advanced past it to 2.6.5). Re-run bump-plugin-version.py so the branch is strictly above upstream main again (2.6.6), per @philyuchkoff's review: the diff had settled back to no version delta at all. --------- Co-authored-by: alvinhui Co-authored-by: Cathryn Lavery <50469282+cathrynlavery@users.noreply.github.com> --- .claude-plugin/plugin.json | 2 +- .codex-plugin/plugin.json | 2 +- .factory-plugin/plugin.json | 2 +- .github/workflows/ci.yml | 12 +++ CONTRIBUTING.md | 4 + scripts/test-verify-semantic-motion.py | 115 +++++++++++++++++++++++++ scripts/test-verify-sequence-oauth.py | 115 +++++++++++++++++++++++++ 7 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 scripts/test-verify-semantic-motion.py create mode 100644 scripts/test-verify-sequence-oauth.py diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index c2a4ee7..e07f4d8 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "diagram-design", "description": "Create branded architecture, IT current-state, flowchart, sequence, state machine, ER/data model, timeline, swimlane, quadrant, radar/spider, polar chart (polar/radial lollipop), loop/flywheel, nested, tree, org chart, layer stack, Venn, pyramid/funnel, treemap, bar, line, Gantt and scatter charts, high-level, process, medallion, data flow, DP integration, DP security matrix, sankey, fishbone, Wardley map, kanban, user journey, deployment, dependency graph, UML class, story map, or database schema diagrams as standalone HTML/SVG/PNG. Redraw .drawio/.drawio.png/.drawio.svg or Mermaid .mmd sources at a chosen size/detail; onboard brand tokens from a website; add semantic patterns, callouts, accessible motion, or sketchy/hand-drawn styling.", - "version": "2.6.5", + "version": "2.6.6", "author": { "name": "Cathryn Lavery", "url": "https://github.com/cathrynlavery" diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 2ac613f..7916e71 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "diagram-design", "description": "Create branded architecture, IT current-state, flowchart, sequence, state machine, ER/data model, timeline, swimlane, quadrant, radar/spider, polar chart (polar/radial lollipop), loop/flywheel, nested, tree, org chart, layer stack, Venn, pyramid/funnel, treemap, bar, line, Gantt and scatter charts, high-level, process, medallion, data flow, DP integration, DP security matrix, sankey, fishbone, Wardley map, kanban, user journey, deployment, dependency graph, UML class, story map, or database schema diagrams as standalone HTML/SVG/PNG. Redraw .drawio/.drawio.png/.drawio.svg or Mermaid .mmd sources at a chosen size/detail; onboard brand tokens from a website; add semantic patterns, callouts, accessible motion, or sketchy/hand-drawn styling.", - "version": "2.6.5", + "version": "2.6.6", "author": { "name": "Cathryn Lavery", "url": "https://github.com/cathrynlavery" diff --git a/.factory-plugin/plugin.json b/.factory-plugin/plugin.json index c2a4ee7..e07f4d8 100644 --- a/.factory-plugin/plugin.json +++ b/.factory-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "diagram-design", "description": "Create branded architecture, IT current-state, flowchart, sequence, state machine, ER/data model, timeline, swimlane, quadrant, radar/spider, polar chart (polar/radial lollipop), loop/flywheel, nested, tree, org chart, layer stack, Venn, pyramid/funnel, treemap, bar, line, Gantt and scatter charts, high-level, process, medallion, data flow, DP integration, DP security matrix, sankey, fishbone, Wardley map, kanban, user journey, deployment, dependency graph, UML class, story map, or database schema diagrams as standalone HTML/SVG/PNG. Redraw .drawio/.drawio.png/.drawio.svg or Mermaid .mmd sources at a chosen size/detail; onboard brand tokens from a website; add semantic patterns, callouts, accessible motion, or sketchy/hand-drawn styling.", - "version": "2.6.5", + "version": "2.6.6", "author": { "name": "Cathryn Lavery", "url": "https://github.com/cathrynlavery" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74b4c3d..2e97777 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,6 +155,16 @@ jobs: id: oauth run: python scripts/verify-sequence-oauth.py + - name: Verify semantic-motion verifier + if: always() + id: semantic_motion_tests + run: python scripts/test-verify-semantic-motion.py + + - name: Verify sequence-oauth verifier + if: always() + id: oauth_tests + run: python scripts/test-verify-sequence-oauth.py + - name: Verify draw.io import script if: always() id: drawio @@ -327,6 +337,8 @@ jobs: echo "| Color Palette & Skin Linter | ${{ steps.skin.outcome == 'success' && '✅ Passed' || (steps.skin.outcome == 'skipped' && '⏭️ Skipped' || '❌ Failed') }} |" >> $GITHUB_STEP_SUMMARY echo "| Quantitative Polar Chart | ${{ steps.polar.outcome == 'success' && '✅ Passed' || (steps.polar.outcome == 'skipped' && '⏭️ Skipped' || '❌ Failed') }} |" >> $GITHUB_STEP_SUMMARY echo "| Sequence Diagram Grammar | ${{ steps.oauth.outcome == 'success' && '✅ Passed' || (steps.oauth.outcome == 'skipped' && '⏭️ Skipped' || '❌ Failed') }} |" >> $GITHUB_STEP_SUMMARY + echo "| Semantic-Motion Verifier Tests | ${{ steps.semantic_motion_tests.outcome == 'success' && '✅ Passed' || (steps.semantic_motion_tests.outcome == 'skipped' && '⏭️ Skipped' || '❌ Failed') }} |" >> $GITHUB_STEP_SUMMARY + echo "| Sequence-OAuth Verifier Tests | ${{ steps.oauth_tests.outcome == 'success' && '✅ Passed' || (steps.oauth_tests.outcome == 'skipped' && '⏭️ Skipped' || '❌ Failed') }} |" >> $GITHUB_STEP_SUMMARY echo "| Draw.io Import Extractor | ${{ steps.drawio.outcome == 'success' && '✅ Passed' || (steps.drawio.outcome == 'skipped' && '⏭️ Skipped' || '❌ Failed') }} |" >> $GITHUB_STEP_SUMMARY echo "| Mermaid Import Extractor | ${{ steps.mermaid.outcome == 'success' && '✅ Passed' || (steps.mermaid.outcome == 'skipped' && '⏭️ Skipped' || '❌ Failed') }} |" >> $GITHUB_STEP_SUMMARY echo "| Optional Motion Contract | ${{ steps.motion.outcome == 'success' && '✅ Passed' || (steps.motion.outcome == 'skipped' && '⏭️ Skipped' || '❌ Failed') }} |" >> $GITHUB_STEP_SUMMARY diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 43ede7b..fd1b68a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,8 @@ The helper refuses to run if the Claude and Codex versions already differ. If an | Quantitative polar encoding and variant parity | `python3 scripts/test-verify-polar.py && python3 scripts/verify-polar.py` | | A single file, e.g. a new example | `python3 scripts/lint-skin.py skills/diagram-design/assets/example-my-type.html` | | Sequence-doc consistency (ATL fragments, budgets) | `python3 scripts/verify-sequence-oauth.py` | +| Semantic-motion verifier behaves (pass + adversarial cases) | `python3 scripts/test-verify-semantic-motion.py` | +| Sequence-oauth verifier behaves (pass + adversarial cases) | `python3 scripts/test-verify-sequence-oauth.py` | | draw.io import path (real extractor vs fixtures + docs sync) | `python3 scripts/verify-drawio-import.py && python3 scripts/test-verify-drawio-import.py` | | Mermaid import path (grammars, adversarial input, caps, docs sync) | `python3 scripts/verify-mermaid-import.py` | | Optional motion contract (fallbacks, controls, budgets, determinism) | `python3 scripts/test-verify-motion.py` | @@ -93,6 +95,8 @@ python3 scripts/test-plugin-package.py \ && python3 scripts/test-verify-polar.py \ && python3 scripts/verify-polar.py \ && python3 scripts/verify-sequence-oauth.py \ + && python3 scripts/test-verify-semantic-motion.py \ + && python3 scripts/test-verify-sequence-oauth.py \ && python3 scripts/verify-drawio-import.py \ && python3 scripts/test-verify-drawio-import.py \ && python3 scripts/verify-mermaid-import.py \ diff --git a/scripts/test-verify-semantic-motion.py b/scripts/test-verify-semantic-motion.py new file mode 100644 index 0000000..e24f0ed --- /dev/null +++ b/scripts/test-verify-semantic-motion.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Regression tests for verify-semantic-motion.py. + +Covers the two entry points (verify_markdown, verify_example) against the +shipped skill docs / animated example, plus a handful of adversarial cases +that mirror the pattern used by test-verify-docs-sync.py and +test-verify-motion.py for the other verifiers in this repo. +""" + +from __future__ import annotations + +import importlib.util +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +VERIFIER = ROOT / "scripts/verify-semantic-motion.py" + + +def load_verifier(): + sys.dont_write_bytecode = True + spec = importlib.util.spec_from_file_location( + "diagram_design_verify_semantic_motion", VERIFIER + ) + if spec is None or spec.loader is None: + raise AssertionError("could not load verify-semantic-motion.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def main() -> int: + module = load_verifier() + + # Shipped docs and example must pass as-is. + markdown_errors = module.verify_markdown() + if markdown_errors: + raise AssertionError(f"shipped skill docs failed verify_markdown: {markdown_errors}") + print("OK: shipped SKILL.md / semantic-patterns.md / animation.md pass verify_markdown") + + example_errors = module.verify_example() + if example_errors: + raise AssertionError(f"shipped example failed verify_example: {example_errors}") + print("OK: shipped policy-trace example passes verify_example") + + original_skill = module.SKILL + try: + with tempfile.TemporaryDirectory(prefix="verify-semantic-motion-") as temp_dir: + scratch = Path(temp_dir) + + # Missing semantic-pattern router link must be rejected. + missing_router = scratch / "missing-router.md" + missing_router.write_text( + original_skill.read_text(encoding="utf-8").replace( + "semantic-patterns.md", "patterns.md" + ), + encoding="utf-8", + ) + module.SKILL = missing_router + errors = module.verify_markdown() + if not any("must link to semantic-patterns.md" in error for error in errors): + raise AssertionError(f"missing semantic router was accepted: {errors}") + print("OK: missing semantic-pattern router link is rejected") + + # Dropping one of the seven named patterns must be rejected. + missing_pattern = scratch / "missing-pattern.md" + missing_pattern.write_text( + original_skill.read_text(encoding="utf-8").replace( + "Fan-in queue / bottleneck", "Fan-in queue removed" + ), + encoding="utf-8", + ) + module.SKILL = missing_pattern + errors = module.verify_markdown() + if not any( + "does not route semantic pattern: Fan-in queue / bottleneck" in error + for error in errors + ): + raise AssertionError(f"missing semantic pattern was accepted: {errors}") + print("OK: missing semantic-pattern name is rejected") + finally: + module.SKILL = original_skill + + # A duplicated HTML/SVG id in the animated example must be rejected. + with tempfile.TemporaryDirectory(prefix="verify-semantic-motion-example-") as temp_dir: + source = module.EXAMPLE.read_text(encoding="utf-8") + first_id_start = source.find(' id="') + if first_id_start < 0: + raise AssertionError("shipped example unexpectedly has no id attributes to duplicate") + # Re-use an existing id value on a second, unrelated element to force a collision. + quote_start = first_id_start + len(' id="') + quote_end = source.find('"', quote_start) + duplicated_id = source[quote_start:quote_end] + insertion_point = source.rfind("") + if insertion_point < 0: + raise AssertionError("shipped example unexpectedly has no to anchor the test") + broken = ( + source[:insertion_point] + + f'
' + + source[insertion_point:] + ) + broken_path = Path(temp_dir) / "duplicate-id.html" + broken_path.write_text(broken, encoding="utf-8") + errors = module.verify_example(broken_path) + if not any("duplicate HTML/SVG IDs" in error for error in errors): + raise AssertionError(f"duplicate id was accepted: {errors}") + print("OK: duplicate HTML/SVG id in the animated example is rejected") + + print("All semantic-motion verifier tests passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test-verify-sequence-oauth.py b/scripts/test-verify-sequence-oauth.py new file mode 100644 index 0000000..263b15a --- /dev/null +++ b/scripts/test-verify-sequence-oauth.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Regression tests for verify-sequence-oauth.py. + +verify-sequence-oauth.py drives its checks against the real shipped +sequence-diagram assets from a top-level main(), so it is exercised here as a +CLI smoke test (pass) plus an adversarial case that mutates a minimal temporary +copy of just the files main() reads before it reaches the dashed-return check +(type-sequence.md plus the cold-cache and oauth example trios) to confirm a +broken guard/return is rejected. This mirrors the CLI smoke-test pattern used +by test-verify-motion.py for verify-motion.py. +""" + +from __future__ import annotations + +import subprocess +import sys +import tempfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +VERIFIER = ROOT / "scripts/verify-sequence-oauth.py" +TYPE_SEQ = ROOT / "skills/diagram-design/references/type-sequence.md" +ASSETS = ROOT / "skills/diagram-design/assets" +COLD = [ + ASSETS / "example-sequence.html", + ASSETS / "example-sequence-dark.html", + ASSETS / "example-sequence-full.html", +] +OAUTH = [ + ASSETS / "example-sequence-oauth.html", + ASSETS / "example-sequence-oauth-dark.html", + ASSETS / "example-sequence-oauth-full.html", +] + + +def run_verifier(cwd: Path, verifier: Path = VERIFIER) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(verifier)], + cwd=cwd, + capture_output=True, + text=True, + check=False, + ) + + +def main() -> int: + # The shipped repo must pass all gates as-is. + result = run_verifier(ROOT) + if result.returncode != 0 or "ALL GATES PASSED" not in result.stdout: + raise AssertionError( + f"shipped repo failed verify-sequence-oauth.py\n{result.stdout}{result.stderr}" + ) + print("OK: shipped repo passes verify-sequence-oauth.py") + + # Removing the dashed-return marker from the 401 response must be rejected. + # verify-sequence-oauth.py resolves every path relative to its own file + # location (ROOT = parent of scripts/), so the adversarial case runs + # against a minimal temporary copy that mirrors just the relative paths + # main() reads before the dashed-return check (type-sequence.md plus the + # cold-cache and oauth example trios) — not the full repository tree. + with tempfile.TemporaryDirectory(prefix="verify-sequence-oauth-") as temp_dir: + temp_root = Path(temp_dir) / "repo" + temp_verifier = temp_root / "scripts/verify-sequence-oauth.py" + temp_verifier.parent.mkdir(parents=True, exist_ok=True) + temp_verifier.write_text(VERIFIER.read_text(encoding="utf-8"), encoding="utf-8") + + temp_type_seq = temp_root / TYPE_SEQ.relative_to(ROOT) + temp_type_seq.parent.mkdir(parents=True, exist_ok=True) + temp_type_seq.write_text(TYPE_SEQ.read_text(encoding="utf-8"), encoding="utf-8") + + for path in (*COLD, *OAUTH): + dest = temp_root / path.relative_to(ROOT) + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text(path.read_text(encoding="utf-8"), encoding="utf-8") + + oauth_copy = temp_root / OAUTH[0].relative_to(ROOT) + original = oauth_copy.read_text(encoding="utf-8") + target_line = ( + '' + ) + if target_line not in original: + raise AssertionError( + "expected shipped example-sequence-oauth.html to contain the dashed " + "401 return line at y1=368" + ) + broken = original.replace( + target_line, + target_line.replace('stroke-dasharray="5,4" ', ""), + 1, + ) + if broken == original: + raise AssertionError("failed to mutate example-sequence-oauth.html for the test") + oauth_copy.write_text(broken, encoding="utf-8") + + # verify-sequence-oauth.py resolves ROOT from its own __file__, so the + # adversarial run must invoke the copy inside temp_root, not VERIFIER. + broken_result = run_verifier(temp_root, verifier=temp_verifier) + if broken_result.returncode == 0: + raise AssertionError( + f"broken dashed return was accepted\n{broken_result.stdout}{broken_result.stderr}" + ) + if "must be dashed" not in broken_result.stdout + broken_result.stderr: + raise AssertionError( + "expected a 'must be dashed' failure, got " + f"{broken_result.stdout}{broken_result.stderr}" + ) + print("OK: removing the dashed 401 return marker is rejected") + + print("All sequence-oauth verifier tests passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main())