fix(import): emit extractor output as UTF-8 on Windows (#122)

* fix(import): emit extractor digests as UTF-8

* fix(import): limit UTF-8 setup to CLI entrypoints

* test(import): normalize cross-platform newline parity

.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "diagram-design",
   "description": "Create branded architecture, IT current-state, flowchart, sequence, state machine, ER/data model, timeline, swimlane, quadrant, radar/spider, loop/flywheel, nested, tree, org chart, layer stack, Venn, pyramid/funnel, treemap, bar, line, Gantt and scatter charts, high-level, process, medallion, data flow, DP integration, DP security matrix, sankey, fishbone, Wardley map, kanban, user journey, deployment, dependency graph, UML class, story map, or database schema diagrams as standalone HTML/SVG/PNG. Redraw .drawio/.drawio.png/.drawio.svg or Mermaid .mmd sources at a chosen size/detail; onboard brand tokens from a website; add semantic patterns, callouts, accessible motion, or sketchy/hand-drawn styling.",
-  "version": "2.5.14",
+  "version": "2.5.15",
   "author": {
     "name": "Cathryn Lavery",
     "url": "https://github.com/cathrynlavery"

.codex-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "diagram-design",
   "description": "Create branded architecture, IT current-state, flowchart, sequence, state machine, ER/data model, timeline, swimlane, quadrant, radar/spider, loop/flywheel, nested, tree, org chart, layer stack, Venn, pyramid/funnel, treemap, bar, line, Gantt and scatter charts, high-level, process, medallion, data flow, DP integration, DP security matrix, sankey, fishbone, Wardley map, kanban, user journey, deployment, dependency graph, UML class, story map, or database schema diagrams as standalone HTML/SVG/PNG. Redraw .drawio/.drawio.png/.drawio.svg or Mermaid .mmd sources at a chosen size/detail; onboard brand tokens from a website; add semantic patterns, callouts, accessible motion, or sketchy/hand-drawn styling.",
-  "version": "2.5.14",
+  "version": "2.5.15",
   "author": {
     "name": "Cathryn Lavery",
     "url": "https://github.com/cathrynlavery"

.factory-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "diagram-design",
   "description": "Create branded architecture, IT current-state, flowchart, sequence, state machine, ER/data model, timeline, swimlane, quadrant, radar/spider, loop/flywheel, nested, tree, org chart, layer stack, Venn, pyramid/funnel, treemap, bar, line, Gantt and scatter charts, high-level, process, medallion, data flow, DP integration, DP security matrix, sankey, fishbone, Wardley map, kanban, user journey, deployment, dependency graph, UML class, story map, or database schema diagrams as standalone HTML/SVG/PNG. Redraw .drawio/.drawio.png/.drawio.svg or Mermaid .mmd sources at a chosen size/detail; onboard brand tokens from a website; add semantic patterns, callouts, accessible motion, or sketchy/hand-drawn styling.",
-  "version": "2.5.14",
+  "version": "2.5.15",
   "author": {
     "name": "Cathryn Lavery",
     "url": "https://github.com/cathrynlavery"

scripts/verify-drawio-import.py

@@ -10,8 +10,11 @@ Exit 0 only when all gates pass. Intended for local/PR checks alongside:
 from __future__ import annotations
 
 import base64
+import contextlib
 import importlib.util
+import io
 import json
+import os
 import re
 import struct
 import subprocess
@@ -43,6 +46,10 @@ def ok(msg: str) -> None:
     print(f"OK: {msg}")
 
 
+def normalize_newlines(text: str) -> str:
+    return text.replace("\r\n", "\n").replace("\r", "\n")
+
+
 def run_extract(args: list[str]) -> str:
     proc = subprocess.run(
         [sys.executable, str(EXTRACT), *args],
@@ -56,6 +63,71 @@ def run_extract(args: list[str]) -> str:
     return proc.stdout
 
 
+def check_legacy_stdout_encoding(tmp: Path) -> None:
+    source = tmp / "unicode-stdout.drawio"
+    source.write_text(
+        """<mxfile><diagram name="日本語">
+<mxGraphModel><root><mxCell id="0"/><mxCell id="1" parent="0"/>
+<mxCell id="2" value="登录&lt;br&gt;続行 ⇒ résumé" vertex="1" parent="1">
+<mxGeometry x="0" y="0" width="120" height="60" as="geometry"/>
+</mxCell></root></mxGraphModel></diagram></mxfile>""",
+        encoding="utf-8",
+    )
+    env = os.environ.copy()
+    env["PYTHONIOENCODING"] = "cp1252"
+    env["PYTHONUTF8"] = "0"
+    process = subprocess.run(
+        [sys.executable, str(EXTRACT), str(source)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=env,
+    )
+    if process.returncode != 0:
+        fail(
+            "draw.io extractor failed with legacy stdout encoding: "
+            + process.stderr.decode("utf-8", errors="replace").strip()
+        )
+    try:
+        output = process.stdout.decode("utf-8", errors="strict")
+    except UnicodeDecodeError as error:
+        fail(f"draw.io extractor did not emit UTF-8 stdout: {error}")
+    for needle in ("日本語", "登录", "続行 ⇒ résumé", "⏎"):
+        if needle not in output:
+            fail(f"UTF-8 draw.io digest lost {needle!r}: {output!r}")
+    if "�" in output:
+        fail("UTF-8 draw.io digest contains a replacement character")
+    destination = tmp / "unicode-stdout.md"
+    file_process = subprocess.run(
+        [sys.executable, str(EXTRACT), str(source), "--out", str(destination)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=env,
+    )
+    if file_process.returncode != 0:
+        fail("draw.io --out failed under a legacy Windows encoding")
+    file_output = destination.read_text(encoding="utf-8")
+    if normalize_newlines(file_output) != normalize_newlines(output):
+        fail("draw.io --out no longer matches its UTF-8 stdout digest")
+
+    class CallerOwnedStdout(io.StringIO):
+        def __init__(self) -> None:
+            super().__init__()
+            self.reconfigured = False
+
+        def reconfigure(self, **_kwargs: object) -> None:
+            self.reconfigured = True
+
+    caller_stdout = CallerOwnedStdout()
+    extractor = load_extractor_module()
+    with contextlib.redirect_stdout(caller_stdout):
+        result = extractor.main([str(source)])
+    if result != 0 or caller_stdout.reconfigured:
+        fail("imported draw.io main() reconfigured its caller-owned stdout")
+    if "登录" not in caller_stdout.getvalue():
+        fail("imported draw.io main() did not write to its caller-owned stdout")
+    ok("draw.io stdout stays lossless UTF-8 under a legacy Windows encoding")
+
+
 def load_extractor_module():
     spec = importlib.util.spec_from_file_location("diagram_design_drawio_extract", EXTRACT)
     if spec is None or spec.loader is None:
@@ -443,6 +515,7 @@ def main() -> int:
         check_files()
         check_parse_raw()
         check_containers(tmp)
+        check_legacy_stdout_encoding(tmp)
         check_digest_escaping(tmp)
         check_security_and_limits(tmp)
         check_docs()

scripts/verify-mermaid-import.py

@@ -8,8 +8,11 @@ command wiring. Exit 0 only when every gate passes.
 
 from __future__ import annotations
 
+import contextlib
 import importlib.util
+import io
 import json
+import os
 import subprocess
 import sys
 import tempfile
@@ -38,6 +41,10 @@ def ok(message: str) -> None:
     print(f"OK: {message}")
 
 
+def normalize_newlines(text: str) -> str:
+    return text.replace("\r\n", "\n").replace("\r", "\n")
+
+
 def invoke(args: list[str]) -> subprocess.CompletedProcess[str]:
     return subprocess.run(
         [sys.executable, str(EXTRACT), *args],
@@ -58,6 +65,67 @@ def run_extract(args: list[str]) -> str:
     return process.stdout
 
 
+def check_legacy_stdout_encoding(tmp: Path) -> None:
+    source = tmp / "unicode-stdout.mmd"
+    source.write_text(
+        'flowchart TD\nA["登录<br/>続行 ⇒"] --> B["résumé"]\n',
+        encoding="utf-8",
+    )
+    env = os.environ.copy()
+    env["PYTHONIOENCODING"] = "cp1252"
+    env["PYTHONUTF8"] = "0"
+    process = subprocess.run(
+        [sys.executable, str(EXTRACT), str(source)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=env,
+    )
+    if process.returncode != 0:
+        fail(
+            "Mermaid extractor failed with legacy stdout encoding: "
+            + process.stderr.decode("utf-8", errors="replace").strip()
+        )
+    try:
+        output = process.stdout.decode("utf-8", errors="strict")
+    except UnicodeDecodeError as error:
+        fail(f"Mermaid extractor did not emit UTF-8 stdout: {error}")
+    for needle in ("登录", "続行 ⇒", "résumé", "⏎"):
+        if needle not in output:
+            fail(f"UTF-8 Mermaid digest lost {needle!r}: {output!r}")
+    if "�" in output:
+        fail("UTF-8 Mermaid digest contains a replacement character")
+    destination = tmp / "unicode-stdout.md"
+    file_process = subprocess.run(
+        [sys.executable, str(EXTRACT), str(source), "--out", str(destination)],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        env=env,
+    )
+    if file_process.returncode != 0:
+        fail("Mermaid --out failed under a legacy Windows encoding")
+    file_output = destination.read_text(encoding="utf-8")
+    if normalize_newlines(file_output) != normalize_newlines(output):
+        fail("Mermaid --out no longer matches its UTF-8 stdout digest")
+
+    class CallerOwnedStdout(io.StringIO):
+        def __init__(self) -> None:
+            super().__init__()
+            self.reconfigured = False
+
+        def reconfigure(self, **_kwargs: object) -> None:
+            self.reconfigured = True
+
+    caller_stdout = CallerOwnedStdout()
+    extractor = load_extractor_module()
+    with contextlib.redirect_stdout(caller_stdout):
+        result = extractor.main([str(source)])
+    if result != 0 or caller_stdout.reconfigured:
+        fail("imported Mermaid main() reconfigured its caller-owned stdout")
+    if "登录" not in caller_stdout.getvalue():
+        fail("imported Mermaid main() did not write to its caller-owned stdout")
+    ok("Mermaid stdout stays lossless UTF-8 under a legacy Windows encoding")
+
+
 def expect_error(args: list[str], message: str) -> None:
     process = invoke(args)
     if process.returncode != 2 or message not in process.stderr:
@@ -708,6 +776,7 @@ def main() -> int:
         check_shape_and_edge_vocabulary(tmp)
         check_frontmatter(tmp)
         check_markdown_and_grammars(tmp)
+        check_legacy_stdout_encoding(tmp)
         check_sequence_grammar_forms(tmp)
         check_adversarial(tmp)
         check_errors_and_limits(tmp)

skills/diagram-design/scripts/drawio_extract.py

@@ -43,6 +43,13 @@ MAX_INPUT_BYTES = 32 * 1024 * 1024
 MAX_XML_BYTES = 64 * 1024 * 1024
 
 
+def _configure_stdout_utf8() -> None:
+    """Emit digests as UTF-8 even when Windows selects a legacy codepage."""
+    reconfigure = getattr(sys.stdout, "reconfigure", None)
+    if reconfigure is not None:
+        reconfigure(encoding="utf-8", errors="strict")
+
+
 class PayloadTooLarge(ValueError):
     """Raised when compressed metadata expands beyond the supported limit."""
 
@@ -886,4 +893,5 @@ def main(argv: list[str] | None = None) -> int:
 
 
 if __name__ == "__main__":
+    _configure_stdout_utf8()
     raise SystemExit(main())

skills/diagram-design/scripts/mermaid_extract.py

@@ -51,6 +51,13 @@ MARKDOWN_SUFFIXES = {".md", ".markdown", ".mdown", ".mkd"}
 MERMAID_SUFFIXES = {".mmd", ".mermaid"}
 
 
+def _configure_stdout_utf8() -> None:
+    """Emit digests as UTF-8 even when Windows selects a legacy codepage."""
+    reconfigure = getattr(sys.stdout, "reconfigure", None)
+    if reconfigure is not None:
+        reconfigure(encoding="utf-8", errors="strict")
+
+
 def _fail(message: str) -> NoReturn:
     print(f"mermaid_extract: {message}", file=sys.stderr)
     raise SystemExit(2)
@@ -1322,4 +1329,5 @@ def main(argv: list[str] | None = None) -> int:
 
 
 if __name__ == "__main__":
+    _configure_stdout_utf8()
     raise SystemExit(main())