global setup and configuration.

skills/document/unslop/scripts/_lang.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""Shared cheap English-detection helpers for banned_phrase_scan.py,
+structure_scan.py, and silhouette_scan.py. All three scanners must decline the
+same non-English inputs, so this table and its two functions live in exactly
+one place.
+
+Also home to the shared prose-view helpers (tokenizer, markdown stripper,
+paragraph splitter) used by structure_scan.py and silhouette_scan.py. Those
+two scanners previously carried private copies of these functions that had
+drifted from each other (structure blanked blockquote lines, silhouette
+didn't; silhouette stripped **bold**, structure didn't). strip_markdown_for_prose
+below is the UNION of both original code paths, gated behind flags so each
+caller reconciles the drift deliberately rather than silently: structure_scan
+passes blank_blockquotes=True, silhouette_scan passes strip_bold=True."""
+
+from __future__ import annotations
+
+import re
+
+
+# A small set of high-frequency English function words used only for a cheap
+# language check. The words are chosen to be distinctively English: Spanish,
+# French, German, etc. rarely use them, so their share of tokens is a robust
+# signal without a language-model dependency.
+ENGLISH_FUNCTION_WORDS = frozenset({
+    "the", "and", "is", "are", "was", "were", "of", "to", "in", "that", "it",
+    "for", "with", "on", "this", "but", "not", "you", "have", "be", "as", "at",
+    "or", "we", "they", "will", "would", "there", "their", "what", "which",
+    "when", "from", "been", "has", "had", "its", "an", "by", "our", "your",
+    "if", "than", "then", "them", "these", "those", "about", "into", "over",
+    "after", "before", "how", "why", "where", "who", "can", "could", "should",
+    "do", "does", "did", "so", "out", "just", "more", "most", "some", "such",
+    "only", "also", "because", "while", "between", "through", "during", "being",
+})
+
+
+def english_function_share(text: str) -> float:
+    """Share of word tokens that are common English function words."""
+    tokens = re.findall(r"[a-z']+", text.lower())
+    if not tokens:
+        return 1.0
+    hits = sum(1 for t in tokens if t in ENGLISH_FUNCTION_WORDS)
+    return hits / len(tokens)
+
+
+def is_probably_english(text: str, threshold: float = 0.10, min_tokens: int = 15) -> bool:
+    """Cheap English detector backing a graceful non-English decline.
+
+    Conservative on purpose: inputs below ``min_tokens`` are always treated as
+    English (too little signal to decline), and ``threshold`` is low enough that
+    even terse or ESL-flavored English clears it. Only prose with almost no
+    English function words (i.e. another language) is declined.
+    """
+    tokens = re.findall(r"[a-z']+", text.lower())
+    if len(tokens) < min_tokens:
+        return True
+    return english_function_share(text) >= threshold
+
+
+def words(text: str) -> list[str]:
+    return re.findall(r"[A-Za-z0-9]+(?:[-'][A-Za-z0-9]+)?", text.lower())
+
+
+def strip_markdown_for_prose(
+    text: str, *, blank_blockquotes: bool = False, strip_bold: bool = False
+) -> str:
+    """Strip markdown structure down to a prose view for cadence/discourse
+    metrics. UNION of structure_scan's and silhouette_scan's original
+    strippers -- each original branch is preserved verbatim, gated behind the
+    flag that reproduces that caller's exact prior behavior.
+
+    blank_blockquotes=True reproduces structure_scan's prior strip (blanks
+    both blockquote and heading lines in one combined check). strip_bold=True
+    reproduces silhouette_scan's prior strip (also collapses **bold** markers
+    after list/ordinal stripping).
+    """
+    text = re.sub(r"```[\s\S]*?```", "\n\n", text)
+    kept = []
+    for line in text.splitlines():
+        if blank_blockquotes:
+            if re.match(r"\s*>", line) or re.match(r"\s{0,3}#{1,6}\s+", line):
+                kept.append("")
+                continue
+        else:
+            if re.match(r"\s{0,3}#{1,6}\s+", line):
+                kept.append("")  # drop heading text from the prose view
+                continue
+        line = re.sub(r"^\s*[-*+]\s+", "", line)
+        line = re.sub(r"^\s*\d+[.)]\s+", "", line)
+        if strip_bold:
+            line = re.sub(r"\*\*([^*]+)\*\*", r"\1", line)
+        kept.append(line)
+    return "\n".join(kept)
+
+
+def paragraphs(
+    text: str, *, blank_blockquotes: bool = False, strip_bold: bool = False
+) -> list[str]:
+    stripped = strip_markdown_for_prose(
+        text, blank_blockquotes=blank_blockquotes, strip_bold=strip_bold
+    )
+    return [
+        re.sub(r"\s+", " ", p).strip()
+        for p in re.split(r"\n\s*\n", stripped)
+        if p.strip()
+    ]