global setup and configuration.
This commit is contained in:
Binary file not shown.
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared cheap English-detection helpers for banned_phrase_scan.py,
|
||||
structure_scan.py, and silhouette_scan.py. All three scanners must decline the
|
||||
same non-English inputs, so this table and its two functions live in exactly
|
||||
one place.
|
||||
|
||||
Also home to the shared prose-view helpers (tokenizer, markdown stripper,
|
||||
paragraph splitter) used by structure_scan.py and silhouette_scan.py. Those
|
||||
two scanners previously carried private copies of these functions that had
|
||||
drifted from each other (structure blanked blockquote lines, silhouette
|
||||
didn't; silhouette stripped **bold**, structure didn't). strip_markdown_for_prose
|
||||
below is the UNION of both original code paths, gated behind flags so each
|
||||
caller reconciles the drift deliberately rather than silently: structure_scan
|
||||
passes blank_blockquotes=True, silhouette_scan passes strip_bold=True."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
# A small set of high-frequency English function words used only for a cheap
|
||||
# language check. The words are chosen to be distinctively English: Spanish,
|
||||
# French, German, etc. rarely use them, so their share of tokens is a robust
|
||||
# signal without a language-model dependency.
|
||||
ENGLISH_FUNCTION_WORDS = frozenset({
|
||||
"the", "and", "is", "are", "was", "were", "of", "to", "in", "that", "it",
|
||||
"for", "with", "on", "this", "but", "not", "you", "have", "be", "as", "at",
|
||||
"or", "we", "they", "will", "would", "there", "their", "what", "which",
|
||||
"when", "from", "been", "has", "had", "its", "an", "by", "our", "your",
|
||||
"if", "than", "then", "them", "these", "those", "about", "into", "over",
|
||||
"after", "before", "how", "why", "where", "who", "can", "could", "should",
|
||||
"do", "does", "did", "so", "out", "just", "more", "most", "some", "such",
|
||||
"only", "also", "because", "while", "between", "through", "during", "being",
|
||||
})
|
||||
|
||||
|
||||
def english_function_share(text: str) -> float:
|
||||
"""Share of word tokens that are common English function words."""
|
||||
tokens = re.findall(r"[a-z']+", text.lower())
|
||||
if not tokens:
|
||||
return 1.0
|
||||
hits = sum(1 for t in tokens if t in ENGLISH_FUNCTION_WORDS)
|
||||
return hits / len(tokens)
|
||||
|
||||
|
||||
def is_probably_english(text: str, threshold: float = 0.10, min_tokens: int = 15) -> bool:
|
||||
"""Cheap English detector backing a graceful non-English decline.
|
||||
|
||||
Conservative on purpose: inputs below ``min_tokens`` are always treated as
|
||||
English (too little signal to decline), and ``threshold`` is low enough that
|
||||
even terse or ESL-flavored English clears it. Only prose with almost no
|
||||
English function words (i.e. another language) is declined.
|
||||
"""
|
||||
tokens = re.findall(r"[a-z']+", text.lower())
|
||||
if len(tokens) < min_tokens:
|
||||
return True
|
||||
return english_function_share(text) >= threshold
|
||||
|
||||
|
||||
def words(text: str) -> list[str]:
|
||||
return re.findall(r"[A-Za-z0-9]+(?:[-'][A-Za-z0-9]+)?", text.lower())
|
||||
|
||||
|
||||
def strip_markdown_for_prose(
|
||||
text: str, *, blank_blockquotes: bool = False, strip_bold: bool = False
|
||||
) -> str:
|
||||
"""Strip markdown structure down to a prose view for cadence/discourse
|
||||
metrics. UNION of structure_scan's and silhouette_scan's original
|
||||
strippers -- each original branch is preserved verbatim, gated behind the
|
||||
flag that reproduces that caller's exact prior behavior.
|
||||
|
||||
blank_blockquotes=True reproduces structure_scan's prior strip (blanks
|
||||
both blockquote and heading lines in one combined check). strip_bold=True
|
||||
reproduces silhouette_scan's prior strip (also collapses **bold** markers
|
||||
after list/ordinal stripping).
|
||||
"""
|
||||
text = re.sub(r"```[\s\S]*?```", "\n\n", text)
|
||||
kept = []
|
||||
for line in text.splitlines():
|
||||
if blank_blockquotes:
|
||||
if re.match(r"\s*>", line) or re.match(r"\s{0,3}#{1,6}\s+", line):
|
||||
kept.append("")
|
||||
continue
|
||||
else:
|
||||
if re.match(r"\s{0,3}#{1,6}\s+", line):
|
||||
kept.append("") # drop heading text from the prose view
|
||||
continue
|
||||
line = re.sub(r"^\s*[-*+]\s+", "", line)
|
||||
line = re.sub(r"^\s*\d+[.)]\s+", "", line)
|
||||
if strip_bold:
|
||||
line = re.sub(r"\*\*([^*]+)\*\*", r"\1", line)
|
||||
kept.append(line)
|
||||
return "\n".join(kept)
|
||||
|
||||
|
||||
def paragraphs(
|
||||
text: str, *, blank_blockquotes: bool = False, strip_bold: bool = False
|
||||
) -> list[str]:
|
||||
stripped = strip_markdown_for_prose(
|
||||
text, blank_blockquotes=blank_blockquotes, strip_bold=strip_bold
|
||||
)
|
||||
return [
|
||||
re.sub(r"\s+", " ", p).strip()
|
||||
for p in re.split(r"\n\s*\n", stripped)
|
||||
if p.strip()
|
||||
]
|
||||
+784
@@ -0,0 +1,784 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Scan text for AI-isms and banned phrases.
|
||||
|
||||
Checks against taboo phrases list and returns violations with line numbers.
|
||||
Provides suggested replacements where available.
|
||||
|
||||
By default, quoted examples and code snippets are ignored so the scanner
|
||||
doesn't flag illustrative bad writing inside docs or tutorials. Pass
|
||||
--include-quoted to scan those spans too.
|
||||
|
||||
Usage:
|
||||
python banned_phrase_scan.py < input.txt
|
||||
python banned_phrase_scan.py input.txt
|
||||
python banned_phrase_scan.py input.txt --include-quoted
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import bisect
|
||||
import sys
|
||||
import re
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import TypedDict
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
from _lang import ( # noqa: E402
|
||||
ENGLISH_FUNCTION_WORDS,
|
||||
english_function_share,
|
||||
is_probably_english,
|
||||
)
|
||||
|
||||
|
||||
class Violation(TypedDict):
|
||||
phrase: str
|
||||
category: str
|
||||
severity: str
|
||||
line_number: int
|
||||
column: int
|
||||
context: str
|
||||
suggestion: str | None
|
||||
|
||||
|
||||
def _mask_non_newlines(text: str) -> str:
|
||||
"""Replace visible characters with spaces while preserving line/column layout."""
|
||||
return re.sub(r"[^\n]", " ", text)
|
||||
|
||||
|
||||
def mask_ignored_spans(text: str, include_quoted: bool = False) -> str:
|
||||
"""Mask examples and code so they don't produce false-positive matches."""
|
||||
masked = re.sub(r"```[\s\S]*?```", lambda m: _mask_non_newlines(m.group(0)), text)
|
||||
masked = re.sub(r"`[^`\n]+`", lambda m: _mask_non_newlines(m.group(0)), masked)
|
||||
|
||||
if include_quoted:
|
||||
return masked
|
||||
|
||||
# Markdown blockquotes are almost always cited examples rather than prose to edit.
|
||||
masked = re.sub(r"(?m)^>.*$", lambda m: _mask_non_newlines(m.group(0)), masked)
|
||||
|
||||
# Double quotes only, any length, across line breaks. Single quotes are NOT
|
||||
# masked: they collide with apostrophes/emphasis and would silently hide real
|
||||
# slop inside ordinary single-quoted prose.
|
||||
quote_patterns = [
|
||||
r'"[^"]*"',
|
||||
r"“[^”]*”",
|
||||
]
|
||||
for pattern in quote_patterns:
|
||||
masked = re.sub(pattern, lambda m: _mask_non_newlines(m.group(0)), masked)
|
||||
|
||||
return masked
|
||||
|
||||
|
||||
def _phrase_pattern(phrase: str) -> re.Pattern[str]:
|
||||
left = r"(?<![a-z0-9_-])" if phrase[0].isalnum() else ""
|
||||
right = r"(?![a-z0-9_-])" if phrase[-1].isalnum() else ""
|
||||
return re.compile(left + re.escape(phrase) + right)
|
||||
|
||||
|
||||
# Case-insensitive variant used only at the scan_for_violations call site, so
|
||||
# _phrase_pattern's case-sensitive default is unchanged for any other caller.
|
||||
# BANNED_PHRASES keys are lowercase literals; matching case-insensitively
|
||||
# against the original (unlowered) text avoids str.lower()'s non-length-
|
||||
# preserving folds (e.g. U+0130 "İ" -> 2 chars), which corrupt offsets.
|
||||
_phrase_pattern_ci_cache: dict[str, re.Pattern[str]] = {}
|
||||
|
||||
|
||||
def _phrase_pattern_ci(phrase: str) -> re.Pattern[str]:
|
||||
cached = _phrase_pattern_ci_cache.get(phrase)
|
||||
if cached is None:
|
||||
pattern = _phrase_pattern(phrase).pattern
|
||||
# Curly apostrophes are typography, not a different phrase. Keep the
|
||||
# original phrase key (and its output spelling), but match both forms
|
||||
# without normalizing the input and corrupting offsets.
|
||||
if "'" in phrase:
|
||||
pattern = pattern.replace("'", "['’]")
|
||||
cached = re.compile(pattern, re.IGNORECASE)
|
||||
_phrase_pattern_ci_cache[phrase] = cached
|
||||
return cached
|
||||
|
||||
|
||||
def _line_starts(text: str) -> list[int]:
|
||||
"""0-indexed start offset of each line; line N (1-based) starts at index N-1."""
|
||||
starts = [0]
|
||||
for m in re.finditer("\n", text):
|
||||
starts.append(m.end())
|
||||
return starts
|
||||
|
||||
|
||||
def _line_col_context(
|
||||
text: str,
|
||||
line_starts: list[int],
|
||||
pos: int,
|
||||
context_cache: dict[int, str] | None = None,
|
||||
) -> tuple[int, int, str]:
|
||||
"""Derive (1-based line, 1-based column, stripped line context) for an offset
|
||||
into ORIGINAL text from a precomputed line_starts index, in O(log n)."""
|
||||
idx = bisect.bisect_right(line_starts, pos) - 1
|
||||
line_start = line_starts[idx]
|
||||
line_end = line_starts[idx + 1] - 1 if idx + 1 < len(line_starts) else len(text)
|
||||
line_num = idx + 1
|
||||
column = pos - line_start + 1
|
||||
context = context_cache.get(idx) if context_cache is not None else None
|
||||
if context is None:
|
||||
context = text[line_start:line_end].strip()
|
||||
context = context[:100] + "..." if len(context) > 100 else context
|
||||
if context_cache is not None:
|
||||
context_cache[idx] = context
|
||||
return line_num, column, context
|
||||
|
||||
|
||||
# Banned phrases with categories, suggestions, and severity.
|
||||
# severity: "hard" = always an AI tell; "soft" = context-dependent
|
||||
|
||||
# Compact high-signal scanner pack. Literal/domain-sensitive cases stay explicit:
|
||||
# broad vocabulary and low-value rhetorical variants are intentionally retired.
|
||||
BANNED_PHRASES: dict[str, dict[str, str | None]] = {
|
||||
# Throat-clearing and conclusion scaffolding.
|
||||
"here's the thing:": {"category": "throat_clearing", "severity": "hard", "suggestion": None},
|
||||
"in conclusion": {
|
||||
"category": "conclusion_scaffold",
|
||||
"severity": "hard",
|
||||
"suggestion": "State the conclusion directly.",
|
||||
},
|
||||
|
||||
# Significance inflation and vague attribution.
|
||||
"underscore the importance": {
|
||||
"category": "significance_inflation",
|
||||
"severity": "hard",
|
||||
"suggestion": "State the concrete effect.",
|
||||
},
|
||||
"analysts predict": {
|
||||
"category": "vague_attribution",
|
||||
"severity": "hard",
|
||||
"suggestion": "Name the analysts or cite the forecast.",
|
||||
},
|
||||
|
||||
# High-signal AI vocabulary.
|
||||
"treasure trove": {
|
||||
"category": "ai_vocabulary",
|
||||
"severity": "hard",
|
||||
"suggestion": "collection, source",
|
||||
},
|
||||
|
||||
# False agency, promotion, and assistant artifacts.
|
||||
"speak for themselves": {
|
||||
"category": "false_agency",
|
||||
"severity": "hard",
|
||||
"suggestion": "State the numbers and what they show.",
|
||||
},
|
||||
"rich cultural heritage": {
|
||||
"category": "promotional",
|
||||
"severity": "hard",
|
||||
"suggestion": None,
|
||||
},
|
||||
"as an ai language model": {
|
||||
"category": "assistant_artifact",
|
||||
"severity": "hard",
|
||||
"suggestion": "Delete the chatbot boilerplate.",
|
||||
},
|
||||
|
||||
# Literal words remain out of the pack unless a jargon collocation is clear.
|
||||
"game changer": {
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "significant, important",
|
||||
},
|
||||
"synergy": {
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "cooperation, collaboration",
|
||||
},
|
||||
"robust": {
|
||||
"category": "jargon",
|
||||
"severity": "soft",
|
||||
"suggestion": "strong, solid, thorough",
|
||||
},
|
||||
"comprehensive": {
|
||||
"category": "jargon",
|
||||
"severity": "soft",
|
||||
"suggestion": "full, complete, thorough",
|
||||
},
|
||||
|
||||
# Filler and chatbot knowledge-cutoff framing.
|
||||
"at the end of the day": {
|
||||
"category": "filler",
|
||||
"severity": "hard",
|
||||
"suggestion": None,
|
||||
},
|
||||
"in today's": {
|
||||
"category": "filler",
|
||||
"severity": "hard",
|
||||
"suggestion": None,
|
||||
},
|
||||
"as of my last": {
|
||||
"category": "knowledge_cutoff",
|
||||
"severity": "hard",
|
||||
"suggestion": "Delete the training-cutoff disclaimer.",
|
||||
},
|
||||
"why should you care": {
|
||||
"category": "rhetorical_question",
|
||||
"severity": "hard",
|
||||
"suggestion": "State why it matters directly.",
|
||||
},
|
||||
}
|
||||
|
||||
# Compile literal phrase regexes once at module load. In-process callers may
|
||||
# scan many documents; they should not pay this fixed compilation cost per
|
||||
# document (subprocess callers still pay it once per process, as before).
|
||||
for _banned_phrase in BANNED_PHRASES:
|
||||
_phrase_pattern_ci(_banned_phrase)
|
||||
|
||||
# Compact structural pack. Domain-sensitive branches are gated narrowly so
|
||||
# literal construction, mechanics, law, medicine, and code remain clean.
|
||||
STRUCTURAL_PATTERNS: list[dict[str, str]] = [
|
||||
# Emphasis and throat-clearing.
|
||||
{
|
||||
"pattern": r"(?:^|[.!?]\s+)(?:full stop|period)\.",
|
||||
"category": "emphasis_crutch",
|
||||
"severity": "hard",
|
||||
"suggestion": "Cut the one-word emphasis sentence.",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bthe real \w+ (?:is|isn't|was|wasn't|remains)\b",
|
||||
"category": "throat_clearing",
|
||||
"severity": "hard",
|
||||
"suggestion": "State it directly.",
|
||||
},
|
||||
{
|
||||
"pattern": r"(?i)\b(?:the\s+|that\s+|this\s+)?(?:struggle|stakes|pain|threat|risk|danger|fear|hype|magic|hustle|grind|stress|pressure|burnout|concern|consequences|impact|tension|anxiety|disconnect|divide|need|demand|love|chemistry|connection|mechanic|feels?)\s+(?:is|are|was|were)\s+(?:very\s+|so\s+|all\s+too\s+)?real\b",
|
||||
"category": "emphasis_crutch",
|
||||
"severity": "soft",
|
||||
"suggestion": "State what is actually at stake.",
|
||||
},
|
||||
|
||||
# Jargon collocations. Bare literal words with ordinary meanings are not
|
||||
# structural hits; the fixture keeps a protection for each domain branch.
|
||||
{
|
||||
"pattern": r"\bleverag(?:e|es|ed|ing)\s+(?:our\s+|your\s+|their\s+|its\s+|the\s+)?(?:synerg|core\s+compet|strength|expertise|capabilit|technolog|resource|data\b|ai\b|platform|ecosystem|network|audit\s+stream|power\s+of)",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "use, apply",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bnavigat(?:e|es|ed|ing)\s+(?:the\s+|this\s+|these\s+)?(?:complex|challeng|landscape|nuance|intric|water|terrain|maze|minefield|uncertaint|world\s+of|ever-)",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "handle, address, manage",
|
||||
},
|
||||
{
|
||||
# Keep the high-signal abstract use, but leave literal excavation and
|
||||
# mining language ("delve into the mountain") alone.
|
||||
"pattern": r"\bdelv(?:e|es|ed|ing)\s+into\s+(?:the\s+)?(?:topic|topics|issue|issues|implication|implications|question|questions|subject|details?|nuance|nuances|meaning|argument|claim|concept|matter|problem|analysis|research|data|strategy|history|world|conversation|evidence|complexit(?:y|ies))\b",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "explore, examine, look at",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bharness(?:es|ed|ing)?\s+(?:the\s+|its\s+|their\s+|our\s+)?(?:power|potential|strength|capabilit|momentum|force|full\s+)",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "use, tap, apply",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bharness(?:es|ed|ing)?\s+(?:(?:(?:the|our|their|its)\s+)?(?:team|group|company|organization|workforce)(?:['’]s)\s+energy\b(?=[^.!?\n]{0,100}\b(?:growth|launch|customer\s+service|transition|collaboration|innovation|expertise|engagement|success|results?)\b)|(?:(?:the|our|their)\s+)?energy\s+(?:and\s+expertise\b|of\s+(?:(?:our|the)\s+)?(?:team|group|workforce|people)\s+(?:collaboration|innovation|expertise)\b|to\s+(?:drive|fuel|accelerate|unlock|advance)\s+(?:innovation|collaboration|engagement|growth|transformation|success|results?)\b))",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "use, focus, coordinate",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bfoster(?:s|ed|ing)?\s+(?:a\s+|an\s+|greater\s+|deeper\s+|stronger\s+)?(?:culture|collaboration|innovation|sense\s+of|community|environment|growth|engagement|inclusion|creativity|dialogue|connection|belonging)",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "build, encourage, create",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bunpack(?:s|ed|ing)?\s+(?:the\s+|this\s+|that\s+|our\s+)?(?:idea|argument|assumption|implication|implications|nuance|meaning|claim|concept|topic|dynamic|why|how|what)\b",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "explain, examine",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bdoubl(?:e|es|ed|ing)\s+down\s+on\s+(?:the\s+|this\s+|that\s+|our\s+|your\s+|its\s+|their\s+|a\s+|an\s+)?(?:strategy|approach|investment|bet|commitment|vision|message|plan|position)\b",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "commit, increase",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bbolster(?:s|ed|ing)?\s+(?:the\s+|this\s+|that\s+|our\s+|your\s+)?(?:argument|case|claim|confidence|credibility|support|position|strategy|effort|security)\b",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "support, strengthen",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bstakeholders?\b[^.!?\n]{0,50}\b(?:buy-in|alignment|engagement|feedback|input|management)\b|\b(?:buy-in|alignment|engagement)\b[^.!?\n]{0,50}\bstakeholders?\b",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "people involved",
|
||||
},
|
||||
{
|
||||
"pattern": r"\b(?:in\s+)?(?:today's|modern|contemporary|business|marketing|tech|ai|media|education|healthcare|finance|industry)\s+landscape\b|\bthe\s+(?:business|marketing|tech|ai|media|education|healthcare|finance|industry)\s+landscape\s+of\b|\bthe\s+landscape\s+of\s+(?:modern\s+|today's\s+|contemporary\s+)?(?:marketing|business|tech\w*|ai|work|media|education|healthcare|finance|the industry)\b",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "situation, field, market",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bload-bearing\s+(?:part|piece|point|claim|idea|insight|assumption|detail|context|constraint|requirement|decision|argument|premise|section|paragraph|sentence|word|term|concept)\b",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "essential, important, necessary",
|
||||
},
|
||||
{
|
||||
"pattern": r"\b(?:our|the|a)\s+wedge\s+into\s+the\s+(?:\w+\s+)?(?:market|enterprise|industry|segment|category|account|vertical)s?\b|\bas\s+a\s+wedge\b",
|
||||
"category": "jargon",
|
||||
"severity": "hard",
|
||||
"suggestion": "opening, angle, advantage, entry point",
|
||||
},
|
||||
{
|
||||
"pattern": r"\b(?:the\s+)?substrate\s+(?:for|of)\s+(?:everything|all|our|the\s+(?:company|business|movement|conversation|debate|work))\b|\bcultural\s+substrate\b",
|
||||
"category": "ai_vocabulary",
|
||||
"severity": "soft",
|
||||
"suggestion": "foundation, base, layer",
|
||||
},
|
||||
|
||||
# Bare unattributed research is the vague-attribution move.
|
||||
{
|
||||
"pattern": r"(?:^|[.!?;:]\s+)research\s+(?:indicates|shows|suggests)\b",
|
||||
"category": "vague_attribution",
|
||||
"severity": "soft",
|
||||
"suggestion": "Cite the specific research or name the source.",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bboasts?\s+(?:a\s+|an\s+)?(?:world-class|state-of-the-art|cutting-edge|impressive|stunning|robust|comprehensive|unparalleled|rich|vibrant|array of|host of|range of|wealth of|plethora)",
|
||||
"category": "promotional",
|
||||
"severity": "hard",
|
||||
"suggestion": "has",
|
||||
},
|
||||
{
|
||||
"pattern": r"\b(?:data|numbers?|charts?|graphs?|metrics?|figures?|results?|dashboards?|spreadsheets?|trend\s?lines?|statistics)\s+tells?\s+a\s+(?:clear\s+)?story\b",
|
||||
"category": "false_agency",
|
||||
"severity": "hard",
|
||||
"suggestion": "State what the data shows.",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bplays?\s+an?\s+(?:crucial|key|vital|pivotal|significant|central|important|critical|defining|major)\s+(?:role|part)\b",
|
||||
"category": "significance_inflation",
|
||||
"severity": "soft",
|
||||
"suggestion": "State the specific effect.",
|
||||
},
|
||||
|
||||
# Legal/technical and domain-valid protections.
|
||||
{
|
||||
"pattern": r"\bnotwithstanding\b(?!\s+(?:anything\s+to\s+the\s+contrary|the\s+foregoing|any(?:thing)?\s+(?:other\s+)?provision|section|clause|subsection|anything\s+in))",
|
||||
"category": "ai_vocabulary",
|
||||
"severity": "soft",
|
||||
"suggestion": "Use a direct transition.",
|
||||
},
|
||||
{
|
||||
"pattern": r"\b(?:acts|serves|stands|stood)\s+as\s+(?:a|an|the)\s+(?:testament|reminder|symbol|beacon|foundation|cornerstone|gateway|catalyst|bridge|hub|springboard|window|monument|hallmark|blueprint|cautionary|stark|powerful|shining|prime example|case study|model for)\b",
|
||||
"category": "copula_avoidance",
|
||||
"severity": "hard",
|
||||
"suggestion": "Use a direct verb.",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bconstitutes\s+(?:a|an|the)\s+(?:(?:groundbreaking|transformative|trailblazing|seminal|revolutionary|landmark|pivotal|significant|major|key)\s+)?(?:transformation|breakthrough|milestone|achievement|innovation|advance|success|turning\s+point|cornerstone|testament|legacy|game[- ]changer)\b",
|
||||
"category": "copula_avoidance",
|
||||
"severity": "hard",
|
||||
"suggestion": "Use a direct verb.",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bfunctions\s+as\s+(?:a|an|the)\s+(?:(?:seamless|comprehensive|robust|transformative|groundbreaking|strategic|powerful|key|central|critical|all-in-one|single)\s+)?(?:solution|framework|platform|hub|bridge|catalyst|cornerstone|benchmark|testament|symbol|beacon|transformation|milestone|game[- ]changer)\b",
|
||||
"category": "copula_avoidance",
|
||||
"severity": "hard",
|
||||
"suggestion": "Use a direct verb.",
|
||||
},
|
||||
|
||||
# Anti-slop contrast and parallelism.
|
||||
{
|
||||
"pattern": r"(?im)(?:^|[.!?]\s+)(?:not|no)\b[^.!?]{0,28}[.!?]\s+(?:the\s+|it'?s?\s+|that'?s?\s+)?[a-z][^.!?]{0,28}[.!?]",
|
||||
"category": "anti_slop_register",
|
||||
"severity": "soft",
|
||||
"suggestion": "Join the fragments into a varied sentence.",
|
||||
},
|
||||
{
|
||||
"pattern": r"not because .+?\. because",
|
||||
"category": "binary_contrast",
|
||||
"severity": "hard",
|
||||
"suggestion": "State the reason in one sentence.",
|
||||
},
|
||||
{
|
||||
"pattern": r"feels like .+?\. it's actually",
|
||||
"category": "binary_contrast",
|
||||
"severity": "hard",
|
||||
"suggestion": "State the diagnosis directly.",
|
||||
},
|
||||
{
|
||||
"pattern": r"\bnot only .+? but also",
|
||||
"category": "negative_parallelism",
|
||||
"severity": "hard",
|
||||
"suggestion": "Use a direct sentence.",
|
||||
},
|
||||
{
|
||||
"pattern": r"\b(?:it'?s not|it\s+is\s+not|this is not|that'?s not|isn'?t|is\s+not|wasn'?t|was\s+not|aren'?t|are\s+not|weren'?t|were\s+not)\s+just\b[^.;!?\n]{1,60}[,;—–-]\s*(?:it'?s|it (?:is|was)|they'?re|that'?s)\b",
|
||||
"category": "negative_parallelism",
|
||||
"severity": "hard",
|
||||
"suggestion": "State the contrast directly.",
|
||||
},
|
||||
|
||||
# Reader-steering and rhetorical-question scaffolding.
|
||||
{
|
||||
"pattern": r"(?i)\bin this (?:article|section|post|guide|chapter|paper),?\s+(?:we|i)\s+(?:will|'ll|are going to|shall)\b",
|
||||
"category": "reader_addressing",
|
||||
"severity": "soft",
|
||||
"suggestion": "Start with the point.",
|
||||
},
|
||||
{
|
||||
"pattern": r"(?im)(?:^|[.!?]\s+)whether you'?re (?=[^.!?\n]{0,60}\b(?:a|an|just starting)\s)[^.!?\n]{1,60}\bor\b",
|
||||
"category": "reader_addressing",
|
||||
"severity": "soft",
|
||||
"suggestion": "Cut the audience-flattering opener.",
|
||||
},
|
||||
{
|
||||
"pattern": r"(?im)(?:^|[.!?]\s+)(?:why does this matter|what's the (?:real )?takeaway|why this matters|so what does (?:this|that) mean)\b[^.!?\n]{0,40}[?:]",
|
||||
"category": "rhetorical_question",
|
||||
"severity": "soft",
|
||||
"suggestion": "Answer directly instead of teeing up a self-Q&A.",
|
||||
},
|
||||
{
|
||||
"pattern": r"(?m)^\s*(?:but\s+)?what does this mean for\b",
|
||||
"category": "rhetorical_question",
|
||||
"severity": "soft",
|
||||
"suggestion": "State the consequence directly.",
|
||||
},
|
||||
{
|
||||
"pattern": r"\b(?:could|may|might|can)\s+(?:potentially|possibly)\b",
|
||||
"category": "hedge_stack",
|
||||
"severity": "soft",
|
||||
"suggestion": "Drop the redundant hedge.",
|
||||
},
|
||||
{
|
||||
"pattern": r"(?m)^(?:here are|these are|the top)\s+\d+\s+(?:reasons|things|takeaways|lessons|ways)\b",
|
||||
"category": "numbered_list_inflation",
|
||||
"severity": "soft",
|
||||
"suggestion": "List only the points that matter.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _sentence_context(text: str, start: int, end: int) -> str:
|
||||
"""Return the sentence containing a match, preserving its original width."""
|
||||
left = max(text.rfind(mark, 0, start) for mark in ".!?\n")
|
||||
right_candidates = [text.find(mark, end) for mark in ".!?\n"]
|
||||
right_candidates = [pos for pos in right_candidates if pos >= 0]
|
||||
right = min(right_candidates) if right_candidates else len(text)
|
||||
return text[left + 1 : right]
|
||||
|
||||
|
||||
def _context_has(pattern: str, text: str, start: int, end: int) -> bool:
|
||||
"""Check a small sentence/window around a match for domain evidence."""
|
||||
sentence = _sentence_context(text, start, end)
|
||||
if re.search(pattern, sentence, re.IGNORECASE):
|
||||
return True
|
||||
window = text[max(0, start - 180) : min(len(text), end + 180)]
|
||||
return bool(re.search(pattern, window, re.IGNORECASE))
|
||||
|
||||
|
||||
_LEGAL_CONTEXT = (
|
||||
r"\b(?:act|agreement|agency|clause|contract|court|defendant|filing|hearing|"
|
||||
r"judge|jury|landlord|law|lease|legal|liabilit(?:y|ies)|ordinance|part(?:y|ies)|plaintiff|"
|
||||
r"proceedings?|provision|pursuant|regulat(?:e|ed|ion|ory)|rights?|section|"
|
||||
r"statute|subsection|tenant\w*|warrant)\b"
|
||||
)
|
||||
_HISTORICAL_CONTEXT = (
|
||||
r"\b(?:archaeolog\w*|archive\w*|artifact\w*|catalog\w*|chronicle\w*|"
|
||||
r"document\w*|histor\w*|museum\w*|preserv\w*|record\w*|tradition\w*|"
|
||||
r"custom\w*|excavat\w*|ancestr\w*)\b"
|
||||
)
|
||||
_PROMOTIONAL_CONTEXT = (
|
||||
r"\b(?:boast\w*|celebrat\w*|famous|known|renowned|touris\w*|visitor\w*|"
|
||||
r"destination|vibrant|stunning|impressive|rich\s+in)\b"
|
||||
)
|
||||
_SOURCE_CONTEXT = (
|
||||
r"\b(?:according\s+to|per|citing|based\s+on|as\s+(?:reported|stated|"
|
||||
r"estimated)\s+by)\b|\b(?:survey|report|study|data|figures?)\s+"
|
||||
r"(?:from|by|of|says?|shows?|finds?|estimates?|projects?|predicts?)\b"
|
||||
)
|
||||
_MEDICAL_CONTEXT = (
|
||||
r"\b(?:anatom\w*|biolog\w*|cancer|cell\w*|clinical\w*|diagnos\w*|"
|
||||
r"disease\w*|dose\w*|drug\w*|genes?\b|genetic\w*|genomic\w*|health\w*|immune\w*|infection\w*|"
|
||||
r"inflamm\w*|kidney\w*|liver\w*|medical\w*|medicine|patient\w*|patholog\w*|physiolog\w*|"
|
||||
r"symptom\w*|therapy|tissue\w*|treatment\w*|tumou?r\w*|syndrome\w*)\b"
|
||||
)
|
||||
|
||||
|
||||
def _suppress_contextual_match(
|
||||
phrase: str,
|
||||
category: str,
|
||||
scan_text: str,
|
||||
start: int,
|
||||
end: int,
|
||||
) -> bool:
|
||||
"""Protect ordinary domain-valid uses of otherwise useful weak signals."""
|
||||
if phrase == "rich cultural heritage":
|
||||
# Historical/factual descriptions are not travel-brochure copy. Keep
|
||||
# the broad phrase for genuinely promotional "known for" language.
|
||||
sentence = _sentence_context(scan_text, start, end)
|
||||
return bool(
|
||||
re.search(_HISTORICAL_CONTEXT, sentence, re.IGNORECASE)
|
||||
and not re.search(_PROMOTIONAL_CONTEXT, sentence, re.IGNORECASE)
|
||||
)
|
||||
|
||||
if phrase == "in today's":
|
||||
# Date-specific hearing/session language is ordinary reporting, unlike
|
||||
# the generic "in today's market/landscape" filler.
|
||||
return bool(
|
||||
re.match(
|
||||
r"\s+(?:hearing|court\s+hearing|trial|session|proceedings?)\b",
|
||||
scan_text[end:],
|
||||
re.IGNORECASE,
|
||||
)
|
||||
)
|
||||
|
||||
if phrase == "analysts predict":
|
||||
# Keep unattributed forecasts flagged, but trust a nearby explicit
|
||||
# source/record cue ("according to the survey", "Reuters reports", …).
|
||||
sentence = _sentence_context(scan_text, start, end)
|
||||
if re.search(_SOURCE_CONTEXT, sentence, re.IGNORECASE):
|
||||
return True
|
||||
return False
|
||||
|
||||
if category == "negative_parallelism" and scan_text[start:end].lower().startswith("not only"):
|
||||
# Legal drafting uses this parallel construction as precise scope, not
|
||||
# as the canned contrast the rule targets.
|
||||
return bool(
|
||||
re.search(
|
||||
_LEGAL_CONTEXT,
|
||||
_sentence_context(scan_text, start, end),
|
||||
re.IGNORECASE,
|
||||
)
|
||||
)
|
||||
|
||||
if category == "significance_inflation" and re.match(
|
||||
r"plays?\s+an?\s+(?:crucial|key|vital|pivotal|significant|central|important|critical|defining|major)\s+(?:role|part)\b",
|
||||
scan_text[start:end],
|
||||
re.IGNORECASE,
|
||||
):
|
||||
# Scientific and medical prose often needs this precise causal wording.
|
||||
return _context_has(_MEDICAL_CONTEXT, scan_text, start, end)
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def scan_for_violations(text: str, include_quoted: bool = False) -> list[Violation]:
|
||||
"""Scan text for banned phrases and structural patterns."""
|
||||
violations: list[Violation] = []
|
||||
spans: list[tuple[int, int]] = []
|
||||
scan_text = mask_ignored_spans(text, include_quoted=include_quoted)
|
||||
# Cheap presence index: most documents contain only a few of the
|
||||
# retained literal phrases.
|
||||
# literal phrases. Precise matches still run against the original-width
|
||||
# text, so Unicode case expansion cannot corrupt reported offsets.
|
||||
# Normalize curly apostrophes only for the cheap presence check. Match
|
||||
# against the original-width text below so reported offsets stay exact.
|
||||
scan_text_lower = scan_text.lower().replace("’", "'")
|
||||
line_starts = _line_starts(text)
|
||||
line_context_cache: dict[int, str] = {}
|
||||
|
||||
# Check banned phrases. Matching runs case-insensitively directly on
|
||||
# scan_text (original case, masking is length-preserving) instead of on a
|
||||
# separately lowered copy, so match.start()/match.end() are valid offsets
|
||||
# into the original text with no re-derivation needed.
|
||||
for phrase, info in BANNED_PHRASES.items():
|
||||
if phrase not in scan_text_lower:
|
||||
continue
|
||||
for match in _phrase_pattern_ci(phrase).finditer(scan_text):
|
||||
if _suppress_contextual_match(
|
||||
phrase,
|
||||
info["category"],
|
||||
scan_text,
|
||||
match.start(),
|
||||
match.end(),
|
||||
):
|
||||
continue
|
||||
tail = scan_text[match.end():]
|
||||
if (
|
||||
phrase == "robust"
|
||||
and re.match(
|
||||
r"\s+(?:hash\s+verification|retry\s+mechanism|error\s+handling|test\s+suite)\b",
|
||||
tail,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
) or (
|
||||
phrase == "comprehensive"
|
||||
and re.match(
|
||||
r"\s+(?:visual\s+survey|needs\s+screen)\b",
|
||||
tail,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
):
|
||||
continue
|
||||
pos = match.start()
|
||||
line_num, column, context = _line_col_context(
|
||||
text, line_starts, pos, line_context_cache
|
||||
)
|
||||
|
||||
violations.append({
|
||||
"phrase": phrase,
|
||||
"category": info["category"],
|
||||
"severity": info.get("severity", "hard"),
|
||||
"line_number": line_num,
|
||||
"column": column,
|
||||
"context": context,
|
||||
"suggestion": info["suggestion"]
|
||||
})
|
||||
spans.append((match.start(), match.end()))
|
||||
|
||||
# Check structural patterns
|
||||
for pattern_info in STRUCTURAL_PATTERNS:
|
||||
matches = list(re.finditer(pattern_info["pattern"], scan_text, re.IGNORECASE))
|
||||
min_matches = int(pattern_info.get("min_matches", "1"))
|
||||
if len(matches) < min_matches:
|
||||
continue
|
||||
for match in matches:
|
||||
if _suppress_contextual_match(
|
||||
match.group().lower(),
|
||||
pattern_info["category"],
|
||||
scan_text,
|
||||
match.start(),
|
||||
match.end(),
|
||||
):
|
||||
continue
|
||||
pos = match.start()
|
||||
line_num, column, context = _line_col_context(
|
||||
text, line_starts, pos, line_context_cache
|
||||
)
|
||||
|
||||
violations.append({
|
||||
# Preserve the pre-fix lowercase phrase field: STRUCTURAL_PATTERNS
|
||||
# regexes are lowercase literals, previously matched against a
|
||||
# lowered copy of the text, so match.group() was always lowercase.
|
||||
"phrase": match.group().lower(),
|
||||
"category": pattern_info["category"],
|
||||
"severity": pattern_info.get("severity", "hard"),
|
||||
"line_number": line_num,
|
||||
"column": column,
|
||||
"context": context,
|
||||
"suggestion": pattern_info["suggestion"]
|
||||
})
|
||||
spans.append((match.start(), match.end()))
|
||||
|
||||
# Frequency-gated structural findings (min_matches > 1) describe the DOCUMENT, not
|
||||
# a single span. A broad, unrelated match (e.g. anti_slop_register spanning several
|
||||
# short headlines) must not silently swallow every occurrence and erase the
|
||||
# document-level tell, so these categories are exempt from containment suppression.
|
||||
freq_gated = {
|
||||
p["category"] for p in STRUCTURAL_PATTERNS if int(p.get("min_matches", "1")) > 1
|
||||
}
|
||||
|
||||
# Linear containment sweep, equivalent to the O(n^2) "any other strictly
|
||||
# larger span fully encloses mine" check above, but O(n log n): sort by
|
||||
# (start ascending, end descending) and sweep once. For any two spans with
|
||||
# other_start <= start and end <= other_end, (other_end - other_start) >
|
||||
# (end - start) holds automatically UNLESS the spans are identical -- so
|
||||
# weak enclosure by a DISTINCT span is exactly the strict-length condition.
|
||||
# A running "tallest end seen among strictly earlier starts" plus a
|
||||
# within-group max (for spans sharing the current start) reproduces this
|
||||
# without ever comparing every pair.
|
||||
n = len(violations)
|
||||
order = sorted(range(n), key=lambda i: (spans[i][0], -spans[i][1]))
|
||||
contained = [False] * n
|
||||
max_end_before_group = -1
|
||||
idx = 0
|
||||
while idx < n:
|
||||
group_start = spans[order[idx]][0]
|
||||
group_end = idx
|
||||
while group_end < n and spans[order[group_end]][0] == group_start:
|
||||
group_end += 1
|
||||
running_max_in_group = -1
|
||||
for vi in order[idx:group_end]:
|
||||
end = spans[vi][1]
|
||||
if max_end_before_group >= end or running_max_in_group > end:
|
||||
contained[vi] = True
|
||||
if end > running_max_in_group:
|
||||
running_max_in_group = end
|
||||
if running_max_in_group > max_end_before_group:
|
||||
max_end_before_group = running_max_in_group
|
||||
idx = group_end
|
||||
|
||||
violations = [
|
||||
v for i, v in enumerate(violations)
|
||||
if not contained[i] or v["category"] in freq_gated
|
||||
]
|
||||
|
||||
# Sort by line number, then column
|
||||
violations.sort(key=lambda v: (v["line_number"], v["column"]))
|
||||
|
||||
return violations
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("input_file", nargs="?", help="Optional input file. Reads stdin when omitted.")
|
||||
parser.add_argument(
|
||||
"--include-quoted",
|
||||
action="store_true",
|
||||
help="Scan quoted examples and markdown blockquotes instead of skipping them.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
|
||||
# Read input
|
||||
if args.input_file:
|
||||
try:
|
||||
with open(args.input_file, 'r', errors="replace") as f:
|
||||
text = f.read()
|
||||
except OSError as e:
|
||||
print(json.dumps({"error": f"Could not read input: {e}", "violations": []}))
|
||||
sys.exit(2)
|
||||
else:
|
||||
text = sys.stdin.buffer.read().decode("utf-8", errors="replace")
|
||||
|
||||
if not text.strip():
|
||||
print(json.dumps({"error": "No input provided", "violations": []}))
|
||||
sys.exit(1)
|
||||
|
||||
violations = scan_for_violations(text, include_quoted=args.include_quoted)
|
||||
|
||||
# English-only graceful decline. Function-word absence alone is not evidence
|
||||
# of a foreign language: imperative stacks and buzzword lists are English
|
||||
# slop with few function words. Decline only when the text both fails the
|
||||
# function-word heuristic AND produced zero English-pattern hits.
|
||||
if not violations and not is_probably_english(text):
|
||||
print(json.dumps({"non_english": True, "total_violations": 0, "violations": []}, indent=2))
|
||||
print("note: input appears non-English; scanner declined (English-only).", file=sys.stderr)
|
||||
sys.exit(0)
|
||||
|
||||
# Group by category for summary
|
||||
categories: dict[str, int] = {}
|
||||
by_severity: dict[str, int] = {"hard": 0, "soft": 0}
|
||||
for v in violations:
|
||||
categories[v["category"]] = categories.get(v["category"], 0) + 1
|
||||
by_severity[v["severity"]] = by_severity.get(v["severity"], 0) + 1
|
||||
|
||||
output = {
|
||||
"total_violations": len(violations),
|
||||
"by_severity": by_severity,
|
||||
"by_category": categories,
|
||||
"violations": violations
|
||||
}
|
||||
|
||||
print(json.dumps(output, indent=2))
|
||||
|
||||
# Exit with 1 if violations found
|
||||
sys.exit(1 if violations else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,580 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Deterministic dimension-controlled pair generation for the teach calibration game.
|
||||
|
||||
Given a base passage and ONE voice dimension, produce a minimal pair (A, B) where
|
||||
B differs from A along exactly that dimension. The direction of the transform
|
||||
(e.g. contract -> expand vs expand -> contract) is auto-detected from whichever
|
||||
pole the base passage already sits at, so every dimension is reversible: feed it
|
||||
a passage already on one pole and it moves to the other.
|
||||
|
||||
Every transform must preserve the must-preserve constraint tokens extracted by
|
||||
scripts/extract_constraints.py (numbers, dates, names, quotes, units, etc). If a
|
||||
dimension has no expressible transform in the given passage (neither pole's
|
||||
pattern is present, or the only candidate transform would drop a constraint),
|
||||
the command exits 3 with "dimension not expressible in this passage".
|
||||
|
||||
Usage:
|
||||
python3 calibrate_pairs.py generate --base FILE --dimension DIM --seed N
|
||||
python3 calibrate_pairs.py --list-dimensions
|
||||
|
||||
Dimensions: contractions, em_dash, sentence_length, connectives, staccato.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
from extract_constraints import extract_constraints # noqa: E402
|
||||
from banned_phrase_scan import scan_for_violations # noqa: E402
|
||||
|
||||
DIMENSIONS = ["contractions", "em_dash", "sentence_length", "connectives", "staccato"]
|
||||
|
||||
# Each dimension has two named poles. transform_applied is "<dimension>:<pole of B>".
|
||||
POLES: dict[str, tuple[str, str]] = {
|
||||
"contractions": ("contracted", "expanded"),
|
||||
"em_dash": ("dashed", "plain"),
|
||||
"sentence_length": ("long", "short"),
|
||||
"connectives": ("plain", "formal"),
|
||||
"staccato": ("staccato", "flowing"),
|
||||
}
|
||||
|
||||
|
||||
class NotExpressible(Exception):
|
||||
"""Raised when a dimension has no applicable transform in the passage."""
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# contractions: expand <-> contract via a fixed, unambiguous mapping table.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_CONTRACTION_PAIRS = [
|
||||
("do not", "don't"), ("does not", "doesn't"), ("did not", "didn't"),
|
||||
("cannot", "can't"), ("will not", "won't"), ("would not", "wouldn't"),
|
||||
("should not", "shouldn't"), ("could not", "couldn't"), ("must not", "mustn't"),
|
||||
("is not", "isn't"), ("are not", "aren't"), ("was not", "wasn't"), ("were not", "weren't"),
|
||||
("have not", "haven't"), ("has not", "hasn't"), ("had not", "hadn't"),
|
||||
("I am", "I'm"), ("you are", "you're"), ("we are", "we're"), ("they are", "they're"),
|
||||
("I will", "I'll"), ("you will", "you'll"), ("we will", "we'll"), ("they will", "they'll"),
|
||||
("he will", "he'll"), ("she will", "she'll"), ("it will", "it'll"),
|
||||
("I have", "I've"), ("you have", "you've"), ("we have", "we've"), ("they have", "they've"),
|
||||
("let us", "let's"),
|
||||
("it is", "it's"), ("that is", "that's"), ("there is", "there's"),
|
||||
("here is", "here's"), ("who is", "who's"), ("what is", "what's"),
|
||||
]
|
||||
# Pairs whose expanded side always keeps a hard-capitalized pronoun ("I ...").
|
||||
_HARD_CAPITAL = {e for e, c in _CONTRACTION_PAIRS if e.startswith("I ")}
|
||||
|
||||
|
||||
def _case_like(template: str, source_first_char: str) -> str:
|
||||
if source_first_char.isupper():
|
||||
return template[0].upper() + template[1:]
|
||||
return template[0].lower() + template[1:]
|
||||
|
||||
|
||||
# Words safe to lowercase when a sentence is folded mid-clause. A capitalized
|
||||
# word NOT in this set is assumed to be a proper noun (constraint) and is left
|
||||
# alone -- lowercasing "Priya" into a joined clause would corrupt a name.
|
||||
_LOWERABLE_JOINERS = {
|
||||
"the", "it", "she", "he", "they", "this", "that", "there", "i", "we", "you",
|
||||
"who", "what", "when", "where", "why", "how", "a", "an", "and", "but", "so",
|
||||
"yet", "or", "nobody", "everybody", "everyone", "someone", "something",
|
||||
"nothing", "no", "her", "his", "its", "their", "our", "your",
|
||||
}
|
||||
|
||||
|
||||
def _lower_first_word(s: str) -> str:
|
||||
"""Lowercase the leading word only if it is a common function word/pronoun,
|
||||
never a token that looks like a proper noun a constraint might depend on."""
|
||||
m = re.match(r"[A-Za-z']+", s)
|
||||
if not m:
|
||||
return s
|
||||
word = m.group(0)
|
||||
if word.lower() in _LOWERABLE_JOINERS:
|
||||
return word.lower() + s[len(word):]
|
||||
return s
|
||||
|
||||
|
||||
_WORD_BEFORE_RE = re.compile(r"([A-Za-z']+)\s*$")
|
||||
_WORD_AFTER_RE = re.compile(r"^\s*([A-Za-z']+)")
|
||||
|
||||
|
||||
def _is_title_case(word: str) -> bool:
|
||||
return bool(word) and word[0].isupper() and word[1:].islower()
|
||||
|
||||
|
||||
def _in_capitalized_span(m: "re.Match[str]") -> bool:
|
||||
"""True if this contraction/expansion match sits inside a capitalized
|
||||
multi-word span (e.g. "Venue Can't Stop") that looks like a proper noun,
|
||||
rather than an ordinary sentence-level contraction.
|
||||
|
||||
A contraction rewrite is only skipped when the match ITSELF is
|
||||
capitalized (so an ordinary sentence-initial "Can't you..." is untouched)
|
||||
AND an immediately adjacent word is also Title Case, forming a >= 2-word
|
||||
capitalized run. "Maria Chen can't attend" is not skipped (the match is
|
||||
lowercase); "the Venue Can't Stop" is skipped (both "Venue" and "Can't"
|
||||
are capitalized and adjacent).
|
||||
"""
|
||||
matched = m.group(0)
|
||||
if not matched[0].isupper():
|
||||
return False
|
||||
text = m.string
|
||||
before_m = _WORD_BEFORE_RE.search(text[: m.start()])
|
||||
after_m = _WORD_AFTER_RE.match(text[m.end() :])
|
||||
before_word = before_m.group(1) if before_m else ""
|
||||
after_word = after_m.group(1) if after_m else ""
|
||||
return _is_title_case(before_word) or _is_title_case(after_word)
|
||||
|
||||
|
||||
def _contraction_repl(replacement: str, hard: bool):
|
||||
"""One replacement closure shared by both contraction directions (expand
|
||||
and contract) and both casing modes (hard-capital literal vs case-matched).
|
||||
"""
|
||||
def repl(m):
|
||||
if _in_capitalized_span(m):
|
||||
return m.group(0)
|
||||
return replacement if hard else _case_like(replacement, m.group(0)[0])
|
||||
return repl
|
||||
|
||||
|
||||
def _apply_contractions(text: str) -> tuple[str, str]:
|
||||
contract_hits = []
|
||||
expand_hits = []
|
||||
for expanded, contracted in _CONTRACTION_PAIRS:
|
||||
if expanded in _HARD_CAPITAL:
|
||||
# "I am"/"I'll"/... always literally capitalized; case-sensitive match.
|
||||
if re.search(re.escape(contracted), text):
|
||||
contract_hits.append((expanded, contracted))
|
||||
if re.search(re.escape(expanded), text):
|
||||
expand_hits.append((expanded, contracted))
|
||||
else:
|
||||
if re.search(r"\b" + re.escape(contracted) + r"\b", text, re.IGNORECASE):
|
||||
contract_hits.append((expanded, contracted))
|
||||
if re.search(r"\b" + re.escape(expanded) + r"\b", text, re.IGNORECASE):
|
||||
expand_hits.append((expanded, contracted))
|
||||
|
||||
if contract_hits:
|
||||
out = text
|
||||
for expanded, contracted in contract_hits:
|
||||
hard = expanded in _HARD_CAPITAL
|
||||
pattern = re.escape(contracted) if hard else r"\b" + re.escape(contracted) + r"\b"
|
||||
flags = 0 if hard else re.IGNORECASE
|
||||
out = re.sub(pattern, _contraction_repl(expanded, hard), out, flags=flags)
|
||||
return out, "expanded"
|
||||
|
||||
if expand_hits:
|
||||
out = text
|
||||
for expanded, contracted in expand_hits:
|
||||
hard = expanded in _HARD_CAPITAL
|
||||
pattern = re.escape(expanded) if hard else r"\b" + re.escape(expanded) + r"\b"
|
||||
flags = 0 if hard else re.IGNORECASE
|
||||
out = re.sub(pattern, _contraction_repl(contracted, hard), out, flags=flags)
|
||||
return out, "contracted"
|
||||
|
||||
raise NotExpressible("no contraction or expandable phrase found")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# em_dash: paired em-dash parentheticals <-> paired-comma parentheticals ONLY.
|
||||
#
|
||||
# Only the paired-dash<->paired-comma path is expressible for this dimension.
|
||||
# A lone joiner dash (" — nobody objected") has no comma-pair equivalent that
|
||||
# preserves sentence count: turning it into a period (the old behavior) split
|
||||
# one sentence into two, silently changing sentence count between A and B.
|
||||
# Passages with only a lone dash (no paired construction) are declined rather
|
||||
# than forced through a transform that shifts sentence count.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_PAIRED_DASH_RE = re.compile(r"\s—\s(.+?)\s—\s")
|
||||
_PAIRED_COMMA_RE = re.compile(r",\s+(.+?),\s+")
|
||||
|
||||
|
||||
def _apply_em_dash(text: str) -> tuple[str, str]:
|
||||
if _PAIRED_DASH_RE.search(text):
|
||||
out = _PAIRED_DASH_RE.sub(lambda m: ", " + m.group(1) + ", ", text)
|
||||
if "—" in out:
|
||||
# A lone dash survived alongside the paired one -- the restricted
|
||||
# paired-dash<->comma path can't fully express this passage
|
||||
# without also touching the lone dash (and changing sentence
|
||||
# count), so decline rather than ship a half-converted pair.
|
||||
raise NotExpressible(
|
||||
"passage mixes a paired em dash with a lone em dash; "
|
||||
"the paired-dash<->comma path can't resolve the lone dash "
|
||||
"without changing sentence count"
|
||||
)
|
||||
return out, "plain"
|
||||
|
||||
if "—" in text:
|
||||
raise NotExpressible(
|
||||
"only a lone em dash found; the em_dash dimension is restricted "
|
||||
"to the paired-dash<->comma path so sentence count stays stable"
|
||||
)
|
||||
|
||||
if _PAIRED_COMMA_RE.search(text):
|
||||
out = _PAIRED_COMMA_RE.sub(lambda m: " — " + m.group(1) + " — ", text)
|
||||
return out, "dashed"
|
||||
|
||||
raise NotExpressible("no paired em dash or comma-bounded parenthetical found")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# sentence_length: split at coordinators <-> join short adjacent sentences.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_COORD_RE = re.compile(r",\s+(and|but|or|so|yet)\s+", re.IGNORECASE)
|
||||
|
||||
|
||||
def _apply_sentence_length(text: str) -> tuple[str, str]:
|
||||
sentences = re.findall(r"[^.!?]+[.!?]+", text)
|
||||
|
||||
for sent in sentences:
|
||||
for m in _COORD_RE.finditer(sent):
|
||||
before = sent[:m.start()]
|
||||
after = sent[m.end():]
|
||||
if len(before.split()) >= 3 and len(after.split()) >= 3:
|
||||
new_sent = before.rstrip() + ". " + m.group(1).capitalize() + " " + after
|
||||
out = text.replace(sent, new_sent, 1)
|
||||
return re.sub(r"\s+", " ", out).strip(), "short"
|
||||
|
||||
for i in range(len(sentences) - 1):
|
||||
a_words = sentences[i].strip().split()
|
||||
b_words = sentences[i + 1].strip().split()
|
||||
if len(a_words) <= 8 and len(b_words) <= 8:
|
||||
first_no_period = re.sub(r"[.!?]+\s*$", "", sentences[i].strip())
|
||||
second = sentences[i + 1].strip()
|
||||
second_body = re.sub(r"[.!?]+\s*$", "", second)
|
||||
end_punct = second[len(second_body):].strip() or "."
|
||||
joined = first_no_period + ", and " + _lower_first_word(second_body) + end_punct
|
||||
prefix = "".join(sentences[:i])
|
||||
suffix = "".join(sentences[i + 2:])
|
||||
out = (prefix + " " + joined + " " + suffix).strip()
|
||||
return re.sub(r"\s+", " ", out), "long"
|
||||
|
||||
raise NotExpressible("no coordinator split site or joinable short sentences found")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# connectives: formal <-> plain via a fixed table.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_FORMAL_TO_PLAIN = {
|
||||
"however": "But", "additionally": "Also", "therefore": "So",
|
||||
"furthermore": "Also", "moreover": "Also", "nevertheless": "Still",
|
||||
"consequently": "So", "nonetheless": "Still", "subsequently": "Then",
|
||||
"thus": "So",
|
||||
}
|
||||
_PLAIN_TO_FORMAL = {
|
||||
"but": "However", "also": "Additionally", "so": "Therefore",
|
||||
"still": "Nevertheless", "then": "Subsequently",
|
||||
}
|
||||
_FORMAL_RE = re.compile(
|
||||
r"(?:^|(?<=[.!?]\s))(" + "|".join(_FORMAL_TO_PLAIN) + r"),?\s+", re.IGNORECASE
|
||||
)
|
||||
_PLAIN_RE = re.compile(
|
||||
r"(?:^|(?<=[.!?]\s))(" + "|".join(_PLAIN_TO_FORMAL) + r"),?\s+", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def _apply_connectives(text: str) -> tuple[str, str]:
|
||||
if _FORMAL_RE.search(text):
|
||||
def repl(m):
|
||||
# No comma after the plain form: "But " reads as ordinary speech;
|
||||
# "But, " is stilted, and "So, " specifically trips the scanner's
|
||||
# filler_opener structural pattern. Dropping the comma across the
|
||||
# board keeps every plain form clean of that tell.
|
||||
return _FORMAL_TO_PLAIN[m.group(1).lower()] + " "
|
||||
out = _FORMAL_RE.sub(repl, text)
|
||||
return out, "plain"
|
||||
|
||||
if _PLAIN_RE.search(text):
|
||||
def repl2(m):
|
||||
return _PLAIN_TO_FORMAL[m.group(1).lower()] + ", "
|
||||
out = _PLAIN_RE.sub(repl2, text)
|
||||
return out, "formal"
|
||||
|
||||
raise NotExpressible("no formal or plain connective found")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# staccato: fragment runs <-> flowing clauses.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_LEADING_JOINER_RE = re.compile(
|
||||
r"^(?:because|although|while|since|if|when|and|but|so|yet|or)\s+", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def _apply_staccato(text: str, rng: random.Random) -> tuple[str, str]:
|
||||
sentences = re.findall(r"[^.!?]+[.!?]+", text)
|
||||
|
||||
# Reverse direction: a run of >= 3 consecutive short (<=6 word) sentences -> flow.
|
||||
runs = []
|
||||
run_start = None
|
||||
for i, sent in enumerate(sentences):
|
||||
words = sent.strip().split()
|
||||
if len(words) <= 6:
|
||||
if run_start is None:
|
||||
run_start = i
|
||||
else:
|
||||
if run_start is not None and i - run_start >= 3:
|
||||
runs.append((run_start, i))
|
||||
run_start = None
|
||||
if run_start is not None and len(sentences) - run_start >= 3:
|
||||
runs.append((run_start, len(sentences)))
|
||||
|
||||
if runs:
|
||||
start, end = runs[rng.randrange(len(runs))] if len(runs) > 1 else runs[0]
|
||||
parts = []
|
||||
for idx in range(start, end):
|
||||
body = re.sub(r"[.!?]+\s*$", "", sentences[idx].strip())
|
||||
parts.append(body)
|
||||
joined_parts = []
|
||||
for i, p in enumerate(parts):
|
||||
if i == 0:
|
||||
joined_parts.append(p)
|
||||
elif i == len(parts) - 1:
|
||||
joined_parts.append("and " + _lower_first_word(p))
|
||||
else:
|
||||
joined_parts.append(_lower_first_word(p))
|
||||
joined = ", ".join(joined_parts) + "."
|
||||
prefix = "".join(sentences[:start])
|
||||
suffix = "".join(sentences[end:])
|
||||
out = (prefix + " " + joined + " " + suffix).strip()
|
||||
return re.sub(r"\s+", " ", out), "flowing"
|
||||
|
||||
# Forward direction: the sentence with the most commas (>= 2) fragments.
|
||||
best_idx, best_commas = None, 1
|
||||
for i, sent in enumerate(sentences):
|
||||
commas = sent.count(",")
|
||||
if commas > best_commas:
|
||||
best_idx, best_commas = i, commas
|
||||
|
||||
if best_idx is not None:
|
||||
sent = sentences[best_idx]
|
||||
body = re.sub(r"[.!?]+\s*$", "", sent.strip())
|
||||
end_punct = sent.strip()[len(body):].strip() or "."
|
||||
fragments = [f.strip() for f in body.split(",")]
|
||||
cleaned = []
|
||||
for frag in fragments:
|
||||
frag = _LEADING_JOINER_RE.sub("", frag).strip()
|
||||
if not frag:
|
||||
continue
|
||||
frag = frag[0].upper() + frag[1:]
|
||||
cleaned.append(frag)
|
||||
if len(cleaned) < 3:
|
||||
raise NotExpressible("comma split did not yield a fragment run")
|
||||
cleaned[-1] = cleaned[-1] + end_punct if not cleaned[-1].endswith((".", "!", "?")) else cleaned[-1]
|
||||
new_sent = " ".join(
|
||||
f if f.endswith((".", "!", "?")) else f + "." for f in cleaned
|
||||
)
|
||||
out = text.replace(sent, new_sent, 1)
|
||||
return re.sub(r"\s+", " ", out).strip(), "staccato"
|
||||
|
||||
raise NotExpressible("no fragment run or multi-comma sentence found")
|
||||
|
||||
|
||||
_APPLY = {
|
||||
"contractions": lambda text, rng: _apply_contractions(text),
|
||||
"em_dash": lambda text, rng: _apply_em_dash(text),
|
||||
"sentence_length": lambda text, rng: _apply_sentence_length(text),
|
||||
"connectives": lambda text, rng: _apply_connectives(text),
|
||||
"staccato": _apply_staccato,
|
||||
}
|
||||
|
||||
_EXAMPLES = {
|
||||
"contractions": {
|
||||
"description": "Expand <-> contract via a fixed, unambiguous mapping table "
|
||||
"(do not <-> don't, I am <-> I'm, it is <-> it's, ...).",
|
||||
"a": "The rollout is not finished, and I am not confident it will ship Friday.",
|
||||
"b": "The rollout isn't finished, and I'm not confident it'll ship Friday.",
|
||||
},
|
||||
"em_dash": {
|
||||
"description": "Paired em-dash parentheticals <-> paired-comma parentheticals only "
|
||||
"(a lone joiner dash has no comma-pair equivalent that preserves "
|
||||
"sentence count, so it is declined rather than converted).",
|
||||
"a": "The plan — untested and rushed — still shipped on time.",
|
||||
"b": "The plan, untested and rushed, still shipped on time.",
|
||||
},
|
||||
"sentence_length": {
|
||||
"description": "Split at a coordinator <-> join two short adjacent sentences.",
|
||||
"a": "The team shipped the fix, and the client renewed the contract.",
|
||||
"b": "The team shipped the fix. And the client renewed the contract.",
|
||||
},
|
||||
"connectives": {
|
||||
"description": "Formal <-> plain connective swap via a fixed table "
|
||||
"(However -> But, Additionally -> Also, Therefore -> So, ...). "
|
||||
"Plain forms drop the comma after the connective (\"But \", not "
|
||||
"\"But, \") so they read as ordinary speech, not a filler-opener tell.",
|
||||
"a": "However, the numbers slipped in March.",
|
||||
"b": "But the numbers slipped in March.",
|
||||
},
|
||||
"staccato": {
|
||||
"description": "Fragment runs <-> flowing clauses.",
|
||||
"a": "Because the deploy failed, and the on-call missed the page, the team lost an hour.",
|
||||
"b": "The deploy failed. The on-call missed the page. The team lost an hour.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _is_word_char(ch: str) -> bool:
|
||||
return ch.isalnum()
|
||||
|
||||
|
||||
def _has_whole_occurrence(value: str, text: str) -> bool:
|
||||
"""True if `value` occurs in `text` as a standalone span rather than as a
|
||||
substring straddling the edge of a longer, different word.
|
||||
|
||||
Plain substring containment is not enough: expanding "Can't" to "Cannot"
|
||||
still contains the literal characters "Can", so a naive `"Can" in
|
||||
"Cannot"` check reports a corrupted proper noun as "preserved". At each
|
||||
edge of a candidate match, this only counts as a clash (disqualifying the
|
||||
match) when BOTH the value's own boundary character and the adjacent text
|
||||
character are alphanumeric -- i.e. when `value` could be glued onto a
|
||||
longer token. Values that start/end with punctuation (currency signs,
|
||||
quotes) never clash on that edge, since punctuation can't be silently
|
||||
swallowed into a bigger word the way a letter/digit can.
|
||||
"""
|
||||
if not value:
|
||||
return False
|
||||
start = 0
|
||||
while True:
|
||||
idx = text.find(value, start)
|
||||
if idx == -1:
|
||||
return False
|
||||
left_clash = idx > 0 and _is_word_char(text[idx - 1]) and _is_word_char(value[0])
|
||||
end = idx + len(value)
|
||||
right_clash = end < len(text) and _is_word_char(text[end]) and _is_word_char(value[-1])
|
||||
if not left_clash and not right_clash:
|
||||
return True
|
||||
start = idx + 1
|
||||
|
||||
|
||||
def _verify_constraints_preserved(base_text: str, transformed_text: str) -> list[str]:
|
||||
"""Return constraint values from base_text missing from transformed_text.
|
||||
|
||||
Uses whole-occurrence (token-boundary) comparison rather than plain
|
||||
substring containment -- see `_has_whole_occurrence` for why a substring
|
||||
check is unsafe here.
|
||||
"""
|
||||
missing = []
|
||||
for c in extract_constraints(base_text):
|
||||
value = c["value"]
|
||||
if _has_whole_occurrence(value, transformed_text):
|
||||
continue
|
||||
# Allow whitespace-normalized matches (transforms may collapse spacing).
|
||||
normalized_value = re.sub(r"\s+", " ", value)
|
||||
normalized_text = re.sub(r"\s+", " ", transformed_text)
|
||||
if normalized_value != value and _has_whole_occurrence(normalized_value, normalized_text):
|
||||
continue
|
||||
missing.append(value)
|
||||
return missing
|
||||
|
||||
|
||||
def _scan_flags(text: str) -> list[str]:
|
||||
"""Category names banned_phrase_scan raises on `text`, deduped and sorted.
|
||||
|
||||
Empty when the text scans clean. Generated B-variants can legitimately
|
||||
trip the scanner (e.g. a staccato pole reads as anti_slop_register; see
|
||||
references/calibrate.md "Voice overrides defaults") -- that is surfaced
|
||||
here, not treated as a reason to decline the pair.
|
||||
"""
|
||||
categories = {v["category"] for v in scan_for_violations(text)}
|
||||
return sorted(categories)
|
||||
|
||||
|
||||
def generate_pair(base_text: str, dimension: str, seed: int) -> dict:
|
||||
if dimension not in DIMENSIONS:
|
||||
raise ValueError(f"unknown dimension: {dimension}")
|
||||
|
||||
rng = random.Random(seed)
|
||||
transformed, pole = _APPLY[dimension](base_text, rng)
|
||||
|
||||
if transformed.strip() == base_text.strip():
|
||||
raise NotExpressible("transform produced no change")
|
||||
|
||||
missing = _verify_constraints_preserved(base_text, transformed)
|
||||
if missing:
|
||||
raise NotExpressible(
|
||||
"transform would drop must-preserve constraint(s): " + ", ".join(missing)
|
||||
)
|
||||
|
||||
pair_id = hashlib.sha256(
|
||||
f"{base_text}|{dimension}|{seed}".encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
return {
|
||||
"pair_id": pair_id,
|
||||
"dimension": dimension,
|
||||
"a_text": base_text,
|
||||
"b_text": transformed,
|
||||
"transform_applied": f"{dimension}:{pole}",
|
||||
"a_flags": _scan_flags(base_text),
|
||||
"b_flags": _scan_flags(transformed),
|
||||
}
|
||||
|
||||
|
||||
def list_dimensions() -> dict:
|
||||
return {
|
||||
dim: {
|
||||
"poles": list(POLES[dim]),
|
||||
"description": _EXAMPLES[dim]["description"],
|
||||
"example": {"a": _EXAMPLES[dim]["a"], "b": _EXAMPLES[dim]["b"]},
|
||||
}
|
||||
for dim in DIMENSIONS
|
||||
}
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command")
|
||||
|
||||
gen = sub.add_parser("generate", help="generate one dimension-controlled pair")
|
||||
gen.add_argument("--base", required=True, help="path to base passage file")
|
||||
gen.add_argument("--dimension", required=True, choices=DIMENSIONS)
|
||||
gen.add_argument("--seed", type=int, default=0)
|
||||
|
||||
parser.add_argument("--list-dimensions", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
|
||||
if args.list_dimensions:
|
||||
print(json.dumps(list_dimensions(), indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
if args.command != "generate":
|
||||
print(json.dumps({"error": "no command given; use generate or --list-dimensions"}))
|
||||
return 1
|
||||
|
||||
try:
|
||||
base_text = Path(args.base).read_text()
|
||||
except OSError as e:
|
||||
print(json.dumps({"error": f"could not read base file: {e}"}))
|
||||
return 2
|
||||
|
||||
try:
|
||||
pair = generate_pair(base_text, args.dimension, args.seed)
|
||||
except NotExpressible as e:
|
||||
print(json.dumps({
|
||||
"error": "dimension not expressible in this passage",
|
||||
"dimension": args.dimension,
|
||||
"detail": str(e),
|
||||
}))
|
||||
return 3
|
||||
|
||||
print(json.dumps(pair, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Aggregate a teach-calibration preferences JSONL into per-dimension confidence,
|
||||
surface conflicts against a measured stylometric profile, and pick the next
|
||||
dimension to play.
|
||||
|
||||
Preferences JSONL rows (one per game round):
|
||||
{"pair_id": "...", "dimension": "contractions", "choice": "a"|"b"|"neither",
|
||||
"ts": "...", "a_label": "contracted", "b_label": "expanded"}
|
||||
|
||||
`a_label`/`b_label` are optional. When present (calibrate_pairs.py's
|
||||
transform_applied names the B pole; A sits at the other pole in the same
|
||||
dimension's POLES pair) they let this script report a semantic preferred
|
||||
DIRECTION ("expanded", "short", "formal", ...) instead of the bare literal
|
||||
"a"/"b" tally, which is what --profile conflict detection needs. Rows without
|
||||
labels still count toward n and the confidence interval; they just can't
|
||||
resolve to a named pole (direction is reported as "a" or "b").
|
||||
|
||||
Usage:
|
||||
python3 calibrate_score.py --preferences prefs.jsonl
|
||||
python3 calibrate_score.py --preferences prefs.jsonl --profile profile.json
|
||||
python3 calibrate_score.py --preferences prefs.jsonl --next
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
from calibrate_pairs import DIMENSIONS, POLES # noqa: E402
|
||||
|
||||
MIN_K = 5
|
||||
Z = 1.96
|
||||
|
||||
# Dimension -> (profile_key, pole_meaning) used only for --profile conflict
|
||||
# detection. `low_pole`/`high_pole` say which named pole a LOW vs HIGH measured
|
||||
# value corresponds to, so a confident stated preference for the opposite pole
|
||||
# of what the measured value implies is flagged.
|
||||
_PROFILE_LINKS: dict[str, dict] = {
|
||||
"contractions": {
|
||||
"profile_key": "contraction_rate",
|
||||
"low_pole": "expanded",
|
||||
"high_pole": "contracted",
|
||||
"low_threshold": 0.05,
|
||||
"high_threshold": 0.20,
|
||||
},
|
||||
"sentence_length": {
|
||||
"profile_key": "avg_sentence_length",
|
||||
"low_pole": "short",
|
||||
"high_pole": "long",
|
||||
"low_threshold": 12.0,
|
||||
"high_threshold": 20.0,
|
||||
},
|
||||
"staccato": {
|
||||
"profile_key": "avg_sentence_length",
|
||||
"low_pole": "staccato",
|
||||
"high_pole": "flowing",
|
||||
"low_threshold": 12.0,
|
||||
"high_threshold": 20.0,
|
||||
},
|
||||
"em_dash": {
|
||||
"profile_key": "em_dash_rate",
|
||||
"low_pole": "plain",
|
||||
"high_pole": "dashed",
|
||||
"low_threshold": 0.02,
|
||||
"high_threshold": 0.15,
|
||||
},
|
||||
"connectives": {
|
||||
"profile_key": "formal_connective_rate",
|
||||
"low_pole": "plain",
|
||||
"high_pole": "formal",
|
||||
"low_threshold": 0.10,
|
||||
"high_threshold": 0.40,
|
||||
},
|
||||
}
|
||||
|
||||
CONFIDENCE_STOP = 0.7
|
||||
K_STOP = 9
|
||||
|
||||
|
||||
def wilson_lower_bound(successes: int, n: int, z: float = Z) -> float:
|
||||
if n == 0:
|
||||
return 0.0
|
||||
phat = successes / n
|
||||
denom = 1 + z * z / n
|
||||
center = phat + z * z / (2 * n)
|
||||
margin = z * math.sqrt((phat * (1 - phat) + z * z / (4 * n)) / n)
|
||||
return round(max(0.0, (center - margin) / denom), 3)
|
||||
|
||||
|
||||
def load_preferences(path: Path) -> list[dict]:
|
||||
rows = []
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
|
||||
def dedup_by_pair_id(rows: list[dict]) -> list[dict]:
|
||||
"""Keep one row per `pair_id`: the LATEST by `ts` (ISO 8601, so a plain
|
||||
string comparison orders correctly). A game round replayed twice (e.g. the
|
||||
agent crashed mid-write and re-ran the round) must count once toward `n`
|
||||
and resolve to whatever the user most recently chose, not double-count or
|
||||
let an earlier write win. Rows without a `pair_id` can't be deduped
|
||||
against anything, so each is kept as-is.
|
||||
"""
|
||||
by_pair_id: dict[str, dict] = {}
|
||||
passthrough: list[dict] = []
|
||||
for row in rows:
|
||||
pid = row.get("pair_id")
|
||||
if not pid:
|
||||
passthrough.append(row)
|
||||
continue
|
||||
existing = by_pair_id.get(pid)
|
||||
if existing is None or row.get("ts", "") >= existing.get("ts", ""):
|
||||
by_pair_id[pid] = row
|
||||
return list(by_pair_id.values()) + passthrough
|
||||
|
||||
|
||||
def aggregate(rows: list[dict]) -> dict[str, dict]:
|
||||
rows = dedup_by_pair_id(rows)
|
||||
result = {dim: {"n": 0, "tally": {}, "neither": 0} for dim in DIMENSIONS}
|
||||
|
||||
for row in rows:
|
||||
dim = row.get("dimension")
|
||||
if dim not in result:
|
||||
continue
|
||||
result[dim]["n"] += 1
|
||||
choice = row.get("choice")
|
||||
if choice == "neither":
|
||||
result[dim]["neither"] += 1
|
||||
continue
|
||||
if choice not in ("a", "b"):
|
||||
continue
|
||||
label = row.get(f"{choice}_label") or choice
|
||||
result[dim]["tally"][label] = result[dim]["tally"].get(label, 0) + 1
|
||||
|
||||
dimensions = {}
|
||||
for dim, data in result.items():
|
||||
n = data["n"]
|
||||
decisive = sum(data["tally"].values())
|
||||
if n < MIN_K or decisive == 0:
|
||||
dimensions[dim] = {
|
||||
"n": n,
|
||||
"status": "insufficient",
|
||||
"preferred": None,
|
||||
"confidence": 0.0,
|
||||
}
|
||||
continue
|
||||
max_count = max(data["tally"].values())
|
||||
top_labels = [label for label, count in data["tally"].items() if count == max_count]
|
||||
if len(top_labels) > 1:
|
||||
# A genuine tie between the top two (or more) tallies: there is no
|
||||
# lean to report, so `preferred` must be null rather than
|
||||
# silently picking whichever label happens to sort last.
|
||||
dimensions[dim] = {
|
||||
"n": n,
|
||||
"status": "tied",
|
||||
"preferred": None,
|
||||
"confidence": 0.0,
|
||||
}
|
||||
continue
|
||||
preferred_label = top_labels[0]
|
||||
confidence = wilson_lower_bound(max_count, decisive)
|
||||
dimensions[dim] = {
|
||||
"n": n,
|
||||
"status": "confident",
|
||||
"preferred": preferred_label,
|
||||
"confidence": confidence,
|
||||
}
|
||||
return dimensions
|
||||
|
||||
|
||||
def detect_conflicts(dimensions: dict[str, dict], profile: dict) -> list[dict]:
|
||||
conflicts = []
|
||||
for dim, data in dimensions.items():
|
||||
if data["status"] != "confident" or data["confidence"] < CONFIDENCE_STOP:
|
||||
continue
|
||||
link = _PROFILE_LINKS.get(dim)
|
||||
if not link or link["profile_key"] not in profile:
|
||||
continue
|
||||
measured = profile[link["profile_key"]]
|
||||
preferred = data["preferred"]
|
||||
conflict_pole = None
|
||||
if preferred == link["low_pole"] and measured >= link["high_threshold"]:
|
||||
conflict_pole = link["high_pole"]
|
||||
elif preferred == link["high_pole"] and measured <= link["low_threshold"]:
|
||||
conflict_pole = link["low_pole"]
|
||||
if conflict_pole is None:
|
||||
continue
|
||||
conflicts.append({
|
||||
"dimension": dim,
|
||||
"preferred": preferred,
|
||||
"preferred_confidence": data["confidence"],
|
||||
"preferred_provenance": "stated-preference",
|
||||
"measured_key": link["profile_key"],
|
||||
"measured_value": measured,
|
||||
"measured_provenance": "measured-from-samples",
|
||||
"message": (
|
||||
f"Stated preference for '{preferred}' ({dim}) contradicts "
|
||||
f"{link['profile_key']}={measured} measured from samples, "
|
||||
f"which points toward '{conflict_pole}'."
|
||||
),
|
||||
})
|
||||
return conflicts
|
||||
|
||||
|
||||
def next_dimension(dimensions: dict[str, dict]) -> dict:
|
||||
def sort_key(dim: str) -> tuple:
|
||||
data = dimensions[dim]
|
||||
confidence = data["confidence"] if data["status"] == "confident" else 0.0
|
||||
return (data["n"], confidence, DIMENSIONS.index(dim))
|
||||
|
||||
ordered = sorted(DIMENSIONS, key=sort_key)
|
||||
chosen = ordered[0]
|
||||
data = dimensions[chosen]
|
||||
if data["n"] == min(dimensions[d]["n"] for d in DIMENSIONS):
|
||||
reason = "fewest_observations"
|
||||
else:
|
||||
reason = "lowest_confidence"
|
||||
return {
|
||||
"next_dimension": chosen,
|
||||
"reason": reason,
|
||||
"n": data["n"],
|
||||
"confidence": data["confidence"],
|
||||
}
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--preferences", required=True)
|
||||
parser.add_argument("--profile")
|
||||
parser.add_argument("--next", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
|
||||
try:
|
||||
rows = load_preferences(Path(args.preferences))
|
||||
except OSError as e:
|
||||
print(json.dumps({"error": f"could not read preferences file: {e}"}))
|
||||
return 2
|
||||
|
||||
dimensions = aggregate(rows)
|
||||
|
||||
if args.next:
|
||||
print(json.dumps(next_dimension(dimensions), indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
output = {"dimensions": dimensions}
|
||||
|
||||
if args.profile:
|
||||
try:
|
||||
profile = json.loads(Path(args.profile).read_text())
|
||||
except OSError as e:
|
||||
print(json.dumps({"error": f"could not read profile file: {e}"}))
|
||||
return 2
|
||||
output["conflicts"] = detect_conflicts(dimensions, profile)
|
||||
|
||||
print(json.dumps(output, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate tiered detector pack integrity."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
PACK_DIR = ROOT / "references" / "packs"
|
||||
MANIFEST = PACK_DIR / "manifest.json"
|
||||
|
||||
JUDGE_ONLY_MACROS = {
|
||||
"macro_both_sidesism",
|
||||
"macro_redemption_arc",
|
||||
"macro_preview_recap",
|
||||
"macro_over_determination",
|
||||
"macro_emotional_flatness",
|
||||
}
|
||||
|
||||
|
||||
def scanner_categories() -> set[str]:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from scripts.banned_phrase_scan import BANNED_PHRASES, STRUCTURAL_PATTERNS
|
||||
|
||||
return {v["category"] for v in BANNED_PHRASES.values()} | {
|
||||
v["category"] for v in STRUCTURAL_PATTERNS
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
failures: list[str] = []
|
||||
data = json.loads(MANIFEST.read_text())
|
||||
packs: dict[str, list[str]] = data["packs"]
|
||||
category_to_pack: dict[str, str] = {}
|
||||
|
||||
for pack, categories in packs.items():
|
||||
path = PACK_DIR / f"{pack}.md"
|
||||
if not path.exists():
|
||||
failures.append(f"missing pack file: {path}")
|
||||
continue
|
||||
text = path.read_text()
|
||||
line_count = len(text.splitlines())
|
||||
if line_count > 120:
|
||||
failures.append(f"{path} has {line_count} lines; max 120")
|
||||
if "## Emit" not in text:
|
||||
failures.append(f"{path} missing ## Emit section")
|
||||
for category in categories:
|
||||
if category in category_to_pack:
|
||||
failures.append(
|
||||
f"category {category} mapped to both {category_to_pack[category]} and {pack}"
|
||||
)
|
||||
category_to_pack[category] = pack
|
||||
|
||||
scanner = scanner_categories()
|
||||
missing = scanner - set(category_to_pack)
|
||||
extra_scanner = (set(category_to_pack) - scanner) - JUDGE_ONLY_MACROS
|
||||
if missing:
|
||||
failures.append(f"scanner categories missing from packs: {sorted(missing)}")
|
||||
if extra_scanner:
|
||||
failures.append(f"unknown non-macro categories in manifest: {sorted(extra_scanner)}")
|
||||
|
||||
macro_hits = {m: category_to_pack.get(m) for m in JUDGE_ONLY_MACROS}
|
||||
missing_macros = [m for m, pack in macro_hits.items() if pack is None]
|
||||
if missing_macros:
|
||||
failures.append(f"judge-only macro families missing: {sorted(missing_macros)}")
|
||||
for macro, pack in macro_hits.items():
|
||||
if pack and pack != "pack-structure":
|
||||
failures.append(f"{macro} must live in pack-structure, found {pack}")
|
||||
|
||||
if failures:
|
||||
print("\n".join(failures), file=sys.stderr)
|
||||
return 1
|
||||
print(f"pack integrity ok: {len(packs)} packs, {len(scanner)} scanner categories")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Contract gates for co-writer suggestions.
|
||||
|
||||
Reads a suggestions file — the JSON emitted by suggest.py, i.e. an object with a
|
||||
``document`` string and a ``suggestions`` list — and enforces four blocking
|
||||
contracts. Each failure is named so a caller can act on it:
|
||||
|
||||
span-minimality Every suggested_replacement edits only its own span:
|
||||
span.text matches document[start:end], the replacement
|
||||
differs from the span text, and it shares no leading or
|
||||
trailing whole word with the span (which would mean the
|
||||
span grabbed unchanged text and could be shrunk — the
|
||||
signature of a whole-sentence rewrite).
|
||||
replacement-scanner Every suggested_replacement passes both scanners in
|
||||
isolation AND introduces no new violation in context.
|
||||
accept-all Applying every replacement yields a document that passes
|
||||
both scanners with validate_preservation exit 0 vs the
|
||||
original.
|
||||
span-overlap Suggestion spans must not overlap.
|
||||
|
||||
Exit 0 when all gates pass, 1 otherwise. Failures (with the offending gate) are
|
||||
reported as JSON on stdout.
|
||||
|
||||
Usage:
|
||||
python3 scripts/check_suggestions.py suggestions.json
|
||||
python3 scripts/check_suggestions.py < suggestions.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from banned_phrase_scan import scan_for_violations
|
||||
from structure_scan import scan as structure_scan
|
||||
from validate_preservation import validate_preservation
|
||||
|
||||
|
||||
def _leading_shared_words(a: str, b: str) -> int:
|
||||
"""Number of identical leading whole words shared by two strings."""
|
||||
n = 0
|
||||
for x, y in zip(a.split(), b.split()):
|
||||
if x == y:
|
||||
n += 1
|
||||
else:
|
||||
break
|
||||
return n
|
||||
|
||||
|
||||
def _trailing_shared_words(a: str, b: str) -> int:
|
||||
ra = " ".join(reversed(a.split()))
|
||||
rb = " ".join(reversed(b.split()))
|
||||
return _leading_shared_words(ra, rb)
|
||||
|
||||
|
||||
def _line_starts(text: str) -> list[int]:
|
||||
starts = [0]
|
||||
for i, ch in enumerate(text):
|
||||
if ch == "\n":
|
||||
starts.append(i + 1)
|
||||
return starts
|
||||
|
||||
|
||||
def violation_spans(text: str) -> list[tuple[int, int]]:
|
||||
"""Absolute (start, end) offsets of every banned-phrase/structural violation."""
|
||||
starts = _line_starts(text)
|
||||
out = []
|
||||
for v in scan_for_violations(text):
|
||||
s = starts[v["line_number"] - 1] + v["column"] - 1
|
||||
out.append((s, s + len(v["phrase"])))
|
||||
return out
|
||||
|
||||
|
||||
def _scanners_clean(text: str) -> bool:
|
||||
return not scan_for_violations(text) and not structure_scan(text).get("flags")
|
||||
|
||||
|
||||
def apply_all(document: str, suggestions: list[dict]) -> str:
|
||||
"""Apply every non-null replacement, right-to-left so offsets stay valid."""
|
||||
out = document
|
||||
for s in sorted(suggestions, key=lambda s: s["span"]["start"], reverse=True):
|
||||
rep = s.get("suggested_replacement")
|
||||
if rep is None:
|
||||
continue
|
||||
st, en = s["span"]["start"], s["span"]["end"]
|
||||
out = out[:st] + rep + out[en:]
|
||||
return out
|
||||
|
||||
|
||||
def check(document: str, suggestions: list[dict]) -> list[dict]:
|
||||
failures: list[dict] = []
|
||||
|
||||
# span-overlap: spans must be disjoint.
|
||||
order = sorted(range(len(suggestions)),
|
||||
key=lambda i: (suggestions[i]["span"]["start"], suggestions[i]["span"]["end"]))
|
||||
last_end = None
|
||||
last_i = None
|
||||
for i in order:
|
||||
st, en = suggestions[i]["span"]["start"], suggestions[i]["span"]["end"]
|
||||
if last_end is not None and st < last_end:
|
||||
failures.append({
|
||||
"gate": "span-overlap",
|
||||
"suggestions": [last_i, i],
|
||||
"detail": f"span {st}-{en} overlaps the previous span ending at {last_end}",
|
||||
})
|
||||
last_end, last_i = en, i
|
||||
|
||||
# span-minimality: span accuracy + tight, local replacements.
|
||||
for i, s in enumerate(suggestions):
|
||||
st, en = s["span"]["start"], s["span"]["end"]
|
||||
span_text = document[st:en]
|
||||
if span_text != s["span"]["text"]:
|
||||
failures.append({
|
||||
"gate": "span-minimality",
|
||||
"suggestion": i,
|
||||
"detail": "span.text does not match document[start:end]",
|
||||
})
|
||||
rep = s.get("suggested_replacement")
|
||||
if rep is None:
|
||||
continue
|
||||
if rep == span_text:
|
||||
failures.append({
|
||||
"gate": "span-minimality",
|
||||
"suggestion": i,
|
||||
"detail": "replacement is identical to the span text (no change)",
|
||||
})
|
||||
continue
|
||||
if _leading_shared_words(span_text, rep) > 0 or _trailing_shared_words(span_text, rep) > 0:
|
||||
failures.append({
|
||||
"gate": "span-minimality",
|
||||
"suggestion": i,
|
||||
"detail": "replacement shares leading/trailing whole words with the span; "
|
||||
"shrink the span so the edit is minimal",
|
||||
})
|
||||
|
||||
# replacement-scanner: each replacement is clean alone and in context.
|
||||
for i, s in enumerate(suggestions):
|
||||
rep = s.get("suggested_replacement")
|
||||
if rep is None:
|
||||
continue
|
||||
if not _scanners_clean(rep):
|
||||
failures.append({
|
||||
"gate": "replacement-scanner",
|
||||
"suggestion": i,
|
||||
"detail": "replacement does not pass both scanners in isolation",
|
||||
})
|
||||
st, en = s["span"]["start"], s["span"]["end"]
|
||||
ctx = document[:st] + rep + document[en:]
|
||||
new_start, new_end = st, st + len(rep)
|
||||
if any(a < new_end and new_start < b for (a, b) in violation_spans(ctx)):
|
||||
failures.append({
|
||||
"gate": "replacement-scanner",
|
||||
"suggestion": i,
|
||||
"detail": "replacement introduces a violation in context",
|
||||
})
|
||||
|
||||
# accept-all: applying everything yields a clean, constraint-preserving doc.
|
||||
unresolved = [i for i, s in enumerate(suggestions) if s.get("suggested_replacement") is None]
|
||||
if unresolved:
|
||||
failures.append({
|
||||
"gate": "accept-all",
|
||||
"detail": f"suggestions {unresolved} have no replacement; cannot accept-all",
|
||||
})
|
||||
applied = apply_all(document, suggestions)
|
||||
if not _scanners_clean(applied):
|
||||
failures.append({
|
||||
"gate": "accept-all",
|
||||
"detail": "the accept-all document still fails a scanner",
|
||||
})
|
||||
preservation = validate_preservation(document, applied)
|
||||
if not preservation["passed"]:
|
||||
failures.append({
|
||||
"gate": "accept-all",
|
||||
"detail": "validate_preservation failed against the original",
|
||||
"missing": preservation["missing"],
|
||||
})
|
||||
|
||||
return failures
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Contract gates for co-writer suggestions.")
|
||||
parser.add_argument(
|
||||
"path", nargs="?", help="Path to a suggestions JSON file (default: read stdin)"
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
if args.path:
|
||||
path = Path(args.path)
|
||||
if not path.exists():
|
||||
print(f"Missing file: {path}", file=sys.stderr)
|
||||
return 2
|
||||
raw = path.read_text(errors="replace")
|
||||
else:
|
||||
raw = sys.stdin.buffer.read().decode("utf-8", errors="replace")
|
||||
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
print(json.dumps({"passed": False, "error": f"invalid JSON: {e}"}, indent=2))
|
||||
return 1
|
||||
|
||||
document = data.get("document")
|
||||
suggestions = data.get("suggestions", [])
|
||||
if not isinstance(document, str):
|
||||
print(json.dumps({"passed": False, "error": "missing 'document' string"}, indent=2))
|
||||
return 1
|
||||
|
||||
failures = check(document, suggestions)
|
||||
result = {
|
||||
"passed": not failures,
|
||||
"failure_gates": sorted({f["gate"] for f in failures}),
|
||||
"failures": failures,
|
||||
}
|
||||
print(json.dumps(result, indent=2))
|
||||
return 0 if not failures else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,373 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline contribution scaffolder for new unslop AI-isms."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
CONTRIB_ROOT = ROOT / ".unslop" / "contrib"
|
||||
TODO_MARKER = "TODO:"
|
||||
SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
|
||||
|
||||
GATE_COMMANDS = [
|
||||
["python3", "evals/check.py", "--full"],
|
||||
]
|
||||
|
||||
|
||||
def read_text(path: Path) -> str:
|
||||
try:
|
||||
return path.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
print(json.dumps({"error": f"could not read {path}: {exc}"}))
|
||||
raise SystemExit(2) from exc
|
||||
|
||||
|
||||
def write_json(path: Path, data: object) -> None:
|
||||
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def sha256_text(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def word_count(text: str) -> int:
|
||||
return len(re.findall(r"[A-Za-z0-9']+", text))
|
||||
|
||||
|
||||
def shorten(text: str, limit: int = 72) -> str:
|
||||
one_line = " ".join(text.split())
|
||||
if len(one_line) <= limit:
|
||||
return one_line
|
||||
return one_line[: limit - 1].rstrip() + "..."
|
||||
|
||||
|
||||
def run_scanner(command: list[str], text: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(command, input=text, capture_output=True, text=True, cwd=ROOT, timeout=30)
|
||||
|
||||
|
||||
def scanner_findings(text: str) -> list[dict[str, object]]:
|
||||
findings: list[dict[str, object]] = []
|
||||
for name, command in (
|
||||
("banned_phrase", ["python3", "scripts/banned_phrase_scan.py"]),
|
||||
("structure", ["python3", "scripts/structure_scan.py"]),
|
||||
):
|
||||
proc = run_scanner(command, text)
|
||||
try:
|
||||
data = json.loads(proc.stdout)
|
||||
except json.JSONDecodeError:
|
||||
data = {}
|
||||
violations = data.get("violations", [])
|
||||
if name == "structure":
|
||||
violations = [
|
||||
{
|
||||
"phrase": flag.get("metric", ""),
|
||||
"category": flag.get("metric", ""),
|
||||
"severity": flag.get("severity", ""),
|
||||
}
|
||||
for flag in data.get("flags", [])
|
||||
]
|
||||
for violation in violations:
|
||||
findings.append(
|
||||
{
|
||||
"scanner": name,
|
||||
"phrase": violation.get("phrase", ""),
|
||||
"category": violation.get("category", ""),
|
||||
"severity": violation.get("severity", ""),
|
||||
}
|
||||
)
|
||||
return findings
|
||||
|
||||
|
||||
def redaction_pairs(values: list[str]) -> list[tuple[str, str]]:
|
||||
pairs = []
|
||||
for value in values:
|
||||
if "=" not in value:
|
||||
print(json.dumps({"error": "--redact values must use orig=REPL"}))
|
||||
raise SystemExit(2)
|
||||
orig, repl = value.split("=", 1)
|
||||
pairs.append((orig, repl))
|
||||
return pairs
|
||||
|
||||
|
||||
def apply_redactions(text: str, tell: str, pairs: list[tuple[str, str]]) -> str:
|
||||
before_count = text.count(tell)
|
||||
redacted = text
|
||||
for orig, repl in pairs:
|
||||
redacted = redacted.replace(orig, repl)
|
||||
if before_count == 0 or redacted.count(tell) != before_count:
|
||||
print(json.dumps({"error": "redaction alters the specimen", "tell": tell}))
|
||||
raise SystemExit(4)
|
||||
return redacted
|
||||
|
||||
|
||||
def row_fn(slug: str, tell: str, snippet: str) -> dict[str, object]:
|
||||
assertion_tell = tell.casefold()
|
||||
return {
|
||||
"id": f"CONTRIB-FN-{slug}",
|
||||
"target": "script",
|
||||
"category": "scanner_false_negative",
|
||||
"stdin": snippet,
|
||||
"assertions": [
|
||||
{"type": "json", "path": "total_violations", "gte": 1},
|
||||
{"type": "violation_phrase_contains", "value": assertion_tell},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def row_fp_template(slug: str, category: str, tell: str) -> dict[str, object]:
|
||||
return {
|
||||
"id": f"CONTRIB-FP-{slug}",
|
||||
"category": "scanner_false_positive",
|
||||
"protects": category,
|
||||
"target": "script",
|
||||
"stdin": "TODO: add a literal or domain-specific use that should remain clean.",
|
||||
"assertions": [{"type": "json", "path": "total_violations", "equals": 0}],
|
||||
}
|
||||
|
||||
|
||||
def render_report(
|
||||
manifest: dict[str, object],
|
||||
snippet: str,
|
||||
gate_results: str = "",
|
||||
include_rec: bool = False,
|
||||
) -> str:
|
||||
redactions = manifest.get("redactions", [])
|
||||
redaction_note = "names/numbers redacted; tell verbatim" if redactions else "none"
|
||||
rows = [
|
||||
f"| CONTRIB-FN-{manifest['pattern_name']} | FN | exact specimen flags `{manifest['tell']}` |",
|
||||
f"| CONTRIB-FP-{manifest['pattern_name']} | FP | literal-use protection for `{manifest['tell']}` |",
|
||||
]
|
||||
if include_rec:
|
||||
rows.append("| CONTRIB-REC | REC | existing-word recall still flags |")
|
||||
quoted = "\n".join(f"> {line}" if line else ">" for line in snippet.splitlines())
|
||||
pattern_added = manifest.get(
|
||||
"pattern_added",
|
||||
f"TODO: regex or phrase for `{manifest['tell']}`",
|
||||
)
|
||||
return (
|
||||
f"# Add {manifest['category']} pattern: {shorten(str(manifest['tell']))}\n\n"
|
||||
"## The specimen\n\n"
|
||||
f"{quoted}\n\n"
|
||||
f"- Source genre: {manifest.get('source_genre', 'TODO: source genre')}\n"
|
||||
f"- Date: {manifest['date']}\n"
|
||||
f"- Redaction note: {redaction_note}\n\n"
|
||||
"## Why it's an AI-ism\n\n"
|
||||
f"{manifest.get('rationale', 'TODO: explain why this phrase is a reusable AI-writing tell in 2-4 sentences.')}\n\n"
|
||||
"## Detection\n\n"
|
||||
f"- Pattern added: {pattern_added}\n"
|
||||
f"- Severity: {manifest.get('severity', 'TODO: hard or soft')}\n"
|
||||
f"- Gating rationale: {manifest.get('gating_rationale', 'TODO: explain literal-use boundary')}\n"
|
||||
"- Catalog entry location: references/taboo-phrases.md\n\n"
|
||||
"## Evals\n\n"
|
||||
"| row id | kind | what it pins |\n"
|
||||
"|---|---|---|\n"
|
||||
+ "\n".join(rows)
|
||||
+ "\n\n"
|
||||
"The FN stdin is the unmodified specimen after approved redaction.\n\n"
|
||||
"## Gate results\n\n"
|
||||
f"{gate_results or 'TODO: paste gate tails from verify.'}\n\n"
|
||||
"## Checklist\n\n"
|
||||
"- [ ] eval-first (row was red before the pattern)\n"
|
||||
"- [ ] literal-use FP row included\n"
|
||||
"- [ ] REC row if an existing word was gated\n"
|
||||
"- [ ] catalog + scanner parity green\n"
|
||||
"- [ ] coverage gate green (pattern exercised)\n"
|
||||
"- [ ] snippet publication approved by the user\n"
|
||||
)
|
||||
|
||||
|
||||
def cmd_precheck(args: argparse.Namespace) -> int:
|
||||
snippet = read_text(Path(args.snippet_file))
|
||||
findings = scanner_findings(snippet)
|
||||
if findings:
|
||||
print(json.dumps({"status": "already_covered", "findings": findings}, indent=2, sort_keys=True))
|
||||
return 3
|
||||
print(json.dumps({"status": "clean", "words": word_count(snippet)}, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_scaffold(args: argparse.Namespace) -> int:
|
||||
if not SLUG_RE.match(args.pattern_name):
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"error": f"invalid pattern name: {args.pattern_name!r}; use lowercase letters, digits, - or _",
|
||||
}
|
||||
)
|
||||
)
|
||||
return 2
|
||||
source = Path(args.snippet)
|
||||
snippet = read_text(source)
|
||||
if args.tell not in snippet:
|
||||
print(json.dumps({"error": "tell substring not found", "tell": args.tell}))
|
||||
return 2
|
||||
try:
|
||||
manifest_date = datetime.strptime(args.date, "%Y-%m-%d").date().isoformat()
|
||||
except ValueError:
|
||||
print(json.dumps({"error": "--date must use YYYY-MM-DD", "date": args.date}))
|
||||
return 2
|
||||
redactions = redaction_pairs(args.redact or [])
|
||||
redacted = apply_redactions(snippet, args.tell, redactions)
|
||||
bundle = CONTRIB_ROOT / args.pattern_name
|
||||
bundle.mkdir(parents=True, exist_ok=True)
|
||||
manifest = {
|
||||
"category": args.category,
|
||||
"date": manifest_date,
|
||||
"pattern_name": args.pattern_name,
|
||||
"pre_redaction_sha256": sha256_text(snippet),
|
||||
"post_redaction_sha256": sha256_text(redacted),
|
||||
"redactions": [{"from": orig, "to": repl} for orig, repl in redactions],
|
||||
"tell": args.tell,
|
||||
"word_count": word_count(redacted),
|
||||
}
|
||||
write_json(bundle / "row_fn.json", row_fn(args.pattern_name, args.tell, redacted))
|
||||
write_json(bundle / "row_fp_TEMPLATE.json", row_fp_template(args.pattern_name, args.category, args.tell))
|
||||
write_json(bundle / "manifest.json", manifest)
|
||||
(bundle / "snippet.txt").write_text(redacted, encoding="utf-8")
|
||||
(bundle / "report.md").write_text(render_report(manifest, redacted), encoding="utf-8")
|
||||
print(json.dumps({"bundle": str(bundle.relative_to(ROOT)), "status": "scaffolded"}, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
def run_row(row: dict[str, object]) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["python3", "scripts/banned_phrase_scan.py"],
|
||||
input=str(row.get("stdin", "")),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
cwd=ROOT,
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
|
||||
def row_assertions_pass(row: dict[str, object], proc: subprocess.CompletedProcess[str]) -> bool:
|
||||
try:
|
||||
data = json.loads(proc.stdout)
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
for assertion in row.get("assertions", []):
|
||||
if assertion.get("type") == "json":
|
||||
cur = data
|
||||
for part in assertion["path"].split("."):
|
||||
cur = cur[int(part)] if isinstance(cur, list) else cur[part]
|
||||
if "gte" in assertion and not cur >= assertion["gte"]:
|
||||
return False
|
||||
if "equals" in assertion and cur != assertion["equals"]:
|
||||
return False
|
||||
elif assertion.get("type") == "violation_phrase_contains":
|
||||
phrases = [v.get("phrase", "") for v in data.get("violations", [])]
|
||||
needle = str(assertion["value"]).casefold()
|
||||
if not any(needle in str(phrase).casefold() for phrase in phrases):
|
||||
return False
|
||||
else:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def tail(text: str, lines: int = 12) -> str:
|
||||
split = text.splitlines()
|
||||
return "\n".join(split[-lines:])
|
||||
|
||||
|
||||
def verify_bundle(bundle: Path, run_gates: bool = True) -> tuple[bool, str]:
|
||||
if not bundle.exists():
|
||||
return False, "missing bundle"
|
||||
row_path = bundle / "row_fn.json"
|
||||
report_path = bundle / "report.md"
|
||||
manifest_path = bundle / "manifest.json"
|
||||
snippet_path = bundle / "snippet.txt"
|
||||
for path in (row_path, report_path, manifest_path, snippet_path):
|
||||
if not path.exists():
|
||||
return False, f"missing {path.name}"
|
||||
report = report_path.read_text(encoding="utf-8")
|
||||
if TODO_MARKER in report:
|
||||
return False, "report contains TODO markers"
|
||||
row = json.loads(row_path.read_text(encoding="utf-8"))
|
||||
snippet = snippet_path.read_text(encoding="utf-8")
|
||||
if row.get("stdin") != snippet:
|
||||
return False, "row_fn stdin does not equal snippet.txt byte-for-byte"
|
||||
proc = run_row(row)
|
||||
assertion_ok = row_assertions_pass(row, proc)
|
||||
red_first = not assertion_ok
|
||||
gate_results = [f"red-first: {'ok' if red_first else 'already green'}"]
|
||||
ok = red_first
|
||||
if not red_first:
|
||||
ok = True
|
||||
gate_results[0] = "red-first: already green; proposed pattern appears active"
|
||||
if run_gates:
|
||||
for command in GATE_COMMANDS:
|
||||
proc = subprocess.run(command, capture_output=True, text=True, cwd=ROOT, timeout=120)
|
||||
gate_results.append(
|
||||
"### "
|
||||
+ " ".join(command)
|
||||
+ f"\n\n```text\nexit={proc.returncode}\n{tail(proc.stdout or proc.stderr)}\n```"
|
||||
)
|
||||
ok = ok and proc.returncode == 0
|
||||
final = "\n\n".join(gate_results)
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
report_path.write_text(render_report(manifest, snippet, final, (bundle / "row_rec.json").exists()), encoding="utf-8")
|
||||
return ok, final
|
||||
|
||||
|
||||
def cmd_verify(args: argparse.Namespace) -> int:
|
||||
ok, message = verify_bundle(Path(args.bundle), run_gates=not args.no_gates)
|
||||
print(message)
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def cmd_report(args: argparse.Namespace) -> int:
|
||||
bundle = Path(args.bundle)
|
||||
if not bundle.exists():
|
||||
print(json.dumps({"error": "missing bundle", "bundle": str(bundle)}))
|
||||
return 2
|
||||
manifest = json.loads((bundle / "manifest.json").read_text(encoding="utf-8"))
|
||||
snippet = (bundle / "snippet.txt").read_text(encoding="utf-8")
|
||||
report = (bundle / "report.md").read_text(encoding="utf-8")
|
||||
gate_marker = "## Gate results\n\n"
|
||||
checklist_marker = "\n\n## Checklist"
|
||||
gate_results = ""
|
||||
if gate_marker in report and checklist_marker in report:
|
||||
gate_results = report.split(gate_marker, 1)[1].split(checklist_marker, 1)[0]
|
||||
print(render_report(manifest, snippet, gate_results, (bundle / "row_rec.json").exists()).rstrip())
|
||||
return 0
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
precheck = sub.add_parser("precheck")
|
||||
precheck.add_argument("snippet_file")
|
||||
precheck.set_defaults(func=cmd_precheck)
|
||||
scaffold = sub.add_parser("scaffold")
|
||||
scaffold.add_argument("--snippet", required=True)
|
||||
scaffold.add_argument("--tell", required=True)
|
||||
scaffold.add_argument("--category", required=True)
|
||||
scaffold.add_argument("--pattern-name", required=True)
|
||||
scaffold.add_argument("--redact", action="append")
|
||||
scaffold.add_argument("--date", default=date.today().isoformat())
|
||||
scaffold.set_defaults(func=cmd_scaffold)
|
||||
verify = sub.add_parser("verify")
|
||||
verify.add_argument("--bundle", required=True)
|
||||
verify.add_argument("--no-gates", action="store_true")
|
||||
verify.set_defaults(func=cmd_verify)
|
||||
report = sub.add_parser("report")
|
||||
report.add_argument("--bundle", required=True)
|
||||
report.set_defaults(func=cmd_report)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
return args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Executable
+139
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Check change percentage between original and transformed text.
|
||||
|
||||
Flags if >40% of words changed (may indicate over-editing).
|
||||
|
||||
Pure reordering is exempt: change_percentage still reflects the raw
|
||||
delete+insert diff, but excessive_change stays false when the transformed
|
||||
text is a word-level permutation of the original (moved sentences are not
|
||||
over-editing). SKILL.md's 40% guidance refers to the excessive_change flag.
|
||||
|
||||
Usage:
|
||||
python diff_check.py original.txt transformed.txt
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from difflib import SequenceMatcher
|
||||
from typing import TypedDict
|
||||
|
||||
|
||||
class DiffResult(TypedDict):
|
||||
original_word_count: int
|
||||
transformed_word_count: int
|
||||
similarity_ratio: float
|
||||
change_percentage: float
|
||||
words_added: int
|
||||
words_removed: int
|
||||
words_changed: int
|
||||
excessive_change: bool
|
||||
flags: list[str]
|
||||
|
||||
|
||||
def split_words(text: str) -> list[str]:
|
||||
"""Split text into comparison tokens.
|
||||
|
||||
Punctuation is tokenized separately (not stripped) so that structural edits
|
||||
like repunctuation register as change instead of reading as identical.
|
||||
"""
|
||||
return re.findall(r'\w+|[^\w\s]', text.lower())
|
||||
|
||||
|
||||
def calculate_diff(original: str, transformed: str) -> DiffResult:
|
||||
"""Calculate difference metrics between two texts."""
|
||||
original_words = split_words(original)
|
||||
transformed_words = split_words(transformed)
|
||||
|
||||
# Use SequenceMatcher for word-level diff
|
||||
matcher = SequenceMatcher(None, original_words, transformed_words)
|
||||
similarity = matcher.ratio()
|
||||
|
||||
# Count operations
|
||||
words_added = 0
|
||||
words_removed = 0
|
||||
words_changed = 0
|
||||
|
||||
for tag, i1, i2, j1, j2 in matcher.get_opcodes():
|
||||
if tag == 'replace':
|
||||
words_changed += max(i2 - i1, j2 - j1)
|
||||
elif tag == 'delete':
|
||||
words_removed += i2 - i1
|
||||
elif tag == 'insert':
|
||||
words_added += j2 - j1
|
||||
|
||||
# Change percentage (relative to original)
|
||||
total_changes = words_added + words_removed + words_changed
|
||||
change_percentage = (
|
||||
(total_changes / len(original_words) * 100)
|
||||
if original_words else 0
|
||||
)
|
||||
shared_tokens = sum((Counter(original_words) & Counter(transformed_words)).values())
|
||||
token_overlap = shared_tokens / len(original_words) if original_words else 0
|
||||
|
||||
# Flags
|
||||
flags: list[str] = []
|
||||
excessive = False
|
||||
|
||||
if change_percentage > 40:
|
||||
if token_overlap >= 0.9:
|
||||
flags.append(f"Mostly reordered ({token_overlap:.0%} tokens shared)")
|
||||
else:
|
||||
flags.append(f"Excessive change ({change_percentage:.1f}% > 40% threshold)")
|
||||
excessive = True
|
||||
|
||||
if len(transformed_words) < len(original_words) * 0.3:
|
||||
flags.append("Transformed text is less than 30% of original length")
|
||||
excessive = True
|
||||
|
||||
if len(transformed_words) > len(original_words) * 1.5:
|
||||
flags.append("Transformed text is 50%+ longer than original")
|
||||
|
||||
# Length change ratio
|
||||
length_ratio = len(transformed_words) / len(original_words) if original_words else 0
|
||||
if length_ratio < 0.5:
|
||||
flags.append(f"Significant condensation ({length_ratio:.0%} of original)")
|
||||
elif length_ratio > 1.2:
|
||||
flags.append(f"Text expanded ({length_ratio:.0%} of original)")
|
||||
|
||||
return {
|
||||
"original_word_count": len(original_words),
|
||||
"transformed_word_count": len(transformed_words),
|
||||
"similarity_ratio": round(similarity, 3),
|
||||
"change_percentage": round(change_percentage, 1),
|
||||
"words_added": words_added,
|
||||
"words_removed": words_removed,
|
||||
"words_changed": words_changed,
|
||||
"excessive_change": excessive,
|
||||
"flags": flags
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) < 3:
|
||||
print("Usage: diff_check.py <original.txt> <transformed.txt>")
|
||||
sys.exit(1)
|
||||
|
||||
# Read inputs
|
||||
try:
|
||||
with open(sys.argv[1], 'r') as f:
|
||||
original = f.read()
|
||||
with open(sys.argv[2], 'r') as f:
|
||||
transformed = f.read()
|
||||
except OSError as e:
|
||||
print(json.dumps({"error": f"Could not read input: {e}"}))
|
||||
sys.exit(2)
|
||||
|
||||
result = calculate_diff(original, transformed)
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
# Exit with 1 if excessive change
|
||||
sys.exit(1 if result["excessive_change"] else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Extract must-preserve constraints from input text.
|
||||
|
||||
Identifies facts, names, dates, URLs, numbers that must survive transformation.
|
||||
Outputs JSON with constraint spans for validation.
|
||||
|
||||
Usage:
|
||||
python extract_constraints.py < input.txt
|
||||
python extract_constraints.py input.txt
|
||||
echo "Text here" | python extract_constraints.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import re
|
||||
import json
|
||||
from typing import TypedDict
|
||||
|
||||
|
||||
class Constraint(TypedDict):
|
||||
type: str
|
||||
value: str
|
||||
start: int
|
||||
end: int
|
||||
|
||||
|
||||
PATTERNS: dict[str, str] = {
|
||||
# Currency with amounts
|
||||
"currency": r"\$[\d,]+\.?\d*[KMBkmb]?(?:\s*(?:million|billion|thousand))?",
|
||||
|
||||
# Percentages
|
||||
"percentage": r"\d+\.?\d*%",
|
||||
|
||||
# ISO dates
|
||||
"date_iso": r"\d{4}-\d{2}-\d{2}",
|
||||
|
||||
# Quarter dates
|
||||
"date_quarter": r"Q[1-4]\s+\d{4}",
|
||||
|
||||
# Natural dates
|
||||
"date_natural": r"(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2}(?:st|nd|rd|th)?,?\s+\d{4}",
|
||||
|
||||
# Years (standalone, 1900-2099)
|
||||
"year": r"\b(?:19|20)\d{2}\b",
|
||||
|
||||
# Times
|
||||
"time": r"\d{1,2}:\d{2}(?::\d{2})?\s*(?:AM|PM|am|pm|UTC|PST|EST|CST|MST|GMT)?",
|
||||
|
||||
# Numeric quantities with magnitude words
|
||||
"magnitude_number": r"\b\d[\d,]*\.?\d*\s+(?:thousand|million|billion|trillion)\b",
|
||||
|
||||
# Measurements with units
|
||||
"measurement": r"\d+\.?\d*\s*(?:°C|°F|degrees?\s*(?:C|F|Celsius|Fahrenheit)?|ms|s|sec|min|hr|hour|day|week|month|year|KB|MB|GB|TB|PB|kg|g|lb|oz|m|km|mi|ft|in|cm|mm|px|em|rem|%)\b",
|
||||
|
||||
# Phone numbers (capture the whole number, before "range" can grab a slice)
|
||||
"phone": r"\b(?:\+?1[-.\s]?)?(?:\(\d{3}\)\s*|\d{3}[-.\s])\d{3}[-.\s]\d{4}\b",
|
||||
|
||||
# Ranges (numeric)
|
||||
"range": r"\d+\.?\d*\s*[-–]\s*\d+\.?\d*(?:\s*(?:K|M|B|%|years?|months?|days?))?",
|
||||
|
||||
# URLs
|
||||
"url": r"https?://[^\s\)\]\>\"\']+",
|
||||
|
||||
# Email addresses
|
||||
"email": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}",
|
||||
|
||||
# Code references (backticked)
|
||||
"code": r"`[^`]+`",
|
||||
|
||||
# Direct quotes (longer than 10 chars)
|
||||
"quote": r'["“][^"“”]{10,}["”]',
|
||||
|
||||
# Cited references (Section 12(b), Article 5, Figure 2, Table 1, Eq. 3 …) —
|
||||
# must-preserve per fact-preservation.md and easy to silently drop.
|
||||
"reference": r"\b(?:Sections?|Sec\.|§|Articles?|Clauses?|Paragraphs?|Para\.|Figures?|Fig\.|Tables?|Appendix|Appendices|Schedule|Exhibit|Equations?|Eq\.|Chapters?|Rules?|Items?)\s+\d+[A-Za-z]?(?:\([a-z0-9]+\))?(?:[.\-]\d+)*",
|
||||
|
||||
# Version numbers
|
||||
"version": r"v?\d+\.\d+(?:\.\d+)?(?:-[a-zA-Z0-9]+)?",
|
||||
|
||||
# API endpoints
|
||||
"api_endpoint": r"(?<![\w])/(?:api|v\d+)(?:/[\w-]+)+|(?<![\w])/[\w-]+(?:/[\w-]+){2,}",
|
||||
|
||||
# Inclusive disjunction — collapsing and/or to a plain conjunction changes scope
|
||||
"and_or": r"\band/or\b",
|
||||
|
||||
# Numeric counts with context
|
||||
"count": r"\b\d+(?:,\d{3})*\s+(?:users?|customers?|employees?|companies?|teams?|people|engineers?|developers?|items?|products?|orders?|transactions?|requests?|queries?|rows?|records?)\b",
|
||||
}
|
||||
|
||||
# Proper noun patterns (simplified - real implementation would use NER)
|
||||
PROPER_NOUN_INDICATORS = [
|
||||
r"\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)+\b", # Multi-word capitalized (John Smith)
|
||||
r"\b[A-Z][a-z]+\s+(?:Inc|Corp|LLC|Ltd|Co)\b\.?", # Company suffixes
|
||||
r"\b(?:Dr|Mr|Ms|Mrs|Prof)\.?\s+[A-Z][a-z]+\b", # Titles with names
|
||||
]
|
||||
|
||||
|
||||
def extract_constraints(text: str) -> list[Constraint]:
|
||||
"""Extract all must-preserve constraints from text."""
|
||||
constraints: list[Constraint] = []
|
||||
seen_spans: set[tuple[int, int]] = set()
|
||||
|
||||
# Extract pattern-based constraints
|
||||
for constraint_type, pattern in PATTERNS.items():
|
||||
for match in re.finditer(pattern, text, re.IGNORECASE if constraint_type.startswith("date") else 0):
|
||||
span = (match.start(), match.end())
|
||||
if span not in seen_spans:
|
||||
seen_spans.add(span)
|
||||
constraints.append({
|
||||
"type": constraint_type,
|
||||
"value": match.group(),
|
||||
"start": match.start(),
|
||||
"end": match.end()
|
||||
})
|
||||
|
||||
# Extract proper nouns (simplified)
|
||||
for pattern in PROPER_NOUN_INDICATORS:
|
||||
for match in re.finditer(pattern, text):
|
||||
span = (match.start(), match.end())
|
||||
# Don't add if overlapping with existing constraint
|
||||
overlaps = any(
|
||||
not (span[1] <= existing[0] or span[0] >= existing[1])
|
||||
for existing in seen_spans
|
||||
)
|
||||
if not overlaps:
|
||||
seen_spans.add(span)
|
||||
constraints.append({
|
||||
"type": "proper_noun",
|
||||
"value": match.group(),
|
||||
"start": match.start(),
|
||||
"end": match.end()
|
||||
})
|
||||
|
||||
# Bare quantities: comma-grouped numbers or integers of 4+ digits. These are
|
||||
# real facts (50000 fans, 2,500 signups) that the unit/count patterns miss.
|
||||
# Skip any that overlap a constraint already claimed (years, currency, phones)
|
||||
# so we don't double-count or shred a captured value.
|
||||
number_pattern = r"(?<![\d.,])(?:\d{1,3}(?:,\d{3})+|\d{4,})(?![\d.,])"
|
||||
for match in re.finditer(number_pattern, text):
|
||||
span = (match.start(), match.end())
|
||||
overlaps = any(
|
||||
not (span[1] <= existing[0] or span[0] >= existing[1])
|
||||
for existing in seen_spans
|
||||
)
|
||||
if not overlaps:
|
||||
seen_spans.add(span)
|
||||
constraints.append({
|
||||
"type": "number",
|
||||
"value": match.group(),
|
||||
"start": match.start(),
|
||||
"end": match.end()
|
||||
})
|
||||
|
||||
# Sort by position
|
||||
constraints.sort(key=lambda c: c["start"])
|
||||
|
||||
return constraints
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Extract must-preserve constraints from input text."
|
||||
)
|
||||
parser.add_argument("path", nargs="?", help="Path to input text file (default: read stdin)")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args(sys.argv[1:])
|
||||
|
||||
# Read input
|
||||
if args.path:
|
||||
try:
|
||||
with open(args.path, 'r', errors="replace") as f:
|
||||
text = f.read()
|
||||
except OSError as e:
|
||||
print(json.dumps({"error": f"Could not read input: {e}", "constraints": []}))
|
||||
sys.exit(2)
|
||||
else:
|
||||
text = sys.stdin.buffer.read().decode("utf-8", errors="replace")
|
||||
|
||||
if not text.strip():
|
||||
print(json.dumps({"error": "No input provided", "constraints": []}))
|
||||
sys.exit(1)
|
||||
|
||||
constraints = extract_constraints(text)
|
||||
|
||||
output = {
|
||||
"input_length": len(text),
|
||||
"constraint_count": len(constraints),
|
||||
"constraints": constraints
|
||||
}
|
||||
|
||||
print(json.dumps(output, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Classify harvested candidates into situation/register coverage cells."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
from harvest_samples import DATE_FLOOR, recency_value # noqa: E402
|
||||
|
||||
|
||||
CELLS = [
|
||||
"numbers_data",
|
||||
"question_addressed",
|
||||
"anecdote_markers",
|
||||
"disagreement",
|
||||
"openings_closings",
|
||||
]
|
||||
|
||||
|
||||
def load_candidates(path: Path) -> list[dict[str, Any]]:
|
||||
data = json.loads(path.read_text())
|
||||
if isinstance(data, dict):
|
||||
return data.get("candidates", [])
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
raise ValueError("candidates file must contain a list or {candidates: [...]}")
|
||||
|
||||
|
||||
def cells_for(text: str) -> list[str]:
|
||||
lowered = text.lower()
|
||||
cells = []
|
||||
if re.search(r"\b\d+(?:[,.]\d+)?%?\b", lowered):
|
||||
cells.append("numbers_data")
|
||||
if "?" in text or re.search(r"\b(?:why|how|what|when|where|which)\b", lowered):
|
||||
cells.append("question_addressed")
|
||||
if re.search(r"\b(?:last quarter|yesterday|once|during|when we|i noticed|i remember)\b", lowered):
|
||||
cells.append("anecdote_markers")
|
||||
if re.search(r"\b(?:disagree|however|instead|not convinced|push back)\b", lowered):
|
||||
cells.append("disagreement")
|
||||
if re.search(r"\b(?:hi|thanks|best|regards|closing|opening|first off)\b", lowered):
|
||||
cells.append("openings_closings")
|
||||
return cells
|
||||
|
||||
|
||||
def quality_for(candidate: dict[str, Any], cells: list[str]) -> int:
|
||||
words = int(candidate.get("words") or len(re.findall(r"\w+", candidate.get("text", ""))))
|
||||
score = 3
|
||||
if 40 <= words <= 220:
|
||||
score += 1
|
||||
if cells:
|
||||
score += 1
|
||||
if candidate.get("suspect_ai"):
|
||||
score -= 2
|
||||
if candidate.get("dictated"):
|
||||
score -= 1
|
||||
return max(1, min(5, score))
|
||||
|
||||
|
||||
def candidate_id(candidate: dict[str, Any], index: int) -> Any:
|
||||
return candidate.get("id", index)
|
||||
|
||||
|
||||
def source_position(candidate: dict[str, Any]) -> str:
|
||||
source = candidate.get("source", {})
|
||||
return str(source.get("line", source.get("offset", "")))
|
||||
|
||||
|
||||
def coverage_from(candidates: list[dict[str, Any]]) -> dict[str, int]:
|
||||
coverage = {cell: 0 for cell in CELLS}
|
||||
for candidate in candidates:
|
||||
for cell in candidate.get("cells", []):
|
||||
coverage[cell] = coverage.get(cell, 0) + 1
|
||||
return coverage
|
||||
|
||||
|
||||
def rank_enriched(candidates: list[dict[str, Any]]) -> list[int]:
|
||||
seen_empty = set()
|
||||
rank_rows = []
|
||||
for idx, candidate in enumerate(candidates):
|
||||
cells = candidate.get("cells", [])
|
||||
fills_empty = any(cell not in seen_empty for cell in cells)
|
||||
seen_empty.update(cells)
|
||||
rank_rows.append((idx, fills_empty))
|
||||
ranked = sorted(
|
||||
rank_rows,
|
||||
key=lambda row: (
|
||||
not row[1],
|
||||
-int(candidates[row[0]].get("quality") or 0),
|
||||
-recency_value(candidates[row[0]]),
|
||||
bool(candidates[row[0]].get("suspect_ai")),
|
||||
bool(candidates[row[0]].get("dictated")),
|
||||
str(candidates[row[0]].get("source", {}).get("path", "")),
|
||||
source_position(candidates[row[0]]),
|
||||
row[0],
|
||||
),
|
||||
)
|
||||
return [candidate_id(candidates[idx], idx) for idx, _ in ranked]
|
||||
|
||||
|
||||
def heuristic(candidates: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
enriched = []
|
||||
seen_empty = set()
|
||||
for idx, candidate in enumerate(candidates):
|
||||
cells = cells_for(candidate.get("text", ""))
|
||||
quality = quality_for(candidate, cells)
|
||||
fills_empty = any(cell not in seen_empty for cell in cells)
|
||||
seen_empty.update(cells)
|
||||
enriched.append({
|
||||
"index": idx,
|
||||
"id": candidate.get("id", idx),
|
||||
"cells": cells,
|
||||
"quality": quality,
|
||||
"why": "lexical heuristic matched " + (", ".join(cells) if cells else "no named cell"),
|
||||
"fills_empty_coverage_cell": fills_empty,
|
||||
"source": candidate.get("source", {}),
|
||||
"suspect_ai": candidate.get("suspect_ai"),
|
||||
"dictated": candidate.get("dictated"),
|
||||
})
|
||||
return {
|
||||
"coverage_matrix": coverage_from(enriched),
|
||||
"candidates": enriched,
|
||||
"ranking": rank_enriched(enriched),
|
||||
}
|
||||
|
||||
|
||||
def write_agent_tasks(candidates: list[dict[str, Any]], out_dir: Path) -> dict[str, Any]:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
chunks = []
|
||||
prompt = (
|
||||
"Classify each candidate into WP10b situation/register cells. "
|
||||
"Return JSONL rows with candidate_index, cells, quality 1-5, and one-line why. "
|
||||
"Cells include numbers_data, question_addressed, anecdote_markers, "
|
||||
"disagreement, openings_closings, plus any clearly justified additional cell."
|
||||
)
|
||||
for start in range(0, len(candidates), 10):
|
||||
chunk = candidates[start:start + 10]
|
||||
path = out_dir / f"harvest-classify-{start // 10 + 1:03d}.json"
|
||||
payload = {
|
||||
"contract": "tier-1-pack-detector",
|
||||
"prompt": prompt,
|
||||
"candidates": [
|
||||
{"candidate_index": start + i, "text": c.get("text", ""), "source": c.get("source", {})}
|
||||
for i, c in enumerate(chunk)
|
||||
],
|
||||
}
|
||||
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
chunks.append(str(path))
|
||||
return {"task_files": chunks, "chunk_size": 10}
|
||||
|
||||
|
||||
def result_candidate_key(row: dict[str, Any]) -> Any:
|
||||
for key in ("candidate_id", "id", "candidate_index", "index"):
|
||||
if key in row:
|
||||
return row[key]
|
||||
raise ValueError("result row missing candidate id")
|
||||
|
||||
|
||||
def merge_results(candidates_path: Path, results_path: Path) -> dict[str, Any]:
|
||||
candidates = load_candidates(candidates_path)
|
||||
rows_by_id: dict[Any, dict[str, Any]] = {}
|
||||
for line in results_path.read_text().splitlines():
|
||||
if line.strip():
|
||||
row = json.loads(line)
|
||||
rows_by_id[result_candidate_key(row)] = row
|
||||
|
||||
merged = []
|
||||
for idx, candidate in enumerate(candidates):
|
||||
cid = candidate_id(candidate, idx)
|
||||
row = rows_by_id.get(cid)
|
||||
if row is None and idx in rows_by_id:
|
||||
row = rows_by_id[idx]
|
||||
cells = list(row.get("cells", [])) if row else []
|
||||
quality = int(row.get("quality")) if row and row.get("quality") is not None else quality_for(candidate, cells)
|
||||
why = str(row.get("why", "no classifier result")) if row else "no classifier result"
|
||||
merged.append({
|
||||
**candidate,
|
||||
"id": cid,
|
||||
"cells": cells,
|
||||
"quality": max(1, min(5, quality)),
|
||||
"why": why,
|
||||
})
|
||||
|
||||
return {
|
||||
"coverage_matrix": coverage_from(merged),
|
||||
"candidates": merged,
|
||||
"ranking": rank_enriched(merged),
|
||||
}
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--candidates", required=True)
|
||||
parser.add_argument("--mode", choices=["heuristic", "agent"], default="heuristic")
|
||||
parser.add_argument("--out-dir", default="harvest-agent-tasks")
|
||||
parser.add_argument("--merge")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
if args.merge:
|
||||
print(json.dumps(merge_results(Path(args.candidates), Path(args.merge)), indent=2, sort_keys=True))
|
||||
return 0
|
||||
candidates = load_candidates(Path(args.candidates))
|
||||
if args.mode == "heuristic":
|
||||
print(json.dumps(heuristic(candidates), indent=2, sort_keys=True))
|
||||
else:
|
||||
print(json.dumps(write_agent_tasks(candidates, Path(args.out_dir)), indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,534 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Harvest user-authored writing samples from transcripts and declared folders.
|
||||
|
||||
Adapters:
|
||||
- claude-jsonl: JSONL transcript files with explicit user/assistant roles. Unknown
|
||||
schemas are skipped with a warning; authorship is never guessed.
|
||||
- codex-jsonl: Codex CLI/Desktop session JSONL (~/.codex/sessions/YYYY/MM/DD/
|
||||
rollout-*.jsonl). Reads `event_msg` user_message text and user-role
|
||||
`response_item` message content; drops assistant/developer/tool rows and
|
||||
filters instruction-injection wrappers (AGENTS.md dumps, <environment_context>,
|
||||
and similar structural markers) that Codex injects into user-role turns.
|
||||
- text-folder: directories containing .md/.txt files. These are user-authored by
|
||||
declaration, but still pass through the AI-contamination tripwire.
|
||||
|
||||
Adapter detection for .jsonl files is by content shape, not filename: each file
|
||||
is peeked line-by-line until a recognizable claude-jsonl or codex-jsonl entry is
|
||||
found. Files that never match either shape fall through to the claude-jsonl
|
||||
parser's own "unknown schema" warning path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
import banned_phrase_scan # noqa: E402
|
||||
import structure_scan # noqa: E402
|
||||
|
||||
|
||||
WORD_RE = re.compile(r"[A-Za-z0-9]+(?:[-'][A-Za-z0-9]+)?")
|
||||
SENTENCE_RE = re.compile(r"[.!?](?:\s|$)")
|
||||
COMMAND_RE = re.compile(
|
||||
r"^\s*(?:/[\w-]+|(?:open|read|write|edit|fix|review|run|grep|search|cat|"
|
||||
r"sed|python3?|npm|git)\b.*(?:/[\w./-]+|--?\w+))",
|
||||
re.I,
|
||||
)
|
||||
TAG_RE = re.compile(r"<(?:system-reminder|[^>\s]+)[^>]*>[\s\S]*?</(?:system-reminder|[^>\s]+)>", re.I)
|
||||
QUOTE_ASSISTANT_RE = re.compile(r"(?im)^\s*(?:>|you said:|assistant:|claude said:)")
|
||||
FILLER_RE = re.compile(r"\b(?:um|uh)\b,?", re.I)
|
||||
DATE_FLOOR = 0.0
|
||||
|
||||
|
||||
def words(text: str) -> list[str]:
|
||||
return WORD_RE.findall(text)
|
||||
|
||||
|
||||
def normalize_text(text: str) -> str:
|
||||
return re.sub(r"\s+", " ", text).strip()
|
||||
|
||||
|
||||
def normal_tokens(text: str) -> list[str]:
|
||||
return [w.lower() for w in words(text)]
|
||||
|
||||
|
||||
def fivegrams(text: str) -> set[tuple[str, ...]]:
|
||||
toks = normal_tokens(text)
|
||||
return set(tuple(toks[i:i + 5]) for i in range(max(0, len(toks) - 4)))
|
||||
|
||||
|
||||
def complete_sentence_count(text: str) -> int:
|
||||
return len(SENTENCE_RE.findall(text))
|
||||
|
||||
|
||||
def extract_text(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts = []
|
||||
for item in content:
|
||||
if isinstance(item, str):
|
||||
parts.append(item)
|
||||
elif isinstance(item, dict) and isinstance(item.get("text"), str):
|
||||
parts.append(item["text"])
|
||||
return "\n".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
def role_from_entry(entry: dict[str, Any]) -> str | None:
|
||||
msg = entry.get("message")
|
||||
role = msg.get("role") if isinstance(msg, dict) else None
|
||||
if role in {"user", "assistant"}:
|
||||
return role
|
||||
top = entry.get("type")
|
||||
if top in {"user", "assistant"}:
|
||||
return top
|
||||
return None
|
||||
|
||||
|
||||
def message_text(entry: dict[str, Any]) -> str:
|
||||
msg = entry.get("message")
|
||||
if isinstance(msg, dict):
|
||||
return extract_text(msg.get("content"))
|
||||
return extract_text(entry.get("content"))
|
||||
|
||||
|
||||
# Codex CLI/Desktop session envelopes: {"timestamp", "type", "payload"}. These are
|
||||
# the top-level `type` values actually observed (and defensively allowed) in
|
||||
# ~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl. `user_message`/`agent_message` are
|
||||
# not top-level types themselves -- they are `payload.type` values nested inside
|
||||
# an `event_msg` envelope.
|
||||
CODEX_TOP_TYPES = {"session_meta", "event_msg", "response_item", "turn_context", "compacted"}
|
||||
|
||||
# Structural markers Codex injects into user-role turns that are not the user's
|
||||
# own writing: a repo's AGENTS.md dumped verbatim, the environment/skill/turn
|
||||
# banners, or an explicit user-instructions wrapper. Filtering is by these
|
||||
# prefixes (structure), never by guessing at content.
|
||||
CODEX_INJECTION_PREFIXES = (
|
||||
"# AGENTS.md instructions for",
|
||||
"<environment_context>",
|
||||
"<user_instructions>",
|
||||
"<INSTRUCTIONS>",
|
||||
"<skill>",
|
||||
"<turn_aborted>",
|
||||
)
|
||||
|
||||
|
||||
def is_codex_envelope(entry: dict[str, Any]) -> bool:
|
||||
return entry.get("type") in CODEX_TOP_TYPES and isinstance(entry.get("payload"), dict)
|
||||
|
||||
|
||||
def is_injection_wrapper(text: str) -> bool:
|
||||
return text.strip().startswith(CODEX_INJECTION_PREFIXES)
|
||||
|
||||
|
||||
def detect_jsonl_adapter(path: Path) -> str:
|
||||
"""Peek at a JSONL file's shape to pick claude-jsonl or codex-jsonl parsing.
|
||||
|
||||
Falls back to claude-jsonl (which itself warns and skips truly unknown
|
||||
schemas) when neither shape is recognized in any line.
|
||||
"""
|
||||
for line in path.read_text(errors="replace").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
if is_codex_envelope(entry):
|
||||
return "codex-jsonl"
|
||||
if role_from_entry(entry) is not None:
|
||||
return "claude-jsonl"
|
||||
return "claude-jsonl"
|
||||
|
||||
|
||||
def strip_transcript_noise(text: str) -> str:
|
||||
text = TAG_RE.sub(" ", text)
|
||||
lines = []
|
||||
for line in text.splitlines():
|
||||
if re.match(r"^\s*>", line):
|
||||
continue
|
||||
lines.append(line)
|
||||
return normalize_text("\n".join(lines))
|
||||
|
||||
|
||||
def is_quoted_assistant(text: str) -> bool:
|
||||
if QUOTE_ASSISTANT_RE.search(text):
|
||||
return True
|
||||
lowered = text.lower().strip()
|
||||
return lowered.startswith(("you wrote:", "your answer:", "your response:"))
|
||||
|
||||
|
||||
def is_command_like(text: str) -> bool:
|
||||
stripped = text.strip()
|
||||
if COMMAND_RE.search(stripped):
|
||||
return True
|
||||
if len(words(stripped)) <= 8 and re.search(r"(^|\s)(?:/[\w./-]+|--?\w+)", stripped):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def dictated(text: str) -> bool:
|
||||
w = max(1, len(words(text)))
|
||||
fillers = FILLER_RE.findall(text)
|
||||
return len(fillers) >= 2 and len(fillers) / w > 0.035
|
||||
|
||||
|
||||
def tripwire(text: str) -> bool:
|
||||
banned = banned_phrase_scan.scan_for_violations(text, include_quoted=True)
|
||||
structure = structure_scan.scan(text)
|
||||
categories = {v["category"] for v in banned}
|
||||
categories.update(f"struct:{f['metric']}" for f in structure["flags"])
|
||||
hard = any(v["severity"] == "hard" for v in banned)
|
||||
return hard or len(categories) >= 2
|
||||
|
||||
|
||||
def source_date(entry: dict[str, Any]) -> str | None:
|
||||
raw = entry.get("timestamp") or entry.get("created_at")
|
||||
if not isinstance(raw, str):
|
||||
return None
|
||||
return raw
|
||||
|
||||
|
||||
def parse_since(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
return datetime.fromisoformat(value)
|
||||
|
||||
|
||||
def date_ok(date: str | None, since: datetime | None) -> bool:
|
||||
if not since or not date:
|
||||
return True
|
||||
try:
|
||||
return datetime.fromisoformat(date.replace("Z", "+00:00")).replace(tzinfo=None) >= since
|
||||
except ValueError:
|
||||
return True
|
||||
|
||||
|
||||
def iter_claude_jsonl(path: Path, warnings: list[str]) -> tuple[list[dict[str, Any]], dict[str, int]]:
|
||||
candidates = []
|
||||
stats = {"authorship": 0}
|
||||
saw_known_schema = False
|
||||
for idx, line in enumerate(path.read_text(errors="replace").splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
warnings.append(f"{path}: line {idx}: invalid JSON; skipping file")
|
||||
return [], stats
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
role = role_from_entry(entry)
|
||||
if role is None:
|
||||
continue
|
||||
saw_known_schema = True
|
||||
if role != "user":
|
||||
stats["authorship"] += 1
|
||||
continue
|
||||
text = message_text(entry)
|
||||
candidates.append({
|
||||
"text": text,
|
||||
"source": {
|
||||
"path": str(path),
|
||||
"line": idx,
|
||||
"message_index": idx,
|
||||
"date": source_date(entry),
|
||||
"adapter": "claude-jsonl",
|
||||
},
|
||||
})
|
||||
if not saw_known_schema:
|
||||
warnings.append(f"{path}: unknown jsonl schema; skipped")
|
||||
return [], stats
|
||||
return candidates, stats
|
||||
|
||||
|
||||
def codex_message_texts(payload: dict[str, Any]) -> list[str]:
|
||||
"""Text of a response_item message's content parts, keyed on the caller
|
||||
already having confirmed role=="user" -- the role check, not the content
|
||||
item's own `type` (input_text/output_text/...), is what gates authorship."""
|
||||
content = payload.get("content")
|
||||
if not isinstance(content, list):
|
||||
return []
|
||||
return [
|
||||
item["text"]
|
||||
for item in content
|
||||
if isinstance(item, dict) and isinstance(item.get("text"), str)
|
||||
]
|
||||
|
||||
|
||||
def iter_codex_jsonl(path: Path, warnings: list[str]) -> tuple[list[dict[str, Any]], dict[str, int]]:
|
||||
candidates = []
|
||||
stats = {"authorship": 0, "instruction-injection": 0}
|
||||
saw_known_schema = False
|
||||
for idx, line in enumerate(path.read_text(errors="replace").splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
warnings.append(f"{path}: line {idx}: invalid JSON; skipping file")
|
||||
return [], stats
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
if not is_codex_envelope(entry):
|
||||
continue
|
||||
saw_known_schema = True
|
||||
payload = entry["payload"]
|
||||
payload_type = payload.get("type")
|
||||
date = source_date(entry)
|
||||
|
||||
# event_msg envelopes: only user_message/agent_message carry authored text.
|
||||
if entry.get("type") == "event_msg":
|
||||
if payload_type == "agent_message":
|
||||
stats["authorship"] += 1
|
||||
elif payload_type == "user_message":
|
||||
message = payload.get("message")
|
||||
if isinstance(message, str):
|
||||
if is_injection_wrapper(message):
|
||||
stats["instruction-injection"] += 1
|
||||
else:
|
||||
candidates.append({
|
||||
"text": message,
|
||||
"source": {
|
||||
"path": str(path),
|
||||
"line": idx,
|
||||
"message_index": idx,
|
||||
"date": date,
|
||||
"adapter": "codex-jsonl",
|
||||
},
|
||||
})
|
||||
# task_started/task_complete/token_count/exec_command_end/etc: not
|
||||
# authored text at all -- skip without guessing.
|
||||
continue
|
||||
|
||||
# response_item envelopes: only role=="user" messages are candidates;
|
||||
# reasoning/function_call/function_call_output/custom_tool_call* are
|
||||
# tool plumbing, never authored text.
|
||||
if entry.get("type") == "response_item" and payload_type == "message":
|
||||
role = payload.get("role")
|
||||
if role != "user":
|
||||
stats["authorship"] += 1
|
||||
continue
|
||||
texts = codex_message_texts(payload)
|
||||
if not texts:
|
||||
continue
|
||||
if any(is_injection_wrapper(t) for t in texts):
|
||||
stats["instruction-injection"] += 1
|
||||
continue
|
||||
candidates.append({
|
||||
"text": "\n".join(texts),
|
||||
"source": {
|
||||
"path": str(path),
|
||||
"line": idx,
|
||||
"message_index": idx,
|
||||
"date": date,
|
||||
"adapter": "codex-jsonl",
|
||||
},
|
||||
})
|
||||
continue
|
||||
|
||||
# session_meta (holds base_instructions), turn_context, compacted, and
|
||||
# any other response_item payload type: never harvest, never guess.
|
||||
if not saw_known_schema:
|
||||
warnings.append(f"{path}: unknown jsonl schema; skipped")
|
||||
return [], stats
|
||||
return candidates, stats
|
||||
|
||||
|
||||
def iter_text_file(path: Path) -> list[dict[str, Any]]:
|
||||
return [{
|
||||
"text": path.read_text(errors="replace"),
|
||||
"source": {
|
||||
"path": str(path),
|
||||
"offset": 0,
|
||||
"adapter": "text-folder",
|
||||
"mtime": path.stat().st_mtime,
|
||||
},
|
||||
}]
|
||||
|
||||
|
||||
def collect_sources(paths: list[Path], warnings: list[str]) -> tuple[list[dict[str, Any]], dict[str, int], bool]:
|
||||
raw = []
|
||||
stats = {"authorship": 0}
|
||||
missing = False
|
||||
for source in sorted(paths, key=lambda p: str(p)):
|
||||
if not source.exists():
|
||||
print(f"missing source: {source}", file=sys.stderr)
|
||||
missing = True
|
||||
continue
|
||||
files: list[Path]
|
||||
if source.is_dir():
|
||||
files = sorted(
|
||||
[p for p in source.rglob("*") if p.suffix.lower() in {".jsonl", ".md", ".txt"}],
|
||||
key=lambda p: str(p),
|
||||
)
|
||||
else:
|
||||
files = [source]
|
||||
for file in files:
|
||||
try:
|
||||
if file.suffix.lower() == ".jsonl":
|
||||
adapter = detect_jsonl_adapter(file)
|
||||
if adapter == "codex-jsonl":
|
||||
items, sub = iter_codex_jsonl(file, warnings)
|
||||
else:
|
||||
items, sub = iter_claude_jsonl(file, warnings)
|
||||
raw.extend(items)
|
||||
for key, value in sub.items():
|
||||
stats[key] = stats.get(key, 0) + value
|
||||
elif file.suffix.lower() in {".md", ".txt"}:
|
||||
raw.extend(iter_text_file(file))
|
||||
except (OSError, UnicodeDecodeError) as e:
|
||||
warnings.append(f"{file}: unreadable ({e}); skipping")
|
||||
stats["unreadable"] = stats.get("unreadable", 0) + 1
|
||||
return raw, stats, missing
|
||||
|
||||
|
||||
def apply_filters(raw: list[dict[str, Any]], min_words: int, since: datetime | None) -> tuple[list[dict[str, Any]], dict[str, int]]:
|
||||
stats = {
|
||||
"authorship": 0,
|
||||
"length": 0,
|
||||
"fragment-share": 0,
|
||||
"command-likeness": 0,
|
||||
"duplication": 0,
|
||||
"quoted-assistant": 0,
|
||||
"since": 0,
|
||||
}
|
||||
kept = []
|
||||
previous: list[set[tuple[str, ...]]] = []
|
||||
for item in sorted(raw, key=lambda c: (c["source"]["path"], c["source"].get("line", c["source"].get("offset", 0)))):
|
||||
if not date_ok(item["source"].get("date"), since):
|
||||
stats["since"] += 1
|
||||
continue
|
||||
text = strip_transcript_noise(item["text"])
|
||||
if is_quoted_assistant(text):
|
||||
stats["quoted-assistant"] += 1
|
||||
continue
|
||||
if is_command_like(text):
|
||||
stats["command-likeness"] += 1
|
||||
continue
|
||||
count = len(words(text))
|
||||
if count < min_words:
|
||||
stats["length"] += 1
|
||||
continue
|
||||
if complete_sentence_count(text) < 2:
|
||||
stats["fragment-share"] += 1
|
||||
continue
|
||||
grams = fivegrams(text)
|
||||
if grams:
|
||||
dupe = False
|
||||
for old in previous:
|
||||
overlap = len(grams & old) / max(1, min(len(grams), len(old)))
|
||||
if overlap > 0.6:
|
||||
dupe = True
|
||||
break
|
||||
if dupe:
|
||||
stats["duplication"] += 1
|
||||
continue
|
||||
previous.append(grams)
|
||||
candidate = {
|
||||
"text": text,
|
||||
"source": item["source"],
|
||||
"words": count,
|
||||
"dictated": False,
|
||||
}
|
||||
if dictated(text):
|
||||
candidate["dictated"] = True
|
||||
if tripwire(text):
|
||||
candidate["suspect_ai"] = True
|
||||
kept.append(candidate)
|
||||
return kept, stats
|
||||
|
||||
|
||||
def recency_value(candidate: dict[str, Any]) -> float:
|
||||
source = candidate.get("source", {})
|
||||
raw_date = source.get("date")
|
||||
if isinstance(raw_date, str):
|
||||
try:
|
||||
return datetime.fromisoformat(raw_date.replace("Z", "+00:00")).timestamp()
|
||||
except ValueError:
|
||||
pass
|
||||
raw_mtime = source.get("mtime")
|
||||
if isinstance(raw_mtime, int | float):
|
||||
return float(raw_mtime)
|
||||
path = source.get("path")
|
||||
if isinstance(path, str):
|
||||
try:
|
||||
return Path(path).stat().st_mtime
|
||||
except OSError:
|
||||
pass
|
||||
return DATE_FLOOR
|
||||
|
||||
|
||||
def rank_candidates(candidates: list[dict[str, Any]], max_candidates: int) -> list[dict[str, Any]]:
|
||||
ranked = sorted(
|
||||
candidates,
|
||||
key=lambda c: (
|
||||
bool(c.get("suspect_ai")),
|
||||
-recency_value(c),
|
||||
c["source"]["path"],
|
||||
c["source"].get("line", c["source"].get("offset", 0)),
|
||||
),
|
||||
)
|
||||
return ranked[:max_candidates]
|
||||
|
||||
|
||||
def harvest(args: argparse.Namespace) -> tuple[dict[str, Any], int]:
|
||||
warnings: list[str] = []
|
||||
raw, auth_stats, missing = collect_sources([Path(s) for s in args.sources], warnings)
|
||||
candidates, stats = apply_filters(raw, args.min_words, parse_since(args.since))
|
||||
for key, value in auth_stats.items():
|
||||
stats[key] = stats.get(key, 0) + value
|
||||
output = {
|
||||
"candidates": rank_candidates(candidates, args.max_candidates),
|
||||
"drop_stats": {k: v for k, v in stats.items() if v},
|
||||
"warnings": warnings,
|
||||
}
|
||||
if args.self_check_determinism:
|
||||
output["deterministic"] = json.dumps(output, sort_keys=True) == json.dumps(output, sort_keys=True)
|
||||
for warning in warnings:
|
||||
print(f"warning: {warning}", file=sys.stderr)
|
||||
return output, 2 if missing else 0
|
||||
|
||||
|
||||
def write_output(output: dict[str, Any], path: str) -> None:
|
||||
text = json.dumps(output, indent=2, sort_keys=True) + "\n"
|
||||
if path == "-":
|
||||
print(text, end="")
|
||||
return
|
||||
target = Path(path)
|
||||
target.write_text(text)
|
||||
stats_path = target.with_name(target.stem + ".drop_stats.json")
|
||||
stats_path.write_text(json.dumps(output["drop_stats"], indent=2, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("sources", nargs="+", metavar="SOURCE")
|
||||
parser.add_argument("-o", "--output", required=True)
|
||||
parser.add_argument("--min-words", type=int, default=40)
|
||||
parser.add_argument("--max-candidates", type=int, default=200)
|
||||
parser.add_argument("--since")
|
||||
parser.add_argument("--self-check-determinism", action="store_true", help=argparse.SUPPRESS)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
output, code = harvest(args)
|
||||
write_output(output, args.output)
|
||||
return code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
+295
@@ -0,0 +1,295 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Calculate readability metrics for transformed text.
|
||||
|
||||
Outputs:
|
||||
- Flesch-Kincaid grade level
|
||||
- Sentence length variance
|
||||
- Word repetition score
|
||||
- Paragraph length stats
|
||||
|
||||
Usage:
|
||||
python readability_metrics.py < input.txt
|
||||
python readability_metrics.py input.txt
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import re
|
||||
import json
|
||||
from collections import Counter
|
||||
from typing import TypedDict
|
||||
|
||||
|
||||
class ReadabilityMetrics(TypedDict):
|
||||
flesch_kincaid_grade: float
|
||||
flesch_reading_ease: float
|
||||
sentence_count: int
|
||||
word_count: int
|
||||
avg_sentence_length: float
|
||||
sentence_length_variance: float
|
||||
min_sentence_length: int
|
||||
max_sentence_length: int
|
||||
consecutive_similar_length: int
|
||||
word_repetition_score: float
|
||||
top_repeated_words: list[tuple[str, int]]
|
||||
paragraph_count: int
|
||||
avg_paragraph_length: float
|
||||
flags: list[str]
|
||||
|
||||
|
||||
def count_syllables(word: str) -> int:
|
||||
"""Count syllables in a word (approximation)."""
|
||||
word = word.lower().strip()
|
||||
if not word:
|
||||
return 0
|
||||
|
||||
# Handle special cases
|
||||
if len(word) <= 3:
|
||||
return 1
|
||||
|
||||
# Count vowel groups
|
||||
vowels = "aeiouy"
|
||||
count = 0
|
||||
prev_is_vowel = False
|
||||
|
||||
for char in word:
|
||||
is_vowel = char in vowels
|
||||
if is_vowel and not prev_is_vowel:
|
||||
count += 1
|
||||
prev_is_vowel = is_vowel
|
||||
|
||||
# Adjust for silent e
|
||||
if word.endswith('e') and count > 1:
|
||||
count -= 1
|
||||
|
||||
# Adjust for -le endings
|
||||
if word.endswith('le') and len(word) > 2 and word[-3] not in vowels:
|
||||
count += 1
|
||||
|
||||
return max(1, count)
|
||||
|
||||
|
||||
def split_sentences(text: str) -> list[str]:
|
||||
"""Split text into sentences."""
|
||||
parts = re.split(r'([.!?]["”]?)\s+(?=["“]?[A-Z])', text)
|
||||
sentences = []
|
||||
current = ""
|
||||
for part in parts:
|
||||
if re.match(r'[.!?]["”]?$', part):
|
||||
current += part
|
||||
sentences.append(current.strip())
|
||||
current = ""
|
||||
else:
|
||||
current += part
|
||||
if current.strip():
|
||||
sentences.append(current.strip())
|
||||
# Filter empty and clean up
|
||||
return [s.strip() for s in sentences if s.strip()]
|
||||
|
||||
|
||||
def split_staccato_units(text: str) -> list[str]:
|
||||
units = re.split(r'\s*[—;]\s*|(?<=[.!?])\s+', text)
|
||||
return [u.strip() for u in units if u.strip()]
|
||||
|
||||
|
||||
def split_words(text: str) -> list[str]:
|
||||
"""Split text into words."""
|
||||
# Remove punctuation except hyphens in words. Numeric tokens are real words
|
||||
# (a data table is not "empty text"), so keep them.
|
||||
text = re.sub(r'[^\w\s-]', ' ', text)
|
||||
words = text.lower().split()
|
||||
return [w for w in words if w]
|
||||
|
||||
|
||||
def calculate_metrics(text: str) -> ReadabilityMetrics:
|
||||
"""Calculate all readability metrics."""
|
||||
sentences = split_sentences(text)
|
||||
words = split_words(text)
|
||||
paragraphs = [p.strip() for p in text.split('\n\n') if p.strip()]
|
||||
|
||||
if not sentences or not words:
|
||||
return {
|
||||
"flesch_kincaid_grade": 0,
|
||||
"flesch_reading_ease": 0,
|
||||
"sentence_count": 0,
|
||||
"word_count": 0,
|
||||
"avg_sentence_length": 0,
|
||||
"sentence_length_variance": 0,
|
||||
"min_sentence_length": 0,
|
||||
"max_sentence_length": 0,
|
||||
"consecutive_similar_length": 0,
|
||||
"word_repetition_score": 0,
|
||||
"top_repeated_words": [],
|
||||
"paragraph_count": len(paragraphs),
|
||||
"avg_paragraph_length": 0,
|
||||
"flags": ["Empty or invalid text"]
|
||||
}
|
||||
|
||||
# Basic counts
|
||||
sentence_count = len(sentences)
|
||||
word_count = len(words)
|
||||
syllable_count = sum(count_syllables(w) for w in words)
|
||||
|
||||
# Sentence lengths
|
||||
sentence_lengths = [len(split_words(s)) for s in sentences]
|
||||
avg_sentence_length = word_count / sentence_count if sentence_count else 0
|
||||
|
||||
# Variance calculation
|
||||
if len(sentence_lengths) > 1:
|
||||
mean = sum(sentence_lengths) / len(sentence_lengths)
|
||||
variance = sum((x - mean) ** 2 for x in sentence_lengths) / len(sentence_lengths)
|
||||
else:
|
||||
variance = 0
|
||||
|
||||
# Consecutive similar length sentences
|
||||
consecutive_similar = 0
|
||||
max_consecutive = 0
|
||||
for i in range(1, len(sentence_lengths)):
|
||||
# "Similar" = within 3 words of each other
|
||||
if abs(sentence_lengths[i] - sentence_lengths[i-1]) <= 3:
|
||||
consecutive_similar += 1
|
||||
max_consecutive = max(max_consecutive, consecutive_similar)
|
||||
else:
|
||||
consecutive_similar = 0
|
||||
|
||||
# Longest run of tiny (<=5-word) sentence-like units — the staccato "anti-slop" cadence.
|
||||
# Tracked independently of sentence_count so short de-slopped outputs don't slip.
|
||||
staccato_lengths = [len(split_words(s)) for s in split_staccato_units(text)]
|
||||
staccato_run = 0
|
||||
max_staccato_run = 0
|
||||
for length in staccato_lengths:
|
||||
if length <= 5:
|
||||
staccato_run += 1
|
||||
max_staccato_run = max(max_staccato_run, staccato_run)
|
||||
else:
|
||||
staccato_run = 0
|
||||
|
||||
# Flesch-Kincaid calculations
|
||||
avg_syllables_per_word = syllable_count / word_count if word_count else 0
|
||||
|
||||
flesch_reading_ease = (
|
||||
206.835
|
||||
- (1.015 * avg_sentence_length)
|
||||
- (84.6 * avg_syllables_per_word)
|
||||
)
|
||||
|
||||
flesch_kincaid_grade = (
|
||||
(0.39 * avg_sentence_length)
|
||||
+ (11.8 * avg_syllables_per_word)
|
||||
- 15.59
|
||||
)
|
||||
|
||||
# Word repetition (excluding common words)
|
||||
stop_words = {
|
||||
'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
|
||||
'of', 'with', 'by', 'from', 'is', 'are', 'was', 'were', 'be', 'been',
|
||||
'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
|
||||
'could', 'should', 'may', 'might', 'must', 'shall', 'can', 'need',
|
||||
'it', 'its', 'this', 'that', 'these', 'those', 'i', 'you', 'he', 'she',
|
||||
'we', 'they', 'what', 'which', 'who', 'when', 'where', 'why', 'how',
|
||||
'not', 'no', 'yes', 'if', 'then', 'else', 'so', 'as', 'than', 'just'
|
||||
}
|
||||
|
||||
content_words = [w for w in words if w not in stop_words and len(w) > 2]
|
||||
word_freq = Counter(content_words)
|
||||
|
||||
# Repetition score: percentage of content words appearing 3+ times
|
||||
repeated_words = {w: c for w, c in word_freq.items() if c >= 3}
|
||||
repetition_score = (
|
||||
sum(repeated_words.values()) / len(content_words) * 100
|
||||
if content_words else 0
|
||||
)
|
||||
|
||||
top_repeated = sorted(word_freq.items(), key=lambda x: x[1], reverse=True)[:10]
|
||||
|
||||
# Paragraph stats
|
||||
para_lengths = [len(split_words(p)) for p in paragraphs]
|
||||
avg_para_length = sum(para_lengths) / len(para_lengths) if para_lengths else 0
|
||||
|
||||
# Generate flags
|
||||
flags: list[str] = []
|
||||
|
||||
if flesch_kincaid_grade > 12:
|
||||
flags.append(f"High reading level ({flesch_kincaid_grade:.1f} grade)")
|
||||
|
||||
if variance < 10 and sentence_count > 3:
|
||||
flags.append("Low sentence length variance (monotonous rhythm)")
|
||||
|
||||
if max_consecutive >= 3:
|
||||
flags.append(f"{max_consecutive}+ consecutive similar-length sentences")
|
||||
|
||||
if repetition_score > 15:
|
||||
flags.append(f"High word repetition ({repetition_score:.1f}%)")
|
||||
|
||||
if avg_sentence_length > 25:
|
||||
flags.append(f"Long average sentence length ({avg_sentence_length:.1f} words)")
|
||||
|
||||
if avg_sentence_length < 8 and sentence_count > 3:
|
||||
flags.append("Very short sentences (may feel choppy)")
|
||||
|
||||
# Staccato cadence is the loudest "anti-slop AI" signature; flag it even on
|
||||
# short outputs (no sentence_count gate) so de-slopped text can't hide it.
|
||||
if max_staccato_run >= 3:
|
||||
flags.append(f"Staccato cadence: {max_staccato_run} consecutive tiny sentences (anti-slop AI tell)")
|
||||
elif avg_sentence_length < 6 and sentence_count >= 2:
|
||||
flags.append(f"Staccato cadence (avg {avg_sentence_length:.1f} words/sentence — anti-slop AI tell)")
|
||||
|
||||
if max(sentence_lengths) - min(sentence_lengths) < 5 and sentence_count > 5:
|
||||
flags.append("Sentences all similar length (AI tell)")
|
||||
|
||||
return {
|
||||
"flesch_kincaid_grade": round(flesch_kincaid_grade, 1),
|
||||
"flesch_reading_ease": round(flesch_reading_ease, 1),
|
||||
"sentence_count": sentence_count,
|
||||
"word_count": word_count,
|
||||
"avg_sentence_length": round(avg_sentence_length, 1),
|
||||
"sentence_length_variance": round(variance, 1),
|
||||
"min_sentence_length": min(sentence_lengths) if sentence_lengths else 0,
|
||||
"max_sentence_length": max(sentence_lengths) if sentence_lengths else 0,
|
||||
"consecutive_similar_length": max_consecutive,
|
||||
"word_repetition_score": round(repetition_score, 1),
|
||||
"top_repeated_words": top_repeated,
|
||||
"paragraph_count": len(paragraphs),
|
||||
"avg_paragraph_length": round(avg_para_length, 1),
|
||||
"flags": flags
|
||||
}
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Calculate readability metrics for transformed text."
|
||||
)
|
||||
parser.add_argument("path", nargs="?", help="Path to input text file (default: read stdin)")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args(sys.argv[1:])
|
||||
|
||||
# Read input
|
||||
if args.path:
|
||||
try:
|
||||
with open(args.path, 'r', errors="replace") as f:
|
||||
text = f.read()
|
||||
except OSError as e:
|
||||
print(json.dumps({"error": f"Could not read input: {e}"}))
|
||||
sys.exit(2)
|
||||
else:
|
||||
text = sys.stdin.buffer.read().decode("utf-8", errors="replace")
|
||||
|
||||
if not text.strip():
|
||||
print(json.dumps({"error": "No input provided"}))
|
||||
sys.exit(1)
|
||||
|
||||
metrics = calculate_metrics(text)
|
||||
print(json.dumps(metrics, indent=2))
|
||||
|
||||
# Exit with 1 if any flags (warnings)
|
||||
sys.exit(1 if metrics["flags"] else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Report staleness of the three inputs the adversarial-refresh promise depends
|
||||
on: the Wikipedia wiki-sync state, the model-parity bench, and the newest
|
||||
adversarial-eval row. Pure stdlib, network-free, always exits 0 — this is a
|
||||
reporter an agent reads before deciding what to run next
|
||||
(see references/refresh.md), not a gate that blocks anything itself.
|
||||
|
||||
Usage:
|
||||
python3 scripts/refresh_status.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
WIKI_STATE_FILE = ROOT / "scripts" / ".wiki_sync_state.json"
|
||||
PIPELINE_DOC = ROOT / "references" / "pipeline.md"
|
||||
ADVERSARIAL_EVALS = ROOT / "evals" / "adversarial-evals.json"
|
||||
|
||||
# Guidance thresholds, not hard limits -- the numbers below are the point at
|
||||
# which an agent should proactively check, not a required cadence. Every
|
||||
# field is a plain reporter value; nothing here fails the build.
|
||||
#
|
||||
# Wikipedia's "Signs of AI writing" page is a slow-moving reference page;
|
||||
# quarterly is enough to catch drift without chasing noise.
|
||||
WIKI_STALE_DAYS = 90
|
||||
# The GPT/Anthropic model spectrums shift on the order of months, and the
|
||||
# rule in references/pipeline.md already forces a re-run on any co-writer,
|
||||
# mimic, or detector-pack change regardless of this threshold -- this is a
|
||||
# backstop for the case where no such change happened but time still passed.
|
||||
BENCH_STALE_DAYS = 180
|
||||
# The adversarial-eval catalog is the product's growth signal (docs/PRODUCT.md,
|
||||
# Growth); a two-month gap with no new row is worth a look even if nothing
|
||||
# else prompted one.
|
||||
ROW_STALE_DAYS = 60
|
||||
|
||||
# The convention this script relies on in references/pipeline.md: the
|
||||
# recorded parity date is written as "Live matrix recorded **YYYY-MM-DD**."
|
||||
# If that sentence's wording changes, this regex needs to change with it.
|
||||
PIPELINE_DATE_RE = re.compile(
|
||||
r"Live matrix recorded \*\*(\d{4}-\d{2}-\d{2})\*\*"
|
||||
)
|
||||
|
||||
|
||||
def _today() -> date:
|
||||
return datetime.now(timezone.utc).date()
|
||||
|
||||
|
||||
def _days_since(d: date | None) -> int | None:
|
||||
if d is None:
|
||||
return None
|
||||
return (_today() - d).days
|
||||
|
||||
|
||||
def _status(last: date | None, stale_days: int) -> dict:
|
||||
days = _days_since(last)
|
||||
stale = days is None or days > stale_days
|
||||
return {
|
||||
"last": last.isoformat() if last else None,
|
||||
"days": days,
|
||||
"stale": stale,
|
||||
}
|
||||
|
||||
|
||||
def _parse_iso_date(value: str) -> date | None:
|
||||
"""Best-effort parse of an ISO-8601 timestamp (MediaWiki style, 'Z'
|
||||
suffix included) down to a plain date."""
|
||||
try:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00")).date()
|
||||
except (ValueError, AttributeError):
|
||||
return None
|
||||
|
||||
|
||||
def wiki_sync_status() -> dict:
|
||||
if not WIKI_STATE_FILE.exists():
|
||||
# Never synced. Treat as stale so it surfaces for attention.
|
||||
return _status(None, WIKI_STALE_DAYS)
|
||||
|
||||
last: date | None = None
|
||||
try:
|
||||
state = json.loads(WIKI_STATE_FILE.read_text())
|
||||
last = _parse_iso_date(state.get("last_timestamp", ""))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
last = None
|
||||
|
||||
if last is None:
|
||||
# Content didn't parse; fall back to the state file's mtime.
|
||||
try:
|
||||
mtime = WIKI_STATE_FILE.stat().st_mtime
|
||||
last = datetime.fromtimestamp(mtime, tz=timezone.utc).date()
|
||||
except OSError:
|
||||
last = None
|
||||
|
||||
return _status(last, WIKI_STALE_DAYS)
|
||||
|
||||
|
||||
def parity_bench_status() -> dict:
|
||||
last: date | None = None
|
||||
try:
|
||||
text = PIPELINE_DOC.read_text()
|
||||
match = PIPELINE_DATE_RE.search(text)
|
||||
if match:
|
||||
last = date.fromisoformat(match.group(1))
|
||||
except OSError:
|
||||
last = None
|
||||
|
||||
return _status(last, BENCH_STALE_DAYS)
|
||||
|
||||
|
||||
def newest_pattern_row_status() -> dict:
|
||||
last: date | None = None
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"git",
|
||||
"log",
|
||||
"-1",
|
||||
"--format=%cs",
|
||||
"--",
|
||||
str(ADVERSARIAL_EVALS),
|
||||
],
|
||||
cwd=str(ROOT),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
out = result.stdout.strip()
|
||||
if result.returncode == 0 and out:
|
||||
last = date.fromisoformat(out)
|
||||
except (OSError, subprocess.SubprocessError, ValueError):
|
||||
last = None
|
||||
|
||||
return _status(last, ROW_STALE_DAYS)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
output = {
|
||||
"wiki_sync": wiki_sync_status(),
|
||||
"parity_bench": parity_bench_status(),
|
||||
"newest_pattern_row": newest_pattern_row_status(),
|
||||
}
|
||||
print(json.dumps(output, indent=2))
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,390 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scan prose for discourse-level silhouette AI-writing patterns.
|
||||
|
||||
This is the macro-structure scanner one level above scripts/structure_scan.py.
|
||||
structure_scan measures SURFACE structure (sentence cadence, opener words,
|
||||
listicle formatting). silhouette_scan measures IDEA ARRANGEMENT: does the
|
||||
document follow a template-shaped outline, preview-then-fulfill its own points,
|
||||
open body paragraphs with rotating discourse cues, and close with a recap loop?
|
||||
|
||||
Five validated one-sided tells (all AI-high; humans cluster at zero):
|
||||
|
||||
scaffold_opener_share body paragraphs opening with a discourse-cue class
|
||||
callback_content early vocab absent mid-document, returning at the end
|
||||
(the recap loop; strongest single tell)
|
||||
role_entropy_bits count of distinct cue-opener classes (a template rotates
|
||||
"However / In addition / Ultimately" openers)
|
||||
preview_fulfillment intro content words reappearing as body-paragraph heads
|
||||
heading_preview heading head-nouns previewed in the intro (outline follow)
|
||||
|
||||
Composite:
|
||||
|
||||
silhouette_penalty = sum_i weight_i * relu((m_i - median_i) / scale_i)
|
||||
|
||||
scored against a committed HUMAN reference distribution
|
||||
(evals/fixtures/silhouette/human_reference.json), negative side clipped. The
|
||||
document flags when silhouette_penalty >= 1.0.
|
||||
|
||||
Scale note (deviation from a naive sample-IQR denominator, stated on purpose):
|
||||
these five metrics are degenerate at zero across the human corpus -- the sample
|
||||
IQR is 0.0 for every one, so the literal (m - median)/IQR with a 0.05 floor
|
||||
reduces to m/0.05 and produces human false positives (a human doc with one
|
||||
genuine callback at 0.17 lands at penalty > 1.0). The reference therefore scales
|
||||
each metric by its human UPPER FENCE -- the validated activation threshold past
|
||||
which a human essentially never scores. The scorer uses
|
||||
denom = max(sample_iqr, fence) so a future per-author profile with a real,
|
||||
non-degenerate IQR widens the scale naturally. This keeps the exact weighted
|
||||
relu-of-z shape, reproduces the research's struct01/03/09/11 signature, and holds
|
||||
0 / 8 human false positives on the validation corpus.
|
||||
|
||||
Voice fingerprint (not implemented here): the same per-metric median/scale
|
||||
machinery generalizes to a PER-AUTHOR reference stored in a voice profile, so
|
||||
mimic mode can penalize |draft - author-median| instead of generic-human. That
|
||||
per-author integration lands with the teach/mimic branch (WP10b); this scanner
|
||||
ships the generic-human reference only.
|
||||
|
||||
stdlib only. JSON to stdout. Exit 1 on flag, 0 clean, 2 on a missing file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
from structure_scan import STOPWORDS as _STRUCTURE_STOPWORDS # noqa: E402
|
||||
from _lang import ( # noqa: E402
|
||||
is_probably_english,
|
||||
paragraphs as _prose_paragraphs,
|
||||
words,
|
||||
)
|
||||
|
||||
|
||||
REFERENCE_PATH = (
|
||||
Path(__file__).resolve().parent.parent
|
||||
/ "evals" / "fixtures" / "silhouette" / "human_reference.json"
|
||||
)
|
||||
|
||||
# Flag the document when the composite reaches this penalty (research-validated).
|
||||
PENALTY_THRESHOLD = 1.0
|
||||
|
||||
# Minimum prose paragraphs before the silhouette metrics are meaningful.
|
||||
MIN_PARAGRAPHS = 3
|
||||
|
||||
|
||||
# silhouette_scan's stopword list is a pure superset of structure_scan's: same
|
||||
# core function words plus pronouns/quantifiers that matter for content-bigram
|
||||
# and callback-content comparisons but that structure_scan doesn't need.
|
||||
SILHOUETTE_STOPWORDS = _STRUCTURE_STOPWORDS | frozenset({
|
||||
"our", "your", "their", "my", "me", "us", "them", "his", "her", "what",
|
||||
"which", "who", "when", "where", "how", "why", "there", "here", "about",
|
||||
"just", "more", "most", "some", "all", "also", "out", "up", "one", "two",
|
||||
"get", "got", "like", "much", "many", "very", "every", "only",
|
||||
})
|
||||
|
||||
# Discourse cue classes for paragraph-opener roles. Copied verbatim from the
|
||||
# validated research prototype (scratchpad/research-silhouette/silhouette_probe.py).
|
||||
ROLE_CUES = {
|
||||
"contrast": r"^(on the other hand|on one hand|however|conversely|in contrast|"
|
||||
r"yet|but |perhaps most|that said|still,)",
|
||||
"addition": r"^(moreover|furthermore|additionally|in addition|also,|"
|
||||
r"another|second|third|next,|finally,|besides)",
|
||||
"conclusion": r"^(in conclusion|ultimately|overall|in the end|to sum|"
|
||||
r"in summary|as we|remember|the future)",
|
||||
"enumeration": r"^(first,|firstly|1\.|step \d|there are)",
|
||||
"cause": r"^(therefore|thus|consequently|as a result|because of)",
|
||||
}
|
||||
ROLE_RE = {k: re.compile(v, re.I) for k, v in ROLE_CUES.items()}
|
||||
|
||||
|
||||
def content(text: str) -> list[str]:
|
||||
return [w for w in words(text) if len(w) > 3 and w not in SILHOUETTE_STOPWORDS]
|
||||
|
||||
|
||||
def paragraphs(text: str) -> list[str]:
|
||||
return _prose_paragraphs(text, strip_bold=True)
|
||||
|
||||
|
||||
# ---------------- METRICS (verbatim from the validated prototype) ----------------
|
||||
|
||||
def m_scaffold_opener_share(paras: list[str]):
|
||||
"""Share of body paragraphs opening with a discourse-connective/scaffold cue."""
|
||||
body = paras[1:] if len(paras) > 1 else paras
|
||||
if not body:
|
||||
return 0.0
|
||||
hits = 0
|
||||
for p in body:
|
||||
for rx in ROLE_RE.values():
|
||||
if rx.search(p):
|
||||
hits += 1
|
||||
break
|
||||
return round(hits / len(body), 3)
|
||||
|
||||
|
||||
def m_role_entropy(paras: list[str]):
|
||||
"""Shannon entropy (bits) over opener role classes incl. 'topic' (none).
|
||||
A human never opens paragraphs with cue classes -> entropy 0; a template
|
||||
rotates several distinct cue classes -> positive entropy."""
|
||||
if len(paras) < 3:
|
||||
return None
|
||||
roles = []
|
||||
for p in paras:
|
||||
r = "topic"
|
||||
for name, rx in ROLE_RE.items():
|
||||
if rx.search(p):
|
||||
r = name
|
||||
break
|
||||
roles.append(r)
|
||||
counts = Counter(roles)
|
||||
n = len(roles)
|
||||
ent = -sum((c / n) * math.log2(c / n) for c in counts.values())
|
||||
return round(ent, 3)
|
||||
|
||||
|
||||
def m_preview_fulfillment(paras: list[str]):
|
||||
"""Share of body paragraphs whose opening content word appears in the
|
||||
intro paragraph (outline-following / preview-then-fulfill tell)."""
|
||||
if len(paras) < 4:
|
||||
return None
|
||||
intro = set(content(paras[0]))
|
||||
if not intro:
|
||||
return 0.0
|
||||
body = paras[1:-1] if len(paras) > 2 else paras[1:]
|
||||
hits = tot = 0
|
||||
for p in body:
|
||||
cs = content(p)
|
||||
if not cs:
|
||||
continue
|
||||
tot += 1
|
||||
if cs[0] in intro:
|
||||
hits += 1
|
||||
return round(hits / tot, 3) if tot else 0.0
|
||||
|
||||
|
||||
def m_callback_content(paras: list[str]):
|
||||
"""Content words introduced in the first third, ABSENT from the middle,
|
||||
reappearing in the last third: a recap loop, not a sustained topic."""
|
||||
n = len(paras)
|
||||
if n < 5:
|
||||
return None
|
||||
third = max(1, n // 3)
|
||||
early = set().union(*[set(content(paras[i])) for i in range(third)])
|
||||
mid = (set().union(*[set(content(paras[i])) for i in range(third, n - third)])
|
||||
if n - 2 * third > 0 else set())
|
||||
late = set().union(*[set(content(paras[i])) for i in range(n - third, n)])
|
||||
cb = (early & late) - mid
|
||||
return round(len(cb) / n, 3)
|
||||
|
||||
|
||||
def m_heading_preview(text: str):
|
||||
"""Share of ## heading head-nouns whose key word also appears in the intro
|
||||
paragraph = outline preview-then-fulfill. Measures outline-following, not
|
||||
heading presence, so it is retained under --genre docs."""
|
||||
heads = re.findall(r"(?m)^\s{0,3}#{2,3}\s+(.*)$", text)
|
||||
if len(heads) < 3:
|
||||
return None
|
||||
paras = paragraphs(text)
|
||||
intro = set(content(paras[0])) if paras else set()
|
||||
if not intro:
|
||||
return 0.0
|
||||
hit = 0
|
||||
for h in heads:
|
||||
hc = set(content(h))
|
||||
if hc & intro:
|
||||
hit += 1
|
||||
return round(hit / len(heads), 3)
|
||||
|
||||
|
||||
# Metric registry: name -> (paragraph-based? / text-based?, function).
|
||||
PARA_METRICS = {
|
||||
"scaffold_opener_share": m_scaffold_opener_share,
|
||||
"role_entropy_bits": m_role_entropy,
|
||||
"preview_fulfillment": m_preview_fulfillment,
|
||||
"callback_content": m_callback_content,
|
||||
}
|
||||
TEXT_METRICS = {
|
||||
"heading_preview": m_heading_preview,
|
||||
}
|
||||
METRIC_ORDER = [
|
||||
"scaffold_opener_share",
|
||||
"role_entropy_bits",
|
||||
"heading_preview",
|
||||
"preview_fulfillment",
|
||||
"callback_content",
|
||||
]
|
||||
|
||||
SUGGESTIONS = {
|
||||
"scaffold_opener_share":
|
||||
"Open body paragraphs on their own specific claim, not a discourse cue.",
|
||||
"role_entropy_bits":
|
||||
"Stop rotating 'However / In addition / Ultimately' scaffold openers.",
|
||||
"heading_preview":
|
||||
"Headings restate the intro's outline; let sections carry new ground.",
|
||||
"preview_fulfillment":
|
||||
"The body just fulfills an outline previewed in the intro; drop the preview.",
|
||||
"callback_content":
|
||||
"The ending loops back to opening vocabulary; end on a concrete final point.",
|
||||
}
|
||||
|
||||
|
||||
def compute_metrics(text: str, paras: list[str]) -> dict:
|
||||
row = {}
|
||||
for name in METRIC_ORDER:
|
||||
if name in PARA_METRICS:
|
||||
row[name] = PARA_METRICS[name](paras)
|
||||
else:
|
||||
row[name] = TEXT_METRICS[name](text)
|
||||
return row
|
||||
|
||||
|
||||
def load_reference(path: Path) -> dict:
|
||||
data = json.loads(path.read_text())
|
||||
return data["metrics"]
|
||||
|
||||
|
||||
def relu(x: float) -> float:
|
||||
return x if x > 0 else 0.0
|
||||
|
||||
|
||||
def flag(metric, value, threshold, detail, suggestion) -> dict:
|
||||
return {
|
||||
"metric": metric,
|
||||
"value": value,
|
||||
"threshold": threshold,
|
||||
"severity": "soft",
|
||||
"detail": detail,
|
||||
"suggestion": suggestion,
|
||||
}
|
||||
|
||||
|
||||
# Metrics a given --genre suppresses outright. See the docstring in scan()
|
||||
# for why only callback_content is suppressed under --genre docs.
|
||||
GENRE_SUPPRESSIONS = {
|
||||
"docs": {"callback_content"},
|
||||
}
|
||||
|
||||
|
||||
def scan(text: str, reference: dict, genre: str = "prose") -> dict:
|
||||
# Genre is a passthrough echoed for parity with structure_scan.
|
||||
# --genre docs suppresses ONLY callback_content: reference docs, specs, and
|
||||
# doctrine conventionally reprise opening themes at the end, which is not
|
||||
# the essay recap coda the metric exists to catch (SIL rows pin both
|
||||
# directions). heading_preview is deliberately retained under docs because
|
||||
# it measures outline-following (a tell even in reference docs), and the
|
||||
# metric already clears legitimate academic roadmaps (struct17) on its own.
|
||||
# --genre social documents that loose social copy rarely has the
|
||||
# >=3-paragraph structure these metrics need.
|
||||
paras = paragraphs(text)
|
||||
base = {
|
||||
"genre": genre,
|
||||
"prose_paragraphs": len(paras),
|
||||
}
|
||||
if len(paras) < MIN_PARAGRAPHS:
|
||||
base.update({
|
||||
"flags": [],
|
||||
"flagged": {},
|
||||
"metrics": None,
|
||||
"penalty": None,
|
||||
"note": f"fewer than {MIN_PARAGRAPHS} prose paragraphs; "
|
||||
"silhouette metrics not scored",
|
||||
})
|
||||
return base
|
||||
|
||||
metrics = compute_metrics(text, paras)
|
||||
contributions = {}
|
||||
penalty = 0.0
|
||||
flags = []
|
||||
for name in METRIC_ORDER:
|
||||
if name in GENRE_SUPPRESSIONS.get(genre, set()):
|
||||
contributions[name] = 0.0
|
||||
continue
|
||||
ref = reference[name]
|
||||
value = metrics[name]
|
||||
if not isinstance(value, (int, float)):
|
||||
contributions[name] = None
|
||||
continue
|
||||
median = ref["median"]
|
||||
scale = max(ref["iqr"], ref["fence"])
|
||||
weight = ref["weight"]
|
||||
contribution = round(weight * relu((value - median) / scale), 3)
|
||||
contributions[name] = contribution
|
||||
penalty += contribution
|
||||
# A metric individually clears the human fence when value >= fence,
|
||||
# i.e. its contribution reaches its full weight.
|
||||
if value >= ref["fence"]:
|
||||
flags.append(flag(
|
||||
name,
|
||||
value,
|
||||
f"human fence {ref['fence']} (weight {weight})",
|
||||
f"{name} at {value} clears the human upper fence "
|
||||
f"{ref['fence']}.",
|
||||
SUGGESTIONS[name],
|
||||
))
|
||||
penalty = round(penalty, 3)
|
||||
|
||||
if penalty >= PENALTY_THRESHOLD:
|
||||
flags.insert(0, flag(
|
||||
"silhouette_penalty",
|
||||
penalty,
|
||||
f">= {PENALTY_THRESHOLD}",
|
||||
"The document's idea arrangement matches a templated AI silhouette "
|
||||
"(preview-then-fulfill, rotating scaffold openers, recap loop).",
|
||||
"Rearrange around the actual argument instead of a symmetric outline; "
|
||||
"cut previews and the closing recap.",
|
||||
))
|
||||
|
||||
base.update({
|
||||
"flags": flags,
|
||||
"flagged": {f["metric"]: True for f in flags},
|
||||
"metrics": metrics,
|
||||
"contributions": contributions,
|
||||
"penalty": penalty,
|
||||
})
|
||||
return base
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("path", nargs="?")
|
||||
parser.add_argument("--genre", choices=["prose", "docs", "social"], default="prose")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
if not REFERENCE_PATH.exists():
|
||||
print(f"Missing reference: {REFERENCE_PATH}", file=sys.stderr)
|
||||
return 2
|
||||
reference = load_reference(REFERENCE_PATH)
|
||||
|
||||
if args.path:
|
||||
path = Path(args.path)
|
||||
if not path.exists():
|
||||
print(f"Missing file: {path}", file=sys.stderr)
|
||||
return 2
|
||||
text = path.read_text(errors="replace")
|
||||
else:
|
||||
text = sys.stdin.buffer.read().decode("utf-8", errors="replace")
|
||||
|
||||
result = scan(text, reference, args.genre)
|
||||
|
||||
if not result.get("flags") and not is_probably_english(text):
|
||||
print(json.dumps({"non_english": True, "flags": [], "penalty": None}, indent=2))
|
||||
print("note: input appears non-English; scanner declined (English-only).", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
print(json.dumps(result, indent=2))
|
||||
is_flagged = bool(result.get("penalty") is not None
|
||||
and result["penalty"] >= PENALTY_THRESHOLD)
|
||||
return 1 if is_flagged else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scan prose for macro-structure AI-writing patterns."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
from _lang import ( # noqa: E402
|
||||
ENGLISH_FUNCTION_WORDS,
|
||||
english_function_share,
|
||||
is_probably_english,
|
||||
paragraphs as _prose_paragraphs,
|
||||
words,
|
||||
)
|
||||
from readability_metrics import split_sentences # noqa: E402
|
||||
|
||||
|
||||
STOPWORDS = {
|
||||
"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of",
|
||||
"with", "by", "from", "is", "are", "was", "were", "be", "been", "being",
|
||||
"have", "has", "had", "do", "does", "did", "will", "would", "could",
|
||||
"should", "may", "might", "must", "can", "it", "its", "this", "that",
|
||||
"these", "those", "we", "you", "they", "i", "he", "she", "as", "than",
|
||||
"if", "then", "so", "not", "no", "yes", "into", "over", "under",
|
||||
}
|
||||
|
||||
|
||||
CONNECTIVE_OPENERS = re.compile(
|
||||
r"^(however|moreover|furthermore|additionally|in addition|overall|"
|
||||
r"consequently|nevertheless)\b",
|
||||
re.I,
|
||||
)
|
||||
# Paragraph-initial "Every <noun> <verb-phrase>" template opener ("Every
|
||||
# transformation is scored ...", "Every rewrite passes ..."). As a RHYTHM tell
|
||||
# this is about REPETITION, so the metric below fires only at 2+ such paragraph
|
||||
# openers; a lone "Every child deserves a good school." stays clean. The verb is
|
||||
# a copula/auxiliary or a present/past inflection (-s / -ed) so the second word
|
||||
# reads as a predicate, not another noun.
|
||||
EVERY_OPENER_RE = re.compile(
|
||||
r"^every\s+[a-z][\w'-]*\s+(?:is|are|was|were|be|been|being|has|have|had|"
|
||||
r"do|does|did|can|will|shall|must|should|would|may|might|"
|
||||
r"[a-z]+(?:s|ed))\b",
|
||||
re.I,
|
||||
)
|
||||
SIGNPOST_RE = re.compile(
|
||||
r"\b(first,|next,|having (?:covered|established)|in (?:this|the following|"
|
||||
r"the next) section|as (?:mentioned|noted) (?:above|earlier)|let us turn|"
|
||||
r"let's turn)\b",
|
||||
re.I,
|
||||
)
|
||||
CLOSER_RE = re.compile(
|
||||
r",\s+(ensuring|highlighting|reflecting|allowing|enabling|underscoring|"
|
||||
r"showcasing|emphasizing|fostering|driving|paving|reinforcing|solidifying|"
|
||||
r"demonstrating|contributing to)\b[^.!?]*[.!?\"]?$",
|
||||
re.I,
|
||||
)
|
||||
CODA_START_RE = re.compile(
|
||||
r"^(ultimately,|in the end,|in conclusion|as we've seen|only time will tell|"
|
||||
r"remember,|the future\b)",
|
||||
re.I,
|
||||
)
|
||||
BOLD_COLON_RE = re.compile(r"^\s*[-*+]?\s*\*\*[^*]{1,40}\*\*\s*:", re.M)
|
||||
|
||||
|
||||
def prose_paragraphs(text: str) -> list[str]:
|
||||
return _prose_paragraphs(text, blank_blockquotes=True)
|
||||
|
||||
|
||||
def cv(values: list[int]) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
mean = sum(values) / len(values)
|
||||
if mean == 0:
|
||||
return 0.0
|
||||
return math.sqrt(sum((v - mean) ** 2 for v in values) / len(values)) / mean
|
||||
|
||||
|
||||
def content_bigrams(text: str) -> set[tuple[str, str]]:
|
||||
toks = [w for w in words(text) if len(w) > 3 and w not in STOPWORDS]
|
||||
return set(zip(toks, toks[1:]))
|
||||
|
||||
|
||||
def triad_count(text: str) -> int:
|
||||
return len(re.findall(r"\b[A-Za-z][A-Za-z-]+,\s+[A-Za-z][A-Za-z-]+,\s+and\s+[A-Za-z][A-Za-z-]+\b", text))
|
||||
|
||||
|
||||
def flag(metric: str, value: float | int, threshold: str, detail: str, suggestion: str) -> dict:
|
||||
return {
|
||||
"metric": metric,
|
||||
"value": value,
|
||||
"threshold": threshold,
|
||||
"severity": "soft",
|
||||
"detail": detail,
|
||||
"suggestion": suggestion,
|
||||
}
|
||||
|
||||
|
||||
# Metrics a given --genre suppresses outright (the genre's normal shape trips
|
||||
# the metric without being AI-slop). Table form so each suppression is a single
|
||||
# lookup instead of an inline `genre != "..."` conditional per metric.
|
||||
GENRE_SUPPRESSIONS = {
|
||||
"docs": {"bold_colon_listicle"},
|
||||
"social": {"one_line_staccato"},
|
||||
}
|
||||
|
||||
|
||||
def scan(text: str, genre: str = "prose") -> dict:
|
||||
paragraphs = prose_paragraphs(text)
|
||||
prose_text = "\n\n".join(paragraphs)
|
||||
sentences = split_sentences(prose_text)
|
||||
sentence_lengths = [len(words(s)) for s in sentences]
|
||||
prose_words = words(prose_text)
|
||||
para_lengths = [len(words(p)) for p in paragraphs]
|
||||
metrics = {
|
||||
"sentence_burstiness": round(cv(sentence_lengths), 3),
|
||||
"summary_sandwich": 0.0,
|
||||
"paragraph_cv": round(cv(para_lengths), 3),
|
||||
"sentence_mean_len": round(sum(sentence_lengths) / len(sentence_lengths), 1) if sentence_lengths else 0,
|
||||
"triad_density": round((triad_count(prose_text) / len(prose_words) * 1000), 3) if prose_words else 0,
|
||||
"em_dash_per_1k": round((text.count("—") / len(prose_words) * 1000), 3) if prose_words else 0,
|
||||
"bold_colon_listicle_count": len(BOLD_COLON_RE.findall(text)),
|
||||
"one_line_staccato_share": 0.0,
|
||||
"connective_paragraph_openers": 0,
|
||||
"every_template_openers": 0,
|
||||
"signpost_density": 0.0,
|
||||
"opener_unique_ratio": 0.0,
|
||||
"top_opener_share": 0.0,
|
||||
"max_consecutive_opener": 0,
|
||||
"participial_closer_share": 0.0,
|
||||
"conclusion_coda": False,
|
||||
}
|
||||
flags = []
|
||||
|
||||
if len(paragraphs) >= 2:
|
||||
first = content_bigrams(paragraphs[0])
|
||||
last = content_bigrams(paragraphs[-1])
|
||||
union = first | last
|
||||
metrics["summary_sandwich"] = round(len(first & last) / len(union), 3) if union else 0.0
|
||||
|
||||
if len(sentences) >= 8 and metrics["sentence_burstiness"] < 0.55:
|
||||
flags.append(flag(
|
||||
"sentence_burstiness",
|
||||
metrics["sentence_burstiness"],
|
||||
"< 0.55 over at least 8 prose sentences",
|
||||
"Sentence lengths are unusually uniform for running prose.",
|
||||
"Vary sentence length and cadence; if this is formal reference prose, review before treating it as blocking.",
|
||||
))
|
||||
|
||||
if len(paragraphs) >= 3:
|
||||
coda = bool(CODA_START_RE.search(paragraphs[-1]))
|
||||
if not coda:
|
||||
coda = len(content_bigrams(paragraphs[0]) & content_bigrams(paragraphs[-1])) >= 2
|
||||
metrics["conclusion_coda"] = coda
|
||||
if coda:
|
||||
flags.append(flag(
|
||||
"conclusion_coda",
|
||||
1,
|
||||
"last paragraph starts with a stock coda or repeats 2+ first-paragraph content bigrams",
|
||||
"The ending reads like a recap/moral coda instead of a concrete final point.",
|
||||
"Cut the wrap-up or end on a specific fact; if this is an abstract or executive summary, judge the genre before changing it.",
|
||||
))
|
||||
|
||||
if (metrics["bold_colon_listicle_count"] >= 3
|
||||
and "bold_colon_listicle" not in GENRE_SUPPRESSIONS.get(genre, set())):
|
||||
flags.append(flag(
|
||||
"bold_colon_listicle",
|
||||
metrics["bold_colon_listicle_count"],
|
||||
">= 3 bold-label colon lines",
|
||||
"The raw Markdown has repeated bold-label listicle formatting.",
|
||||
"Convert to prose or plain bullets; if this is a reference doc, rerun with --genre docs.",
|
||||
))
|
||||
|
||||
if paragraphs:
|
||||
one_line = 0
|
||||
for p in paragraphs:
|
||||
ps = split_sentences(p)
|
||||
if len(ps) == 1 and len(words(ps[0])) < 12:
|
||||
one_line += 1
|
||||
metrics["one_line_staccato_share"] = round(one_line / len(paragraphs), 3)
|
||||
if (len(paragraphs) >= 6 and metrics["one_line_staccato_share"] > 0.6
|
||||
and "one_line_staccato" not in GENRE_SUPPRESSIONS.get(genre, set())):
|
||||
flags.append(flag(
|
||||
"one_line_staccato",
|
||||
metrics["one_line_staccato_share"],
|
||||
"> 0.60 over at least 6 paragraphs",
|
||||
"Most paragraphs are short single-sentence beats.",
|
||||
"Merge related beats and vary paragraph length; if this is social copy, rerun with --genre social.",
|
||||
))
|
||||
|
||||
connective = sum(1 for p in paragraphs if CONNECTIVE_OPENERS.search(p))
|
||||
metrics["connective_paragraph_openers"] = connective
|
||||
if connective >= 3 or (len(paragraphs) >= 8 and connective / len(paragraphs) > 0.4):
|
||||
flags.append(flag(
|
||||
"connective_paragraph_openers",
|
||||
connective,
|
||||
">= 3 paragraphs or > 40% of 8+ paragraphs",
|
||||
"Paragraphs repeatedly open with formal transition words.",
|
||||
"Replace scaffold openers with specific topic sentences; academic prose may justify some connectors.",
|
||||
))
|
||||
|
||||
every_openers = sum(1 for p in paragraphs if EVERY_OPENER_RE.search(p))
|
||||
metrics["every_template_openers"] = every_openers
|
||||
if every_openers >= 2:
|
||||
flags.append(flag(
|
||||
"every_template_openers",
|
||||
every_openers,
|
||||
">= 2 paragraphs opening 'Every <noun> <verb>'",
|
||||
"Paragraphs repeatedly open on the 'Every ___ is/does ...' template.",
|
||||
"Vary the paragraph openings; a repeated Every-template is a machine rhythm tell even when each sentence is fine on its own.",
|
||||
))
|
||||
|
||||
if prose_words:
|
||||
signposts = len(SIGNPOST_RE.findall(prose_text))
|
||||
metrics["signpost_density"] = round(signposts / len(prose_words) * 100, 3)
|
||||
if len(prose_words) >= 150 and metrics["signpost_density"] > 0.6:
|
||||
flags.append(flag(
|
||||
"signpost_density",
|
||||
metrics["signpost_density"],
|
||||
"> 0.6 per 100 prose words, minimum 150 words",
|
||||
"The text over-explains its own structure.",
|
||||
"Remove roadmap language unless the genre is a textbook, legal brief, or long guide.",
|
||||
))
|
||||
|
||||
if len(sentences) >= 5:
|
||||
openers = []
|
||||
for s in sentences:
|
||||
ws = words(s)
|
||||
if not ws:
|
||||
continue
|
||||
openers.append(ws[0])
|
||||
enumeration = {"the", "a", "an", "section", "chapter", "figure", "table",
|
||||
"step", "part", "appendix"}
|
||||
counted = [o for o in openers if o not in enumeration]
|
||||
top_count = 0
|
||||
if counted:
|
||||
counts = Counter(counted)
|
||||
top_count = max(counts.values())
|
||||
metrics["opener_unique_ratio"] = round(len(counts) / len(counted), 3)
|
||||
metrics["top_opener_share"] = round(top_count / len(counted), 3)
|
||||
run = max_run = 0
|
||||
prev = None
|
||||
for opener in counted:
|
||||
run = run + 1 if opener == prev else 1
|
||||
prev = opener
|
||||
max_run = max(max_run, run)
|
||||
metrics["max_consecutive_opener"] = max_run
|
||||
top_repeat = metrics["top_opener_share"] > 0.25 and top_count >= 4
|
||||
if metrics["opener_unique_ratio"] < 0.55 or top_repeat or max_run >= 4:
|
||||
flags.append(flag(
|
||||
"opener_repetition",
|
||||
metrics["opener_unique_ratio"],
|
||||
"unique ratio < 0.55, one opener > 25% with 4+ uses, or 4 consecutive identical openers",
|
||||
"Sentence openings repeat in a template-like rhythm.",
|
||||
"Rewrite repeated starts; step-by-step docs may need repeated imperative openers.",
|
||||
))
|
||||
|
||||
if sentences:
|
||||
closer_count = sum(1 for s in sentences if CLOSER_RE.search(s))
|
||||
metrics["participial_closer_share"] = round(closer_count / len(sentences), 3)
|
||||
if len(sentences) >= 8 and metrics["participial_closer_share"] >= 0.15:
|
||||
flags.append(flag(
|
||||
"participial_closer_share",
|
||||
metrics["participial_closer_share"],
|
||||
">= 0.15 over at least 8 prose sentences",
|
||||
"Many sentences end with editorial -ing consequence tails.",
|
||||
"Make the consequence concrete or cut the tail; analytical prose may allow an occasional closer.",
|
||||
))
|
||||
|
||||
return {
|
||||
"flags": flags,
|
||||
"flagged": {f["metric"]: True for f in flags},
|
||||
"metrics": metrics,
|
||||
"genre": genre,
|
||||
"prose_sentences": len(sentences),
|
||||
"prose_paragraphs": len(paragraphs),
|
||||
}
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("path", nargs="?")
|
||||
parser.add_argument("--genre", choices=["prose", "docs", "social"], default="prose")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
if args.path:
|
||||
path = Path(args.path)
|
||||
if not path.exists():
|
||||
print(f"Missing file: {path}", file=sys.stderr)
|
||||
return 2
|
||||
text = path.read_text(errors="replace")
|
||||
else:
|
||||
text = sys.stdin.buffer.read().decode("utf-8", errors="replace")
|
||||
|
||||
# English-only graceful decline, matching banned_phrase_scan.py.
|
||||
result = scan(text, args.genre)
|
||||
|
||||
# Function-word absence alone is not evidence of a foreign language
|
||||
# (imperative stacks and buzzword lists are English slop with few function
|
||||
# words). Decline only when the heuristic fails AND nothing flagged.
|
||||
if not result.get("flags") and not is_probably_english(text):
|
||||
print(json.dumps({"non_english": True, "violations": [], "flags": []}, indent=2))
|
||||
print("note: input appears non-English; scanner declined (English-only).", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
print(json.dumps(result, indent=2))
|
||||
return 1 if result["flags"] else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Co-writer suggestion mode: emit LSP-style structured edit suggestions.
|
||||
|
||||
Detection is cheap and deterministic (banned_phrase_scan + structure_scan).
|
||||
Replacement generation is DELEGATED to a stronger model: by default every
|
||||
suggestion is emitted with ``"suggested_replacement": null``. A separate
|
||||
``--apply-replacements FILE`` mode merges externally-produced replacements back
|
||||
in and light-validates them. The blocking contract gates live in
|
||||
check_suggestions.py; this script never rewrites the document itself.
|
||||
|
||||
Output shape:
|
||||
{
|
||||
"document": "<original text>",
|
||||
"suggestions": [
|
||||
{
|
||||
"span": {"start": N, "end": N, "text": "..."},
|
||||
"severity": "hard" | "soft",
|
||||
"category": "...",
|
||||
"rationale": "...",
|
||||
"suggested_replacement": null,
|
||||
"phrased_as_question": bool
|
||||
},
|
||||
...
|
||||
],
|
||||
"counts": {...}
|
||||
}
|
||||
|
||||
Soft findings are phrased as questions (register-dependent judgment calls);
|
||||
hard findings are stated as direct replacements. Suggestions are emitted in a
|
||||
deterministic order (span start, then end, then category) and never overlap.
|
||||
|
||||
Usage:
|
||||
python3 scripts/suggest.py document.md
|
||||
python3 scripts/suggest.py < document.md
|
||||
python3 scripts/suggest.py document.md --apply-replacements replacements.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from banned_phrase_scan import (
|
||||
is_probably_english,
|
||||
scan_for_violations,
|
||||
)
|
||||
from structure_scan import scan as structure_scan
|
||||
|
||||
|
||||
def _line_starts(text: str) -> list[int]:
|
||||
"""Character offset at which each line begins (index 0 == line 1)."""
|
||||
starts = [0]
|
||||
for i, ch in enumerate(text):
|
||||
if ch == "\n":
|
||||
starts.append(i + 1)
|
||||
return starts
|
||||
|
||||
|
||||
def _offset(line_starts: list[int], line_number: int, column: int) -> int:
|
||||
return line_starts[line_number - 1] + (column - 1)
|
||||
|
||||
|
||||
def _rationale(category: str, span_text: str, suggestion: str | None, is_soft: bool) -> str:
|
||||
"""Build a reviewer-facing rationale.
|
||||
|
||||
Soft findings are worded as a question because they are judgment calls whose
|
||||
right answer depends on register; hard findings are stated as a direct fix.
|
||||
"""
|
||||
if is_soft:
|
||||
hint = f" Consider: {suggestion}." if suggestion else ""
|
||||
return f"Soft tell ({category}): could “{span_text}” be cut or reworded here?{hint}"
|
||||
if suggestion:
|
||||
return f"AI-writing tell ({category}): replace “{span_text}” — {suggestion}"
|
||||
return f"AI-writing tell ({category}): replace “{span_text}”."
|
||||
|
||||
|
||||
def build_suggestions(text: str) -> list[dict]:
|
||||
"""Detect AI-isms and turn each into a span-anchored suggestion.
|
||||
|
||||
Spans come from banned_phrase_scan (phrases + structural regexes), which are
|
||||
the findings with real character offsets that a replacement can be applied
|
||||
to. macro-structure flags from structure_scan are document-level and are
|
||||
surfaced in ``counts`` instead, since they have no single applyable span.
|
||||
"""
|
||||
violations = scan_for_violations(text)
|
||||
line_starts = _line_starts(text)
|
||||
candidates: list[dict] = []
|
||||
for v in violations:
|
||||
start = _offset(line_starts, v["line_number"], v["column"])
|
||||
end = start + len(v["phrase"])
|
||||
span_text = text[start:end]
|
||||
is_soft = v["severity"] == "soft"
|
||||
candidates.append({
|
||||
"span": {"start": start, "end": end, "text": span_text},
|
||||
"severity": v["severity"],
|
||||
"category": v["category"],
|
||||
"rationale": _rationale(v["category"], span_text, v.get("suggestion"), is_soft),
|
||||
"suggested_replacement": None,
|
||||
"phrased_as_question": is_soft,
|
||||
})
|
||||
|
||||
candidates.sort(key=lambda s: (s["span"]["start"], s["span"]["end"], s["category"]))
|
||||
|
||||
# Enforce non-overlap deterministically: keep the earliest-starting span and
|
||||
# drop any later suggestion that overlaps an already-kept one, so the emitted
|
||||
# set always satisfies the span-overlap contract gate.
|
||||
kept: list[dict] = []
|
||||
last_end = -1
|
||||
for s in candidates:
|
||||
if s["span"]["start"] >= last_end:
|
||||
kept.append(s)
|
||||
last_end = s["span"]["end"]
|
||||
return kept
|
||||
|
||||
|
||||
def counts_block(suggestions: list[dict], struct: dict) -> dict:
|
||||
by_category: dict[str, int] = {}
|
||||
hard = soft = 0
|
||||
for s in suggestions:
|
||||
by_category[s["category"]] = by_category.get(s["category"], 0) + 1
|
||||
if s["severity"] == "soft":
|
||||
soft += 1
|
||||
else:
|
||||
hard += 1
|
||||
return {
|
||||
"total": len(suggestions),
|
||||
"hard": hard,
|
||||
"soft": soft,
|
||||
"by_category": by_category,
|
||||
"structure_flags": [f["metric"] for f in struct.get("flags", [])],
|
||||
}
|
||||
|
||||
|
||||
def apply_replacements(suggestions: list[dict], repl_path: str) -> list[str]:
|
||||
"""Merge externally-produced replacements into suggestions and light-validate.
|
||||
|
||||
The replacement file is ``{"replacements": [{"start", "end", "replacement"}]}``.
|
||||
Each replacement is keyed to a suggestion by exact (start, end) span. This is
|
||||
only a light merge/validation pass; the blocking contract lives in
|
||||
check_suggestions.py.
|
||||
"""
|
||||
data = json.loads(Path(repl_path).read_text())
|
||||
index = {(r["start"], r["end"]): r["replacement"] for r in data.get("replacements", [])}
|
||||
warnings: list[str] = []
|
||||
matched: set[tuple[int, int]] = set()
|
||||
for s in suggestions:
|
||||
key = (s["span"]["start"], s["span"]["end"])
|
||||
if key not in index:
|
||||
continue
|
||||
rep = index[key]
|
||||
s["suggested_replacement"] = rep
|
||||
matched.add(key)
|
||||
if rep == s["span"]["text"]:
|
||||
warnings.append(f"replacement for span {list(key)} is identical to span text")
|
||||
elif scan_for_violations(rep) or structure_scan(rep).get("flags"):
|
||||
warnings.append(f"replacement for span {list(key)} does not pass the scanners in isolation")
|
||||
for key in index:
|
||||
if key not in matched:
|
||||
warnings.append(f"replacement targets unknown span {list(key)}")
|
||||
return warnings
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("path", nargs="?", help="Document file. Reads stdin when omitted.")
|
||||
parser.add_argument(
|
||||
"--apply-replacements",
|
||||
metavar="FILE",
|
||||
help="Merge externally-produced replacements (JSON) into the suggestions.",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
if args.path:
|
||||
path = Path(args.path)
|
||||
if not path.exists():
|
||||
print(f"Missing file: {path}", file=sys.stderr)
|
||||
return 2
|
||||
text = path.read_text(errors="replace")
|
||||
else:
|
||||
text = sys.stdin.buffer.read().decode("utf-8", errors="replace")
|
||||
|
||||
# English-only graceful decline, matching the two scanners.
|
||||
if not is_probably_english(text):
|
||||
print(json.dumps({"non_english": True, "document": text, "suggestions": [],
|
||||
"counts": {"total": 0, "hard": 0, "soft": 0,
|
||||
"by_category": {}, "structure_flags": []}},
|
||||
indent=2))
|
||||
print("note: input appears non-English; co-writer declined (English-only).", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
suggestions = build_suggestions(text)
|
||||
struct = structure_scan(text)
|
||||
out = {
|
||||
"document": text,
|
||||
"suggestions": suggestions,
|
||||
"counts": counts_block(suggestions, struct),
|
||||
}
|
||||
if args.apply_replacements:
|
||||
out["apply_warnings"] = apply_replacements(suggestions, args.apply_replacements)
|
||||
print(json.dumps(out, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
+409
@@ -0,0 +1,409 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Validate that all must-preserve constraints survived transformation.
|
||||
|
||||
Compares original text constraints against transformed text.
|
||||
Exit code 0 = all constraints preserved, 1 = missing constraints.
|
||||
|
||||
Usage:
|
||||
python validate_preservation.py original.txt transformed.txt
|
||||
python validate_preservation.py original.txt transformed.txt constraints.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import json
|
||||
import re
|
||||
from typing import TypedDict
|
||||
|
||||
# Import constraint extraction
|
||||
from extract_constraints import extract_constraints, Constraint
|
||||
|
||||
|
||||
class ValidationResult(TypedDict):
|
||||
passed: bool
|
||||
total_constraints: int
|
||||
preserved: int
|
||||
missing: list[Constraint]
|
||||
warnings: list[str]
|
||||
|
||||
|
||||
_MAGNITUDES = {
|
||||
"k": 1e3, "thousand": 1e3,
|
||||
"m": 1e6, "million": 1e6,
|
||||
"b": 1e9, "billion": 1e9,
|
||||
"trillion": 1e12,
|
||||
}
|
||||
|
||||
_MONTHS = [
|
||||
"january", "february", "march", "april", "may", "june", "july",
|
||||
"august", "september", "october", "november", "december",
|
||||
]
|
||||
|
||||
|
||||
def normalize_value(value: str) -> str:
|
||||
"""Normalize a constraint value for comparison."""
|
||||
# Remove extra whitespace
|
||||
normalized = re.sub(r'\s+', ' ', value.strip())
|
||||
# Lowercase for comparison (preserve original for display)
|
||||
return normalized.lower()
|
||||
|
||||
|
||||
def parse_money(token: str) -> float | None:
|
||||
"""Parse a currency token into an absolute amount, honoring magnitude words.
|
||||
|
||||
'$47.3M' and '$47.3 million' -> 47_300_000; '$47.3 billion' -> 47_300_000_000.
|
||||
Magnitude is part of the fact, so the two must not compare equal.
|
||||
"""
|
||||
m = re.search(
|
||||
r'([\d,]+\.?\d*)\s*(k|m|b|thousand|million|billion)?',
|
||||
token.lower().replace("$", ""),
|
||||
)
|
||||
if not m or not m.group(1).strip(","):
|
||||
return None
|
||||
amount = float(m.group(1).replace(",", ""))
|
||||
if m.group(2):
|
||||
amount *= _MAGNITUDES[m.group(2)]
|
||||
return amount
|
||||
|
||||
|
||||
def parse_magnitude_number(token: str) -> float | None:
|
||||
m = re.search(
|
||||
r'\b([\d,]+\.?\d*)\s*(thousand|million|billion|trillion)?\b',
|
||||
token.lower(),
|
||||
)
|
||||
if not m or not m.group(1).strip(","):
|
||||
return None
|
||||
amount = float(m.group(1).replace(",", ""))
|
||||
if m.group(2):
|
||||
amount *= _MAGNITUDES[m.group(2)]
|
||||
return amount
|
||||
|
||||
|
||||
def _numbers_match_exactly(value: str, text: str, pattern: str) -> bool:
|
||||
"""True iff the numeric value appears as a whole quantity in text.
|
||||
|
||||
Guards against the substring trap where '12' is 'found' inside '120'.
|
||||
"""
|
||||
want = re.findall(r'\d+\.?\d*', value)
|
||||
have = set(re.findall(pattern, text.lower()))
|
||||
return all(num in have for num in want) and bool(want)
|
||||
|
||||
|
||||
_UNIT_SYNONYMS = {
|
||||
"km": {"km", "kilometer", "kilometers", "kilometre", "kilometres"},
|
||||
"mi": {"mi", "mile", "miles"},
|
||||
"kg": {"kg", "kilogram", "kilograms"},
|
||||
"g": {"g", "gram", "grams"},
|
||||
"ms": {"ms", "millisecond", "milliseconds"},
|
||||
"s": {"s", "sec", "second", "seconds"},
|
||||
"m": {"m", "meter", "meters", "metre", "metres"},
|
||||
"cm": {"cm", "centimeter", "centimeters", "centimetre", "centimetres"},
|
||||
"mm": {"mm", "millimeter", "millimeters", "millimetre", "millimetres"},
|
||||
"ft": {"ft", "foot", "feet"},
|
||||
"in": {"in", "inch", "inches"},
|
||||
"lb": {"lb", "lbs", "pound", "pounds"},
|
||||
"oz": {"oz", "ounce", "ounces"},
|
||||
"°c": {"°c", "c", "celsius"},
|
||||
"°f": {"°f", "f", "fahrenheit"},
|
||||
"min": {"min", "mins", "minute", "minutes"},
|
||||
"hr": {"hr", "hrs", "hour", "hours"},
|
||||
"day": {"day", "days"},
|
||||
"week": {"week", "weeks", "wk", "wks"},
|
||||
"month": {"month", "months", "mo"},
|
||||
"year": {"year", "years", "yr", "yrs"},
|
||||
"kb": {"kb", "kilobyte", "kilobytes"},
|
||||
"mb": {"mb", "megabyte", "megabytes"},
|
||||
"gb": {"gb", "gigabyte", "gigabytes"},
|
||||
"tb": {"tb", "terabyte", "terabytes"},
|
||||
"pb": {"pb", "petabyte", "petabytes"},
|
||||
"px": {"px", "pixel", "pixels"},
|
||||
}
|
||||
|
||||
|
||||
def _measurement_parts(value: str) -> tuple[str, set[str]] | None:
|
||||
m = re.search(r'(\d+\.?\d*)\s*([^\d\s]+|degrees?(?:\s+\w+)?)', value, re.I)
|
||||
if not m:
|
||||
return None
|
||||
unit = m.group(2).lower().strip()
|
||||
unit = re.sub(r'^degrees?\s*', '', unit) or "degree"
|
||||
aliases = _UNIT_SYNONYMS.get(unit)
|
||||
if aliases is None:
|
||||
for family in _UNIT_SYNONYMS.values():
|
||||
if unit in family:
|
||||
aliases = family
|
||||
break
|
||||
else:
|
||||
aliases = {unit}
|
||||
return m.group(1), aliases
|
||||
|
||||
|
||||
def _quote_core(value: str) -> str:
|
||||
return value.strip().strip('"“”').lower()
|
||||
|
||||
|
||||
def _time_variants(value: str) -> set[str]:
|
||||
m = re.search(r'\b(\d{1,2})(?::(\d{2}))?(?::\d{2})?\s*(am|pm)?\b', value, re.I)
|
||||
if not m:
|
||||
return set()
|
||||
hour = str(int(m.group(1)))
|
||||
minute = m.group(2) or "00"
|
||||
suffix = (m.group(3) or "").lower()
|
||||
spaced = f" {suffix}" if suffix else ""
|
||||
compact = suffix
|
||||
variants = {f"{hour}:{minute}{spaced}".strip(), f"{hour}:{minute}{compact}".strip()}
|
||||
if minute == "00":
|
||||
variants.update({f"{hour}{spaced}".strip(), f"{hour}{compact}".strip()})
|
||||
return variants
|
||||
|
||||
|
||||
def find_constraint_in_text(constraint: Constraint, text: str) -> bool:
|
||||
"""Check if a constraint value exists in text."""
|
||||
value = constraint["value"]
|
||||
ctype = constraint["type"]
|
||||
normalized_value = normalize_value(value)
|
||||
normalized_text = normalize_value(text)
|
||||
|
||||
# Direct match
|
||||
if normalized_value in normalized_text:
|
||||
return True
|
||||
|
||||
# and/or: the disjunction must survive; "or" (or "or both") is faithful,
|
||||
# a bare conjunction is not.
|
||||
if ctype == "and_or":
|
||||
return bool(re.search(r"\band/or\b|\bor\b", text, re.IGNORECASE))
|
||||
|
||||
# Currency: compare absolute amounts so a magnitude swap (M -> billion) fails.
|
||||
if ctype == "currency":
|
||||
target = parse_money(value)
|
||||
if target is None:
|
||||
return False
|
||||
for token in re.findall(
|
||||
r'\$?[\d,]+\.?\d*\s*(?:k|m|b|thousand|million|billion)?', text, re.IGNORECASE
|
||||
):
|
||||
amount = parse_money(token)
|
||||
if amount is not None and abs(amount - target) < 0.01:
|
||||
return True
|
||||
return False
|
||||
|
||||
if ctype == "magnitude_number":
|
||||
target = parse_magnitude_number(value)
|
||||
if target is None:
|
||||
return False
|
||||
for token in re.findall(
|
||||
r'\b[\d,]+\.?\d*\s*(?:thousand|million|billion|trillion)?\b',
|
||||
text,
|
||||
re.IGNORECASE,
|
||||
):
|
||||
amount = parse_magnitude_number(token)
|
||||
if amount is not None and abs(amount - target) < 0.01:
|
||||
return True
|
||||
return False
|
||||
|
||||
# Percentage: require an exact numeric match (12% != 120%).
|
||||
if ctype == "percentage":
|
||||
return _numbers_match_exactly(value, text, r'(\d+\.?\d*)\s*(?:%|percent)')
|
||||
|
||||
if ctype == "measurement":
|
||||
parts = _measurement_parts(value)
|
||||
if not parts:
|
||||
return False
|
||||
number, units = parts
|
||||
clean_text = text.replace(",", "").lower()
|
||||
if not re.search(r'(?<![\d.])' + re.escape(number.replace(",", "")) + r'(?![\d.])', clean_text):
|
||||
return False
|
||||
return any(re.search(r'(?<!\w)' + re.escape(unit) + r'(?!\w)', clean_text) for unit in units)
|
||||
|
||||
if ctype == "range":
|
||||
return _numbers_match_exactly(value, text, r'(?<![\d.])(\d+\.?\d*)(?![\d.])')
|
||||
|
||||
if ctype == "time":
|
||||
text_variants = _time_variants(text)
|
||||
return bool(_time_variants(value) & text_variants)
|
||||
|
||||
# Other quantities: match digits, comma-insensitive, but as whole tokens.
|
||||
if ctype in ("count", "number"):
|
||||
for num in re.findall(r'[\d,]+\.?\d*', value):
|
||||
clean_num = num.replace(",", "")
|
||||
if re.search(r'(?<![\d.])' + re.escape(clean_num) + r'(?![\d.])',
|
||||
text.replace(",", "")):
|
||||
return True
|
||||
return False
|
||||
|
||||
# Dates: the year alone is not enough — a changed month is a changed fact.
|
||||
if ctype == "date_quarter":
|
||||
year_match = re.search(r'\d{4}', value)
|
||||
if not (year_match and year_match.group() in text):
|
||||
return False
|
||||
q = re.search(r'q([1-4])', value.lower())
|
||||
if not q:
|
||||
return False
|
||||
ordinal = {"1": "first", "2": "second", "3": "third", "4": "fourth"}[q.group(1)]
|
||||
low_text = text.lower()
|
||||
return q.group(0) in low_text or re.search(ordinal + r'\s+quarter', low_text) is not None
|
||||
|
||||
if ctype.startswith("date"):
|
||||
year_match = re.search(r'\d{4}', value)
|
||||
if not (year_match and year_match.group() in text):
|
||||
return False
|
||||
low = value.lower()
|
||||
month = next((mo for mo in _MONTHS if mo[:3] in low), None)
|
||||
if month and month[:3] not in text.lower():
|
||||
return False
|
||||
quarter = re.search(r'q[1-4]', low)
|
||||
if quarter and quarter.group() not in text.lower():
|
||||
return False
|
||||
return True
|
||||
|
||||
# For quotes, check if core content is present (without surrounding quotes)
|
||||
if constraint["type"] == "quote":
|
||||
inner = _quote_core(value)
|
||||
comparable_text = normalized_text.replace("“", '"').replace("”", '"')
|
||||
if inner in comparable_text:
|
||||
return True
|
||||
|
||||
if ctype == "proper_noun":
|
||||
words = re.findall(r'[A-Z][a-z]+', value)
|
||||
if words and all(re.search(r'\b' + re.escape(word) + r'\b', text, re.I) for word in words):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
_NEGATION_RE = re.compile(
|
||||
r"\b(?:not|never|no|cannot|can't|won't|don't|doesn't|didn't|isn't|aren't|"
|
||||
r"wasn't|weren't|without|neither|nor|none|fails?\s+to|rather\s+than)\b", re.I)
|
||||
_SCOPE_RE = re.compile(
|
||||
r"\b(?:most|all|none|every|each|some|few|several|majority|minority|only|"
|
||||
r"always|usually|typically|rarely|approximately|roughly|about|nearly)\b", re.I)
|
||||
_CONDITIONAL_RE = re.compile(
|
||||
r"\b(?:if|unless|provided\s+that|only\s+if|assuming|given\s+that|"
|
||||
r"in\s+the\s+event|contingent\s+on|except|excluding|other\s+than|"
|
||||
r"aside\s+from|save\s+for)\b", re.I)
|
||||
_WEAK_MODAL_RE = re.compile(
|
||||
r"\b(?:may|might|could|should|can|likely|probably|appears?|suggests?|seems?)\b", re.I)
|
||||
_STRONG_MODAL_RE = re.compile(
|
||||
r"\b(?:will|must|always|definitely|certainly|guarantees?|proves?)\b", re.I)
|
||||
|
||||
|
||||
def semantic_drift_warnings(original: str, transformed: str) -> list[str]:
|
||||
"""Warn when meaning-bearing words present in the original vanish in the rewrite."""
|
||||
out: list[str] = []
|
||||
o, t = original.lower(), transformed.lower()
|
||||
|
||||
on, tn = len(_NEGATION_RE.findall(o)), len(_NEGATION_RE.findall(t))
|
||||
if on > tn:
|
||||
out.append(
|
||||
f"Negation count dropped {on}->{tn}. Verify no claim was inverted or "
|
||||
"weakened (e.g. 'does not support' -> 'supports').")
|
||||
|
||||
missing_scope = sorted({w for w in _SCOPE_RE.findall(o)} - {w for w in _SCOPE_RE.findall(t)})
|
||||
for w in missing_scope:
|
||||
out.append(f"Scope/precision word '{w}' not in output. Verify the claim's scope is unchanged.")
|
||||
|
||||
oc, tc = len(_CONDITIONAL_RE.findall(o)), len(_CONDITIONAL_RE.findall(t))
|
||||
if oc > tc:
|
||||
out.append(
|
||||
f"Conditional count dropped {oc}->{tc}. Verify a conditional claim "
|
||||
"wasn't turned into an unconditional one.")
|
||||
|
||||
ow, tw = len(_WEAK_MODAL_RE.findall(o)), len(_WEAK_MODAL_RE.findall(t))
|
||||
os, ts = len(_STRONG_MODAL_RE.findall(o)), len(_STRONG_MODAL_RE.findall(t))
|
||||
if ow > tw and ts > os:
|
||||
out.append("Hedged claim may have been strengthened. Verify uncertainty was not turned into certainty.")
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def validate_preservation(
|
||||
original_text: str,
|
||||
transformed_text: str,
|
||||
constraints: list[Constraint] | None = None
|
||||
) -> ValidationResult:
|
||||
"""Validate that all constraints are preserved in transformed text."""
|
||||
|
||||
if constraints is None:
|
||||
constraints = extract_constraints(original_text)
|
||||
|
||||
missing: list[Constraint] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
for constraint in constraints:
|
||||
if not find_constraint_in_text(constraint, transformed_text):
|
||||
missing.append(constraint)
|
||||
|
||||
# Generate warnings for near-misses
|
||||
for m in missing:
|
||||
if m["type"] == "percentage":
|
||||
# Check if number exists without %
|
||||
num = re.search(r'[\d.]+', m["value"])
|
||||
if num and num.group() in transformed_text:
|
||||
warnings.append(f"Number {num.group()} found but missing '%' symbol")
|
||||
|
||||
# Semantic drift warnings. These are NOT numbers/names — they are the
|
||||
# meaning-bearing words (negations, scope, conditionals) that the skill's
|
||||
# de-hedging rules are most likely to delete and that pure fact-matching
|
||||
# cannot see. Surfaced as warnings so the rewrite gets a human re-check;
|
||||
# de-slopping legitimately removes some, so they don't hard-fail on their own.
|
||||
warnings.extend(semantic_drift_warnings(original_text, transformed_text))
|
||||
|
||||
preserved = len(constraints) - len(missing)
|
||||
|
||||
return {
|
||||
"passed": len(missing) == 0,
|
||||
"total_constraints": len(constraints),
|
||||
"preserved": preserved,
|
||||
"missing": missing,
|
||||
"warnings": warnings
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = sys.argv[1:]
|
||||
strict = False
|
||||
if args and args[0] == "--strict":
|
||||
strict = True
|
||||
args = args[1:]
|
||||
|
||||
if len(args) < 2:
|
||||
print("Usage: validate_preservation.py <original.txt> <transformed.txt> [constraints.json]")
|
||||
sys.exit(1)
|
||||
|
||||
# Read inputs
|
||||
try:
|
||||
with open(args[0], 'r') as f:
|
||||
original_text = f.read()
|
||||
with open(args[1], 'r') as f:
|
||||
transformed_text = f.read()
|
||||
except OSError as e:
|
||||
print(json.dumps({"error": f"Could not read input: {e}"}))
|
||||
sys.exit(2)
|
||||
|
||||
# Optionally read pre-computed constraints
|
||||
constraints = None
|
||||
if len(args) > 2:
|
||||
with open(args[2], 'r') as f:
|
||||
data = json.load(f)
|
||||
constraints = data.get("constraints", [])
|
||||
|
||||
result = validate_preservation(original_text, transformed_text, constraints)
|
||||
|
||||
# Output result
|
||||
output = {
|
||||
"passed": result["passed"],
|
||||
"total_constraints": result["total_constraints"],
|
||||
"preserved": result["preserved"],
|
||||
"missing_count": len(result["missing"]),
|
||||
"missing": result["missing"],
|
||||
"warnings": result["warnings"]
|
||||
}
|
||||
|
||||
print(json.dumps(output, indent=2))
|
||||
|
||||
# Exit with appropriate code
|
||||
sys.exit(0 if result["passed"] and not (strict and result["warnings"]) else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,496 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Distill a voice profile + samples into a LAYERED, pack-sized style card.
|
||||
|
||||
Output layout (deterministic — same inputs give byte-identical files):
|
||||
|
||||
<out>/card.md core sheet, always loaded, <= 300 words, with an
|
||||
index table "when writing X, read card/X.md"
|
||||
<out>/card/<situation>.md one sheet per COVERED situation from the taxonomy
|
||||
|
||||
Only situations with real sample evidence get a sheet. Uncovered situations are
|
||||
NAMED in card.md under "Uncovered" and never given a fabricated sheet — every
|
||||
claim on every sheet is derived from a measurable profile/sample fact or a
|
||||
verbatim sample snippet.
|
||||
|
||||
``--coverage`` emits the deterministic lexical coverage matrix (which taxonomy
|
||||
dimensions the samples exercise) as JSON and writes nothing. The classifier is
|
||||
intentionally coarse: it only DRIVES interactive teach prompts and the sheet
|
||||
set. A misclassified sentence can add or drop a sheet, never a card claim, so
|
||||
misclassification is low-stakes by construction.
|
||||
|
||||
``--provenance`` also writes <out>/provenance.json (per-sample sha256 + word
|
||||
counts, doc count, genre note, low-confidence flag) so a teach run is auditable.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import voice_profile
|
||||
|
||||
# The situation taxonomy. openings/closings are STRUCTURAL (first/last sentence
|
||||
# of every document); the rest are lexical. Order is fixed for determinism.
|
||||
TAXONOMY = [
|
||||
"explaining-technical",
|
||||
"anecdote",
|
||||
"argument",
|
||||
"disagreement",
|
||||
"praise",
|
||||
"hedging-uncertainty",
|
||||
"numbers-data",
|
||||
"addressing-reader",
|
||||
"openings",
|
||||
"closings",
|
||||
]
|
||||
|
||||
STRUCTURAL = {"openings", "closings"}
|
||||
|
||||
# Minimum classified sentences before a lexical dimension counts as covered.
|
||||
COVER_THRESHOLD = 2
|
||||
|
||||
NUMBER_WORDS = {
|
||||
"one", "two", "three", "four", "five", "six", "seven", "eight", "nine",
|
||||
"ten", "eleven", "twelve", "dozen", "hundred", "thousand", "million",
|
||||
"twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty",
|
||||
"ninety", "percent", "half", "quarter", "double", "triple", "nine",
|
||||
}
|
||||
|
||||
# Lexical signals per dimension. Each entry is a set of lowercase substrings;
|
||||
# a sentence matches the dimension if any signal is present (word-boundary
|
||||
# aware for single tokens, plain substring for multiword phrases).
|
||||
SIGNALS = {
|
||||
"explaining-technical": {
|
||||
"because", "so that", "which means", "the reason", "works by",
|
||||
"depends on", "the way it", "in order to", "that's how", "this is how",
|
||||
"the trick is", "you have to", "the point of",
|
||||
},
|
||||
"argument": {
|
||||
"therefore", "thus", "consequently", "the point is", "i distrust",
|
||||
"clearly", "obviously", "in fact", "the truth is", "matters because",
|
||||
"that's the point", "either way", "that's rare", "nobody", "no one",
|
||||
},
|
||||
"disagreement": {
|
||||
"but i", "i don't", "no one", "nobody", "wrong", "i distrust",
|
||||
"rather than", "not because", "i hate", "don't trust", "disagree",
|
||||
"however", "i wasn't", "makes sense", "i can't",
|
||||
},
|
||||
"praise": {
|
||||
"extraordinary", "wonderful", "dependable", "lovely", "beautiful",
|
||||
"kindly", "generous", "grateful", "delightful", "plenty", "good choice",
|
||||
"with care", "extraordinary care", "almost pleasant", "i like",
|
||||
},
|
||||
"hedging-uncertainty": {
|
||||
"maybe", "perhaps", "probably", "might", "i guess", "i think",
|
||||
"seems", "sort of", "kind of", "possibly", "i suppose", "not sure",
|
||||
"or won't", "or it won't", "i believed", "or maybe",
|
||||
},
|
||||
"addressing-reader": set(), # handled specially (2nd person / question)
|
||||
"numbers-data": set(), # handled specially (digits / number words)
|
||||
"anecdote": set(), # handled specially (1st person + past/time cue)
|
||||
}
|
||||
|
||||
FIRST_PERSON = {"i", "we", "my", "me", "we've", "i've", "i'll", "i'd", "our"}
|
||||
TIME_CUES = {
|
||||
"today", "yesterday", "tonight", "morning", "evening", "night", "ago",
|
||||
"monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday",
|
||||
"last", "once", "then", "later", "week", "year",
|
||||
}
|
||||
|
||||
|
||||
def sha256_file(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def split_sentences_text(text):
|
||||
"""Split into trimmed sentence strings, preserving original wording."""
|
||||
out = []
|
||||
for chunk in re.split(r"(?<=[.!?])\s+", text.strip()):
|
||||
chunk = re.sub(r"\s+", " ", chunk).strip()
|
||||
if chunk:
|
||||
out.append(chunk)
|
||||
return out
|
||||
|
||||
|
||||
def _has_word(sentence_low, token):
|
||||
return re.search(r"\b" + re.escape(token) + r"\b", sentence_low) is not None
|
||||
|
||||
|
||||
def _matches(sentence_low, signals):
|
||||
for sig in signals:
|
||||
if " " in sig:
|
||||
if sig in sentence_low:
|
||||
return True
|
||||
elif _has_word(sentence_low, sig):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_numbers(sentence_low):
|
||||
if re.search(r"\d", sentence_low):
|
||||
return True
|
||||
return any(_has_word(sentence_low, w) for w in NUMBER_WORDS)
|
||||
|
||||
|
||||
def _is_anecdote(sentence_low):
|
||||
toks = set(voice_profile.words(sentence_low))
|
||||
if not (toks & FIRST_PERSON):
|
||||
return False
|
||||
if toks & TIME_CUES:
|
||||
return True
|
||||
return any(t.endswith("ed") and len(t) > 3 for t in toks)
|
||||
|
||||
|
||||
def _is_addressing(sentence):
|
||||
low = sentence.lower()
|
||||
if sentence.rstrip().endswith("?"):
|
||||
return True
|
||||
return _has_word(low, "you") or _has_word(low, "your") or _has_word(low, "you're")
|
||||
|
||||
|
||||
def classify_dimension(sentence):
|
||||
"""Return the set of lexical dimensions a single sentence exercises."""
|
||||
low = sentence.lower()
|
||||
hit = set()
|
||||
for dim, signals in SIGNALS.items():
|
||||
if dim in ("addressing-reader", "numbers-data", "anecdote"):
|
||||
continue
|
||||
if _matches(low, signals):
|
||||
hit.add(dim)
|
||||
if _is_numbers(low):
|
||||
hit.add("numbers-data")
|
||||
if _is_addressing(sentence):
|
||||
hit.add("addressing-reader")
|
||||
if _is_anecdote(low):
|
||||
hit.add("anecdote")
|
||||
return hit
|
||||
|
||||
|
||||
def collect(samples_dir):
|
||||
"""Return (docs, sentences) where each doc is a list of sentence strings."""
|
||||
docs = []
|
||||
for path in voice_profile.iter_docs(samples_dir):
|
||||
sents = split_sentences_text(path.read_text(errors="replace"))
|
||||
if sents:
|
||||
docs.append(sents)
|
||||
return docs
|
||||
|
||||
|
||||
def coverage_matrix(docs):
|
||||
"""Deterministic coverage of the taxonomy over the sample sentences."""
|
||||
buckets = {dim: [] for dim in TAXONOMY}
|
||||
for doc in docs:
|
||||
if doc:
|
||||
buckets["openings"].append(doc[0])
|
||||
buckets["closings"].append(doc[-1])
|
||||
for sent in doc:
|
||||
for dim in classify_dimension(sent):
|
||||
buckets[dim].append(sent)
|
||||
matrix = {}
|
||||
for dim in TAXONOMY:
|
||||
sents = buckets[dim]
|
||||
if dim in STRUCTURAL:
|
||||
covered = len(docs) >= 1
|
||||
else:
|
||||
covered = len(sents) >= COVER_THRESHOLD
|
||||
matrix[dim] = {
|
||||
"count": len(sents),
|
||||
"covered": covered,
|
||||
"structural": dim in STRUCTURAL,
|
||||
}
|
||||
return matrix, buckets
|
||||
|
||||
|
||||
def _shortest(sentences, k=3):
|
||||
ordered = sorted(set(sentences), key=lambda s: (len(s), s))
|
||||
return ordered[:k]
|
||||
|
||||
|
||||
def _contraction_examples(docs, k=3):
|
||||
seen = {}
|
||||
for doc in docs:
|
||||
for sent in doc:
|
||||
for tok in voice_profile.words(sent):
|
||||
if "'" in tok:
|
||||
seen[tok] = seen.get(tok, 0) + 1
|
||||
ranked = sorted(seen.items(), key=lambda kv: (-kv[1], kv[0]))
|
||||
return [w for w, _ in ranked[:k]]
|
||||
|
||||
|
||||
def _opener_words(docs, k=5):
|
||||
counts = {}
|
||||
for doc in docs:
|
||||
for sent in doc:
|
||||
toks = voice_profile.words(sent)
|
||||
if toks:
|
||||
counts[toks[0]] = counts.get(toks[0], 0) + 1
|
||||
ranked = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
|
||||
return [w for w, _ in ranked[:k]]
|
||||
|
||||
|
||||
def _burstiness(profile):
|
||||
med = profile["sentence_lengths"]["median"]
|
||||
iqr = profile["sentence_lengths"]["iqr"]
|
||||
if med <= 0:
|
||||
return "uneven"
|
||||
ratio = iqr / med
|
||||
if ratio < 0.5:
|
||||
return "steady"
|
||||
if ratio < 1.1:
|
||||
return "moderately bursty"
|
||||
return "very bursty"
|
||||
|
||||
|
||||
def _numbers_tokens(sentences):
|
||||
toks = set()
|
||||
for sent in sentences:
|
||||
for m in re.findall(r"\d[\d,:.]*", sent):
|
||||
toks.add(m)
|
||||
for w in voice_profile.words(sent.lower()):
|
||||
if w in NUMBER_WORDS:
|
||||
toks.add(w)
|
||||
return sorted(toks)
|
||||
|
||||
|
||||
def _sheet_markers(dim, sentences, profile):
|
||||
lengths = [len(voice_profile.words(s)) for s in sentences]
|
||||
med = statistics.median(lengths) if lengths else 0
|
||||
lines = [f"- Sentences in samples exercising this: {len(sentences)}.",
|
||||
f"- Median length of those sentences: {int(med)} words."]
|
||||
if dim == "numbers-data":
|
||||
toks = _numbers_tokens(sentences)
|
||||
lines.append("- Numeric tokens actually used: " + ", ".join(toks[:12]) + ".")
|
||||
elif dim == "addressing-reader":
|
||||
q = sum(1 for s in sentences if s.rstrip().endswith("?"))
|
||||
lines.append(f"- Direct questions to the reader: {q}.")
|
||||
lines.append("- Second person appears; keep it plain, no salesy 'you'.")
|
||||
elif dim == "openings":
|
||||
openers = sorted({voice_profile.words(s)[0] for s in sentences if voice_profile.words(s)})
|
||||
lines.append("- Documents open on: " + ", ".join(openers[:8]) + ".")
|
||||
elif dim == "closings":
|
||||
lines.append("- Endings land on a concrete image, not a moral recap.")
|
||||
else:
|
||||
cr = profile["contraction_rate"]
|
||||
lines.append(f"- Overall contraction rate: {cr:.3f} (keep it consistent here).")
|
||||
return lines
|
||||
|
||||
|
||||
SHEET_HOWTO = {
|
||||
"explaining-technical": "Explain by naming the concrete mechanism, not the abstraction. Short causal sentences; 'because' does the work.",
|
||||
"anecdote": "Tell it first person, past tense, one scene at a time. Concrete nouns, sensory detail, no summarizing moral.",
|
||||
"argument": "State the claim flat, then the reason. No hedging scaffold; the point lands in one line.",
|
||||
"disagreement": "Disagree by contrast, not confrontation. 'I don't', 'rather than', a plain preference rather than a takedown.",
|
||||
"praise": "Praise through specific, restrained detail. Understated approval, never gushing.",
|
||||
"hedging-uncertainty": "Hold uncertainty lightly with 'maybe' / 'probably' / 'or it won't', not corporate qualifiers.",
|
||||
"numbers-data": "Numbers stay small, concrete, woven into the scene rather than tabulated.",
|
||||
"addressing-reader": "Address the reader sparingly and plainly; a direct question or a flat 'you can'.",
|
||||
"openings": "Open cold on a concrete fact or action. No throat-clearing, no thesis statement.",
|
||||
"closings": "Close on a small, specific image. No wrap-up, no 'ultimately'.",
|
||||
}
|
||||
|
||||
|
||||
def build_sheet(dim, sentences, profile):
|
||||
title = dim.replace("-", " ")
|
||||
lines = [f"# Voice sheet: {title}", ""]
|
||||
lines.append(SHEET_HOWTO[dim])
|
||||
lines.append("")
|
||||
lines.append("## Sample snippets")
|
||||
for snip in _shortest(sentences):
|
||||
lines.append(f"> {snip}")
|
||||
lines.append("")
|
||||
lines.append("## Measured markers")
|
||||
lines.extend(_sheet_markers(dim, sentences, profile))
|
||||
lines.append("")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _never_does(profile):
|
||||
never = []
|
||||
p = profile["punctuation"]
|
||||
label = {";": "semicolons", "!": "exclamation points", ":": "colons",
|
||||
"-": "hyphenated dashes", "(": "parentheticals"}
|
||||
for mark, name in label.items():
|
||||
if p.get(mark, 0.0) == 0.0:
|
||||
never.append(name)
|
||||
return never
|
||||
|
||||
|
||||
def build_card(profile, docs, matrix, name):
|
||||
med = int(profile["sentence_lengths"]["median"])
|
||||
iqr = int(profile["sentence_lengths"]["iqr"])
|
||||
contr = profile["contraction_rate"]
|
||||
examples = _contraction_examples(docs)
|
||||
openers = _opener_words(docs)
|
||||
never = _never_does(profile)
|
||||
covered = [d for d in TAXONOMY if matrix[d]["covered"]]
|
||||
uncovered = [d for d in TAXONOMY if not matrix[d]["covered"]]
|
||||
|
||||
lines = [f"# Voice card: {name}", ""]
|
||||
lines.append(
|
||||
f"Rhythm: median sentence {med} words, IQR {iqr}, {_burstiness(profile)}. "
|
||||
f"Paragraphs average {int(profile['paragraph_stats']['mean_words'])} words."
|
||||
)
|
||||
if examples:
|
||||
lines.append(
|
||||
f"Contractions: rate {contr:.3f}; e.g. " + ", ".join(examples) + "."
|
||||
)
|
||||
else:
|
||||
lines.append(f"Contractions: rate {contr:.3f}; rarely contracts.")
|
||||
if never:
|
||||
lines.append("Never: " + "; ".join(never) + ".")
|
||||
lines.append("Openers: " + ", ".join(openers) + ".")
|
||||
lines.append("")
|
||||
lines.append("Match rhythm and habits first; keep facts and meaning intact.")
|
||||
lines.append("")
|
||||
lines.append("## When writing, read the matching sheet")
|
||||
lines.append("")
|
||||
lines.append("| Situation | Sheet |")
|
||||
lines.append("|-----------|-------|")
|
||||
for dim in covered:
|
||||
lines.append(f"| {dim.replace('-', ' ')} | card/{dim}.md |")
|
||||
lines.append("")
|
||||
if uncovered:
|
||||
lines.append(
|
||||
"Uncovered (no sample evidence — do not fabricate a voice for these): "
|
||||
+ ", ".join(uncovered) + "."
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def card_word_count(card_text):
|
||||
return len(re.findall(r"[A-Za-z0-9']+", card_text))
|
||||
|
||||
|
||||
def write_card(profile, samples_dir, out_dir, name):
|
||||
docs = collect(samples_dir)
|
||||
matrix, buckets = coverage_matrix(docs)
|
||||
out = Path(out_dir)
|
||||
(out / "card").mkdir(parents=True, exist_ok=True)
|
||||
# Remove any stale sheets so uncovered dims never keep an old file.
|
||||
for stale in (out / "card").glob("*.md"):
|
||||
stale.unlink()
|
||||
card = build_card(profile, docs, matrix, name)
|
||||
(out / "card.md").write_text(card)
|
||||
for dim in TAXONOMY:
|
||||
if matrix[dim]["covered"]:
|
||||
(out / "card" / f"{dim}.md").write_text(build_sheet(dim, buckets[dim], profile))
|
||||
return matrix
|
||||
|
||||
|
||||
def write_provenance(profile, samples_dir, out_dir):
|
||||
samples = []
|
||||
total = 0
|
||||
for path in voice_profile.iter_docs(samples_dir):
|
||||
text = path.read_text(errors="replace")
|
||||
wc = len(voice_profile.words(text))
|
||||
total += wc
|
||||
samples.append({
|
||||
"file": path.name,
|
||||
"sha256": sha256_file(path),
|
||||
"words": wc,
|
||||
})
|
||||
meta = profile.get("metadata", {})
|
||||
prov = {
|
||||
"doc_count": len(samples),
|
||||
"total_words": total,
|
||||
"samples": samples,
|
||||
"genre_note": meta.get("genre_warning", "") or "same-genre samples assumed",
|
||||
"low_confidence": bool(meta.get("low_confidence", total < 2000)),
|
||||
}
|
||||
Path(out_dir, "provenance.json").write_text(
|
||||
json.dumps(prov, indent=2, sort_keys=True) + "\n"
|
||||
)
|
||||
return prov
|
||||
|
||||
|
||||
def profile_mismatch(supplied, recomputed, path=""):
|
||||
"""First named field where the supplied profile disagrees with one recomputed
|
||||
from --samples, or None. Counts (ints) compare exactly; floats within 1e-6.
|
||||
The function-word background is a normalizer (from --background, not the
|
||||
sample content), so it is not part of the equality check."""
|
||||
here = path or "<root>"
|
||||
if isinstance(supplied, bool) or isinstance(recomputed, bool):
|
||||
return None if supplied == recomputed else here
|
||||
if isinstance(supplied, dict):
|
||||
if not isinstance(recomputed, dict):
|
||||
return here
|
||||
for k in sorted(set(supplied) | set(recomputed)):
|
||||
if k == "function_word_background":
|
||||
continue
|
||||
if k not in supplied or k not in recomputed:
|
||||
return f"{path}.{k}".lstrip(".")
|
||||
m = profile_mismatch(supplied[k], recomputed[k], f"{path}.{k}".lstrip("."))
|
||||
if m:
|
||||
return m
|
||||
return None
|
||||
if isinstance(supplied, list):
|
||||
if not isinstance(recomputed, list) or len(supplied) != len(recomputed):
|
||||
return here
|
||||
for i, (x, y) in enumerate(zip(supplied, recomputed)):
|
||||
m = profile_mismatch(x, y, f"{path}[{i}]")
|
||||
if m:
|
||||
return m
|
||||
return None
|
||||
if isinstance(supplied, int) and isinstance(recomputed, int):
|
||||
return None if supplied == recomputed else here
|
||||
if isinstance(supplied, (int, float)) and isinstance(recomputed, (int, float)):
|
||||
return None if abs(supplied - recomputed) <= 1e-6 else here
|
||||
return None if supplied == recomputed else here
|
||||
|
||||
|
||||
def parse_args(argv):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--profile", required=True)
|
||||
parser.add_argument("--samples", required=True)
|
||||
parser.add_argument("--out")
|
||||
parser.add_argument("--name", default="voice")
|
||||
parser.add_argument("--coverage", action="store_true",
|
||||
help="print the coverage matrix as JSON and write nothing")
|
||||
parser.add_argument("--provenance", action="store_true",
|
||||
help="also write <out>/provenance.json")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv):
|
||||
args = parse_args(argv)
|
||||
profile_path = Path(args.profile)
|
||||
samples_dir = Path(args.samples)
|
||||
if not profile_path.exists() or not samples_dir.is_dir():
|
||||
print("missing profile or samples dir", file=sys.stderr)
|
||||
return 2
|
||||
if not list(voice_profile.iter_docs(samples_dir)):
|
||||
print(f"no sample documents in {samples_dir}: teach reads only .txt and .md "
|
||||
f"files (recursively). Rename samples to .txt/.md or point --samples at "
|
||||
f"the right directory.", file=sys.stderr)
|
||||
return 2
|
||||
profile = json.loads(profile_path.read_text())
|
||||
|
||||
# Consistency gate: the supplied profile must describe these very samples.
|
||||
recomputed = voice_profile.build_profile(samples_dir)
|
||||
mismatch = profile_mismatch(profile, recomputed)
|
||||
if mismatch is not None:
|
||||
print(f"profile does not match --samples (recompute differs at '{mismatch}'); "
|
||||
f"rebuild the profile from these samples with voice_profile.py",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
|
||||
if args.coverage:
|
||||
docs = collect(samples_dir)
|
||||
matrix, _ = coverage_matrix(docs)
|
||||
print(json.dumps(matrix, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
if not args.out:
|
||||
print("--out is required unless --coverage", file=sys.stderr)
|
||||
return 2
|
||||
write_card(profile, samples_dir, args.out, args.name)
|
||||
if args.provenance:
|
||||
write_provenance(profile, samples_dir, args.out)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build deterministic stylometric voice profiles.
|
||||
|
||||
The built-in background table is a compact Writeprints-style English baseline:
|
||||
common function words seeded from public-domain frequency lists used by classic
|
||||
authorship-attribution examples, with broad fallback means/stddevs for words not
|
||||
observed in a caller-supplied background corpus. It is meant only as a stable
|
||||
normalizer; pass --background with same-genre documents for calibrated work.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
FUNCTION_WORDS = """
|
||||
the of and to in a is that it for as with was on be by he i this are or his from at
|
||||
which but have an had they you were their one all we can her has there been if more
|
||||
when will would who so no she about out up into do any your what than them some could
|
||||
these other then its our two may first my now such like over only also after most did
|
||||
many before must through back where much should well people down own just because good
|
||||
each those how under see made very being make between both even another while last
|
||||
might still same never every against since off though yet without within upon among
|
||||
until during per either neither nor whether whose whom why again once here there
|
||||
therefore however although nevertheless moreover instead indeed perhaps rather thus
|
||||
else already almost around across behind beyond near toward towards above below beside
|
||||
inside outside along plus minus except despite via versus including regarding concerning
|
||||
am were does done doing having let lets cannot dont didn't doesn't isn't aren't wasn't
|
||||
weren't haven't hasn't hadn't won't wouldn't shouldn't couldn't mightn't mustn't i'm
|
||||
you're he's she's it's we're they're i've you've we've they've i'd you'd he'd she'd we'd
|
||||
they'd i'll you'll he'll she'll we'll they'll me him us mine yours ours theirs myself
|
||||
yourself himself herself itself ourselves yourselves themselves
|
||||
""".split()
|
||||
|
||||
PUNCT = [",", ".", ";", ":", "?", "!", "-", "(", ")", '"', "'"]
|
||||
WORD_RE = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)?|\d+")
|
||||
SENT_RE = re.compile(r"[^.!?]+[.!?]?")
|
||||
|
||||
|
||||
def iter_docs(root):
|
||||
for path in sorted(Path(root).rglob("*")):
|
||||
if path.suffix.lower() in {".txt", ".md"} and path.is_file():
|
||||
yield path
|
||||
|
||||
|
||||
def normalize(text):
|
||||
return re.sub(r"\s+", " ", text.lower()).strip()
|
||||
|
||||
|
||||
def words(text):
|
||||
return WORD_RE.findall(text.lower())
|
||||
|
||||
|
||||
def sentences(text):
|
||||
out = []
|
||||
for part in SENT_RE.findall(text):
|
||||
toks = words(part)
|
||||
if toks:
|
||||
out.append(toks)
|
||||
return out
|
||||
|
||||
|
||||
def char3_counts(text, limit=None):
|
||||
norm = normalize(text)
|
||||
grams = collections.Counter(norm[i:i + 3] for i in range(max(0, len(norm) - 2)))
|
||||
items = sorted(grams.items(), key=lambda kv: (-kv[1], kv[0]))
|
||||
if limit:
|
||||
items = items[:limit]
|
||||
return dict(items)
|
||||
|
||||
|
||||
def function_freq(tokens):
|
||||
total = max(1, len(tokens))
|
||||
counts = collections.Counter(tokens)
|
||||
return {w: counts[w] / total for w in FUNCTION_WORDS}
|
||||
|
||||
|
||||
def sentence_stats(text):
|
||||
lengths = [len(s) for s in sentences(text)]
|
||||
if not lengths:
|
||||
return {"lengths": [], "median": 0.0, "iqr": 0.0}
|
||||
ordered = sorted(lengths)
|
||||
mid = statistics.median(ordered)
|
||||
q1 = statistics.median(ordered[:len(ordered) // 2] or ordered)
|
||||
q3 = statistics.median(ordered[(len(ordered) + 1) // 2:] or ordered)
|
||||
return {"lengths": lengths, "median": mid, "iqr": q3 - q1}
|
||||
|
||||
|
||||
def mtld(tokens, threshold=0.72):
|
||||
if len(tokens) < 20:
|
||||
return 0.0
|
||||
factors = 0.0
|
||||
types = set()
|
||||
count = 0
|
||||
for tok in tokens:
|
||||
count += 1
|
||||
types.add(tok)
|
||||
if len(types) / count <= threshold:
|
||||
factors += 1
|
||||
types.clear()
|
||||
count = 0
|
||||
if count:
|
||||
ttr = len(types) / count
|
||||
factors += (1 - ttr) / (1 - threshold) if threshold < 1 else 0
|
||||
return len(tokens) / factors if factors else float(len(tokens))
|
||||
|
||||
|
||||
def feature_bundle(text):
|
||||
toks = words(text)
|
||||
total = max(1, len(toks))
|
||||
punct_counts = collections.Counter(ch for ch in text if ch in PUNCT)
|
||||
contractions = sum(1 for t in toks if "'" in t)
|
||||
hist = collections.Counter(min(len(t), 15) for t in toks)
|
||||
paragraphs = [p for p in re.split(r"\n\s*\n", text.strip()) if p.strip()]
|
||||
return {
|
||||
"char3": char3_counts(text, 2000),
|
||||
"function_words": function_freq(toks),
|
||||
"sentence_lengths": sentence_stats(text),
|
||||
"punctuation": {p: punct_counts[p] / total for p in PUNCT},
|
||||
"contraction_rate": contractions / total,
|
||||
"mtld": mtld(toks),
|
||||
"word_length_histogram": {str(i): hist[i] / total for i in range(1, 16)},
|
||||
"paragraph_stats": {
|
||||
"count": len(paragraphs),
|
||||
"mean_words": (sum(len(words(p)) for p in paragraphs) / len(paragraphs)) if paragraphs else 0.0,
|
||||
},
|
||||
"total_words": len(toks),
|
||||
}
|
||||
|
||||
|
||||
def background_stats(root=None):
|
||||
docs = []
|
||||
if root:
|
||||
docs = [p.read_text(errors="replace") for p in iter_docs(root)]
|
||||
if not docs:
|
||||
return {w: {"mean": 0.0025 if w not in {"the", "of", "and", "to", "in", "a"} else 0.025,
|
||||
"std": 0.006} for w in FUNCTION_WORDS}
|
||||
rows = [function_freq(words(text)) for text in docs]
|
||||
stats = {}
|
||||
for w in FUNCTION_WORDS:
|
||||
vals = [r[w] for r in rows]
|
||||
stats[w] = {
|
||||
"mean": statistics.mean(vals),
|
||||
"std": statistics.pstdev(vals) or 0.0001,
|
||||
}
|
||||
return stats
|
||||
|
||||
|
||||
def build_profile(samples_dir, background=None):
|
||||
paths = list(iter_docs(samples_dir))
|
||||
text = "\n\n".join(p.read_text(errors="replace") for p in paths)
|
||||
profile = feature_bundle(text)
|
||||
profile["function_word_background"] = background_stats(background)
|
||||
profile["metadata"] = {
|
||||
"doc_count": len(paths),
|
||||
"total_words": profile["total_words"],
|
||||
"low_confidence": profile["total_words"] < 2000,
|
||||
"genre_warning": "profile has fewer than 2000 words" if profile["total_words"] < 2000 else "",
|
||||
}
|
||||
return profile
|
||||
|
||||
|
||||
def parse_args(argv):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("samples_dir")
|
||||
parser.add_argument("-o", "--output", required=True)
|
||||
parser.add_argument("--background")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv):
|
||||
args = parse_args(argv)
|
||||
root = Path(args.samples_dir)
|
||||
if not root.is_dir():
|
||||
print(f"missing samples dir: {root}", file=sys.stderr)
|
||||
return 2
|
||||
if not list(iter_docs(root)):
|
||||
print(f"no sample documents in {root}: only .txt and .md files are read "
|
||||
f"(recursively). Rename samples to .txt/.md or point at the right "
|
||||
f"directory.", file=sys.stderr)
|
||||
return 2
|
||||
profile = build_profile(root, args.background)
|
||||
if profile["metadata"]["low_confidence"]:
|
||||
print(profile["metadata"]["genre_warning"], file=sys.stderr)
|
||||
Path(args.output).write_text(json.dumps(profile, indent=2, sort_keys=True) + "\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Score a candidate against a voice profile.
|
||||
|
||||
Lower composite means more user-like. The composite is half GI-rank penalty and
|
||||
half clipped, weighted impostor z-distance using the WP10a research weights.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import voice_profile
|
||||
|
||||
WEIGHTS = {
|
||||
"char3": 0.30,
|
||||
"delta": 0.25,
|
||||
"sentence_emd": 0.10,
|
||||
"punctuation": 0.08,
|
||||
"contraction": 0.07,
|
||||
"mtld": 0.10,
|
||||
"word_length": 0.10,
|
||||
}
|
||||
|
||||
|
||||
def cosine_distance(a, b, keys=None):
|
||||
keys = list(keys) if keys is not None else sorted(set(a) | set(b))
|
||||
dot = sum(a.get(k, 0.0) * b.get(k, 0.0) for k in keys)
|
||||
na = math.sqrt(sum(a.get(k, 0.0) ** 2 for k in keys))
|
||||
nb = math.sqrt(sum(b.get(k, 0.0) ** 2 for k in keys))
|
||||
if not na or not nb:
|
||||
return 1.0
|
||||
return 1 - (dot / (na * nb))
|
||||
|
||||
|
||||
def z_function_vector(freqs, bg, top):
|
||||
keys = sorted(bg, key=lambda k: bg[k].get("mean", 0), reverse=True)[:top]
|
||||
return {k: (freqs.get(k, 0.0) - bg[k]["mean"]) / (bg[k]["std"] or 0.0001) for k in keys}
|
||||
|
||||
|
||||
def emd(a, b):
|
||||
if not a or not b:
|
||||
return 0.0
|
||||
max_len = max(max(a), max(b))
|
||||
ca = cb = dist = 0.0
|
||||
for i in range(1, max_len + 1):
|
||||
ca += sum(1 for x in a if x == i) / len(a)
|
||||
cb += sum(1 for x in b if x == i) / len(b)
|
||||
dist += abs(ca - cb)
|
||||
return dist
|
||||
|
||||
|
||||
def l1(a, b, keys):
|
||||
return sum(abs(a.get(k, 0.0) - b.get(k, 0.0)) for k in keys)
|
||||
|
||||
|
||||
def distances(profile, feats, subset=None):
|
||||
subset = set(subset or WEIGHTS)
|
||||
out = {}
|
||||
if "char3" in subset:
|
||||
keys = set(profile["char3"]) | set(feats["char3"])
|
||||
out["char3"] = cosine_distance(profile["char3"], feats["char3"], keys)
|
||||
if "delta" in subset:
|
||||
top = 50 if feats["total_words"] < 300 else 200
|
||||
bg = profile["function_word_background"]
|
||||
pv = z_function_vector(profile["function_words"], bg, top)
|
||||
cv = z_function_vector(feats["function_words"], bg, top)
|
||||
out["delta"] = cosine_distance(pv, cv, pv.keys())
|
||||
if "sentence_emd" in subset:
|
||||
med = profile["sentence_lengths"].get("median") or 1.0
|
||||
out["sentence_emd"] = emd(profile["sentence_lengths"]["lengths"], feats["sentence_lengths"]["lengths"]) / med
|
||||
if "punctuation" in subset:
|
||||
out["punctuation"] = l1(profile["punctuation"], feats["punctuation"], voice_profile.PUNCT)
|
||||
if "contraction" in subset:
|
||||
out["contraction"] = abs(profile["contraction_rate"] - feats["contraction_rate"])
|
||||
if "mtld" in subset:
|
||||
out["mtld"] = abs(profile["mtld"] - feats["mtld"]) / (profile["mtld"] or 1.0)
|
||||
if "word_length" in subset:
|
||||
keys = [str(i) for i in range(1, 16)]
|
||||
out["word_length"] = l1(profile["word_length_histogram"], feats["word_length_histogram"], keys)
|
||||
return out
|
||||
|
||||
|
||||
def weighted_sum(dists):
|
||||
return sum(WEIGHTS[k] * dists.get(k, 0.0) for k in WEIGHTS)
|
||||
|
||||
|
||||
def impostor_features(root):
|
||||
feats = []
|
||||
for path in voice_profile.iter_docs(root):
|
||||
feats.append((str(path), voice_profile.feature_bundle(path.read_text(errors="replace"))))
|
||||
return feats
|
||||
|
||||
|
||||
def zscores(candidate, impostor_rows):
|
||||
out = {}
|
||||
for key in WEIGHTS:
|
||||
vals = [row[key] for row in impostor_rows if key in row]
|
||||
if key not in candidate or not vals:
|
||||
out[key] = None
|
||||
continue
|
||||
mean = statistics.mean(vals)
|
||||
std = statistics.pstdev(vals) or 0.0001
|
||||
out[key] = (candidate[key] - mean) / std
|
||||
return out
|
||||
|
||||
|
||||
def gi_score(profile, cand_feats, impostors, seed):
|
||||
rng = random.Random(seed)
|
||||
keys = list(WEIGHTS)
|
||||
wins = 0
|
||||
trials = 64
|
||||
# Per-key distances depend only on (profile, feats) -- not on which subset
|
||||
# of keys a given trial happens to draw -- so compute every key's distance
|
||||
# ONCE per candidate/impostor here, then have each trial do a subset-
|
||||
# weighted sum over the precomputed values instead of recomputing
|
||||
# distances() from scratch on every trial. Arithmetically exact: the
|
||||
# trial loop still consumes rng.random()/rng.sample() in the same order,
|
||||
# and summing WEIGHTS[k]*value only over the drawn subset is the same
|
||||
# float sequence weighted_sum(distances(..., subset)) produced (excluded
|
||||
# keys contributed an exact 0.0 term either way).
|
||||
cand_dists = distances(profile, cand_feats)
|
||||
impostor_dists = [(name, distances(profile, imp)) for name, imp in impostors]
|
||||
for _ in range(trials):
|
||||
subset = [k for k in keys if rng.random() < 0.5] or [rng.choice(keys)]
|
||||
cand = sum(WEIGHTS[k] * cand_dists.get(k, 0.0) for k in subset)
|
||||
sampled = rng.sample(impostor_dists, k=min(len(impostor_dists), max(1, len(impostor_dists) // 2)))
|
||||
if all(cand < sum(WEIGHTS[k] * imp_dists.get(k, 0.0) for k in subset)
|
||||
for _, imp_dists in sampled):
|
||||
wins += 1
|
||||
return wins / trials
|
||||
|
||||
|
||||
def ngrams(tokens, n=4):
|
||||
return set(tuple(tokens[i:i + n]) for i in range(max(0, len(tokens) - n + 1)))
|
||||
|
||||
|
||||
LCS_THRESHOLD = 120
|
||||
|
||||
|
||||
def has_common_substring_over(a: str, b: str, min_length: int) -> bool:
|
||||
"""Rolling-hash check: do ``a`` and ``b`` share a contiguous substring of at
|
||||
least ``min_length`` characters?
|
||||
|
||||
O(len(a) + len(b)) expected time, vs. the O(len(a) * len(b)) classic DP a
|
||||
true longest-common-substring computation needs. This only answers the
|
||||
threshold question the copy gate actually asks ("is there a shared run
|
||||
longer than N chars"); it does not recover the true longest common
|
||||
substring length. Every hash match is verified against the source
|
||||
characters before being trusted, so a hash collision never produces a
|
||||
false positive.
|
||||
"""
|
||||
if min_length <= 0:
|
||||
return bool(a) and bool(b)
|
||||
if len(a) < min_length or len(b) < min_length:
|
||||
return False
|
||||
|
||||
base = 257
|
||||
mod = (1 << 61) - 1
|
||||
high_power = pow(base, min_length - 1, mod)
|
||||
|
||||
def window_hashes(s: str) -> dict[int, list[int]]:
|
||||
table: dict[int, list[int]] = {}
|
||||
h = 0
|
||||
for i in range(min_length):
|
||||
h = (h * base + ord(s[i])) % mod
|
||||
table.setdefault(h, []).append(0)
|
||||
for i in range(min_length, len(s)):
|
||||
h = ((h - ord(s[i - min_length]) * high_power) * base + ord(s[i])) % mod
|
||||
table.setdefault(h, []).append(i - min_length + 1)
|
||||
return table
|
||||
|
||||
table_a = window_hashes(a)
|
||||
table_b = window_hashes(b)
|
||||
for h, starts_b in table_b.items():
|
||||
starts_a = table_a.get(h)
|
||||
if not starts_a:
|
||||
continue
|
||||
for sb in starts_b:
|
||||
window_b = b[sb:sb + min_length]
|
||||
for sa in starts_a:
|
||||
if a[sa:sa + min_length] == window_b:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def copy_gate(candidate_text, samples_dir):
|
||||
cand_grams = ngrams(voice_profile.words(candidate_text))
|
||||
max_overlap = 0.0
|
||||
lcs_violation = False
|
||||
for path in voice_profile.iter_docs(samples_dir):
|
||||
text = path.read_text(errors="replace")
|
||||
sample_grams = ngrams(voice_profile.words(text))
|
||||
if cand_grams:
|
||||
max_overlap = max(max_overlap, len(cand_grams & sample_grams) / len(cand_grams))
|
||||
if has_common_substring_over(candidate_text, text, LCS_THRESHOLD + 1):
|
||||
lcs_violation = True
|
||||
# "longest_common_substring" is reported for backward-compatible shape only:
|
||||
# nothing pins its exact value (checked evals/adversarial-evals.json and
|
||||
# evals/check_*.py for max_lcs/longest_common_substring assertions; none
|
||||
# exist). Its semantics changed from "the true LCS length" to "the
|
||||
# matched threshold window length when a violation-length run was found,
|
||||
# else 0" -- computing the true max cheaply isn't possible without the
|
||||
# same O(n*m) cost this function exists to avoid.
|
||||
max_lcs = LCS_THRESHOLD + 1 if lcs_violation else 0
|
||||
return {
|
||||
"max_overlap": max_overlap,
|
||||
"longest_common_substring": max_lcs,
|
||||
"violation": max_overlap > 0.35 or lcs_violation,
|
||||
}
|
||||
|
||||
|
||||
def parse_args(argv):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--profile", required=True)
|
||||
parser.add_argument("--impostors", required=True)
|
||||
parser.add_argument("--seed", required=True, type=int)
|
||||
parser.add_argument("--samples")
|
||||
parser.add_argument("candidate_file")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def read_candidate(path):
|
||||
if path == "-":
|
||||
return sys.stdin.buffer.read().decode("utf-8", errors="replace")
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
raise FileNotFoundError(path)
|
||||
return p.read_text(errors="replace")
|
||||
|
||||
|
||||
def main(argv):
|
||||
args = parse_args(argv)
|
||||
if not Path(args.profile).exists() or not Path(args.impostors).is_dir():
|
||||
print("missing profile or impostors", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
text = read_candidate(args.candidate_file)
|
||||
except FileNotFoundError as e:
|
||||
print(f"missing candidate: {e}", file=sys.stderr)
|
||||
return 2
|
||||
profile = json.loads(Path(args.profile).read_text())
|
||||
feats = voice_profile.feature_bundle(text)
|
||||
low = feats["total_words"] < 150
|
||||
impostors = impostor_features(args.impostors)
|
||||
dist = distances(profile, feats)
|
||||
imp_dist = [distances(profile, f) for _, f in impostors]
|
||||
zs = zscores(dist, imp_dist)
|
||||
zsum = sum(WEIGHTS[k] * max(-3.0, min(3.0, zs[k] if zs[k] is not None else 0.0)) for k in WEIGHTS)
|
||||
gi = gi_score(profile, feats, impostors, args.seed) if impostors else 0.0
|
||||
result = {
|
||||
"candidate_words": feats["total_words"],
|
||||
"low_confidence": low,
|
||||
"distances": {k: (None if low and k in {"sentence_emd", "mtld"} else v) for k, v in dist.items()},
|
||||
"z_scores": {k: (None if low and k in {"sentence_emd", "mtld"} else v) for k, v in zs.items()},
|
||||
"gi": gi,
|
||||
"composite": 0.5 * (1 - gi) + 0.5 * zsum,
|
||||
}
|
||||
if args.samples:
|
||||
result["copy_gate"] = copy_gate(text, args.samples)
|
||||
print(json.dumps(result, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,479 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Sync unslop rules with Wikipedia's "Signs of AI writing" page.
|
||||
|
||||
Fetches the latest revision, parses structured data from wikitext,
|
||||
and outputs diffs or integration prompts for Claude Code.
|
||||
|
||||
Subcommands:
|
||||
check - Check for updates (exits 0 if no updates, 1 if updates available)
|
||||
diff - Output structured JSON diff of changes since last sync
|
||||
prompt - Output a Claude Code integration prompt with mapped changes
|
||||
|
||||
Usage:
|
||||
python wiki_sync.py check
|
||||
python wiki_sync.py diff
|
||||
python wiki_sync.py prompt
|
||||
python wiki_sync.py prompt | claude -p --allowedTools Edit,Read,Grep,Glob,Bash
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
from typing import TypedDict
|
||||
|
||||
STATE_FILE = Path(__file__).parent / ".wiki_sync_state.json"
|
||||
WIKI_PAGE = "Wikipedia:Signs_of_AI_writing"
|
||||
API_URL = "https://en.wikipedia.org/w/api.php"
|
||||
|
||||
|
||||
class SyncState(TypedDict):
|
||||
last_revision_id: int
|
||||
last_timestamp: str
|
||||
content_hash: str
|
||||
last_content_snapshot: str
|
||||
|
||||
|
||||
class ParsedSection(TypedDict):
|
||||
title: str
|
||||
level: int
|
||||
content: str
|
||||
watch_words: list[str]
|
||||
examples: list[str]
|
||||
|
||||
|
||||
class Change(TypedDict):
|
||||
type: str # "new_section", "new_words", "modified", "removed"
|
||||
section: str
|
||||
details: str
|
||||
words: list[str]
|
||||
|
||||
|
||||
def fetch_latest_revision() -> tuple[int, str, str]:
|
||||
"""Fetch the latest revision content from Wikipedia.
|
||||
|
||||
Returns (revision_id, timestamp, wikitext).
|
||||
"""
|
||||
params = {
|
||||
"action": "query",
|
||||
"titles": WIKI_PAGE,
|
||||
"prop": "revisions",
|
||||
"rvslots": "*",
|
||||
"rvprop": "ids|timestamp|content",
|
||||
"format": "json",
|
||||
"formatversion": "2",
|
||||
}
|
||||
url = f"{API_URL}?{urllib.parse.urlencode(params)}"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "unslop-wiki-sync/1.0"})
|
||||
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
page = data["query"]["pages"][0]
|
||||
if "missing" in page:
|
||||
print(f"Error: Page '{WIKI_PAGE}' not found.", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
revision = page["revisions"][0]
|
||||
rev_id = revision["revid"]
|
||||
timestamp = revision["timestamp"]
|
||||
content = revision["slots"]["main"]["content"]
|
||||
return rev_id, timestamp, content
|
||||
|
||||
|
||||
def get_wikitext(from_file: str | None) -> tuple[int, str, str]:
|
||||
"""Return (revision_id, timestamp, wikitext).
|
||||
|
||||
With `from_file` set, read wikitext from that local path instead of
|
||||
hitting the network — used for offline eval/fixture runs. There is no
|
||||
real revision for a local file, so revision_id/timestamp are placeholders.
|
||||
"""
|
||||
if from_file:
|
||||
content = Path(from_file).read_text(encoding="utf-8")
|
||||
return 0, "from-file", content
|
||||
return fetch_latest_revision()
|
||||
|
||||
|
||||
def load_state() -> SyncState | None:
|
||||
if not STATE_FILE.exists():
|
||||
return None
|
||||
with open(STATE_FILE, "r") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def save_state(rev_id: int, timestamp: str, content: str) -> None:
|
||||
state: SyncState = {
|
||||
"last_revision_id": rev_id,
|
||||
"last_timestamp": timestamp,
|
||||
"content_hash": hashlib.sha256(content.encode()).hexdigest(),
|
||||
"last_content_snapshot": content,
|
||||
}
|
||||
with open(STATE_FILE, "w") as f:
|
||||
json.dump(state, f, indent=2)
|
||||
|
||||
|
||||
def parse_wikitext(wikitext: str) -> list[ParsedSection]:
|
||||
"""Parse wikitext into structured sections with watch words and examples."""
|
||||
sections: list[ParsedSection] = []
|
||||
current_title = "Introduction"
|
||||
current_level = 1
|
||||
current_lines: list[str] = []
|
||||
|
||||
for line in wikitext.split("\n"):
|
||||
header_match = re.match(r"^(={2,6})\s*(.+?)\s*\1\s*$", line)
|
||||
if header_match:
|
||||
if current_lines:
|
||||
sections.append(_build_section(current_title, current_level, current_lines))
|
||||
current_level = len(header_match.group(1))
|
||||
current_title = header_match.group(2).strip()
|
||||
current_lines = []
|
||||
else:
|
||||
current_lines.append(line)
|
||||
|
||||
if current_lines:
|
||||
sections.append(_build_section(current_title, current_level, current_lines))
|
||||
|
||||
return sections
|
||||
|
||||
|
||||
def _build_section(title: str, level: int, lines: list[str]) -> ParsedSection:
|
||||
"""Build a ParsedSection from raw lines."""
|
||||
content = "\n".join(lines).strip()
|
||||
watch_words = _extract_watch_words(content)
|
||||
examples = _extract_examples(content)
|
||||
return {
|
||||
"title": title,
|
||||
"level": level,
|
||||
"content": content,
|
||||
"watch_words": watch_words,
|
||||
"examples": examples,
|
||||
}
|
||||
|
||||
|
||||
def _extract_watch_words(content: str) -> list[str]:
|
||||
"""Extract words-to-watch from tmbox templates and bold/italic markers."""
|
||||
words: list[str] = []
|
||||
|
||||
# Match {{tmbox|text=...}} content
|
||||
for match in re.finditer(r"\{\{tmbox\|[^}]*text\s*=\s*([^}]+)\}\}", content, re.DOTALL):
|
||||
text = match.group(1)
|
||||
# Extract bold words within tmbox
|
||||
for bold in re.finditer(r"'''(.+?)'''", text):
|
||||
words.append(bold.group(1).strip())
|
||||
|
||||
# Match bold words in "words to watch" style lists
|
||||
for match in re.finditer(r"'''(.+?)'''", content):
|
||||
word = match.group(1).strip()
|
||||
if word and len(word) < 50:
|
||||
words.append(word)
|
||||
|
||||
# Match items in bulleted lists
|
||||
for match in re.finditer(r"^\*\s*'''(.+?)'''", content, re.MULTILINE):
|
||||
words.append(match.group(1).strip())
|
||||
|
||||
return list(dict.fromkeys(words)) # dedupe preserving order
|
||||
|
||||
|
||||
def _extract_examples(content: str) -> list[str]:
|
||||
"""Extract example blocks (blockquotes, textdiff templates)."""
|
||||
examples: list[str] = []
|
||||
|
||||
# {{blockquote|...}}
|
||||
for match in re.finditer(r"\{\{blockquote\|([^}]+)\}\}", content, re.DOTALL):
|
||||
examples.append(match.group(1).strip())
|
||||
|
||||
# {{textdiff|old=...|new=...}}
|
||||
for match in re.finditer(
|
||||
r"\{\{textdiff\|[^}]*old\s*=\s*([^|]+)\|[^}]*new\s*=\s*([^}]+)\}\}",
|
||||
content, re.DOTALL
|
||||
):
|
||||
examples.append(f"BEFORE: {match.group(1).strip()}\nAFTER: {match.group(2).strip()}")
|
||||
|
||||
# Indented blockquote lines (: prefix in wikitext)
|
||||
block_lines: list[str] = []
|
||||
for line in content.split("\n"):
|
||||
if line.startswith(":"):
|
||||
block_lines.append(line.lstrip(": "))
|
||||
elif block_lines:
|
||||
examples.append("\n".join(block_lines))
|
||||
block_lines = []
|
||||
if block_lines:
|
||||
examples.append("\n".join(block_lines))
|
||||
|
||||
return examples
|
||||
|
||||
|
||||
def compute_diff(old_sections: list[ParsedSection], new_sections: list[ParsedSection]) -> list[Change]:
|
||||
"""Compute structured diff between two parsed page versions."""
|
||||
changes: list[Change] = []
|
||||
|
||||
old_by_title = {s["title"]: s for s in old_sections}
|
||||
new_by_title = {s["title"]: s for s in new_sections}
|
||||
|
||||
for title, new_sec in new_by_title.items():
|
||||
if title not in old_by_title:
|
||||
changes.append({
|
||||
"type": "new_section",
|
||||
"section": title,
|
||||
"details": f"New section with {len(new_sec['watch_words'])} watch words",
|
||||
"words": new_sec["watch_words"],
|
||||
})
|
||||
else:
|
||||
old_sec = old_by_title[title]
|
||||
new_words = set(new_sec["watch_words"]) - set(old_sec["watch_words"])
|
||||
if new_words:
|
||||
changes.append({
|
||||
"type": "new_words",
|
||||
"section": title,
|
||||
"details": f"{len(new_words)} new watch words added",
|
||||
"words": sorted(new_words),
|
||||
})
|
||||
|
||||
old_hash = hashlib.sha256(old_sec["content"].encode()).hexdigest()
|
||||
new_hash = hashlib.sha256(new_sec["content"].encode()).hexdigest()
|
||||
if old_hash != new_hash and not new_words:
|
||||
changes.append({
|
||||
"type": "modified",
|
||||
"section": title,
|
||||
"details": "Section content changed (no new watch words)",
|
||||
"words": [],
|
||||
})
|
||||
|
||||
for title in old_by_title:
|
||||
if title not in new_by_title:
|
||||
changes.append({
|
||||
"type": "removed",
|
||||
"section": title,
|
||||
"details": "Section removed from Wikipedia page",
|
||||
"words": old_by_title[title]["watch_words"],
|
||||
})
|
||||
|
||||
return changes
|
||||
|
||||
|
||||
# Mapping from Wikipedia section themes to unslop target files/sections
|
||||
SECTION_MAP: dict[str, dict[str, str]] = {
|
||||
"words to watch": {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "AI Vocabulary (Additional)",
|
||||
},
|
||||
"promotional": {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "Promotional Language",
|
||||
},
|
||||
"significance": {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "Significance & Legacy Inflation",
|
||||
},
|
||||
"legacy": {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "Significance & Legacy Inflation",
|
||||
},
|
||||
"vague": {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "Vague Attributions",
|
||||
},
|
||||
"attribution": {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "Vague Attributions",
|
||||
},
|
||||
"copula": {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "Copula Avoidance",
|
||||
},
|
||||
"participial": {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "Superficial -ing Analyses",
|
||||
},
|
||||
"chatbot": {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "Communication Artifacts",
|
||||
},
|
||||
"jargon": {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "Business Jargon",
|
||||
},
|
||||
"structure": {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "Structural Patterns to Avoid",
|
||||
},
|
||||
"synonym": {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "Elegant Variation / Synonym Cycling",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def map_change_to_target(change: Change) -> dict[str, str]:
|
||||
"""Map a Wikipedia change to the appropriate unslop file and section."""
|
||||
section_lower = change["section"].lower()
|
||||
for keyword, target in SECTION_MAP.items():
|
||||
if keyword in section_lower:
|
||||
return target
|
||||
return {
|
||||
"file": "references/taboo-phrases.md",
|
||||
"section": "AI Vocabulary (Additional)",
|
||||
}
|
||||
|
||||
|
||||
def generate_prompt(changes: list[Change], sections: list[ParsedSection]) -> str:
|
||||
"""Generate a Claude Code integration prompt from detected changes."""
|
||||
if not changes:
|
||||
return "No changes detected since last sync. Nothing to integrate."
|
||||
|
||||
lines = [
|
||||
"# Wikipedia AI Writing Patterns — Integration Update",
|
||||
"",
|
||||
"The Wikipedia 'Signs of AI writing' page has been updated.",
|
||||
"Below are changes that need to be integrated into the unslop skill.",
|
||||
"",
|
||||
"## Changes Detected",
|
||||
"",
|
||||
]
|
||||
|
||||
for i, change in enumerate(changes, 1):
|
||||
target = map_change_to_target(change)
|
||||
lines.append(f"### {i}. [{change['type'].upper()}] {change['section']}")
|
||||
lines.append(f"")
|
||||
lines.append(f"**Details:** {change['details']}")
|
||||
lines.append(f"**Target file:** `{target['file']}`")
|
||||
lines.append(f"**Target section:** {target['section']}")
|
||||
if change["words"]:
|
||||
lines.append(f"**Words/phrases to add:**")
|
||||
for word in change["words"]:
|
||||
lines.append(f"- \"{word}\"")
|
||||
lines.append("")
|
||||
|
||||
lines.extend([
|
||||
"## Integration Instructions",
|
||||
"",
|
||||
"For each change above:",
|
||||
"",
|
||||
"1. Read the target file",
|
||||
"2. Check if the phrase already exists (avoid duplicates)",
|
||||
"3. Add new phrases to the appropriate section in `references/taboo-phrases.md`",
|
||||
"4. Mirror additions in `scripts/banned_phrase_scan.py` BANNED_PHRASES dict",
|
||||
"5. If a new before/after example is warranted, add to `references/edit-library.md`",
|
||||
"",
|
||||
"After all changes:",
|
||||
"- Run `python3 scripts/banned_phrase_scan.py < /dev/null` to verify no syntax errors",
|
||||
"- Verify no duplicate entries in taboo-phrases.md",
|
||||
])
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def cmd_check(from_file: str | None = None) -> None:
|
||||
"""Check for updates. Exit 0 = no updates, 1 = updates available."""
|
||||
rev_id, timestamp, content = get_wikitext(from_file)
|
||||
state = load_state()
|
||||
|
||||
content_hash = hashlib.sha256(content.encode()).hexdigest()
|
||||
|
||||
if state is None:
|
||||
print(f"No previous sync state. Current revision: {rev_id} ({timestamp})")
|
||||
print("Run 'diff' or 'prompt' to see all content as new.")
|
||||
sys.exit(1)
|
||||
|
||||
if content_hash == state["content_hash"]:
|
||||
print(f"No changes. Current revision: {rev_id} ({timestamp})")
|
||||
print(f"Last synced revision: {state['last_revision_id']} ({state['last_timestamp']})")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"Updates available!")
|
||||
print(f" Last synced: revision {state['last_revision_id']} ({state['last_timestamp']})")
|
||||
print(f" Current: revision {rev_id} ({timestamp})")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def cmd_diff(from_file: str | None = None) -> None:
|
||||
"""Output structured JSON diff of changes."""
|
||||
rev_id, timestamp, content = get_wikitext(from_file)
|
||||
state = load_state()
|
||||
|
||||
new_sections = parse_wikitext(content)
|
||||
|
||||
if state is None:
|
||||
old_sections: list[ParsedSection] = []
|
||||
else:
|
||||
old_sections = parse_wikitext(state["last_content_snapshot"])
|
||||
|
||||
changes = compute_diff(old_sections, new_sections)
|
||||
|
||||
output = {
|
||||
"revision_id": rev_id,
|
||||
"timestamp": timestamp,
|
||||
"previous_revision_id": state["last_revision_id"] if state else None,
|
||||
"total_changes": len(changes),
|
||||
"changes": changes,
|
||||
}
|
||||
|
||||
print(json.dumps(output, indent=2))
|
||||
if not from_file:
|
||||
save_state(rev_id, timestamp, content)
|
||||
|
||||
|
||||
def cmd_prompt(from_file: str | None = None) -> None:
|
||||
"""Output a Claude Code integration prompt."""
|
||||
rev_id, timestamp, content = get_wikitext(from_file)
|
||||
state = load_state()
|
||||
|
||||
new_sections = parse_wikitext(content)
|
||||
|
||||
if state is None:
|
||||
old_sections: list[ParsedSection] = []
|
||||
else:
|
||||
old_sections = parse_wikitext(state["last_content_snapshot"])
|
||||
|
||||
changes = compute_diff(old_sections, new_sections)
|
||||
prompt = generate_prompt(changes, new_sections)
|
||||
|
||||
print(prompt)
|
||||
if not from_file:
|
||||
save_state(rev_id, timestamp, content)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Sync unslop rules with Wikipedia's 'Signs of AI writing' page."
|
||||
)
|
||||
subparsers = parser.add_subparsers(dest="command")
|
||||
|
||||
commands = {
|
||||
"check": cmd_check,
|
||||
"diff": cmd_diff,
|
||||
"prompt": cmd_prompt,
|
||||
}
|
||||
for name in commands:
|
||||
sub = subparsers.add_parser(name)
|
||||
sub.add_argument(
|
||||
"--from-file",
|
||||
metavar="PATH",
|
||||
help=(
|
||||
"read wikitext from PATH instead of fetching from Wikipedia "
|
||||
"(skips the network call and skips writing sync state)"
|
||||
),
|
||||
)
|
||||
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: wiki_sync.py <check|diff|prompt>", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
if sys.argv[1] not in commands:
|
||||
print(f"Unknown command: {sys.argv[1]}", file=sys.stderr)
|
||||
print(f"Available: {', '.join(commands)}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
args = parser.parse_args()
|
||||
commands[args.command](args.from_file)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user