global setup and configuration.
This commit is contained in:
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build deterministic stylometric voice profiles.
|
||||
|
||||
The built-in background table is a compact Writeprints-style English baseline:
|
||||
common function words seeded from public-domain frequency lists used by classic
|
||||
authorship-attribution examples, with broad fallback means/stddevs for words not
|
||||
observed in a caller-supplied background corpus. It is meant only as a stable
|
||||
normalizer; pass --background with same-genre documents for calibrated work.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
FUNCTION_WORDS = """
|
||||
the of and to in a is that it for as with was on be by he i this are or his from at
|
||||
which but have an had they you were their one all we can her has there been if more
|
||||
when will would who so no she about out up into do any your what than them some could
|
||||
these other then its our two may first my now such like over only also after most did
|
||||
many before must through back where much should well people down own just because good
|
||||
each those how under see made very being make between both even another while last
|
||||
might still same never every against since off though yet without within upon among
|
||||
until during per either neither nor whether whose whom why again once here there
|
||||
therefore however although nevertheless moreover instead indeed perhaps rather thus
|
||||
else already almost around across behind beyond near toward towards above below beside
|
||||
inside outside along plus minus except despite via versus including regarding concerning
|
||||
am were does done doing having let lets cannot dont didn't doesn't isn't aren't wasn't
|
||||
weren't haven't hasn't hadn't won't wouldn't shouldn't couldn't mightn't mustn't i'm
|
||||
you're he's she's it's we're they're i've you've we've they've i'd you'd he'd she'd we'd
|
||||
they'd i'll you'll he'll she'll we'll they'll me him us mine yours ours theirs myself
|
||||
yourself himself herself itself ourselves yourselves themselves
|
||||
""".split()
|
||||
|
||||
PUNCT = [",", ".", ";", ":", "?", "!", "-", "(", ")", '"', "'"]
|
||||
WORD_RE = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)?|\d+")
|
||||
SENT_RE = re.compile(r"[^.!?]+[.!?]?")
|
||||
|
||||
|
||||
def iter_docs(root):
|
||||
for path in sorted(Path(root).rglob("*")):
|
||||
if path.suffix.lower() in {".txt", ".md"} and path.is_file():
|
||||
yield path
|
||||
|
||||
|
||||
def normalize(text):
|
||||
return re.sub(r"\s+", " ", text.lower()).strip()
|
||||
|
||||
|
||||
def words(text):
|
||||
return WORD_RE.findall(text.lower())
|
||||
|
||||
|
||||
def sentences(text):
|
||||
out = []
|
||||
for part in SENT_RE.findall(text):
|
||||
toks = words(part)
|
||||
if toks:
|
||||
out.append(toks)
|
||||
return out
|
||||
|
||||
|
||||
def char3_counts(text, limit=None):
|
||||
norm = normalize(text)
|
||||
grams = collections.Counter(norm[i:i + 3] for i in range(max(0, len(norm) - 2)))
|
||||
items = sorted(grams.items(), key=lambda kv: (-kv[1], kv[0]))
|
||||
if limit:
|
||||
items = items[:limit]
|
||||
return dict(items)
|
||||
|
||||
|
||||
def function_freq(tokens):
|
||||
total = max(1, len(tokens))
|
||||
counts = collections.Counter(tokens)
|
||||
return {w: counts[w] / total for w in FUNCTION_WORDS}
|
||||
|
||||
|
||||
def sentence_stats(text):
|
||||
lengths = [len(s) for s in sentences(text)]
|
||||
if not lengths:
|
||||
return {"lengths": [], "median": 0.0, "iqr": 0.0}
|
||||
ordered = sorted(lengths)
|
||||
mid = statistics.median(ordered)
|
||||
q1 = statistics.median(ordered[:len(ordered) // 2] or ordered)
|
||||
q3 = statistics.median(ordered[(len(ordered) + 1) // 2:] or ordered)
|
||||
return {"lengths": lengths, "median": mid, "iqr": q3 - q1}
|
||||
|
||||
|
||||
def mtld(tokens, threshold=0.72):
|
||||
if len(tokens) < 20:
|
||||
return 0.0
|
||||
factors = 0.0
|
||||
types = set()
|
||||
count = 0
|
||||
for tok in tokens:
|
||||
count += 1
|
||||
types.add(tok)
|
||||
if len(types) / count <= threshold:
|
||||
factors += 1
|
||||
types.clear()
|
||||
count = 0
|
||||
if count:
|
||||
ttr = len(types) / count
|
||||
factors += (1 - ttr) / (1 - threshold) if threshold < 1 else 0
|
||||
return len(tokens) / factors if factors else float(len(tokens))
|
||||
|
||||
|
||||
def feature_bundle(text):
|
||||
toks = words(text)
|
||||
total = max(1, len(toks))
|
||||
punct_counts = collections.Counter(ch for ch in text if ch in PUNCT)
|
||||
contractions = sum(1 for t in toks if "'" in t)
|
||||
hist = collections.Counter(min(len(t), 15) for t in toks)
|
||||
paragraphs = [p for p in re.split(r"\n\s*\n", text.strip()) if p.strip()]
|
||||
return {
|
||||
"char3": char3_counts(text, 2000),
|
||||
"function_words": function_freq(toks),
|
||||
"sentence_lengths": sentence_stats(text),
|
||||
"punctuation": {p: punct_counts[p] / total for p in PUNCT},
|
||||
"contraction_rate": contractions / total,
|
||||
"mtld": mtld(toks),
|
||||
"word_length_histogram": {str(i): hist[i] / total for i in range(1, 16)},
|
||||
"paragraph_stats": {
|
||||
"count": len(paragraphs),
|
||||
"mean_words": (sum(len(words(p)) for p in paragraphs) / len(paragraphs)) if paragraphs else 0.0,
|
||||
},
|
||||
"total_words": len(toks),
|
||||
}
|
||||
|
||||
|
||||
def background_stats(root=None):
|
||||
docs = []
|
||||
if root:
|
||||
docs = [p.read_text(errors="replace") for p in iter_docs(root)]
|
||||
if not docs:
|
||||
return {w: {"mean": 0.0025 if w not in {"the", "of", "and", "to", "in", "a"} else 0.025,
|
||||
"std": 0.006} for w in FUNCTION_WORDS}
|
||||
rows = [function_freq(words(text)) for text in docs]
|
||||
stats = {}
|
||||
for w in FUNCTION_WORDS:
|
||||
vals = [r[w] for r in rows]
|
||||
stats[w] = {
|
||||
"mean": statistics.mean(vals),
|
||||
"std": statistics.pstdev(vals) or 0.0001,
|
||||
}
|
||||
return stats
|
||||
|
||||
|
||||
def build_profile(samples_dir, background=None):
|
||||
paths = list(iter_docs(samples_dir))
|
||||
text = "\n\n".join(p.read_text(errors="replace") for p in paths)
|
||||
profile = feature_bundle(text)
|
||||
profile["function_word_background"] = background_stats(background)
|
||||
profile["metadata"] = {
|
||||
"doc_count": len(paths),
|
||||
"total_words": profile["total_words"],
|
||||
"low_confidence": profile["total_words"] < 2000,
|
||||
"genre_warning": "profile has fewer than 2000 words" if profile["total_words"] < 2000 else "",
|
||||
}
|
||||
return profile
|
||||
|
||||
|
||||
def parse_args(argv):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("samples_dir")
|
||||
parser.add_argument("-o", "--output", required=True)
|
||||
parser.add_argument("--background")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv):
|
||||
args = parse_args(argv)
|
||||
root = Path(args.samples_dir)
|
||||
if not root.is_dir():
|
||||
print(f"missing samples dir: {root}", file=sys.stderr)
|
||||
return 2
|
||||
if not list(iter_docs(root)):
|
||||
print(f"no sample documents in {root}: only .txt and .md files are read "
|
||||
f"(recursively). Rename samples to .txt/.md or point at the right "
|
||||
f"directory.", file=sys.stderr)
|
||||
return 2
|
||||
profile = build_profile(root, args.background)
|
||||
if profile["metadata"]["low_confidence"]:
|
||||
print(profile["metadata"]["genre_warning"], file=sys.stderr)
|
||||
Path(args.output).write_text(json.dumps(profile, indent=2, sort_keys=True) + "\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
Reference in New Issue
Block a user