global setup and configuration.
This commit is contained in:
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Aggregate a teach-calibration preferences JSONL into per-dimension confidence,
|
||||
surface conflicts against a measured stylometric profile, and pick the next
|
||||
dimension to play.
|
||||
|
||||
Preferences JSONL rows (one per game round):
|
||||
{"pair_id": "...", "dimension": "contractions", "choice": "a"|"b"|"neither",
|
||||
"ts": "...", "a_label": "contracted", "b_label": "expanded"}
|
||||
|
||||
`a_label`/`b_label` are optional. When present (calibrate_pairs.py's
|
||||
transform_applied names the B pole; A sits at the other pole in the same
|
||||
dimension's POLES pair) they let this script report a semantic preferred
|
||||
DIRECTION ("expanded", "short", "formal", ...) instead of the bare literal
|
||||
"a"/"b" tally, which is what --profile conflict detection needs. Rows without
|
||||
labels still count toward n and the confidence interval; they just can't
|
||||
resolve to a named pole (direction is reported as "a" or "b").
|
||||
|
||||
Usage:
|
||||
python3 calibrate_score.py --preferences prefs.jsonl
|
||||
python3 calibrate_score.py --preferences prefs.jsonl --profile profile.json
|
||||
python3 calibrate_score.py --preferences prefs.jsonl --next
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(HERE))
|
||||
|
||||
from calibrate_pairs import DIMENSIONS, POLES # noqa: E402
|
||||
|
||||
MIN_K = 5
|
||||
Z = 1.96
|
||||
|
||||
# Dimension -> (profile_key, pole_meaning) used only for --profile conflict
|
||||
# detection. `low_pole`/`high_pole` say which named pole a LOW vs HIGH measured
|
||||
# value corresponds to, so a confident stated preference for the opposite pole
|
||||
# of what the measured value implies is flagged.
|
||||
_PROFILE_LINKS: dict[str, dict] = {
|
||||
"contractions": {
|
||||
"profile_key": "contraction_rate",
|
||||
"low_pole": "expanded",
|
||||
"high_pole": "contracted",
|
||||
"low_threshold": 0.05,
|
||||
"high_threshold": 0.20,
|
||||
},
|
||||
"sentence_length": {
|
||||
"profile_key": "avg_sentence_length",
|
||||
"low_pole": "short",
|
||||
"high_pole": "long",
|
||||
"low_threshold": 12.0,
|
||||
"high_threshold": 20.0,
|
||||
},
|
||||
"staccato": {
|
||||
"profile_key": "avg_sentence_length",
|
||||
"low_pole": "staccato",
|
||||
"high_pole": "flowing",
|
||||
"low_threshold": 12.0,
|
||||
"high_threshold": 20.0,
|
||||
},
|
||||
"em_dash": {
|
||||
"profile_key": "em_dash_rate",
|
||||
"low_pole": "plain",
|
||||
"high_pole": "dashed",
|
||||
"low_threshold": 0.02,
|
||||
"high_threshold": 0.15,
|
||||
},
|
||||
"connectives": {
|
||||
"profile_key": "formal_connective_rate",
|
||||
"low_pole": "plain",
|
||||
"high_pole": "formal",
|
||||
"low_threshold": 0.10,
|
||||
"high_threshold": 0.40,
|
||||
},
|
||||
}
|
||||
|
||||
CONFIDENCE_STOP = 0.7
|
||||
K_STOP = 9
|
||||
|
||||
|
||||
def wilson_lower_bound(successes: int, n: int, z: float = Z) -> float:
|
||||
if n == 0:
|
||||
return 0.0
|
||||
phat = successes / n
|
||||
denom = 1 + z * z / n
|
||||
center = phat + z * z / (2 * n)
|
||||
margin = z * math.sqrt((phat * (1 - phat) + z * z / (4 * n)) / n)
|
||||
return round(max(0.0, (center - margin) / denom), 3)
|
||||
|
||||
|
||||
def load_preferences(path: Path) -> list[dict]:
|
||||
rows = []
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
|
||||
def dedup_by_pair_id(rows: list[dict]) -> list[dict]:
|
||||
"""Keep one row per `pair_id`: the LATEST by `ts` (ISO 8601, so a plain
|
||||
string comparison orders correctly). A game round replayed twice (e.g. the
|
||||
agent crashed mid-write and re-ran the round) must count once toward `n`
|
||||
and resolve to whatever the user most recently chose, not double-count or
|
||||
let an earlier write win. Rows without a `pair_id` can't be deduped
|
||||
against anything, so each is kept as-is.
|
||||
"""
|
||||
by_pair_id: dict[str, dict] = {}
|
||||
passthrough: list[dict] = []
|
||||
for row in rows:
|
||||
pid = row.get("pair_id")
|
||||
if not pid:
|
||||
passthrough.append(row)
|
||||
continue
|
||||
existing = by_pair_id.get(pid)
|
||||
if existing is None or row.get("ts", "") >= existing.get("ts", ""):
|
||||
by_pair_id[pid] = row
|
||||
return list(by_pair_id.values()) + passthrough
|
||||
|
||||
|
||||
def aggregate(rows: list[dict]) -> dict[str, dict]:
|
||||
rows = dedup_by_pair_id(rows)
|
||||
result = {dim: {"n": 0, "tally": {}, "neither": 0} for dim in DIMENSIONS}
|
||||
|
||||
for row in rows:
|
||||
dim = row.get("dimension")
|
||||
if dim not in result:
|
||||
continue
|
||||
result[dim]["n"] += 1
|
||||
choice = row.get("choice")
|
||||
if choice == "neither":
|
||||
result[dim]["neither"] += 1
|
||||
continue
|
||||
if choice not in ("a", "b"):
|
||||
continue
|
||||
label = row.get(f"{choice}_label") or choice
|
||||
result[dim]["tally"][label] = result[dim]["tally"].get(label, 0) + 1
|
||||
|
||||
dimensions = {}
|
||||
for dim, data in result.items():
|
||||
n = data["n"]
|
||||
decisive = sum(data["tally"].values())
|
||||
if n < MIN_K or decisive == 0:
|
||||
dimensions[dim] = {
|
||||
"n": n,
|
||||
"status": "insufficient",
|
||||
"preferred": None,
|
||||
"confidence": 0.0,
|
||||
}
|
||||
continue
|
||||
max_count = max(data["tally"].values())
|
||||
top_labels = [label for label, count in data["tally"].items() if count == max_count]
|
||||
if len(top_labels) > 1:
|
||||
# A genuine tie between the top two (or more) tallies: there is no
|
||||
# lean to report, so `preferred` must be null rather than
|
||||
# silently picking whichever label happens to sort last.
|
||||
dimensions[dim] = {
|
||||
"n": n,
|
||||
"status": "tied",
|
||||
"preferred": None,
|
||||
"confidence": 0.0,
|
||||
}
|
||||
continue
|
||||
preferred_label = top_labels[0]
|
||||
confidence = wilson_lower_bound(max_count, decisive)
|
||||
dimensions[dim] = {
|
||||
"n": n,
|
||||
"status": "confident",
|
||||
"preferred": preferred_label,
|
||||
"confidence": confidence,
|
||||
}
|
||||
return dimensions
|
||||
|
||||
|
||||
def detect_conflicts(dimensions: dict[str, dict], profile: dict) -> list[dict]:
|
||||
conflicts = []
|
||||
for dim, data in dimensions.items():
|
||||
if data["status"] != "confident" or data["confidence"] < CONFIDENCE_STOP:
|
||||
continue
|
||||
link = _PROFILE_LINKS.get(dim)
|
||||
if not link or link["profile_key"] not in profile:
|
||||
continue
|
||||
measured = profile[link["profile_key"]]
|
||||
preferred = data["preferred"]
|
||||
conflict_pole = None
|
||||
if preferred == link["low_pole"] and measured >= link["high_threshold"]:
|
||||
conflict_pole = link["high_pole"]
|
||||
elif preferred == link["high_pole"] and measured <= link["low_threshold"]:
|
||||
conflict_pole = link["low_pole"]
|
||||
if conflict_pole is None:
|
||||
continue
|
||||
conflicts.append({
|
||||
"dimension": dim,
|
||||
"preferred": preferred,
|
||||
"preferred_confidence": data["confidence"],
|
||||
"preferred_provenance": "stated-preference",
|
||||
"measured_key": link["profile_key"],
|
||||
"measured_value": measured,
|
||||
"measured_provenance": "measured-from-samples",
|
||||
"message": (
|
||||
f"Stated preference for '{preferred}' ({dim}) contradicts "
|
||||
f"{link['profile_key']}={measured} measured from samples, "
|
||||
f"which points toward '{conflict_pole}'."
|
||||
),
|
||||
})
|
||||
return conflicts
|
||||
|
||||
|
||||
def next_dimension(dimensions: dict[str, dict]) -> dict:
|
||||
def sort_key(dim: str) -> tuple:
|
||||
data = dimensions[dim]
|
||||
confidence = data["confidence"] if data["status"] == "confident" else 0.0
|
||||
return (data["n"], confidence, DIMENSIONS.index(dim))
|
||||
|
||||
ordered = sorted(DIMENSIONS, key=sort_key)
|
||||
chosen = ordered[0]
|
||||
data = dimensions[chosen]
|
||||
if data["n"] == min(dimensions[d]["n"] for d in DIMENSIONS):
|
||||
reason = "fewest_observations"
|
||||
else:
|
||||
reason = "lowest_confidence"
|
||||
return {
|
||||
"next_dimension": chosen,
|
||||
"reason": reason,
|
||||
"n": data["n"],
|
||||
"confidence": data["confidence"],
|
||||
}
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--preferences", required=True)
|
||||
parser.add_argument("--profile")
|
||||
parser.add_argument("--next", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
args = parse_args(argv)
|
||||
|
||||
try:
|
||||
rows = load_preferences(Path(args.preferences))
|
||||
except OSError as e:
|
||||
print(json.dumps({"error": f"could not read preferences file: {e}"}))
|
||||
return 2
|
||||
|
||||
dimensions = aggregate(rows)
|
||||
|
||||
if args.next:
|
||||
print(json.dumps(next_dimension(dimensions), indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
output = {"dimensions": dimensions}
|
||||
|
||||
if args.profile:
|
||||
try:
|
||||
profile = json.loads(Path(args.profile).read_text())
|
||||
except OSError as e:
|
||||
print(json.dumps({"error": f"could not read profile file: {e}"}))
|
||||
return 2
|
||||
output["conflicts"] = detect_conflicts(dimensions, profile)
|
||||
|
||||
print(json.dumps(output, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Reference in New Issue
Block a user