"""Schema-v1, secret-free evidence records and private atomic storage.""" from __future__ import annotations import datetime as _dt import hashlib import json import os import re import secrets import stat from collections.abc import Mapping, Sequence from dataclasses import dataclass from pathlib import Path from typing import Any from .redaction import assert_no_leaks SCHEMA_VERSION = 1 RETENTION_DAYS = 30 _SHA256 = re.compile(r"^[0-9a-f]{64}$") _COMMIT = re.compile(r"^[0-9a-f]{40}$") _ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") _QUARANTINE_ID = re.compile(r"^\.purge-[0-9a-f]{32}$") _UTC = re.compile( r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?Z$" ) _STATUSES = frozenset({"passed", "failed", "blocked", "not-run"}) _OUTCOME_VALUES = frozenset({"allowed", "denied", "unknown", "not-run"}) _TOP_LEVEL_FIELDS = frozenset( { "schemaVersion", "runId", "testId", "suite", "repositoryCommit", "policySnapshotDigest", "installedSnapshotDigest", "nonoVersion", "status", "approver", "startedAt", "finishedAt", "rawEvidence", "resultHashes", "observations", } ) _OBSERVATION_FIELDS = frozenset( {"id", "capability", "expected", "actual", "status", "hash"} ) _RAW_FIELD_NAMES = frozenset( { "argv", "args", "command", "env", "environment", "exception", "output", "raw", "stderr", "stdout", } ) class EvidenceValidationError(ValueError): """A summary or evidence envelope does not satisfy the closed contract.""" @dataclass(frozen=True) class RawEvidenceRef: run_id: str path: Path metadata_path: Path sha256: str retained_until: str def utc_timestamp(value: _dt.datetime | None = None) -> str: current = value or _dt.datetime.now(_dt.timezone.utc) if current.tzinfo is None: current = current.replace(tzinfo=_dt.timezone.utc) current = current.astimezone(_dt.timezone.utc) return current.isoformat(timespec="seconds").replace("+00:00", "Z") def sha256_bytes(value: bytes) -> str: return hashlib.sha256(value).hexdigest() def sha256_json(value: Any) -> str: return sha256_bytes(_canonical_json(value)) def _canonical_json(value: Any) -> bytes: try: return json.dumps( value, ensure_ascii=False, sort_keys=True, separators=(",", ":") ).encode("utf-8") except (TypeError, ValueError) as error: raise EvidenceValidationError(f"evidence is not JSON serializable: {error}") from error def _safe_id(value: object, field: str) -> str: if not isinstance(value, str) or _ID.fullmatch(value) is None: raise EvidenceValidationError(f"{field} must be a safe identifier") return value def _safe_digest(value: object, field: str) -> str: if not isinstance(value, str) or _SHA256.fullmatch(value) is None: raise EvidenceValidationError(f"{field} must be a lowercase SHA-256 digest") return value def _timestamp(value: object, field: str) -> _dt.datetime: if not isinstance(value, str) or _UTC.fullmatch(value) is None: raise EvidenceValidationError(f"{field} must be a UTC timestamp ending in Z") try: parsed = _dt.datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError as error: raise EvidenceValidationError(f"{field} is not a valid timestamp") from error if parsed.tzinfo != _dt.timezone.utc: raise EvidenceValidationError(f"{field} must use UTC") return parsed def _assert_no_raw_fields(value: Any, path: str = "$") -> None: if isinstance(value, Mapping): for key, child in value.items(): if isinstance(key, str) and key.lower() in _RAW_FIELD_NAMES: raise EvidenceValidationError(f"raw evidence field is not allowed in summary: {path}.{key}") _assert_no_raw_fields(child, f"{path}.{key}") elif isinstance(value, (list, tuple)): for index, child in enumerate(value): _assert_no_raw_fields(child, f"{path}[{index}]") def _normalize_observation( observation: Mapping[str, Any], ) -> dict[str, str]: if not isinstance(observation, Mapping): raise EvidenceValidationError("each observation must be an object") # Hash is intentionally explicit. Callers must hash an observation before # it enters a summary instead of accidentally putting command output here. result = dict(observation) if "observationHash" in result and "hash" not in result: result["hash"] = result.pop("observationHash") expected_fields = _OBSERVATION_FIELDS if set(result) != expected_fields: missing = sorted(expected_fields - set(result)) extra = sorted(set(result) - expected_fields) raise EvidenceValidationError( f"observation fields mismatch (missing={missing}, extra={extra})" ) if any(not isinstance(result[field], str) for field in expected_fields): raise EvidenceValidationError("observation fields must be strings") _safe_id(result["id"], "observation.id") _safe_id(result["capability"], "observation.capability") if result["expected"] not in _OUTCOME_VALUES: raise EvidenceValidationError("observation.expected is invalid") if result["actual"] not in _OUTCOME_VALUES: raise EvidenceValidationError("observation.actual is invalid") if result["status"] not in _STATUSES: raise EvidenceValidationError("observation.status is invalid") _safe_digest(result["hash"], "observation.hash") return {field: result[field] for field in expected_fields} def create_summary( *, run_id: str, test_id: str, suite: str, repository_commit: str, policy_snapshot_digest: str, installed_snapshot_digest: str, nono_version: str, status: str, started_at: str, finished_at: str, raw_evidence_sha256: str, raw_evidence_retained_until: str, result_hashes: Mapping[str, str], observations: Sequence[Mapping[str, Any]], approver: str | None = None, canaries: Sequence[str] = (), credential_name_patterns: Sequence[str] | None = None, ) -> dict[str, Any]: """Create and validate a redacted summary; raw command fields are rejected.""" summary: dict[str, Any] = { "schemaVersion": SCHEMA_VERSION, "runId": _safe_id(run_id, "runId"), "testId": _safe_id(test_id, "testId"), "suite": suite, "repositoryCommit": repository_commit, "policySnapshotDigest": _safe_digest(policy_snapshot_digest, "policySnapshotDigest"), "installedSnapshotDigest": _safe_digest( installed_snapshot_digest, "installedSnapshotDigest" ), "nonoVersion": nono_version, "status": status, "approver": approver, "startedAt": started_at, "finishedAt": finished_at, "rawEvidence": { "sha256": raw_evidence_sha256, "retainedUntil": raw_evidence_retained_until, }, "resultHashes": dict(result_hashes), "observations": [ _normalize_observation(item) for item in observations ], } _assert_no_raw_fields(summary) # A caller that forgot to redact gets an explicit failure, never a best-effort # committed summary. assert_no_leaks( summary, canaries=canaries, credential_name_patterns=credential_name_patterns, ) validate_summary(summary) return summary def validate_summary(summary: Any) -> None: if not isinstance(summary, Mapping): raise EvidenceValidationError("summary must be a JSON object") if set(summary) != _TOP_LEVEL_FIELDS: raise EvidenceValidationError( f"summary fields mismatch (missing={sorted(_TOP_LEVEL_FIELDS - set(summary))}, " f"extra={sorted(set(summary) - _TOP_LEVEL_FIELDS)})" ) if summary["schemaVersion"] != SCHEMA_VERSION: raise EvidenceValidationError("unsupported schemaVersion") _safe_id(summary["runId"], "runId") _safe_id(summary["testId"], "testId") if summary["suite"] not in {"hermetic", "host", "live"}: raise EvidenceValidationError("suite must be hermetic, host, or live") if not isinstance(summary["repositoryCommit"], str) or _COMMIT.fullmatch( summary["repositoryCommit"] ) is None: raise EvidenceValidationError("repositoryCommit must be a 40-character commit SHA") _safe_digest(summary["policySnapshotDigest"], "policySnapshotDigest") _safe_digest(summary["installedSnapshotDigest"], "installedSnapshotDigest") if not isinstance(summary["nonoVersion"], str) or re.fullmatch( r"0\.74\.\d+", summary["nonoVersion"] ) is None: raise EvidenceValidationError("nonoVersion must be in the 0.74.x line") if summary["status"] not in _STATUSES: raise EvidenceValidationError("invalid summary status") if summary["suite"] == "live": if not isinstance(summary["approver"], str) or not summary["approver"].strip(): raise EvidenceValidationError("live evidence requires an approver") elif summary["approver"] is not None and not isinstance(summary["approver"], str): raise EvidenceValidationError("approver must be null outside live mode") started = _timestamp(summary["startedAt"], "startedAt") finished = _timestamp(summary["finishedAt"], "finishedAt") if finished < started: raise EvidenceValidationError("finishedAt precedes startedAt") raw = summary["rawEvidence"] if not isinstance(raw, Mapping) or set(raw) != {"sha256", "retainedUntil"}: raise EvidenceValidationError("rawEvidence fields mismatch") _safe_digest(raw["sha256"], "rawEvidence.sha256") _timestamp(raw["retainedUntil"], "rawEvidence.retainedUntil") hashes = summary["resultHashes"] if not isinstance(hashes, Mapping): raise EvidenceValidationError("resultHashes must be an object") for key, digest in hashes.items(): _safe_id(key, "resultHashes key") _safe_digest(digest, f"resultHashes.{key}") observations = summary["observations"] if not isinstance(observations, list): raise EvidenceValidationError("observations must be an array") for item in observations: _normalize_observation(item) _assert_no_raw_fields(summary) assert_no_leaks(summary) class EvidenceStore: """Store raw evidence and summaries beneath caller-injected roots only.""" def __init__(self, state_root: str | os.PathLike[str], summary_root: str | os.PathLike[str]): self.state_root = self._root(Path(state_root), "state_root") self.summary_root = self._root(Path(summary_root), "summary_root") @staticmethod def _root(path: Path, name: str) -> Path: if not path.is_absolute(): raise ValueError(f"{name} must be absolute") if _has_symlink_component(path): raise ValueError(f"{name} cannot contain a symlinked path component") if path.exists(): if not path.is_dir(): raise ValueError(f"{name} must be a regular directory, not a symlink") if path.stat().st_uid != os.geteuid() or stat_mode(path) != 0o700: raise ValueError(f"{name} must be caller-owned mode 0700") else: path.mkdir(parents=True, mode=0o700) os.chmod(path, 0o700) return path def _raw_dir(self, run_id: str, *, create: bool = True) -> Path: _safe_id(run_id, "runId") base = self.state_root / "pi-sbx" / "evidence" directory = base / run_id if create: _private_dir(base) _private_dir(directory) else: if ( not base.is_dir() or base.is_symlink() or base.stat().st_uid != os.geteuid() or stat_mode(base) != 0o700 ): raise EvidenceValidationError("retained evidence root does not exist privately") if ( not directory.is_dir() or directory.is_symlink() or directory.stat().st_uid != os.geteuid() or stat_mode(directory) != 0o700 ): raise EvidenceValidationError("retained evidence run directory is unavailable") return directory def write_raw_evidence( self, run_id: str, evidence: Any, *, now: _dt.datetime | None = None, ) -> RawEvidenceRef: raw_dir = self._raw_dir(run_id) started = now or _dt.datetime.now(_dt.timezone.utc) retained = utc_timestamp(started + _dt.timedelta(days=RETENTION_DAYS)) payload = _canonical_json(evidence) digest = sha256_bytes(payload) raw_path = raw_dir / "raw.json" metadata_path = raw_dir / "metadata.json" _atomic_write(raw_path, payload) metadata = { "schemaVersion": SCHEMA_VERSION, "runId": run_id, "sha256": digest, "createdAt": utc_timestamp(started), "retainedUntil": retained, } _atomic_write(metadata_path, _canonical_json(metadata)) return RawEvidenceRef(run_id, raw_path, metadata_path, digest, retained) def read_raw_evidence( self, run_id: str, *, now: _dt.datetime | None = None, allow_expired: bool = False, ) -> tuple[Any, dict[str, Any]]: raw_dir = self._raw_dir(run_id, create=False) directory_fd = _private_directory_fd(raw_dir) try: return self._read_raw_evidence_fd( run_id, directory_fd, now=now, allow_expired=allow_expired, ) finally: os.close(directory_fd) def _read_raw_evidence_fd( self, run_id: str, directory_fd: int, *, now: _dt.datetime | None, allow_expired: bool, ) -> tuple[Any, dict[str, Any]]: raw_bytes = _read_nofollow_at(directory_fd, "raw.json", "raw.json") metadata_bytes = _read_nofollow_at( directory_fd, "metadata.json", "metadata.json", ) try: payload = json.loads(raw_bytes.decode("utf-8")) metadata = json.loads(metadata_bytes.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as error: raise EvidenceValidationError(f"invalid retained raw evidence: {error}") from error if not isinstance(metadata, dict): raise EvidenceValidationError("raw evidence metadata is not an object") metadata_fields = {"schemaVersion", "runId", "sha256", "createdAt", "retainedUntil"} if set(metadata) != metadata_fields or metadata.get("schemaVersion") != SCHEMA_VERSION: raise EvidenceValidationError("raw evidence metadata fields mismatch") if metadata.get("sha256") != sha256_bytes(raw_bytes): raise EvidenceValidationError("raw evidence hash does not match metadata") if metadata.get("runId") != run_id: raise EvidenceValidationError("raw evidence run ID mismatch") created = _timestamp(metadata.get("createdAt"), "raw metadata createdAt") retained = _timestamp(metadata.get("retainedUntil"), "raw metadata retainedUntil") if retained != created + _dt.timedelta(days=RETENTION_DAYS): raise EvidenceValidationError("raw evidence retention must be exactly 30 days") current = now or _dt.datetime.now(_dt.timezone.utc) if not allow_expired and retained <= current.astimezone(_dt.timezone.utc): raise EvidenceValidationError("raw evidence retention has expired") return payload, metadata def purge_expired(self, *, now: _dt.datetime | None = None) -> list[str]: base = self.state_root / "pi-sbx" / "evidence" if not base.exists(): return [] current = now or _dt.datetime.now(_dt.timezone.utc) removed: list[str] = [] base_fd = _private_directory_fd(base) try: for name in sorted(os.listdir(base_fd)): before = os.stat(name, dir_fd=base_fd, follow_symlinks=False) if _QUARANTINE_ID.fullmatch(name): if ( not stat.S_ISDIR(before.st_mode) or stat.S_IMODE(before.st_mode) != 0o700 or before.st_uid != os.geteuid() ): raise EvidenceValidationError( f"unsafe interrupted purge entry: {base / name}" ) _remove_tree_at( base_fd, name, (before.st_dev, before.st_ino), ) continue run_id = _safe_id(name, "retained runId") if ( not stat.S_ISDIR(before.st_mode) or stat.S_IMODE(before.st_mode) != 0o700 or before.st_uid != os.geteuid() ): raise EvidenceValidationError(f"unsafe evidence entry: {base / name}") run_fd = os.open( name, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0), dir_fd=base_fd, ) try: opened = os.fstat(run_fd) if (opened.st_dev, opened.st_ino) != (before.st_dev, before.st_ino): raise EvidenceValidationError( f"evidence directory changed before purge read: {base / name}" ) _payload, metadata = self._read_raw_evidence_fd( run_id, run_fd, now=current, allow_expired=True, ) finally: os.close(run_fd) after = os.stat(name, dir_fd=base_fd, follow_symlinks=False) if (before.st_dev, before.st_ino) != (after.st_dev, after.st_ino): raise EvidenceValidationError( f"evidence directory changed during purge: {base / name}" ) retained = _timestamp( metadata["retainedUntil"], "raw metadata retainedUntil", ) if retained <= current.astimezone(_dt.timezone.utc): quarantine = f".purge-{secrets.token_hex(16)}" os.rename( name, quarantine, src_dir_fd=base_fd, dst_dir_fd=base_fd, ) quarantined = os.stat( quarantine, dir_fd=base_fd, follow_symlinks=False, ) if (quarantined.st_dev, quarantined.st_ino) != ( before.st_dev, before.st_ino, ): raise EvidenceValidationError( f"evidence directory changed during quarantine: {base / name}" ) _remove_tree_at( base_fd, quarantine, (before.st_dev, before.st_ino), ) removed.append(run_id) finally: os.close(base_fd) return removed def summary_path(self, repository_id: str, policy_digest: str, run_id: str) -> Path: _safe_id(repository_id, "repository_id") _safe_digest(policy_digest, "policy_digest") _safe_id(run_id, "run_id") directory = self.summary_root / repository_id / policy_digest _private_dir(directory) return directory / f"{run_id}.json" def write_summary( self, summary: Mapping[str, Any], *, repository_id: str, policy_digest: str, ) -> Path: validate_summary(summary) path = self.summary_path(repository_id, policy_digest, str(summary["runId"])) _atomic_write(path, _canonical_json(summary)) return path def stat_mode(path: Path) -> int: return path.stat().st_mode & 0o777 def _has_symlink_component(path: Path) -> bool: current = path while True: if current.is_symlink(): return True if current.parent == current: return False current = current.parent def _private_dir(path: Path) -> None: if _has_symlink_component(path): raise ValueError(f"evidence directory cannot contain a symlinked component: {path}") if path.exists(): if ( path.is_symlink() or not path.is_dir() or path.stat().st_uid != os.geteuid() or stat_mode(path) != 0o700 ): raise ValueError(f"evidence directory must be caller-owned mode-0700: {path}") return missing: list[Path] = [] current = path while not current.exists(): missing.append(current) current = current.parent if ( current.is_symlink() or not current.is_dir() or current.stat().st_uid != os.geteuid() or stat_mode(current) != 0o700 ): raise ValueError(f"evidence parent must be caller-owned mode-0700: {current}") for directory in reversed(missing): directory.mkdir(mode=0o700) os.chmod(directory, 0o700) def _private_directory_fd(path: Path) -> int: flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) descriptor = os.open(path, flags) directory_stat = os.fstat(descriptor) if ( not stat.S_ISDIR(directory_stat.st_mode) or stat.S_IMODE(directory_stat.st_mode) != 0o700 or directory_stat.st_uid != os.geteuid() ): os.close(descriptor) raise EvidenceValidationError( f"evidence directory is not caller-owned mode-0700: {path}" ) return descriptor def _remove_tree_at( parent_fd: int, name: str, expected_identity: tuple[int, int], ) -> None: flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) directory_fd = os.open(name, flags, dir_fd=parent_fd) try: directory_stat = os.fstat(directory_fd) if (directory_stat.st_dev, directory_stat.st_ino) != expected_identity: raise EvidenceValidationError( f"evidence directory identity changed before removal: {name}" ) for child in os.listdir(directory_fd): child_stat = os.stat(child, dir_fd=directory_fd, follow_symlinks=False) if stat.S_ISDIR(child_stat.st_mode): _remove_tree_at( directory_fd, child, (child_stat.st_dev, child_stat.st_ino), ) elif stat.S_ISREG(child_stat.st_mode): os.unlink(child, dir_fd=directory_fd) else: raise EvidenceValidationError( f"unsafe entry in expired evidence directory: {name}/{child}" ) finally: os.close(directory_fd) final_stat = os.stat(name, dir_fd=parent_fd, follow_symlinks=False) if (final_stat.st_dev, final_stat.st_ino) != expected_identity: raise EvidenceValidationError( f"evidence directory identity changed before final removal: {name}" ) os.rmdir(name, dir_fd=parent_fd) def _atomic_write(path: Path, payload: bytes) -> None: directory_fd = _private_directory_fd(path.parent) temporary_name = f".{path.name}.{secrets.token_hex(8)}" descriptor = -1 try: descriptor = os.open( temporary_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600, dir_fd=directory_fd, ) written = 0 while written < len(payload): written += os.write(descriptor, payload[written:]) os.fchmod(descriptor, 0o600) os.fsync(descriptor) os.close(descriptor) descriptor = -1 os.rename( temporary_name, path.name, src_dir_fd=directory_fd, dst_dir_fd=directory_fd, ) os.fsync(directory_fd) finally: if descriptor != -1: os.close(descriptor) try: os.unlink(temporary_name, dir_fd=directory_fd) except FileNotFoundError: pass os.close(directory_fd) def _read_nofollow_at(directory_fd: int, name: str, display: str) -> bytes: flags = ( os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) ) try: descriptor = os.open(name, flags, dir_fd=directory_fd) except OSError as error: raise EvidenceValidationError( f"cannot read retained evidence {display}: {error}" ) from error try: file_stat = os.fstat(descriptor) if ( not stat.S_ISREG(file_stat.st_mode) or stat.S_IMODE(file_stat.st_mode) != 0o600 or file_stat.st_uid != os.geteuid() ): raise EvidenceValidationError( "retained evidence is not a caller-owned mode-0600 regular file: " + display ) remaining = file_stat.st_size chunks: list[bytes] = [] while remaining: chunk = os.read(descriptor, remaining) if not chunk: break chunks.append(chunk) remaining -= len(chunk) return b"".join(chunks) finally: os.close(descriptor) validate_evidence_summary = validate_summary __all__ = [ "EvidenceStore", "EvidenceValidationError", "RawEvidenceRef", "RETENTION_DAYS", "SCHEMA_VERSION", "create_summary", "sha256_bytes", "sha256_json", "utc_timestamp", "validate_evidence_summary", "validate_summary", ]