from __future__ import annotations import hashlib import importlib.util import json import os import shutil import subprocess import sys import tempfile import unittest from pathlib import Path ROOT = Path(__file__).parents[2] SCRIPT = ROOT / "scripts" / "nono-evidence.py" SPEC = importlib.util.spec_from_file_location("nono_evidence_cli", SCRIPT) if SPEC is None or SPEC.loader is None: raise RuntimeError("cannot load evidence CLI") CLI = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(CLI) class EvidenceCliTests(unittest.TestCase): def _snapshot(self, root: Path) -> tuple[Path, Path, Path, str]: snapshot = root / "installed-snapshot" (snapshot / "scripts").mkdir(parents=True, mode=0o700) shutil.copy2(SCRIPT, snapshot / "scripts" / "nono-evidence.py") shutil.copy2(ROOT / "scripts" / "test_nono.py", snapshot / "scripts" / "test_nono.py") shutil.copytree( ROOT / "tests" / "nono", snapshot / "tests" / "nono", ignore=shutil.ignore_patterns("__pycache__", "*.pyc"), ) (snapshot / "config" / "nono").mkdir(parents=True, mode=0o700) shutil.copy2( ROOT / "config" / "nono" / "evidence.schema.json", snapshot / "config" / "nono" / "evidence.schema.json", ) (snapshot / ".pi-sbx-trusted-snapshot").write_text( "pi-sbx-trusted-snapshot-v1\n", encoding="utf-8" ) fake_nono = snapshot / "bin" / "nono" fake_nono.parent.mkdir(mode=0o700) fake_nono.write_text( "#!/bin/sh\nprintf '%s\\n' 'nono 0.74.0'\n", encoding="utf-8", ) fake_nono.chmod(0o700) os.chmod(snapshot, 0o700) installed_script = snapshot / "scripts" / "nono-evidence.py" return snapshot, installed_script, fake_nono, CLI._snapshot_digest(snapshot) def _metadata(self, nono: Path) -> list[str]: commit = subprocess.run( ["git", "-C", str(ROOT), "rev-parse", "HEAD"], check=True, capture_output=True, text=True, ).stdout.strip() policy = ROOT / "tests" / "nono" / "fixtures" / "core" / "minimal.json" digest = hashlib.sha256(policy.read_bytes()).hexdigest() return [ "--repository-root", str(ROOT), "--repository-commit", commit, "--policy-file", str(policy), "--policy-digest", digest, "--nono", str(nono), ] def _base( self, snapshot: Path, installed_script: Path, digest: str, state: Path, summary: Path, ) -> list[str]: return [ sys.executable, str(installed_script), "--trusted-snapshot", str(snapshot), "--trusted-snapshot-digest", digest, "--state-root", str(state), "--summary-root", str(summary), ] def test_identity_reader_rejects_fifo_without_blocking(self) -> None: with tempfile.TemporaryDirectory() as directory: fifo = Path(directory) / "identity.fifo" os.mkfifo(fifo) with self.assertRaisesRegex(CLI.EvidenceCliError, "regular file"): CLI._regular_file_identity(fifo, max_size=1024) def test_source_fails_closed_without_explicit_trusted_snapshot(self) -> None: result = subprocess.run( [sys.executable, str(SCRIPT), "run", "hermetic-core"], cwd=ROOT, capture_output=True, text=True, check=False, ) self.assertNotEqual(result.returncode, 0) self.assertIn("required", result.stderr) def test_hermetic_core_runs_real_harness_tests_before_passing(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) snapshot, installed_script, fake_nono, digest = self._snapshot(root) state, summary = root / "state", root / "summary" state.mkdir(mode=0o700) summary.mkdir(mode=0o700) base = self._base(snapshot, installed_script, digest, state, summary) command = [ *base, "run", *self._metadata(fake_nono), "hermetic-core", ] run = subprocess.run( command, cwd=ROOT, capture_output=True, text=True, check=False, ) self.assertEqual(run.returncode, 0, run.stderr) record = json.loads(run.stdout) self.assertEqual(record["status"], "passed") raw = json.loads(Path(record["rawEvidence"]).read_text(encoding="utf-8")) self.assertIn("Ran", raw["result"]["stderr"]) repeated = subprocess.run( command, cwd=ROOT, capture_output=True, text=True, check=False, ) self.assertEqual(repeated.returncode, 0, repeated.stderr) self.assertEqual(list(snapshot.rglob("__pycache__")), []) def test_live_mode_requires_approval_and_missing_check_is_blocked(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) snapshot, installed_script, fake_nono, digest = self._snapshot(root) state, summary = root / "state", root / "summary" state.mkdir(mode=0o700) summary.mkdir(mode=0o700) base = self._base(snapshot, installed_script, digest, state, summary) metadata = self._metadata(fake_nono) denied = subprocess.run( [*base, "run", *metadata, "--suite", "live", "git-live"], cwd=ROOT, capture_output=True, text=True, check=False, ) self.assertNotEqual(denied.returncode, 0) self.assertIn("approval", denied.stderr) run = subprocess.run( [ *base, "run", *metadata, "--suite", "live", "--approve-live", "--approver", "reviewer", "git-live", ], cwd=ROOT, capture_output=True, text=True, check=False, ) self.assertNotEqual(run.returncode, 0) run_record = json.loads(run.stdout) self.assertEqual(run_record["status"], "blocked") summarized = subprocess.run( [*base, "summarize", run_record["runId"]], cwd=ROOT, capture_output=True, text=True, check=False, ) self.assertNotEqual(summarized.returncode, 0) self.assertEqual(json.loads(summarized.stdout)["status"], "blocked") summary_file = next(summary.rglob("*.json")) committed = json.loads(summary_file.read_text(encoding="utf-8")) self.assertEqual(committed["status"], "blocked") self.assertEqual(committed["approver"], "reviewer") self.assertNotIn("stdout", json.dumps(committed)) def test_writable_nested_snapshot_directory_is_rejected(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) snapshot, installed_script, fake_nono, digest = self._snapshot(root) os.chmod(snapshot / "tests", 0o777) state, summary = root / "state", root / "summary" state.mkdir(mode=0o700) summary.mkdir(mode=0o700) base = self._base(snapshot, installed_script, digest, state, summary) result = subprocess.run( [ *base, "run", *self._metadata(fake_nono), "hermetic-core", ], cwd=ROOT, capture_output=True, text=True, check=False, ) self.assertEqual(result.returncode, 2) self.assertIn("not group/other writable", result.stderr) def test_source_cannot_borrow_an_unrelated_snapshot_identity(self) -> None: with tempfile.TemporaryDirectory() as directory: snapshot, _installed_script, _fake_nono, digest = self._snapshot( Path(directory) ) with self.assertRaisesRegex( CLI.TrustedSnapshotError, "source checkout cannot be a trusted production snapshot", ): CLI.verify_trusted_snapshot(snapshot, digest) def test_checkout_cannot_be_declared_trusted(self) -> None: with self.assertRaises(CLI.TrustedSnapshotError): CLI.verify_trusted_snapshot(ROOT, "0" * 64) if __name__ == "__main__": unittest.main()