feat(nono): add isolated evidence harness
This commit is contained in:
@@ -0,0 +1,245 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "nono-evidence.py"
|
||||
SPEC = importlib.util.spec_from_file_location("nono_evidence_cli", SCRIPT)
|
||||
if SPEC is None or SPEC.loader is None:
|
||||
raise RuntimeError("cannot load evidence CLI")
|
||||
CLI = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(CLI)
|
||||
|
||||
|
||||
class EvidenceCliTests(unittest.TestCase):
|
||||
def _snapshot(self, root: Path) -> tuple[Path, Path, Path, str]:
|
||||
snapshot = root / "installed-snapshot"
|
||||
(snapshot / "scripts").mkdir(parents=True, mode=0o700)
|
||||
shutil.copy2(SCRIPT, snapshot / "scripts" / "nono-evidence.py")
|
||||
shutil.copy2(ROOT / "scripts" / "test_nono.py", snapshot / "scripts" / "test_nono.py")
|
||||
shutil.copytree(
|
||||
ROOT / "tests" / "nono",
|
||||
snapshot / "tests" / "nono",
|
||||
ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
|
||||
)
|
||||
(snapshot / "config" / "nono").mkdir(parents=True, mode=0o700)
|
||||
shutil.copy2(
|
||||
ROOT / "config" / "nono" / "evidence.schema.json",
|
||||
snapshot / "config" / "nono" / "evidence.schema.json",
|
||||
)
|
||||
(snapshot / ".pi-sbx-trusted-snapshot").write_text(
|
||||
"pi-sbx-trusted-snapshot-v1\n", encoding="utf-8"
|
||||
)
|
||||
fake_nono = snapshot / "bin" / "nono"
|
||||
fake_nono.parent.mkdir(mode=0o700)
|
||||
fake_nono.write_text(
|
||||
"#!/bin/sh\nprintf '%s\\n' 'nono 0.74.0'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fake_nono.chmod(0o700)
|
||||
os.chmod(snapshot, 0o700)
|
||||
installed_script = snapshot / "scripts" / "nono-evidence.py"
|
||||
return snapshot, installed_script, fake_nono, CLI._snapshot_digest(snapshot)
|
||||
|
||||
def _metadata(self, nono: Path) -> list[str]:
|
||||
commit = subprocess.run(
|
||||
["git", "-C", str(ROOT), "rev-parse", "HEAD"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
policy = ROOT / "tests" / "nono" / "fixtures" / "core" / "minimal.json"
|
||||
digest = hashlib.sha256(policy.read_bytes()).hexdigest()
|
||||
return [
|
||||
"--repository-root",
|
||||
str(ROOT),
|
||||
"--repository-commit",
|
||||
commit,
|
||||
"--policy-file",
|
||||
str(policy),
|
||||
"--policy-digest",
|
||||
digest,
|
||||
"--nono",
|
||||
str(nono),
|
||||
]
|
||||
|
||||
def _base(
|
||||
self,
|
||||
snapshot: Path,
|
||||
installed_script: Path,
|
||||
digest: str,
|
||||
state: Path,
|
||||
summary: Path,
|
||||
) -> list[str]:
|
||||
return [
|
||||
sys.executable,
|
||||
str(installed_script),
|
||||
"--trusted-snapshot",
|
||||
str(snapshot),
|
||||
"--trusted-snapshot-digest",
|
||||
digest,
|
||||
"--state-root",
|
||||
str(state),
|
||||
"--summary-root",
|
||||
str(summary),
|
||||
]
|
||||
|
||||
def test_identity_reader_rejects_fifo_without_blocking(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
fifo = Path(directory) / "identity.fifo"
|
||||
os.mkfifo(fifo)
|
||||
with self.assertRaisesRegex(CLI.EvidenceCliError, "regular file"):
|
||||
CLI._regular_file_identity(fifo, max_size=1024)
|
||||
|
||||
def test_source_fails_closed_without_explicit_trusted_snapshot(self) -> None:
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "run", "hermetic-core"],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn("required", result.stderr)
|
||||
|
||||
def test_hermetic_core_runs_real_harness_tests_before_passing(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
snapshot, installed_script, fake_nono, digest = self._snapshot(root)
|
||||
state, summary = root / "state", root / "summary"
|
||||
state.mkdir(mode=0o700)
|
||||
summary.mkdir(mode=0o700)
|
||||
base = self._base(snapshot, installed_script, digest, state, summary)
|
||||
command = [
|
||||
*base,
|
||||
"run",
|
||||
*self._metadata(fake_nono),
|
||||
"hermetic-core",
|
||||
]
|
||||
run = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(run.returncode, 0, run.stderr)
|
||||
record = json.loads(run.stdout)
|
||||
self.assertEqual(record["status"], "passed")
|
||||
raw = json.loads(Path(record["rawEvidence"]).read_text(encoding="utf-8"))
|
||||
self.assertIn("Ran", raw["result"]["stderr"])
|
||||
repeated = subprocess.run(
|
||||
command,
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(repeated.returncode, 0, repeated.stderr)
|
||||
self.assertEqual(list(snapshot.rglob("__pycache__")), [])
|
||||
|
||||
def test_live_mode_requires_approval_and_missing_check_is_blocked(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
snapshot, installed_script, fake_nono, digest = self._snapshot(root)
|
||||
state, summary = root / "state", root / "summary"
|
||||
state.mkdir(mode=0o700)
|
||||
summary.mkdir(mode=0o700)
|
||||
base = self._base(snapshot, installed_script, digest, state, summary)
|
||||
metadata = self._metadata(fake_nono)
|
||||
denied = subprocess.run(
|
||||
[*base, "run", *metadata, "--suite", "live", "git-live"],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertNotEqual(denied.returncode, 0)
|
||||
self.assertIn("approval", denied.stderr)
|
||||
run = subprocess.run(
|
||||
[
|
||||
*base,
|
||||
"run",
|
||||
*metadata,
|
||||
"--suite",
|
||||
"live",
|
||||
"--approve-live",
|
||||
"--approver",
|
||||
"reviewer",
|
||||
"git-live",
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertNotEqual(run.returncode, 0)
|
||||
run_record = json.loads(run.stdout)
|
||||
self.assertEqual(run_record["status"], "blocked")
|
||||
summarized = subprocess.run(
|
||||
[*base, "summarize", run_record["runId"]],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertNotEqual(summarized.returncode, 0)
|
||||
self.assertEqual(json.loads(summarized.stdout)["status"], "blocked")
|
||||
summary_file = next(summary.rglob("*.json"))
|
||||
committed = json.loads(summary_file.read_text(encoding="utf-8"))
|
||||
self.assertEqual(committed["status"], "blocked")
|
||||
self.assertEqual(committed["approver"], "reviewer")
|
||||
self.assertNotIn("stdout", json.dumps(committed))
|
||||
|
||||
def test_writable_nested_snapshot_directory_is_rejected(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
snapshot, installed_script, fake_nono, digest = self._snapshot(root)
|
||||
os.chmod(snapshot / "tests", 0o777)
|
||||
state, summary = root / "state", root / "summary"
|
||||
state.mkdir(mode=0o700)
|
||||
summary.mkdir(mode=0o700)
|
||||
base = self._base(snapshot, installed_script, digest, state, summary)
|
||||
result = subprocess.run(
|
||||
[
|
||||
*base,
|
||||
"run",
|
||||
*self._metadata(fake_nono),
|
||||
"hermetic-core",
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(result.returncode, 2)
|
||||
self.assertIn("not group/other writable", result.stderr)
|
||||
|
||||
def test_source_cannot_borrow_an_unrelated_snapshot_identity(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
snapshot, _installed_script, _fake_nono, digest = self._snapshot(
|
||||
Path(directory)
|
||||
)
|
||||
with self.assertRaisesRegex(
|
||||
CLI.TrustedSnapshotError,
|
||||
"source checkout cannot be a trusted production snapshot",
|
||||
):
|
||||
CLI.verify_trusted_snapshot(snapshot, digest)
|
||||
|
||||
def test_checkout_cannot_be_declared_trusted(self) -> None:
|
||||
with self.assertRaises(CLI.TrustedSnapshotError):
|
||||
CLI.verify_trusted_snapshot(ROOT, "0" * 64)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user