246 lines
9.0 KiB
Python
246 lines
9.0 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).parents[2]
|
|
SCRIPT = ROOT / "scripts" / "nono-evidence.py"
|
|
SPEC = importlib.util.spec_from_file_location("nono_evidence_cli", SCRIPT)
|
|
if SPEC is None or SPEC.loader is None:
|
|
raise RuntimeError("cannot load evidence CLI")
|
|
CLI = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(CLI)
|
|
|
|
|
|
class EvidenceCliTests(unittest.TestCase):
|
|
def _snapshot(self, root: Path) -> tuple[Path, Path, Path, str]:
|
|
snapshot = root / "installed-snapshot"
|
|
(snapshot / "scripts").mkdir(parents=True, mode=0o700)
|
|
shutil.copy2(SCRIPT, snapshot / "scripts" / "nono-evidence.py")
|
|
shutil.copy2(ROOT / "scripts" / "test_nono.py", snapshot / "scripts" / "test_nono.py")
|
|
shutil.copytree(
|
|
ROOT / "tests" / "nono",
|
|
snapshot / "tests" / "nono",
|
|
ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
|
|
)
|
|
(snapshot / "config" / "nono").mkdir(parents=True, mode=0o700)
|
|
shutil.copy2(
|
|
ROOT / "config" / "nono" / "evidence.schema.json",
|
|
snapshot / "config" / "nono" / "evidence.schema.json",
|
|
)
|
|
(snapshot / ".pi-sbx-trusted-snapshot").write_text(
|
|
"pi-sbx-trusted-snapshot-v1\n", encoding="utf-8"
|
|
)
|
|
fake_nono = snapshot / "bin" / "nono"
|
|
fake_nono.parent.mkdir(mode=0o700)
|
|
fake_nono.write_text(
|
|
"#!/bin/sh\nprintf '%s\\n' 'nono 0.74.0'\n",
|
|
encoding="utf-8",
|
|
)
|
|
fake_nono.chmod(0o700)
|
|
os.chmod(snapshot, 0o700)
|
|
installed_script = snapshot / "scripts" / "nono-evidence.py"
|
|
return snapshot, installed_script, fake_nono, CLI._snapshot_digest(snapshot)
|
|
|
|
def _metadata(self, nono: Path) -> list[str]:
|
|
commit = subprocess.run(
|
|
["git", "-C", str(ROOT), "rev-parse", "HEAD"],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout.strip()
|
|
policy = ROOT / "tests" / "nono" / "fixtures" / "core" / "minimal.json"
|
|
digest = hashlib.sha256(policy.read_bytes()).hexdigest()
|
|
return [
|
|
"--repository-root",
|
|
str(ROOT),
|
|
"--repository-commit",
|
|
commit,
|
|
"--policy-file",
|
|
str(policy),
|
|
"--policy-digest",
|
|
digest,
|
|
"--nono",
|
|
str(nono),
|
|
]
|
|
|
|
def _base(
|
|
self,
|
|
snapshot: Path,
|
|
installed_script: Path,
|
|
digest: str,
|
|
state: Path,
|
|
summary: Path,
|
|
) -> list[str]:
|
|
return [
|
|
sys.executable,
|
|
str(installed_script),
|
|
"--trusted-snapshot",
|
|
str(snapshot),
|
|
"--trusted-snapshot-digest",
|
|
digest,
|
|
"--state-root",
|
|
str(state),
|
|
"--summary-root",
|
|
str(summary),
|
|
]
|
|
|
|
def test_identity_reader_rejects_fifo_without_blocking(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
fifo = Path(directory) / "identity.fifo"
|
|
os.mkfifo(fifo)
|
|
with self.assertRaisesRegex(CLI.EvidenceCliError, "regular file"):
|
|
CLI._regular_file_identity(fifo, max_size=1024)
|
|
|
|
def test_source_fails_closed_without_explicit_trusted_snapshot(self) -> None:
|
|
result = subprocess.run(
|
|
[sys.executable, str(SCRIPT), "run", "hermetic-core"],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertIn("required", result.stderr)
|
|
|
|
def test_hermetic_core_runs_real_harness_tests_before_passing(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
snapshot, installed_script, fake_nono, digest = self._snapshot(root)
|
|
state, summary = root / "state", root / "summary"
|
|
state.mkdir(mode=0o700)
|
|
summary.mkdir(mode=0o700)
|
|
base = self._base(snapshot, installed_script, digest, state, summary)
|
|
command = [
|
|
*base,
|
|
"run",
|
|
*self._metadata(fake_nono),
|
|
"hermetic-core",
|
|
]
|
|
run = subprocess.run(
|
|
command,
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
self.assertEqual(run.returncode, 0, run.stderr)
|
|
record = json.loads(run.stdout)
|
|
self.assertEqual(record["status"], "passed")
|
|
raw = json.loads(Path(record["rawEvidence"]).read_text(encoding="utf-8"))
|
|
self.assertIn("Ran", raw["result"]["stderr"])
|
|
repeated = subprocess.run(
|
|
command,
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
self.assertEqual(repeated.returncode, 0, repeated.stderr)
|
|
self.assertEqual(list(snapshot.rglob("__pycache__")), [])
|
|
|
|
def test_live_mode_requires_approval_and_missing_check_is_blocked(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
snapshot, installed_script, fake_nono, digest = self._snapshot(root)
|
|
state, summary = root / "state", root / "summary"
|
|
state.mkdir(mode=0o700)
|
|
summary.mkdir(mode=0o700)
|
|
base = self._base(snapshot, installed_script, digest, state, summary)
|
|
metadata = self._metadata(fake_nono)
|
|
denied = subprocess.run(
|
|
[*base, "run", *metadata, "--suite", "live", "git-live"],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
self.assertNotEqual(denied.returncode, 0)
|
|
self.assertIn("approval", denied.stderr)
|
|
run = subprocess.run(
|
|
[
|
|
*base,
|
|
"run",
|
|
*metadata,
|
|
"--suite",
|
|
"live",
|
|
"--approve-live",
|
|
"--approver",
|
|
"reviewer",
|
|
"git-live",
|
|
],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
self.assertNotEqual(run.returncode, 0)
|
|
run_record = json.loads(run.stdout)
|
|
self.assertEqual(run_record["status"], "blocked")
|
|
summarized = subprocess.run(
|
|
[*base, "summarize", run_record["runId"]],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
self.assertNotEqual(summarized.returncode, 0)
|
|
self.assertEqual(json.loads(summarized.stdout)["status"], "blocked")
|
|
summary_file = next(summary.rglob("*.json"))
|
|
committed = json.loads(summary_file.read_text(encoding="utf-8"))
|
|
self.assertEqual(committed["status"], "blocked")
|
|
self.assertEqual(committed["approver"], "reviewer")
|
|
self.assertNotIn("stdout", json.dumps(committed))
|
|
|
|
def test_writable_nested_snapshot_directory_is_rejected(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
snapshot, installed_script, fake_nono, digest = self._snapshot(root)
|
|
os.chmod(snapshot / "tests", 0o777)
|
|
state, summary = root / "state", root / "summary"
|
|
state.mkdir(mode=0o700)
|
|
summary.mkdir(mode=0o700)
|
|
base = self._base(snapshot, installed_script, digest, state, summary)
|
|
result = subprocess.run(
|
|
[
|
|
*base,
|
|
"run",
|
|
*self._metadata(fake_nono),
|
|
"hermetic-core",
|
|
],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertIn("not group/other writable", result.stderr)
|
|
|
|
def test_source_cannot_borrow_an_unrelated_snapshot_identity(self) -> None:
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
snapshot, _installed_script, _fake_nono, digest = self._snapshot(
|
|
Path(directory)
|
|
)
|
|
with self.assertRaisesRegex(
|
|
CLI.TrustedSnapshotError,
|
|
"source checkout cannot be a trusted production snapshot",
|
|
):
|
|
CLI.verify_trusted_snapshot(snapshot, digest)
|
|
|
|
def test_checkout_cannot_be_declared_trusted(self) -> None:
|
|
with self.assertRaises(CLI.TrustedSnapshotError):
|
|
CLI.verify_trusted_snapshot(ROOT, "0" * 64)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|