From 1371995cbeddc5da65e74bb89f6077972e1a9ffd Mon Sep 17 00:00:00 2001 From: JB Date: Sat, 19 Sep 2026 00:08:01 -0400 Subject: [PATCH] test: add rollout smoke and restore evidence runbook --- .gitea/workflows/ci.yml | 2 + README.md | 1 + docs/release-safety.md | 9 ++ docs/restore-rehearsal.md | 184 ++++++++++++++++++++++++++++++++++ package.json | 3 +- scripts/release-gate-smoke.ts | 72 +++++++++++++ tests/workflow-config.test.ts | 30 ++++++ 7 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 docs/restore-rehearsal.md create mode 100644 scripts/release-gate-smoke.ts create mode 100644 tests/workflow-config.test.ts diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 8d979bb..42b4042 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -26,6 +26,8 @@ jobs: run: APP_MODE=demo APP_RELEASE_ID=fixture bun run release:check - name: Isolated fixture smoke run: APP_MODE=demo APP_RELEASE_ID=fixture bun run test:fixture + - name: Disabled capability HTTP smoke + run: APP_MODE=demo APP_RELEASE_ID=fixture bun run test:release-gates - name: Strict staging release safety check run: >- RELEASE_ENV=staging APP_MODE=live diff --git a/README.md b/README.md index ab45119..a0080fc 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ docs/research/ current official-source evidence and captured ESI con - [Convex/Bun platform evidence](docs/research/platform.md) - [Security invariants](docs/security.md) - [Executed checks and remaining verification gates](docs/verification.md) +- [Isolated restore/migration rehearsal runbook](docs/restore-rehearsal.md) ## Operational limits diff --git a/docs/release-safety.md b/docs/release-safety.md index d371701..a6cd8f3 100644 --- a/docs/release-safety.md +++ b/docs/release-safety.md @@ -77,6 +77,15 @@ and remove the project only after recording the result. The release gate does not itself perform migrations, backups, restores, or cleanup of an operator's persistent deployment. +## Restore and migration evidence + +The synthetic restore/migration rehearsal is specified in +[`docs/restore-rehearsal.md`](restore-rehearsal.md). It includes Convex data +and instance-credential backup/restore, application rollback, migration +rollback prerequisites, session/grant invalidation decisions, resource +budgets, and a redacted evidence template. It has not been executed in this +lane. + ## Evidence boundaries Read-only `tea` inspection on 2026-09-19 reported no Gitea workflows or runs diff --git a/docs/restore-rehearsal.md b/docs/restore-rehearsal.md new file mode 100644 index 0000000..6b90e02 --- /dev/null +++ b/docs/restore-rehearsal.md @@ -0,0 +1,184 @@ +# Isolated application/Convex restore rehearsal (T01) + +This is a runbook and evidence template, not an executed restore. It is +intended for a disposable self-hosted Convex deployment and synthetic users +only. Never point it at the persistent development deployment, copy a live +`.env`, or use production Infisical credentials. + +## Scope and ownership + +T01 owns the application/Compose release procedure, Convex data and instance +credential backup/restore, configuration/key backup handling, and post-restore +session/grant invalidation evidence. T03 owns immutable reference artifact +rollback and profile-compatible `PREVIOUS` handling; this runbook does not +change or claim that path. + +The current repository has no automated Convex migration rollback command and +no implemented global post-restore session/grant invalidation migration. Those +are explicit blockers below, not steps that may be silently skipped. + +## Prerequisites and isolation + +- A clean T01 worktree at a recorded commit; Bun 1.3.14 and Docker/Compose. +- A unique project name, for example `eden-restore-$USER-$RANDOM`, and unique + volume names. Keep published ports on loopback and use a disposable backend. +- At least **2x the current Convex data volume bytes plus 10%** free disk for + source volume, backup archive, and restored volume. Record `du -sb` before + starting. +- Budget **2 CPUs, 4 GiB RAM, and 30 minutes** for the disposable backend and + application rehearsal. A reference rebuild is a separate T03 operation; + the reviewed production worker budget is 2 CPUs/4 GiB and must not be + assumed as Convex restore evidence. +- A temporary mode-0600 directory outside the repository for synthetic keys, + backup archives, logs, and evidence. Do not print key values or put them in + command arguments. + +```sh +set -euo pipefail +repo="$PWD" +project="eden-restore-${USER:-operator}-$(date +%s)-$$" +work="$(mktemp -d "${TMPDIR:-/tmp}/eden-restore.XXXXXX")" +chmod 700 "$work" +trap 'docker compose -p "$project" down -v >/dev/null 2>&1 || true; rm -rf "$work"' EXIT +(cd "$work" && bun "$repo/scripts/generate-keys.ts") +chmod 600 "$work/.env.keys" +``` + +Load only those synthetic values into a temporary shell environment. Set +`APP_MODE=demo`, `APP_RELEASE_ID=restore-fixture`, +`APP_CAPABILITY_ALLOWLIST=asset_search,asset_valuation,asset_sync,assistant_dispatch,session_token,identity,subscriptions`, +and an issuer such as `http://127.0.0.1:18080`. Do not use a production +callback or provider key. + +## Backup rehearsal + +1. Start a disposable backend with a unique Compose project and record the + backend image digest, git commit, volume name, and container ID. + + ```sh + export COMPOSE_PROJECT_NAME="$project" + docker compose -p "$project" up -d backend + volume="${project}_convex-data" + mount="$(docker volume inspect -f '{{ .Mountpoint }}' "$volume")" + du -sb "$mount" + ``` + +2. Configure the synthetic Convex environment using the temporary key file, + synthetic issuer/JWKS, bridge secret, and allowlist. Use the existing + `bun run convex:configure` path only with the disposable URL/admin key; + never pass a persistent admin key through this procedure. + +3. Run the existing synthetic write smoke against this backend, with + `SMOKE_ALLOW_WRITE=1`, and record its output without recording secrets. It + exercises login/session, staged publication, failed publication preserving + the previous generation, tenant isolation, JWT audience checks, and local + revocation. + +4. Create a consistent filesystem backup while the disposable backend is + stopped. The Convex data volume contains the backend data and instance + credential material; back up it as one unit. A host administrator may need + root access to the Docker volume mount. + + ```sh + docker compose -p "$project" stop backend + tar --xattrs --acls --numeric-owner -C "$mount" \ + -czf "$work/convex-data.tar.gz" . + sha256sum "$work/convex-data.tar.gz" > "$work/convex-data.tar.gz.sha256" + cp --preserve=mode "$work/.env.keys" "$work/application-keys.synthetic" + sha256sum "$work/application-keys.synthetic" > "$work/application-keys.synthetic.sha256" + ``` + + Record archive byte size, SHA-256, volume size, and stop/start timestamps. + Do not include raw keys in the evidence bundle; retain the mode-0600 file in + the protected temporary directory until cleanup. + +## Restore and isolation checks + +1. Create a second unique Compose project and volume. Extract only the + recorded archive into the new volume, preserving ownership and xattrs. + + ```sh + restore_project="${project}-restored" + restore_volume="${restore_project}_convex-data" + docker volume create "$restore_volume" >/dev/null + restore_mount="$(docker volume inspect -f '{{ .Mountpoint }}' "$restore_volume")" + tar --xattrs --acls --numeric-owner -C "$restore_mount" \ + -xzf "$work/convex-data.tar.gz" + ``` + +2. Start the restored backend under the second project and configure the web + container with **only** the synthetic application keys. Confirm that the + restored instance credential is the backed-up credential, not a newly + generated one; record only a redacted fingerprint/metadata, never its + value. + +3. Repeat the synthetic Convex smoke with fresh synthetic IDs. Verify: + + - restored schema/functions start without a migration error; + - a newly created synthetic user cannot read the source project's records; + - restored records remain tenant-isolated; + - the restored reference/config volume is separate and mounted read-only by + web; + - the old pre-backup session/JWT is rejected after the approved invalidation + step; and + - the old encrypted grant cannot service synchronization until explicit + reauthorization. + +4. **Current blocker:** the repository does not yet provide a global + post-restore session/grant invalidation command. Do not claim this check + passed. The operator must choose and approve one of: a Convex migration that + revokes all sessions and quarantines token records, a deployment-specific + credential-version fence, or an equivalent bounded mechanism. Record the + maximum stale-authority window and the decision before reopening traffic. + +5. Test application rollback by stopping the restored web service, starting the + immediately previous digest-pinned web image against the restored backend, + and exercising read-only synthetic status/assets paths. Do not use a mutable + `latest` tag. If the schema is not backward compatible, record rollback as + blocked; do not mutate the source volume to make it pass. + +## Migration rollback rehearsal + +There is no migration history/rollback command in this baseline. A valid +rehearsal therefore requires a parent-approved migration fixture and a chosen +rollback mechanism. At minimum: + +1. back up the disposable volume before migration; +2. deploy the migration to the disposable backend; +3. run schema/functions and tenant-isolation checks; +4. restore the pre-migration volume into a fresh project (never in place); +5. start the previous application image and rerun the checks; and +6. record whether the migration is backward compatible and which data loss, + RPO, and RTO limits apply. + +A `docker compose down -v` or a successful process restart is not migration +rollback evidence. T03's reference artifact rollback is a separate test and +must verify profile, importer version, manifest integrity, and live-reader +compatibility. + +## Evidence template + +Copy this table into the release record and attach only redacted logs/checksums: + +| Field | Record | +|---|---| +| Worktree commit / image digests | | +| Source and restored Compose project names | | +| Convex source/restored volume names | | +| Synthetic key IDs/fingerprints (not values) | | +| Backup archive bytes / SHA-256 | | +| Pre/post volume byte counts | | +| Schema/migration version and rollback mechanism | | +| Session/grant invalidation decision and stale window | | +| Commands and timestamps (UTC) | | +| Synthetic smoke result | | +| Tenant-isolation result | | +| Old-session rejection result | | +| Old-grant sync rejection result | | +| Browser/provider/restore steps not exercised | | +| Cleanup confirmation and retained artifacts/expiry | | +| Operator approval and unresolved blockers | | + +The evidence must label mock, fixture, disposable-backend, and consenting-live +observations separately. Passing local tests, CI, or a fixture container does +not constitute a restore, browser, provider, or production-readiness claim. diff --git a/package.json b/package.json index 99ff8da..cc37d35 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "secrets:status": "bun scripts/secrets.ts status", "secrets:run": "bun scripts/secrets.ts run", "release:check": "bun scripts/release-check.ts", - "test:fixture": "bun scripts/fixture-smoke.ts" + "test:fixture": "bun scripts/fixture-smoke.ts", + "test:release-gates": "bun scripts/release-gate-smoke.ts" }, "dependencies": { "@sveltejs/kit": "2.70.3", diff --git a/scripts/release-gate-smoke.ts b/scripts/release-gate-smoke.ts new file mode 100644 index 0000000..cee7d88 --- /dev/null +++ b/scripts/release-gate-smoke.ts @@ -0,0 +1,72 @@ +import { createServer } from 'node:net'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const capabilities = 'asset_search,asset_valuation'; +const safeEnv = { PATH: process.env.PATH ?? '/usr/bin:/bin', HOME: process.env.HOME ?? '/tmp', TMPDIR: process.env.TMPDIR ?? '/tmp' }; +async function freePort(): Promise { + return await new Promise((resolve, reject) => { + const probe = createServer(); + probe.once('error', reject); + probe.listen(0, '127.0.0.1', () => { + const address = probe.address(); + const port = typeof address === 'object' && address ? address.port : 0; + probe.close((error) => error ? reject(error) : resolve(port)); + }); + }); +} +const port = await freePort(); +const base = `http://127.0.0.1:${port}`; +const releaseId = `gate-${crypto.randomUUID()}`; +const referenceDir = await mkdtemp(join(tmpdir(), 'eden-gate-reference-')); +let server: Bun.Subprocess | undefined; +async function run(args: string[], env: Record): Promise { + const child = Bun.spawn(args, { env: { ...safeEnv, ...env }, stdout: 'pipe', stderr: 'pipe', stdin: 'ignore' }); + if (await child.exited !== 0) throw new Error(`Command failed: ${args.join(' ')}`); +} +async function waitForServer(): Promise { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + const response = await fetch(base); + if (response.status === 200 && response.headers.get('x-application-release') === releaseId) return; + } catch { /* startup race */ } + await Bun.sleep(250); + } + throw new Error('Rollout-gate smoke server did not become ready'); +} +function assertStatus(response: Response, expected: number, label: string): void { + if (response.status !== expected) throw new Error(`${label}: expected ${expected}, got ${response.status}`); +} +try { + await run(['bun', 'workers/reference.ts', '--fixture', '--directory', referenceDir], { APP_MODE: 'demo', REFERENCE_DIR: referenceDir }); + server = Bun.spawn(['bun', 'build/index.js'], { + env: { + ...safeEnv, APP_MODE: 'demo', APP_ORIGIN: base, ORIGIN: base, HOST: '127.0.0.1', PORT: String(port), + REFERENCE_DIR: referenceDir, APP_RELEASE_ID: releaseId, APP_CAPABILITY_ALLOWLIST: capabilities, + TYPESAFE_API_KEY: 'sentinel-not-used-when-assistant-is-disabled', + }, stdout: 'pipe', stderr: 'pipe', stdin: 'ignore', + }); + await waitForServer(); + const headers = { origin: base, 'content-type': 'application/json' }; + const chat = await fetch(`${base}/api/chat`, { method: 'POST', headers, body: JSON.stringify({ message: '/value', requestId: crypto.randomUUID() }) }); + assertStatus(chat, 503, 'assistant dispatch'); + if (!(await chat.text()).includes('assistant_dispatch')) throw new Error('assistant gate reason missing'); + const sync = await fetch(`${base}/api/sync`, { method: 'POST', headers: { origin: base } }); + assertStatus(sync, 503, 'asset sync'); + const token = await fetch(`${base}/api/session/token`); + assertStatus(token, 503, 'session token'); + const characters = await fetch(`${base}/characters`); + assertStatus(characters, 200, 'character page'); + if (!(await characters.text()).includes('identity')) throw new Error('identity gate reason missing'); + const oauth = await fetch(`${base}/auth/eve`, { method: 'POST', headers: { origin: base, 'content-type': 'application/x-www-form-urlencoded' }, redirect: 'manual' }); + assertStatus(oauth, 503, 'OAuth initiation'); + const callback = await fetch(`${base}/auth/eve/callback?state=synthetic`, { redirect: 'manual' }); + assertStatus(callback, 303, 'OAuth callback'); + if (!callback.headers.get('location')?.includes('auth=disabled')) throw new Error('OAuth callback gate reason missing'); + console.log(`Rollout-gate HTTP smoke passed: disabled capabilities returned fail-closed responses; ${base}`); +} finally { + if (server) { server.kill('SIGTERM'); await server.exited; } + await rm(referenceDir, { recursive: true, force: true }); +} diff --git a/tests/workflow-config.test.ts b/tests/workflow-config.test.ts new file mode 100644 index 0000000..3fab530 --- /dev/null +++ b/tests/workflow-config.test.ts @@ -0,0 +1,30 @@ +import { expect, test } from 'bun:test'; +import { readFileSync } from 'node:fs'; + +const workflow = readFileSync('.gitea/workflows/ci.yml', 'utf8'); +const compose = readFileSync('compose.yaml', 'utf8'); +const dockerfile = readFileSync('Dockerfile', 'utf8'); +const dockerignore = readFileSync('.dockerignore', 'utf8'); + +test('Gitea workflow separates PR source checks from trusted container builds', () => { + expect(workflow).toContain('pull_request:'); + expect(workflow).toContain("if: ${{ gitea.event_name == 'push' }}"); + expect(workflow).toContain('RELEASE_ENV=staging'); + expect(workflow).toContain('bun run test:release-gates'); + expect(workflow).toContain('docker compose config --quiet'); + expect(workflow).toContain('docker build --build-arg APP_RELEASE_ID=fixture-ci'); + expect(workflow).toContain('--network none'); +}); + +test('Compose and Docker defaults are isolated and digest-pinned', () => { + expect(compose).toContain('CONVEX_BIND_ADDRESS:-127.0.0.1'); + expect(compose).toContain('WEB_BIND_ADDRESS:-127.0.0.1'); + expect(compose).toMatch(/CONVEX_IMAGE:-[^\n]*@sha256:[a-f0-9]{64}/); + expect(compose).toMatch(/CONVEX_DASHBOARD_IMAGE:-[^\n]*@sha256:[a-f0-9]{64}/); + const bases = [...dockerfile.matchAll(/^FROM\s+([^\s]+)/gim)].map((match) => match[1]); + expect(bases.length).toBeGreaterThan(0); + expect(bases.every((image) => /@sha256:[a-f0-9]{64}$/i.test(image))).toBe(true); + for (const pattern of ['*.pem', '*.key', '*.p12', '*.pfx', 'credentials*', 'secrets*', '.npmrc']) { + expect(dockerignore).toContain(pattern); + } +});