From 76013656214c321e8758eaa2d398571c8c19569c Mon Sep 17 00:00:00 2001 From: JB Date: Sat, 19 Sep 2026 02:56:24 -0400 Subject: [PATCH] fix(reference): verify station coverage and rollback inputs --- README.md | 2 +- src/lib/server/reference/compiler.ts | 19 ++++++++++++++----- tests/reference-readers.test.ts | 27 ++++++++++++++++++++++++++- tests/reference.test.ts | 19 +++++++++++++++++++ workers/check-reference.ts | 10 +++++++++- 5 files changed, 69 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index cf5b3dd..ab45119 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ docs/research/ current official-source evidence and captured ESI con ## Operational limits - The configured development deployment has passed live TypeSafe routing and public Convex WebSocket checks; the operator has confirmed real EVE sign-in, asset synchronization, and matching Assets/chat valuation totals for one character. Live multi-character linking and revocation/transfer scenarios remain separate checks. -- Production imports use reviewed source adapters and immutable SQLite artifacts. Station data is not imported; 21 dangling blueprint references in the reviewed release are preserved explicitly rather than fabricated. See reference operations and the verification record for executed import checks and limitations. +- Production imports use reviewed source adapters and immutable SQLite artifacts. CCP NPC station IDs, system/type/owner/position facts are imported when `npcStations.jsonl` is present; station names remain optional and the production profile explicitly marks station coverage incomplete until an isolated measured replay. Twenty-one dangling blueprint references in the reviewed release are preserved explicitly rather than fabricated. See reference operations and the verification record for executed import checks and limitations. - Asset synchronization runs on demand in a single Bun web process. Convex leases/CAS protect staging and token rotation, but durable background character-sync job recovery and scheduled refresh are future work. - ESI page collections are not transactional snapshots. Stability checks reduce risk; cache rollover/mutation may require a later retry. - Location IDs are shown explicitly as unresolved. Fitted/nested assets are distinct ESI items; no contents multiplier is applied to ship prices. diff --git a/src/lib/server/reference/compiler.ts b/src/lib/server/reference/compiler.ts index 449b8f1..8401ad3 100644 --- a/src/lib/server/reference/compiler.ts +++ b/src/lib/server/reference/compiler.ts @@ -339,7 +339,7 @@ export class ReferenceCompiler { if (!manifest.active || !manifest.previous) throw new Error('no previous reference artifact is available for rollback'); const previousPath = artifactDatabasePath(this.directory, manifest.previous.database); if (!await exists(previousPath)) throw new Error('previous reference database is missing'); - validateRollbackArtifact(previousPath, manifest.previous, this.profile, this.importerVersion, this.thresholds); + await validateRollbackArtifact(this.directory, previousPath, manifest.previous, this.profile, this.importerVersion, this.thresholds); const next: ReferenceManifest = { ...manifest, active: manifest.previous, previous: manifest.active, lastAttempt: { startedAt: iso(this.now()), finishedAt: iso(this.now()), status: 'success', profile: manifest.previous.profile, version: manifest.previous.version } }; await atomicWriteJson(this.directory, MANIFEST_FILE, next); return { activeVersion: next.active!.version, previousVersion: next.previous?.version }; @@ -347,19 +347,28 @@ export class ReferenceCompiler { } } -function validateRollbackArtifact(path: string, artifact: ActiveManifest, expectedProfile: BuildProfile, expectedImporterVersion: string, thresholds: ValidationThresholds): void { - if (artifact.profile !== expectedProfile || artifact.importerVersion !== expectedImporterVersion || artifact.stationCoverage !== thresholds.stationCoverage) throw new Error('previous reference artifact is incompatible with the current compiler/profile'); +async function validateRollbackArtifact(directory: string, path: string, artifact: ActiveManifest, expectedProfile: BuildProfile, expectedImporterVersion: string, thresholds: ValidationThresholds): Promise { + if (artifact.profile !== expectedProfile || artifact.importerVersion !== expectedImporterVersion || artifact.stationCoverage !== thresholds.stationCoverage || basename(path) !== `${artifact.version}.sqlite`) throw new Error('previous reference artifact is incompatible with the current compiler/profile'); if (!artifact.sourceInputs || !artifact.counts) throw new Error('previous reference artifact is missing compatibility metadata'); let db: Database | undefined; try { + for (const source of REFERENCE_SOURCES) { + const expected = artifact.sourceInputs[source]; + if (!expected) throw new Error(`previous reference artifact is missing source input metadata for ${source}`); + await readSourceCache(directory, source, expected.cacheFile, cacheExpectation(source, expected, expectedImporterVersion, expectedProfile)); + } db = new Database(path, { readonly: true, strict: true }); const counts = validateCandidate(db, thresholds, artifact.profile); - for (const [table, expected] of Object.entries(artifact.counts)) if (counts[table] !== expected) throw new Error(`previous reference artifact count mismatch for ${table}: expected ${expected}, got ${counts[table]}`); + for (const key of Object.keys(counts)) { + if (!(key in artifact.counts)) throw new Error(`previous reference artifact is missing count metadata for ${key}`); + const expected = artifact.counts[key]; + if (counts[key] !== expected) throw new Error(`previous reference artifact count mismatch for ${key}: expected ${expected}, got ${counts[key]}`); + } const rows = db.query<{ source: string; fingerprint: string; local_sha256: string }, []>('SELECT source,fingerprint,local_sha256 FROM source_provenance').all(); const bySource = new Map(rows.map((row) => [row.source, row])); for (const source of REFERENCE_SOURCES) { const expected = artifact.sourceInputs[source]; const row = bySource.get(source); - if (!expected || !row || row.fingerprint !== expected.fingerprint || row.local_sha256 !== expected.localSha256) throw new Error(`previous reference artifact provenance mismatch for ${source}`); + if (!row || row.fingerprint !== expected.fingerprint || row.local_sha256 !== expected.localSha256) throw new Error(`previous reference artifact provenance mismatch for ${source}`); } } finally { try { db?.close(); } catch { /* best effort */ } } } diff --git a/tests/reference-readers.test.ts b/tests/reference-readers.test.ts index 5dad1af..3ccb325 100644 --- a/tests/reference-readers.test.ts +++ b/tests/reference-readers.test.ts @@ -2,7 +2,7 @@ import { expect, test } from 'bun:test'; import { cp, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; -import { ReferenceCompiler, SqliteReferenceRepository } from '../src/lib/server/reference'; +import { ReferenceCompiler, SqliteReferenceRepository, readReferenceManifest } from '../src/lib/server/reference'; import { fixtureSources } from '../src/lib/server/reference/sources'; test('reference readers search normalized types and expose group/category/system/region/station joins', async () => { @@ -57,6 +57,31 @@ test('readers reject empty type searches and preserve unknown station metadata', } finally { await rm(root, { recursive: true, force: true }); } }); +test('offline production-profile candidate records verified station counts', async () => { + const root = await mkdtemp(join(tmpdir(), 'eden-reference-production-stations-')); + const sources = join(root, 'fixtures'); + const directory = join(root, 'reference'); + try { + await cp('fixtures/reference', sources, { recursive: true }); + const universePath = join(sources, 'systems.json'); + const universe = JSON.parse(await readFile(universePath, 'utf8')) as Record; + universe.regions = { '10000001': { region_id: 10000001, name: 'The Forge' } }; + universe.constellations = { '20000020': { constellation_id: 20000020, name: 'Kimotoro', region_id: 10000001 } }; + universe.stations = { '60003760': { station_id: 60003760, system_id: 30000142, type_id: 587, owner_id: 1000035, position: { x: 1, y: 2, z: 3 } } }; + await writeFile(universePath, JSON.stringify(universe)); + const result = await new ReferenceCompiler({ directory, profile: 'production', thresholds: { systemsMinimum: 1, stationsMinimum: 1 }, sources: fixtureSources(sources) }).compile(); + expect(result.status).toBe('success'); + const manifest = await readReferenceManifest(directory); + expect(manifest.active?.profile).toBe('production'); + expect(manifest.active?.counts.stations).toBe(1); + expect(manifest.active?.counts.verified_stations).toBe(1); + expect(manifest.active?.stationCoverage).toBe('incomplete'); + const reader = new SqliteReferenceRepository(directory, 'production'); + try { expect((await reader.stations([60003760]))[0]?.sourceVerified).toBe(true); } + finally { reader.close(); } + } finally { await rm(root, { recursive: true, force: true }); } +}); + test('unknown system station IDs cannot satisfy verified production station coverage', async () => { const root = await mkdtemp(join(tmpdir(), 'eden-reference-station-gate-')); const sources = join(root, 'fixtures'); diff --git a/tests/reference.test.ts b/tests/reference.test.ts index 46fd65d..d89ae47 100644 --- a/tests/reference.test.ts +++ b/tests/reference.test.ts @@ -321,6 +321,25 @@ describe('fixture reference compiler', () => { } finally { await rm(root, { recursive: true, force: true }); } }); + test('rejects incomplete PREVIOUS count metadata before rollback', async () => { + const { root, sources } = await fixtureDirectory(); + const directory = join(root, 'reference'); + try { + const first = await new ReferenceCompiler({ directory, profile: 'fixture', sources: fixtureSources(sources) }).compile(); + expect(first.status).toBe('success'); + if (first.status !== 'success') throw new Error('initial fixture build did not publish'); + await writeFile(join(sources, 'reference-data.json'), `${await readFile(join(sources, 'reference-data.json'), 'utf8')}\n`); + const second = await new ReferenceCompiler({ directory, profile: 'fixture', sources: fixtureSources(sources) }).compile(); + expect(second.status).toBe('success'); + const manifest = await readReferenceManifest(directory); + if (!manifest.previous) throw new Error('fixture build did not retain PREVIOUS'); + const malformed = { ...manifest, previous: { ...manifest.previous, counts: Object.fromEntries(Object.entries(manifest.previous.counts).filter(([key]) => key !== 'stations')) } }; + await writeFile(join(directory, 'manifest.json'), `${JSON.stringify(malformed)}\n`); + await expect(new ReferenceCompiler({ directory, profile: 'fixture', sources: fixtureSources(sources) }).rollback()).rejects.toThrow(/missing count metadata/); + expect((await readReferenceManifest(directory)).active?.version).toBe(manifest.active?.version); + } finally { await rm(root, { recursive: true, force: true }); } + }); + test('recovers a validated orphan database after manifest publication crash', async () => { const { root, sources } = await fixtureDirectory(); const directory = join(root, 'reference'); diff --git a/workers/check-reference.ts b/workers/check-reference.ts index 067592c..7692f7c 100644 --- a/workers/check-reference.ts +++ b/workers/check-reference.ts @@ -13,7 +13,7 @@ try { assert.equal(names.get(34), 'Tritanium'); assert.equal(names.get(587), 'Rifter'); const counts: Record = {}; - for (const table of ['types', 'blueprints', 'systems', 'gates', 'unresolved_references']) { + for (const table of ['types', 'blueprints', 'systems', 'gates', 'stations', 'unresolved_references']) { const row = db.query<{ n: number }, []>(`SELECT COUNT(*) AS n FROM ${table}`).get(); counts[table] = Number(row?.n); assert.equal(counts[table], manifest.active.counts[table], `${table} must match the published manifest`); @@ -22,6 +22,14 @@ try { assert(counts.systems > 5_000); assert(counts.blueprints > 0); assert(counts.gates > 0); + const verifiedStations = Number((db.query<{ n: number }, []>('SELECT COUNT(*) AS n FROM stations WHERE source_verified = 1').get() as { n: number }).n); + assert.equal(verifiedStations, manifest.active.counts.verified_stations, 'verified station count must match the published manifest'); + assert(verifiedStations > 0); + const joinedStations = Number((db.query<{ n: number }, []>('SELECT COUNT(*) AS n FROM stations st JOIN systems s ON s.system_id = st.system_id JOIN constellations c ON c.constellation_id = s.constellation_id JOIN regions r ON r.region_id = c.region_id').get() as { n: number }).n); + assert.equal(joinedStations, counts.stations, 'every station must join through system, constellation, and region'); + const verifiedStation = (await repository.stations({ limit: 1 })).find((station) => station.sourceVerified); + assert(verifiedStation, 'reader must expose a verified station row'); + assert.deepEqual(await repository.stations([Number.MAX_SAFE_INTEGER]), [], 'unknown station IDs must remain absent rather than fabricated'); const integrity = db.query<{ integrity_check: string }, []>('PRAGMA integrity_check').all(); assert.deepEqual(integrity.map(row => row.integrity_check), ['ok']); assert.deepEqual(db.query('PRAGMA foreign_key_check').all(), []);