jb/eden
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(reference): preserve station evidence and rollback schema

Browse Source
This commit is contained in:
jb
2026-09-19 09:53:46 -04:00
parent f7b604f40f
commit 24660edb09
8 changed files with 83 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/server/reference/compiler.ts
+5 -4
View File
@@ -3,7 +3,7 @@ import { createHash } from 'node:crypto';
import { access, mkdir, open, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises';
import { basename, isAbsolute, join, resolve } from 'node:path';
import { normalizeAll } from './normalize';
import { createReferenceSchema, insertHoboleaksData, insertReferenceData, insertSourceMetadata, insertUniverseData, SCHEMA_VERSION } from './schema';
import { assertReferenceReaderSchema, createReferenceSchema, insertHoboleaksData, insertReferenceData, insertSourceMetadata, insertUniverseData, SCHEMA_VERSION } from './schema';
import { fixtureSources, sha256 } from './sources';
import type {
ActiveManifest,
@@ -153,7 +153,7 @@ export class ReferenceCompiler {
failureResults = sourceResults;
const attempts: Partial<Record<ReferenceSourceName, SourceAttempt>> = {};
const loaded: Partial<Record<ReferenceSourceName, LoadedSource>> = {};
let changed = !manifest.active || manifest.active.profile !== this.profile || manifest.active.importerVersion !== this.importerVersion;
let changed = !manifest.active || manifest.active.profile !== this.profile || manifest.active.importerVersion !== this.importerVersion || manifest.active.stationCoverage !== this.thresholds.stationCoverage;
const currentFailures: string[] = [];
for (const source of REFERENCE_SOURCES) {
@@ -264,7 +264,7 @@ export class ReferenceCompiler {
return [source, { fingerprint: item.check.fingerprint!, cacheFile: item.cacheFile, localSha256: item.localSha256, canonicalSha256: item.canonicalSha256, archiveByteLength: item.archiveByteLength, checkedAt: item.check.checkedAt, url: item.check.url, etag: item.check.etag, etagIsChecksum: item.check.etagIsChecksum, lastModified: item.check.lastModified, byteLength: item.check.byteLength, importerVersion: this.importerVersion, expectedSha256: item.expectedSha256, sourceRevision: item.check.sourceRevision, sourceBuild: item.check.sourceBuild, stale: item.check.stale } satisfies ActiveSourceInput];
})) as Record<ReferenceSourceName, ActiveSourceInput>;
const committedSources = commitSourceResults(manifest.sources, sourceResults, loaded, this.importerVersion, this.now);
const active: ActiveManifest = { version, database: join(DATABASE_DIRECTORY, `${version}.sqlite`), createdAt: iso(this.now()), profile: this.profile, importerVersion: this.importerVersion, sources: Object.fromEntries(REFERENCE_SOURCES.map((source) => [source, loaded[source]!.check.fingerprint!])) as Record<ReferenceSourceName, string>, sourceInputs, counts };
const active: ActiveManifest = { version, database: join(DATABASE_DIRECTORY, `${version}.sqlite`), createdAt: iso(this.now()), profile: this.profile, importerVersion: this.importerVersion, sources: Object.fromEntries(REFERENCE_SOURCES.map((source) => [source, loaded[source]!.check.fingerprint!])) as Record<ReferenceSourceName, string>, sourceInputs, counts, stationCoverage: this.thresholds.stationCoverage };
const sourceErrors = Object.values(committedSources).filter((result) => result?.status === 'failed').map((result) => `${result!.source}: ${result!.error ?? 'source failed'}`);
const next: ReferenceManifest = { format: 1, active, previous: manifest.active, sources: committedSources, lastAttempt: { startedAt, finishedAt: iso(this.now()), status: 'success', profile: this.profile, version, ...(sourceErrors.length ? { error: sourceErrors.join('; ') } : {}) } };
await atomicWriteJson(this.directory, MANIFEST_FILE, next);
@@ -348,7 +348,7 @@ export class ReferenceCompiler {
}
function validateRollbackArtifact(path: string, artifact: ActiveManifest, expectedProfile: BuildProfile, expectedImporterVersion: string, thresholds: ValidationThresholds): void {
if (artifact.profile !== expectedProfile || artifact.importerVersion !== expectedImporterVersion) throw new Error('previous reference artifact is incompatible with the current compiler/profile');
if (artifact.profile !== expectedProfile || artifact.importerVersion !== expectedImporterVersion || artifact.stationCoverage !== thresholds.stationCoverage) throw new Error('previous reference artifact is incompatible with the current compiler/profile');
if (!artifact.sourceInputs || !artifact.counts) throw new Error('previous reference artifact is missing compatibility metadata');
let db: Database | undefined;
try {
@@ -538,6 +538,7 @@ async function validateExistingArtifact(path: string, input: ReturnType<typeof n
export function validateCandidate(db: Database, thresholds: ValidationThresholds, profile: string): Record<string, number> {
const schemaVersion = Number((db.query<{ user_version: number }, []>('PRAGMA user_version').get() as { user_version: number } | null)?.user_version ?? 0);
if (schemaVersion !== SCHEMA_VERSION) throw new Error(`SQLite schema version ${schemaVersion} does not match compiler ${SCHEMA_VERSION}`);
assertReferenceReaderSchema(db);
const integrity = db.query<{ integrity_check: string }, []>('PRAGMA integrity_check').all();
if (integrity.length !== 1 || integrity[0].integrity_check !== 'ok') throw new Error(`SQLite integrity_check failed: ${JSON.stringify(integrity)}`);
const foreignKeys = db.query<unknown, []>('PRAGMA foreign_key_check').all(); if (foreignKeys.length) throw new Error(`SQLite foreign_key_check failed: ${JSON.stringify(foreignKeys)}`);
src/lib/server/reference/repository.ts
+10 -27
View File
@@ -22,19 +22,19 @@ import type {
ReferenceTypeRecord,
} from './types';
import { MANIFEST_FILE, DATABASE_DIRECTORY } from './compiler';
import { SCHEMA_VERSION } from './schema';
import { assertReferenceReaderSchema } from './schema';
const MAX_IN_BATCH = 500;
const MAX_READER_LIMIT = 500;
const MAX_READER_OFFSET = 100_000;
type TypeRow = { type_id: number; name_json: string; category_id: number; volume: number | null };
type TypeRow = { type_id: number; name_json: string; category_id: number; group_id: number; volume: number | null };
type SearchTypeRow = { type_id: number; name: string; category_id: number; group_id: number; published: number | null; volume: number | null };
type CategoryRow = { category_id: number; name_json: string; published: number | null };
type GroupRow = { group_id: number; category_id: number; name_json: string; published: number | null };
type RegionRow = { region_id: number; name: string | null };
type ConstellationRow = { constellation_id: number; name: string | null; region_id: number | null };
type SystemRow = { system_id: number; name: string; constellation_id: number; region_id: number | null; x: number; y: number; z: number; security_status: number; security_class: string | null };
type StationRow = { station_id: number; system_id: number; system_name: string | null; region_id: number | null; name: string | null; type_id: number | null; owner_id: number | null; x: number | null; y: number | null; z: number | null };
type StationRow = { station_id: number; system_id: number; system_name: string | null; region_id: number | null; name: string | null; type_id: number | null; owner_id: number | null; x: number | null; y: number | null; z: number | null; source_verified: number };
type ListSelection = ReferenceListOptions;
type IdSelection = ReferenceIdSelection;
type GroupSelection = ReferenceGroupSelection;
@@ -75,6 +75,7 @@ export class SqliteReferenceRepository implements ReferenceReader, ReferenceSnap
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as ReferenceManifest;
if (manifest.format !== 1 || !manifest.active) throw new Error(`reference manifest has no active artifact: ${manifestPath}`);
if (expectedProfile && manifest.active.profile !== expectedProfile) throw new Error(`reference artifact must use the ${expectedProfile} profile`);
if (manifest.active.stationCoverage !== 'incomplete' && manifest.active.stationCoverage !== 'measured') throw new Error('reference artifact is missing station coverage profile metadata');
this.pinnedVersion = manifest.active.version;
this.databasePath = assertArtifactPath(root, manifest.active.database);
if (basename(this.databasePath) !== `${this.pinnedVersion}.sqlite`) throw new Error('active reference database filename does not match its manifest version');
@@ -191,31 +192,13 @@ function queryStations(db: Database, selection: StationSelection): Promise<Refer
if (ids) { const clause = inClause(ids, 'st.station_id'); conditions.push(clause.sql); args.push(...clause.args); }
const systemId = optionValue(selection, 'systemId'); if (typeof systemId === 'number') { conditions.push('st.system_id = ?'); args.push(systemId); }
const query = optionValue(selection, 'query'); if (typeof query === 'string' && query.trim()) { conditions.push("st.name LIKE ? ESCAPE '\\'"); args.push(like(query)); }
const rows = db.query<StationRow, (string | number)[]>(`SELECT st.station_id,st.system_id,s.name AS system_name,c.region_id,st.name,st.type_id,st.owner_id,st.x,st.y,st.z FROM stations st JOIN systems s ON s.system_id = st.system_id LEFT JOIN constellations c ON c.constellation_id = s.constellation_id${conditions.length ? ` WHERE ${conditions.join(' AND ')}` : ''} ORDER BY st.station_id LIMIT ? OFFSET ?`).all(...args, limit(Number(optionValue(selection, 'limit'))), bounded(Number(optionValue(selection, 'offset')), 0));
return Promise.resolve(rows.map((row) => ({ id: row.station_id, systemId: row.system_id, systemName: row.system_name, regionId: row.region_id, name: row.name, typeId: row.type_id, ownerId: row.owner_id, x: row.x, y: row.y, z: row.z })));
const rows = db.query<StationRow, (string | number)[]>(`SELECT st.station_id,st.system_id,s.name AS system_name,c.region_id,st.name,st.type_id,st.owner_id,st.x,st.y,st.z,st.source_verified FROM stations st JOIN systems s ON s.system_id = st.system_id LEFT JOIN constellations c ON c.constellation_id = s.constellation_id${conditions.length ? ` WHERE ${conditions.join(' AND ')}` : ''} ORDER BY st.station_id LIMIT ? OFFSET ?`).all(...args, limit(Number(optionValue(selection, 'limit'))), bounded(Number(optionValue(selection, 'offset')), 0));
return Promise.resolve(rows.map((row) => ({ id: row.station_id, systemId: row.system_id, systemName: row.system_name, regionId: row.region_id, sourceVerified: row.source_verified === 1, name: row.name, typeId: row.type_id, ownerId: row.owner_id, x: row.x, y: row.y, z: row.z })));
}
/** Cheap metadata check only; full integrity_check belongs to the compiler. */
/** Reader construction closes malformed handles before surfacing the schema error. */
function assertReaderSchema(db: Database): void {
const version = Number((db.query<{ user_version: number }, []>('PRAGMA user_version').get() as { user_version: number } | null)?.user_version ?? 0);
const row = db.query<{ present: number }, []>("SELECT COUNT(*) AS present FROM sqlite_master WHERE type='table' AND name IN ('types','type_search','type_search_fts','categories','groups','regions','constellations','systems','stations')").get();
const index = db.query<{ present: number }, []>("SELECT COUNT(*) AS present FROM sqlite_master WHERE type='index' AND name='type_search_normalized_name_idx'").get();
const requiredColumns: Record<string, string[]> = {
types: ['type_id', 'group_id', 'category_id', 'name_json', 'published', 'volume'],
type_search: ['type_id', 'language', 'name', 'normalized_name'],
type_search_fts: ['type_id', 'language', 'name', 'normalized_name'],
categories: ['category_id', 'name_json', 'published'],
groups: ['group_id', 'category_id', 'name_json', 'published'],
regions: ['region_id', 'name'],
constellations: ['constellation_id', 'region_id', 'name'],
stations: ['station_id', 'system_id', 'name', 'type_id', 'owner_id', 'x', 'y', 'z', 'source_verified'],
systems: ['system_id', 'constellation_id', 'name', 'x', 'y', 'z', 'security_status', 'security_class'],
};
const columnsValid = Object.entries(requiredColumns).every(([table, columns]) => {
const present = new Set(db.query<{ name: string }, []>(`PRAGMA table_info(${table})`).all().map((column) => column.name));
return columns.every((column) => present.has(column));
});
if (version !== SCHEMA_VERSION || !row || Number(row.present) !== 9 || !index || Number(index.present) !== 1 || !columnsValid) { db.close(); throw new Error(`active reference database has an incompatible schema version (expected ${SCHEMA_VERSION})`); }
try { assertReferenceReaderSchema(db); } catch (error) { db.close(); throw error; }
}
async function queryTypes(db: Database, requested: number[]): Promise<InventoryType[]> {
@@ -227,12 +210,12 @@ async function queryTypes(db: Database, requested: number[]): Promise<InventoryT
for (let offset = 0; offset < unique.length; offset += MAX_IN_BATCH) {
const batch = unique.slice(offset, offset + MAX_IN_BATCH);
const placeholders = batch.map(() => '?').join(',');
rows.push(...db.query<TypeRow, number[]>(`SELECT type_id,name_json,category_id,volume FROM types WHERE type_id IN (${placeholders})`).all(...batch));
rows.push(...db.query<TypeRow, number[]>(`SELECT type_id,name_json,category_id,group_id,volume FROM types WHERE type_id IN (${placeholders})`).all(...batch));
}
const byId = new Map(rows.map((row) => [row.type_id, row]));
return unique.flatMap((id) => {
const row = byId.get(id); if (!row) return [];
const names = JSON.parse(row.name_json) as Record<string, string>;
return [{ id: row.type_id, name: names.en ?? Object.values(names)[0] ?? String(row.type_id), categoryId: row.category_id, ...(row.volume === null ? {} : { volume: row.volume }) }];
return [{ id: row.type_id, name: names.en ?? Object.values(names)[0] ?? String(row.type_id), categoryId: row.category_id, groupId: row.group_id, ...(row.volume === null ? {} : { volume: row.volume }) }];
});
}
src/lib/server/reference/schema.ts
+23
View File
@@ -205,6 +205,29 @@ export function createReferenceSchema(db: Database): void {
`);
}
/** Validate the complete read schema before a runtime handle or rollback can use an artifact. */
export function assertReferenceReaderSchema(db: Database): void {
const version = Number((db.query<{ user_version: number }, []>('PRAGMA user_version').get() as { user_version: number } | null)?.user_version ?? 0);
const row = db.query<{ present: number }, []>("SELECT COUNT(*) AS present FROM sqlite_master WHERE type='table' AND name IN ('types','type_search','type_search_fts','categories','groups','regions','constellations','systems','stations')").get();
const index = db.query<{ present: number }, []>("SELECT COUNT(*) AS present FROM sqlite_master WHERE type='index' AND name='type_search_normalized_name_idx'").get();
const requiredColumns: Record<string, string[]> = {
types: ['type_id', 'group_id', 'category_id', 'name_json', 'published', 'volume'],
type_search: ['type_id', 'language', 'name', 'normalized_name'],
type_search_fts: ['type_id', 'language', 'name', 'normalized_name'],
categories: ['category_id', 'name_json', 'published'],
groups: ['group_id', 'category_id', 'name_json', 'published'],
regions: ['region_id', 'name'],
constellations: ['constellation_id', 'region_id', 'name'],
stations: ['station_id', 'system_id', 'name', 'type_id', 'owner_id', 'x', 'y', 'z', 'source_verified'],
systems: ['system_id', 'constellation_id', 'name', 'x', 'y', 'z', 'security_status', 'security_class'],
};
const columnsValid = Object.entries(requiredColumns).every(([table, columns]) => {
const present = new Set(db.query<{ name: string }, []>(`PRAGMA table_info(${table})`).all().map((column) => column.name));
return columns.every((column) => present.has(column));
});
if (version !== SCHEMA_VERSION || !row || Number(row.present) !== 9 || !index || Number(index.present) !== 1 || !columnsValid) throw new Error(`active reference database has an incompatible schema version (expected ${SCHEMA_VERSION})`);
}
const bool = (value: boolean | undefined): number | null => value === undefined ? null : value ? 1 : 0;
const json = (value: unknown): string => JSON.stringify(value);
const normalizedName = (value: string): string => value.normalize('NFKC').trim().replace(/\s+/g, ' ').toLocaleLowerCase('en-US');
src/lib/server/reference/types.ts
+9 -2
View File
@@ -178,6 +178,8 @@ export interface ReferenceStationRecord {
systemId: number;
systemName: string | null;
regionId: number | null;
/** True only for a station row imported from the authoritative station member. */
sourceVerified: boolean;
name: string | null;
typeId: number | null;
ownerId: number | null;
@@ -260,15 +262,18 @@ export interface ProductionArchiveSourceOptions {
}
export type BuildProfile = 'fixture' | 'production';
/** Station rows are present but not completeness-verified until a measured release replay is captured. */
export type StationCoverageProfile = 'incomplete' | 'measured';
export interface ValidationThresholds {
systemsMinimum: number;
stationsMinimum: number;
stationCoverage: StationCoverageProfile;
requiredCategories: number[];
blueprintsMinimum: number;
}
export const PROFILE_THRESHOLDS: Record<BuildProfile, ValidationThresholds> = {
fixture: { systemsMinimum: 1, stationsMinimum: 0, requiredCategories: [4, 6, 9], blueprintsMinimum: 1 },
production: { systemsMinimum: 5001, stationsMinimum: 1, requiredCategories: [4, 6, 9], blueprintsMinimum: 1 },
fixture: { systemsMinimum: 1, stationsMinimum: 0, stationCoverage: 'incomplete', requiredCategories: [4, 6, 9], blueprintsMinimum: 1 },
production: { systemsMinimum: 5001, stationsMinimum: 1, stationCoverage: 'incomplete', requiredCategories: [4, 6, 9], blueprintsMinimum: 1 },
};
export interface ReferenceCompilerOptions {
@@ -318,6 +323,8 @@ export interface ActiveManifest {
/** Content-addressed source cache selected by this artifact. */
sourceInputs: Record<ReferenceSourceName, ActiveSourceInput>;
counts: Record<string, number>;
/** Completeness is explicit; production remains incomplete until measured replay evidence exists. */
stationCoverage?: StationCoverageProfile;
}
export interface ActiveSourceInput {
fingerprint: string;