feat(reference): add indexed readers and station source

This commit is contained in:
jb
2026-09-19 09:53:46 -04:00
parent 9ddd12fbc5
commit f7b604f40f
10 changed files with 429 additions and 25 deletions
+26 -2
View File
@@ -337,7 +337,9 @@ export class ReferenceCompiler {
try {
const manifest = await readManifest(this.directory);
if (!manifest.active || !manifest.previous) throw new Error('no previous reference artifact is available for rollback');
if (!await exists(artifactDatabasePath(this.directory, manifest.previous.database))) throw new Error('previous reference database is missing');
const previousPath = artifactDatabasePath(this.directory, manifest.previous.database);
if (!await exists(previousPath)) throw new Error('previous reference database is missing');
validateRollbackArtifact(previousPath, manifest.previous, this.profile, this.importerVersion, this.thresholds);
const next: ReferenceManifest = { ...manifest, active: manifest.previous, previous: manifest.active, lastAttempt: { startedAt: iso(this.now()), finishedAt: iso(this.now()), status: 'success', profile: manifest.previous.profile, version: manifest.previous.version } };
await atomicWriteJson(this.directory, MANIFEST_FILE, next);
return { activeVersion: next.active!.version, previousVersion: next.previous?.version };
@@ -345,6 +347,23 @@ export class ReferenceCompiler {
}
}
function validateRollbackArtifact(path: string, artifact: ActiveManifest, expectedProfile: BuildProfile, expectedImporterVersion: string, thresholds: ValidationThresholds): void {
if (artifact.profile !== expectedProfile || artifact.importerVersion !== expectedImporterVersion) throw new Error('previous reference artifact is incompatible with the current compiler/profile');
if (!artifact.sourceInputs || !artifact.counts) throw new Error('previous reference artifact is missing compatibility metadata');
let db: Database | undefined;
try {
db = new Database(path, { readonly: true, strict: true });
const counts = validateCandidate(db, thresholds, artifact.profile);
for (const [table, expected] of Object.entries(artifact.counts)) if (counts[table] !== expected) throw new Error(`previous reference artifact count mismatch for ${table}: expected ${expected}, got ${counts[table]}`);
const rows = db.query<{ source: string; fingerprint: string; local_sha256: string }, []>('SELECT source,fingerprint,local_sha256 FROM source_provenance').all();
const bySource = new Map(rows.map((row) => [row.source, row]));
for (const source of REFERENCE_SOURCES) {
const expected = artifact.sourceInputs[source]; const row = bySource.get(source);
if (!expected || !row || row.fingerprint !== expected.fingerprint || row.local_sha256 !== expected.localSha256) throw new Error(`previous reference artifact provenance mismatch for ${source}`);
}
} finally { try { db?.close(); } catch { /* best effort */ } }
}
function verifyPayload(check: SourceCheck, payload: SourcePayload): void {
if (payload.source !== check.source) throw new Error(`loaded source mismatch: expected ${check.source}, got ${payload.source}`);
if (payload.check.fingerprint !== check.fingerprint) throw new Error('loaded source fingerprint differs from checked fingerprint');
@@ -492,7 +511,8 @@ function canonicalCounts(input: ReturnType<typeof normalizeAll>): Record<string,
regions: Object.keys(input.universe.regions ?? {}).length,
constellations: Object.keys(input.universe.constellations ?? {}).length,
gates: Object.keys(input.universe.gates ?? {}).length,
stations: 0,
stations: Object.keys(input.universe.stations ?? {}).length + Object.values(input.universe.systems).reduce((count, system) => count + (system.stations ?? []).filter((stationId) => !input.universe.stations?.[String(stationId)]).length, 0),
verified_stations: Object.keys(input.universe.stations ?? {}).length,
unresolved_references: [...Object.values(input.reference.blueprints)].reduce((count, blueprint) => count + Object.values(blueprint.activities).reduce((activityCount, activity) => activityCount + [...Object.values(activity.materials), ...Object.values(activity.products)].filter((row) => !input.reference.types[String(row.type_id)]).length, 0), 0),
};
}
@@ -516,12 +536,16 @@ async function validateExistingArtifact(path: string, input: ReturnType<typeof n
}
export function validateCandidate(db: Database, thresholds: ValidationThresholds, profile: string): Record<string, number> {
const schemaVersion = Number((db.query<{ user_version: number }, []>('PRAGMA user_version').get() as { user_version: number } | null)?.user_version ?? 0);
if (schemaVersion !== SCHEMA_VERSION) throw new Error(`SQLite schema version ${schemaVersion} does not match compiler ${SCHEMA_VERSION}`);
const integrity = db.query<{ integrity_check: string }, []>('PRAGMA integrity_check').all();
if (integrity.length !== 1 || integrity[0].integrity_check !== 'ok') throw new Error(`SQLite integrity_check failed: ${JSON.stringify(integrity)}`);
const foreignKeys = db.query<unknown, []>('PRAGMA foreign_key_check').all(); if (foreignKeys.length) throw new Error(`SQLite foreign_key_check failed: ${JSON.stringify(foreignKeys)}`);
const count = (table: string): number => Number((db.query<{ count: number }, []>(`SELECT COUNT(*) AS count FROM ${table}`).get() as { count: number }).count);
const counts: Record<string, number> = {}; for (const table of ['categories', 'groups', 'types', 'market_groups', 'blueprints', 'activities', 'materials', 'products', 'unresolved_references', 'systems', 'regions', 'constellations', 'gates', 'stations']) counts[table] = count(table);
counts.verified_stations = Number((db.query<{ count: number }, []>('SELECT COUNT(*) AS count FROM stations WHERE source_verified = 1').get() as { count: number }).count);
if (counts.systems < thresholds.systemsMinimum) throw new Error(`${profile} validation requires at least ${thresholds.systemsMinimum} systems, got ${counts.systems}`);
if (counts.verified_stations < thresholds.stationsMinimum) throw new Error(`${profile} validation requires at least ${thresholds.stationsMinimum} verified stations, got ${counts.verified_stations}`);
if (counts.blueprints < thresholds.blueprintsMinimum) throw new Error(`${profile} validation requires at least ${thresholds.blueprintsMinimum} blueprints, got ${counts.blueprints}`);
for (const category of thresholds.requiredCategories) { const present = Number((db.query<{ count: number }, [number]>('SELECT COUNT(*) AS count FROM categories WHERE category_id = ?').get(category) as { count: number }).count); if (!present) throw new Error(`${profile} validation requires category ${category}`); }
db.query('SELECT type_id FROM types WHERE type_id IN (34,35,587)').all(); db.query('SELECT blueprint_type_id FROM blueprints LIMIT 1').all();
+30 -1
View File
@@ -289,12 +289,18 @@ export function normalizeUniverse(raw: unknown): EsiUniversePayload {
const constellations = root.constellations === undefined ? undefined : normalizeConstellations(root.constellations);
const regions = root.regions === undefined ? undefined : normalizeRegions(root.regions);
const gates = root.gates === undefined ? undefined : normalizeGates(root.gates);
const stations = root.stations === undefined ? undefined : normalizeStations(root.stations);
for (const row of Object.values(systems)) if (constellations && !constellations[String(row.constellation_id)]) throw new Error(`ESI system ${row.system_id} references missing constellation ${row.constellation_id}`);
for (const row of Object.values(systems)) if (gates) for (const gateId of row.stargates ?? []) {
const gate = gates[String(gateId)];
if (!gate || gate.system_id !== row.system_id) throw new Error(`ESI system ${row.system_id} references missing or mismatched gate ${gateId}`);
}
return { systems, constellations, regions, gates };
for (const row of Object.values(stations ?? {})) if (!systems[String(row.system_id)]) throw new Error(`station ${row.station_id} references missing system ${row.system_id}`);
for (const system of Object.values(systems)) for (const stationId of system.stations ?? []) {
const station = stations?.[String(stationId)];
if (station && station.system_id !== system.system_id) throw new Error(`system ${system.system_id} lists station ${stationId} owned by system ${station.system_id}`);
}
return { systems, constellations, regions, gates, stations };
}
function sourceName(value: unknown, label: string): string {
if (typeof value === 'string' && value) return value;
@@ -324,6 +330,26 @@ function normalizeGates(input: unknown): NonNullable<EsiUniversePayload['gates']
}
return output;
}
function normalizeStations(input: unknown): NonNullable<EsiUniversePayload['stations']> {
const output: NonNullable<EsiUniversePayload['stations']> = {};
for (const [key, entry] of Object.entries(record(input, 'NPC stations'))) {
const row = record(entry, `NPC station ${key}`);
const position = row.position === undefined ? undefined : record(row.position, `NPC station ${key}.position`);
output[key] = {
station_id: idKey(key, row.station_id, `NPC station ${key}.station_id`),
system_id: integer(row.system_id, `NPC station ${key}.system_id`)!,
name: row.name === undefined || row.name === null ? undefined : sourceName(row.name, `NPC station ${key}.name`),
type_id: integer(row.type_id, `NPC station ${key}.type_id`, true),
owner_id: integer(row.owner_id, `NPC station ${key}.owner_id`, true),
position: position === undefined ? undefined : {
x: finite(position.x, `NPC station ${key}.position.x`)!,
y: finite(position.y, `NPC station ${key}.position.y`)!,
z: finite(position.z, `NPC station ${key}.position.z`)!,
},
};
}
return output;
}
export function normalizeAll(reference: unknown, hoboleaks: unknown, universe: unknown): NormalizedInputs {
const normalized = { reference: normalizeReference(reference), hoboleaks: normalizeHoboleaks(hoboleaks), universe: normalizeUniverse(universe) };
@@ -345,4 +371,7 @@ export function validateCrossSource(data: NormalizedInputs): void {
for (const gate of Object.values(data.universe.gates ?? {})) {
if (!systems.has(gate.system_id) || !systems.has(gate.destination_system_id)) throw new Error(`ESI gate ${gate.stargate_id} references missing system`);
}
for (const station of Object.values(data.universe.stations ?? {})) {
if (!systems.has(station.system_id)) throw new Error(`NPC station ${station.station_id} references missing system ${station.system_id}`);
}
}
+13 -4
View File
@@ -302,9 +302,9 @@ async function consumeLines(stream: ReadableStream<Uint8Array>, label: string, o
async function ccpUniverse(path: string): Promise<EsiUniversePayload> {
const { file, entries } = await zipEntries(path);
try {
const required = ['mapSolarSystems.jsonl', 'mapConstellations.jsonl', 'mapRegions.jsonl', 'mapStargates.jsonl'];
const required = ['mapSolarSystems.jsonl', 'mapConstellations.jsonl', 'mapRegions.jsonl', 'mapStargates.jsonl', 'npcStations.jsonl'];
for (const name of required) if (!entries.has(name)) throw new Error(`CCP SDE ZIP is missing ${name}`);
const systems: NonNullable<EsiUniversePayload['systems']> = {}; const constellations: NonNullable<EsiUniversePayload['constellations']> = {}; const regions: NonNullable<EsiUniversePayload['regions']> = {}; const gates: NonNullable<EsiUniversePayload['gates']> = {};
const systems: NonNullable<EsiUniversePayload['systems']> = {}; const constellations: NonNullable<EsiUniversePayload['constellations']> = {}; const regions: NonNullable<EsiUniversePayload['regions']> = {}; const gates: NonNullable<EsiUniversePayload['gates']> = {}; const stations: NonNullable<EsiUniversePayload['stations']> = {};
const read = async (name: string, handler: (value: AnyMap, line: number) => void) => {
const entry = entries.get(name)!; if (entry.directory) throw new Error(`${name} is a directory`); const stream = await zipMemberStream(file, entry); const stats = await consumeLines(stream, name, (value, line) => handler(object(value, `${name}:${line}`), line), ARCHIVE_LIMITS.memberBytes); if (stats.byteLength !== entry.uncompressedSize) throw new Error(`${name} decompressed size ${stats.byteLength} does not match ZIP metadata ${entry.uncompressedSize}`); if (stats.crc32 !== entry.crc32) throw new Error(`${name} CRC-32 does not match ZIP metadata`);
};
@@ -312,10 +312,19 @@ async function ccpUniverse(path: string): Promise<EsiUniversePayload> {
await read('mapConstellations.jsonl', (row, line) => { const key = id(row._key, `mapConstellations:${line}._key`); if (Object.prototype.hasOwnProperty.call(constellations, String(key))) throw new Error(`mapConstellations:${line} duplicate _key ${key}`); constellations[String(key)] = { constellation_id: key, name: sourceName(row.name, `mapConstellations:${line}.name`), region_id: id(row.regionID, `mapConstellations:${line}.regionID`) }; });
await read('mapSolarSystems.jsonl', (row, line) => {
const key = id(row._key, `mapSolarSystems:${line}._key`); const position = object(row.position, `mapSolarSystems:${line}.position`); const stargates = row.stargateIDs === undefined ? undefined : (Array.isArray(row.stargateIDs) ? row.stargateIDs.map((value: unknown, index: number) => id(value, `mapSolarSystems:${line}.stargateIDs[${index}]`)) : (() => { throw new Error(`mapSolarSystems:${line}.stargateIDs must be an array`); })());
if (Object.prototype.hasOwnProperty.call(systems, String(key))) throw new Error(`mapSolarSystems:${line} duplicate _key ${key}`); systems[String(key)] = { system_id: key, name: sourceName(row.name, `mapSolarSystems:${line}.name`), constellation_id: id(row.constellationID, `mapSolarSystems:${line}.constellationID`), position: { x: number(position.x, `mapSolarSystems:${line}.position.x`), y: number(position.y, `mapSolarSystems:${line}.position.y`), z: number(position.z, `mapSolarSystems:${line}.position.z`) }, security_status: number(row.securityStatus, `mapSolarSystems:${line}.securityStatus`), security_class: row.securityClass === undefined || row.securityClass === null ? row.securityClass : String(row.securityClass), stargates };
const stationIDs = row.stationIDs === undefined ? undefined : (Array.isArray(row.stationIDs) ? row.stationIDs.map((value: unknown, index: number) => id(value, `mapSolarSystems:${line}.stationIDs[${index}]`)) : (() => { throw new Error(`mapSolarSystems:${line}.stationIDs must be an array`); })());
if (Object.prototype.hasOwnProperty.call(systems, String(key))) throw new Error(`mapSolarSystems:${line} duplicate _key ${key}`); systems[String(key)] = { system_id: key, name: sourceName(row.name, `mapSolarSystems:${line}.name`), constellation_id: id(row.constellationID, `mapSolarSystems:${line}.constellationID`), position: { x: number(position.x, `mapSolarSystems:${line}.position.x`), y: number(position.y, `mapSolarSystems:${line}.position.y`), z: number(position.z, `mapSolarSystems:${line}.position.z`) }, security_status: number(row.securityStatus, `mapSolarSystems:${line}.securityStatus`), security_class: row.securityClass === undefined || row.securityClass === null ? row.securityClass : String(row.securityClass), stargates, stations: stationIDs };
});
await read('mapStargates.jsonl', (row, line) => { const key = id(row._key, `mapStargates:${line}._key`); if (Object.prototype.hasOwnProperty.call(gates, String(key))) throw new Error(`mapStargates:${line} duplicate _key ${key}`); const destination = object(row.destination, `mapStargates:${line}.destination`); gates[String(key)] = { stargate_id: key, system_id: id(row.solarSystemID, `mapStargates:${line}.solarSystemID`), destination_system_id: id(destination.solarSystemID, `mapStargates:${line}.destination.solarSystemID`) }; });
return { systems, constellations, regions, gates };
await read('npcStations.jsonl', (row, line) => {
const key = id(row._key, `npcStations:${line}._key`); if (Object.prototype.hasOwnProperty.call(stations, String(key))) throw new Error(`npcStations:${line} duplicate _key ${key}`);
const rawPosition = object(row.position, `npcStations:${line}.position`);
const position = { x: number(rawPosition.x, `npcStations:${line}.position.x`), y: number(rawPosition.y, `npcStations:${line}.position.y`), z: number(rawPosition.z, `npcStations:${line}.position.z`) };
// The official CCP member has no station-name field. Names are accepted
// only by separate, explicitly selected adapters; never infer one here.
stations[String(key)] = { station_id: key, system_id: id(row.solarSystemID, `npcStations:${line}.solarSystemID`), type_id: id(row.typeID, `npcStations:${line}.typeID`), owner_id: id(row.ownerID, `npcStations:${line}.ownerID`), position };
});
return { systems, constellations, regions, gates, stations };
} finally { await file.close(); }
}
+153 -5
View File
@@ -1,12 +1,56 @@
import { Database } from 'bun:sqlite';
import { existsSync, readFileSync } from 'node:fs';
import { basename, extname, isAbsolute, join, relative, resolve } from 'node:path';
import type { InventoryType, ReferenceRepository } from '../../domain/model';
import type { BuildProfile, ReferenceManifest, ReferenceSnapshot } from './types';
import type { InventoryType } from '../../domain/model';
import type {
BuildProfile,
ReferenceCategoryRecord,
ReferenceConstellationRecord,
ReferenceGroupRecord,
ReferenceGroupSelection,
ReferenceIdSelection,
ReferenceListOptions,
ReferenceManifest,
ReferenceReader,
ReferenceRegionRecord,
ReferenceSearchOptions,
ReferenceSnapshot,
ReferenceStationRecord,
ReferenceStationSelection,
ReferenceSystemRecord,
ReferenceSystemSelection,
ReferenceTypeRecord,
} from './types';
import { MANIFEST_FILE, DATABASE_DIRECTORY } from './compiler';
import { SCHEMA_VERSION } from './schema';
const MAX_IN_BATCH = 500;
const MAX_READER_LIMIT = 500;
const MAX_READER_OFFSET = 100_000;
type TypeRow = { type_id: number; name_json: string; category_id: number; volume: number | null };
type SearchTypeRow = { type_id: number; name: string; category_id: number; group_id: number; published: number | null; volume: number | null };
type CategoryRow = { category_id: number; name_json: string; published: number | null };
type GroupRow = { group_id: number; category_id: number; name_json: string; published: number | null };
type RegionRow = { region_id: number; name: string | null };
type ConstellationRow = { constellation_id: number; name: string | null; region_id: number | null };
type SystemRow = { system_id: number; name: string; constellation_id: number; region_id: number | null; x: number; y: number; z: number; security_status: number; security_class: string | null };
type StationRow = { station_id: number; system_id: number; system_name: string | null; region_id: number | null; name: string | null; type_id: number | null; owner_id: number | null; x: number | null; y: number | null; z: number | null };
type ListSelection = ReferenceListOptions;
type IdSelection = ReferenceIdSelection;
type GroupSelection = ReferenceGroupSelection;
type SystemSelection = ReferenceSystemSelection;
type StationSelection = ReferenceStationSelection;
const normalize = (value: string): string => value.normalize('NFKC').trim().replace(/\s+/g, ' ').toLocaleLowerCase('en-US');
const ftsQuery = (value: string): string => normalize(value).split(/[^\p{L}\p{N}]+/u).filter(Boolean).map((token) => `${token}*`).join(' AND ');
const nameFromJson = (value: string): string => { const names = JSON.parse(value) as Record<string, string>; return names.en ?? Object.values(names)[0] ?? ''; };
function bounded(value: number | undefined, fallback: number): number { return Math.max(0, Math.min(MAX_READER_OFFSET, Number.isSafeInteger(value) ? value! : fallback)); }
function limit(value: number | undefined, fallback = 100): number { return Math.max(1, Math.min(MAX_READER_LIMIT, Number.isSafeInteger(value) ? value! : fallback)); }
function idsFrom(selection: IdSelection): number[] | undefined { if (Array.isArray(selection)) return [...new Set(selection.filter((id) => Number.isSafeInteger(id) && id >= 0))]; if (!selection || selection.ids === undefined) return undefined; if (!Array.isArray(selection.ids)) return []; return [...new Set(selection.ids.filter((id) => Number.isSafeInteger(id) && id >= 0))]; }
function optionValue<T extends object>(selection: IdSelection, key: string): unknown { return !Array.isArray(selection) && selection && key in selection ? (selection as Record<string, unknown>)[key] : undefined; }
function like(value: string): string { return `%${normalize(value).replace(/[\\%_]/g, (part) => `\\${part}`)}%`; }
function assertSelectionBounded(ids: number[] | undefined, label: string): void { if (ids && ids.length > MAX_IN_BATCH) throw new Error(`${label} selection exceeds ${MAX_IN_BATCH} IDs`); }
function inClause(ids: number[], column: string): { sql: string; args: number[] } { return { sql: `${column} IN (${ids.map(() => '?').join(',')})`, args: ids }; }
function assertArtifactPath(root: string, database: string): string {
if (!database || isAbsolute(database) || database.includes('\\') || database.split('/').includes('..')) throw new Error('active reference database path is not a safe relative artifact path');
@@ -18,7 +62,7 @@ function assertArtifactPath(root: string, database: string): string {
}
/** Runtime reader: a read-only handle pinned to one immutable manifest artifact. */
export class SqliteReferenceRepository implements ReferenceRepository, ReferenceSnapshot {
export class SqliteReferenceRepository implements ReferenceReader, ReferenceSnapshot {
private readonly db: Database;
private closed = false;
readonly pinnedVersion: string;
@@ -44,6 +88,15 @@ export class SqliteReferenceRepository implements ReferenceRepository, Reference
snapshot(): SqliteReferenceSnapshot { return this.pin(); }
async types(ids: number[]): Promise<InventoryType[]> { this.assertOpen(); return queryTypes(this.db, ids); }
/** Indexed normalized-name search. Empty queries are rejected to avoid accidental full-catalog scans. */
async searchTypes(options: ReferenceSearchOptions): Promise<ReferenceTypeRecord[]> { this.assertOpen(); return queryTypesSearch(this.db, options); }
async searchTypesByName(query: string, options: Omit<ReferenceSearchOptions, 'query'> = {}): Promise<ReferenceTypeRecord[]> { return this.searchTypes({ ...options, query }); }
async categories(selection?: IdSelection): Promise<ReferenceCategoryRecord[]> { this.assertOpen(); return queryCategories(this.db, selection); }
async groups(selection?: GroupSelection): Promise<ReferenceGroupRecord[]> { this.assertOpen(); return queryGroups(this.db, selection); }
async regions(selection?: IdSelection): Promise<ReferenceRegionRecord[]> { this.assertOpen(); return queryRegions(this.db, selection); }
async constellations(selection?: IdSelection): Promise<ReferenceConstellationRecord[]> { this.assertOpen(); return queryConstellations(this.db, selection); }
async systems(selection?: SystemSelection): Promise<ReferenceSystemRecord[]> { this.assertOpen(); return querySystems(this.db, selection); }
async stations(selection?: StationSelection): Promise<ReferenceStationRecord[]> { this.assertOpen(); return queryStations(this.db, selection); }
async version(): Promise<string> { this.assertOpen(); return this.pinnedVersion; }
close(): void { if (!this.closed) { this.closed = true; this.db.close(); } }
[Symbol.dispose](): void { this.close(); }
@@ -58,16 +111,111 @@ export class SqliteReferenceSnapshot implements ReferenceSnapshot {
assertReaderSchema(this.db);
}
async types(ids: number[]): Promise<InventoryType[]> { this.assertOpen(); return queryTypes(this.db, ids); }
async searchTypes(options: ReferenceSearchOptions): Promise<ReferenceTypeRecord[]> { this.assertOpen(); return queryTypesSearch(this.db, options); }
async searchTypesByName(query: string, options: Omit<ReferenceSearchOptions, 'query'> = {}): Promise<ReferenceTypeRecord[]> { return this.searchTypes({ ...options, query }); }
async categories(selection?: IdSelection): Promise<ReferenceCategoryRecord[]> { this.assertOpen(); return queryCategories(this.db, selection); }
async groups(selection?: GroupSelection): Promise<ReferenceGroupRecord[]> { this.assertOpen(); return queryGroups(this.db, selection); }
async regions(selection?: IdSelection): Promise<ReferenceRegionRecord[]> { this.assertOpen(); return queryRegions(this.db, selection); }
async constellations(selection?: IdSelection): Promise<ReferenceConstellationRecord[]> { this.assertOpen(); return queryConstellations(this.db, selection); }
async systems(selection?: SystemSelection): Promise<ReferenceSystemRecord[]> { this.assertOpen(); return querySystems(this.db, selection); }
async stations(selection?: StationSelection): Promise<ReferenceStationRecord[]> { this.assertOpen(); return queryStations(this.db, selection); }
async version(): Promise<string> { this.assertOpen(); return this.pinnedVersion; }
close(): void { if (!this.closed) { this.closed = true; this.db.close(); } }
[Symbol.dispose](): void { this.close(); }
private assertOpen(): void { if (this.closed) throw new Error('reference snapshot is closed'); }
}
function queryTypesSearch(db: Database, options: ReferenceSearchOptions): Promise<ReferenceTypeRecord[]> {
const query = typeof options.query === 'string' ? options.query.trim() : '';
const match = ftsQuery(query);
if (!match) return Promise.reject(new Error('type search query must contain letters or numbers'));
const conditions = ['s.type_id IN (SELECT CAST(type_id AS INTEGER) FROM type_search_fts WHERE type_search_fts MATCH ?)']; const args: (string | number)[] = [match];
if (options.categoryId !== undefined) { conditions.push('t.category_id = ?'); args.push(options.categoryId); }
if (options.groupId !== undefined) { conditions.push('t.group_id = ?'); args.push(options.groupId); }
if (options.published !== undefined) { conditions.push('t.published = ?'); args.push(options.published ? 1 : 0); }
const rows = db.query<SearchTypeRow, (string | number)[]>(`SELECT t.type_id, COALESCE((SELECT name FROM type_search WHERE type_id = t.type_id AND language = 'en' LIMIT 1), MIN(s.name)) AS name, t.category_id, t.group_id, t.published, t.volume FROM type_search s JOIN types t ON t.type_id = s.type_id WHERE ${conditions.join(' AND ')} GROUP BY t.type_id ORDER BY s.normalized_name, t.type_id LIMIT ? OFFSET ?`).all(...args, limit(options.limit, 50), bounded(options.offset, 0));
return Promise.resolve(rows.map((row) => ({ id: row.type_id, name: row.name, categoryId: row.category_id, groupId: row.group_id, ...(row.published === null ? {} : { published: row.published === 1 }), ...(row.volume === null ? {} : { volume: row.volume }) })));
}
function queryCategories(db: Database, selection: IdSelection): Promise<ReferenceCategoryRecord[]> {
const conditions: string[] = []; const args: (string | number)[] = []; const ids = idsFrom(selection);
if (ids && !ids.length) return Promise.resolve([]);
assertSelectionBounded(ids, 'category');
if (ids) { const clause = inClause(ids, 'category_id'); conditions.push(clause.sql); args.push(...clause.args); }
const query = optionValue(selection, 'query'); if (typeof query === 'string' && query.trim()) { conditions.push("name_json LIKE ? ESCAPE '\\'"); args.push(like(query)); }
const rows = db.query<CategoryRow, (string | number)[]>(`SELECT category_id,name_json,published FROM categories${conditions.length ? ` WHERE ${conditions.join(' AND ')}` : ''} ORDER BY category_id LIMIT ? OFFSET ?`).all(...args, limit(Number(optionValue(selection, 'limit'))), bounded(Number(optionValue(selection, 'offset')), 0));
return Promise.resolve(rows.map((row) => ({ id: row.category_id, name: nameFromJson(row.name_json), ...(row.published === null ? {} : { published: row.published === 1 }) })));
}
function queryGroups(db: Database, selection: GroupSelection): Promise<ReferenceGroupRecord[]> {
const conditions: string[] = []; const args: (string | number)[] = []; const ids = idsFrom(selection);
if (ids && !ids.length) return Promise.resolve([]);
assertSelectionBounded(ids, 'group');
if (ids) { const clause = inClause(ids, 'group_id'); conditions.push(clause.sql); args.push(...clause.args); }
const categoryId = optionValue(selection, 'categoryId'); if (typeof categoryId === 'number') { conditions.push('category_id = ?'); args.push(categoryId); }
const query = optionValue(selection, 'query'); if (typeof query === 'string' && query.trim()) { conditions.push("name_json LIKE ? ESCAPE '\\'"); args.push(like(query)); }
const rows = db.query<GroupRow, (string | number)[]>(`SELECT group_id,category_id,name_json,published FROM groups${conditions.length ? ` WHERE ${conditions.join(' AND ')}` : ''} ORDER BY group_id LIMIT ? OFFSET ?`).all(...args, limit(Number(optionValue(selection, 'limit'))), bounded(Number(optionValue(selection, 'offset')), 0));
return Promise.resolve(rows.map((row) => ({ id: row.group_id, categoryId: row.category_id, name: nameFromJson(row.name_json), ...(row.published === null ? {} : { published: row.published === 1 }) })));
}
function queryRegions(db: Database, selection: IdSelection): Promise<ReferenceRegionRecord[]> {
const conditions: string[] = []; const args: (string | number)[] = []; const ids = idsFrom(selection);
if (ids && !ids.length) return Promise.resolve([]);
assertSelectionBounded(ids, 'region');
if (ids) { const clause = inClause(ids, 'region_id'); conditions.push(clause.sql); args.push(...clause.args); }
const query = optionValue(selection, 'query'); if (typeof query === 'string' && query.trim()) { conditions.push("name LIKE ? ESCAPE '\\'"); args.push(like(query)); }
const rows = db.query<RegionRow, (string | number)[]>(`SELECT region_id,name FROM regions${conditions.length ? ` WHERE ${conditions.join(' AND ')}` : ''} ORDER BY region_id LIMIT ? OFFSET ?`).all(...args, limit(Number(optionValue(selection, 'limit'))), bounded(Number(optionValue(selection, 'offset')), 0));
return Promise.resolve(rows.map((row) => ({ id: row.region_id, name: row.name })));
}
function queryConstellations(db: Database, selection: IdSelection): Promise<ReferenceConstellationRecord[]> {
const conditions: string[] = []; const args: (string | number)[] = []; const ids = idsFrom(selection);
if (ids && !ids.length) return Promise.resolve([]);
assertSelectionBounded(ids, 'constellation');
if (ids) { const clause = inClause(ids, 'constellation_id'); conditions.push(clause.sql); args.push(...clause.args); }
const query = optionValue(selection, 'query'); if (typeof query === 'string' && query.trim()) { conditions.push("name LIKE ? ESCAPE '\\'"); args.push(like(query)); }
const rows = db.query<ConstellationRow, (string | number)[]>(`SELECT constellation_id,name,region_id FROM constellations${conditions.length ? ` WHERE ${conditions.join(' AND ')}` : ''} ORDER BY constellation_id LIMIT ? OFFSET ?`).all(...args, limit(Number(optionValue(selection, 'limit'))), bounded(Number(optionValue(selection, 'offset')), 0));
return Promise.resolve(rows.map((row) => ({ id: row.constellation_id, name: row.name, regionId: row.region_id })));
}
function querySystems(db: Database, selection: SystemSelection): Promise<ReferenceSystemRecord[]> {
const conditions: string[] = []; const args: (string | number)[] = []; const ids = idsFrom(selection);
if (ids && !ids.length) return Promise.resolve([]);
assertSelectionBounded(ids, 'system');
if (ids) { const clause = inClause(ids, 's.system_id'); conditions.push(clause.sql); args.push(...clause.args); }
const constellationId = optionValue(selection, 'constellationId'); if (typeof constellationId === 'number') { conditions.push('s.constellation_id = ?'); args.push(constellationId); }
const regionId = optionValue(selection, 'regionId'); if (typeof regionId === 'number') { conditions.push('c.region_id = ?'); args.push(regionId); }
const query = optionValue(selection, 'query'); if (typeof query === 'string' && query.trim()) { conditions.push("s.name LIKE ? ESCAPE '\\'"); args.push(like(query)); }
const rows = db.query<SystemRow, (string | number)[]>(`SELECT s.system_id,s.name,s.constellation_id,c.region_id,s.x,s.y,s.z,s.security_status,s.security_class FROM systems s LEFT JOIN constellations c ON c.constellation_id = s.constellation_id${conditions.length ? ` WHERE ${conditions.join(' AND ')}` : ''} ORDER BY s.system_id LIMIT ? OFFSET ?`).all(...args, limit(Number(optionValue(selection, 'limit'))), bounded(Number(optionValue(selection, 'offset')), 0));
return Promise.resolve(rows.map((row) => ({ id: row.system_id, name: row.name, constellationId: row.constellation_id, regionId: row.region_id, x: row.x, y: row.y, z: row.z, securityStatus: row.security_status, securityClass: row.security_class })));
}
function queryStations(db: Database, selection: StationSelection): Promise<ReferenceStationRecord[]> {
const conditions: string[] = []; const args: (string | number)[] = []; const ids = idsFrom(selection);
if (ids && !ids.length) return Promise.resolve([]);
assertSelectionBounded(ids, 'station');
if (ids) { const clause = inClause(ids, 'st.station_id'); conditions.push(clause.sql); args.push(...clause.args); }
const systemId = optionValue(selection, 'systemId'); if (typeof systemId === 'number') { conditions.push('st.system_id = ?'); args.push(systemId); }
const query = optionValue(selection, 'query'); if (typeof query === 'string' && query.trim()) { conditions.push("st.name LIKE ? ESCAPE '\\'"); args.push(like(query)); }
const rows = db.query<StationRow, (string | number)[]>(`SELECT st.station_id,st.system_id,s.name AS system_name,c.region_id,st.name,st.type_id,st.owner_id,st.x,st.y,st.z FROM stations st JOIN systems s ON s.system_id = st.system_id LEFT JOIN constellations c ON c.constellation_id = s.constellation_id${conditions.length ? ` WHERE ${conditions.join(' AND ')}` : ''} ORDER BY st.station_id LIMIT ? OFFSET ?`).all(...args, limit(Number(optionValue(selection, 'limit'))), bounded(Number(optionValue(selection, 'offset')), 0));
return Promise.resolve(rows.map((row) => ({ id: row.station_id, systemId: row.system_id, systemName: row.system_name, regionId: row.region_id, name: row.name, typeId: row.type_id, ownerId: row.owner_id, x: row.x, y: row.y, z: row.z })));
}
/** Cheap metadata check only; full integrity_check belongs to the compiler. */
function assertReaderSchema(db: Database): void {
const row = db.query<{ present: number }, []>("SELECT COUNT(*) AS present FROM sqlite_master WHERE type='table' AND name IN ('types','categories')").get();
if (!row || Number(row.present) !== 2) { db.close(); throw new Error('active reference database has an unexpected schema'); }
const version = Number((db.query<{ user_version: number }, []>('PRAGMA user_version').get() as { user_version: number } | null)?.user_version ?? 0);
const row = db.query<{ present: number }, []>("SELECT COUNT(*) AS present FROM sqlite_master WHERE type='table' AND name IN ('types','type_search','type_search_fts','categories','groups','regions','constellations','systems','stations')").get();
const index = db.query<{ present: number }, []>("SELECT COUNT(*) AS present FROM sqlite_master WHERE type='index' AND name='type_search_normalized_name_idx'").get();
const requiredColumns: Record<string, string[]> = {
types: ['type_id', 'group_id', 'category_id', 'name_json', 'published', 'volume'],
type_search: ['type_id', 'language', 'name', 'normalized_name'],
type_search_fts: ['type_id', 'language', 'name', 'normalized_name'],
categories: ['category_id', 'name_json', 'published'],
groups: ['group_id', 'category_id', 'name_json', 'published'],
regions: ['region_id', 'name'],
constellations: ['constellation_id', 'region_id', 'name'],
stations: ['station_id', 'system_id', 'name', 'type_id', 'owner_id', 'x', 'y', 'z', 'source_verified'],
systems: ['system_id', 'constellation_id', 'name', 'x', 'y', 'z', 'security_status', 'security_class'],
};
const columnsValid = Object.entries(requiredColumns).every(([table, columns]) => {
const present = new Set(db.query<{ name: string }, []>(`PRAGMA table_info(${table})`).all().map((column) => column.name));
return columns.every((column) => present.has(column));
});
if (version !== SCHEMA_VERSION || !row || Number(row.present) !== 9 || !index || Number(index.present) !== 1 || !columnsValid) { db.close(); throw new Error(`active reference database has an incompatible schema version (expected ${SCHEMA_VERSION})`); }
}
async function queryTypes(db: Database, requested: number[]): Promise<InventoryType[]> {
+27 -4
View File
@@ -9,13 +9,14 @@ import type {
ReferenceType,
} from './types';
export const SCHEMA_VERSION = 1;
export const SCHEMA_VERSION = 3;
/**
* The database is built from scratch and opened read-only by the serving side.
* Foreign keys are deliberately explicit even for currently empty future tables.
*/
export function createReferenceSchema(db: Database): void {
db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);
db.run('PRAGMA foreign_keys = ON');
db.run('PRAGMA journal_mode = DELETE');
db.run('PRAGMA synchronous = FULL');
@@ -56,6 +57,13 @@ export function createReferenceSchema(db: Database): void {
radius REAL,
base_price REAL
);
CREATE TABLE type_search (
type_id INTEGER NOT NULL REFERENCES types(type_id),
language TEXT NOT NULL,
name TEXT NOT NULL,
normalized_name TEXT NOT NULL,
PRIMARY KEY (type_id, language)
);
CREATE TABLE market_groups (
market_group_id INTEGER PRIMARY KEY,
parent_group_id INTEGER REFERENCES market_groups(market_group_id),
@@ -161,7 +169,8 @@ export function createReferenceSchema(db: Database): void {
owner_id INTEGER,
x REAL,
y REAL,
z REAL
z REAL,
source_verified INTEGER NOT NULL CHECK (source_verified IN (0, 1))
);
CREATE TABLE source_provenance (
source TEXT PRIMARY KEY,
@@ -176,6 +185,8 @@ export function createReferenceSchema(db: Database): void {
CREATE INDEX types_group_idx ON types(group_id);
CREATE INDEX types_category_idx ON types(category_id);
CREATE INDEX types_name_idx ON types(name_json);
CREATE INDEX type_search_normalized_name_idx ON type_search(normalized_name);
CREATE VIRTUAL TABLE type_search_fts USING fts5(type_id UNINDEXED, language UNINDEXED, name, normalized_name);
CREATE INDEX market_groups_parent_idx ON market_groups(parent_group_id);
CREATE INDEX activities_blueprint_idx ON activities(blueprint_type_id);
CREATE INDEX materials_type_idx ON materials(type_id);
@@ -196,6 +207,7 @@ export function createReferenceSchema(db: Database): void {
const bool = (value: boolean | undefined): number | null => value === undefined ? null : value ? 1 : 0;
const json = (value: unknown): string => JSON.stringify(value);
const normalizedName = (value: string): string => value.normalize('NFKC').trim().replace(/\s+/g, ' ').toLocaleLowerCase('en-US');
const keyId = (key: string, label: string): number => {
const id = Number(key);
if (!Number.isSafeInteger(id) || id < 0) throw new Error(`${label} key is not a safe integer: ${key}`);
@@ -229,6 +241,9 @@ export function insertReferenceData(db: Database, data: ReferenceDataPayload): v
type.run(id, row.group_id, row.category_id, json(row.name), row.description === undefined ? null : json(row.description), bool(row.published), bool(row.is_blueprint), bool(row.is_skill), bool(row.is_ore), bool(row.is_repackable), row.volume ?? null, row.packaged_volume ?? null, row.portion_size ?? null, row.mass ?? null, row.capacity ?? null, row.radius ?? null, row.base_price ?? null);
}
insertTypeMaps(db, data.types);
const typeSearch = db.prepare('INSERT INTO type_search(type_id,language,name,normalized_name) VALUES (?,?,?,?)');
const typeSearchFts = db.prepare('INSERT INTO type_search_fts(type_id,language,name,normalized_name) VALUES (?,?,?,?)');
for (const row of Object.values(data.types)) for (const [language, name] of Object.entries(row.name)) { typeSearch.run(row.type_id, language, name, normalizedName(name)); typeSearchFts.run(row.type_id, language, name, normalizedName(name)); }
const marketGroup = db.prepare('INSERT INTO market_groups(market_group_id,parent_group_id,name_json,description_json,icon_id,has_types) VALUES (?,?,?,?,?,?)');
for (const [key, value] of Object.entries(data.market_groups ?? {})) {
const row = value as Record<string, unknown>;
@@ -279,8 +294,16 @@ export function insertUniverseData(db: Database, data: EsiUniversePayload): void
}
const gate = db.prepare('INSERT INTO gates(system_id,stargate_id,destination_system_id) VALUES (?,?,?)');
for (const row of Object.values(data.gates ?? {})) gate.run(row.system_id, row.stargate_id, row.destination_system_id);
const station = db.prepare('INSERT INTO stations(station_id,system_id,name,type_id,owner_id,x,y,z) VALUES (?,?,?,?,?,?,?,?)');
for (const row of Object.values(data.systems)) for (const stationId of row.stations ?? []) station.run(stationId, row.system_id, null, null, null, null, null, null);
const station = db.prepare('INSERT INTO stations(station_id,system_id,name,type_id,owner_id,x,y,z,source_verified) VALUES (?,?,?,?,?,?,?,?,?)');
const stationIds = new Set<number>();
for (const row of Object.values(data.stations ?? {})) {
const position = row.position;
station.run(row.station_id, row.system_id, row.name ?? null, row.type_id ?? null, row.owner_id ?? null, position?.x ?? null, position?.y ?? null, position?.z ?? null, 1);
stationIds.add(row.station_id);
}
// ESI system responses expose station IDs but not station metadata. Preserve
// those IDs as explicitly unknown rather than fabricating a name or type.
for (const row of Object.values(data.systems)) for (const stationId of row.stations ?? []) if (!stationIds.has(stationId)) station.run(stationId, row.system_id, null, null, null, null, null, null, 0);
}
export function insertHoboleaksData(db: Database, data: HoboleaksPayload): void {
+89 -3
View File
@@ -121,12 +121,97 @@ export interface UniverseGate {
destination_system_id: number;
}
export interface ReferenceStation {
station_id: number;
system_id: number;
/** A station name is absent when the selected source only proves the ID. */
name?: string;
type_id?: number;
owner_id?: number;
position?: { x: number; y: number; z: number };
}
export interface EsiUniversePayload {
systems: Record<string, EsiUniverseSystem>;
/** Optional ESI/SDE complements, kept separate from the ESI system contract. */
constellations?: Record<string, { constellation_id: number; name?: string; region_id?: number }>;
regions?: Record<string, { region_id: number; name?: string }>;
gates?: Record<string, UniverseGate>;
/** Verified NPC station records, when the selected universe source provides them. */
stations?: Record<string, ReferenceStation>;
}
export interface ReferenceCategoryRecord {
id: number;
name: string;
published?: boolean;
}
export interface ReferenceGroupRecord {
id: number;
categoryId: number;
name: string;
published?: boolean;
}
export interface ReferenceTypeRecord {
id: number;
name: string;
categoryId: number;
groupId: number;
published?: boolean;
volume?: number;
}
export interface ReferenceRegionRecord { id: number; name: string | null; }
export interface ReferenceConstellationRecord { id: number; name: string | null; regionId: number | null; }
export interface ReferenceSystemRecord {
id: number;
name: string;
constellationId: number;
regionId: number | null;
x: number;
y: number;
z: number;
securityStatus: number;
securityClass: string | null;
}
export interface ReferenceStationRecord {
id: number;
systemId: number;
systemName: string | null;
regionId: number | null;
name: string | null;
typeId: number | null;
ownerId: number | null;
x: number | null;
y: number | null;
z: number | null;
}
export interface ReferenceSearchOptions {
query: string;
categoryId?: number;
groupId?: number;
published?: boolean;
limit?: number;
offset?: number;
}
export interface ReferenceListOptions {
ids?: number[];
query?: string;
limit?: number;
offset?: number;
}
export type ReferenceIdSelection = number[] | ReferenceListOptions | undefined;
export type ReferenceGroupSelection = number[] | (ReferenceListOptions & { categoryId?: number }) | undefined;
export type ReferenceSystemSelection = number[] | (ReferenceListOptions & { constellationId?: number; regionId?: number }) | undefined;
export type ReferenceStationSelection = number[] | (ReferenceListOptions & { systemId?: number }) | undefined;
export interface ReferenceReader extends ReferenceRepository {
searchTypes(options: ReferenceSearchOptions): Promise<ReferenceTypeRecord[]>;
searchTypesByName(query: string, options?: Omit<ReferenceSearchOptions, 'query'>): Promise<ReferenceTypeRecord[]>;
categories(selection?: ReferenceIdSelection): Promise<ReferenceCategoryRecord[]>;
groups(selection?: ReferenceGroupSelection): Promise<ReferenceGroupRecord[]>;
regions(selection?: ReferenceIdSelection): Promise<ReferenceRegionRecord[]>;
constellations(selection?: ReferenceIdSelection): Promise<ReferenceConstellationRecord[]>;
systems(selection?: ReferenceSystemSelection): Promise<ReferenceSystemRecord[]>;
stations(selection?: ReferenceStationSelection): Promise<ReferenceStationRecord[]>;
}
export interface SourceCheck {
@@ -177,12 +262,13 @@ export interface ProductionArchiveSourceOptions {
export type BuildProfile = 'fixture' | 'production';
export interface ValidationThresholds {
systemsMinimum: number;
stationsMinimum: number;
requiredCategories: number[];
blueprintsMinimum: number;
}
export const PROFILE_THRESHOLDS: Record<BuildProfile, ValidationThresholds> = {
fixture: { systemsMinimum: 1, requiredCategories: [4, 6, 9], blueprintsMinimum: 1 },
production: { systemsMinimum: 5001, requiredCategories: [4, 6, 9], blueprintsMinimum: 1 },
fixture: { systemsMinimum: 1, stationsMinimum: 0, requiredCategories: [4, 6, 9], blueprintsMinimum: 1 },
production: { systemsMinimum: 5001, stationsMinimum: 1, requiredCategories: [4, 6, 9], blueprintsMinimum: 1 },
};
export interface ReferenceCompilerOptions {
@@ -281,7 +367,7 @@ export interface SourceCheckResult {
error?: string;
}
export interface ReferenceSnapshot extends ReferenceRepository {
export interface ReferenceSnapshot extends ReferenceReader {
close(): void;
readonly pinnedVersion: string;
}