From 24660edb09e7b1d5f2b6b2a16b067209b86b48b6 Mon Sep 17 00:00:00 2001 From: JB Date: Sat, 19 Sep 2026 02:54:00 -0400 Subject: [PATCH] fix(reference): preserve station evidence and rollback schema --- docs/reference-operations.md | 4 +-- docs/research/production-import.md | 2 +- src/lib/server/reference/compiler.ts | 9 ++++--- src/lib/server/reference/repository.ts | 37 +++++++------------------- src/lib/server/reference/schema.ts | 23 ++++++++++++++++ src/lib/server/reference/types.ts | 11 ++++++-- tests/reference-readers.test.ts | 9 ++++++- tests/reference.test.ts | 28 ++++++++++++++++--- 8 files changed, 83 insertions(+), 40 deletions(-) diff --git a/docs/reference-operations.md b/docs/reference-operations.md index 9565c89..087ac95 100644 --- a/docs/reference-operations.md +++ b/docs/reference-operations.md @@ -21,7 +21,7 @@ bun workers/reference.ts --production bun workers/reference.ts --production --offline --archive-dir data/reference-live ``` -The default production archive and artifact directory is `data/reference-live` (ignored by git); production publishes `data/reference-live/manifest.json`, while fixture builds remain under `data/reference`. An explicit `--directory` wins over `REFERENCE_DIR`, which wins over these defaults. `REFERENCE_LIVE_DIR` controls archive storage only. Compose explicitly publishes to `/app/data/reference`, the shared worker/web volume; the production web reader rejects fixture-profile artifacts. Runtime images must provide the `xz` executable (Debian package `xz-utils`) for streaming `.tar.xz` decompression; ZIP deflate uses the Bun runtime. No npm archive dependency is required. Production validation requires more than 5,000 systems, at least one station, and imports CCP's `mapSolarSystems`, `mapConstellations`, `mapRegions`, `mapStargates`, and `npcStations` JSONL members. A production candidate missing `npcStations.jsonl` or containing no station rows fails closed rather than claiming station coverage. The importer parses official station ID/system/type/owner/position facts; names remain unknown unless a separately verified name-bearing adapter is selected. The current worktree does not download or promote a production artifact, so deployed station coverage remains unverified until an isolated replay validates the member and its joins. +The default production archive and artifact directory is `data/reference-live` (ignored by git); production publishes `data/reference-live/manifest.json`, while fixture builds remain under `data/reference`. An explicit `--directory` wins over `REFERENCE_DIR`, which wins over these defaults. `REFERENCE_LIVE_DIR` controls archive storage only. Compose explicitly publishes to `/app/data/reference`, the shared worker/web volume; the production web reader rejects fixture-profile artifacts. Runtime images must provide the `xz` executable (Debian package `xz-utils`) for streaming `.tar.xz` decompression; ZIP deflate uses the Bun runtime. No npm archive dependency is required. Production validation requires more than 5,000 systems, at least one station, and imports CCP's `mapSolarSystems`, `mapConstellations`, `mapRegions`, `mapStargates`, and `npcStations` JSONL members. A production candidate missing `npcStations.jsonl` or containing no station rows fails closed rather than claiming any station rows. The importer parses official station ID/system/type/owner/position facts; names remain unknown unless a separately verified name-bearing adapter is selected. The production profile records station coverage as explicitly `incomplete` until a measured release replay proves source-consistent coverage; `stationsMinimum: 1` is only a non-empty safety gate, not a completeness claim. The current worktree does not download or promote a production artifact, so deployed station coverage remains unverified until an isolated replay validates the member, its joins, and its measured count. `fixtures/reference/` contains small schema-faithful EVE Ref keyed records for categories, groups, types, and blueprints (including types 34 Tritanium, 35 Pyerite, 587 Rifter, and blueprint 687). The Rifter recipe is synthetic and is labelled as such. Hoboleaks `industryactivities`, `repackagedvolumes`, and `meta` are independent raw maps. `systems.json` is explicitly an ESI-universe fixture: it preserves the official system fields and XYZ coordinate object and is not presented as a Reference Data `/systems` record. @@ -59,7 +59,7 @@ The lock prevents concurrent builds. A crash can leave a lock, staging directory ## Rollback and checks ```sh -REFERENCE_DIR=data/reference bun workers/reference.ts --rollback +REFERENCE_DIR=data/reference bun workers/reference.ts --fixture --rollback REFERENCE_DIR=data/reference bun workers/reference.ts --check --fixture bun workers/reference.ts --help ``` diff --git a/docs/research/production-import.md b/docs/research/production-import.md index 5b0ac48..f85a93e 100644 --- a/docs/research/production-import.md +++ b/docs/research/production-import.md @@ -166,7 +166,7 @@ Hoboleaks metadata also carries per-file `md5`, `revision`, `stale`, and `deprec The CCP JSON Lines SDE includes `npcStations.jsonl`; CCP's [schema changelog](https://developers.eveonline.com/static-data/tranquility/schema-changelog.yaml) records the station-schema migration and the [Map Data guide](https://developers.eveonline.com/docs/guides/map-data/) establishes the map hierarchy. A bounded inspection of the build-3503375 station member found 5,210 records with `_key`, `solarSystemID`, `ownerID`, `typeID`, and nested `position` (`x`, `y`, `z`); none carried a `name` field. The third-party schema page was used only as a cross-check, not as authority; this worktree did not download or promote the production archive. -The importer therefore treats CCP station IDs, system relationships, owner/type IDs, and coordinates as source facts, while station names remain unknown for the official SDE. The normalized station contract permits an optional `name` only when a separately verified name-bearing adapter supplies it; the production CCP adapter ignores all name-like fields and never derives a name from an ID or type. A station record must reference a system in the same candidate; malformed position/IDs or mismatched supplied system station lists fail closed. The selected production CCP parser requires `npcStations.jsonl` and the production validation profile requires at least one parsed station row; a missing member or empty station set fails closed instead of degrading to fabricated or incomplete station coverage. +The importer therefore treats CCP station IDs, system relationships, owner/type IDs, and coordinates as source facts, while station names remain unknown for the official SDE. The normalized station contract permits an optional `name` only when a separately verified name-bearing adapter supplies it; the production CCP adapter ignores all name-like fields and never derives a name from an ID or type. A station record must reference a system in the same candidate; malformed position/IDs or mismatched supplied system station lists fail closed. The selected production CCP parser requires `npcStations.jsonl` and the production validation profile requires at least one verified station row; a missing member or empty station set fails closed instead of degrading to fabricated coverage. This is explicitly an incomplete station-coverage profile until an isolated replay measures a source-consistent release count; it is not a universal completeness threshold. This worktree has not downloaded or replayed the production ZIP, so actual release member presence and station row coverage remain an acceptance blocker for deployed NPC station coverage. diff --git a/src/lib/server/reference/compiler.ts b/src/lib/server/reference/compiler.ts index 8084ab9..449b8f1 100644 --- a/src/lib/server/reference/compiler.ts +++ b/src/lib/server/reference/compiler.ts @@ -3,7 +3,7 @@ import { createHash } from 'node:crypto'; import { access, mkdir, open, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'; import { basename, isAbsolute, join, resolve } from 'node:path'; import { normalizeAll } from './normalize'; -import { createReferenceSchema, insertHoboleaksData, insertReferenceData, insertSourceMetadata, insertUniverseData, SCHEMA_VERSION } from './schema'; +import { assertReferenceReaderSchema, createReferenceSchema, insertHoboleaksData, insertReferenceData, insertSourceMetadata, insertUniverseData, SCHEMA_VERSION } from './schema'; import { fixtureSources, sha256 } from './sources'; import type { ActiveManifest, @@ -153,7 +153,7 @@ export class ReferenceCompiler { failureResults = sourceResults; const attempts: Partial> = {}; const loaded: Partial> = {}; - let changed = !manifest.active || manifest.active.profile !== this.profile || manifest.active.importerVersion !== this.importerVersion; + let changed = !manifest.active || manifest.active.profile !== this.profile || manifest.active.importerVersion !== this.importerVersion || manifest.active.stationCoverage !== this.thresholds.stationCoverage; const currentFailures: string[] = []; for (const source of REFERENCE_SOURCES) { @@ -264,7 +264,7 @@ export class ReferenceCompiler { return [source, { fingerprint: item.check.fingerprint!, cacheFile: item.cacheFile, localSha256: item.localSha256, canonicalSha256: item.canonicalSha256, archiveByteLength: item.archiveByteLength, checkedAt: item.check.checkedAt, url: item.check.url, etag: item.check.etag, etagIsChecksum: item.check.etagIsChecksum, lastModified: item.check.lastModified, byteLength: item.check.byteLength, importerVersion: this.importerVersion, expectedSha256: item.expectedSha256, sourceRevision: item.check.sourceRevision, sourceBuild: item.check.sourceBuild, stale: item.check.stale } satisfies ActiveSourceInput]; })) as Record; const committedSources = commitSourceResults(manifest.sources, sourceResults, loaded, this.importerVersion, this.now); - const active: ActiveManifest = { version, database: join(DATABASE_DIRECTORY, `${version}.sqlite`), createdAt: iso(this.now()), profile: this.profile, importerVersion: this.importerVersion, sources: Object.fromEntries(REFERENCE_SOURCES.map((source) => [source, loaded[source]!.check.fingerprint!])) as Record, sourceInputs, counts }; + const active: ActiveManifest = { version, database: join(DATABASE_DIRECTORY, `${version}.sqlite`), createdAt: iso(this.now()), profile: this.profile, importerVersion: this.importerVersion, sources: Object.fromEntries(REFERENCE_SOURCES.map((source) => [source, loaded[source]!.check.fingerprint!])) as Record, sourceInputs, counts, stationCoverage: this.thresholds.stationCoverage }; const sourceErrors = Object.values(committedSources).filter((result) => result?.status === 'failed').map((result) => `${result!.source}: ${result!.error ?? 'source failed'}`); const next: ReferenceManifest = { format: 1, active, previous: manifest.active, sources: committedSources, lastAttempt: { startedAt, finishedAt: iso(this.now()), status: 'success', profile: this.profile, version, ...(sourceErrors.length ? { error: sourceErrors.join('; ') } : {}) } }; await atomicWriteJson(this.directory, MANIFEST_FILE, next); @@ -348,7 +348,7 @@ export class ReferenceCompiler { } function validateRollbackArtifact(path: string, artifact: ActiveManifest, expectedProfile: BuildProfile, expectedImporterVersion: string, thresholds: ValidationThresholds): void { - if (artifact.profile !== expectedProfile || artifact.importerVersion !== expectedImporterVersion) throw new Error('previous reference artifact is incompatible with the current compiler/profile'); + if (artifact.profile !== expectedProfile || artifact.importerVersion !== expectedImporterVersion || artifact.stationCoverage !== thresholds.stationCoverage) throw new Error('previous reference artifact is incompatible with the current compiler/profile'); if (!artifact.sourceInputs || !artifact.counts) throw new Error('previous reference artifact is missing compatibility metadata'); let db: Database | undefined; try { @@ -538,6 +538,7 @@ async function validateExistingArtifact(path: string, input: ReturnType { const schemaVersion = Number((db.query<{ user_version: number }, []>('PRAGMA user_version').get() as { user_version: number } | null)?.user_version ?? 0); if (schemaVersion !== SCHEMA_VERSION) throw new Error(`SQLite schema version ${schemaVersion} does not match compiler ${SCHEMA_VERSION}`); + assertReferenceReaderSchema(db); const integrity = db.query<{ integrity_check: string }, []>('PRAGMA integrity_check').all(); if (integrity.length !== 1 || integrity[0].integrity_check !== 'ok') throw new Error(`SQLite integrity_check failed: ${JSON.stringify(integrity)}`); const foreignKeys = db.query('PRAGMA foreign_key_check').all(); if (foreignKeys.length) throw new Error(`SQLite foreign_key_check failed: ${JSON.stringify(foreignKeys)}`); diff --git a/src/lib/server/reference/repository.ts b/src/lib/server/reference/repository.ts index e1f861d..3fda133 100644 --- a/src/lib/server/reference/repository.ts +++ b/src/lib/server/reference/repository.ts @@ -22,19 +22,19 @@ import type { ReferenceTypeRecord, } from './types'; import { MANIFEST_FILE, DATABASE_DIRECTORY } from './compiler'; -import { SCHEMA_VERSION } from './schema'; +import { assertReferenceReaderSchema } from './schema'; const MAX_IN_BATCH = 500; const MAX_READER_LIMIT = 500; const MAX_READER_OFFSET = 100_000; -type TypeRow = { type_id: number; name_json: string; category_id: number; volume: number | null }; +type TypeRow = { type_id: number; name_json: string; category_id: number; group_id: number; volume: number | null }; type SearchTypeRow = { type_id: number; name: string; category_id: number; group_id: number; published: number | null; volume: number | null }; type CategoryRow = { category_id: number; name_json: string; published: number | null }; type GroupRow = { group_id: number; category_id: number; name_json: string; published: number | null }; type RegionRow = { region_id: number; name: string | null }; type ConstellationRow = { constellation_id: number; name: string | null; region_id: number | null }; type SystemRow = { system_id: number; name: string; constellation_id: number; region_id: number | null; x: number; y: number; z: number; security_status: number; security_class: string | null }; -type StationRow = { station_id: number; system_id: number; system_name: string | null; region_id: number | null; name: string | null; type_id: number | null; owner_id: number | null; x: number | null; y: number | null; z: number | null }; +type StationRow = { station_id: number; system_id: number; system_name: string | null; region_id: number | null; name: string | null; type_id: number | null; owner_id: number | null; x: number | null; y: number | null; z: number | null; source_verified: number }; type ListSelection = ReferenceListOptions; type IdSelection = ReferenceIdSelection; type GroupSelection = ReferenceGroupSelection; @@ -75,6 +75,7 @@ export class SqliteReferenceRepository implements ReferenceReader, ReferenceSnap const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as ReferenceManifest; if (manifest.format !== 1 || !manifest.active) throw new Error(`reference manifest has no active artifact: ${manifestPath}`); if (expectedProfile && manifest.active.profile !== expectedProfile) throw new Error(`reference artifact must use the ${expectedProfile} profile`); + if (manifest.active.stationCoverage !== 'incomplete' && manifest.active.stationCoverage !== 'measured') throw new Error('reference artifact is missing station coverage profile metadata'); this.pinnedVersion = manifest.active.version; this.databasePath = assertArtifactPath(root, manifest.active.database); if (basename(this.databasePath) !== `${this.pinnedVersion}.sqlite`) throw new Error('active reference database filename does not match its manifest version'); @@ -191,31 +192,13 @@ function queryStations(db: Database, selection: StationSelection): Promise(`SELECT st.station_id,st.system_id,s.name AS system_name,c.region_id,st.name,st.type_id,st.owner_id,st.x,st.y,st.z FROM stations st JOIN systems s ON s.system_id = st.system_id LEFT JOIN constellations c ON c.constellation_id = s.constellation_id${conditions.length ? ` WHERE ${conditions.join(' AND ')}` : ''} ORDER BY st.station_id LIMIT ? OFFSET ?`).all(...args, limit(Number(optionValue(selection, 'limit'))), bounded(Number(optionValue(selection, 'offset')), 0)); - return Promise.resolve(rows.map((row) => ({ id: row.station_id, systemId: row.system_id, systemName: row.system_name, regionId: row.region_id, name: row.name, typeId: row.type_id, ownerId: row.owner_id, x: row.x, y: row.y, z: row.z }))); + const rows = db.query(`SELECT st.station_id,st.system_id,s.name AS system_name,c.region_id,st.name,st.type_id,st.owner_id,st.x,st.y,st.z,st.source_verified FROM stations st JOIN systems s ON s.system_id = st.system_id LEFT JOIN constellations c ON c.constellation_id = s.constellation_id${conditions.length ? ` WHERE ${conditions.join(' AND ')}` : ''} ORDER BY st.station_id LIMIT ? OFFSET ?`).all(...args, limit(Number(optionValue(selection, 'limit'))), bounded(Number(optionValue(selection, 'offset')), 0)); + return Promise.resolve(rows.map((row) => ({ id: row.station_id, systemId: row.system_id, systemName: row.system_name, regionId: row.region_id, sourceVerified: row.source_verified === 1, name: row.name, typeId: row.type_id, ownerId: row.owner_id, x: row.x, y: row.y, z: row.z }))); } -/** Cheap metadata check only; full integrity_check belongs to the compiler. */ +/** Reader construction closes malformed handles before surfacing the schema error. */ function assertReaderSchema(db: Database): void { - const version = Number((db.query<{ user_version: number }, []>('PRAGMA user_version').get() as { user_version: number } | null)?.user_version ?? 0); - const row = db.query<{ present: number }, []>("SELECT COUNT(*) AS present FROM sqlite_master WHERE type='table' AND name IN ('types','type_search','type_search_fts','categories','groups','regions','constellations','systems','stations')").get(); - const index = db.query<{ present: number }, []>("SELECT COUNT(*) AS present FROM sqlite_master WHERE type='index' AND name='type_search_normalized_name_idx'").get(); - const requiredColumns: Record = { - types: ['type_id', 'group_id', 'category_id', 'name_json', 'published', 'volume'], - type_search: ['type_id', 'language', 'name', 'normalized_name'], - type_search_fts: ['type_id', 'language', 'name', 'normalized_name'], - categories: ['category_id', 'name_json', 'published'], - groups: ['group_id', 'category_id', 'name_json', 'published'], - regions: ['region_id', 'name'], - constellations: ['constellation_id', 'region_id', 'name'], - stations: ['station_id', 'system_id', 'name', 'type_id', 'owner_id', 'x', 'y', 'z', 'source_verified'], - systems: ['system_id', 'constellation_id', 'name', 'x', 'y', 'z', 'security_status', 'security_class'], - }; - const columnsValid = Object.entries(requiredColumns).every(([table, columns]) => { - const present = new Set(db.query<{ name: string }, []>(`PRAGMA table_info(${table})`).all().map((column) => column.name)); - return columns.every((column) => present.has(column)); - }); - if (version !== SCHEMA_VERSION || !row || Number(row.present) !== 9 || !index || Number(index.present) !== 1 || !columnsValid) { db.close(); throw new Error(`active reference database has an incompatible schema version (expected ${SCHEMA_VERSION})`); } + try { assertReferenceReaderSchema(db); } catch (error) { db.close(); throw error; } } async function queryTypes(db: Database, requested: number[]): Promise { @@ -227,12 +210,12 @@ async function queryTypes(db: Database, requested: number[]): Promise '?').join(','); - rows.push(...db.query(`SELECT type_id,name_json,category_id,volume FROM types WHERE type_id IN (${placeholders})`).all(...batch)); + rows.push(...db.query(`SELECT type_id,name_json,category_id,group_id,volume FROM types WHERE type_id IN (${placeholders})`).all(...batch)); } const byId = new Map(rows.map((row) => [row.type_id, row])); return unique.flatMap((id) => { const row = byId.get(id); if (!row) return []; const names = JSON.parse(row.name_json) as Record; - return [{ id: row.type_id, name: names.en ?? Object.values(names)[0] ?? String(row.type_id), categoryId: row.category_id, ...(row.volume === null ? {} : { volume: row.volume }) }]; + return [{ id: row.type_id, name: names.en ?? Object.values(names)[0] ?? String(row.type_id), categoryId: row.category_id, groupId: row.group_id, ...(row.volume === null ? {} : { volume: row.volume }) }]; }); } diff --git a/src/lib/server/reference/schema.ts b/src/lib/server/reference/schema.ts index 1e814de..745ef6b 100644 --- a/src/lib/server/reference/schema.ts +++ b/src/lib/server/reference/schema.ts @@ -205,6 +205,29 @@ export function createReferenceSchema(db: Database): void { `); } +/** Validate the complete read schema before a runtime handle or rollback can use an artifact. */ +export function assertReferenceReaderSchema(db: Database): void { + const version = Number((db.query<{ user_version: number }, []>('PRAGMA user_version').get() as { user_version: number } | null)?.user_version ?? 0); + const row = db.query<{ present: number }, []>("SELECT COUNT(*) AS present FROM sqlite_master WHERE type='table' AND name IN ('types','type_search','type_search_fts','categories','groups','regions','constellations','systems','stations')").get(); + const index = db.query<{ present: number }, []>("SELECT COUNT(*) AS present FROM sqlite_master WHERE type='index' AND name='type_search_normalized_name_idx'").get(); + const requiredColumns: Record = { + types: ['type_id', 'group_id', 'category_id', 'name_json', 'published', 'volume'], + type_search: ['type_id', 'language', 'name', 'normalized_name'], + type_search_fts: ['type_id', 'language', 'name', 'normalized_name'], + categories: ['category_id', 'name_json', 'published'], + groups: ['group_id', 'category_id', 'name_json', 'published'], + regions: ['region_id', 'name'], + constellations: ['constellation_id', 'region_id', 'name'], + stations: ['station_id', 'system_id', 'name', 'type_id', 'owner_id', 'x', 'y', 'z', 'source_verified'], + systems: ['system_id', 'constellation_id', 'name', 'x', 'y', 'z', 'security_status', 'security_class'], + }; + const columnsValid = Object.entries(requiredColumns).every(([table, columns]) => { + const present = new Set(db.query<{ name: string }, []>(`PRAGMA table_info(${table})`).all().map((column) => column.name)); + return columns.every((column) => present.has(column)); + }); + if (version !== SCHEMA_VERSION || !row || Number(row.present) !== 9 || !index || Number(index.present) !== 1 || !columnsValid) throw new Error(`active reference database has an incompatible schema version (expected ${SCHEMA_VERSION})`); +} + const bool = (value: boolean | undefined): number | null => value === undefined ? null : value ? 1 : 0; const json = (value: unknown): string => JSON.stringify(value); const normalizedName = (value: string): string => value.normalize('NFKC').trim().replace(/\s+/g, ' ').toLocaleLowerCase('en-US'); diff --git a/src/lib/server/reference/types.ts b/src/lib/server/reference/types.ts index a309c9c..e80744a 100644 --- a/src/lib/server/reference/types.ts +++ b/src/lib/server/reference/types.ts @@ -178,6 +178,8 @@ export interface ReferenceStationRecord { systemId: number; systemName: string | null; regionId: number | null; + /** True only for a station row imported from the authoritative station member. */ + sourceVerified: boolean; name: string | null; typeId: number | null; ownerId: number | null; @@ -260,15 +262,18 @@ export interface ProductionArchiveSourceOptions { } export type BuildProfile = 'fixture' | 'production'; +/** Station rows are present but not completeness-verified until a measured release replay is captured. */ +export type StationCoverageProfile = 'incomplete' | 'measured'; export interface ValidationThresholds { systemsMinimum: number; stationsMinimum: number; + stationCoverage: StationCoverageProfile; requiredCategories: number[]; blueprintsMinimum: number; } export const PROFILE_THRESHOLDS: Record = { - fixture: { systemsMinimum: 1, stationsMinimum: 0, requiredCategories: [4, 6, 9], blueprintsMinimum: 1 }, - production: { systemsMinimum: 5001, stationsMinimum: 1, requiredCategories: [4, 6, 9], blueprintsMinimum: 1 }, + fixture: { systemsMinimum: 1, stationsMinimum: 0, stationCoverage: 'incomplete', requiredCategories: [4, 6, 9], blueprintsMinimum: 1 }, + production: { systemsMinimum: 5001, stationsMinimum: 1, stationCoverage: 'incomplete', requiredCategories: [4, 6, 9], blueprintsMinimum: 1 }, }; export interface ReferenceCompilerOptions { @@ -318,6 +323,8 @@ export interface ActiveManifest { /** Content-addressed source cache selected by this artifact. */ sourceInputs: Record; counts: Record; + /** Completeness is explicit; production remains incomplete until measured replay evidence exists. */ + stationCoverage?: StationCoverageProfile; } export interface ActiveSourceInput { fingerprint: string; diff --git a/tests/reference-readers.test.ts b/tests/reference-readers.test.ts index 1e2394d..5dad1af 100644 --- a/tests/reference-readers.test.ts +++ b/tests/reference-readers.test.ts @@ -16,6 +16,7 @@ test('reference readers search normalized types and expose group/category/system universe.regions = { '10000001': { region_id: 10000001, name: 'The Forge' } }; universe.constellations = { '20000020': { constellation_id: 20000020, name: 'Kimotoro', region_id: 10000001 } }; universe.stations = { '60003760': { station_id: 60003760, system_id: 30000142, name: 'Jita IV - Moon 4 - Caldari Navy Assembly Plant', type_id: 587, owner_id: 1000035, position: { x: 1, y: 2, z: 3 } } }; + (universe.systems as Record>)['30000142'].stations = [60003761]; await writeFile(universePath, JSON.stringify(universe)); const result = await new ReferenceCompiler({ directory, profile: 'fixture', sources: fixtureSources(sources) }).compile(); @@ -28,7 +29,13 @@ test('reference readers search normalized types and expose group/category/system expect(await reader.groups({ categoryId: 4 })).toEqual([{ id: 18, categoryId: 4, name: 'Mineral', published: true }]); expect(await reader.regions({ query: 'forge' })).toEqual([{ id: 10000001, name: 'The Forge' }]); expect(await reader.systems({ regionId: 10000001 })).toEqual([{ id: 30000142, name: 'Jita', constellationId: 20000020, regionId: 10000001, x: -129064861735000000, y: 60755306910000000, z: 117469227060000000, securityStatus: 0.9459131360054016, securityClass: null }]); - expect(await reader.stations({ systemId: 30000142 })).toEqual([{ id: 60003760, systemId: 30000142, systemName: 'Jita', regionId: 10000001, name: 'Jita IV - Moon 4 - Caldari Navy Assembly Plant', typeId: 587, ownerId: 1000035, x: 1, y: 2, z: 3 }]); + const stations = await reader.stations({ systemId: 30000142 }); + expect(stations).toHaveLength(2); + expect(stations).toEqual([ + { id: 60003760, systemId: 30000142, systemName: 'Jita', regionId: 10000001, sourceVerified: true, name: 'Jita IV - Moon 4 - Caldari Navy Assembly Plant', typeId: 587, ownerId: 1000035, x: 1, y: 2, z: 3 }, + { id: 60003761, systemId: 30000142, systemName: 'Jita', regionId: 10000001, sourceVerified: false, name: null, typeId: null, ownerId: null, x: null, y: null, z: null }, + ]); + expect(await reader.stations({ query: 'assembly' })).toEqual([stations[0]]); const pinned = reader.pin(); try { expect((await pinned.searchTypesByName('pyerite'))[0]?.id).toBe(35); } finally { pinned.close(); } diff --git a/tests/reference.test.ts b/tests/reference.test.ts index d925b7d..46fd65d 100644 --- a/tests/reference.test.ts +++ b/tests/reference.test.ts @@ -24,9 +24,9 @@ describe('fixture reference compiler', () => { const resultVersion = result.version; const repository = new SqliteReferenceRepository(join(root, 'reference')); expect(await repository.types([34, 35, 587, 999999])).toEqual([ - { id: 34, name: 'Tritanium', categoryId: 4, volume: 0.01 }, - { id: 35, name: 'Pyerite', categoryId: 4, volume: 0.01 }, - { id: 587, name: 'Rifter', categoryId: 6, volume: 2500 }, + expect.objectContaining({ id: 34, name: 'Tritanium', categoryId: 4, groupId: 18, volume: 0.01 }), + expect.objectContaining({ id: 35, name: 'Pyerite', categoryId: 4, groupId: 18, volume: 0.01 }), + expect.objectContaining({ id: 587, name: 'Rifter', categoryId: 6, groupId: 25, volume: 2500 }), ]); expect(await repository.version()).toBe(resultVersion); repository.close(); @@ -299,6 +299,28 @@ describe('fixture reference compiler', () => { } finally { await rm(root, { recursive: true, force: true }); } }); + test('rejects a malformed PREVIOUS artifact before rollback', async () => { + const { root, sources } = await fixtureDirectory(); + const directory = join(root, 'reference'); + try { + const first = await new ReferenceCompiler({ directory, profile: 'fixture', sources: fixtureSources(sources) }).compile(); + expect(first.status).toBe('success'); + if (first.status !== 'success' || !first.version) throw new Error('initial fixture build did not publish a version'); + await writeFile(join(sources, 'reference-data.json'), `${await readFile(join(sources, 'reference-data.json'), 'utf8')}\n`); + const second = await new ReferenceCompiler({ directory, profile: 'fixture', sources: fixtureSources(sources) }).compile(); + expect(second.status).toBe('success'); + if (second.status !== 'success' || !second.version) throw new Error('second fixture build did not publish a version'); + const manifest = await readReferenceManifest(directory); + if (!manifest.previous) throw new Error('fixture build did not retain PREVIOUS'); + const previousPath = join(directory, 'databases', `${manifest.previous.version}.sqlite`); + const malformed = new Database(previousPath); + malformed.run('DROP INDEX type_search_normalized_name_idx'); + malformed.close(); + await expect(new ReferenceCompiler({ directory, profile: 'fixture', sources: fixtureSources(sources) }).rollback()).rejects.toThrow(/incompatible schema version/); + expect((await readReferenceManifest(directory)).active?.version).toBe(second.version); + } finally { await rm(root, { recursive: true, force: true }); } + }); + test('recovers a validated orphan database after manifest publication crash', async () => { const { root, sources } = await fixtureDirectory(); const directory = join(root, 'reference');