325 lines
18 KiB
TypeScript
325 lines
18 KiB
TypeScript
import type { Database } from 'bun:sqlite';
|
|
import type {
|
|
EsiUniversePayload,
|
|
HoboleaksPayload,
|
|
ReferenceBlueprint,
|
|
ReferenceCategory,
|
|
ReferenceDataPayload,
|
|
ReferenceGroup,
|
|
ReferenceType,
|
|
} from './types';
|
|
|
|
export const SCHEMA_VERSION = 3;
|
|
|
|
/**
|
|
* The database is built from scratch and opened read-only by the serving side.
|
|
* Foreign keys are deliberately explicit even for currently empty future tables.
|
|
*/
|
|
export function createReferenceSchema(db: Database): void {
|
|
db.run(`PRAGMA user_version = ${SCHEMA_VERSION}`);
|
|
db.run('PRAGMA foreign_keys = ON');
|
|
db.run('PRAGMA journal_mode = DELETE');
|
|
db.run('PRAGMA synchronous = FULL');
|
|
db.run(`
|
|
CREATE TABLE categories (
|
|
category_id INTEGER PRIMARY KEY,
|
|
name_json TEXT NOT NULL,
|
|
published INTEGER,
|
|
icon_id INTEGER
|
|
);
|
|
CREATE TABLE groups (
|
|
group_id INTEGER PRIMARY KEY,
|
|
category_id INTEGER NOT NULL REFERENCES categories(category_id),
|
|
name_json TEXT NOT NULL,
|
|
published INTEGER,
|
|
icon_id INTEGER,
|
|
anchorable INTEGER,
|
|
anchored INTEGER,
|
|
fittable_non_singleton INTEGER,
|
|
use_base_price INTEGER
|
|
);
|
|
CREATE TABLE types (
|
|
type_id INTEGER PRIMARY KEY,
|
|
group_id INTEGER NOT NULL REFERENCES groups(group_id),
|
|
category_id INTEGER NOT NULL REFERENCES categories(category_id),
|
|
name_json TEXT NOT NULL,
|
|
description_json TEXT,
|
|
published INTEGER,
|
|
is_blueprint INTEGER,
|
|
is_skill INTEGER,
|
|
is_ore INTEGER,
|
|
is_repackable INTEGER,
|
|
volume REAL,
|
|
packaged_volume REAL,
|
|
portion_size INTEGER,
|
|
mass REAL,
|
|
capacity REAL,
|
|
radius REAL,
|
|
base_price REAL
|
|
);
|
|
CREATE TABLE type_search (
|
|
type_id INTEGER NOT NULL REFERENCES types(type_id),
|
|
language TEXT NOT NULL,
|
|
name TEXT NOT NULL,
|
|
normalized_name TEXT NOT NULL,
|
|
PRIMARY KEY (type_id, language)
|
|
);
|
|
CREATE TABLE market_groups (
|
|
market_group_id INTEGER PRIMARY KEY,
|
|
parent_group_id INTEGER REFERENCES market_groups(market_group_id),
|
|
name_json TEXT,
|
|
description_json TEXT,
|
|
icon_id INTEGER,
|
|
has_types INTEGER
|
|
);
|
|
CREATE TABLE blueprints (
|
|
blueprint_type_id INTEGER PRIMARY KEY REFERENCES types(type_id),
|
|
max_production_limit INTEGER
|
|
);
|
|
CREATE TABLE activities (
|
|
activity_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
blueprint_type_id INTEGER NOT NULL REFERENCES blueprints(blueprint_type_id),
|
|
activity_name TEXT NOT NULL,
|
|
required_skills_json TEXT,
|
|
time INTEGER NOT NULL,
|
|
UNIQUE (blueprint_type_id, activity_name)
|
|
);
|
|
CREATE TABLE materials (
|
|
activity_id INTEGER NOT NULL REFERENCES activities(activity_id),
|
|
type_id INTEGER NOT NULL,
|
|
quantity INTEGER NOT NULL,
|
|
probability REAL,
|
|
PRIMARY KEY (activity_id, type_id)
|
|
);
|
|
CREATE TABLE products (
|
|
activity_id INTEGER NOT NULL REFERENCES activities(activity_id),
|
|
type_id INTEGER NOT NULL,
|
|
quantity INTEGER NOT NULL,
|
|
probability REAL,
|
|
PRIMARY KEY (activity_id, type_id)
|
|
);
|
|
CREATE TABLE unresolved_references (
|
|
kind TEXT NOT NULL,
|
|
owner_id INTEGER NOT NULL,
|
|
activity_name TEXT NOT NULL,
|
|
target_id INTEGER NOT NULL,
|
|
PRIMARY KEY (kind, owner_id, activity_name, target_id)
|
|
);
|
|
CREATE TABLE type_materials (
|
|
type_id INTEGER NOT NULL REFERENCES types(type_id),
|
|
material_type_id INTEGER NOT NULL REFERENCES types(type_id),
|
|
quantity INTEGER NOT NULL,
|
|
PRIMARY KEY (type_id, material_type_id)
|
|
);
|
|
CREATE TABLE dogma_attributes (
|
|
type_id INTEGER NOT NULL REFERENCES types(type_id),
|
|
attribute_id INTEGER NOT NULL,
|
|
value REAL NOT NULL,
|
|
PRIMARY KEY (type_id, attribute_id)
|
|
);
|
|
CREATE TABLE dogma_effects (
|
|
type_id INTEGER NOT NULL REFERENCES types(type_id),
|
|
effect_id INTEGER NOT NULL,
|
|
value REAL,
|
|
is_default INTEGER,
|
|
PRIMARY KEY (type_id, effect_id)
|
|
);
|
|
CREATE TABLE required_skills (
|
|
type_id INTEGER NOT NULL REFERENCES types(type_id),
|
|
skill_type_id INTEGER NOT NULL REFERENCES types(type_id),
|
|
level INTEGER NOT NULL,
|
|
PRIMARY KEY (type_id, skill_type_id)
|
|
);
|
|
CREATE TABLE produced_by_blueprints (
|
|
type_id INTEGER NOT NULL REFERENCES types(type_id),
|
|
blueprint_type_id INTEGER NOT NULL REFERENCES blueprints(blueprint_type_id),
|
|
activity_name TEXT,
|
|
PRIMARY KEY (type_id, blueprint_type_id, activity_name)
|
|
);
|
|
CREATE TABLE regions (
|
|
region_id INTEGER PRIMARY KEY,
|
|
name TEXT
|
|
);
|
|
CREATE TABLE constellations (
|
|
constellation_id INTEGER PRIMARY KEY,
|
|
region_id INTEGER REFERENCES regions(region_id),
|
|
name TEXT
|
|
);
|
|
CREATE TABLE systems (
|
|
system_id INTEGER PRIMARY KEY,
|
|
constellation_id INTEGER NOT NULL REFERENCES constellations(constellation_id),
|
|
name TEXT NOT NULL,
|
|
x REAL NOT NULL,
|
|
y REAL NOT NULL,
|
|
z REAL NOT NULL,
|
|
security_status REAL NOT NULL,
|
|
security_class TEXT
|
|
);
|
|
CREATE TABLE gates (
|
|
system_id INTEGER NOT NULL REFERENCES systems(system_id),
|
|
stargate_id INTEGER NOT NULL,
|
|
destination_system_id INTEGER NOT NULL REFERENCES systems(system_id),
|
|
PRIMARY KEY (system_id, stargate_id)
|
|
);
|
|
CREATE TABLE stations (
|
|
station_id INTEGER PRIMARY KEY,
|
|
system_id INTEGER NOT NULL REFERENCES systems(system_id),
|
|
name TEXT,
|
|
type_id INTEGER REFERENCES types(type_id),
|
|
owner_id INTEGER,
|
|
x REAL,
|
|
y REAL,
|
|
z REAL,
|
|
source_verified INTEGER NOT NULL CHECK (source_verified IN (0, 1))
|
|
);
|
|
CREATE TABLE source_provenance (
|
|
source TEXT PRIMARY KEY,
|
|
fingerprint TEXT NOT NULL,
|
|
checked_at TEXT NOT NULL,
|
|
local_sha256 TEXT NOT NULL,
|
|
source_revision TEXT,
|
|
source_build TEXT,
|
|
stale INTEGER
|
|
);
|
|
CREATE INDEX groups_category_idx ON groups(category_id);
|
|
CREATE INDEX types_group_idx ON types(group_id);
|
|
CREATE INDEX types_category_idx ON types(category_id);
|
|
CREATE INDEX types_name_idx ON types(name_json);
|
|
CREATE INDEX type_search_normalized_name_idx ON type_search(normalized_name);
|
|
CREATE VIRTUAL TABLE type_search_fts USING fts5(type_id UNINDEXED, language UNINDEXED, name, normalized_name);
|
|
CREATE INDEX market_groups_parent_idx ON market_groups(parent_group_id);
|
|
CREATE INDEX activities_blueprint_idx ON activities(blueprint_type_id);
|
|
CREATE INDEX materials_type_idx ON materials(type_id);
|
|
CREATE INDEX products_type_idx ON products(type_id);
|
|
CREATE INDEX unresolved_references_target_idx ON unresolved_references(target_id);
|
|
CREATE INDEX type_materials_material_idx ON type_materials(material_type_id);
|
|
CREATE INDEX dogma_attributes_type_idx ON dogma_attributes(type_id);
|
|
CREATE INDEX dogma_effects_type_idx ON dogma_effects(type_id);
|
|
CREATE INDEX required_skills_type_idx ON required_skills(type_id);
|
|
CREATE INDEX produced_by_blueprints_type_idx ON produced_by_blueprints(type_id);
|
|
CREATE INDEX constellations_region_idx ON constellations(region_id);
|
|
CREATE INDEX systems_constellation_idx ON systems(constellation_id);
|
|
CREATE INDEX systems_name_idx ON systems(name);
|
|
CREATE INDEX gates_destination_idx ON gates(destination_system_id);
|
|
CREATE INDEX stations_system_idx ON stations(system_id);
|
|
`);
|
|
}
|
|
|
|
const bool = (value: boolean | undefined): number | null => value === undefined ? null : value ? 1 : 0;
|
|
const json = (value: unknown): string => JSON.stringify(value);
|
|
const normalizedName = (value: string): string => value.normalize('NFKC').trim().replace(/\s+/g, ' ').toLocaleLowerCase('en-US');
|
|
const keyId = (key: string, label: string): number => {
|
|
const id = Number(key);
|
|
if (!Number.isSafeInteger(id) || id < 0) throw new Error(`${label} key is not a safe integer: ${key}`);
|
|
return id;
|
|
};
|
|
|
|
export interface SourceDatabaseMetadata {
|
|
source: string;
|
|
fingerprint: string;
|
|
checkedAt: string;
|
|
localSha256: string;
|
|
sourceRevision?: string | number;
|
|
sourceBuild?: string;
|
|
stale?: boolean;
|
|
}
|
|
|
|
export function insertReferenceData(db: Database, data: ReferenceDataPayload): void {
|
|
const category = db.prepare('INSERT INTO categories(category_id,name_json,published,icon_id) VALUES (?,?,?,?)');
|
|
for (const [key, row] of Object.entries(data.categories)) {
|
|
const id = keyId(key, 'category');
|
|
category.run(id, json(row.name), bool(row.published), row.icon_id ?? null);
|
|
}
|
|
const group = db.prepare('INSERT INTO groups(group_id,category_id,name_json,published,icon_id,anchorable,anchored,fittable_non_singleton,use_base_price) VALUES (?,?,?,?,?,?,?,?,?)');
|
|
for (const [key, row] of Object.entries(data.groups)) {
|
|
const id = keyId(key, 'group');
|
|
group.run(id, row.category_id, json(row.name), bool(row.published), row.icon_id ?? null, bool(row.anchorable), bool(row.anchored), bool(row.fittable_non_singleton), bool(row.use_base_price));
|
|
}
|
|
const type = db.prepare('INSERT INTO types(type_id,group_id,category_id,name_json,description_json,published,is_blueprint,is_skill,is_ore,is_repackable,volume,packaged_volume,portion_size,mass,capacity,radius,base_price) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)');
|
|
for (const [key, row] of Object.entries(data.types)) {
|
|
const id = keyId(key, 'type');
|
|
type.run(id, row.group_id, row.category_id, json(row.name), row.description === undefined ? null : json(row.description), bool(row.published), bool(row.is_blueprint), bool(row.is_skill), bool(row.is_ore), bool(row.is_repackable), row.volume ?? null, row.packaged_volume ?? null, row.portion_size ?? null, row.mass ?? null, row.capacity ?? null, row.radius ?? null, row.base_price ?? null);
|
|
}
|
|
insertTypeMaps(db, data.types);
|
|
const typeSearch = db.prepare('INSERT INTO type_search(type_id,language,name,normalized_name) VALUES (?,?,?,?)');
|
|
const typeSearchFts = db.prepare('INSERT INTO type_search_fts(type_id,language,name,normalized_name) VALUES (?,?,?,?)');
|
|
for (const row of Object.values(data.types)) for (const [language, name] of Object.entries(row.name)) { typeSearch.run(row.type_id, language, name, normalizedName(name)); typeSearchFts.run(row.type_id, language, name, normalizedName(name)); }
|
|
const marketGroup = db.prepare('INSERT INTO market_groups(market_group_id,parent_group_id,name_json,description_json,icon_id,has_types) VALUES (?,?,?,?,?,?)');
|
|
for (const [key, value] of Object.entries(data.market_groups ?? {})) {
|
|
const row = value as Record<string, unknown>;
|
|
marketGroup.run(keyId(key, 'market group'), typeof row.parent_group_id === 'number' ? row.parent_group_id : null, row.name === undefined ? null : json(row.name), row.description === undefined ? null : json(row.description), typeof row.icon_id === 'number' ? row.icon_id : null, typeof row.has_types === 'boolean' ? bool(row.has_types) : null);
|
|
}
|
|
const blueprint = db.prepare('INSERT INTO blueprints(blueprint_type_id,max_production_limit) VALUES (?,?)');
|
|
const activity = db.prepare('INSERT INTO activities(blueprint_type_id,activity_name,required_skills_json,time) VALUES (?,?,?,?)');
|
|
const material = db.prepare('INSERT INTO materials(activity_id,type_id,quantity,probability) VALUES (?,?,?,?)');
|
|
const product = db.prepare('INSERT INTO products(activity_id,type_id,quantity,probability) VALUES (?,?,?,?)');
|
|
const unresolved = db.prepare('INSERT INTO unresolved_references(kind,owner_id,activity_name,target_id) VALUES (?,?,?,?)');
|
|
const knownTypes = new Set(Object.keys(data.types).map(Number));
|
|
for (const [key, row] of Object.entries(data.blueprints)) {
|
|
const blueprintId = keyId(key, 'blueprint');
|
|
blueprint.run(blueprintId, row.max_production_limit ?? null);
|
|
for (const [activityName, activityRow] of Object.entries(row.activities)) {
|
|
const result = activity.run(blueprintId, activityName, activityRow.required_skills === undefined ? null : json(activityRow.required_skills), activityRow.time);
|
|
const activityId = Number(result.lastInsertRowid);
|
|
for (const materialRow of Object.values(activityRow.materials)) { material.run(activityId, materialRow.type_id, materialRow.quantity, materialRow.probability ?? null); if (!knownTypes.has(materialRow.type_id)) unresolved.run('blueprint-material', blueprintId, activityName, materialRow.type_id); }
|
|
for (const productRow of Object.values(activityRow.products)) { product.run(activityId, productRow.type_id, productRow.quantity, productRow.probability ?? null); if (!knownTypes.has(productRow.type_id)) unresolved.run('blueprint-product', blueprintId, activityName, productRow.type_id); }
|
|
}
|
|
}
|
|
}
|
|
|
|
function insertTypeMaps(db: Database, types: Record<string, ReferenceType>): void {
|
|
const material = db.prepare('INSERT INTO type_materials(type_id,material_type_id,quantity) VALUES (?,?,?)');
|
|
const attribute = db.prepare('INSERT INTO dogma_attributes(type_id,attribute_id,value) VALUES (?,?,?)');
|
|
const effect = db.prepare('INSERT INTO dogma_effects(type_id,effect_id,value,is_default) VALUES (?,?,?,?)');
|
|
const skill = db.prepare('INSERT INTO required_skills(type_id,skill_type_id,level) VALUES (?,?,?)');
|
|
const produced = db.prepare('INSERT INTO produced_by_blueprints(type_id,blueprint_type_id,activity_name) VALUES (?,?,?)');
|
|
for (const row of Object.values(types)) {
|
|
for (const [key, item] of Object.entries(row.type_materials ?? {})) material.run(row.type_id, item.type_id, item.quantity);
|
|
for (const [key, value] of Object.entries(row.dogma_attributes ?? {})) attribute.run(row.type_id, value.attribute_id, value.value);
|
|
for (const [key, value] of Object.entries(row.dogma_effects ?? {})) effect.run(row.type_id, value.effect_id, null, value.is_default === undefined ? null : value.is_default ? 1 : 0);
|
|
for (const [key, value] of Object.entries(row.required_skills ?? {})) skill.run(row.type_id, keyId(key, 'required skill'), value);
|
|
for (const [key, value] of Object.entries(row.produced_by_blueprints ?? {})) produced.run(row.type_id, value.blueprint_type_id, value.blueprint_activity ?? null);
|
|
}
|
|
}
|
|
|
|
export function insertUniverseData(db: Database, data: EsiUniversePayload): void {
|
|
const region = db.prepare('INSERT INTO regions(region_id,name) VALUES (?,?)');
|
|
for (const [key, row] of Object.entries(data.regions ?? {})) region.run(keyId(key, 'region'), row.name ?? null);
|
|
const constellation = db.prepare('INSERT INTO constellations(constellation_id,region_id,name) VALUES (?,?,?)');
|
|
for (const [key, row] of Object.entries(data.constellations ?? {})) constellation.run(keyId(key, 'constellation'), row.region_id ?? null, row.name ?? null);
|
|
const system = db.prepare('INSERT INTO systems(system_id,constellation_id,name,x,y,z,security_status,security_class) VALUES (?,?,?,?,?,?,?,?)');
|
|
for (const [key, row] of Object.entries(data.systems)) {
|
|
// ESI's system contract, including XYZ, is intentionally copied without adding Ref-only fields.
|
|
system.run(keyId(key, 'system'), row.constellation_id, row.name, row.position.x, row.position.y, row.position.z, row.security_status, row.security_class ?? null);
|
|
}
|
|
const gate = db.prepare('INSERT INTO gates(system_id,stargate_id,destination_system_id) VALUES (?,?,?)');
|
|
for (const row of Object.values(data.gates ?? {})) gate.run(row.system_id, row.stargate_id, row.destination_system_id);
|
|
const station = db.prepare('INSERT INTO stations(station_id,system_id,name,type_id,owner_id,x,y,z,source_verified) VALUES (?,?,?,?,?,?,?,?,?)');
|
|
const stationIds = new Set<number>();
|
|
for (const row of Object.values(data.stations ?? {})) {
|
|
const position = row.position;
|
|
station.run(row.station_id, row.system_id, row.name ?? null, row.type_id ?? null, row.owner_id ?? null, position?.x ?? null, position?.y ?? null, position?.z ?? null, 1);
|
|
stationIds.add(row.station_id);
|
|
}
|
|
// ESI system responses expose station IDs but not station metadata. Preserve
|
|
// those IDs as explicitly unknown rather than fabricating a name or type.
|
|
for (const row of Object.values(data.systems)) for (const stationId of row.stations ?? []) if (!stationIds.has(stationId)) station.run(stationId, row.system_id, null, null, null, null, null, null, 0);
|
|
}
|
|
|
|
export function insertHoboleaksData(db: Database, data: HoboleaksPayload): void {
|
|
// Industry activities are provenance/source facts. The normalized blueprint activity contract is sourced from Ref.
|
|
// Keep the raw Hoboleaks values in a compact metadata table rather than conflating the two schemas.
|
|
db.run('CREATE TABLE IF NOT EXISTS hoboleaks_industryactivities(activity_id INTEGER PRIMARY KEY, description TEXT, activity_name TEXT NOT NULL)');
|
|
db.run('CREATE TABLE IF NOT EXISTS hoboleaks_repackagedvolumes(type_id INTEGER PRIMARY KEY, volume REAL NOT NULL)');
|
|
const activity = db.prepare('INSERT INTO hoboleaks_industryactivities(activity_id,description,activity_name) VALUES (?,?,?)');
|
|
for (const [key, row] of Object.entries(data.industryactivities)) activity.run(keyId(key, 'industry activity'), row.description ?? null, row.activityName);
|
|
const volume = db.prepare('INSERT INTO hoboleaks_repackagedvolumes(type_id,volume) VALUES (?,?)');
|
|
for (const [key, value] of Object.entries(data.repackagedvolumes)) volume.run(keyId(key, 'repackaged volume'), value);
|
|
db.run('CREATE INDEX IF NOT EXISTS hoboleaks_repackagedvolumes_volume_idx ON hoboleaks_repackagedvolumes(volume)');
|
|
}
|
|
|
|
export function insertSourceMetadata(db: Database, metadata: SourceDatabaseMetadata[]): void {
|
|
const statement = db.prepare('INSERT INTO source_provenance(source,fingerprint,checked_at,local_sha256,source_revision,source_build,stale) VALUES (?,?,?,?,?,?,?)');
|
|
for (const item of metadata) statement.run(item.source, item.fingerprint, item.checkedAt, item.localSha256, item.sourceRevision?.toString() ?? null, item.sourceBuild ?? null, item.stale === undefined ? null : item.stale ? 1 : 0);
|
|
}
|