checkpoint ISAC registries and system monitoring
This commit is contained in:
JB
@@ -0,0 +1,17 @@
|
||||
<script lang="ts">
|
||||
import type { Snippet } from 'svelte';
|
||||
|
||||
let {
|
||||
id,
|
||||
label,
|
||||
children
|
||||
}: {
|
||||
id: string;
|
||||
label: string;
|
||||
children: Snippet;
|
||||
} = $props();
|
||||
</script>
|
||||
|
||||
<section class="registry-inline-editor" {id} aria-label={label}>
|
||||
{@render children()}
|
||||
</section>
|
||||
@@ -87,6 +87,6 @@
|
||||
{/each}
|
||||
<a class:disabled={pagination.page === pagination.pageCount} aria-disabled={pagination.page === pagination.pageCount} href={href(Math.min(pagination.pageCount, pagination.page + 1))}>NEXT →</a>
|
||||
</div>
|
||||
<div class="pagination-sizes"><span>ROWS</span>{#each [10, 25, 50, 100] as size}<a class:active={size === pagination.perPage} href={href(1, size)}>{size}</a>{/each}</div>
|
||||
{/if}
|
||||
<div class="pagination-sizes"><span>ROWS</span>{#each [10, 25, 50, 100] as size}<a class:active={size === pagination.perPage} href={href(1, size)}>{size}</a>{/each}</div>
|
||||
</nav>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<script lang="ts">
|
||||
type Item = { id: string; name: string; pending?: boolean; proposerName?: string };
|
||||
type Item = { id: string; name: string; unit?: string; pending?: boolean; proposerName?: string };
|
||||
let {
|
||||
items,
|
||||
selected = $bindable<string[]>([]),
|
||||
@@ -33,10 +33,12 @@
|
||||
const normalizedQuery = $derived(query.normalize('NFKC').trim().toLocaleLowerCase());
|
||||
const available = $derived(items.filter((item) =>
|
||||
!excludedIds.includes(item.id)
|
||||
&& (!normalizedQuery || item.name.toLocaleLowerCase().includes(normalizedQuery))
|
||||
&& (!normalizedQuery || `${item.name} ${item.unit ?? ''}`.toLocaleLowerCase().includes(normalizedQuery))
|
||||
));
|
||||
const visible = $derived(available.slice(0, 30));
|
||||
const selectedItems = $derived(selectedIds.map((id) => items.find((item) => item.id === id)).filter((item): item is Item => Boolean(item)));
|
||||
const unitLabel = (item: Item) => item.unit === undefined ? '' : item.unit || 'FLAT';
|
||||
const accessibleName = (item: Item) => unitLabel(item) ? `${item.name}, unit ${unitLabel(item)}` : item.name;
|
||||
const toggle = (id: string) => {
|
||||
if (mode === 'single') {
|
||||
value = value === id ? '' : id;
|
||||
@@ -57,7 +59,7 @@
|
||||
<div class="assignment-summary"><span>{mode === 'single' ? 'SELECTED' : 'ASSIGNED'}</span><b>{selectedIds.length}</b></div>
|
||||
<div class="assignment-chips">
|
||||
{#each selectedItems as item}
|
||||
<button type="button" class:pending-reference={item.pending} onclick={() => toggle(item.id)} aria-label={`Remove ${item.name}`}><span>{item.name}{#if item.pending}<small>PENDING</small>{/if}</span><b>×</b></button>
|
||||
<button type="button" class:pending-reference={item.pending} onclick={() => toggle(item.id)} aria-label={`Remove ${accessibleName(item)}`}><span>{item.name}{#if unitLabel(item)}<small class="assignment-item-unit">UNIT · {unitLabel(item)}</small>{/if}{#if item.pending}<small>PENDING</small>{/if}</span><b>×</b></button>
|
||||
{:else}
|
||||
<small>{emptySelectionLabel}</small>
|
||||
{/each}
|
||||
@@ -68,7 +70,7 @@
|
||||
<div class="assignment-results" role="list">
|
||||
{#each visible as item}
|
||||
<button type="button" class:selected={selectedIds.includes(item.id)} class:pending-reference={item.pending} aria-pressed={selectedIds.includes(item.id)} onclick={() => toggle(item.id)}>
|
||||
<span>{item.name}{#if item.pending}<small>PENDING · {item.proposerName ?? 'CONTRIBUTOR'}</small>{/if}</span><b>{selectedIds.includes(item.id) ? (mode === 'single' ? 'SELECTED' : 'ASSIGNED') : (mode === 'single' ? 'SELECT →' : 'ADD ＋')}</b>
|
||||
<span>{item.name}{#if unitLabel(item)}<small class="assignment-item-unit">UNIT · {unitLabel(item)}</small>{/if}{#if item.pending}<small>PENDING · {item.proposerName ?? 'CONTRIBUTOR'}</small>{/if}</span><b>{selectedIds.includes(item.id) ? (mode === 'single' ? 'SELECTED' : 'ASSIGNED') : (mode === 'single' ? 'SELECT →' : 'ADD ＋')}</b>
|
||||
</button>
|
||||
{:else}
|
||||
<p>{emptyLabel}</p>
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
<script lang="ts">
|
||||
import { page } from '$app/state';
|
||||
import type { RegistrySort, RegistrySortDirection } from '$lib/server/pagination';
|
||||
|
||||
let {
|
||||
label,
|
||||
column,
|
||||
sort,
|
||||
namespace = '',
|
||||
defaultDirection = 'asc',
|
||||
class: className = ''
|
||||
}: {
|
||||
label: string;
|
||||
column: string;
|
||||
sort: Pick<RegistrySort, 'key' | 'direction'>;
|
||||
namespace?: string;
|
||||
defaultDirection?: RegistrySortDirection;
|
||||
class?: string;
|
||||
} = $props();
|
||||
|
||||
const active = $derived(sort.key === column);
|
||||
const nextDirection = $derived<RegistrySortDirection>(
|
||||
active ? (sort.direction === 'asc' ? 'desc' : 'asc') : defaultDirection
|
||||
);
|
||||
const sortKey = $derived(namespace ? `${namespace}Sort` : 'sort');
|
||||
const directionKey = $derived(namespace ? `${namespace}Direction` : 'direction');
|
||||
const pageKey = $derived(namespace ? `${namespace}Page` : 'page');
|
||||
const href = $derived.by(() => {
|
||||
const params = new URLSearchParams(page.url.searchParams);
|
||||
params.set(sortKey, column);
|
||||
params.set(directionKey, nextDirection);
|
||||
params.delete(pageKey);
|
||||
const query = params.toString();
|
||||
return `${page.url.pathname}${query ? `?${query}` : ''}`;
|
||||
});
|
||||
</script>
|
||||
|
||||
<a
|
||||
class="sortable-column {className}"
|
||||
class:active
|
||||
{href}
|
||||
role="columnheader"
|
||||
aria-sort={active ? (sort.direction === 'asc' ? 'ascending' : 'descending') : 'none'}
|
||||
aria-label={`${label}: ${active ? `sorted ${sort.direction}. Activate to sort ${nextDirection}` : `activate to sort ${nextDirection}`}`}
|
||||
>
|
||||
<span class="sort-direction" aria-hidden="true">{active ? (sort.direction === 'asc' ? '↑' : '↓') : '↕'}</span>
|
||||
<span>{label}</span>
|
||||
</a>
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { getPagination, getRegistrySort } from './pagination.js';
|
||||
|
||||
const options = {
|
||||
name: { asc: 'name ASC', desc: 'name DESC' },
|
||||
updated: { asc: 'updated_at ASC', desc: 'updated_at DESC', defaultDirection: 'desc' as const }
|
||||
};
|
||||
|
||||
describe('getRegistrySort', () => {
|
||||
it('returns only allowlisted sort expressions', () => {
|
||||
const url = new URL('https://shd.build/isac/attributes?sort=name&direction=desc');
|
||||
expect(getRegistrySort(url, options, 'name')).toEqual({
|
||||
key: 'name',
|
||||
direction: 'desc',
|
||||
orderBy: 'name DESC'
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back for unknown keys and directions', () => {
|
||||
const url = new URL('https://shd.build/isac/attributes?sort=DROP%20TABLE&direction=sideways');
|
||||
expect(getRegistrySort(url, options, 'updated')).toEqual({
|
||||
key: 'updated',
|
||||
direction: 'desc',
|
||||
orderBy: 'updated_at DESC'
|
||||
});
|
||||
});
|
||||
|
||||
it('supports namespaced registry parameters', () => {
|
||||
const url = new URL('https://shd.build/isac/users/1?loginSort=name&loginDirection=asc');
|
||||
expect(getRegistrySort(url, options, 'updated', 'desc', 'login')).toMatchObject({
|
||||
key: 'name',
|
||||
direction: 'asc'
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('getPagination', () => {
|
||||
it('keeps supported row counts and clamps the selected page', () => {
|
||||
const url = new URL('https://shd.build/isac/attributes?perPage=50&page=9');
|
||||
expect(getPagination(url, 75)).toMatchObject({ perPage: 50, page: 2, pageCount: 2 });
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,39 @@ import type { URL } from 'node:url';
|
||||
|
||||
export const PAGE_SIZE_PRESETS = [10, 25, 50, 100] as const;
|
||||
|
||||
export type RegistrySortDirection = 'asc' | 'desc';
|
||||
export type RegistrySort = {
|
||||
key: string;
|
||||
direction: RegistrySortDirection;
|
||||
orderBy: string;
|
||||
};
|
||||
export type RegistrySortOptions = Record<
|
||||
string,
|
||||
{ asc: string; desc: string; defaultDirection?: RegistrySortDirection }
|
||||
>;
|
||||
|
||||
export function getRegistrySort(
|
||||
url: URL,
|
||||
options: RegistrySortOptions,
|
||||
defaultKey: string,
|
||||
defaultDirection: RegistrySortDirection = 'asc',
|
||||
namespace = ''
|
||||
): RegistrySort {
|
||||
const sortKey = namespace ? `${namespace}Sort` : 'sort';
|
||||
const directionKey = namespace ? `${namespace}Direction` : 'direction';
|
||||
const requestedKey = String(url.searchParams.get(sortKey) ?? '');
|
||||
const key = Object.hasOwn(options, requestedKey) ? requestedKey : defaultKey;
|
||||
const option = options[key];
|
||||
if (!option) throw new Error(`Unknown default registry sort: ${defaultKey}`);
|
||||
const requestedDirection = String(url.searchParams.get(directionKey) ?? '').toLowerCase();
|
||||
const fallbackDirection = option.defaultDirection ?? defaultDirection;
|
||||
const direction: RegistrySortDirection =
|
||||
requestedDirection === 'asc' || requestedDirection === 'desc'
|
||||
? requestedDirection
|
||||
: fallbackDirection;
|
||||
return { key, direction, orderBy: option[direction] };
|
||||
}
|
||||
|
||||
export function getRegistrySearch(url: URL, namespace = '') {
|
||||
const key = namespace ? `${namespace}Q` : 'q';
|
||||
const search = String(url.searchParams.get(key) ?? '').normalize('NFKC').trim().slice(0, 80);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import sharp from 'sharp';
|
||||
import { recordRemoveBgCharge } from '$lib/server/remove-bg-monitor';
|
||||
|
||||
const REMOVE_BG_ENDPOINT = 'https://api.remove.bg/v1.0/removebg';
|
||||
const REMOVE_BG_PREVIEW_EDGE = 1500;
|
||||
@@ -41,6 +42,12 @@ export async function removeBackgroundPreview(input: Buffer): Promise<Buffer<Arr
|
||||
throw new BackgroundRemovalError('remove.bg could not be reached. Try again or upload a transparent image.');
|
||||
}
|
||||
if (!response.ok) throw new BackgroundRemovalError(apiMessage(response.status));
|
||||
const charged = Number(response.headers.get('X-Credits-Charged'));
|
||||
if (Number.isFinite(charged) && charged > 0) {
|
||||
await recordRemoveBgCharge(charged).catch((error) => {
|
||||
console.error('Unable to update the cached remove.bg credit balance.', error);
|
||||
});
|
||||
}
|
||||
const output = Buffer.from(await response.arrayBuffer());
|
||||
const metadata = await sharp(output, { failOn: 'warning' }).metadata();
|
||||
if (!metadata.hasAlpha) throw new BackgroundRemovalError('remove.bg returned an image without transparency.');
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { parseRemoveBgAccount } from './remove-bg-monitor';
|
||||
|
||||
describe('parseRemoveBgAccount', () => {
|
||||
it('normalizes the official account response fields', () => {
|
||||
expect(parseRemoveBgAccount({
|
||||
data: {
|
||||
attributes: {
|
||||
credits: {
|
||||
total: 23.5,
|
||||
subscription: 20,
|
||||
payg: 3.5,
|
||||
enterprise: 0
|
||||
},
|
||||
api: {
|
||||
free_calls: 17,
|
||||
sizes: 'preview'
|
||||
}
|
||||
}
|
||||
}
|
||||
})).toEqual({
|
||||
total: 23.5,
|
||||
subscription: 20,
|
||||
payg: 3.5,
|
||||
enterprise: 0,
|
||||
freeCalls: 17
|
||||
});
|
||||
});
|
||||
|
||||
it('floors fractional free-call balances and accepts numeric API strings', () => {
|
||||
expect(parseRemoveBgAccount({
|
||||
data: {
|
||||
attributes: {
|
||||
credits: { total: '7.25', subscription: '5', payg: '2.25' },
|
||||
api: { free_calls: '4.9' }
|
||||
}
|
||||
}
|
||||
})).toEqual({
|
||||
total: 7.25,
|
||||
subscription: 5,
|
||||
payg: 2.25,
|
||||
enterprise: 0,
|
||||
freeCalls: 4
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
undefined,
|
||||
{},
|
||||
{ data: { attributes: {} } },
|
||||
{ data: { attributes: { credits: { total: 'unknown' }, api: { free_calls: 10 } } } },
|
||||
{ data: { attributes: { credits: { total: 10 }, api: { free_calls: 'unknown' } } } },
|
||||
{ data: { attributes: { credits: { total: -1 }, api: { free_calls: 10 } } } },
|
||||
{ data: { attributes: { credits: { total: 10 }, api: { free_calls: -1 } } } }
|
||||
])('rejects a malformed or negative required balance: %j', (payload) => {
|
||||
expect(() => parseRemoveBgAccount(payload)).toThrow('remove.bg returned an invalid account balance.');
|
||||
});
|
||||
|
||||
it('defaults malformed or negative optional credit buckets to zero', () => {
|
||||
expect(parseRemoveBgAccount({
|
||||
data: {
|
||||
attributes: {
|
||||
credits: {
|
||||
total: 12,
|
||||
subscription: -1,
|
||||
payg: 'not-a-number',
|
||||
enterprise: Number.POSITIVE_INFINITY
|
||||
},
|
||||
api: { free_calls: 8 }
|
||||
}
|
||||
}
|
||||
})).toEqual({
|
||||
total: 12,
|
||||
subscription: 0,
|
||||
payg: 0,
|
||||
enterprise: 0,
|
||||
freeCalls: 8
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,236 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { getDatabase } from '$lib/server/database';
|
||||
import { createNotification } from '$lib/server/notifications';
|
||||
|
||||
const PROVIDER = 'remove_bg';
|
||||
const ACCOUNT_ENDPOINT = 'https://api.remove.bg/v1.0/account';
|
||||
|
||||
export type RemoveBgCreditStatus = {
|
||||
configured: boolean;
|
||||
total: number | null;
|
||||
subscription: number | null;
|
||||
payg: number | null;
|
||||
enterprise: number | null;
|
||||
freeCalls: number | null;
|
||||
checkedAt: string | null;
|
||||
nextCheckAt: string | null;
|
||||
lastCharge: number | null;
|
||||
lastChargeAt: string | null;
|
||||
error: string | null;
|
||||
severity: 'unknown' | 'ok' | 'warning' | 'critical';
|
||||
warningCreditThreshold: number;
|
||||
warningFreeCallThreshold: number;
|
||||
};
|
||||
|
||||
const finiteNonNegative = (value: unknown) => {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
|
||||
};
|
||||
|
||||
export function parseRemoveBgAccount(payload: unknown) {
|
||||
const attributes = (payload as { data?: { attributes?: Record<string, unknown> } })?.data?.attributes;
|
||||
const credits = attributes?.credits as Record<string, unknown> | undefined;
|
||||
const api = attributes?.api as Record<string, unknown> | undefined;
|
||||
const total = finiteNonNegative(credits?.total);
|
||||
const freeCalls = finiteNonNegative(api?.free_calls);
|
||||
if (total === null || freeCalls === null) throw new Error('remove.bg returned an invalid account balance.');
|
||||
return {
|
||||
total,
|
||||
subscription: finiteNonNegative(credits?.subscription) ?? 0,
|
||||
payg: finiteNonNegative(credits?.payg) ?? 0,
|
||||
enterprise: finiteNonNegative(credits?.enterprise) ?? 0,
|
||||
freeCalls: Math.floor(freeCalls)
|
||||
};
|
||||
}
|
||||
|
||||
function settings() {
|
||||
const warningCreditThreshold = finiteNonNegative(env.REMOVE_BG_LOW_CREDIT_THRESHOLD) ?? 5;
|
||||
const warningFreeCallThreshold = finiteNonNegative(env.REMOVE_BG_LOW_FREE_CALL_THRESHOLD) ?? 10;
|
||||
const checkHours = Math.max(1, finiteNonNegative(env.REMOVE_BG_CREDIT_CHECK_HOURS) ?? 6);
|
||||
return { warningCreditThreshold, warningFreeCallThreshold, checkHours };
|
||||
}
|
||||
|
||||
function severity(total: number, freeCalls: number) {
|
||||
const { warningCreditThreshold, warningFreeCallThreshold } = settings();
|
||||
if (total <= 0 && freeCalls <= 0) return 'critical' as const;
|
||||
if (total <= warningCreditThreshold && freeCalls <= warningFreeCallThreshold) return 'warning' as const;
|
||||
return 'ok' as const;
|
||||
}
|
||||
|
||||
async function activeAdministrators() {
|
||||
const database = await getDatabase();
|
||||
return (await database.prepare(`
|
||||
SELECT DISTINCT users.id
|
||||
FROM users
|
||||
LEFT JOIN deleted_accounts deleted ON deleted.user_id=users.id
|
||||
LEFT JOIN user_status status ON status.user_id=users.id
|
||||
LEFT JOIN user_roles ON user_roles.user_id=users.id
|
||||
LEFT JOIN role_permissions ON role_permissions.role_id=user_roles.role_id
|
||||
LEFT JOIN permissions role_permission ON role_permission.id=role_permissions.permission_id
|
||||
LEFT JOIN user_permissions ON user_permissions.user_id=users.id
|
||||
LEFT JOIN permissions direct_permission ON direct_permission.id=user_permissions.permission_id
|
||||
WHERE deleted.user_id IS NULL AND status.blocked_at IS NULL
|
||||
AND (status.banned_until IS NULL OR status.banned_until<=?)
|
||||
AND (users.role='admin' OR user_roles.role_id='role-admin'
|
||||
OR role_permission.name='system.admin' OR direct_permission.name='system.admin')
|
||||
`)).all(new Date().toISOString());
|
||||
}
|
||||
|
||||
async function queueAlertEpisode(episodeId: string, alertSeverity: 'warning' | 'critical') {
|
||||
const database = await getDatabase();
|
||||
const now = new Date().toISOString();
|
||||
for (const administrator of await activeAdministrators()) {
|
||||
await (await database.prepare(`INSERT OR IGNORE INTO external_service_alert_deliveries
|
||||
(id,provider,episode_id,severity,user_id,created_at)
|
||||
VALUES(?,?,?,?,?,?)`)).run(
|
||||
randomUUID(), PROVIDER, episodeId, alertSeverity, administrator.id, now
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function persistBalance(balance: ReturnType<typeof parseRemoveBgAccount>) {
|
||||
const database = await getDatabase();
|
||||
const previous = await (await database.prepare(
|
||||
'SELECT severity,episode_id FROM external_service_status WHERE provider=?'
|
||||
)).get(PROVIDER);
|
||||
const now = new Date();
|
||||
const nextCheck = new Date(now.getTime() + settings().checkHours * 60 * 60_000);
|
||||
const nextSeverity = severity(balance.total, balance.freeCalls);
|
||||
const previousSeverity = String(previous?.severity ?? 'unknown');
|
||||
const recovered = nextSeverity === 'ok' && (previousSeverity === 'warning' || previousSeverity === 'critical');
|
||||
const startsEpisode = (nextSeverity === 'warning' || nextSeverity === 'critical') &&
|
||||
(previousSeverity === 'unknown' || previousSeverity === 'ok');
|
||||
const episodeId = startsEpisode ? randomUUID() : String(previous?.episode_id ?? '') || null;
|
||||
await (await database.prepare(`INSERT INTO external_service_status(
|
||||
provider,credits_total,credits_subscription,credits_payg,credits_enterprise,free_calls,
|
||||
checked_at,next_check_at,error_message,severity,episode_id,episode_started_at,recovered_at,updated_at
|
||||
) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(provider) DO UPDATE SET
|
||||
credits_total=excluded.credits_total,credits_subscription=excluded.credits_subscription,
|
||||
credits_payg=excluded.credits_payg,credits_enterprise=excluded.credits_enterprise,
|
||||
free_calls=excluded.free_calls,checked_at=excluded.checked_at,next_check_at=excluded.next_check_at,
|
||||
error_message=NULL,severity=excluded.severity,episode_id=excluded.episode_id,
|
||||
episode_started_at=CASE WHEN excluded.episode_id<>external_service_status.episode_id THEN excluded.episode_started_at ELSE external_service_status.episode_started_at END,
|
||||
recovered_at=excluded.recovered_at,updated_at=excluded.updated_at`)).run(
|
||||
PROVIDER, balance.total, balance.subscription, balance.payg, balance.enterprise, balance.freeCalls,
|
||||
now.toISOString(), nextCheck.toISOString(), null, nextSeverity,
|
||||
recovered ? null : episodeId, startsEpisode ? now.toISOString() : null,
|
||||
recovered ? now.toISOString() : null, now.toISOString()
|
||||
);
|
||||
if (episodeId && (startsEpisode || (previousSeverity === 'warning' && nextSeverity === 'critical'))) {
|
||||
await queueAlertEpisode(episodeId, nextSeverity);
|
||||
}
|
||||
}
|
||||
|
||||
export async function refreshRemoveBgCredits(force = false) {
|
||||
const database = await getDatabase();
|
||||
const existing = await (await database.prepare(
|
||||
'SELECT next_check_at FROM external_service_status WHERE provider=?'
|
||||
)).get(PROVIDER);
|
||||
if (!force && existing?.next_check_at && String(existing.next_check_at) > new Date().toISOString()) {
|
||||
return getRemoveBgCreditStatus();
|
||||
}
|
||||
const apiKey = env.REMOVE_BG_API_KEY?.trim();
|
||||
if (!apiKey) return getRemoveBgCreditStatus();
|
||||
try {
|
||||
const response = await fetch(ACCOUNT_ENDPOINT, {
|
||||
headers: { 'X-Api-Key': apiKey },
|
||||
signal: AbortSignal.timeout(15_000)
|
||||
});
|
||||
if (!response.ok) throw new Error(`remove.bg account check failed with status ${response.status}.`);
|
||||
await persistBalance(parseRemoveBgAccount(await response.json()));
|
||||
} catch (error) {
|
||||
const now = new Date();
|
||||
const retryAt = new Date(now.getTime() + 60 * 60_000);
|
||||
const message = error instanceof Error ? error.message : 'remove.bg account check failed.';
|
||||
await (await database.prepare(`INSERT INTO external_service_status(provider,next_check_at,error_message,updated_at)
|
||||
VALUES(?,?,?,?) ON CONFLICT(provider) DO UPDATE SET
|
||||
next_check_at=excluded.next_check_at,error_message=excluded.error_message,updated_at=excluded.updated_at`))
|
||||
.run(PROVIDER, retryAt.toISOString(), message.slice(0, 240), now.toISOString());
|
||||
}
|
||||
return getRemoveBgCreditStatus();
|
||||
}
|
||||
|
||||
export async function recordRemoveBgCharge(charged: number) {
|
||||
if (!Number.isFinite(charged) || charged <= 0) return;
|
||||
const database = await getDatabase();
|
||||
const row = await (await database.prepare(
|
||||
'SELECT credits_total,free_calls,severity,episode_id FROM external_service_status WHERE provider=?'
|
||||
)).get(PROVIDER);
|
||||
if (row?.credits_total === null || row?.credits_total === undefined) return;
|
||||
const now = new Date().toISOString();
|
||||
const total = Math.max(0, Number(row.credits_total) - charged);
|
||||
const nextSeverity = severity(total, Number(row.free_calls ?? 0));
|
||||
const previousSeverity = String(row.severity ?? 'unknown');
|
||||
const startsEpisode = (nextSeverity === 'warning' || nextSeverity === 'critical') &&
|
||||
(previousSeverity === 'unknown' || previousSeverity === 'ok');
|
||||
const episodeId = startsEpisode ? randomUUID() : String(row.episode_id ?? '') || null;
|
||||
await (await database.prepare(`UPDATE external_service_status SET credits_total=?,last_charge=?,last_charge_at=?,
|
||||
severity=?,episode_id=?,episode_started_at=CASE WHEN ? THEN ? ELSE episode_started_at END,updated_at=?
|
||||
WHERE provider=?`)).run(
|
||||
total, charged, now, nextSeverity, episodeId, startsEpisode ? 1 : 0, now, now, PROVIDER
|
||||
);
|
||||
if (episodeId && (startsEpisode || (previousSeverity === 'warning' && nextSeverity === 'critical'))) {
|
||||
await queueAlertEpisode(episodeId, nextSeverity);
|
||||
}
|
||||
}
|
||||
|
||||
export async function deliverRemoveBgCreditAlerts() {
|
||||
const database = await getDatabase();
|
||||
const pending = await (await database.prepare(`SELECT delivery.*,status.credits_total,status.free_calls
|
||||
FROM external_service_alert_deliveries delivery
|
||||
JOIN external_service_status status ON status.provider=delivery.provider
|
||||
WHERE delivery.delivered_at IS NULL ORDER BY delivery.created_at LIMIT 100`)).all();
|
||||
let delivered = 0;
|
||||
for (const alert of pending) {
|
||||
const title = alert.severity === 'critical'
|
||||
? 'remove.bg credits exhausted'
|
||||
: 'remove.bg credits running low';
|
||||
const body = `remove.bg has ${Number(alert.credits_total ?? 0).toLocaleString()} paid credits and ${Number(alert.free_calls ?? 0).toLocaleString()} free calls remaining. Review image-processing capacity in System Metrics.`;
|
||||
const notificationId = await createNotification({
|
||||
userId: String(alert.user_id),
|
||||
category: 'isac',
|
||||
source: 'ISAC SYSTEM OVERSIGHT',
|
||||
title,
|
||||
body,
|
||||
link: '/isac/metrics',
|
||||
actions: [{ label: 'VIEW SYSTEM METRICS', href: '/isac/metrics' }],
|
||||
mandatory: true
|
||||
});
|
||||
await (await database.prepare(`UPDATE external_service_alert_deliveries
|
||||
SET notification_id=?,delivered_at=? WHERE id=? AND delivered_at IS NULL`))
|
||||
.run(notificationId, new Date().toISOString(), alert.id);
|
||||
delivered++;
|
||||
}
|
||||
return delivered;
|
||||
}
|
||||
|
||||
export async function getRemoveBgCreditStatus(): Promise<RemoveBgCreditStatus> {
|
||||
const database = await getDatabase();
|
||||
const row = await (await database.prepare('SELECT * FROM external_service_status WHERE provider=?')).get(PROVIDER);
|
||||
const { warningCreditThreshold, warningFreeCallThreshold } = settings();
|
||||
return {
|
||||
configured: Boolean(env.REMOVE_BG_API_KEY?.trim()),
|
||||
total: finiteNonNegative(row?.credits_total),
|
||||
subscription: finiteNonNegative(row?.credits_subscription),
|
||||
payg: finiteNonNegative(row?.credits_payg),
|
||||
enterprise: finiteNonNegative(row?.credits_enterprise),
|
||||
freeCalls: finiteNonNegative(row?.free_calls),
|
||||
checkedAt: row?.checked_at ? String(row.checked_at) : null,
|
||||
nextCheckAt: row?.next_check_at ? String(row.next_check_at) : null,
|
||||
lastCharge: finiteNonNegative(row?.last_charge),
|
||||
lastChargeAt: row?.last_charge_at ? String(row.last_charge_at) : null,
|
||||
error: row?.error_message ? String(row.error_message) : null,
|
||||
severity: (['ok', 'warning', 'critical'].includes(String(row?.severity))
|
||||
? String(row?.severity)
|
||||
: 'unknown') as RemoveBgCreditStatus['severity'],
|
||||
warningCreditThreshold,
|
||||
warningFreeCallThreshold
|
||||
};
|
||||
}
|
||||
|
||||
export async function runRemoveBgCreditMonitor() {
|
||||
await refreshRemoveBgCredits(false);
|
||||
return deliverRemoveBgCreditAlerts();
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { getDatabase } from '$lib/server/database';
|
||||
import nodemailer from 'nodemailer';
|
||||
import { runRemoveBgCreditMonitor } from '$lib/server/remove-bg-monitor';
|
||||
|
||||
function mailTransport() {
|
||||
if (!env.SMTP_HOST || !env.SMTP_FROM) return null;
|
||||
@@ -159,14 +160,15 @@ async function deliverDailyDigests() {
|
||||
return sent;
|
||||
}
|
||||
|
||||
let running: Promise<{ system: number; oversight: number; digests: number }> | null = null;
|
||||
let running: Promise<{ system: number; oversight: number; digests: number; serviceAlerts: number }> | null = null;
|
||||
export function runScheduledNotificationTasks() {
|
||||
if (running) return running;
|
||||
running = (async () => {
|
||||
const system = await deliverSystemMessages();
|
||||
const oversight = await deliverOversightMessages();
|
||||
const digests = await deliverDailyDigests();
|
||||
return { system, oversight, digests };
|
||||
const serviceAlerts = await runRemoveBgCreditMonitor();
|
||||
return { system, oversight, digests, serviceAlerts };
|
||||
})()
|
||||
.finally(() => { running = null; });
|
||||
return running;
|
||||
|
||||
Reference in New Issue
Block a user