feat: add automated community moderation
This commit is contained in:
JB
@@ -0,0 +1,63 @@
|
||||
<script lang="ts">
|
||||
import PendingForm from '$lib/components/PendingForm.svelte';
|
||||
|
||||
let {
|
||||
contentType,
|
||||
contentId,
|
||||
label = 'REPORT',
|
||||
result
|
||||
}: {
|
||||
contentType: 'comment' | 'profile_bio' | 'guide' | 'avatar';
|
||||
contentId: string;
|
||||
label?: string;
|
||||
result?: Record<string, unknown> | null;
|
||||
} = $props();
|
||||
|
||||
const categories = [
|
||||
{ value: 'spam', label: 'SPAM OR PROMOTION' },
|
||||
{ value: 'harassment', label: 'HARASSMENT OR BULLYING' },
|
||||
{ value: 'hate', label: 'HATEFUL CONTENT' },
|
||||
{ value: 'sexual', label: 'SEXUAL CONTENT' },
|
||||
{ value: 'violence', label: 'VIOLENCE OR THREATS' },
|
||||
{ value: 'self_harm', label: 'SELF-HARM' },
|
||||
{ value: 'privacy', label: 'PRIVATE INFORMATION' },
|
||||
{ value: 'other', label: 'OTHER POLICY CONCERN' }
|
||||
];
|
||||
const matchesResult = $derived(
|
||||
String(result?.userReportContentId ?? '') === contentId
|
||||
&& String(result?.userReportContentType ?? '') === contentType
|
||||
);
|
||||
</script>
|
||||
|
||||
<details class="content-report">
|
||||
<summary>{label}</summary>
|
||||
<div class="content-report-panel">
|
||||
<strong>REQUEST OVERSIGHT REVIEW</strong>
|
||||
<p>Send this content to Community Oversight. A report does not remove it automatically.</p>
|
||||
<PendingForm action="?/reportContent" toast={false}>
|
||||
{#snippet children(pending)}
|
||||
<input type="hidden" name="contentType" value={contentType}/>
|
||||
<input type="hidden" name="contentId" value={contentId}/>
|
||||
<label class="field">CONCERN
|
||||
<select name="category" required>
|
||||
{#each categories as category}<option value={category.value}>{category.label}</option>{/each}
|
||||
</select>
|
||||
</label>
|
||||
<label class="field">DETAILS — OPTIONAL
|
||||
<textarea name="details" maxlength="500" rows="3" placeholder="Give Oversight useful context without repeating sensitive information."></textarea>
|
||||
</label>
|
||||
{#if matchesResult && result?.userReportError}<p class="error">{String(result.userReportError)}</p>{/if}
|
||||
{#if matchesResult && result?.userReportSaved}<p class="success">Report sent to Community Oversight.</p>{/if}
|
||||
<div class="content-report-actions">
|
||||
<button class="primary" disabled={pending} aria-busy={pending}>{pending ? 'SENDING…' : 'SEND REPORT →'}</button>
|
||||
</div>
|
||||
{/snippet}
|
||||
</PendingForm>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<style>
|
||||
.content-report{position:relative}.content-report summary{list-style:none;min-height:32px;display:inline-flex;align-items:center;padding:0 7px;border:1px solid var(--shd-border);color:var(--shd-text-muted);font-size:var(--shd-type-telemetry);font-weight:900;letter-spacing:.7px;cursor:pointer}.content-report summary::-webkit-details-marker{display:none}.content-report summary:hover,.content-report summary:focus-visible{border-color:var(--shd-danger);color:var(--shd-danger)}.content-report[open] summary{border-color:var(--shd-orange);color:var(--shd-orange)}.content-report-panel{position:absolute;z-index:8;right:0;width:min(380px,calc(100vw - 32px));display:grid;gap:8px;padding:10px;border:1px solid var(--shd-border-strong);background:var(--shd-surface);box-shadow:0 18px 46px #000a}.content-report-panel>strong{color:var(--shd-orange);font-size:var(--shd-type-telemetry);letter-spacing:.8px}.content-report-panel>p{margin:0;color:var(--shd-text-muted);font-size:var(--shd-type-supporting)}.content-report-panel :global(form fieldset){display:grid;gap:8px}.content-report-panel :global(.field){margin:0}.content-report-panel textarea{min-height:80px}.content-report-actions{display:flex;justify-content:flex-end}.content-report-actions button{min-height:40px}
|
||||
@media(max-width:650px){.content-report-panel{position:fixed;right:16px;bottom:16px;left:16px;width:auto}.content-report summary{min-height:44px}.content-report-actions button{width:100%;min-height:44px}}
|
||||
@media(prefers-reduced-motion:reduce){.content-report summary{transition:none}}
|
||||
</style>
|
||||
@@ -0,0 +1,17 @@
|
||||
<aside class="moderation-disclosure" aria-label="Automated safety review">
|
||||
<span aria-hidden="true">i</span>
|
||||
<p>
|
||||
This submission may receive automated safety review.
|
||||
<a href="/terms#privacy">TERMS & PRIVACY →</a>
|
||||
</p>
|
||||
</aside>
|
||||
|
||||
<style>
|
||||
.moderation-disclosure{display:flex;align-items:flex-start;gap:8px;margin:8px 0;padding:8px 9px;border:1px solid var(--shd-border-soft);background:var(--shd-surface);color:var(--shd-text-muted)}
|
||||
.moderation-disclosure>span{flex:0 0 20px;display:grid;place-items:center;width:20px;height:20px;border:1px solid var(--shd-border-strong);color:var(--shd-orange);font-size:var(--shd-type-telemetry);font-weight:900}
|
||||
.moderation-disclosure p{margin:0;font-size:var(--shd-type-telemetry);line-height:1.5}
|
||||
.moderation-disclosure a{display:inline-block;margin-left:4px;color:var(--shd-orange);font-weight:900;letter-spacing:.4px}
|
||||
.moderation-disclosure a:hover{text-decoration:underline}
|
||||
.moderation-disclosure a:focus-visible{outline:2px solid var(--shd-orange);outline-offset:3px}
|
||||
@media(max-width:650px){.moderation-disclosure{padding:10px}.moderation-disclosure a{display:block;min-height:44px;margin:4px 0 0;padding-top:8px}}
|
||||
</style>
|
||||
@@ -49,7 +49,12 @@ export async function processAvatarUpload(file: File): Promise<Buffer> {
|
||||
}
|
||||
}
|
||||
|
||||
export async function saveUploadedAvatar(database: Database, userId: string, output: Buffer): Promise<string> {
|
||||
export async function saveUploadedAvatar(
|
||||
database: Database,
|
||||
userId: string,
|
||||
output: Buffer,
|
||||
options: { queueReview: boolean } = { queueReview: true }
|
||||
): Promise<{ uploadedAt: string; reviewId: string | null }> {
|
||||
const key = storageKey(userId);
|
||||
const directory = avatarStorageDirectory();
|
||||
await mkdir(directory, { recursive: true });
|
||||
@@ -61,18 +66,55 @@ export async function saveUploadedAvatar(database: Database, userId: string, out
|
||||
.run(userId, 'upload', key, output.length, now);
|
||||
await (await database.prepare("UPDATE avatar_reviews SET status='superseded',reviewed_at=? WHERE user_id=? AND status='pending'"))
|
||||
.run(now, userId);
|
||||
let reviewId: string | null = null;
|
||||
if (options.queueReview) {
|
||||
reviewId = await queueCurrentAvatarReview(
|
||||
database,
|
||||
userId,
|
||||
{
|
||||
origin: 'automated',
|
||||
summary: 'Automated safety checks flagged a custom avatar for Oversight review.'
|
||||
}
|
||||
);
|
||||
}
|
||||
return { uploadedAt: now, reviewId };
|
||||
}
|
||||
|
||||
export async function queueCurrentAvatarReview(
|
||||
database: Database,
|
||||
userId: string,
|
||||
options: {
|
||||
origin: 'automated' | 'user_report' | 'legacy';
|
||||
summary?: string;
|
||||
} = { origin: 'user_report' }
|
||||
): Promise<string | null> {
|
||||
const avatar = await (await database.prepare(`SELECT source,value,byte_size,updated_at
|
||||
FROM user_avatars WHERE user_id=?`)).get(userId);
|
||||
if (avatar?.source !== 'upload' || typeof avatar.value !== 'string') return null;
|
||||
const existing = await (await database.prepare(
|
||||
"SELECT id FROM avatar_reviews WHERE user_id=? AND avatar_updated_at=? AND status='pending'"
|
||||
)).get(userId, avatar.updated_at);
|
||||
if (existing?.id) return String(existing.id);
|
||||
const reviewId = randomUUID();
|
||||
await (await database.prepare(`INSERT INTO avatar_reviews
|
||||
(id,user_id,avatar_updated_at,storage_key,byte_size,status,submitted_at)
|
||||
VALUES(?,?,?,?,?,'pending',?)`)).run(reviewId, userId, now, key, output.length, now);
|
||||
(id,user_id,avatar_updated_at,storage_key,byte_size,status,review_origin,submitted_at)
|
||||
VALUES(?,?,?,?,?,'pending',?,?)`)).run(
|
||||
reviewId,
|
||||
userId,
|
||||
avatar.updated_at,
|
||||
avatar.value,
|
||||
avatar.byte_size,
|
||||
options.origin,
|
||||
new Date().toISOString()
|
||||
);
|
||||
await notifyOversightReview({
|
||||
category: 'avatars',
|
||||
itemId: reviewId,
|
||||
title: 'Avatar awaiting review',
|
||||
summary: 'A new custom agent avatar was uploaded and is ready for Oversight review.',
|
||||
summary: options.summary ?? 'An Agent report requested Oversight review of a custom avatar.',
|
||||
link: '/isac/avatar-oversight?status=pending'
|
||||
}).catch((error) => console.error('Unable to notify Avatar Oversight reviewers', error));
|
||||
return now;
|
||||
return reviewId;
|
||||
}
|
||||
|
||||
export async function selectPresetAvatar(database: Database, userId: string, preset: string): Promise<void> {
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
assessCommunityModeration,
|
||||
findLocalModerationMatches,
|
||||
normalizeModerationTerm,
|
||||
type CommunityModerationPolicy
|
||||
} from './community-moderation';
|
||||
|
||||
const policy = (overrides: Partial<CommunityModerationPolicy> = {}): CommunityModerationPolicy => ({
|
||||
automatedEnabled: true,
|
||||
autoPublishClean: true,
|
||||
holdOnProviderError: true,
|
||||
scanComments: true,
|
||||
scanProfiles: true,
|
||||
scanGuides: true,
|
||||
scanAvatars: true,
|
||||
providerModel: 'omni-moderation-latest',
|
||||
detectPii: true,
|
||||
maxLinks: 1,
|
||||
duplicateWindowSeconds: 300,
|
||||
policyVersion: 1,
|
||||
thresholds: [{
|
||||
category: 'harassment',
|
||||
enabled: true,
|
||||
reviewThreshold: 0.5,
|
||||
priorityThreshold: 0.85
|
||||
}],
|
||||
terms: [],
|
||||
...overrides
|
||||
});
|
||||
|
||||
describe('community moderation policy', () => {
|
||||
it('normalizes configured terms consistently', () => {
|
||||
expect(normalizeModerationTerm(' Rogue\u00a0Agent ')).toBe('rogue agent');
|
||||
});
|
||||
|
||||
it('allows an exact local exception to suppress a matching banned term', () => {
|
||||
const matches = findLocalModerationMatches('Rogue Agent tactics', 'comment', policy({
|
||||
detectPii: false,
|
||||
terms: [
|
||||
{ id: 'ban', listType: 'ban', matchMode: 'phrase', value: 'Rogue Agent', normalizedValue: 'rogue agent', scopes: ['comment'], enabled: true },
|
||||
{ id: 'allow', listType: 'allow', matchMode: 'phrase', value: 'Rogue Agent', normalizedValue: 'rogue agent', scopes: ['comment'], enabled: true }
|
||||
]
|
||||
}));
|
||||
expect(matches).toEqual([]);
|
||||
});
|
||||
|
||||
it('finds scoped banned terms, PII, and excessive links', () => {
|
||||
const matches = findLocalModerationMatches(
|
||||
'Contact test@example.com about forbidden plans at https://one.test and https://two.test',
|
||||
'comment',
|
||||
policy({
|
||||
terms: [{
|
||||
id: 'ban',
|
||||
listType: 'ban',
|
||||
matchMode: 'word',
|
||||
value: 'forbidden',
|
||||
normalizedValue: 'forbidden',
|
||||
scopes: ['comment'],
|
||||
enabled: true
|
||||
}]
|
||||
})
|
||||
);
|
||||
expect(matches.map((match) => match.kind)).toEqual(['banned_term', 'pii', 'link_limit']);
|
||||
});
|
||||
|
||||
it('routes priority scores ahead of ordinary review scores', () => {
|
||||
expect(assessCommunityModeration({
|
||||
policy: policy(),
|
||||
contentType: 'comment',
|
||||
providerFlagged: true,
|
||||
categoryScores: { harassment: 0.9 },
|
||||
ruleMatches: []
|
||||
}).decision).toBe('priority_review');
|
||||
});
|
||||
|
||||
it('holds provider failures under the default fail-closed policy', () => {
|
||||
expect(assessCommunityModeration({
|
||||
policy: policy(),
|
||||
contentType: 'profile_bio',
|
||||
providerFlagged: false,
|
||||
categoryScores: {},
|
||||
ruleMatches: [],
|
||||
providerError: 'timeout'
|
||||
}).decision).toBe('error');
|
||||
});
|
||||
|
||||
it('publishes clean content only when auto publication is enabled', () => {
|
||||
expect(assessCommunityModeration({
|
||||
policy: policy(),
|
||||
contentType: 'comment',
|
||||
providerFlagged: false,
|
||||
categoryScores: { harassment: 0.01 },
|
||||
ruleMatches: []
|
||||
}).decision).toBe('approved');
|
||||
expect(assessCommunityModeration({
|
||||
policy: policy({ autoPublishClean: false }),
|
||||
contentType: 'comment',
|
||||
providerFlagged: false,
|
||||
categoryScores: { harassment: 0.01 },
|
||||
ruleMatches: []
|
||||
}).decision).toBe('review');
|
||||
});
|
||||
|
||||
it('publishes locally clean content when provider scanning is disabled', () => {
|
||||
expect(assessCommunityModeration({
|
||||
policy: policy({ automatedEnabled: false }),
|
||||
contentType: 'comment',
|
||||
providerFlagged: false,
|
||||
categoryScores: {},
|
||||
ruleMatches: []
|
||||
}).decision).toBe('approved');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,611 @@
|
||||
import { env } from '$env/dynamic/private';
|
||||
import { getDatabase } from '$lib/server/database';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import OpenAI from 'openai';
|
||||
|
||||
export const MODERATION_CATEGORIES = [
|
||||
'harassment',
|
||||
'harassment/threatening',
|
||||
'hate',
|
||||
'hate/threatening',
|
||||
'illicit',
|
||||
'illicit/violent',
|
||||
'self-harm',
|
||||
'self-harm/intent',
|
||||
'self-harm/instructions',
|
||||
'sexual',
|
||||
'sexual/minors',
|
||||
'violence',
|
||||
'violence/graphic'
|
||||
] as const;
|
||||
|
||||
export type ModerationCategory = (typeof MODERATION_CATEGORIES)[number];
|
||||
export type ModeratedContentType = 'comment' | 'profile_bio' | 'guide' | 'avatar';
|
||||
export type ModerationDecision = 'approved' | 'review' | 'priority_review' | 'error';
|
||||
|
||||
export type ModerationThreshold = {
|
||||
category: ModerationCategory;
|
||||
enabled: boolean;
|
||||
reviewThreshold: number;
|
||||
priorityThreshold: number;
|
||||
};
|
||||
|
||||
export type ModerationTerm = {
|
||||
id: string;
|
||||
listType: 'allow' | 'ban';
|
||||
matchMode: 'word' | 'phrase';
|
||||
value: string;
|
||||
normalizedValue: string;
|
||||
scopes: ModeratedContentType[];
|
||||
enabled: boolean;
|
||||
};
|
||||
|
||||
export type CommunityModerationPolicy = {
|
||||
automatedEnabled: boolean;
|
||||
autoPublishClean: boolean;
|
||||
holdOnProviderError: boolean;
|
||||
scanComments: boolean;
|
||||
scanProfiles: boolean;
|
||||
scanGuides: boolean;
|
||||
scanAvatars: boolean;
|
||||
providerModel: 'omni-moderation-latest' | 'omni-moderation-2024-09-26';
|
||||
detectPii: boolean;
|
||||
maxLinks: number;
|
||||
duplicateWindowSeconds: number;
|
||||
policyVersion: number;
|
||||
thresholds: ModerationThreshold[];
|
||||
terms: ModerationTerm[];
|
||||
};
|
||||
|
||||
export type LocalModerationMatch = {
|
||||
kind: 'banned_term' | 'pii' | 'link_limit';
|
||||
label: string;
|
||||
excerpt: string;
|
||||
termId?: string;
|
||||
};
|
||||
|
||||
export type CommunityModerationResult = {
|
||||
contentHash: string;
|
||||
decision: ModerationDecision;
|
||||
riskLevel: 'low' | 'medium' | 'high';
|
||||
reason: string;
|
||||
policyVersion: number;
|
||||
provider: 'openai' | 'local';
|
||||
model: string;
|
||||
requestId: string | null;
|
||||
categoryScores: Partial<Record<ModerationCategory, number>>;
|
||||
categoryFlags: Partial<Record<ModerationCategory, boolean>>;
|
||||
ruleMatches: LocalModerationMatch[];
|
||||
error: string | null;
|
||||
};
|
||||
|
||||
const DEFAULT_POLICY: CommunityModerationPolicy = {
|
||||
automatedEnabled: false,
|
||||
autoPublishClean: true,
|
||||
holdOnProviderError: true,
|
||||
scanComments: true,
|
||||
scanProfiles: true,
|
||||
scanGuides: true,
|
||||
scanAvatars: true,
|
||||
providerModel: 'omni-moderation-latest',
|
||||
detectPii: true,
|
||||
maxLinks: 3,
|
||||
duplicateWindowSeconds: 300,
|
||||
policyVersion: 1,
|
||||
thresholds: [],
|
||||
terms: []
|
||||
};
|
||||
|
||||
let cachedPolicy: { value: CommunityModerationPolicy; expiresAt: number } | null = null;
|
||||
let client: OpenAI | null = null;
|
||||
|
||||
export function invalidateCommunityModerationPolicy() {
|
||||
cachedPolicy = null;
|
||||
}
|
||||
|
||||
export function normalizeModerationTerm(value: string) {
|
||||
return value.normalize('NFKC').trim().replace(/\s+/g, ' ').toLocaleLowerCase();
|
||||
}
|
||||
|
||||
export function hashModeratedContent(value: string) {
|
||||
return createHash('sha256').update(value.normalize('NFC')).digest('hex');
|
||||
}
|
||||
|
||||
function parseScopes(value: unknown): ModeratedContentType[] {
|
||||
try {
|
||||
const parsed = typeof value === 'string' ? JSON.parse(value) : value;
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed.filter((scope): scope is ModeratedContentType =>
|
||||
scope === 'comment' || scope === 'profile_bio' || scope === 'guide' || scope === 'avatar'
|
||||
);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function boundedNumber(value: unknown, fallback: number, min: number, max: number) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) ? Math.min(max, Math.max(min, number)) : fallback;
|
||||
}
|
||||
|
||||
export async function getCommunityModerationPolicy(): Promise<CommunityModerationPolicy> {
|
||||
if (cachedPolicy && cachedPolicy.expiresAt > Date.now()) return cachedPolicy.value;
|
||||
const database = await getDatabase();
|
||||
const [settings, thresholdRows, termRows] = await Promise.all([
|
||||
(await database.prepare("SELECT * FROM community_moderation_settings WHERE id='primary'")).get(),
|
||||
(await database.prepare('SELECT * FROM community_moderation_thresholds ORDER BY category')).all(),
|
||||
(await database.prepare('SELECT * FROM community_moderation_terms WHERE enabled=1 ORDER BY list_kind,value')).all()
|
||||
]);
|
||||
const model = String(settings?.provider_model ?? DEFAULT_POLICY.providerModel);
|
||||
const value: CommunityModerationPolicy = {
|
||||
automatedEnabled: Boolean(settings?.automated_enabled),
|
||||
autoPublishClean: settings?.auto_publish_clean === undefined
|
||||
? DEFAULT_POLICY.autoPublishClean
|
||||
: Boolean(settings.auto_publish_clean),
|
||||
holdOnProviderError: settings?.hold_on_provider_error === undefined
|
||||
? DEFAULT_POLICY.holdOnProviderError
|
||||
: Boolean(settings.hold_on_provider_error),
|
||||
scanComments: settings?.scan_comments === undefined ? true : Boolean(settings.scan_comments),
|
||||
scanProfiles: settings?.scan_profiles === undefined ? true : Boolean(settings.scan_profiles),
|
||||
scanGuides: settings?.scan_guides === undefined ? true : Boolean(settings.scan_guides),
|
||||
scanAvatars: settings?.scan_avatars === undefined ? true : Boolean(settings.scan_avatars),
|
||||
providerModel: model === 'omni-moderation-2024-09-26'
|
||||
? model
|
||||
: 'omni-moderation-latest',
|
||||
detectPii: settings?.detect_pii === undefined ? true : Boolean(settings.detect_pii),
|
||||
maxLinks: boundedNumber(settings?.max_links, DEFAULT_POLICY.maxLinks, 0, 20),
|
||||
duplicateWindowSeconds: boundedNumber(
|
||||
settings?.duplicate_window_seconds,
|
||||
DEFAULT_POLICY.duplicateWindowSeconds,
|
||||
0,
|
||||
3_600
|
||||
),
|
||||
policyVersion: Math.max(1, Number(settings?.policy_version ?? 1)),
|
||||
thresholds: thresholdRows.flatMap((row) => {
|
||||
const category = String(row.category);
|
||||
if (!MODERATION_CATEGORIES.includes(category as ModerationCategory)) return [];
|
||||
return [{
|
||||
category: category as ModerationCategory,
|
||||
enabled: Boolean(row.enabled),
|
||||
reviewThreshold: boundedNumber(row.review_threshold, 0.5, 0, 1),
|
||||
priorityThreshold: boundedNumber(row.priority_threshold, 0.85, 0, 1)
|
||||
}];
|
||||
}),
|
||||
terms: termRows.flatMap((row) => {
|
||||
const listType = String(row.list_kind);
|
||||
const matchMode = String(row.match_mode);
|
||||
if (!['allow', 'ban'].includes(listType) || !['word', 'phrase'].includes(matchMode)) return [];
|
||||
return [{
|
||||
id: String(row.id),
|
||||
listType: listType as 'allow' | 'ban',
|
||||
matchMode: matchMode as 'word' | 'phrase',
|
||||
value: String(row.value),
|
||||
normalizedValue: String(row.normalized_value),
|
||||
scopes: parseScopes(row.scopes_json),
|
||||
enabled: Boolean(row.enabled)
|
||||
}];
|
||||
})
|
||||
};
|
||||
cachedPolicy = { value, expiresAt: Date.now() + 30_000 };
|
||||
return value;
|
||||
}
|
||||
|
||||
function scopeEnabled(policy: CommunityModerationPolicy, contentType: ModeratedContentType) {
|
||||
if (contentType === 'comment') return policy.scanComments;
|
||||
if (contentType === 'profile_bio') return policy.scanProfiles;
|
||||
if (contentType === 'guide') return policy.scanGuides;
|
||||
return policy.scanAvatars;
|
||||
}
|
||||
|
||||
function wordPattern(value: string) {
|
||||
const escaped = value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
return new RegExp(`(^|[^\\p{L}\\p{N}_])${escaped}(?=$|[^\\p{L}\\p{N}_])`, 'iu');
|
||||
}
|
||||
|
||||
function matchesTerm(text: string, term: ModerationTerm) {
|
||||
return term.matchMode === 'word'
|
||||
? wordPattern(term.normalizedValue).test(text)
|
||||
: text.includes(term.normalizedValue);
|
||||
}
|
||||
|
||||
function excerpt(value: string, limit = 80) {
|
||||
const compact = value.replace(/\s+/g, ' ').trim();
|
||||
return compact.length > limit ? `${compact.slice(0, limit - 1)}…` : compact;
|
||||
}
|
||||
|
||||
const piiPatterns = [
|
||||
{ label: 'EMAIL ADDRESS', pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/iu },
|
||||
{ label: 'PHONE NUMBER', pattern: /(?:\+?\d[\d\s().-]{7,}\d)/u },
|
||||
{ label: 'IP ADDRESS', pattern: /\b(?:\d{1,3}\.){3}\d{1,3}\b/u }
|
||||
] as const;
|
||||
|
||||
const linkPattern = /\bhttps?:\/\/[^\s<>()]+/giu;
|
||||
|
||||
export function findLocalModerationMatches(
|
||||
text: string,
|
||||
contentType: ModeratedContentType,
|
||||
policy: Pick<CommunityModerationPolicy, 'terms' | 'detectPii' | 'maxLinks'>
|
||||
): LocalModerationMatch[] {
|
||||
const normalized = normalizeModerationTerm(text);
|
||||
const scoped = policy.terms.filter((term) =>
|
||||
term.enabled && (term.scopes.length === 0 || term.scopes.includes(contentType))
|
||||
);
|
||||
const allowKeys = new Set(scoped.filter((term) => term.listType === 'allow' && matchesTerm(normalized, term))
|
||||
.map((term) => `${term.matchMode}:${term.normalizedValue}`));
|
||||
const matches: LocalModerationMatch[] = scoped
|
||||
.filter((term) =>
|
||||
term.listType === 'ban'
|
||||
&& !allowKeys.has(`${term.matchMode}:${term.normalizedValue}`)
|
||||
&& matchesTerm(normalized, term)
|
||||
)
|
||||
.map((term) => ({
|
||||
kind: 'banned_term',
|
||||
label: `BANNED ${term.matchMode.toLocaleUpperCase()}`,
|
||||
excerpt: excerpt(term.value),
|
||||
termId: term.id
|
||||
}));
|
||||
if (policy.detectPii) {
|
||||
for (const definition of piiPatterns) {
|
||||
const match = text.match(definition.pattern);
|
||||
if (match?.[0]) matches.push({
|
||||
kind: 'pii',
|
||||
label: definition.label,
|
||||
excerpt: excerpt(match[0])
|
||||
});
|
||||
}
|
||||
}
|
||||
const links = [...text.matchAll(linkPattern)];
|
||||
if (links.length > policy.maxLinks) matches.push({
|
||||
kind: 'link_limit',
|
||||
label: 'LINK LIMIT',
|
||||
excerpt: `${links.length} links submitted; policy allows ${policy.maxLinks}`
|
||||
});
|
||||
return matches;
|
||||
}
|
||||
|
||||
export function assessCommunityModeration(input: {
|
||||
policy: CommunityModerationPolicy;
|
||||
contentType: ModeratedContentType;
|
||||
providerFlagged: boolean;
|
||||
categoryScores: Partial<Record<ModerationCategory, number>>;
|
||||
categoryFlags?: Partial<Record<ModerationCategory, boolean>>;
|
||||
ruleMatches: LocalModerationMatch[];
|
||||
providerError?: string | null;
|
||||
}): Pick<CommunityModerationResult, 'decision' | 'riskLevel' | 'reason'> {
|
||||
if (!input.policy.automatedEnabled || !scopeEnabled(input.policy, input.contentType)) {
|
||||
if (input.ruleMatches.some((match) => match.kind === 'banned_term')) {
|
||||
return { decision: 'priority_review', riskLevel: 'high', reason: 'A configured banned term requires priority review.' };
|
||||
}
|
||||
if (input.ruleMatches.length) {
|
||||
return { decision: 'review', riskLevel: 'medium', reason: 'A deterministic safety rule requires review.' };
|
||||
}
|
||||
return { decision: 'approved', riskLevel: 'low', reason: 'Automated provider scanning is disabled and no local rule was triggered.' };
|
||||
}
|
||||
if (input.providerError) {
|
||||
return input.policy.holdOnProviderError
|
||||
? { decision: 'error', riskLevel: 'medium', reason: 'The moderation provider could not classify this content.' }
|
||||
: { decision: 'approved', riskLevel: 'low', reason: 'Provider errors are configured to allow publication.' };
|
||||
}
|
||||
if (input.ruleMatches.some((match) => match.kind === 'banned_term')) {
|
||||
return { decision: 'priority_review', riskLevel: 'high', reason: 'A configured banned term requires priority review.' };
|
||||
}
|
||||
if (input.ruleMatches.length) {
|
||||
return { decision: 'review', riskLevel: 'medium', reason: 'A deterministic safety rule requires review.' };
|
||||
}
|
||||
const enabledThresholds = input.policy.thresholds.filter((threshold) => threshold.enabled);
|
||||
if (enabledThresholds.some((threshold) =>
|
||||
Number(input.categoryScores[threshold.category] ?? 0) >= threshold.priorityThreshold
|
||||
)) {
|
||||
return { decision: 'priority_review', riskLevel: 'high', reason: 'A provider category exceeded its priority threshold.' };
|
||||
}
|
||||
if (input.providerFlagged || enabledThresholds.some((threshold) =>
|
||||
Number(input.categoryScores[threshold.category] ?? 0) >= threshold.reviewThreshold
|
||||
)) {
|
||||
return { decision: 'review', riskLevel: 'medium', reason: 'The moderation provider flagged this content for review.' };
|
||||
}
|
||||
return input.policy.autoPublishClean
|
||||
? { decision: 'approved', riskLevel: 'low', reason: 'No configured moderation rule was triggered.' }
|
||||
: { decision: 'review', riskLevel: 'medium', reason: 'Clean content requires human approval under the current policy.' };
|
||||
}
|
||||
|
||||
function moderationClient() {
|
||||
const apiKey = env.OPENAI_MODERATION_API_KEY?.trim() || env.OPENAI_MECHANIC_API_KEY?.trim();
|
||||
if (!apiKey) return null;
|
||||
if (!client) client = new OpenAI({ apiKey, timeout: 15_000, maxRetries: 1 });
|
||||
return client;
|
||||
}
|
||||
|
||||
export async function moderateCommunityText(input: {
|
||||
contentType: ModeratedContentType;
|
||||
text: string;
|
||||
policy?: CommunityModerationPolicy;
|
||||
}): Promise<CommunityModerationResult> {
|
||||
const policy = input.policy ?? await getCommunityModerationPolicy();
|
||||
const contentHash = hashModeratedContent(input.text);
|
||||
const ruleMatches = findLocalModerationMatches(input.text, input.contentType, policy);
|
||||
if (!policy.automatedEnabled || !scopeEnabled(policy, input.contentType)) {
|
||||
const assessment = assessCommunityModeration({
|
||||
policy,
|
||||
contentType: input.contentType,
|
||||
providerFlagged: false,
|
||||
categoryScores: {},
|
||||
ruleMatches
|
||||
});
|
||||
return {
|
||||
...assessment,
|
||||
contentHash,
|
||||
policyVersion: policy.policyVersion,
|
||||
provider: 'local',
|
||||
model: 'local-rules-v1',
|
||||
requestId: null,
|
||||
categoryScores: {},
|
||||
categoryFlags: {},
|
||||
ruleMatches,
|
||||
error: null
|
||||
};
|
||||
}
|
||||
const openai = moderationClient();
|
||||
if (!openai) {
|
||||
const error = 'OpenAI moderation is not configured.';
|
||||
const assessment = assessCommunityModeration({
|
||||
policy,
|
||||
contentType: input.contentType,
|
||||
providerFlagged: false,
|
||||
categoryScores: {},
|
||||
ruleMatches,
|
||||
providerError: error
|
||||
});
|
||||
return {
|
||||
...assessment,
|
||||
contentHash,
|
||||
policyVersion: policy.policyVersion,
|
||||
provider: 'openai',
|
||||
model: policy.providerModel,
|
||||
requestId: null,
|
||||
categoryScores: {},
|
||||
categoryFlags: {},
|
||||
ruleMatches,
|
||||
error
|
||||
};
|
||||
}
|
||||
try {
|
||||
const response = await openai.moderations.create({
|
||||
model: policy.providerModel,
|
||||
input: input.text
|
||||
});
|
||||
const result = response.results[0];
|
||||
const scores = Object.fromEntries(MODERATION_CATEGORIES.map((category) => [
|
||||
category,
|
||||
Number(result?.category_scores[category] ?? 0)
|
||||
])) as Record<ModerationCategory, number>;
|
||||
const flags = Object.fromEntries(MODERATION_CATEGORIES.map((category) => [
|
||||
category,
|
||||
Boolean(result?.categories[category])
|
||||
])) as Record<ModerationCategory, boolean>;
|
||||
const assessment = assessCommunityModeration({
|
||||
policy,
|
||||
contentType: input.contentType,
|
||||
providerFlagged: Boolean(result?.flagged),
|
||||
categoryScores: scores,
|
||||
categoryFlags: flags,
|
||||
ruleMatches
|
||||
});
|
||||
return {
|
||||
...assessment,
|
||||
contentHash,
|
||||
policyVersion: policy.policyVersion,
|
||||
provider: 'openai',
|
||||
model: String(response.model),
|
||||
requestId: String((response as typeof response & { _request_id?: string })._request_id ?? response.id),
|
||||
categoryScores: scores,
|
||||
categoryFlags: flags,
|
||||
ruleMatches,
|
||||
error: null
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message.slice(0, 500) : 'Moderation provider request failed.';
|
||||
const assessment = assessCommunityModeration({
|
||||
policy,
|
||||
contentType: input.contentType,
|
||||
providerFlagged: false,
|
||||
categoryScores: {},
|
||||
ruleMatches,
|
||||
providerError: message
|
||||
});
|
||||
return {
|
||||
...assessment,
|
||||
contentHash,
|
||||
policyVersion: policy.policyVersion,
|
||||
provider: 'openai',
|
||||
model: policy.providerModel,
|
||||
requestId: null,
|
||||
categoryScores: {},
|
||||
categoryFlags: {},
|
||||
ruleMatches,
|
||||
error: message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function moderateCommunityImage(input: {
|
||||
contentType: 'avatar';
|
||||
image: Buffer;
|
||||
policy?: CommunityModerationPolicy;
|
||||
}): Promise<CommunityModerationResult> {
|
||||
const policy = input.policy ?? await getCommunityModerationPolicy();
|
||||
const contentHash = createHash('sha256').update(input.image).digest('hex');
|
||||
if (!policy.automatedEnabled || !scopeEnabled(policy, input.contentType)) {
|
||||
return {
|
||||
contentHash,
|
||||
decision: 'approved',
|
||||
riskLevel: 'low',
|
||||
reason: 'Automated image scanning is disabled.',
|
||||
policyVersion: policy.policyVersion,
|
||||
provider: 'local',
|
||||
model: 'local-rules-v1',
|
||||
requestId: null,
|
||||
categoryScores: {},
|
||||
categoryFlags: {},
|
||||
ruleMatches: [],
|
||||
error: null
|
||||
};
|
||||
}
|
||||
const openai = moderationClient();
|
||||
if (!openai) {
|
||||
const error = 'OpenAI moderation is not configured.';
|
||||
const assessment = assessCommunityModeration({
|
||||
policy,
|
||||
contentType: input.contentType,
|
||||
providerFlagged: false,
|
||||
categoryScores: {},
|
||||
ruleMatches: [],
|
||||
providerError: error
|
||||
});
|
||||
return {
|
||||
...assessment,
|
||||
contentHash,
|
||||
policyVersion: policy.policyVersion,
|
||||
provider: 'openai',
|
||||
model: policy.providerModel,
|
||||
requestId: null,
|
||||
categoryScores: {},
|
||||
categoryFlags: {},
|
||||
ruleMatches: [],
|
||||
error
|
||||
};
|
||||
}
|
||||
try {
|
||||
const response = await openai.moderations.create({
|
||||
model: policy.providerModel,
|
||||
input: [{
|
||||
type: 'image_url',
|
||||
image_url: { url: `data:image/webp;base64,${input.image.toString('base64')}` }
|
||||
}]
|
||||
});
|
||||
const result = response.results[0];
|
||||
const scores = Object.fromEntries(MODERATION_CATEGORIES.map((category) => [
|
||||
category,
|
||||
Number(result?.category_scores[category] ?? 0)
|
||||
])) as Record<ModerationCategory, number>;
|
||||
const flags = Object.fromEntries(MODERATION_CATEGORIES.map((category) => [
|
||||
category,
|
||||
Boolean(result?.categories[category])
|
||||
])) as Record<ModerationCategory, boolean>;
|
||||
const assessment = assessCommunityModeration({
|
||||
policy,
|
||||
contentType: input.contentType,
|
||||
providerFlagged: Boolean(result?.flagged),
|
||||
categoryScores: scores,
|
||||
categoryFlags: flags,
|
||||
ruleMatches: []
|
||||
});
|
||||
return {
|
||||
...assessment,
|
||||
contentHash,
|
||||
policyVersion: policy.policyVersion,
|
||||
provider: 'openai',
|
||||
model: String(response.model),
|
||||
requestId: String((response as typeof response & { _request_id?: string })._request_id ?? response.id),
|
||||
categoryScores: scores,
|
||||
categoryFlags: flags,
|
||||
ruleMatches: [],
|
||||
error: null
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message.slice(0, 500) : 'Moderation provider request failed.';
|
||||
const assessment = assessCommunityModeration({
|
||||
policy,
|
||||
contentType: input.contentType,
|
||||
providerFlagged: false,
|
||||
categoryScores: {},
|
||||
ruleMatches: [],
|
||||
providerError: message
|
||||
});
|
||||
return {
|
||||
...assessment,
|
||||
contentHash,
|
||||
policyVersion: policy.policyVersion,
|
||||
provider: 'openai',
|
||||
model: policy.providerModel,
|
||||
requestId: null,
|
||||
categoryScores: {},
|
||||
categoryFlags: {},
|
||||
ruleMatches: [],
|
||||
error: message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function recordCommunityModerationScan(input: {
|
||||
contentType: ModeratedContentType;
|
||||
contentId: string;
|
||||
revisionId?: string | null;
|
||||
authorId: string;
|
||||
result: CommunityModerationResult;
|
||||
}) {
|
||||
const database = await getDatabase();
|
||||
const id = randomUUID();
|
||||
const now = new Date().toISOString();
|
||||
const result = await (await database.prepare(`INSERT OR IGNORE INTO content_moderation_scans
|
||||
(id,content_type,content_id,revision_id,author_id,content_hash,policy_version,
|
||||
provider,model,provider_request_id,status,risk_level,reason,scores_json,
|
||||
flags_json,rule_matches_json,error_code,error_message,created_at,completed_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)).run(
|
||||
id,
|
||||
input.contentType,
|
||||
input.contentId,
|
||||
input.revisionId ?? null,
|
||||
input.authorId,
|
||||
input.result.contentHash,
|
||||
input.result.policyVersion,
|
||||
input.result.provider,
|
||||
input.result.model,
|
||||
input.result.requestId,
|
||||
input.result.decision,
|
||||
input.result.riskLevel,
|
||||
input.result.reason,
|
||||
JSON.stringify(input.result.categoryScores),
|
||||
JSON.stringify(input.result.categoryFlags),
|
||||
JSON.stringify(input.result.ruleMatches),
|
||||
input.result.error ? 'provider_error' : null,
|
||||
input.result.error,
|
||||
now,
|
||||
now
|
||||
);
|
||||
if (result.changes) return id;
|
||||
const existing = await (await database.prepare(`SELECT id FROM content_moderation_scans
|
||||
WHERE content_type=? AND content_id=? AND content_hash=? AND policy_version=?
|
||||
AND provider=? AND model=?`)).get(
|
||||
input.contentType,
|
||||
input.contentId,
|
||||
input.result.contentHash,
|
||||
input.result.policyVersion,
|
||||
input.result.provider,
|
||||
input.result.model
|
||||
);
|
||||
return String(existing?.id ?? id);
|
||||
}
|
||||
|
||||
export async function createUserContentReport(input: {
|
||||
contentType: ModeratedContentType;
|
||||
contentId: string;
|
||||
reporterId: string;
|
||||
category: 'spam' | 'harassment' | 'hate' | 'sexual' | 'violence' | 'self_harm' | 'privacy' | 'other';
|
||||
details?: string | null;
|
||||
}) {
|
||||
const database = await getDatabase();
|
||||
const id = randomUUID();
|
||||
const createdAt = new Date().toISOString();
|
||||
const result = await (await database.prepare(`INSERT OR IGNORE INTO user_content_reports
|
||||
(id,content_type,content_id,reporter_id,category,details,status,created_at)
|
||||
VALUES(?,?,?,?,?,?,'pending',?)`)).run(
|
||||
id,
|
||||
input.contentType,
|
||||
input.contentId,
|
||||
input.reporterId,
|
||||
input.category,
|
||||
input.details?.trim() || null,
|
||||
createdAt
|
||||
);
|
||||
return result.changes ? { id, createdAt } : null;
|
||||
}
|
||||
Reference in New Issue
Block a user