feat: add automated community moderation

src/lib/server/community-moderation.test.ts

@@ -0,0 +1,114 @@
+import { describe, expect, it } from 'vitest';
+import {
+	assessCommunityModeration,
+	findLocalModerationMatches,
+	normalizeModerationTerm,
+	type CommunityModerationPolicy
+} from './community-moderation';
+
+const policy = (overrides: Partial<CommunityModerationPolicy> = {}): CommunityModerationPolicy => ({
+	automatedEnabled: true,
+	autoPublishClean: true,
+	holdOnProviderError: true,
+	scanComments: true,
+	scanProfiles: true,
+	scanGuides: true,
+	scanAvatars: true,
+	providerModel: 'omni-moderation-latest',
+	detectPii: true,
+	maxLinks: 1,
+	duplicateWindowSeconds: 300,
+	policyVersion: 1,
+	thresholds: [{
+		category: 'harassment',
+		enabled: true,
+		reviewThreshold: 0.5,
+		priorityThreshold: 0.85
+	}],
+	terms: [],
+	...overrides
+});
+
+describe('community moderation policy', () => {
+	it('normalizes configured terms consistently', () => {
+		expect(normalizeModerationTerm('  Rogue\u00a0Agent  ')).toBe('rogue agent');
+	});
+
+	it('allows an exact local exception to suppress a matching banned term', () => {
+		const matches = findLocalModerationMatches('Rogue Agent tactics', 'comment', policy({
+			detectPii: false,
+			terms: [
+				{ id: 'ban', listType: 'ban', matchMode: 'phrase', value: 'Rogue Agent', normalizedValue: 'rogue agent', scopes: ['comment'], enabled: true },
+				{ id: 'allow', listType: 'allow', matchMode: 'phrase', value: 'Rogue Agent', normalizedValue: 'rogue agent', scopes: ['comment'], enabled: true }
+			]
+		}));
+		expect(matches).toEqual([]);
+	});
+
+	it('finds scoped banned terms, PII, and excessive links', () => {
+		const matches = findLocalModerationMatches(
+			'Contact test@example.com about forbidden plans at https://one.test and https://two.test',
+			'comment',
+			policy({
+				terms: [{
+					id: 'ban',
+					listType: 'ban',
+					matchMode: 'word',
+					value: 'forbidden',
+					normalizedValue: 'forbidden',
+					scopes: ['comment'],
+					enabled: true
+				}]
+			})
+		);
+		expect(matches.map((match) => match.kind)).toEqual(['banned_term', 'pii', 'link_limit']);
+	});
+
+	it('routes priority scores ahead of ordinary review scores', () => {
+		expect(assessCommunityModeration({
+			policy: policy(),
+			contentType: 'comment',
+			providerFlagged: true,
+			categoryScores: { harassment: 0.9 },
+			ruleMatches: []
+		}).decision).toBe('priority_review');
+	});
+
+	it('holds provider failures under the default fail-closed policy', () => {
+		expect(assessCommunityModeration({
+			policy: policy(),
+			contentType: 'profile_bio',
+			providerFlagged: false,
+			categoryScores: {},
+			ruleMatches: [],
+			providerError: 'timeout'
+		}).decision).toBe('error');
+	});
+
+	it('publishes clean content only when auto publication is enabled', () => {
+		expect(assessCommunityModeration({
+			policy: policy(),
+			contentType: 'comment',
+			providerFlagged: false,
+			categoryScores: { harassment: 0.01 },
+			ruleMatches: []
+		}).decision).toBe('approved');
+		expect(assessCommunityModeration({
+			policy: policy({ autoPublishClean: false }),
+			contentType: 'comment',
+			providerFlagged: false,
+			categoryScores: { harassment: 0.01 },
+			ruleMatches: []
+		}).decision).toBe('review');
+	});
+
+	it('publishes locally clean content when provider scanning is disabled', () => {
+		expect(assessCommunityModeration({
+			policy: policy({ automatedEnabled: false }),
+			contentType: 'comment',
+			providerFlagged: false,
+			categoryScores: {},
+			ruleMatches: []
+		}).decision).toBe('approved');
+	});
+});