shd.build/src/lib/server/community-moderation.test.ts

import { describe, expect, it } from 'vitest';
import {
  assessCommunityModeration,
  findLocalModerationMatches,
  normalizeModerationTerm,
  parseModerationTermBatch,
  type CommunityModerationPolicy,
} from './community-moderation';

const policy = (
  overrides: Partial<CommunityModerationPolicy> = {},
): CommunityModerationPolicy => ({
  automatedEnabled: true,
  autoPublishClean: true,
  holdOnProviderError: true,
  scanComments: true,
  scanProfiles: true,
  scanGuides: true,
  scanAvatars: true,
  providerModel: 'omni-moderation-latest',
  detectPii: true,
  maxLinks: 1,
  duplicateWindowSeconds: 300,
  policyVersion: 1,
  thresholds: [
    {
      category: 'harassment',
      enabled: true,
      reviewThreshold: 0.5,
      priorityThreshold: 0.85,
    },
  ],
  terms: [],
  ...overrides,
});

describe('community moderation policy', () => {
  it('holds provider-required Guide content when classification fails', () => {
    expect(
      assessCommunityModeration({
        policy: policy({ automatedEnabled: false, holdOnProviderError: false }),
        contentType: 'guide',
        providerFlagged: false,
        categoryScores: {},
        ruleMatches: [],
        providerError: 'unavailable',
        providerRequired: true,
      }).decision,
    ).toBe('error');
  });

  it('normalizes configured terms consistently', () => {
    expect(normalizeModerationTerm('  Rogue\u00a0Agent  ')).toBe('rogue agent');
  });

  it('parses pasted terms with automatic word and phrase matching', () => {
    expect(
      parseModerationTermBatch('weapon\nkill, machine gun; assault rifle'),
    ).toEqual({
      terms: [
        { value: 'weapon', normalizedValue: 'weapon', matchMode: 'word' },
        { value: 'kill', normalizedValue: 'kill', matchMode: 'word' },
        {
          value: 'machine gun',
          normalizedValue: 'machine gun',
          matchMode: 'phrase',
        },
        {
          value: 'assault rifle',
          normalizedValue: 'assault rifle',
          matchMode: 'phrase',
        },
      ],
      duplicateCount: 0,
    });
  });

  it('deduplicates pasted terms after compatibility and case normalization', () => {
    const parsed = parseModerationTermBatch(
      'Weapon\nｗｅａｐｏｎ\nMACHINE   GUN\nmachine gun',
    );
    expect(parsed.terms).toEqual([
      { value: 'Weapon', normalizedValue: 'weapon', matchMode: 'word' },
      {
        value: 'MACHINE GUN',
        normalizedValue: 'machine gun',
        matchMode: 'phrase',
      },
    ]);
    expect(parsed.duplicateCount).toBe(2);
  });

  it('allows an exact local exception to suppress a matching banned term', () => {
    const matches = findLocalModerationMatches(
      'Rogue Agent tactics',
      'comment',
      policy({
        detectPii: false,
        terms: [
          {
            id: 'ban',
            listType: 'ban',
            matchMode: 'phrase',
            value: 'Rogue Agent',
            normalizedValue: 'rogue agent',
            scopes: ['comment'],
            enabled: true,
          },
          {
            id: 'allow',
            listType: 'allow',
            matchMode: 'phrase',
            value: 'Rogue Agent',
            normalizedValue: 'rogue agent',
            scopes: ['comment'],
            enabled: true,
          },
        ],
      }),
    );
    expect(matches).toEqual([]);
  });

  it('finds scoped banned terms, PII, and excessive links', () => {
    const matches = findLocalModerationMatches(
      'Contact test@example.com about forbidden plans at https://one.test and https://two.test',
      'comment',
      policy({
        terms: [
          {
            id: 'ban',
            listType: 'ban',
            matchMode: 'word',
            value: 'forbidden',
            normalizedValue: 'forbidden',
            scopes: ['comment'],
            enabled: true,
          },
        ],
      }),
    );
    expect(matches.map((match) => match.kind)).toEqual([
      'banned_term',
      'pii',
      'link_limit',
    ]);
  });

  it('routes priority scores ahead of ordinary review scores', () => {
    expect(
      assessCommunityModeration({
        policy: policy(),
        contentType: 'comment',
        providerFlagged: true,
        categoryScores: { harassment: 0.9 },
        ruleMatches: [],
      }).decision,
    ).toBe('priority_review');
  });

  it('holds provider failures under the default fail-closed policy', () => {
    expect(
      assessCommunityModeration({
        policy: policy(),
        contentType: 'profile_bio',
        providerFlagged: false,
        categoryScores: {},
        ruleMatches: [],
        providerError: 'timeout',
      }).decision,
    ).toBe('error');
  });

  it('publishes clean content only when auto publication is enabled', () => {
    expect(
      assessCommunityModeration({
        policy: policy(),
        contentType: 'comment',
        providerFlagged: false,
        categoryScores: { harassment: 0.01 },
        ruleMatches: [],
      }).decision,
    ).toBe('approved');
    expect(
      assessCommunityModeration({
        policy: policy({ autoPublishClean: false }),
        contentType: 'comment',
        providerFlagged: false,
        categoryScores: { harassment: 0.01 },
        ruleMatches: [],
      }).decision,
    ).toBe('review');
  });

  it('publishes locally clean content when provider scanning is disabled', () => {
    expect(
      assessCommunityModeration({
        policy: policy({ automatedEnabled: false }),
        contentType: 'comment',
        providerFlagged: false,
        categoryScores: {},
        ruleMatches: [],
      }).decision,
    ).toBe('approved');
  });
});