feat: link official news to Ubisoft
This commit is contained in:
@@ -302,6 +302,8 @@ CREATE TABLE IF NOT EXISTS external_news_sources (
|
|||||||
provider TEXT NOT NULL CHECK(provider IN ('ubisoft')),
|
provider TEXT NOT NULL CHECK(provider IN ('ubisoft')),
|
||||||
external_id TEXT NOT NULL,
|
external_id TEXT NOT NULL,
|
||||||
canonical_url TEXT NOT NULL,
|
canonical_url TEXT NOT NULL,
|
||||||
|
thumbnail_url TEXT,
|
||||||
|
thumbnail_alt TEXT NOT NULL DEFAULT '',
|
||||||
source_published_at TEXT NOT NULL,
|
source_published_at TEXT NOT NULL,
|
||||||
content_hash TEXT NOT NULL CHECK(length(content_hash)=64),
|
content_hash TEXT NOT NULL CHECK(length(content_hash)=64),
|
||||||
first_seen_at TEXT NOT NULL,
|
first_seen_at TEXT NOT NULL,
|
||||||
|
|||||||
@@ -606,6 +606,11 @@ editorial group, and Recommended Guides carry a textual badge and filter. Search
|
|||||||
relevance is not overridden by curation. Guide cards pair a cover or
|
relevance is not overridden by curation. Guide cards pair a cover or
|
||||||
field-manual fallback with title, deck, author, publication date, tags, and the
|
field-manual fallback with title, deck, author, publication date, tags, and the
|
||||||
rating count. Unlisted and private articles never enter discovery or search.
|
rating count. Unlisted and private articles never enter discovery or search.
|
||||||
|
Official Ubisoft News cards use the validated source thumbnail through the
|
||||||
|
same-origin image proxy and link their cover and headline to the canonical
|
||||||
|
Ubisoft article. The `BY UBISOFT` metadata identifies the external destination
|
||||||
|
before activation. Internal News and all Guides continue to use SHD.build
|
||||||
|
article routes.
|
||||||
|
|
||||||
Profile conversations begin with a collapsed `LEAVE A MESSAGE` action. A reply
|
Profile conversations begin with a collapsed `LEAVE A MESSAGE` action. A reply
|
||||||
opens the same editor beneath the selected message, and only one editor remains
|
opens the same editor beneath the selected message, and only one editor remains
|
||||||
|
|||||||
+15
-10
@@ -123,25 +123,30 @@ comments still use community moderation.
|
|||||||
|
|
||||||
The Ubisoft News importer reads the English-language Division Resurgence feed
|
The Ubisoft News importer reads the English-language Division Resurgence feed
|
||||||
from Ubisoft Nimbus. It stores each Nimbus item ID as the stable external ID and
|
from Ubisoft Nimbus. It stores each Nimbus item ID as the stable external ID and
|
||||||
publishes only the source title, excerpt, and canonical Ubisoft link. It does
|
publishes the source title, excerpt, canonical Ubisoft link, and validated
|
||||||
not copy the source article body or images. Imported articles use Ubisoft's
|
Ubisoft-hosted thumbnail. SHD.build does not copy the article body or image
|
||||||
original publication timestamp rather than the import time.
|
file. A bounded same-origin endpoint proxies each thumbnail so opening the News
|
||||||
|
listing does not send visitor metadata to Ubisoft. News cards link to the
|
||||||
|
canonical Ubisoft article. Imported articles use Ubisoft's original publication
|
||||||
|
timestamp rather than the import time.
|
||||||
|
|
||||||
`external_news_sources` maps each Nimbus ID to one SHD.build article. A SHA-256
|
`external_news_sources` maps each Nimbus ID to one SHD.build article. A SHA-256
|
||||||
content hash covers the normalized title, excerpt, canonical URL, and source
|
content hash covers the normalized title, excerpt, canonical URL, and source
|
||||||
publication timestamp. An unchanged hash updates `last_seen_at` without creating
|
publication timestamp. An unchanged hash updates `last_seen_at` without creating
|
||||||
a revision. A changed hash creates and publishes an immutable revision. The
|
a revision. Thumbnail changes update the audited source metadata without
|
||||||
importer does not delete or unpublish an article when Nimbus omits it from a
|
creating an article revision. A changed content hash creates and publishes an
|
||||||
later feed, and it does not restore an article that an editor deleted. The
|
immutable revision. The importer does not delete or unpublish an article when
|
||||||
Newsroom treats source-linked articles as read-only so local edits cannot retain
|
Nimbus omits it from a later feed, and it does not restore an article that an
|
||||||
Ubisoft attribution. The importer defers future-dated items until their source
|
editor deleted. The Newsroom treats source-linked articles as read-only so local
|
||||||
publication time.
|
edits cannot retain Ubisoft attribution. The importer defers future-dated items
|
||||||
|
until their source publication time.
|
||||||
|
|
||||||
The seeded, non-public `Ubisoft Official News` service identity owns imported
|
The seeded, non-public `Ubisoft Official News` service identity owns imported
|
||||||
articles and acts on their audit events. A first import records
|
articles and acts on their audit events. A first import records
|
||||||
`editorial_content / news.imported`; a later content sync records
|
`editorial_content / news.imported`; a later content sync records
|
||||||
`editorial_content / news.import_synced`. Audit snapshots identify the provider,
|
`editorial_content / news.import_synced`. Audit snapshots identify the provider,
|
||||||
Nimbus ID, canonical URL, source timestamp, and resulting revision.
|
Nimbus ID, canonical URL, thumbnail metadata, source timestamp, and resulting
|
||||||
|
revision.
|
||||||
|
|
||||||
The existing bearer-authenticated scheduler endpoint invokes the importer. A
|
The existing bearer-authenticated scheduler endpoint invokes the importer. A
|
||||||
successful run sets the next check one hour later. The importer stores the last
|
successful run sets the next check one hour later. The importer stores the last
|
||||||
|
|||||||
@@ -679,6 +679,27 @@ if (articleRevisionColumns.some((column) => column.name === 'plaint_text')) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const externalNewsSourceColumns = await (
|
||||||
|
await database.prepare("PRAGMA table_info('external_news_sources')")
|
||||||
|
).all();
|
||||||
|
if (
|
||||||
|
externalNewsSourceColumns.length &&
|
||||||
|
!externalNewsSourceColumns.some((column) => column.name === 'thumbnail_url')
|
||||||
|
) {
|
||||||
|
console.log('Adding Ubisoft News thumbnail metadata.');
|
||||||
|
await database.exec(
|
||||||
|
'ALTER TABLE external_news_sources ADD COLUMN thumbnail_url TEXT',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
externalNewsSourceColumns.length &&
|
||||||
|
!externalNewsSourceColumns.some((column) => column.name === 'thumbnail_alt')
|
||||||
|
) {
|
||||||
|
await database.exec(
|
||||||
|
"ALTER TABLE external_news_sources ADD COLUMN thumbnail_alt TEXT NOT NULL DEFAULT ''",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const currentModerationScanColumns = await (
|
const currentModerationScanColumns = await (
|
||||||
await database.prepare("PRAGMA table_info('content_moderation_scans')")
|
await database.prepare("PRAGMA table_info('content_moderation_scans')")
|
||||||
).all();
|
).all();
|
||||||
|
|||||||
@@ -1,13 +1,15 @@
|
|||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
let { article, kind = 'guide', prominent = false } = $props();
|
let { article, kind = 'guide', prominent = false } = $props();
|
||||||
|
const href = $derived(article.source?.url ?? `/${kind === 'guide' ? 'guides' : 'news'}/${encodeURIComponent(article.slug)}`);
|
||||||
|
const external = $derived(Boolean(article.source));
|
||||||
</script>
|
</script>
|
||||||
<article class="article-card" class:prominent>
|
<article class="article-card" class:prominent>
|
||||||
<a class="article-card-cover" href={`/${kind === 'guide' ? 'guides' : 'news'}/${encodeURIComponent(article.slug)}`}>
|
<a class="article-card-cover" {href} rel={external ? 'external noreferrer' : undefined}>
|
||||||
{#if article.coverUrl}<img src={article.coverUrl} alt=""/>{:else}<div><span>SHD // {kind === 'guide' ? 'FIELD MANUAL' : 'TRANSMISSION'}</span><b>{article.tags?.[0]?.name ?? article.channel ?? 'COMMUNITY INTELLIGENCE'}</b></div>{/if}
|
{#if article.coverUrl}<img src={article.coverUrl} alt={article.coverAlt ?? ''}/>{:else}<div><span>SHD // {kind === 'guide' ? 'FIELD MANUAL' : 'TRANSMISSION'}</span><b>{article.tags?.[0]?.name ?? article.channel ?? 'COMMUNITY INTELLIGENCE'}</b></div>{/if}
|
||||||
</a>
|
</a>
|
||||||
<div class="article-card-body">
|
<div class="article-card-body">
|
||||||
<header><span>{article.channel ?? article.tags?.[0]?.name ?? kind.toUpperCase()}</span>{#if article.recommended}<b>RECOMMENDED</b>{/if}{#if article.featured}<b>FEATURED</b>{/if}</header>
|
<header><span>{article.channel ?? article.tags?.[0]?.name ?? kind.toUpperCase()}</span>{#if article.recommended}<b>RECOMMENDED</b>{/if}{#if article.featured}<b>FEATURED</b>{/if}</header>
|
||||||
<h2><a href={`/${kind === 'guide' ? 'guides' : 'news'}/${encodeURIComponent(article.slug)}`}>{article.title}</a></h2>
|
<h2><a {href} rel={external ? 'external noreferrer' : undefined}>{article.title}</a></h2>
|
||||||
<p>{article.deck}</p>
|
<p>{article.deck}</p>
|
||||||
<footer><span>BY {(article.source?.name ?? article.author).toUpperCase()} · {new Date(article.publishedAt).toLocaleDateString()}</span>{#if kind === 'guide'}<b>{article.votes ? `★ ${article.rating.toFixed(1)} · ${article.votes}` : 'UNRATED'}</b>{/if}</footer>
|
<footer><span>BY {(article.source?.name ?? article.author).toUpperCase()} · {new Date(article.publishedAt).toLocaleDateString()}</span>{#if kind === 'guide'}<b>{article.votes ? `★ ${article.rating.toFixed(1)} · ${article.votes}` : 'UNRATED'}</b>{/if}</footer>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -29,7 +29,7 @@
|
|||||||
{#if data.article.source}<aside class="article-source-notice"><div><span>OFFICIAL SOURCE</span><strong>Originally published by {data.article.source.name}</strong></div><a href={data.article.source.url} target="_blank" rel="noopener noreferrer">READ ON UBISOFT</a></aside>{/if}
|
{#if data.article.source}<aside class="article-source-notice"><div><span>OFFICIAL SOURCE</span><strong>Originally published by {data.article.source.name}</strong></div><a href={data.article.source.url} target="_blank" rel="noopener noreferrer">READ ON UBISOFT</a></aside>{/if}
|
||||||
{#if data.tags.length}<nav class="article-reader-tags" aria-label="Article tags">{#each data.tags as tag}<a href={`/${path}?tag=${encodeURIComponent(tag.slug)}`}>{tag.name}</a>{/each}</nav>{/if}
|
{#if data.tags.length}<nav class="article-reader-tags" aria-label="Article tags">{#each data.tags as tag}<a href={`/${path}?tag=${encodeURIComponent(tag.slug)}`}>{tag.name}</a>{/each}</nav>{/if}
|
||||||
</header>
|
</header>
|
||||||
{#if data.article.coverUrl}<figure class="article-hero-image"><img src={data.article.coverUrl} alt={data.article.coverAlt}/>{#if data.article.coverCaption}<figcaption>{data.article.coverCaption}</figcaption>{/if}</figure>{/if}
|
{#if data.article.coverUrl}<figure class="article-hero-image"><img src={data.article.coverUrl} alt={data.article.coverAlt} referrerpolicy={data.article.source ? 'no-referrer' : undefined}/>{#if data.article.coverCaption}<figcaption>{data.article.coverCaption}</figcaption>{/if}</figure>{/if}
|
||||||
<div class="article-reader-layout" class:article-reader-layout-wide={data.article.kind === 'news'}>
|
<div class="article-reader-layout" class:article-reader-layout-wide={data.article.kind === 'news'}>
|
||||||
<article><ArticleContent document={data.document} builds={data.builds} images={data.images}/></article>
|
<article><ArticleContent document={data.document} builds={data.builds} images={data.images}/></article>
|
||||||
{#if data.article.kind === 'guide'}<aside>
|
{#if data.article.kind === 'guide'}<aside>
|
||||||
|
|||||||
@@ -896,6 +896,7 @@ export async function loadPublishedArticle(
|
|||||||
COALESCE(profile.display_name,users.username,'Deleted Agent') AS author,profile.public_slug,
|
COALESCE(profile.display_name,users.username,'Deleted Agent') AS author,profile.public_slug,
|
||||||
images.updated_at AS cover_updated_at,
|
images.updated_at AS cover_updated_at,
|
||||||
source.provider AS external_provider,source.canonical_url AS external_canonical_url,
|
source.provider AS external_provider,source.canonical_url AS external_canonical_url,
|
||||||
|
source.thumbnail_url AS external_thumbnail_url,source.thumbnail_alt AS external_thumbnail_alt,
|
||||||
source.source_published_at AS external_published_at,
|
source.source_published_at AS external_published_at,
|
||||||
COALESCE(rating.average,0) AS rating,COALESCE(rating.votes,0) AS votes,
|
COALESCE(rating.average,0) AS rating,COALESCE(rating.votes,0) AS votes,
|
||||||
curation.pinned_order,COALESCE(curation.featured,0) AS featured,COALESCE(curation.recommended,0) AS recommended
|
curation.pinned_order,COALESCE(curation.featured,0) AS featured,COALESCE(curation.recommended,0) AS recommended
|
||||||
@@ -995,8 +996,12 @@ export async function loadPublishedArticle(
|
|||||||
channel: row.channel ? String(row.channel) : null,
|
channel: row.channel ? String(row.channel) : null,
|
||||||
coverUrl: row.cover_image_id
|
coverUrl: row.cover_image_id
|
||||||
? managedImageUrl(String(row.cover_image_id), row.cover_updated_at)
|
? managedImageUrl(String(row.cover_image_id), row.cover_updated_at)
|
||||||
: '',
|
: row.external_thumbnail_url
|
||||||
coverAlt: String(row.cover_alt),
|
? `/images/news/${encodeURIComponent(String(row.article_id))}`
|
||||||
|
: '',
|
||||||
|
coverAlt: row.cover_image_id
|
||||||
|
? String(row.cover_alt)
|
||||||
|
: String(row.external_thumbnail_alt ?? ''),
|
||||||
coverCaption: String(row.cover_caption),
|
coverCaption: String(row.cover_caption),
|
||||||
rating: Number(row.rating),
|
rating: Number(row.rating),
|
||||||
votes: Number(row.votes),
|
votes: Number(row.votes),
|
||||||
@@ -1110,6 +1115,7 @@ export async function listPublishedArticles(input: {
|
|||||||
await database.prepare(`${withClause} SELECT articles.id,articles.slug,articles.owner_id,articles.published_at,
|
await database.prepare(`${withClause} SELECT articles.id,articles.slug,articles.owner_id,articles.published_at,
|
||||||
revision.id AS revision_id,revision.title,revision.deck,revision.channel,revision.cover_image_id,cover.updated_at AS cover_updated_at,
|
revision.id AS revision_id,revision.title,revision.deck,revision.channel,revision.cover_image_id,cover.updated_at AS cover_updated_at,
|
||||||
source.provider AS external_provider,source.canonical_url AS external_canonical_url,
|
source.provider AS external_provider,source.canonical_url AS external_canonical_url,
|
||||||
|
source.thumbnail_url AS external_thumbnail_url,source.thumbnail_alt AS external_thumbnail_alt,
|
||||||
COALESCE(profile.display_name,users.username,'Deleted Agent') AS author,profile.public_slug,
|
COALESCE(profile.display_name,users.username,'Deleted Agent') AS author,profile.public_slug,
|
||||||
COALESCE(ratings.rating,0) AS rating,COALESCE(ratings.votes,0) AS votes,
|
COALESCE(ratings.rating,0) AS rating,COALESCE(ratings.votes,0) AS votes,
|
||||||
curation.pinned_order,COALESCE(curation.featured,0) AS featured,COALESCE(curation.recommended,0) AS recommended,
|
curation.pinned_order,COALESCE(curation.featured,0) AS featured,COALESCE(curation.recommended,0) AS recommended,
|
||||||
@@ -1142,7 +1148,12 @@ export async function listPublishedArticles(input: {
|
|||||||
: null,
|
: null,
|
||||||
coverUrl: row.cover_image_id
|
coverUrl: row.cover_image_id
|
||||||
? managedImageUrl(String(row.cover_image_id), row.cover_updated_at)
|
? managedImageUrl(String(row.cover_image_id), row.cover_updated_at)
|
||||||
: '',
|
: row.external_thumbnail_url
|
||||||
|
? `/images/news/${encodeURIComponent(String(row.id))}`
|
||||||
|
: '',
|
||||||
|
coverAlt: row.cover_image_id
|
||||||
|
? ''
|
||||||
|
: String(row.external_thumbnail_alt ?? ''),
|
||||||
rating: Number(row.rating),
|
rating: Number(row.rating),
|
||||||
votes: Number(row.votes),
|
votes: Number(row.votes),
|
||||||
pinnedOrder: row.pinned_order === null ? null : Number(row.pinned_order),
|
pinnedOrder: row.pinned_order === null ? null : Number(row.pinned_order),
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { describe, expect, it, vi } from 'vitest';
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
import {
|
import {
|
||||||
extractNimbusApiKey,
|
extractNimbusApiKey,
|
||||||
|
fetchUbisoftNewsThumbnail,
|
||||||
fetchUbisoftNews,
|
fetchUbisoftNews,
|
||||||
UbisoftNewsSourceError,
|
UbisoftNewsSourceError,
|
||||||
} from './ubisoft-news-client';
|
} from './ubisoft-news-client';
|
||||||
@@ -29,6 +30,10 @@ function item(id: string, overrides: Record<string, unknown> = {}) {
|
|||||||
abstract: 'Read the latest official update.',
|
abstract: 'Read the latest official update.',
|
||||||
date: '2024-03-11T15:20:00.000Z',
|
date: '2024-03-11T15:20:00.000Z',
|
||||||
button: { buttonUrl: `/${id}/division-resurgence-update` },
|
button: { buttonUrl: `/${id}/division-resurgence-update` },
|
||||||
|
thumbnail: {
|
||||||
|
url: `https://staticctf.ubisoft.com/news/${id}/cover.jpg`,
|
||||||
|
description: 'Division Resurgence key art',
|
||||||
|
},
|
||||||
...overrides,
|
...overrides,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -68,6 +73,10 @@ describe('Ubisoft News client', () => {
|
|||||||
expect(result[0].canonicalUrl).toBe(
|
expect(result[0].canonicalUrl).toBe(
|
||||||
`https://www.ubisoft.com/en-us/game/the-division/resurgence/news-updates/${externalIdOne}/earlier-update`,
|
`https://www.ubisoft.com/en-us/game/the-division/resurgence/news-updates/${externalIdOne}/earlier-update`,
|
||||||
);
|
);
|
||||||
|
expect(result[0].thumbnailUrl).toBe(
|
||||||
|
`https://staticctf.ubisoft.com/news/${externalIdOne}/cover.jpg`,
|
||||||
|
);
|
||||||
|
expect(result[0].thumbnailAlt).toBe('Division Resurgence key art');
|
||||||
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
expect(fetchImpl).toHaveBeenCalledTimes(3);
|
||||||
const secondRequest = fetchImpl.mock.calls[1];
|
const secondRequest = fetchImpl.mock.calls[1];
|
||||||
expect(String(secondRequest[0])).toContain('locale=en-us');
|
expect(String(secondRequest[0])).toContain('locale=en-us');
|
||||||
@@ -109,6 +118,26 @@ describe('Ubisoft News client', () => {
|
|||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('rejects thumbnails outside Ubisoft image hosting', async () => {
|
||||||
|
const fetchImpl = vi
|
||||||
|
.fn<typeof fetch>()
|
||||||
|
.mockResolvedValueOnce(landing())
|
||||||
|
.mockResolvedValueOnce(
|
||||||
|
page(1, [
|
||||||
|
item(externalIdOne, {
|
||||||
|
thumbnail: {
|
||||||
|
url: 'https://example.com/cover.jpg',
|
||||||
|
description: 'Untrusted image',
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(fetchUbisoftNews(fetchImpl)).rejects.toThrow(
|
||||||
|
'Ubisoft news thumbnail was outside the approved image host.',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
it('does not follow provider redirects', async () => {
|
it('does not follow provider redirects', async () => {
|
||||||
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
|
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
|
||||||
new Response(null, {
|
new Response(null, {
|
||||||
@@ -122,4 +151,54 @@ describe('Ubisoft News client', () => {
|
|||||||
);
|
);
|
||||||
expect(fetchImpl.mock.calls[0][1]?.redirect).toBe('error');
|
expect(fetchImpl.mock.calls[0][1]?.redirect).toBe('error');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('fetches bounded raster thumbnails without following redirects', async () => {
|
||||||
|
const bytes = new Uint8Array([1, 2, 3]);
|
||||||
|
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
|
||||||
|
new Response(bytes, {
|
||||||
|
headers: { 'content-type': 'image/jpeg; charset=binary' },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
const image = await fetchUbisoftNewsThumbnail(
|
||||||
|
'https://staticctf.ubisoft.com/news/cover.jpg',
|
||||||
|
fetchImpl,
|
||||||
|
);
|
||||||
|
expect(image.contentType).toBe('image/jpeg');
|
||||||
|
expect(image.body).toEqual(bytes);
|
||||||
|
expect(fetchImpl.mock.calls[0][1]?.redirect).toBe('error');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects thumbnail responses that are not raster images', async () => {
|
||||||
|
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
|
||||||
|
new Response('<svg/>', {
|
||||||
|
headers: { 'content-type': 'image/svg+xml' },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
fetchUbisoftNewsThumbnail(
|
||||||
|
'https://staticctf.ubisoft.com/news/cover.svg',
|
||||||
|
fetchImpl,
|
||||||
|
),
|
||||||
|
).rejects.toThrow('not a supported image');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects thumbnails larger than the response bound', async () => {
|
||||||
|
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValueOnce(
|
||||||
|
new Response(new Uint8Array([1]), {
|
||||||
|
headers: {
|
||||||
|
'content-type': 'image/jpeg',
|
||||||
|
'content-length': String(5 * 1024 * 1024 + 1),
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
fetchUbisoftNewsThumbnail(
|
||||||
|
'https://staticctf.ubisoft.com/news/large-cover.jpg',
|
||||||
|
fetchImpl,
|
||||||
|
),
|
||||||
|
).rejects.toThrow('exceeded the size limit');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,9 +2,11 @@ const UBISOFT_NEWS_ORIGIN = 'https://www.ubisoft.com';
|
|||||||
export const UBISOFT_NEWS_LISTING_URL = `${UBISOFT_NEWS_ORIGIN}/en-us/game/the-division/resurgence/news-updates`;
|
export const UBISOFT_NEWS_LISTING_URL = `${UBISOFT_NEWS_ORIGIN}/en-us/game/the-division/resurgence/news-updates`;
|
||||||
const UBISOFT_NEWS_PATH = '/en-us/game/the-division/resurgence/news-updates';
|
const UBISOFT_NEWS_PATH = '/en-us/game/the-division/resurgence/news-updates';
|
||||||
const NIMBUS_ITEMS_URL = 'https://nimbus.ubisoft.com/api/v1/items';
|
const NIMBUS_ITEMS_URL = 'https://nimbus.ubisoft.com/api/v1/items';
|
||||||
|
const UBISOFT_IMAGE_ORIGIN = 'https://staticctf.ubisoft.com';
|
||||||
const RESURGENCE_TAGS = 'BR-the-division GA-resurgence';
|
const RESURGENCE_TAGS = 'BR-the-division GA-resurgence';
|
||||||
const MAX_LANDING_BYTES = 600_000;
|
const MAX_LANDING_BYTES = 600_000;
|
||||||
const MAX_API_BYTES = 2_000_000;
|
const MAX_API_BYTES = 2_000_000;
|
||||||
|
const MAX_THUMBNAIL_BYTES = 5 * 1024 * 1024;
|
||||||
const PAGE_SIZE = 15;
|
const PAGE_SIZE = 15;
|
||||||
const MAX_PAGES = 10;
|
const MAX_PAGES = 10;
|
||||||
const ITEM_ID_PATTERN = /^[A-Za-z0-9]{20,24}$/;
|
const ITEM_ID_PATTERN = /^[A-Za-z0-9]{20,24}$/;
|
||||||
@@ -14,6 +16,8 @@ export type UbisoftNewsItem = {
|
|||||||
title: string;
|
title: string;
|
||||||
excerpt: string;
|
excerpt: string;
|
||||||
canonicalUrl: string;
|
canonicalUrl: string;
|
||||||
|
thumbnailUrl: string;
|
||||||
|
thumbnailAlt: string;
|
||||||
publishedAt: string;
|
publishedAt: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -23,14 +27,14 @@ function isRecord(value: unknown): value is Record<string, unknown> {
|
|||||||
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function readBoundedText(response: Response, maxBytes: number) {
|
async function readBoundedBytes(response: Response, maxBytes: number) {
|
||||||
const declaredLength = Number(response.headers.get('content-length'));
|
const declaredLength = Number(response.headers.get('content-length'));
|
||||||
if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
|
if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
|
||||||
throw new UbisoftNewsSourceError(
|
throw new UbisoftNewsSourceError(
|
||||||
'Ubisoft response exceeded the size limit.',
|
'Ubisoft response exceeded the size limit.',
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (!response.body) return '';
|
if (!response.body) return new Uint8Array();
|
||||||
const reader = response.body.getReader();
|
const reader = response.body.getReader();
|
||||||
const chunks: Uint8Array[] = [];
|
const chunks: Uint8Array[] = [];
|
||||||
let size = 0;
|
let size = 0;
|
||||||
@@ -52,7 +56,11 @@ async function readBoundedText(response: Response, maxBytes: number) {
|
|||||||
bytes.set(chunk, offset);
|
bytes.set(chunk, offset);
|
||||||
offset += chunk.byteLength;
|
offset += chunk.byteLength;
|
||||||
}
|
}
|
||||||
return new TextDecoder().decode(bytes);
|
return bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readBoundedText(response: Response, maxBytes: number) {
|
||||||
|
return new TextDecoder().decode(await readBoundedBytes(response, maxBytes));
|
||||||
}
|
}
|
||||||
|
|
||||||
export function extractNimbusApiKey(html: string) {
|
export function extractNimbusApiKey(html: string) {
|
||||||
@@ -144,6 +152,95 @@ function canonicalUrl(externalId: string, title: string, candidate: unknown) {
|
|||||||
return url.toString();
|
return url.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function validateUbisoftThumbnailUrl(value: unknown) {
|
||||||
|
if (typeof value !== 'string') {
|
||||||
|
throw new UbisoftNewsSourceError(
|
||||||
|
'Ubisoft returned an invalid news thumbnail.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let url: URL;
|
||||||
|
try {
|
||||||
|
url = new URL(value);
|
||||||
|
} catch {
|
||||||
|
throw new UbisoftNewsSourceError(
|
||||||
|
'Ubisoft returned an invalid news thumbnail.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
url.origin !== UBISOFT_IMAGE_ORIGIN ||
|
||||||
|
url.username ||
|
||||||
|
url.password ||
|
||||||
|
value.length > 2_000
|
||||||
|
) {
|
||||||
|
throw new UbisoftNewsSourceError(
|
||||||
|
'Ubisoft news thumbnail was outside the approved image host.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
url.hash = '';
|
||||||
|
return url.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function thumbnail(value: unknown, title: string) {
|
||||||
|
if (!isRecord(value)) {
|
||||||
|
throw new UbisoftNewsSourceError(
|
||||||
|
'Ubisoft returned an invalid news thumbnail.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const url = validateUbisoftThumbnailUrl(value.url);
|
||||||
|
const description = normalizeText(value.description, 300, false);
|
||||||
|
if (description === null) {
|
||||||
|
throw new UbisoftNewsSourceError(
|
||||||
|
'Ubisoft returned an invalid news thumbnail.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
url,
|
||||||
|
alt: description || title,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchUbisoftNewsThumbnail(
|
||||||
|
sourceUrl: string,
|
||||||
|
fetchImpl: typeof fetch = fetch,
|
||||||
|
) {
|
||||||
|
const url = validateUbisoftThumbnailUrl(sourceUrl);
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
response = await fetchImpl(url, {
|
||||||
|
headers: { accept: 'image/avif,image/webp,image/png,image/jpeg' },
|
||||||
|
redirect: 'error',
|
||||||
|
signal: AbortSignal.timeout(10_000),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
throw new UbisoftNewsSourceError('Ubisoft thumbnail request failed.');
|
||||||
|
}
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new UbisoftNewsSourceError(
|
||||||
|
`Ubisoft thumbnail request returned HTTP ${response.status}.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const contentType = response.headers
|
||||||
|
.get('content-type')
|
||||||
|
?.split(';', 1)[0]
|
||||||
|
?.trim()
|
||||||
|
.toLocaleLowerCase('en-US');
|
||||||
|
if (
|
||||||
|
!contentType ||
|
||||||
|
!['image/avif', 'image/jpeg', 'image/png', 'image/webp'].includes(
|
||||||
|
contentType,
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
throw new UbisoftNewsSourceError(
|
||||||
|
'Ubisoft thumbnail response was not a supported image.',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const body = await readBoundedBytes(response, MAX_THUMBNAIL_BYTES);
|
||||||
|
if (!body.byteLength) {
|
||||||
|
throw new UbisoftNewsSourceError('Ubisoft thumbnail response was empty.');
|
||||||
|
}
|
||||||
|
return { body, contentType };
|
||||||
|
}
|
||||||
|
|
||||||
function parseItem(value: unknown): UbisoftNewsItem {
|
function parseItem(value: unknown): UbisoftNewsItem {
|
||||||
if (!isRecord(value))
|
if (!isRecord(value))
|
||||||
throw new UbisoftNewsSourceError('Ubisoft returned an invalid news item.');
|
throw new UbisoftNewsSourceError('Ubisoft returned an invalid news item.');
|
||||||
@@ -162,11 +259,14 @@ function parseItem(value: unknown): UbisoftNewsItem {
|
|||||||
throw new UbisoftNewsSourceError('Ubisoft returned an invalid news item.');
|
throw new UbisoftNewsSourceError('Ubisoft returned an invalid news item.');
|
||||||
}
|
}
|
||||||
const button = isRecord(value.button) ? value.button.buttonUrl : null;
|
const button = isRecord(value.button) ? value.button.buttonUrl : null;
|
||||||
|
const image = thumbnail(value.thumbnail, title);
|
||||||
return {
|
return {
|
||||||
externalId,
|
externalId,
|
||||||
title,
|
title,
|
||||||
excerpt,
|
excerpt,
|
||||||
canonicalUrl: canonicalUrl(externalId, title, button),
|
canonicalUrl: canonicalUrl(externalId, title, button),
|
||||||
|
thumbnailUrl: image.url,
|
||||||
|
thumbnailAlt: image.alt,
|
||||||
publishedAt: published.toISOString(),
|
publishedAt: published.toISOString(),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,11 @@ import {
|
|||||||
const externalId = '64f1234567890abcdef12345';
|
const externalId = '64f1234567890abcdef12345';
|
||||||
const sourceDate = '2024-02-20T10:30:00.000Z';
|
const sourceDate = '2024-02-20T10:30:00.000Z';
|
||||||
|
|
||||||
function fetchSource(excerpt: string, publishedAt = sourceDate) {
|
function fetchSource(
|
||||||
|
excerpt: string,
|
||||||
|
publishedAt = sourceDate,
|
||||||
|
thumbnailUrl = 'https://staticctf.ubisoft.com/news/cover.jpg',
|
||||||
|
) {
|
||||||
return vi
|
return vi
|
||||||
.fn<typeof fetch>()
|
.fn<typeof fetch>()
|
||||||
.mockResolvedValueOnce(
|
.mockResolvedValueOnce(
|
||||||
@@ -28,6 +32,10 @@ function fetchSource(excerpt: string, publishedAt = sourceDate) {
|
|||||||
abstract: excerpt,
|
abstract: excerpt,
|
||||||
date: publishedAt,
|
date: publishedAt,
|
||||||
button: { buttonUrl: `/${externalId}/official-news` },
|
button: { buttonUrl: `/${externalId}/official-news` },
|
||||||
|
thumbnail: {
|
||||||
|
url: thumbnailUrl,
|
||||||
|
description: 'Official source artwork',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
}),
|
}),
|
||||||
@@ -65,7 +73,8 @@ async function createDatabase() {
|
|||||||
);
|
);
|
||||||
CREATE TABLE external_news_sources (
|
CREATE TABLE external_news_sources (
|
||||||
article_id TEXT PRIMARY KEY, provider TEXT NOT NULL, external_id TEXT NOT NULL, canonical_url TEXT NOT NULL,
|
article_id TEXT PRIMARY KEY, provider TEXT NOT NULL, external_id TEXT NOT NULL, canonical_url TEXT NOT NULL,
|
||||||
source_published_at TEXT NOT NULL, content_hash TEXT NOT NULL, first_seen_at TEXT NOT NULL,
|
thumbnail_url TEXT, thumbnail_alt TEXT NOT NULL DEFAULT '', source_published_at TEXT NOT NULL,
|
||||||
|
content_hash TEXT NOT NULL, first_seen_at TEXT NOT NULL,
|
||||||
last_seen_at TEXT NOT NULL, last_synced_at TEXT NOT NULL, UNIQUE(provider, external_id)
|
last_seen_at TEXT NOT NULL, last_synced_at TEXT NOT NULL, UNIQUE(provider, external_id)
|
||||||
);
|
);
|
||||||
CREATE TABLE external_news_sync_state (
|
CREATE TABLE external_news_sync_state (
|
||||||
@@ -125,6 +134,10 @@ describe('Ubisoft News import', () => {
|
|||||||
await database.prepare('SELECT * FROM external_news_sources')
|
await database.prepare('SELECT * FROM external_news_sources')
|
||||||
).get();
|
).get();
|
||||||
expect(source?.source_published_at).toBe(sourceDate);
|
expect(source?.source_published_at).toBe(sourceDate);
|
||||||
|
expect(source?.thumbnail_url).toBe(
|
||||||
|
'https://staticctf.ubisoft.com/news/cover.jpg',
|
||||||
|
);
|
||||||
|
expect(source?.thumbnail_alt).toBe('Official source artwork');
|
||||||
const audit = await (
|
const audit = await (
|
||||||
await database.prepare('SELECT * FROM audit_log')
|
await database.prepare('SELECT * FROM audit_log')
|
||||||
).get();
|
).get();
|
||||||
@@ -135,6 +148,7 @@ describe('Ubisoft News import', () => {
|
|||||||
});
|
});
|
||||||
expect(JSON.parse(String(audit?.after_json))).toMatchObject({
|
expect(JSON.parse(String(audit?.after_json))).toMatchObject({
|
||||||
externalId,
|
externalId,
|
||||||
|
thumbnailUrl: 'https://staticctf.ubisoft.com/news/cover.jpg',
|
||||||
sourcePublishedAt: sourceDate,
|
sourcePublishedAt: sourceDate,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -230,6 +244,105 @@ describe('Ubisoft News import', () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('synchronizes a changed Ubisoft thumbnail', async () => {
|
||||||
|
await syncUbisoftNews({
|
||||||
|
database,
|
||||||
|
fetchImpl: fetchSource('Stable excerpt.'),
|
||||||
|
now: new Date('2026-08-01T12:00:00.000Z'),
|
||||||
|
force: true,
|
||||||
|
});
|
||||||
|
const result = await syncUbisoftNews({
|
||||||
|
database,
|
||||||
|
fetchImpl: fetchSource(
|
||||||
|
'Stable excerpt.',
|
||||||
|
sourceDate,
|
||||||
|
'https://staticctf.ubisoft.com/news/revised-cover.jpg',
|
||||||
|
),
|
||||||
|
now: new Date('2026-08-01T14:00:00.000Z'),
|
||||||
|
force: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ status: 'ok', updated: 1 });
|
||||||
|
const source = await (
|
||||||
|
await database.prepare(
|
||||||
|
'SELECT thumbnail_url,thumbnail_alt FROM external_news_sources',
|
||||||
|
)
|
||||||
|
).get();
|
||||||
|
expect(source).toMatchObject({
|
||||||
|
thumbnail_url: 'https://staticctf.ubisoft.com/news/revised-cover.jpg',
|
||||||
|
thumbnail_alt: 'Official source artwork',
|
||||||
|
});
|
||||||
|
expect(
|
||||||
|
Number(
|
||||||
|
(
|
||||||
|
await (
|
||||||
|
await database.prepare(
|
||||||
|
'SELECT COUNT(*) AS count FROM article_revisions',
|
||||||
|
)
|
||||||
|
).get()
|
||||||
|
)?.count,
|
||||||
|
),
|
||||||
|
).toBe(1);
|
||||||
|
const audit = await (
|
||||||
|
await database.prepare(
|
||||||
|
"SELECT after_json FROM audit_log WHERE action='news.import_synced' ORDER BY created_at DESC LIMIT 1",
|
||||||
|
)
|
||||||
|
).get();
|
||||||
|
expect(JSON.parse(String(audit?.after_json))).toMatchObject({
|
||||||
|
thumbnailUrl: 'https://staticctf.ubisoft.com/news/revised-cover.jpg',
|
||||||
|
sourceMetadataOnly: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('audits source changes without restoring a deleted article', async () => {
|
||||||
|
await syncUbisoftNews({
|
||||||
|
database,
|
||||||
|
fetchImpl: fetchSource('First excerpt.'),
|
||||||
|
now: new Date('2026-08-01T12:00:00.000Z'),
|
||||||
|
force: true,
|
||||||
|
});
|
||||||
|
await (
|
||||||
|
await database.prepare(
|
||||||
|
"UPDATE articles SET deleted_at='2026-08-01T13:00:00.000Z'",
|
||||||
|
)
|
||||||
|
).run();
|
||||||
|
|
||||||
|
const result = await syncUbisoftNews({
|
||||||
|
database,
|
||||||
|
fetchImpl: fetchSource('Changed after deletion.'),
|
||||||
|
now: new Date('2026-08-01T14:00:00.000Z'),
|
||||||
|
force: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(result).toMatchObject({ status: 'ok', updated: 1 });
|
||||||
|
const article = await (
|
||||||
|
await database.prepare(
|
||||||
|
'SELECT deleted_at,published_revision_id FROM articles',
|
||||||
|
)
|
||||||
|
).get();
|
||||||
|
expect(article?.deleted_at).toBe('2026-08-01T13:00:00.000Z');
|
||||||
|
expect(
|
||||||
|
Number(
|
||||||
|
(
|
||||||
|
await (
|
||||||
|
await database.prepare(
|
||||||
|
'SELECT COUNT(*) AS count FROM article_revisions',
|
||||||
|
)
|
||||||
|
).get()
|
||||||
|
)?.count,
|
||||||
|
),
|
||||||
|
).toBe(1);
|
||||||
|
const audit = await (
|
||||||
|
await database.prepare(
|
||||||
|
"SELECT after_json FROM audit_log WHERE action='news.import_synced' ORDER BY created_at DESC LIMIT 1",
|
||||||
|
)
|
||||||
|
).get();
|
||||||
|
expect(JSON.parse(String(audit?.after_json))).toMatchObject({
|
||||||
|
articleDeleted: true,
|
||||||
|
sourceMetadataOnly: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
it('skips network work until the persisted next-check time', async () => {
|
it('skips network work until the persisted next-check time', async () => {
|
||||||
const now = new Date('2026-08-01T12:00:00.000Z');
|
const now = new Date('2026-08-01T12:00:00.000Z');
|
||||||
await syncUbisoftNews({
|
await syncUbisoftNews({
|
||||||
|
|||||||
@@ -96,6 +96,60 @@ function revisionSnapshot(item: UbisoftNewsItem, document: ArticleDocument) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function syncSourceMetadata(
|
||||||
|
database: Database,
|
||||||
|
existing: Record<string, unknown>,
|
||||||
|
item: UbisoftNewsItem,
|
||||||
|
contentHash: string,
|
||||||
|
now: string,
|
||||||
|
) {
|
||||||
|
const articleId = String(existing.article_id);
|
||||||
|
await (
|
||||||
|
await database.prepare(`UPDATE external_news_sources SET canonical_url=?,thumbnail_url=?,thumbnail_alt=?,
|
||||||
|
source_published_at=?,content_hash=?,last_seen_at=?,last_synced_at=? WHERE article_id=?`)
|
||||||
|
).run(
|
||||||
|
item.canonicalUrl,
|
||||||
|
item.thumbnailUrl,
|
||||||
|
item.thumbnailAlt,
|
||||||
|
item.publishedAt,
|
||||||
|
contentHash,
|
||||||
|
now,
|
||||||
|
now,
|
||||||
|
articleId,
|
||||||
|
);
|
||||||
|
await recordIsacAudit({
|
||||||
|
actorId: UBISOFT_NEWS_SERVICE_USER_ID,
|
||||||
|
category: 'editorial_content',
|
||||||
|
action: 'news.import_synced',
|
||||||
|
entityId: articleId,
|
||||||
|
database,
|
||||||
|
createdAt: now,
|
||||||
|
before: {
|
||||||
|
contentHash: String(existing.content_hash),
|
||||||
|
canonicalUrl: String(existing.canonical_url),
|
||||||
|
thumbnailUrl: existing.thumbnail_url
|
||||||
|
? String(existing.thumbnail_url)
|
||||||
|
: null,
|
||||||
|
thumbnailAlt: String(existing.thumbnail_alt ?? ''),
|
||||||
|
sourcePublishedAt: String(existing.source_published_at),
|
||||||
|
publishedRevisionId: existing.published_revision_id ?? null,
|
||||||
|
},
|
||||||
|
after: {
|
||||||
|
provider: UBISOFT_NEWS_PROVIDER,
|
||||||
|
externalId: item.externalId,
|
||||||
|
canonicalUrl: item.canonicalUrl,
|
||||||
|
thumbnailUrl: item.thumbnailUrl,
|
||||||
|
thumbnailAlt: item.thumbnailAlt,
|
||||||
|
sourcePublishedAt: item.publishedAt,
|
||||||
|
contentHash,
|
||||||
|
publishedRevisionId: existing.published_revision_id ?? null,
|
||||||
|
sourceMetadataOnly: true,
|
||||||
|
articleDeleted: Boolean(existing.deleted_at),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return 'updated' as const;
|
||||||
|
}
|
||||||
|
|
||||||
async function importItem(
|
async function importItem(
|
||||||
database: Database,
|
database: Database,
|
||||||
item: UbisoftNewsItem,
|
item: UbisoftNewsItem,
|
||||||
@@ -110,7 +164,13 @@ async function importItem(
|
|||||||
WHERE source.provider=? AND source.external_id=?`)
|
WHERE source.provider=? AND source.external_id=?`)
|
||||||
).get(UBISOFT_NEWS_PROVIDER, item.externalId);
|
).get(UBISOFT_NEWS_PROVIDER, item.externalId);
|
||||||
|
|
||||||
if (existing && String(existing.content_hash) === contentHash) {
|
const contentUnchanged =
|
||||||
|
existing && String(existing.content_hash) === contentHash;
|
||||||
|
const thumbnailUnchanged =
|
||||||
|
existing &&
|
||||||
|
String(existing.thumbnail_url ?? '') === item.thumbnailUrl &&
|
||||||
|
String(existing.thumbnail_alt ?? '') === item.thumbnailAlt;
|
||||||
|
if (contentUnchanged && thumbnailUnchanged) {
|
||||||
await (
|
await (
|
||||||
await database.prepare(
|
await database.prepare(
|
||||||
'UPDATE external_news_sources SET last_seen_at=? WHERE provider=? AND external_id=?',
|
'UPDATE external_news_sources SET last_seen_at=? WHERE provider=? AND external_id=?',
|
||||||
@@ -118,6 +178,9 @@ async function importItem(
|
|||||||
).run(now, UBISOFT_NEWS_PROVIDER, item.externalId);
|
).run(now, UBISOFT_NEWS_PROVIDER, item.externalId);
|
||||||
return 'unchanged' as const;
|
return 'unchanged' as const;
|
||||||
}
|
}
|
||||||
|
if (existing && (contentUnchanged || existing.deleted_at)) {
|
||||||
|
return syncSourceMetadata(database, existing, item, contentHash, now);
|
||||||
|
}
|
||||||
|
|
||||||
const document = articleDocument(item);
|
const document = articleDocument(item);
|
||||||
const parsed = parseArticleDocument(document);
|
const parsed = parseArticleDocument(document);
|
||||||
@@ -181,13 +244,16 @@ async function importItem(
|
|||||||
).run(articleId, item.title, item.excerpt, parsed.plainText, NEWS_CHANNEL);
|
).run(articleId, item.title, item.excerpt, parsed.plainText, NEWS_CHANNEL);
|
||||||
await (
|
await (
|
||||||
await database.prepare(`INSERT INTO external_news_sources(
|
await database.prepare(`INSERT INTO external_news_sources(
|
||||||
article_id,provider,external_id,canonical_url,source_published_at,content_hash,first_seen_at,last_seen_at,last_synced_at
|
article_id,provider,external_id,canonical_url,thumbnail_url,thumbnail_alt,source_published_at,
|
||||||
) VALUES(?,?,?,?,?,?,?,?,?)`)
|
content_hash,first_seen_at,last_seen_at,last_synced_at
|
||||||
|
) VALUES(?,?,?,?,?,?,?,?,?,?,?)`)
|
||||||
).run(
|
).run(
|
||||||
articleId,
|
articleId,
|
||||||
UBISOFT_NEWS_PROVIDER,
|
UBISOFT_NEWS_PROVIDER,
|
||||||
item.externalId,
|
item.externalId,
|
||||||
item.canonicalUrl,
|
item.canonicalUrl,
|
||||||
|
item.thumbnailUrl,
|
||||||
|
item.thumbnailAlt,
|
||||||
item.publishedAt,
|
item.publishedAt,
|
||||||
contentHash,
|
contentHash,
|
||||||
now,
|
now,
|
||||||
@@ -205,6 +271,8 @@ async function importItem(
|
|||||||
provider: UBISOFT_NEWS_PROVIDER,
|
provider: UBISOFT_NEWS_PROVIDER,
|
||||||
externalId: item.externalId,
|
externalId: item.externalId,
|
||||||
canonicalUrl: item.canonicalUrl,
|
canonicalUrl: item.canonicalUrl,
|
||||||
|
thumbnailUrl: item.thumbnailUrl,
|
||||||
|
thumbnailAlt: item.thumbnailAlt,
|
||||||
sourcePublishedAt: item.publishedAt,
|
sourcePublishedAt: item.publishedAt,
|
||||||
publishedRevisionId: revisionId,
|
publishedRevisionId: revisionId,
|
||||||
snapshotHash,
|
snapshotHash,
|
||||||
@@ -214,20 +282,6 @@ async function importItem(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const articleId = String(existing.article_id);
|
const articleId = String(existing.article_id);
|
||||||
if (existing.deleted_at) {
|
|
||||||
await (
|
|
||||||
await database.prepare(`UPDATE external_news_sources SET canonical_url=?,source_published_at=?,content_hash=?,
|
|
||||||
last_seen_at=?,last_synced_at=? WHERE article_id=?`)
|
|
||||||
).run(
|
|
||||||
item.canonicalUrl,
|
|
||||||
item.publishedAt,
|
|
||||||
contentHash,
|
|
||||||
now,
|
|
||||||
now,
|
|
||||||
articleId,
|
|
||||||
);
|
|
||||||
return 'unchanged' as const;
|
|
||||||
}
|
|
||||||
const next = await (
|
const next = await (
|
||||||
await database.prepare(
|
await database.prepare(
|
||||||
'SELECT COALESCE(MAX(revision_number),0)+1 AS number FROM article_revisions WHERE article_id=?',
|
'SELECT COALESCE(MAX(revision_number),0)+1 AS number FROM article_revisions WHERE article_id=?',
|
||||||
@@ -287,9 +341,18 @@ async function importItem(
|
|||||||
)
|
)
|
||||||
).run(articleId, item.title, item.excerpt, parsed.plainText, NEWS_CHANNEL);
|
).run(articleId, item.title, item.excerpt, parsed.plainText, NEWS_CHANNEL);
|
||||||
await (
|
await (
|
||||||
await database.prepare(`UPDATE external_news_sources SET canonical_url=?,source_published_at=?,content_hash=?,
|
await database.prepare(`UPDATE external_news_sources SET canonical_url=?,thumbnail_url=?,thumbnail_alt=?,
|
||||||
last_seen_at=?,last_synced_at=? WHERE article_id=?`)
|
source_published_at=?,content_hash=?,last_seen_at=?,last_synced_at=? WHERE article_id=?`)
|
||||||
).run(item.canonicalUrl, item.publishedAt, contentHash, now, now, articleId);
|
).run(
|
||||||
|
item.canonicalUrl,
|
||||||
|
item.thumbnailUrl,
|
||||||
|
item.thumbnailAlt,
|
||||||
|
item.publishedAt,
|
||||||
|
contentHash,
|
||||||
|
now,
|
||||||
|
now,
|
||||||
|
articleId,
|
||||||
|
);
|
||||||
await recordIsacAudit({
|
await recordIsacAudit({
|
||||||
actorId: UBISOFT_NEWS_SERVICE_USER_ID,
|
actorId: UBISOFT_NEWS_SERVICE_USER_ID,
|
||||||
category: 'editorial_content',
|
category: 'editorial_content',
|
||||||
@@ -300,6 +363,10 @@ async function importItem(
|
|||||||
before: {
|
before: {
|
||||||
contentHash: String(existing.content_hash),
|
contentHash: String(existing.content_hash),
|
||||||
canonicalUrl: String(existing.canonical_url),
|
canonicalUrl: String(existing.canonical_url),
|
||||||
|
thumbnailUrl: existing.thumbnail_url
|
||||||
|
? String(existing.thumbnail_url)
|
||||||
|
: null,
|
||||||
|
thumbnailAlt: String(existing.thumbnail_alt ?? ''),
|
||||||
sourcePublishedAt: String(existing.source_published_at),
|
sourcePublishedAt: String(existing.source_published_at),
|
||||||
publishedRevisionId: existing.published_revision_id ?? null,
|
publishedRevisionId: existing.published_revision_id ?? null,
|
||||||
},
|
},
|
||||||
@@ -307,6 +374,8 @@ async function importItem(
|
|||||||
provider: UBISOFT_NEWS_PROVIDER,
|
provider: UBISOFT_NEWS_PROVIDER,
|
||||||
externalId: item.externalId,
|
externalId: item.externalId,
|
||||||
canonicalUrl: item.canonicalUrl,
|
canonicalUrl: item.canonicalUrl,
|
||||||
|
thumbnailUrl: item.thumbnailUrl,
|
||||||
|
thumbnailAlt: item.thumbnailAlt,
|
||||||
sourcePublishedAt: item.publishedAt,
|
sourcePublishedAt: item.publishedAt,
|
||||||
contentHash,
|
contentHash,
|
||||||
publishedRevisionId: revisionId,
|
publishedRevisionId: revisionId,
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { error } from '@sveltejs/kit';
|
||||||
|
import { getDatabase } from '$lib/server/database';
|
||||||
|
import { fetchUbisoftNewsThumbnail } from '$lib/server/ubisoft-news-client';
|
||||||
|
import type { RequestHandler } from './$types';
|
||||||
|
|
||||||
|
const ID_PATTERN =
|
||||||
|
/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||||
|
const CACHE_TTL_MS = 60 * 60 * 1_000;
|
||||||
|
const CACHE_LIMIT_BYTES = 16 * 1024 * 1024;
|
||||||
|
type CachedImage = Awaited<ReturnType<typeof fetchUbisoftNewsThumbnail>> & {
|
||||||
|
sourceUrl: string;
|
||||||
|
etag: string;
|
||||||
|
expiresAt: number;
|
||||||
|
};
|
||||||
|
const cache = new Map<string, CachedImage>();
|
||||||
|
const pending = new Map<string, Promise<CachedImage>>();
|
||||||
|
let cacheBytes = 0;
|
||||||
|
|
||||||
|
function removeCached(id: string) {
|
||||||
|
const image = cache.get(id);
|
||||||
|
if (!image) return;
|
||||||
|
cache.delete(id);
|
||||||
|
cacheBytes -= image.body.byteLength;
|
||||||
|
}
|
||||||
|
|
||||||
|
function pruneCache(now: number) {
|
||||||
|
for (const [id, image] of cache) {
|
||||||
|
if (image.expiresAt <= now) removeCached(id);
|
||||||
|
}
|
||||||
|
while (cacheBytes > CACHE_LIMIT_BYTES) {
|
||||||
|
const oldest = cache.keys().next().value;
|
||||||
|
if (!oldest) break;
|
||||||
|
removeCached(oldest);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadImage(id: string, sourceUrl: string) {
|
||||||
|
const now = Date.now();
|
||||||
|
pruneCache(now);
|
||||||
|
const cached = cache.get(id);
|
||||||
|
if (cached?.sourceUrl === sourceUrl && cached.expiresAt > now) {
|
||||||
|
cache.delete(id);
|
||||||
|
cache.set(id, cached);
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
if (cached) removeCached(id);
|
||||||
|
const key = `${id}:${sourceUrl}`;
|
||||||
|
const active = pending.get(key);
|
||||||
|
if (active) return active;
|
||||||
|
|
||||||
|
const imageRequest = fetchUbisoftNewsThumbnail(sourceUrl).then((image) => {
|
||||||
|
const loaded: CachedImage = {
|
||||||
|
...image,
|
||||||
|
sourceUrl,
|
||||||
|
etag: `"${createHash('sha256').update(image.body).digest('hex')}"`,
|
||||||
|
expiresAt: Date.now() + CACHE_TTL_MS,
|
||||||
|
};
|
||||||
|
if (loaded.body.byteLength <= CACHE_LIMIT_BYTES) {
|
||||||
|
removeCached(id);
|
||||||
|
cache.set(id, loaded);
|
||||||
|
cacheBytes += loaded.body.byteLength;
|
||||||
|
pruneCache(Date.now());
|
||||||
|
}
|
||||||
|
return loaded;
|
||||||
|
});
|
||||||
|
pending.set(key, imageRequest);
|
||||||
|
try {
|
||||||
|
return await imageRequest;
|
||||||
|
} finally {
|
||||||
|
pending.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const GET: RequestHandler = async ({ params, request }) => {
|
||||||
|
if (!ID_PATTERN.test(params.id)) error(404, 'Image not found');
|
||||||
|
const row = await (
|
||||||
|
await (
|
||||||
|
await getDatabase()
|
||||||
|
).prepare(`SELECT source.thumbnail_url FROM external_news_sources source
|
||||||
|
JOIN articles ON articles.id=source.article_id
|
||||||
|
WHERE source.article_id=? AND source.provider='ubisoft' AND source.thumbnail_url IS NOT NULL
|
||||||
|
AND articles.visibility IN ('public','unlisted') AND articles.deleted_at IS NULL`)
|
||||||
|
).get(params.id);
|
||||||
|
if (!row?.thumbnail_url) error(404, 'Image not found');
|
||||||
|
|
||||||
|
let image: CachedImage;
|
||||||
|
try {
|
||||||
|
image = await loadImage(params.id, String(row.thumbnail_url));
|
||||||
|
} catch {
|
||||||
|
error(502, 'Source image unavailable');
|
||||||
|
}
|
||||||
|
const etag = image.etag;
|
||||||
|
if (
|
||||||
|
request.headers
|
||||||
|
.get('if-none-match')
|
||||||
|
?.split(',')
|
||||||
|
.some((value) => value.trim() === etag)
|
||||||
|
) {
|
||||||
|
return new Response(null, {
|
||||||
|
status: 304,
|
||||||
|
headers: { etag, 'cache-control': 'public, max-age=3600' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return new Response(image.body, {
|
||||||
|
headers: {
|
||||||
|
'cache-control': 'public, max-age=3600, stale-while-revalidate=86400',
|
||||||
|
'cross-origin-resource-policy': 'same-origin',
|
||||||
|
'content-length': String(image.body.byteLength),
|
||||||
|
'content-type': image.contentType,
|
||||||
|
etag,
|
||||||
|
'x-content-type-options': 'nosniff',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
};
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
const mocks = vi.hoisted(() => ({
|
||||||
|
get: vi.fn(),
|
||||||
|
fetchThumbnail: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('$lib/server/database', () => ({
|
||||||
|
getDatabase: vi.fn(async () => ({
|
||||||
|
prepare: vi.fn(async () => ({ get: mocks.get })),
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
|
||||||
|
vi.mock('$lib/server/ubisoft-news-client', () => ({
|
||||||
|
fetchUbisoftNewsThumbnail: mocks.fetchThumbnail,
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { GET } from './+server';
|
||||||
|
|
||||||
|
const articleId = '9890c398-0522-4e8c-8fdb-78de9c66deb6';
|
||||||
|
const sourceUrl = 'https://staticctf.ubisoft.com/news/cover.jpg';
|
||||||
|
|
||||||
|
describe('Ubisoft News image proxy', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
mocks.get.mockReset().mockResolvedValue({ thumbnail_url: sourceUrl });
|
||||||
|
mocks.fetchThumbnail.mockReset().mockResolvedValue({
|
||||||
|
body: new Uint8Array([1, 2, 3]),
|
||||||
|
contentType: 'image/jpeg',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('serves a same-origin cached image and honors its ETag', async () => {
|
||||||
|
const first = await GET({
|
||||||
|
params: { id: articleId },
|
||||||
|
request: new Request(`http://localhost/images/news/${articleId}`),
|
||||||
|
} as never);
|
||||||
|
expect(first.status).toBe(200);
|
||||||
|
expect(first.headers.get('content-type')).toBe('image/jpeg');
|
||||||
|
expect(first.headers.get('cross-origin-resource-policy')).toBe(
|
||||||
|
'same-origin',
|
||||||
|
);
|
||||||
|
const etag = first.headers.get('etag');
|
||||||
|
expect(etag).toBeTruthy();
|
||||||
|
|
||||||
|
const second = await GET({
|
||||||
|
params: { id: articleId },
|
||||||
|
request: new Request(`http://localhost/images/news/${articleId}`, {
|
||||||
|
headers: { 'if-none-match': String(etag) },
|
||||||
|
}),
|
||||||
|
} as never);
|
||||||
|
expect(second.status).toBe(304);
|
||||||
|
expect(mocks.fetchThumbnail).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user