Fix #218/#219/#220: identify ShadowBroker on Wikipedia + Wikidata calls (#284)

Wikimedia's User-Agent policy asks API clients to identify themselves
with a stable, contactable identifier so their operators can rate-limit
or coordinate. Before this change, ShadowBroker was sending:

- Backend (region_dossier.py): generic project default UA only; no
  Api-User-Agent.
- Frontend (useRegionDossier.ts, WikiImage.tsx, NewsFeed.tsx): zero
  identifying header at all; three separate copy-pasted anonymous
  fetches with their own module-local caches.

Three separate components doing the same broken thing meant policy
fixes had to happen in three places, with no shared cache or kill
switch.

Fix (no UX change, zero hostility):

== Backend ==

`backend/services/region_dossier.py` now sets explicit `User-Agent` +
`Api-User-Agent` headers on every outbound Wikidata and Wikipedia
request via a new `_WIKIMEDIA_REQUEST_HEADERS` constant. The identifier
includes a contact path (issues page on the public GitHub repo).

== Frontend ==

New shared helper `frontend/src/lib/wikimediaClient.ts`:
- `fetchWikipediaSummary(title)` — single source of truth for Wikipedia
  REST summary lookups, with one shared LRU cache (in-flight requests
  deduplicated, 512-entry cap), `Api-User-Agent` on every fetch.
- `fetchWikidataSparql(query)` — same shape for Wikidata SPARQL.
- `WIKIMEDIA_API_USER_AGENT` — exported constant; one place to update
  if Wikimedia ever asks us to back off.

Refactored three components to use the shared client:
- `frontend/src/hooks/useRegionDossier.ts` — fetchLeader() and
  fetchLocalWikiSummary() now route through the shared helpers.
- `frontend/src/components/WikiImage.tsx` — uses fetchWikipediaSummary,
  proper React state instead of module-mutation + forceUpdate trick.
- `frontend/src/components/NewsFeed.tsx` — same shape.

UX: byte-for-byte identical. Same thumbnails, same dossier content,
same load behavior. The only observable difference is the outgoing
request header.

Note on #239 (route duplication): an audit-grade inventory shows 166
main.py routes are shadowed by router modules. That cleanup is too
large to land safely in this PR; it will be staged as a separate
ladder of small PRs grouped by router module.

Tests:
- `backend/tests/test_region_dossier_wikimedia_ua.py` — 3 tests
  asserting backend headers are present.
- `frontend/src/__tests__/utils/wikimediaClient.test.ts` — 9 tests
  covering Api-User-Agent presence, shared cache, concurrent
  deduplication, disambiguation/HTTP-error/network-error fallthroughs,
  empty-input safety.

Local: backend 76/76 security suite green, frontend 716/716 vitest
suite green.

Credit: tg12 (external security audit).
This commit is contained in:
Shadowbroker
2026-05-21 10:48:05 -06:00
committed by GitHub
parent e125467721
commit 0fee36e8f7
7 changed files with 514 additions and 68 deletions
@@ -0,0 +1,164 @@
/**
* Issues #218 / #219 / #220 (tg12 external audit):
*
* Every browser-direct call to Wikipedia or Wikidata must send the
* `Api-User-Agent` header that Wikimedia's UA policy asks for. These
* tests pin that requirement on the shared `lib/wikimediaClient`
* helper that WikiImage, NewsFeed, and useRegionDossier all route
* through, so a future refactor that drops the header gets a loud
* test failure rather than a silent ToS regression.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
WIKIMEDIA_API_USER_AGENT,
fetchWikipediaSummary,
fetchWikidataSparql,
_resetWikimediaClientCacheForTests,
} from '@/lib/wikimediaClient';
const originalFetch = globalThis.fetch;
describe('lib/wikimediaClient', () => {
beforeEach(() => {
_resetWikimediaClientCacheForTests();
});
afterEach(() => {
globalThis.fetch = originalFetch;
vi.restoreAllMocks();
});
it('exposes a stable Api-User-Agent identifier with a contact path', () => {
expect(WIKIMEDIA_API_USER_AGENT).toContain('Shadowbroker');
expect(WIKIMEDIA_API_USER_AGENT.toLowerCase()).toContain('github.com');
expect(WIKIMEDIA_API_USER_AGENT.toLowerCase()).toContain('issues');
});
it('sends Api-User-Agent on Wikipedia summary fetch', async () => {
const calls: Array<{ url: string; init?: RequestInit }> = [];
globalThis.fetch = vi.fn(async (url: any, init?: RequestInit) => {
calls.push({ url: String(url), init });
return new Response(
JSON.stringify({
type: 'standard',
title: 'Boeing 747',
description: 'aircraft',
extract: 'long extract',
thumbnail: { source: 'https://example.org/thumb.jpg' },
}),
{ status: 200 },
);
}) as any;
const summary = await fetchWikipediaSummary('Boeing 747');
expect(summary?.thumbnail).toBe('https://example.org/thumb.jpg');
expect(calls).toHaveLength(1);
const headers = (calls[0].init?.headers || {}) as Record<string, string>;
expect(headers['Api-User-Agent']).toBe(WIKIMEDIA_API_USER_AGENT);
});
it('sends Api-User-Agent on Wikidata SPARQL fetch', async () => {
const calls: Array<{ url: string; init?: RequestInit }> = [];
globalThis.fetch = vi.fn(async (url: any, init?: RequestInit) => {
calls.push({ url: String(url), init });
return new Response(
JSON.stringify({
results: {
bindings: [
{
leaderLabel: { value: 'Test Leader' },
govTypeLabel: { value: 'Test Government' },
},
],
},
}),
{ status: 200 },
);
}) as any;
const bindings = await fetchWikidataSparql('SELECT * WHERE { ?s ?p ?o }');
expect(bindings).toHaveLength(1);
const headers = (calls[0].init?.headers || {}) as Record<string, string>;
expect(headers['Api-User-Agent']).toBe(WIKIMEDIA_API_USER_AGENT);
expect(headers['Accept']).toBe('application/sparql-results+json');
});
it('shares cache across consecutive callers for the same Wikipedia title', async () => {
let fetchCount = 0;
globalThis.fetch = vi.fn(async () => {
fetchCount++;
return new Response(
JSON.stringify({
type: 'standard',
title: 'Eiffel Tower',
description: 'iron lattice tower',
extract: '...',
thumbnail: { source: 'https://example.org/eiffel.jpg' },
}),
{ status: 200 },
);
}) as any;
const a = await fetchWikipediaSummary('Eiffel Tower');
const b = await fetchWikipediaSummary('Eiffel Tower');
expect(fetchCount).toBe(1);
expect(a?.thumbnail).toBe(b?.thumbnail);
});
it('deduplicates concurrent in-flight requests for the same title', async () => {
let fetchCount = 0;
globalThis.fetch = vi.fn(async () => {
fetchCount++;
await new Promise((r) => setTimeout(r, 5));
return new Response(
JSON.stringify({
type: 'standard',
title: 'Mount Fuji',
description: 'stratovolcano',
extract: '...',
thumbnail: { source: 'https://example.org/fuji.jpg' },
}),
{ status: 200 },
);
}) as any;
const [a, b, c] = await Promise.all([
fetchWikipediaSummary('Mount Fuji'),
fetchWikipediaSummary('Mount Fuji'),
fetchWikipediaSummary('Mount Fuji'),
]);
expect(fetchCount).toBe(1);
expect(a?.thumbnail).toBe('https://example.org/fuji.jpg');
expect(b).toEqual(a);
expect(c).toEqual(a);
});
it('returns null on disambiguation pages without throwing', async () => {
globalThis.fetch = vi.fn(async () =>
new Response(JSON.stringify({ type: 'disambiguation' }), { status: 200 }),
) as any;
const summary = await fetchWikipediaSummary('Mercury');
expect(summary).toBeNull();
});
it('returns null on HTTP error without throwing', async () => {
globalThis.fetch = vi.fn(async () => new Response('not found', { status: 404 })) as any;
const summary = await fetchWikipediaSummary('Nonexistent Article 12345');
expect(summary).toBeNull();
});
it('returns null on network error without throwing', async () => {
globalThis.fetch = vi.fn(async () => {
throw new Error('network down');
}) as any;
const summary = await fetchWikipediaSummary('Anything');
expect(summary).toBeNull();
});
it('returns null on empty input', async () => {
globalThis.fetch = vi.fn(async () => new Response('{}', { status: 200 })) as any;
expect(await fetchWikipediaSummary('')).toBeNull();
expect(await fetchWikipediaSummary(' ')).toBeNull();
expect(globalThis.fetch).not.toHaveBeenCalled();
});
});