CalvinBackup/Shadowbroker
1
0
Fork 0
mirror of https://github.com/BigBodyCobain/Shadowbroker.git synced 2026-08-10 20:50:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

Route dossier, geocode, and Wikimedia through the backend (#351, #352, #360)

Browse Source 
Proxy region dossier, Sentinel search, Wikipedia, and Wikidata via self-hosted
APIs; remove LocateBar client-side Nominatim fallback; migrate legacy shadow-
operator handles to operator- prefix.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
BigBodyCobain
2026-06-02 15:20:44 -06:00
co-authored by Cursor
parent c3dd95f6a9
commit f08781bdc9
9 changed files with 250 additions and 622 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/__tests__/page/pageDecomposition.test.ts
+13
View File
@@ -203,6 +203,19 @@ describe('page.tsx decomposition — no admin-session/proxy regression', () => {
const locateBar = readAppFile('LocateBar.tsx');
expect(locateBar).toContain('API_BASE');
expect(locateBar).toContain('/api/geocode/search');
expect(locateBar).not.toContain('nominatim.openstreetmap.org');
});
it('useRegionDossier uses backend dossier APIs (no browser-direct enrichment)', () => {
const hook = fs.readFileSync(
path.resolve(__dirname, '../../hooks/useRegionDossier.ts'),
'utf-8',
);
expect(hook).toContain('/api/region-dossier');
expect(hook).toContain('/api/sentinel2/search');
expect(hook).not.toContain('nominatim.openstreetmap.org');
expect(hook).not.toContain('planetarycomputer.microsoft.com');
expect(hook).not.toContain('restcountries.com');
});
});
frontend/src/__tests__/utils/wikimediaClient.test.ts
+36 -177
View File
@@ -1,21 +1,8 @@
/**
* Issues #218 / #219 / #220 (tg12 external audit) + Round 7a:
*
* Every browser-direct call to Wikipedia or Wikidata must send the
* `Api-User-Agent` header that Wikimedia's UA policy asks for, AND must
* embed the per-install operator handle so Wikimedia can rate-limit /
* contact the specific operator instead of treating "Shadowbroker" as
* one giant entity.
*
* These tests pin both requirements on the shared `lib/wikimediaClient`
* helper that WikiImage, NewsFeed, and useRegionDossier all route
* through. A future refactor that drops either the header OR the
* per-operator handle gets a loud test failure rather than a silent
* ToS / privacy regression.
* #360: Wikipedia / Wikidata traffic is proxied via the self-hosted backend.
*/
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
buildWikimediaUserAgent,
fetchWikipediaSummary,
fetchWikidataSparql,
_resetWikimediaClientCacheForTests,
@@ -23,18 +10,6 @@ import {
const originalFetch = globalThis.fetch;
// Helper: stub fetch so calls to /api/settings/operator-handle return a
// known handle, and everything else proxies to whatever the test set up.
function withHandle(handle: string, otherFetch: typeof globalThis.fetch) {
return vi.fn(async (input: any, init?: RequestInit) => {
const url = String(input);
if (url.endsWith('/api/settings/operator-handle')) {
return new Response(JSON.stringify({ handle }), { status: 200 });
}
return otherFetch(input, init);
});
}
describe('lib/wikimediaClient', () => {
beforeEach(() => {
_resetWikimediaClientCacheForTests();
@@ -45,194 +20,78 @@ describe('lib/wikimediaClient', () => {
vi.restoreAllMocks();
});
it('builds a stable per-operator Api-User-Agent with contact path', async () => {
globalThis.fetch = withHandle(
'operator-abc123',
vi.fn(async () => new Response('{}', { status: 200 })) as any,
) as any;
const ua = await buildWikimediaUserAgent('wikipedia-summary');
expect(ua).toContain('Shadowbroker');
expect(ua.toLowerCase()).toContain('github.com');
expect(ua.toLowerCase()).toContain('issues');
expect(ua).toContain('operator: operator-abc123');
expect(ua).toContain('purpose: wikipedia-summary');
});
it('falls back to "operator-offline" when handle endpoint is unreachable', async () => {
it('fetches Wikipedia summary through backend proxy', async () => {
const calls: string[] = [];
globalThis.fetch = vi.fn(async (input: any) => {
const url = String(input);
if (url.endsWith('/api/settings/operator-handle')) {
return new Response('forbidden', { status: 403 });
}
return new Response('{}', { status: 200 });
}) as any;
const ua = await buildWikimediaUserAgent('test');
expect(ua).toContain('operator: operator-offline');
});
it('sends per-operator Api-User-Agent on Wikipedia summary fetch', async () => {
const wikiCalls: Array<{ url: string; init?: RequestInit }> = [];
const baseFetch = vi.fn(async (url: any, init?: RequestInit) => {
wikiCalls.push({ url: String(url), init });
calls.push(String(input));
return new Response(
JSON.stringify({
type: 'standard',
title: 'Boeing 747',
description: 'aircraft',
extract: 'long extract',
thumbnail: { source: 'https://example.org/thumb.jpg' },
}),
{ status: 200 },
);
});
globalThis.fetch = withHandle('operator-test01', baseFetch as any) as any;
const summary = await fetchWikipediaSummary('Boeing 747');
expect(summary?.thumbnail).toBe('https://example.org/thumb.jpg');
// wikiCalls only captures calls to non-handle URLs.
expect(wikiCalls).toHaveLength(1);
const headers = (wikiCalls[0].init?.headers || {}) as Record<string, string>;
expect(headers['Api-User-Agent']).toContain('operator: operator-test01');
expect(headers['Api-User-Agent']).toContain('purpose: wikipedia-summary');
});
it('sends per-operator Api-User-Agent on Wikidata SPARQL fetch', async () => {
const calls: Array<{ url: string; init?: RequestInit }> = [];
const baseFetch = vi.fn(async (url: any, init?: RequestInit) => {
calls.push({ url: String(url), init });
return new Response(
JSON.stringify({
results: { bindings: [{ leaderLabel: { value: 'Test Leader' } }] },
}),
{ status: 200 },
);
});
globalThis.fetch = withHandle('operator-sparql', baseFetch as any) as any;
const bindings = await fetchWikidataSparql('SELECT * WHERE { ?s ?p ?o }');
expect(bindings).toHaveLength(1);
const headers = (calls[0].init?.headers || {}) as Record<string, string>;
expect(headers['Api-User-Agent']).toContain('operator: operator-sparql');
expect(headers['Api-User-Agent']).toContain('purpose: wikidata-sparql');
expect(headers['Accept']).toBe('application/sparql-results+json');
});
it('handle endpoint is queried only ONCE across many wiki fetches', async () => {
let handleCalls = 0;
let wikiCalls = 0;
globalThis.fetch = vi.fn(async (input: any) => {
const url = String(input);
if (url.endsWith('/api/settings/operator-handle')) {
handleCalls++;
return new Response(JSON.stringify({ handle: 'operator-cache' }), { status: 200 });
}
wikiCalls++;
return new Response(
JSON.stringify({
thumbnail: 'https://example.org/thumb.jpg',
type: 'standard',
title: 'X',
description: '',
extract: '',
thumbnail: { source: 'https://example.org/x.jpg' },
}),
{ status: 200 },
);
}) as any;
await fetchWikipediaSummary('Eiffel Tower');
await fetchWikipediaSummary('Mount Fuji');
await fetchWikipediaSummary('Statue of Liberty');
expect(handleCalls).toBe(1);
expect(wikiCalls).toBe(3);
const summary = await fetchWikipediaSummary('Boeing 747');
expect(summary?.thumbnail).toBe('https://example.org/thumb.jpg');
expect(calls).toHaveLength(1);
expect(calls[0]).toContain('/api/wikipedia/summary');
expect(calls[0]).not.toContain('wikipedia.org');
});
it('shares cache across consecutive callers for the same Wikipedia title', async () => {
let fetchCount = 0;
const baseFetch = vi.fn(async () => {
fetchCount++;
it('fetches Wikidata SPARQL through backend proxy', async () => {
const calls: Array<{ url: string; init?: RequestInit }> = [];
globalThis.fetch = vi.fn(async (url: any, init?: RequestInit) => {
calls.push({ url: String(url), init });
return new Response(
JSON.stringify({
type: 'standard',
title: 'Eiffel Tower',
description: 'iron lattice tower',
extract: '...',
thumbnail: { source: 'https://example.org/eiffel.jpg' },
bindings: [{ leaderLabel: { value: 'Test Leader' } }],
}),
{ status: 200 },
);
});
globalThis.fetch = withHandle('operator-cache', baseFetch as any) as any;
}) as any;
const a = await fetchWikipediaSummary('Eiffel Tower');
const b = await fetchWikipediaSummary('Eiffel Tower');
expect(fetchCount).toBe(1);
expect(a?.thumbnail).toBe(b?.thumbnail);
const bindings = await fetchWikidataSparql('SELECT * WHERE { ?s ?p ?o }');
expect(bindings).toHaveLength(1);
expect(calls).toHaveLength(1);
expect(calls[0].url).toContain('/api/wikidata/sparql');
expect(calls[0].init?.method).toBe('POST');
expect(calls[0].url).not.toContain('wikidata.org');
});
it('deduplicates concurrent in-flight requests for the same title', async () => {
let fetchCount = 0;
const baseFetch = vi.fn(async () => {
fetchCount++;
await new Promise((r) => setTimeout(r, 5));
it('deduplicates concurrent Wikipedia summary requests', async () => {
let hits = 0;
globalThis.fetch = vi.fn(async () => {
hits += 1;
return new Response(
JSON.stringify({
type: 'standard',
title: 'Mount Fuji',
description: 'stratovolcano',
extract: '...',
thumbnail: { source: 'https://example.org/fuji.jpg' },
description: 'mountain',
extract: 'extract',
thumbnail: '',
type: 'standard',
}),
{ status: 200 },
);
});
globalThis.fetch = withHandle('operator-cache', baseFetch as any) as any;
}) as any;
const [a, b, c] = await Promise.all([
fetchWikipediaSummary('Mount Fuji'),
fetchWikipediaSummary('Mount Fuji'),
fetchWikipediaSummary('Mount Fuji'),
]);
expect(fetchCount).toBe(1);
expect(a?.thumbnail).toBe('https://example.org/fuji.jpg');
expect(a?.title).toBe('Mount Fuji');
expect(b).toEqual(a);
expect(c).toEqual(a);
expect(hits).toBe(1);
});
it('returns null on disambiguation pages without throwing', async () => {
globalThis.fetch = withHandle(
'operator-cache',
vi.fn(async () =>
new Response(JSON.stringify({ type: 'disambiguation' }), { status: 200 }),
) as any,
) as any;
const summary = await fetchWikipediaSummary('Mercury');
expect(summary).toBeNull();
});
it('returns null on HTTP error without throwing', async () => {
globalThis.fetch = withHandle(
'operator-cache',
vi.fn(async () => new Response('not found', { status: 404 })) as any,
) as any;
const summary = await fetchWikipediaSummary('Nonexistent Article 12345');
expect(summary).toBeNull();
});
it('returns null on network error without throwing', async () => {
globalThis.fetch = withHandle(
'operator-cache',
vi.fn(async () => {
throw new Error('network down');
}) as any,
) as any;
const summary = await fetchWikipediaSummary('Anything');
expect(summary).toBeNull();
});
it('returns null on empty input without fetching anything', async () => {
globalThis.fetch = vi.fn(async () => new Response('{}', { status: 200 })) as any;
expect(await fetchWikipediaSummary('')).toBeNull();
expect(await fetchWikipediaSummary(' ')).toBeNull();
expect(globalThis.fetch).not.toHaveBeenCalled();
it('returns null on Wikipedia 404', async () => {
globalThis.fetch = vi.fn(async () => new Response('{}', { status: 404 })) as any;
expect(await fetchWikipediaSummary('Nonexistent Article 12345')).toBeNull();
});
});