remove spans, log geocoding shit bc of cost

This commit is contained in:
Will Freeman
2026-04-29 21:34:51 -06:00
parent d446cfcd15
commit ec8bb9cdca
10 changed files with 171 additions and 291 deletions
-9
View File
@@ -9,12 +9,9 @@
"@opentelemetry/api-logs": "^0.215.0", "@opentelemetry/api-logs": "^0.215.0",
"@opentelemetry/exporter-logs-otlp-http": "^0.215.0", "@opentelemetry/exporter-logs-otlp-http": "^0.215.0",
"@opentelemetry/exporter-metrics-otlp-http": "^0.215.0", "@opentelemetry/exporter-metrics-otlp-http": "^0.215.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.215.0",
"@opentelemetry/resources": "^2.7.0", "@opentelemetry/resources": "^2.7.0",
"@opentelemetry/sdk-logs": "^0.215.0", "@opentelemetry/sdk-logs": "^0.215.0",
"@opentelemetry/sdk-metrics": "^2.7.0", "@opentelemetry/sdk-metrics": "^2.7.0",
"@opentelemetry/sdk-trace-node": "^2.7.0",
"@opentelemetry/semantic-conventions": "^1.40.0",
"@sinclair/typebox": "^0.34.48", "@sinclair/typebox": "^0.34.48",
"cache-manager": "^7.2.8", "cache-manager": "^7.2.8",
"cache-manager-fs-hash": "^3.0.0", "cache-manager-fs-hash": "^3.0.0",
@@ -50,16 +47,12 @@
"@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.215.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-xrFlqhdhUyO8wSRn6DjE0145/HPWSJ5Nm0C7vWua6TdL/FSEAZvEyvdsa9CRXuxo9ebb7j/NEPhEcO62IJ0qUA=="], "@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.215.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-xrFlqhdhUyO8wSRn6DjE0145/HPWSJ5Nm0C7vWua6TdL/FSEAZvEyvdsa9CRXuxo9ebb7j/NEPhEcO62IJ0qUA=="],
"@opentelemetry/context-async-hooks": ["@opentelemetry/context-async-hooks@2.7.0", "", { "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-MWXggArM+Y11mPS8VOrqxOj+YMGQSRuvhM91eSBX4xFpJa05mpkeVvM8pPux5ElkEjV5RMgrkisrlP/R83SpBQ=="],
"@opentelemetry/core": ["@opentelemetry/core@2.7.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-DT12SXVwV2eoJrGf4nnsvZojxxeQo+LlNAsoYGRRObPWTeN6APiqZ2+nqDCQDvQX40eLi1AePONS0onoASp3yQ=="], "@opentelemetry/core": ["@opentelemetry/core@2.7.0", "", { "dependencies": { "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-DT12SXVwV2eoJrGf4nnsvZojxxeQo+LlNAsoYGRRObPWTeN6APiqZ2+nqDCQDvQX40eLi1AePONS0onoASp3yQ=="],
"@opentelemetry/exporter-logs-otlp-http": ["@opentelemetry/exporter-logs-otlp-http@0.215.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.215.0", "@opentelemetry/core": "2.7.0", "@opentelemetry/otlp-exporter-base": "0.215.0", "@opentelemetry/otlp-transformer": "0.215.0", "@opentelemetry/sdk-logs": "0.215.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-U7Qb+TVX2GZH5RSC+Gx9aE5zChKP1kPg87X3PlI/41lWVPJdBIzmgMmuE28MmQlrK84nLHCIqUOOben8YkSzBw=="], "@opentelemetry/exporter-logs-otlp-http": ["@opentelemetry/exporter-logs-otlp-http@0.215.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.215.0", "@opentelemetry/core": "2.7.0", "@opentelemetry/otlp-exporter-base": "0.215.0", "@opentelemetry/otlp-transformer": "0.215.0", "@opentelemetry/sdk-logs": "0.215.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-U7Qb+TVX2GZH5RSC+Gx9aE5zChKP1kPg87X3PlI/41lWVPJdBIzmgMmuE28MmQlrK84nLHCIqUOOben8YkSzBw=="],
"@opentelemetry/exporter-metrics-otlp-http": ["@opentelemetry/exporter-metrics-otlp-http@0.215.0", "", { "dependencies": { "@opentelemetry/core": "2.7.0", "@opentelemetry/otlp-exporter-base": "0.215.0", "@opentelemetry/otlp-transformer": "0.215.0", "@opentelemetry/resources": "2.7.0", "@opentelemetry/sdk-metrics": "2.7.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-FRydO5j7MWnXK9ghfykKxiSM8I5UeiicK/UNl3/mv86xoEKkb+LKz1I3WXgkuYVOQf22VNqbPO58s2W1mVWtEQ=="], "@opentelemetry/exporter-metrics-otlp-http": ["@opentelemetry/exporter-metrics-otlp-http@0.215.0", "", { "dependencies": { "@opentelemetry/core": "2.7.0", "@opentelemetry/otlp-exporter-base": "0.215.0", "@opentelemetry/otlp-transformer": "0.215.0", "@opentelemetry/resources": "2.7.0", "@opentelemetry/sdk-metrics": "2.7.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-FRydO5j7MWnXK9ghfykKxiSM8I5UeiicK/UNl3/mv86xoEKkb+LKz1I3WXgkuYVOQf22VNqbPO58s2W1mVWtEQ=="],
"@opentelemetry/exporter-trace-otlp-http": ["@opentelemetry/exporter-trace-otlp-http@0.215.0", "", { "dependencies": { "@opentelemetry/core": "2.7.0", "@opentelemetry/otlp-exporter-base": "0.215.0", "@opentelemetry/otlp-transformer": "0.215.0", "@opentelemetry/resources": "2.7.0", "@opentelemetry/sdk-trace-base": "2.7.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-k4J9ISeGpb0Bm/wCrlcrbroMFTkiWMrdhNxQGrlktxLy127Yzd4/7nrTawn5d/ApktYTknvdixsE6++34Qfi1w=="],
"@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.215.0", "", { "dependencies": { "@opentelemetry/core": "2.7.0", "@opentelemetry/otlp-transformer": "0.215.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-lHrfbmeLSmesGSkkHiqDwOzfaEMSWXdc7q6UoLfbW8byONCb+bE/zkAr0kapN4US1baT/2nbpNT7Cn9XoB96Vg=="], "@opentelemetry/otlp-exporter-base": ["@opentelemetry/otlp-exporter-base@0.215.0", "", { "dependencies": { "@opentelemetry/core": "2.7.0", "@opentelemetry/otlp-transformer": "0.215.0" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-lHrfbmeLSmesGSkkHiqDwOzfaEMSWXdc7q6UoLfbW8byONCb+bE/zkAr0kapN4US1baT/2nbpNT7Cn9XoB96Vg=="],
"@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.215.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.215.0", "@opentelemetry/core": "2.7.0", "@opentelemetry/resources": "2.7.0", "@opentelemetry/sdk-logs": "0.215.0", "@opentelemetry/sdk-metrics": "2.7.0", "@opentelemetry/sdk-trace-base": "2.7.0", "protobufjs": "^8.0.1" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-cWwBvaV+vkXHkSoTYR8hGw+AW03UlgTr6xtrUKOMeum3T+8vffYXIfXu6KY5MLu8O9QtoBKqaKWw9I5xoOepng=="], "@opentelemetry/otlp-transformer": ["@opentelemetry/otlp-transformer@0.215.0", "", { "dependencies": { "@opentelemetry/api-logs": "0.215.0", "@opentelemetry/core": "2.7.0", "@opentelemetry/resources": "2.7.0", "@opentelemetry/sdk-logs": "0.215.0", "@opentelemetry/sdk-metrics": "2.7.0", "@opentelemetry/sdk-trace-base": "2.7.0", "protobufjs": "^8.0.1" }, "peerDependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-cWwBvaV+vkXHkSoTYR8hGw+AW03UlgTr6xtrUKOMeum3T+8vffYXIfXu6KY5MLu8O9QtoBKqaKWw9I5xoOepng=="],
@@ -72,8 +65,6 @@
"@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.7.0", "", { "dependencies": { "@opentelemetry/core": "2.7.0", "@opentelemetry/resources": "2.7.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-Yg9zEXJB50DLVLpsKPk7NmNqlPlS+OvqhJGh0A8oawIOTPOwlm4eXs9BMJV7L79lvEwI+dWtAj+YjTyddV336A=="], "@opentelemetry/sdk-trace-base": ["@opentelemetry/sdk-trace-base@2.7.0", "", { "dependencies": { "@opentelemetry/core": "2.7.0", "@opentelemetry/resources": "2.7.0", "@opentelemetry/semantic-conventions": "^1.29.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.3.0 <1.10.0" } }, "sha512-Yg9zEXJB50DLVLpsKPk7NmNqlPlS+OvqhJGh0A8oawIOTPOwlm4eXs9BMJV7L79lvEwI+dWtAj+YjTyddV336A=="],
"@opentelemetry/sdk-trace-node": ["@opentelemetry/sdk-trace-node@2.7.0", "", { "dependencies": { "@opentelemetry/context-async-hooks": "2.7.0", "@opentelemetry/core": "2.7.0", "@opentelemetry/sdk-trace-base": "2.7.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-RrFHOXw0IYp/OThew6QORdybnnLitUAUMCJKcQNBYS0hDkCYarO2vTkVxfrGxCIqd5XHSMvbCpBd/T8ZMw8oSg=="],
"@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.40.0", "", {}, "sha512-cifvXDhcqMwwTlTK04GBNeIe7yyo28Mfby85QXFe1Yk8nmi36Ab/5UQwptOx84SsoGNRg+EVSjwzfSZMy6pmlw=="], "@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.40.0", "", {}, "sha512-cifvXDhcqMwwTlTK04GBNeIe7yyo28Mfby85QXFe1Yk8nmi36Ab/5UQwptOx84SsoGNRg+EVSjwzfSZMy6pmlw=="],
"@pinojs/redact": ["@pinojs/redact@0.4.0", "", {}, "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg=="], "@pinojs/redact": ["@pinojs/redact@0.4.0", "", {}, "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg=="],
+2 -20
View File
@@ -1,6 +1,6 @@
# OpenTelemetry Collector configuration for deflock-api # OpenTelemetry Collector configuration for deflock-api
# Receives traces and logs from the API, forwards to Grafana Cloud via the # Receives logs and metrics from the API, forwards to Grafana Cloud via the
# unified OTLP gateway (routes traces → Tempo, logs → Loki automatically). # unified OTLP gateway (routes logs → Loki, metrics → Mimir automatically).
# #
# Required environment variables (set in /etc/systemd/system/otelcol.service): # Required environment variables (set in /etc/systemd/system/otelcol.service):
# GRAFANA_OTLP_ENDPOINT e.g. https://otlp-gateway-prod-us-east-0.grafana.net/otlp # GRAFANA_OTLP_ENDPOINT e.g. https://otlp-gateway-prod-us-east-0.grafana.net/otlp
@@ -19,20 +19,6 @@ processors:
timeout: 5s timeout: 5s
send_batch_size: 256 send_batch_size: 256
# Drop healthcheck spans to reduce noise
filter/drop_healthcheck:
error_mode: ignore
traces:
span:
- 'attributes["http.route"] == "/healthcheck"'
# Only forward spans that resulted in an error
filter/errors_only:
error_mode: ignore
traces:
span:
- 'status.code != STATUS_CODE_ERROR'
# Drop 404 log records to reduce noise # Drop 404 log records to reduce noise
filter/drop_404_logs: filter/drop_404_logs:
error_mode: ignore error_mode: ignore
@@ -61,10 +47,6 @@ service:
level: none level: none
extensions: [basicauth/grafana, health_check] extensions: [basicauth/grafana, health_check]
pipelines: pipelines:
traces:
receivers: [otlp]
processors: [filter/drop_healthcheck, filter/errors_only, batch]
exporters: [otlp_http/grafana]
logs: logs:
receivers: [otlp] receivers: [otlp]
processors: [filter/drop_404_logs, batch] processors: [filter/drop_404_logs, batch]
-3
View File
@@ -13,12 +13,9 @@
"@opentelemetry/api-logs": "^0.215.0", "@opentelemetry/api-logs": "^0.215.0",
"@opentelemetry/exporter-logs-otlp-http": "^0.215.0", "@opentelemetry/exporter-logs-otlp-http": "^0.215.0",
"@opentelemetry/exporter-metrics-otlp-http": "^0.215.0", "@opentelemetry/exporter-metrics-otlp-http": "^0.215.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.215.0",
"@opentelemetry/resources": "^2.7.0", "@opentelemetry/resources": "^2.7.0",
"@opentelemetry/sdk-logs": "^0.215.0", "@opentelemetry/sdk-logs": "^0.215.0",
"@opentelemetry/sdk-metrics": "^2.7.0", "@opentelemetry/sdk-metrics": "^2.7.0",
"@opentelemetry/sdk-trace-node": "^2.7.0",
"@opentelemetry/semantic-conventions": "^1.40.0",
"@sinclair/typebox": "^0.34.48", "@sinclair/typebox": "^0.34.48",
"cache-manager": "^7.2.8", "cache-manager": "^7.2.8",
"cache-manager-fs-hash": "^3.0.0", "cache-manager-fs-hash": "^3.0.0",
+36 -66
View File
@@ -1,15 +1,12 @@
import './telemetry'; import './telemetry';
import { tracer, otelLogger, SeverityNumber, meter } from './telemetry'; import { otelLogger, SeverityNumber, meter } from './telemetry';
import { type Span, SpanKind, SpanStatusCode, context, trace } from '@opentelemetry/api';
import Fastify, { FastifyInstance, FastifyError } from 'fastify'; import Fastify, { FastifyInstance, FastifyError } from 'fastify';
declare module 'fastify' { declare module 'fastify' {
interface FastifyRequest { interface FastifyRequest {
span?: Span;
traceId?: string;
errorHandled?: boolean; errorHandled?: boolean;
meta?: Record<string, unknown>; meta?: Record<string, string>;
} }
} }
@@ -32,7 +29,7 @@ function classifyByStatus(statusCode: number): string {
} }
import cors from '@fastify/cors'; import cors from '@fastify/cors';
import { NominatimClient, NominatimResultSchema } from './services/NominatimClient'; import { NominatimClient, NominatimResultSchema } from './services/NominatimClient';
import { isZipCode } from './services/ZipCodeService'; import { classifyGeoQuery } from './services/GeoQueryClassifier';
import { GithubClient, SponsorsResponseSchema } from './services/GithubClient'; import { GithubClient, SponsorsResponseSchema } from './services/GithubClient';
import { TurnstileClient } from './services/TurnstileClient'; import { TurnstileClient } from './services/TurnstileClient';
import { ZammadClient, ContactMessageBodySchema, ContactMessageBody } from './services/ZammadClient'; import { ZammadClient, ContactMessageBodySchema, ContactMessageBody } from './services/ZammadClient';
@@ -57,13 +54,6 @@ const start = async () => {
server.setErrorHandler((error: FastifyError, request, reply) => { server.setErrorHandler((error: FastifyError, request, reply) => {
const errorType = classifyError(error); const errorType = classifyError(error);
const statusCode = error.statusCode ?? 500; const statusCode = error.statusCode ?? 500;
const { span } = request;
if (span) {
span.setAttribute('error.type', errorType);
span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
span.recordException(error);
}
otelLogger.emit({ otelLogger.emit({
severityNumber: SeverityNumber.ERROR, severityNumber: SeverityNumber.ERROR,
@@ -76,7 +66,6 @@ const start = async () => {
'http.response.status_code': statusCode, 'http.response.status_code': statusCode,
'exception.message': error.message, 'exception.message': error.message,
'exception.stacktrace': error.stack ?? '', 'exception.stacktrace': error.stack ?? '',
'trace.id': request.traceId ?? '',
}, },
}); });
@@ -86,7 +75,6 @@ const start = async () => {
server.log.error({ server.log.error({
url: request.url, url: request.url,
method: request.method, method: request.method,
traceId: request.traceId,
error: error.message, error: error.message,
stack: error.stack, stack: error.stack,
}, 'Request error'); }, 'Request error');
@@ -115,52 +103,28 @@ const start = async () => {
methods: ['GET', 'HEAD', 'POST'], methods: ['GET', 'HEAD', 'POST'],
}); });
server.addHook('onRequest', (request, _reply, done) => {
const route = (request.routeOptions as { url?: string })?.url ?? request.url.split('?')[0];
const span = tracer.startSpan(`${request.method} ${route}`, {
kind: SpanKind.SERVER,
attributes: {
'http.request.method': request.method,
'http.route': route,
'http.url': request.url,
'network.peer.address': request.ip,
},
});
request.span = span;
request.traceId = span.spanContext().traceId;
done();
});
server.addHook('onResponse', (request, reply, done) => { server.addHook('onResponse', (request, reply, done) => {
const { span } = request; const statusCode = reply.statusCode;
if (span) { const route = (request.routeOptions as { url?: string })?.url ?? request.url.split('?')[0];
const statusCode = reply.statusCode; requestCounter.add(1, {
const route = (request.routeOptions as { url?: string })?.url ?? request.url.split('?')[0]; 'http.route': route,
requestCounter.add(1, { 'http.request.method': request.method,
'http.route': route, 'http.response.status_code': statusCode,
'http.request.method': request.method, ...request.meta,
'http.response.status_code': statusCode, });
...(request.meta?.geocodeSource ? { 'geocode.source': request.meta.geocodeSource as string } : {}), if (!request.errorHandled && statusCode >= 400 && statusCode !== 404) {
otelLogger.emit({
severityNumber: statusCode >= 500 ? SeverityNumber.ERROR : SeverityNumber.WARN,
severityText: statusCode >= 500 ? 'ERROR' : 'WARN',
body: `HTTP ${statusCode} ${request.method} ${route}`,
attributes: {
'error.type': classifyByStatus(statusCode),
'http.route': route,
'http.request.method': request.method,
'http.response.status_code': statusCode,
...request.meta,
},
}); });
span.setAttribute('http.response.status_code', statusCode);
if (statusCode >= 500) {
span.setStatus({ code: SpanStatusCode.ERROR });
}
if (!request.errorHandled && statusCode >= 400) {
otelLogger.emit({
severityNumber: statusCode >= 500 ? SeverityNumber.ERROR : SeverityNumber.WARN,
severityText: statusCode >= 500 ? 'ERROR' : 'WARN',
body: `HTTP ${statusCode} ${request.method} ${(request.routeOptions as { url?: string })?.url ?? request.url}`,
attributes: {
'error.type': classifyByStatus(statusCode),
'http.route': (request.routeOptions as { url?: string })?.url ?? '',
'http.request.method': request.method,
'http.response.status_code': statusCode,
'trace.id': request.traceId ?? '',
},
});
}
span.end();
} }
done(); done();
}); });
@@ -201,7 +165,7 @@ const start = async () => {
}, async (request, reply) => { }, async (request, reply) => {
const { query } = request.query as { query: string }; const { query } = request.query as { query: string };
reply.header('Cache-Control', 'public, max-age=86400, s-maxage=86400'); reply.header('Cache-Control', 'public, max-age=86400, s-maxage=86400');
const result = await nominatim.geocodeSingleResult(query, request.span); const result = await nominatim.geocodeSingleResult(query);
if (!result) { if (!result) {
return reply.status(404).send({ error: 'No results found' }); return reply.status(404).send({ error: 'No results found' });
} }
@@ -214,6 +178,7 @@ const start = async () => {
type: 'object', type: 'object',
properties: { properties: {
query: { type: 'string' }, query: { type: 'string' },
source: { type: 'string' },
}, },
required: ['query'], required: ['query'],
}, },
@@ -226,10 +191,15 @@ const start = async () => {
}, },
}, },
}, async (request, reply) => { }, async (request, reply) => {
const { query } = request.query as { query: string }; const { query, source } = request.query as { query: string, source?: string };
request.meta = { geocodeSource: isZipCode(query) ? 'local_zip' : 'nominatim' }; const geoType = classifyGeoQuery(query);
request.meta = {
geocodeSource: geoType === 'zip_code' ? 'local_zip' : 'nominatim',
geocodeInitiator: source ?? '',
geocodeQueryType: geoType,
};
reply.header('Cache-Control', 'public, max-age=86400, s-maxage=86400'); reply.header('Cache-Control', 'public, max-age=86400, s-maxage=86400');
const result = await nominatim.geocodePhrase(query, false, request.span); const result = await nominatim.geocodePhrase(query, false);
return result; return result;
}); });
@@ -249,7 +219,7 @@ const start = async () => {
}, async (request, reply) => { }, async (request, reply) => {
const { username } = request.query as { username?: string }; const { username } = request.query as { username?: string };
reply.header('Cache-Control', 'public, max-age=60, s-maxage=600'); reply.header('Cache-Control', 'public, max-age=60, s-maxage=600');
const result = await githubClient.getSponsors(username || 'frillweeman', request.span); const result = await githubClient.getSponsors(username || 'frillweeman');
return result; return result;
}); });
@@ -266,12 +236,12 @@ const start = async () => {
const { name, email, topic, subject, message, turnstileToken } = request.body as ContactMessageBody; const { name, email, topic, subject, message, turnstileToken } = request.body as ContactMessageBody;
const remoteIp = request.ip; const remoteIp = request.ip;
const valid = await turnstileClient.verify(turnstileToken, remoteIp, request.span); const valid = await turnstileClient.verify(turnstileToken, remoteIp);
if (!valid) { if (!valid) {
return reply.status(400).send({ error: 'Invalid captcha' }); return reply.status(400).send({ error: 'Invalid captcha' });
} }
await zammadClient.createTicket({ name, email, topic, subject, message }, request.span); await zammadClient.createTicket({ name, email, topic, subject, message });
return reply.status(201).send({}); return reply.status(201).send({});
}); });
+50
View File
@@ -0,0 +1,50 @@
/**
* Classifies a geocoding query string into a common geographic query type.
*/
export type GeoQueryType =
| 'zip_code' // 5-digit US ZIP, e.g. "90210"
| 'coordinates' // lat/lon pair, e.g. "34.0522, -118.2437"
| 'address' // street address, e.g. "123 Main St, Springfield, IL"
| 'city_state' // city + 2-letter state, e.g. "Portland, OR"
| 'city' // bare city name, e.g. "Portland"
| 'state' // US state name or abbreviation, e.g. "Oregon" or "OR"
| 'other';
const ZIP_RE = /^\d{5}$/;
const COORDS_RE = /^-?\d{1,3}(\.\d+)?\s*,\s*-?\d{1,3}(\.\d+)?$/;
// Street address: starts with a number followed by a street name fragment
const ADDRESS_RE = /^\d+\s+\w/;
// "City, ST" — word(s), comma, exactly 2 uppercase letters
const CITY_STATE_RE = /^[\w\s.''-]+,\s*[A-Z]{2}$/;
const US_STATES = new Set([
'alabama','alaska','arizona','arkansas','california','colorado','connecticut',
'delaware','florida','georgia','hawaii','idaho','illinois','indiana','iowa',
'kansas','kentucky','louisiana','maine','maryland','massachusetts','michigan',
'minnesota','mississippi','missouri','montana','nebraska','nevada',
'new hampshire','new jersey','new mexico','new york','north carolina',
'north dakota','ohio','oklahoma','oregon','pennsylvania','rhode island',
'south carolina','south dakota','tennessee','texas','utah','vermont',
'virginia','washington','west virginia','wisconsin','wyoming',
'district of columbia',
// Abbreviations
'al','ak','az','ar','ca','co','ct','de','fl','ga','hi','id','il','in','ia',
'ks','ky','la','me','md','ma','mi','mn','ms','mo','mt','ne','nv','nh','nj',
'nm','ny','nc','nd','oh','ok','or','pa','ri','sc','sd','tn','tx','ut','vt',
'va','wa','wv','wi','wy','dc',
]);
export function classifyGeoQuery(query: string): GeoQueryType {
const q = query.trim();
if (ZIP_RE.test(q)) return 'zip_code';
if (COORDS_RE.test(q)) return 'coordinates';
if (ADDRESS_RE.test(q)) return 'address';
if (CITY_STATE_RE.test(q)) return 'city_state';
if (US_STATES.has(q.toLowerCase())) return 'state';
// Single word or short multi-word without punctuation — treat as city
if (/^[\w\s.'']+$/.test(q) && q.length >= 2) return 'city';
return 'other';
}
+14 -31
View File
@@ -1,7 +1,5 @@
import { Type, Static } from '@sinclair/typebox'; import { Type, Static } from '@sinclair/typebox';
import { type Span, SpanKind, SpanStatusCode, context, trace } from '@opentelemetry/api';
import { tracer } from '../telemetry';
const GITHUB_TOKEN = process.env.GITHUB_TOKEN || ''; const GITHUB_TOKEN = process.env.GITHUB_TOKEN || '';
const graphQLEndpoint = 'https://api.github.com/graphql'; const graphQLEndpoint = 'https://api.github.com/graphql';
@@ -20,37 +18,22 @@ export type Sponsor = Static<typeof SponsorSchema>;
export const SponsorsResponseSchema = Type.Array(SponsorSchema); export const SponsorsResponseSchema = Type.Array(SponsorSchema);
export class GithubClient { export class GithubClient {
async getSponsors(username: string, parentSpan?: Span): Promise<Sponsor[]> { async getSponsors(username: string): Promise<Sponsor[]> {
const query = `query { user(login: "${username}") { sponsorshipsAsMaintainer(first: 100) { nodes { sponsor { login name avatarUrl url } } } } }`; const query = `query { user(login: "${username}") { sponsorshipsAsMaintainer(first: 100) { nodes { sponsor { login name avatarUrl url } } } } }`;
const body = JSON.stringify({ query, variables: '' }); const body = JSON.stringify({ query, variables: '' });
const ctx = parentSpan ? trace.setSpan(context.active(), parentSpan) : context.active(); const response = await fetch(graphQLEndpoint, {
const span = tracer.startSpan('github.getSponsors', { method: 'POST',
kind: SpanKind.CLIENT, headers: {
attributes: { 'peer.service': 'github', 'http.request.method': 'POST' }, 'Authorization': `Bearer ${GITHUB_TOKEN}`,
}, ctx); 'User-Agent': 'Shotgun',
try { 'Content-Type': 'application/json',
const response = await fetch(graphQLEndpoint, { },
method: 'POST', body,
headers: { });
'Authorization': `Bearer ${GITHUB_TOKEN}`, if (!response.ok) {
'User-Agent': 'Shotgun', throw new Error(`Failed to get sponsors: ${response.status}`);
'Content-Type': 'application/json',
},
body,
});
span.setAttribute('http.response.status_code', response.status);
if (!response.ok) {
throw new Error(`Failed to get sponsors: ${response.status}`);
}
const json = await response.json();
return json?.data?.user?.sponsorshipsAsMaintainer?.nodes || [];
} catch (err) {
span.recordException(err as Error);
span.setStatus({ code: SpanStatusCode.ERROR, message: (err as Error).message });
span.setAttribute('error.type', 'upstream_error');
throw err;
} finally {
span.end();
} }
const json = await response.json();
return json?.data?.user?.sponsorshipsAsMaintainer?.nodes || [];
} }
} }
+7 -19
View File
@@ -1,9 +1,9 @@
import { createCache, Cache } from 'cache-manager'; import { createCache, Cache } from 'cache-manager';
import { Type, Static } from '@sinclair/typebox'; import { Type, Static } from '@sinclair/typebox';
import { type Span, SpanKind, SpanStatusCode, context, trace } from '@opentelemetry/api'; import { otelLogger, SeverityNumber } from '../telemetry';
import { tracer, otelLogger, SeverityNumber } from '../telemetry'; import { classifyGeoQuery } from './GeoQueryClassifier';
import { isZipCode, lookupZipCode } from './ZipCodeService'; import { lookupZipCode } from './ZipCodeService';
const { DiskStore } = require('cache-manager-fs-hash'); const { DiskStore } = require('cache-manager-fs-hash');
export const NominatimResultSchema = Type.Object({ export const NominatimResultSchema = Type.Object({
@@ -56,9 +56,9 @@ const cache: Cache = createCache({
export class NominatimClient { export class NominatimClient {
baseUrl = 'https://nominatim.openstreetmap.org/search'; baseUrl = 'https://nominatim.openstreetmap.org/search';
async geocodePhrase(query: string, includeGeoJson: boolean = false, parentSpan?: Span): Promise<NominatimResult[]> { async geocodePhrase(query: string, includeGeoJson: boolean = false): Promise<NominatimResult[]> {
// Short-circuit for ZIP codes — serve from local data, no Nominatim call needed // Short-circuit for ZIP codes — serve from local data, no Nominatim call needed
if (isZipCode(query)) { if (classifyGeoQuery(query) === 'zip_code') {
const zipResult = lookupZipCode(query.trim()); const zipResult = lookupZipCode(query.trim());
return zipResult ? [zipResult] : []; return zipResult ? [zipResult] : [];
} }
@@ -70,16 +70,10 @@ export class NominatimClient {
} }
const geojsonParam = includeGeoJson ? '&polygon_geojson=1' : ''; const geojsonParam = includeGeoJson ? '&polygon_geojson=1' : '';
const url = `${this.baseUrl}?q=${encodeURIComponent(query)}&format=json&addressdetails=1&limit=8&countrycodes=us&dedupe=1${geojsonParam}`; const url = `${this.baseUrl}?q=${encodeURIComponent(query)}&format=json&addressdetails=1&limit=8&countrycodes=us&dedupe=1${geojsonParam}`;
const ctx = parentSpan ? trace.setSpan(context.active(), parentSpan) : context.active();
const span = tracer.startSpan('nominatim.geocode', {
kind: SpanKind.CLIENT,
attributes: { 'peer.service': 'nominatim', 'http.request.method': 'GET' },
}, ctx);
try { try {
const response = await fetch(url, { const response = await fetch(url, {
headers: { 'User-Agent': 'DeFlock/1.2' }, headers: { 'User-Agent': 'DeFlock/1.2' },
}); });
span.setAttribute('http.response.status_code', response.status);
if (!response.ok) { if (!response.ok) {
const body = await response.text(); const body = await response.text();
otelLogger.emit({ otelLogger.emit({
@@ -91,7 +85,6 @@ export class NominatimClient {
'nominatim.response_body': body, 'nominatim.response_body': body,
'http.url': url, 'http.url': url,
}, },
context: trace.setSpan(context.active(), span),
}); });
throw new Error(`Failed to geocode phrase: ${response.status}`); throw new Error(`Failed to geocode phrase: ${response.status}`);
} }
@@ -99,17 +92,12 @@ export class NominatimClient {
await cache.set(cacheKey, json); await cache.set(cacheKey, json);
return json; return json;
} catch (err) { } catch (err) {
span.recordException(err as Error);
span.setStatus({ code: SpanStatusCode.ERROR, message: (err as Error).message });
span.setAttribute('error.type', 'upstream_error');
throw err; throw err;
} finally {
span.end();
} }
} }
async geocodeSingleResult(query: string, parentSpan?: Span): Promise<NominatimResult | null> { async geocodeSingleResult(query: string): Promise<NominatimResult | null> {
const results = await this.geocodePhrase(query, true, parentSpan); const results = await this.geocodePhrase(query, true);
if (!results.length) return null; if (!results.length) return null;
+10 -35
View File
@@ -1,47 +1,22 @@
import { type Span, SpanKind, SpanStatusCode, context, trace } from '@opentelemetry/api';
import { tracer } from '../telemetry';
const TURNSTILE_SECRET_KEY = process.env.TURNSTILE_SECRET_KEY || ''; const TURNSTILE_SECRET_KEY = process.env.TURNSTILE_SECRET_KEY || '';
const SITEVERIFY_URL = 'https://challenges.cloudflare.com/turnstile/v0/siteverify'; const SITEVERIFY_URL = 'https://challenges.cloudflare.com/turnstile/v0/siteverify';
export class TurnstileClient { export class TurnstileClient {
async verify(token: string, remoteIp?: string, parentSpan?: Span): Promise<boolean> { async verify(token: string, remoteIp?: string): Promise<boolean> {
const body = new URLSearchParams({ const body = new URLSearchParams({
secret: TURNSTILE_SECRET_KEY, secret: TURNSTILE_SECRET_KEY,
response: token, response: token,
...(remoteIp ? { remoteip: remoteIp } : {}), ...(remoteIp ? { remoteip: remoteIp } : {}),
}); });
const ctx = parentSpan ? trace.setSpan(context.active(), parentSpan) : context.active(); const response = await fetch(SITEVERIFY_URL, {
const span = tracer.startSpan('turnstile.verify', { method: 'POST',
kind: SpanKind.CLIENT, headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
attributes: { 'peer.service': 'turnstile', 'http.request.method': 'POST' }, body: body.toString(),
}, ctx); });
try { if (!response.ok) {
const response = await fetch(SITEVERIFY_URL, { throw new Error(`Turnstile siteverify request failed: ${response.status}`);
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: body.toString(),
});
span.setAttribute('http.response.status_code', response.status);
if (!response.ok) {
throw new Error(`Turnstile siteverify request failed: ${response.status}`);
}
const json = await response.json() as { success: boolean };
const success = json.success === true;
if (!success) {
span.setAttribute('error.type', 'captcha_failure');
span.setStatus({ code: SpanStatusCode.ERROR, message: 'Captcha verification failed' });
}
return success;
} catch (err) {
span.recordException(err as Error);
span.setStatus({ code: SpanStatusCode.ERROR, message: (err as Error).message });
if (!(err instanceof Error && err.message.startsWith('Turnstile'))) {
span.setAttribute('error.type', 'upstream_error');
}
throw err;
} finally {
span.end();
} }
const json = await response.json() as { success: boolean };
return json.success;
} }
} }
+51 -96
View File
@@ -1,6 +1,4 @@
import { Type, Static } from '@sinclair/typebox'; import { Type, Static } from '@sinclair/typebox';
import { type Span, SpanKind, SpanStatusCode, context, trace } from '@opentelemetry/api';
import { tracer } from '../telemetry';
const ZAMMAD_URL = process.env.ZAMMAD_URL || ''; const ZAMMAD_URL = process.env.ZAMMAD_URL || '';
const ZAMMAD_TOKEN = process.env.ZAMMAD_TOKEN || ''; const ZAMMAD_TOKEN = process.env.ZAMMAD_TOKEN || '';
@@ -46,115 +44,72 @@ export interface CreateTicketPayload {
} }
export class ZammadClient { export class ZammadClient {
private async upsertCustomer(name: string, email: string, parentSpan: Span): Promise<number> { private async upsertCustomer(name: string, email: string): Promise<number> {
const normalizedEmail = email.toLowerCase(); const normalizedEmail = email.toLowerCase();
const ctx = trace.setSpan(context.active(), parentSpan);
// Search for existing user by email // Search for existing user by email
const searchSpan = tracer.startSpan('zammad.upsertCustomer.search', { const searchResponse = await fetch(
kind: SpanKind.CLIENT, `${ZAMMAD_URL}/api/v1/users/search?query=${encodeURIComponent(normalizedEmail)}&limit=1`,
attributes: { 'peer.service': 'zammad', 'http.request.method': 'GET' }, {
}, ctx); headers: { 'Authorization': `Token token=${ZAMMAD_TOKEN}` },
try {
const searchResponse = await fetch(
`${ZAMMAD_URL}/api/v1/users/search?query=${encodeURIComponent(normalizedEmail)}&limit=1`,
{
headers: { 'Authorization': `Token token=${ZAMMAD_TOKEN}` },
}
);
searchSpan.setAttribute('http.response.status_code', searchResponse.status);
if (searchResponse.ok) {
const users = await searchResponse.json() as Array<{ id: number; email: string }>;
const match = users.find(u => u.email?.toLowerCase() === normalizedEmail);
if (match) return match.id;
} }
} catch (err) { );
searchSpan.recordException(err as Error); if (searchResponse.ok) {
searchSpan.setStatus({ code: SpanStatusCode.ERROR, message: (err as Error).message }); const users = await searchResponse.json() as Array<{ id: number; email: string }>;
throw err; const match = users.find(u => u.email?.toLowerCase() === normalizedEmail);
} finally { if (match) return match.id;
searchSpan.end();
} }
// Create the customer if not found // Create the customer if not found
const createSpan = tracer.startSpan('zammad.upsertCustomer.create', { const createResponse = await fetch(`${ZAMMAD_URL}/api/v1/users`, {
kind: SpanKind.CLIENT, method: 'POST',
attributes: { 'peer.service': 'zammad', 'http.request.method': 'POST' }, headers: {
}, ctx); 'Authorization': `Token token=${ZAMMAD_TOKEN}`,
try { 'Content-Type': 'application/json',
const createResponse = await fetch(`${ZAMMAD_URL}/api/v1/users`, { },
method: 'POST', body: JSON.stringify({ firstname: name, email: normalizedEmail, roles: ['Customer'] }),
headers: { });
'Authorization': `Token token=${ZAMMAD_TOKEN}`, if (!createResponse.ok) {
'Content-Type': 'application/json', const text = await createResponse.text();
}, throw new Error(`Zammad customer creation failed: ${createResponse.status} ${text}`);
body: JSON.stringify({ firstname: name, email: normalizedEmail, roles: ['Customer'] }),
});
createSpan.setAttribute('http.response.status_code', createResponse.status);
if (!createResponse.ok) {
const text = await createResponse.text();
throw new Error(`Zammad customer creation failed: ${createResponse.status} ${text}`);
}
const user = await createResponse.json() as { id: number };
return user.id;
} catch (err) {
createSpan.recordException(err as Error);
createSpan.setStatus({ code: SpanStatusCode.ERROR, message: (err as Error).message });
throw err;
} finally {
createSpan.end();
} }
const user = await createResponse.json() as { id: number };
return user.id;
} }
async createTicket(payload: CreateTicketPayload, parentSpan?: Span): Promise<void> { async createTicket(payload: CreateTicketPayload): Promise<void> {
const { name, email, topic, subject, message } = payload; const { name, email, topic, subject, message } = payload;
const group = TOPIC_GROUP_MAP[topic]; const group = TOPIC_GROUP_MAP[topic];
const ctx = parentSpan ? trace.setSpan(context.active(), parentSpan) : context.active(); const customerId = await this.upsertCustomer(name, email);
const span = tracer.startSpan('zammad.createTicket', {
kind: SpanKind.CLIENT,
attributes: { 'peer.service': 'zammad', 'http.request.method': 'POST' },
}, ctx);
try {
const customerId = await this.upsertCustomer(name, email, span);
const body = JSON.stringify({ const body = JSON.stringify({
title: subject, title: subject,
group, group,
priority: topic === 'media' ? '3 high' : '2 normal', priority: topic === 'media' ? '3 high' : '2 normal',
customer_id: customerId, customer_id: customerId,
article: { article: {
subject, subject,
body: message, body: message,
type: 'email', type: 'email',
sender: 'Customer', sender: 'Customer',
from: `${name} <${email}>`, from: `${name} <${email}>`,
to: 'contact@deflock.org', to: 'contact@deflock.org',
internal: false, internal: false,
}, },
}); });
const ctx = trace.setSpan(context.active(), span); const response = await fetch(`${ZAMMAD_URL}/api/v1/tickets`, {
const response = await context.with(ctx, () => fetch(`${ZAMMAD_URL}/api/v1/tickets`, { method: 'POST',
method: 'POST', headers: {
headers: { 'Authorization': `Token token=${ZAMMAD_TOKEN}`,
'Authorization': `Token token=${ZAMMAD_TOKEN}`, 'Content-Type': 'application/json',
'Content-Type': 'application/json', },
}, body,
body, });
})); if (!response.ok) {
span.setAttribute('http.response.status_code', response.status); const text = await response.text();
if (!response.ok) { throw new Error(`Zammad ticket creation failed: ${response.status} ${text}`);
const text = await response.text();
throw new Error(`Zammad ticket creation failed: ${response.status} ${text}`);
}
} catch (err) {
span.recordException(err as Error);
span.setStatus({ code: SpanStatusCode.ERROR, message: (err as Error).message });
span.setAttribute('error.type', 'upstream_error');
throw err;
} finally {
span.end();
} }
} }
} }
+1 -12
View File
@@ -1,11 +1,9 @@
import { NodeTracerProvider, BatchSpanProcessor } from '@opentelemetry/sdk-trace-node';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
import { LoggerProvider, BatchLogRecordProcessor } from '@opentelemetry/sdk-logs'; import { LoggerProvider, BatchLogRecordProcessor } from '@opentelemetry/sdk-logs';
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
import { resourceFromAttributes } from '@opentelemetry/resources'; import { resourceFromAttributes } from '@opentelemetry/resources';
import { trace, metrics } from '@opentelemetry/api'; import { metrics } from '@opentelemetry/api';
import { logs, SeverityNumber } from '@opentelemetry/api-logs'; import { logs, SeverityNumber } from '@opentelemetry/api-logs';
export { SeverityNumber }; export { SeverityNumber };
@@ -17,14 +15,6 @@ const resource = resourceFromAttributes({
'deployment.environment': process.env.NODE_ENV ?? 'production', 'deployment.environment': process.env.NODE_ENV ?? 'production',
}); });
const tracerProvider = new NodeTracerProvider({
resource,
spanProcessors: [
new BatchSpanProcessor(new OTLPTraceExporter({ url: `${OTEL_ENDPOINT}/v1/traces` })),
],
});
tracerProvider.register();
const loggerProvider = new LoggerProvider({ const loggerProvider = new LoggerProvider({
resource, resource,
processors: [ processors: [
@@ -44,6 +34,5 @@ const meterProvider = new MeterProvider({
}); });
metrics.setGlobalMeterProvider(meterProvider); metrics.setGlobalMeterProvider(meterProvider);
export const tracer = trace.getTracer('deflock-api', '1.0.0');
export const otelLogger = logs.getLogger('deflock-api', '1.0.0'); export const otelLogger = logs.getLogger('deflock-api', '1.0.0');
export const meter = metrics.getMeter('deflock-api', '1.0.0'); export const meter = metrics.getMeter('deflock-api', '1.0.0');