🔧 OPTIMIZATION: Implemented batch fetching for user and leak data in queries to resolve N+1 query issues, enhancing performance across multiple functions. Updated App component to remove blocking auth loading pattern for improved user experience.

This commit is contained in:
joaodunas
2025-12-21 14:45:43 +00:00
parent 44381858f8
commit ebd62cdeae
3 changed files with 161 additions and 152 deletions

View File

@@ -622,6 +622,10 @@ export const insertVerifiedLeak = internalMutation({
* Get all verified leaks from the database.
* Returns an array of leak documents that have been fully verified.
* Includes the names of users who verified each leak.
*
* OPTIMIZATION: Uses batch fetching to avoid N+1 query problem.
* Instead of fetching each user individually, we collect all unique user IDs
* and fetch them in one pass, then map the results.
*/
export const getVerifiedLeaks = query({
args: {},
@@ -661,55 +665,45 @@ export const getVerifiedLeaks = query({
.withIndex("by_isFullyVerified", (q) => q.eq("isFullyVerified", true))
.collect();
// Fetch submitter and verifier names
const leaksWithNames: Array<{
_id: Id<"leaks">;
_creationTime: number;
requiresLogin?: boolean;
isPaid?: boolean;
hasToolPrompts?: boolean;
accessNotes?: string;
leakText: string;
leakContext?: string;
url?: string;
targetType: "model" | "app" | "tool" | "agent" | "plugin" | "custom";
targetName: string;
provider: string;
submittedBy?: Id<"users">;
verifiedBy?: Array<Id<"users">>;
isFullyVerified: boolean;
requestId?: Id<"requests">;
submitterName?: string;
verifierNames?: Array<string>;
}> = [];
// Collect all unique user IDs (submitters + verifiers) to batch fetch
const userIds = new Set<Id<"users">>();
for (const leak of leaks) {
let submitterName: string | undefined = undefined;
let verifierNames: Array<string> | undefined = undefined;
// Get submitter name if exists
if (leak.submittedBy) {
const submitter = await ctx.db.get(leak.submittedBy);
submitterName = submitter?.name || "Unknown";
userIds.add(leak.submittedBy);
}
// Get verifier names if exists
if (leak.verifiedBy && leak.verifiedBy.length > 0) {
verifierNames = [];
if (leak.verifiedBy) {
for (const verifierId of leak.verifiedBy) {
const verifier = await ctx.db.get(verifierId);
if (verifier) {
verifierNames.push(verifier.name);
}
userIds.add(verifierId);
}
}
}
leaksWithNames.push({
// Batch fetch all users at once (single pass instead of N+1)
const userMap = new Map<Id<"users">, string>();
await Promise.all(
Array.from(userIds).map(async (userId) => {
const user = await ctx.db.get(userId);
userMap.set(userId, user?.name || "Unknown");
}),
);
// Map leaks to include user names from the cached map
const leaksWithNames = leaks.map((leak) => {
const submitterName = leak.submittedBy
? userMap.get(leak.submittedBy)
: undefined;
const verifierNames = leak.verifiedBy
? leak.verifiedBy
.map((id) => userMap.get(id))
.filter((name): name is string => name !== undefined)
: undefined;
return {
...leak,
submitterName,
verifierNames,
});
}
};
});
return leaksWithNames;
},
@@ -838,6 +832,8 @@ export const getProviders = query({
/**
* Get all verified leaks for a specific provider.
* Returns an array of leak documents with submitter and verifier names.
*
* OPTIMIZATION: Uses batch fetching to avoid N+1 query problem.
*/
export const getLeaksByProvider = query({
args: {
@@ -880,60 +876,48 @@ export const getLeaksByProvider = query({
.withIndex("by_provider_and_verified", (q) =>
q.eq("provider", args.provider).eq("isFullyVerified", true),
)
.order("desc") // Order by creation time directly in query
.collect();
// Fetch submitter and verifier names
const leaksWithNames: Array<{
_id: Id<"leaks">;
_creationTime: number;
requiresLogin?: boolean;
isPaid?: boolean;
hasToolPrompts?: boolean;
accessNotes?: string;
leakText: string;
leakContext?: string;
url?: string;
targetType: "model" | "app" | "tool" | "agent" | "plugin" | "custom";
targetName: string;
provider: string;
submittedBy?: Id<"users">;
verifiedBy?: Array<Id<"users">>;
isFullyVerified: boolean;
requestId?: Id<"requests">;
submitterName?: string;
verifierNames?: Array<string>;
}> = [];
// Collect all unique user IDs to batch fetch
const userIds = new Set<Id<"users">>();
for (const leak of leaks) {
let submitterName: string | undefined = undefined;
let verifierNames: Array<string> | undefined = undefined;
// Get submitter name if exists
if (leak.submittedBy) {
const submitter = await ctx.db.get(leak.submittedBy);
submitterName = submitter?.name || "Unknown";
userIds.add(leak.submittedBy);
}
// Get verifier names if exists
if (leak.verifiedBy && leak.verifiedBy.length > 0) {
verifierNames = [];
if (leak.verifiedBy) {
for (const verifierId of leak.verifiedBy) {
const verifier = await ctx.db.get(verifierId);
if (verifier) {
verifierNames.push(verifier.name);
}
userIds.add(verifierId);
}
}
}
leaksWithNames.push({
// Batch fetch all users at once
const userMap = new Map<Id<"users">, string>();
await Promise.all(
Array.from(userIds).map(async (userId) => {
const user = await ctx.db.get(userId);
userMap.set(userId, user?.name || "Unknown");
}),
);
// Map leaks with user names from the cached map
const leaksWithNames = leaks.map((leak) => {
const submitterName = leak.submittedBy
? userMap.get(leak.submittedBy)
: undefined;
const verifierNames = leak.verifiedBy
? leak.verifiedBy
.map((id) => userMap.get(id))
.filter((name): name is string => name !== undefined)
: undefined;
return {
...leak,
submitterName,
verifierNames,
});
}
// Sort by creation time (newest first)
leaksWithNames.sort((a, b) => b._creationTime - a._creationTime);
};
});
return leaksWithNames;
},

View File

@@ -6,6 +6,8 @@ import { Id } from "./_generated/dataModel";
/**
* Get all open (non-closed) requests from the database.
* Returns requests with submitter names populated.
*
* OPTIMIZATION: Uses batch fetching to avoid N+1 query problem.
*/
export const getOpenRequests = query({
args: {},
@@ -37,28 +39,27 @@ export const getOpenRequests = query({
.order("desc")
.collect();
// Fetch submitter names
const requestsWithNames: Array<{
_id: Id<"requests">;
_creationTime: number;
targetName: string;
provider: string;
targetType: "model" | "app" | "tool" | "agent" | "plugin" | "custom";
targetUrl: string;
closed: boolean;
submittedBy: Id<"users">;
submitterName: string;
leaks: Array<Id<"leaks">>;
}> = [];
// Collect unique user IDs to batch fetch
const userIds = new Set<Id<"users">>();
for (const request of requests) {
const user = await ctx.db.get(request.submittedBy);
requestsWithNames.push({
...request,
submitterName: user?.name || "Unknown",
});
userIds.add(request.submittedBy);
}
// Batch fetch all users at once
const userMap = new Map<Id<"users">, string>();
await Promise.all(
Array.from(userIds).map(async (userId) => {
const user = await ctx.db.get(userId);
userMap.set(userId, user?.name || "Unknown");
}),
);
// Map requests with user names from the cached map
const requestsWithNames = requests.map((request) => ({
...request,
submitterName: userMap.get(request.submittedBy) || "Unknown",
}));
return requestsWithNames;
},
});
@@ -274,6 +275,8 @@ export const closeRequest = mutation({
* Uses full-text search to find matching requests.
* Only returns open (non-closed) requests.
* Returns up to 10 results.
*
* OPTIMIZATION: Uses batch fetching to avoid N+1 query problem.
*/
export const searchRequests = query({
args: {
@@ -312,27 +315,25 @@ export const searchRequests = query({
)
.take(10);
const requestsWithNames: Array<{
_id: Id<"requests">;
_creationTime: number;
targetName: string;
provider: string;
targetType: "model" | "app" | "tool" | "agent" | "plugin" | "custom";
targetUrl: string;
closed: boolean;
submittedBy: Id<"users">;
submitterName: string;
leaks: Array<Id<"leaks">>;
}> = [];
// Batch fetch all unique users at once
const userIds = new Set<Id<"users">>();
for (const request of requests) {
const user = await ctx.db.get(request.submittedBy);
requestsWithNames.push({
...request,
submitterName: user?.name || "Unknown",
});
userIds.add(request.submittedBy);
}
const userMap = new Map<Id<"users">, string>();
await Promise.all(
Array.from(userIds).map(async (userId) => {
const user = await ctx.db.get(userId);
userMap.set(userId, user?.name || "Unknown");
}),
);
const requestsWithNames = requests.map((request) => ({
...request,
submitterName: userMap.get(request.submittedBy) || "Unknown",
}));
return requestsWithNames;
},
});
@@ -344,6 +345,8 @@ export const searchRequests = query({
* - Number of leak confirmations
* - Number of unique submitters
* - Submitter name
*
* OPTIMIZATION: Uses batch fetching for both users and leaks to avoid N+1 queries.
*/
export const getRequestsWithVerificationStatus = query({
args: {},
@@ -377,40 +380,61 @@ export const getRequestsWithVerificationStatus = query({
.order("desc")
.collect();
const requestsWithStatus: Array<{
_id: Id<"requests">;
_creationTime: number;
targetName: string;
provider: string;
targetType: "model" | "app" | "tool" | "agent" | "plugin" | "custom";
targetUrl: string;
closed: boolean;
submittedBy: Id<"users">;
submitterName: string;
leaks: Array<Id<"leaks">>;
confirmationCount: number;
uniqueSubmitters: number;
}> = [];
// Collect all unique user IDs and leak IDs to batch fetch
const userIds = new Set<Id<"users">>();
const leakIds = new Set<Id<"leaks">>();
for (const request of requests) {
const user = await ctx.db.get(request.submittedBy);
userIds.add(request.submittedBy);
for (const leakId of request.leaks) {
leakIds.add(leakId);
}
}
// Calculate unique submitters
// Batch fetch all users and leaks in parallel
const [userMap, leakMap] = await Promise.all([
// Fetch all users
(async () => {
const map = new Map<Id<"users">, string>();
await Promise.all(
Array.from(userIds).map(async (userId) => {
const user = await ctx.db.get(userId);
map.set(userId, user?.name || "Unknown");
}),
);
return map;
})(),
// Fetch all leaks
(async () => {
const map = new Map<Id<"leaks">, Id<"users"> | undefined>();
await Promise.all(
Array.from(leakIds).map(async (leakId) => {
const leak = await ctx.db.get(leakId);
map.set(leakId, leak?.submittedBy);
}),
);
return map;
})(),
]);
// Map requests with pre-fetched data
const requestsWithStatus = requests.map((request) => {
// Calculate unique submitters from cached leak data
const submitters = new Set<Id<"users">>();
for (const leakId of request.leaks) {
const leak = await ctx.db.get(leakId);
if (leak && leak.submittedBy) {
submitters.add(leak.submittedBy);
const submittedBy = leakMap.get(leakId);
if (submittedBy) {
submitters.add(submittedBy);
}
}
requestsWithStatus.push({
return {
...request,
submitterName: user?.name || "Unknown",
submitterName: userMap.get(request.submittedBy) || "Unknown",
confirmationCount: request.leaks.length,
uniqueSubmitters: submitters.size,
});
}
};
});
return requestsWithStatus;
},

View File

@@ -1,6 +1,6 @@
"use client";
import { Authenticated, Unauthenticated, useConvexAuth } from "convex/react";
import { Authenticated, Unauthenticated } from "convex/react";
import { useNavigate, Routes, Route, useLocation } from "react-router-dom";
import { Navbar } from "./components/navbar";
import Leaderboard from "./pages/Leaderboard";
@@ -11,21 +11,22 @@ import { LeakLibrary } from "./components/leakLibrary";
import { SubmitLeakForm } from "./components/submitLeakForm";
import Dashboard from "./pages/Dashboard";
/**
* Main App component.
*
* OPTIMIZATION: Removed the blocking auth loading pattern.
* Previously, the entire app would wait for authentication to complete before
* rendering anything, causing the "stuck" feeling on cold starts.
*
* Now, the app renders immediately and components handle their own loading states.
* The Authenticated/Unauthenticated components will automatically show/hide content
* based on auth state without blocking the entire UI.
*/
export default function App() {
const { isLoading } = useConvexAuth();
const navigate = useNavigate();
const location = useLocation();
const isHomePage = location.pathname === "/";
// Show only loading spinner during auth initialization
if (isLoading) {
return (
<div className="flex justify-center items-center h-screen bg-[#0f0f0f]">
<div className="animate-spin rounded-full h-12 w-12 border-t-2 border-b-2 border-[#00ff88]"></div>
</div>
);
}
return (
<>
<div className="min-h-screen bg-[#0f0f0f] text-[#e0e0e0] overflow-x-hidden font-sans relative">