perf(dashboard): optimize random book lookups and scroller batching

Add index on books table for random dashboard scroller queries, batch dashboard scroller requests, instrument pg pool logging, and cover dashboard scroller batching in e2e tests.
This commit is contained in:
neon
2026-08-05 21:48:23 +00:00
parent 854b1a75d1
commit 25bc7a21d4
38 changed files with 17212 additions and 117 deletions
+59 -4
View File
@@ -98,6 +98,10 @@ const routeInventory = loadRouteInventory();
const supportedMethods = new Set<SupportedHttpMethod>(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']);
function isSupportedMethod(method: string): method is SupportedHttpMethod {
return supportedMethods.has(method as SupportedHttpMethod);
}
function loadRouteInventory(): RouteInventory {
const inventoryDir = join(currentDir, 'e2e/authorization-matrix/route-inventory');
const manifest = JSON.parse(readFileSync(join(inventoryDir, 'manifest.json'), 'utf8')) as RouteInventoryManifest;
@@ -113,6 +117,36 @@ function loadRouteInventory(): RouteInventory {
};
}
function liveRouteLabels(app: NestFastifyApplication): string[] {
const allMethodPaths = new Set(routeInventory.routes.filter((route) => route.httpMethod === 'ALL').map((route) => route.path));
const labels = new Set<string>();
const printedRoutes = app.getHttpAdapter().getInstance().printRoutes({ commonPrefix: false });
const pathAtDepth: string[] = [];
// This test-only parser intentionally follows Fastify's printRoutes tree layout.
// A Fastify formatting change should fail parity and be updated here explicitly.
for (const line of printedRoutes.split('\n')) {
const match = line.match(/^((?:│ {3}| {4})*)[├└]── (.+?)(?: \(([^)]+)\))?$/);
if (!match) continue;
const [, indentation = '', pathPart, rawMethods] = match;
const depth = indentation.length / 4;
pathAtDepth.length = depth;
pathAtDepth.push(pathPart ?? '');
if (!rawMethods) continue;
const prefixedPath = pathAtDepth.join('');
if (!prefixedPath?.startsWith('/api/v1')) continue;
const path = prefixedPath.slice('/api/v1'.length) || '/';
if (allMethodPaths.has(path)) continue;
for (const method of rawMethods?.split(', ') ?? []) {
if (isSupportedMethod(method)) labels.add(`${method} ${path}`);
}
}
return [...labels].sort();
}
describe('Authorization matrix (e2e)', () => {
let ctx: AuthorizationMatrixE2EContext;
let libraryA: CreatedLibrary;
@@ -440,6 +474,31 @@ describe('Authorization matrix (e2e)', () => {
});
describe('guard matrix - jwt/permission/library/default-password', () => {
it('keeps the dashboard scroller inventory in parity with the live application', () => {
expect(routeInventory.routes).toHaveLength(routeInventory.totalRoutes);
const inventoryLabels = routeInventory.routes
.filter((route) => isSupportedMethod(route.httpMethod) && route.path.startsWith('/dashboard/scrollers'))
.map(routeLabel)
.sort();
const liveLabels = liveRouteLabels(ctx.app).filter((label) => label.includes(' /dashboard/scrollers'));
expect(liveLabels).toEqual(inventoryLabels);
});
it('serves the authenticated random scroller through the batch route', async () => {
const response = await ctx.app.inject({
method: 'POST',
url: '/api/v1/dashboard/scrollers/batch',
headers: authHeader(personas.allPermsUser.accessToken),
payload: { items: [{ id: 'random', type: 'random', limit: 3 }] },
});
expect(response.statusCode).toBe(200);
expect(response.json()).toEqual({
items: [{ id: 'random', books: expect.any(Array), failed: false }],
});
});
it('rejects unauthenticated access to every non-public route', async () => {
const failures: MatrixFailure[] = [];
const protectedRoutes = routeInventory.routes.filter((route) => !route.isPublic && isSupportedMethod(route.httpMethod));
@@ -1298,10 +1357,6 @@ describe('Authorization matrix (e2e)', () => {
});
});
function isSupportedMethod(method: string): method is SupportedHttpMethod {
return supportedMethods.has(method as SupportedHttpMethod);
}
function routeLabel(route: Pick<RouteInventoryRoute, 'httpMethod' | 'path'>): string {
return `${route.httpMethod} ${route.path}`;
}
+71
View File
@@ -0,0 +1,71 @@
import { randomUUID } from 'crypto';
import * as schema from '../src/db/schema';
import { DashboardRepository } from '../src/modules/dashboard/dashboard.repository';
import {
closeAuthorizationMatrixE2EContext,
createAuthorizationMatrixE2EContext,
createLibraryWithFolder,
createUserAndLogin,
grantLibraryAccess,
type AuthorizationMatrixE2EContext,
} from './e2e/authorization-matrix/authorization-matrix-harness';
const SCENARIO_TIMEOUT_MS = 60_000;
describe('Dashboard random scroller (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () => {
let ctx!: AuthorizationMatrixE2EContext;
let repository!: DashboardRepository;
let libraryId!: number;
let userId!: number;
let bookIds!: number[];
beforeAll(async () => {
ctx = await createAuthorizationMatrixE2EContext();
repository = ctx.app.get(DashboardRepository);
const library = await createLibraryWithFolder(ctx, { name: `dashboard-random-${randomUUID()}` });
const user = await createUserAndLogin(ctx);
libraryId = library.libraryId;
userId = user.userId;
await grantLibraryAccess(ctx, userId, libraryId);
const rows = await ctx.db
.insert(schema.books)
.values(
Array.from({ length: 6 }, (_, index) => ({
libraryId,
libraryFolderId: library.libraryFolderId,
folderPath: `dashboard-random-${index}`,
status: 'present' as const,
})),
)
.returning({ id: schema.books.id });
bookIds = rows.map((row) => row.id);
await ctx.db.insert(schema.userBookStatus).values([
{ userId, bookId: bookIds.at(-1)!, status: 'read', source: 'manual' },
{ userId, bookId: bookIds.at(-2)!, status: 'reading', source: 'manual' },
]);
});
afterEach(() => {
vi.restoreAllMocks();
});
afterAll(async () => {
await closeAuthorizationMatrixE2EContext(ctx);
});
it('uses the nearest eligible lower id when the forward probe has no candidate', async () => {
vi.spyOn(Math, 'random').mockReturnValue(0.99);
await expect(repository.findRandomBookIds([libraryId], userId, 1)).resolves.toEqual([bookIds.at(-3)!]);
});
it('uses the forward candidate without requiring the wrap branch', async () => {
vi.spyOn(Math, 'random').mockReturnValue(0);
await expect(repository.findRandomBookIds([libraryId], userId, 1)).resolves.toEqual([bookIds[0]!]);
});
});
@@ -1,7 +1,7 @@
{
"totalRoutes": 370,
"totalRoutes": 371,
"byPermission": {
"(none)": 165,
"(none)": 166,
"Permission.ManageMetadataConfig": 32,
"Permission.EmailSend": 30,
"Permission.LibraryEditMetadata": 33,
@@ -215,6 +215,18 @@
"allowDefault": false,
"useGuards": []
},
{
"file": "server/src/modules/dashboard/dashboard.controller.ts",
"className": "DashboardController",
"methodName": "getScrollers",
"httpMethod": "POST",
"path": "/dashboard/scrollers/batch",
"permissions": [],
"libraryAccess": [],
"isPublic": false,
"allowDefault": false,
"useGuards": []
},
{
"file": "server/src/modules/email/email-admin-log.controller.ts",
"className": "EmailAdminLogController",