import type { BookCard, BooksPage, JumpBucketsResponse, SortSpec, TemporalJumpBucketGranularity } from '@bookorbit/types'; import { eq } from 'drizzle-orm'; import { refreshPrimaryAuthorSortNamesForBooks } from '../src/db/book-author-sort-key'; import { authors, bookAuthors, bookFiles, bookMetadata, books, bookSeries, readingProgress, userBookStatus, users } from '../src/db/schema'; import { authHeader, closeMetadataWriteE2EContext, createLibraryWithFolder, createMetadataWriteE2EContext, createUserAndLogin, type MetadataWriteE2EContext, } from './e2e/metadata-write/metadata-write-harness'; const SCENARIO_TIMEOUT_MS = 120_000; type SeededBook = { id: number; title: string | null; authorSortName: string | null; publisher: string | null; language: string | null; format: string | null; readStatus: string; publishedYear: number | null; seriesName: string | null; addedAt: string; updatedAt: string; lastReadAt: string | null; startedAt: string | null; finishedAt: string | null; }; // Mirrors the SQL letter bucket expression: empty -> null, A-Z -> letter, else '#'. function letterOf(value: string | null): string | null { const trimmed = (value ?? '').trim(); if (trimmed === '') return null; const first = trimmed[0]!.toUpperCase(); const unaccented = first .normalize('NFKD') .replace(/\p{Mark}/gu, '')[0] ?.toUpperCase() ?? first; if (/^[A-Z]$/.test(unaccented)) return unaccented; return /^\p{Letter}$/u.test(first) ? first : '#'; } function expectedBucketKey( book: SeededBook, field: 'title' | 'author' | 'series' | 'publisher' | 'publishedYear', collapsed: boolean, granularity: TemporalJumpBucketGranularity | null, ): string | null { if (field === 'publishedYear') { if (book.publishedYear === null) return '__unknown__'; const step = granularity?.step ?? 1; return String(Math.floor(book.publishedYear / step) * step); } if (field === 'author') return letterOf(book.authorSortName); if (field === 'series') return letterOf(book.seriesName); if (field === 'publisher') return letterOf(book.publisher); if (collapsed) return letterOf(book.seriesName ?? book.title); return letterOf(book.title); } function expectedCategoryBucketKey(book: SeededBook, field: 'language' | 'format' | 'readStatus'): string { if (field === 'readStatus') return book.readStatus; return book[field] ?? '__unknown__'; } function expectedTemporalBucketKey(value: string | null, granularity: TemporalJumpBucketGranularity | null): string { if (value === null) return '__unknown__'; if (!granularity || granularity.unit === 'day') return value; if (granularity.unit === 'month') return value.slice(0, 7); const year = Number(value.slice(0, 4)); return String(Math.floor(year / granularity.step) * granularity.step); } describe('Jump buckets invariant (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () => { let ctx!: MetadataWriteE2EContext; let libraryId!: number; const seededById = new Map(); async function seedBook(input: { title: string | null; publisher?: string | null; language?: string | null; publishedYear?: number | null; seriesId?: number | null; seriesName?: string | null; seriesIndex?: number | null; authorIds?: number[]; addedAt: Date; libraryFolderId: number; }): Promise { const [book] = await ctx.db .insert(books) .values({ libraryId, libraryFolderId: input.libraryFolderId, folderPath: `/seed/${Math.random().toString(36).slice(2)}`, status: 'present', addedAt: input.addedAt, updatedAt: input.addedAt, }) .returning({ id: books.id }); await ctx.db.insert(bookMetadata).values({ bookId: book!.id, title: input.title, publisher: input.publisher ?? null, language: input.language ?? null, publishedYear: input.publishedYear ?? null, seriesId: input.seriesId ?? null, seriesName: input.seriesName ?? null, seriesIndex: input.seriesIndex ?? null, }); for (const [order, authorId] of (input.authorIds ?? []).entries()) { await ctx.db.insert(bookAuthors).values({ bookId: book!.id, authorId, displayOrder: order }); } return book!.id; } beforeAll(async () => { ctx = await createMetadataWriteE2EContext(); const library = await createLibraryWithFolder(ctx, { name: 'jump-buckets-invariant' }); libraryId = library.libraryId; const [duneSaga] = await ctx.db.insert(bookSeries).values({ name: 'Dune Saga', normalizedName: 'dune saga' }).returning({ id: bookSeries.id }); const [zetaSeries] = await ctx.db .insert(bookSeries) .values({ name: 'zeta chronicles', normalizedName: 'zeta chronicles' }) .returning({ id: bookSeries.id }); const authorRows = await ctx.db .insert(authors) .values([ { name: 'Isaac Asimov', sortName: 'Asimov, Isaac' }, { name: 'Roger Zelazny', sortName: 'Zelazny, Roger' }, { name: 'banana namesort', sortName: null }, { name: '99 Cent Press', sortName: '99 Cent Press' }, ]) .returning({ id: authors.id, name: authors.name, sortName: authors.sortName }); const [asimov, zelazny, banana, cheap] = authorRows; const seedSpecs: Array<{ title: string | null; publishedYear?: number | null; series?: { id: number; name: string; index: number }; author?: { id: number; sortName: string | null; name: string }; addedAt: string; }> = [ { title: 'Alpha Centauri', publishedYear: 1991, author: { id: asimov!.id, sortName: 'Asimov, Isaac', name: 'Isaac Asimov' }, addedAt: '2024-01-01', }, { title: 'apple pie', publishedYear: 2005, author: { id: banana!.id, sortName: null, name: 'banana namesort' }, addedAt: '2024-01-02' }, { title: '1984', publishedYear: 1949, author: { id: zelazny!.id, sortName: 'Zelazny, Roger', name: 'Roger Zelazny' }, addedAt: '2024-01-03' }, { title: 'Émile of the North', publishedYear: 1949, addedAt: '2024-01-04' }, { title: 'Same Title', publishedYear: 2001, author: { id: asimov!.id, sortName: 'Asimov, Isaac', name: 'Isaac Asimov' }, addedAt: '2024-01-05', }, { title: 'Same Title', publishedYear: 2002, author: { id: cheap!.id, sortName: '99 Cent Press', name: '99 Cent Press' }, addedAt: '2024-01-06', }, { title: 'Same Title', publishedYear: 2003, addedAt: '2024-01-07' }, { title: 'Beta Waves', publishedYear: null, author: { id: banana!.id, sortName: null, name: 'banana namesort' }, addedAt: '2024-01-08' }, { title: null, publishedYear: 1980, addedAt: '2024-01-09' }, { title: 'zebra crossing', publishedYear: 2010, author: { id: zelazny!.id, sortName: 'Zelazny, Roger', name: 'Roger Zelazny' }, addedAt: '2024-01-10', }, { title: 'Dune', publishedYear: 1965, series: { id: duneSaga!.id, name: 'Dune Saga', index: 1 }, author: { id: asimov!.id, sortName: 'Asimov, Isaac', name: 'Isaac Asimov' }, addedAt: '2024-01-11', }, { title: 'Dune Messiah', publishedYear: 1969, series: { id: duneSaga!.id, name: 'Dune Saga', index: 2 }, author: { id: asimov!.id, sortName: 'Asimov, Isaac', name: 'Isaac Asimov' }, addedAt: '2024-01-12', }, { title: 'Children of Dune', publishedYear: 1976, series: { id: duneSaga!.id, name: 'Dune Saga', index: 3 }, author: { id: zelazny!.id, sortName: 'Zelazny, Roger', name: 'Roger Zelazny' }, addedAt: '2024-01-13', }, { title: 'Awakening', publishedYear: 2015, series: { id: zetaSeries!.id, name: 'zeta chronicles', index: 1 }, author: { id: zelazny!.id, sortName: 'Zelazny, Roger', name: 'Roger Zelazny' }, addedAt: '2024-01-14', }, { title: 'Burning', publishedYear: 2016, series: { id: zetaSeries!.id, name: 'zeta chronicles', index: 2 }, author: { id: banana!.id, sortName: null, name: 'banana namesort' }, addedAt: '2024-01-15', }, { title: 'Quantum Garden', publishedYear: 2016, author: { id: cheap!.id, sortName: '99 Cent Press', name: '99 Cent Press' }, addedAt: '2024-01-16', }, { title: 'Дальний берег', publishedYear: 1991, addedAt: '2024-01-17' }, ]; const allIds: number[] = []; for (const [index, spec] of seedSpecs.entries()) { const publisher = index % 6 === 0 ? null : ['Orbit', 'Penguin', 'Éditions du Nord'][index % 3]!; const language = index % 7 === 0 ? null : ['en', 'de', 'pt'][index % 3]!; const id = await seedBook({ title: spec.title, publisher, language, publishedYear: spec.publishedYear ?? null, seriesId: spec.series?.id ?? null, seriesName: spec.series?.name ?? null, seriesIndex: spec.series?.index ?? null, authorIds: spec.author ? [spec.author.id] : [], addedAt: new Date(`${spec.addedAt}T00:00:00.000Z`), libraryFolderId: library.libraryFolderId, }); allIds.push(id); seededById.set(id, { id, title: spec.title, authorSortName: spec.author ? (spec.author.sortName ?? spec.author.name) : null, publisher, language, format: null, readStatus: 'unread', publishedYear: spec.publishedYear ?? null, seriesName: spec.series?.name ?? null, addedAt: spec.addedAt, updatedAt: spec.addedAt, lastReadAt: null, startedAt: null, finishedAt: null, }); } await refreshPrimaryAuthorSortNamesForBooks(ctx.db, allIds); const [admin] = await ctx.db.select({ id: users.id }).from(users).where(eq(users.username, 'metadata-write-e2e-admin')).limit(1); if (!admin) throw new Error('e2e admin user not found'); const seededStatuses = ['reading', 'read', 'want_to_read'] as const; const statusRows = allIds.flatMap((bookId, index) => { if (index % 5 === 0) return []; const day = String(index + 1).padStart(2, '0'); const startedAt = `2025-03-${day}`; const finishedAt = `2025-04-${day}`; const seeded = seededById.get(bookId)!; seeded.startedAt = startedAt; seeded.finishedAt = finishedAt; seeded.readStatus = seededStatuses[index % seededStatuses.length]!; return [ { userId: admin.id, bookId, status: seeded.readStatus as (typeof seededStatuses)[number], source: 'manual' as const, startedAt: new Date(`${startedAt}T12:00:00Z`), finishedAt: new Date(`${finishedAt}T12:00:00Z`), }, ]; }); await ctx.db.insert(userBookStatus).values(statusRows); const seededFormats = ['epub', 'pdf', 'mobi'] as const; const insertedFiles = await ctx.db .insert(bookFiles) .values( allIds.map((bookId, index) => ({ bookId, libraryFolderId: library.libraryFolderId, absolutePath: `/seed/progress-${bookId}.epub`, relPath: `progress-${bookId}.epub`, ino: BigInt(900_000 + index), format: seededFormats[index % seededFormats.length], role: 'content', })), ) .returning({ id: bookFiles.id, bookId: bookFiles.bookId }); for (const [index, file] of insertedFiles.entries()) { const seeded = seededById.get(file.bookId)!; await ctx.db .update(books) .set({ primaryFileId: file.id, updatedAt: new Date(`${seeded.updatedAt}T00:00:00.000Z`) }) .where(eq(books.id, file.bookId)); seeded.format = seededFormats[index % seededFormats.length]!; } await ctx.db.insert(readingProgress).values( insertedFiles .filter((_, index) => index % 5 !== 0) .map((file, index) => { const day = String(index + 1).padStart(2, '0'); const lastReadAt = `2025-05-${day}`; seededById.get(file.bookId)!.lastReadAt = lastReadAt; const readAt = new Date(`${lastReadAt}T12:00:00Z`); return { bookFileId: file.id, userId: admin.id, percentage: 50, updatedAt: readAt, lastReadAt: readAt }; }), ); }); afterAll(async () => { await closeMetadataWriteE2EContext(ctx); }); async function fetchBuckets(sort: SortSpec[], collapseSeries: boolean): Promise { const response = await ctx.app.inject({ method: 'POST', url: `/api/v1/libraries/${libraryId}/books/jump-buckets`, headers: authHeader(ctx.adminToken), payload: { sort, pagination: { page: 0, size: 50 }, maxBuckets: 32, ...(collapseSeries ? { collapseSeries: true } : {}) }, }); expect(response.statusCode).toBe(201); return response.json() as JumpBucketsResponse; } async function fetchListing(sort: SortSpec[], collapseSeries: boolean): Promise { const response = await ctx.app.inject({ method: 'POST', url: `/api/v1/libraries/${libraryId}/books`, headers: authHeader(ctx.adminToken), payload: { sort, pagination: { page: 0, size: 200 }, ...(collapseSeries ? { collapseSeries: true } : {}) }, }); expect(response.statusCode).toBe(201); return response.json() as BooksPage; } const fields = ['title', 'author', 'series', 'publisher', 'publishedYear'] as const; const dirs = ['asc', 'desc'] as const; const collapseModes = [false, true] as const; for (const field of fields) { for (const dir of dirs) { for (const collapsed of collapseModes) { it(`anchors ${field} ${dir} collapse=${collapsed} buckets to the exact listing offsets`, async () => { const sort: SortSpec[] = [{ field, dir }]; const [bucketsResponse, listing] = await Promise.all([fetchBuckets(sort, collapsed), fetchListing(sort, collapsed)]); expect(listing.items.length).toBe(listing.total); expect(bucketsResponse.total).toBe(listing.total); expect(bucketsResponse.buckets.length).toBeGreaterThan(1); expect(bucketsResponse.buckets.length).toBeLessThanOrEqual(32); let previousIndex = -1; for (const bucket of bucketsResponse.buckets) { expect(bucket.index).toBeGreaterThan(previousIndex); previousIndex = bucket.index; expect(bucket.index).toBeLessThan(listing.total); const itemAtAnchor = listing.items[bucket.index] as BookCard; const seeded = seededById.get(itemAtAnchor.id); expect(seeded, `listing item ${itemAtAnchor.id} at index ${bucket.index} must be a seeded book`).toBeDefined(); expect( expectedBucketKey(seeded!, field, collapsed, bucketsResponse.granularity), `bucket ${bucket.key} -> book ${itemAtAnchor.id} (${seeded!.title})`, ).toBe(bucket.key); if (bucket.index > 0) { const itemBefore = listing.items[bucket.index - 1] as BookCard; const seededBefore = seededById.get(itemBefore.id); expect(seededBefore).toBeDefined(); expect( expectedBucketKey(seededBefore!, field, collapsed, bucketsResponse.granularity), `item before bucket ${bucket.key} must belong to a different bucket`, ).not.toBe(bucket.key); } } }); } } } it('applies a secondary metadataScore tier to collapsed buckets without breaking the anchors', async () => { const sort: SortSpec[] = [ { field: 'title', dir: 'asc' }, { field: 'metadataScore', dir: 'desc' }, ]; const [bucketsResponse, listing] = await Promise.all([fetchBuckets(sort, true), fetchListing(sort, true)]); expect(bucketsResponse.total).toBe(listing.total); expect(bucketsResponse.buckets.length).toBeGreaterThan(1); for (const bucket of bucketsResponse.buckets) { const seeded = seededById.get((listing.items[bucket.index] as BookCard).id); expect(seeded).toBeDefined(); expect(expectedBucketKey(seeded!, 'title', true, bucketsResponse.granularity)).toBe(bucket.key); } }); for (const dir of dirs) { it(`lists collapsed series stacks before standalone books by title when sorting by series ${dir}`, async () => { const [seriesListing, titleListing] = await Promise.all([ fetchListing([{ field: 'series', dir }], true), fetchListing([{ field: 'title', dir }], true), ]); const rows = seriesListing.items.map((item) => seededById.get((item as BookCard).id)!); const firstStandalone = rows.findIndex((row) => row.seriesName === null); expect(rows.slice(0, firstStandalone).map((row) => row.seriesName)).toEqual( dir === 'asc' ? ['Dune Saga', 'zeta chronicles'] : ['zeta chronicles', 'Dune Saga'], ); expect(rows.slice(firstStandalone).every((row) => row.seriesName === null)).toBe(true); expect(rows.slice(firstStandalone).map((row) => row.id)).toEqual( titleListing.items.map((item) => (item as BookCard).id).filter((id) => seededById.get(id)!.seriesName === null), ); }); } it('preserves every populated year when sparse years fit the bounded capacity', async () => { const response = await fetchBuckets([{ field: 'publishedYear', dir: 'asc' }], false); const expectedYears = [...new Set([...seededById.values()].flatMap((book) => (book.publishedYear === null ? [] : [String(book.publishedYear)])))]; const returnedKeys = new Set(response.buckets.map((bucket) => bucket.key)); expect(response.granularity).toEqual({ unit: 'year', step: 1 }); for (const year of expectedYears) expect(returnedKeys.has(year)).toBe(true); }); const categoryFields = ['language', 'format', 'readStatus'] as const; for (const field of categoryFields) { for (const dir of dirs) { for (const collapsed of collapseModes) { it(`anchors ${field} ${dir} collapse=${collapsed} category buckets to exact listing offsets`, async () => { const sort: SortSpec[] = [{ field, dir }]; const [bucketsResponse, listing] = await Promise.all([fetchBuckets(sort, collapsed), fetchListing(sort, collapsed)]); expect(bucketsResponse.kind).toBe('category'); expect(bucketsResponse.buckets.length).toBeGreaterThan(1); expect(bucketsResponse.buckets.length).toBeLessThanOrEqual(32); expect(bucketsResponse.total).toBe(listing.total); let previousIndex = -1; for (const bucket of bucketsResponse.buckets) { expect(bucket.index).toBeGreaterThan(previousIndex); previousIndex = bucket.index; const itemAtAnchor = listing.items[bucket.index] as BookCard; const seeded = seededById.get(itemAtAnchor.id)!; expect(expectedCategoryBucketKey(seeded, field)).toBe(bucket.key); if (bucket.index > 0) { const itemBefore = listing.items[bucket.index - 1] as BookCard; const seededBefore = seededById.get(itemBefore.id)!; expect(expectedCategoryBucketKey(seededBefore, field)).not.toBe(bucket.key); } } }); } } } const temporalFields = ['addedAt', 'updatedAt', 'lastReadAt', 'startedAt', 'finishedAt'] as const; for (const field of temporalFields) { for (const dir of dirs) { it(`anchors ${field} ${dir} temporal buckets to exact flat-listing offsets`, async () => { const sort: SortSpec[] = [{ field, dir }]; const [bucketsResponse, listing] = await Promise.all([fetchBuckets(sort, false), fetchListing(sort, false)]); expect(bucketsResponse.kind).toBe('temporal'); expect(bucketsResponse.buckets.length).toBeLessThanOrEqual(32); expect(bucketsResponse.total).toBe(listing.total); for (const bucket of bucketsResponse.buckets) { const itemAtAnchor = listing.items[bucket.index] as BookCard; const seeded = seededById.get(itemAtAnchor.id)!; expect(expectedTemporalBucketKey(seeded[field], bucketsResponse.granularity)).toBe(bucket.key); if (bucket.index > 0) { const itemBefore = listing.items[bucket.index - 1] as BookCard; const seededBefore = seededById.get(itemBefore.id)!; expect(expectedTemporalBucketKey(seededBefore[field], bucketsResponse.granularity)).not.toBe(bucket.key); } } }); } } it('rejects ineligible sorts with 400', async () => { const response = await ctx.app.inject({ method: 'POST', url: `/api/v1/libraries/${libraryId}/books/jump-buckets`, headers: authHeader(ctx.adminToken), payload: { sort: [{ field: 'rating', dir: 'desc' }], pagination: { page: 0, size: 50 }, maxBuckets: 32 }, }); expect(response.statusCode).toBe(400); }); it('denies access to users without library access, matching the listing endpoint', async () => { const limitedUser = await createUserAndLogin(ctx); const payload = { sort: [{ field: 'title', dir: 'asc' }], pagination: { page: 0, size: 50 } }; const [bucketsResponse, listingResponse] = await Promise.all([ ctx.app.inject({ method: 'POST', url: `/api/v1/libraries/${libraryId}/books/jump-buckets`, headers: authHeader(limitedUser.accessToken), payload, }), ctx.app.inject({ method: 'POST', url: `/api/v1/libraries/${libraryId}/books`, headers: authHeader(limitedUser.accessToken), payload, }), ]); expect(bucketsResponse.statusCode).toBeGreaterThanOrEqual(403); expect(bucketsResponse.statusCode).toBe(listingResponse.statusCode); }); });