perf(dashboard): optimize random book lookups and scroller batching

Add index on books table for random dashboard scroller queries, batch dashboard scroller requests, instrument pg pool logging, and cover dashboard scroller batching in e2e tests.
This commit is contained in:
neon
2026-08-05 21:48:23 +00:00
parent 854b1a75d1
commit 25bc7a21d4
38 changed files with 17212 additions and 117 deletions
@@ -87,10 +87,11 @@ async function flush() {
await nextTick() await nextTick()
} }
function setup(options: { scope?: number | null; viewMode?: string; collapse?: boolean; q?: string; railEnabled?: boolean } = {}) { function setup(options: { scope?: number | null; viewMode?: string; collapse?: boolean; q?: string; railEnabled?: boolean; enabled?: boolean } = {}) {
const scopeId = ref<number | null>(options.scope === undefined ? 1 : options.scope) const scopeId = ref<number | null>(options.scope === undefined ? 1 : options.scope)
const viewMode = ref(options.viewMode ?? 'table') const viewMode = ref(options.viewMode ?? 'table')
const railEnabled = ref(options.railEnabled ?? true) const railEnabled = ref(options.railEnabled ?? true)
const enabled = ref(options.enabled ?? true)
const collapseEnabled = ref(options.collapse ?? false) const collapseEnabled = ref(options.collapse ?? false)
const q = ref(options.q ?? '') const q = ref(options.q ?? '')
const win = useBookViewWindow({ const win = useBookViewWindow({
@@ -101,8 +102,9 @@ function setup(options: { scope?: number | null; viewMode?: string; collapse?: b
railEnabled, railEnabled,
collapseEnabled, collapseEnabled,
q, q,
enabled,
}) })
return { win, scopeId, viewMode, railEnabled, collapseEnabled, q } return { win, scopeId, viewMode, railEnabled, enabled, collapseEnabled, q }
} }
function listRequests(): number[] { function listRequests(): number[] {
@@ -162,6 +164,25 @@ describe('useBookViewWindow', () => {
expect(win.total.value).toBe(0) expect(win.total.value).toBe(0)
}) })
it('waits until enabled and sends only the prepared sort', async () => {
mockApi(250)
const { win, enabled } = setup({ enabled: false })
await flush()
expect(fetchMock).not.toHaveBeenCalled()
win.sort.value = [{ field: 'author', dir: 'desc' }]
enabled.value = true
await flush()
const listCalls = fetchMock.mock.calls.filter(([url]) => !url.includes('jump-buckets'))
expect(listCalls).toHaveLength(1)
expect(JSON.parse(String(listCalls[0]?.[1]?.body))).toMatchObject({
sort: [{ field: 'author', dir: 'desc' }],
pagination: { page: 0 },
})
})
it('loadMorePrefix resolves only after the next contiguous chunk has loaded', async () => { it('loadMorePrefix resolves only after the next contiguous chunk has loaded', async () => {
mockApi(500) mockApi(500)
const { win } = setup() const { win } = setup()
@@ -182,6 +182,18 @@ describe('useBookWindow', () => {
expect(window.slots.value).toHaveLength(80) expect(window.slots.value).toHaveLength(80)
}) })
it('does not refetch when only query property order changes', async () => {
mockBlocks(80)
const { query } = makeWindow()
await flush()
fetchMock.mockClear()
query.value = { sort: [{ dir: 'asc', field: 'title' }] }
await flush()
expect(fetchMock).not.toHaveBeenCalled()
})
it('exposes the contiguous prefix for list mode', async () => { it('exposes the contiguous prefix for list mode', async () => {
mockBlocks(250) mockBlocks(250)
const { window } = makeWindow() const { window } = makeWindow()
@@ -32,6 +32,7 @@ export function useBookViewWindow(options: {
railViewport?: Ref<HTMLElement | null> railViewport?: Ref<HTMLElement | null>
collapseEnabled?: Ref<boolean> collapseEnabled?: Ref<boolean>
q?: Ref<string> q?: Ref<string>
enabled?: Ref<boolean>
}) { }) {
const filter = ref<GroupRule | undefined>(undefined) const filter = ref<GroupRule | undefined>(undefined)
const sort = ref<SortSpec[]>([{ field: 'title', dir: 'asc' }]) const sort = ref<SortSpec[]>([{ field: 'title', dir: 'asc' }])
@@ -43,8 +44,13 @@ export function useBookViewWindow(options: {
...(options.q?.value.trim() ? { q: options.q.value.trim() } : {}), ...(options.q?.value.trim() ? { q: options.q.value.trim() } : {}),
})) }))
const listEndpoint = computed(() => (isValidScopeId(options.scopeId.value) ? options.listEndpoint(options.scopeId.value) : null)) const queryEnabled = computed(() => options.enabled?.value ?? true)
const bucketsEndpoint = computed(() => (isValidScopeId(options.scopeId.value) ? options.bucketsEndpoint(options.scopeId.value) : null)) const listEndpoint = computed(() =>
queryEnabled.value && isValidScopeId(options.scopeId.value) ? options.listEndpoint(options.scopeId.value) : null,
)
const bucketsEndpoint = computed(() =>
queryEnabled.value && isValidScopeId(options.scopeId.value) ? options.bucketsEndpoint(options.scopeId.value) : null,
)
const window = useBookWindow({ endpoint: listEndpoint, query }) const window = useBookWindow({ endpoint: listEndpoint, query })
@@ -15,6 +15,17 @@ export function isBookPlaceholder(slot: BookSlot): slot is BookPlaceholder {
return (slot as BookPlaceholder).placeholder === true return (slot as BookPlaceholder).placeholder === true
} }
function canonicalizeQueryValue(value: unknown): unknown {
if (Array.isArray(value)) return value.map(canonicalizeQueryValue)
if (value === null || typeof value !== 'object') return value
const record = value as Record<string, unknown>
return Object.fromEntries(
Object.keys(record)
.sort()
.map((key) => [key, canonicalizeQueryValue(record[key])]),
)
}
/** /**
* Placeholder-window store for book listings: holds a slot array sized to the * Placeholder-window store for book listings: holds a slot array sized to the
* query total, where unloaded positions are placeholder objects rendered as * query total, where unloaded positions are placeholder objects rendered as
@@ -253,7 +264,7 @@ export function useBookWindow(options: { endpoint: Ref<string | null>; query: Re
}, },
}) })
const queryKey = computed(() => `${options.endpoint.value ?? ''}|${JSON.stringify(options.query.value)}`) const queryKey = computed(() => `${options.endpoint.value ?? ''}|${JSON.stringify(canonicalizeQueryValue(options.query.value))}`)
watch(queryKey, reset, { immediate: true }) watch(queryKey, reset, { immediate: true })
@@ -32,6 +32,18 @@ function mockResponse(data: unknown, ok = true): Response {
} as Response } as Response
} }
function mockSuccessfulBatch(books: unknown[], failed = false): void {
mockApi.mockImplementation(async (_input, init) => {
const body = JSON.parse(String(init?.body)) as { items: Array<{ id: string }> }
return mockResponse({ items: body.items.map((item) => ({ id: item.id, books, failed })) })
})
}
function batchBody(): { items: Array<{ id: string; type: string; limit: number; smartScopeId?: number }> } {
const init = mockApi.mock.calls.at(-1)?.[1]
return JSON.parse(String(init?.body))
}
function mountComposable(type: ScrollerType, limit = 20, smartScopeId?: number) { function mountComposable(type: ScrollerType, limit = 20, smartScopeId?: number) {
let result!: ReturnType<typeof useDashboardScroller> let result!: ReturnType<typeof useDashboardScroller>
mount( mount(
@@ -56,41 +68,60 @@ describe('useDashboardScroller', () => {
}) })
it('loads books for up-next-in-series on mount', async () => { it('loads books for up-next-in-series on mount', async () => {
mockApi.mockResolvedValue(mockResponse([{ id: 7 }, { id: 2 }])) mockSuccessfulBatch([{ id: 7 }, { id: 2 }])
const state = mountComposable('up-next-in-series', 12) const state = mountComposable('up-next-in-series', 12)
expect(state.loading.value).toBe(true) expect(state.loading.value).toBe(true)
await flushPromises() await flushPromises()
expect(mockApi).toHaveBeenCalledWith('/api/v1/dashboard/scrollers/up-next-in-series?limit=12') expect(mockApi).toHaveBeenCalledWith('/api/v1/dashboard/scrollers/batch', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: expect.any(String),
})
expect(batchBody().items).toEqual([{ id: expect.any(String), type: 'up-next-in-series', limit: 12 }])
expect(state.books.value).toEqual([{ id: 7 }, { id: 2 }]) expect(state.books.value).toEqual([{ id: 7 }, { id: 2 }])
expect(state.error.value).toBe(false) expect(state.error.value).toBe(false)
expect(state.loading.value).toBe(false) expect(state.loading.value).toBe(false)
}) })
it('includes smartScopeId only for smart-scope requests', async () => { it('includes smartScopeId only for smart-scope requests', async () => {
mockApi.mockResolvedValue(mockResponse([])) mockSuccessfulBatch([])
mountComposable('smart-scope', 30, 99) mountComposable('smart-scope', 30, 99)
await flushPromises() await flushPromises()
expect(mockApi).toHaveBeenCalledWith('/api/v1/dashboard/scrollers/smart-scope?limit=30&smartScopeId=99') expect(batchBody().items).toEqual([{ id: expect.any(String), type: 'smart-scope', limit: 30, smartScopeId: 99 }])
}) })
it.each([ it.each([
['continue-listening', '/api/v1/dashboard/scrollers/continue-listening?limit=8'], ['continue-listening', 8],
['want-to-read', '/api/v1/dashboard/scrollers/want-to-read?limit=9'], ['want-to-read', 9],
] as const)('loads books for %s on mount', async (type, expectedPath) => { ] as const)('loads books for %s on mount', async (type, limit) => {
mockApi.mockResolvedValue(mockResponse([{ id: 1 }])) mockSuccessfulBatch([{ id: 1 }])
mountComposable(type, type === 'continue-listening' ? 8 : 9) mountComposable(type, limit)
await flushPromises() await flushPromises()
expect(mockApi).toHaveBeenCalledWith(expectedPath) expect(batchBody().items[0]).toMatchObject({ type, limit })
})
it('consolidates shelves mounted in the same turn into one request', async () => {
mockSuccessfulBatch([{ id: 1 }])
mountComposable('recently-added', 20)
mountComposable('want-to-read', 10)
await flushPromises()
expect(mockApi).toHaveBeenCalledOnce()
expect(batchBody().items.map(({ type, limit }) => ({ type, limit }))).toEqual([
{ type: 'recently-added', limit: 20 },
{ type: 'want-to-read', limit: 10 },
])
}) })
it('sets error=true when API response is not ok', async () => { it('sets error=true when API response is not ok', async () => {
mockApi.mockResolvedValue(mockResponse([], false)) mockApi.mockResolvedValue(mockResponse({}, false))
const state = mountComposable('continue-reading', 5) const state = mountComposable('continue-reading', 5)
await flushPromises() await flushPromises()
@@ -101,7 +132,10 @@ describe('useDashboardScroller', () => {
}) })
it('refresh retries after an error and updates books', async () => { it('refresh retries after an error and updates books', async () => {
mockApi.mockResolvedValueOnce(mockResponse([], false)).mockResolvedValueOnce(mockResponse([{ id: 42 }], true)) mockApi.mockResolvedValueOnce(mockResponse({}, false)).mockImplementationOnce(async (_input, init) => {
const body = JSON.parse(String(init?.body)) as { items: Array<{ id: string }> }
return mockResponse({ items: body.items.map((item) => ({ id: item.id, books: [{ id: 42 }], failed: false })) })
})
const state = mountComposable('continue-reading', 5) const state = mountComposable('continue-reading', 5)
await flushPromises() await flushPromises()
@@ -116,7 +150,7 @@ describe('useDashboardScroller', () => {
it('debounces progress events and refreshes dashboard membership', async () => { it('debounces progress events and refreshes dashboard membership', async () => {
vi.useFakeTimers() vi.useFakeTimers()
mockApi.mockResolvedValue(mockResponse([{ id: 42 }])) mockSuccessfulBatch([{ id: 42 }])
mountComposable('continue-reading', 5) mountComposable('continue-reading', 5)
await flushPromises() await flushPromises()
mockApi.mockClear() mockApi.mockClear()
@@ -128,6 +162,7 @@ describe('useDashboardScroller', () => {
await vi.advanceTimersByTimeAsync(1) await vi.advanceTimersByTimeAsync(1)
await flushPromises() await flushPromises()
expect(mockApi).toHaveBeenCalledExactlyOnceWith('/api/v1/dashboard/scrollers/continue-reading?limit=5') expect(mockApi).toHaveBeenCalledOnce()
expect(batchBody().items[0]).toMatchObject({ type: 'continue-reading', limit: 5 })
}) })
}) })
@@ -1,9 +1,76 @@
import { onMounted, ref } from 'vue' import { onMounted, ref } from 'vue'
import type { BookCard, ScrollerType } from '@bookorbit/types' import {
DASHBOARD_SCROLLER_BATCH_MAX,
type BookCard,
type DashboardScrollerBatchRequest,
type DashboardScrollerBatchResponse,
type DashboardScrollerBatchResult,
type ScrollerType,
} from '@bookorbit/types'
import { api } from '@/lib/api' import { api } from '@/lib/api'
import { useBookProgressRefresh } from '@/features/book/composables/useBookProgressRefresh' import { useBookProgressRefresh } from '@/features/book/composables/useBookProgressRefresh'
type PendingScrollerRequest = {
item: DashboardScrollerBatchRequest['items'][number]
resolve: (result: DashboardScrollerBatchResult) => void
reject: (reason?: unknown) => void
}
const pendingRequests: PendingScrollerRequest[] = []
let batchScheduled = false
let requestSequence = 0
function scheduleBatch(): void {
if (batchScheduled) return
batchScheduled = true
queueMicrotask(() => void flushBatch())
}
async function flushBatch(): Promise<void> {
batchScheduled = false
const batch = pendingRequests.splice(0, DASHBOARD_SCROLLER_BATCH_MAX)
if (batch.length === 0) return
if (pendingRequests.length > 0) scheduleBatch()
try {
const body: DashboardScrollerBatchRequest = { items: batch.map((request) => request.item) }
const response = await api('/api/v1/dashboard/scrollers/batch', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
})
if (!response.ok) throw new Error('Dashboard scroller batch failed')
const payload: DashboardScrollerBatchResponse = await response.json()
const resultsById = new Map(payload.items.map((item) => [item.id, item]))
for (const request of batch) {
const result = resultsById.get(request.item.id)
if (result) request.resolve(result)
else request.reject(new Error('Dashboard scroller batch result missing'))
}
} catch (error) {
for (const request of batch) request.reject(error)
}
}
function requestScroller(type: ScrollerType, limit: number, smartScopeId?: number): Promise<DashboardScrollerBatchResult> {
return new Promise((resolve, reject) => {
requestSequence += 1
pendingRequests.push({
item: {
id: String(requestSequence),
type,
limit,
...(type === 'smart-scope' && smartScopeId ? { smartScopeId } : {}),
},
resolve,
reject,
})
scheduleBatch()
})
}
export function useDashboardScroller(type: ScrollerType, limit = 20, smartScopeId?: number) { export function useDashboardScroller(type: ScrollerType, limit = 20, smartScopeId?: number) {
const books = ref<BookCard[]>([]) const books = ref<BookCard[]>([])
const loading = ref(true) const loading = ref(true)
@@ -13,11 +80,9 @@ export function useDashboardScroller(type: ScrollerType, limit = 20, smartScopeI
loading.value = true loading.value = true
error.value = false error.value = false
try { try {
const params = new URLSearchParams({ limit: String(limit) }) const result = await requestScroller(type, limit, smartScopeId)
if (type === 'smart-scope' && smartScopeId) params.set('smartScopeId', String(smartScopeId)) books.value = result.books
const res = await api(`/api/v1/dashboard/scrollers/${type}?${params}`) error.value = result.failed
if (!res.ok) throw new Error()
books.value = await res.json()
} catch { } catch {
error.value = true error.value = true
} finally { } finally {
@@ -381,6 +381,27 @@ describe('fetchUserPeakReadingHours', () => {
await expect(fetchUserPeakReadingHours(filters)).rejects.toThrow('User peak hours request failed: 503') await expect(fetchUserPeakReadingHours(filters)).rejects.toThrow('User peak hours request failed: 503')
}) })
it('coalesces concurrent requests and starts a fresh request after settlement', async () => {
const data = [{ hour: 9, minutes: 60 }]
let resolveResponse: ((response: Response) => void) | undefined
mockedApi.mockReturnValueOnce(
new Promise<Response>((resolve) => {
resolveResponse = resolve
}),
)
const first = fetchUserPeakReadingHours(filters)
const second = fetchUserPeakReadingHours(filters)
expect(mockedApi).toHaveBeenCalledTimes(1)
resolveResponse?.(mockOkResponse(data))
await expect(Promise.all([first, second])).resolves.toEqual([data, data])
mockedApi.mockResolvedValueOnce(mockOkResponse(data))
await expect(fetchUserPeakReadingHours(filters)).resolves.toEqual(data)
expect(mockedApi).toHaveBeenCalledTimes(2)
})
}) })
describe('fetchUserFavoriteReadingDays', () => { describe('fetchUserFavoriteReadingDays', () => {
@@ -421,6 +442,27 @@ describe('fetchUserCompletionTimeline', () => {
await expect(fetchUserCompletionTimeline(filters)).rejects.toThrow('User completion timeline request failed: 503') await expect(fetchUserCompletionTimeline(filters)).rejects.toThrow('User completion timeline request failed: 503')
}) })
it('coalesces concurrent requests and starts a fresh request after settlement', async () => {
const data = [{ date: '2026-01-01', completedBooks: 1 }]
let resolveResponse: ((response: Response) => void) | undefined
mockedApi.mockReturnValueOnce(
new Promise<Response>((resolve) => {
resolveResponse = resolve
}),
)
const first = fetchUserCompletionTimeline(filters)
const second = fetchUserCompletionTimeline(filters)
expect(mockedApi).toHaveBeenCalledTimes(1)
resolveResponse?.(mockOkResponse(data))
await expect(Promise.all([first, second])).resolves.toEqual([data, data])
mockedApi.mockResolvedValueOnce(mockOkResponse(data))
await expect(fetchUserCompletionTimeline(filters)).resolves.toEqual(data)
expect(mockedApi).toHaveBeenCalledTimes(2)
})
}) })
describe('fetchUserGoalTrajectory', () => { describe('fetchUserGoalTrajectory', () => {
@@ -43,6 +43,19 @@ async function parseResult<T>(res: Response): Promise<StatisticsResult<T>> {
return res.json() as Promise<StatisticsResult<T>> return res.json() as Promise<StatisticsResult<T>>
} }
// Only endpoints with known concurrent chart consumers opt into request coalescing.
const inFlightRequests = new Map<string, Promise<unknown>>()
function coalesceRequest<T>(key: string, request: () => Promise<T>): Promise<T> {
const existing = inFlightRequests.get(key) as Promise<T> | undefined
if (existing) return existing
const pending = request().finally(() => {
if (inFlightRequests.get(key) === pending) inFlightRequests.delete(key)
})
inFlightRequests.set(key, pending)
return pending
}
function buildParams(filters: StatisticsFilterConfig, extra?: Record<string, string>): string { function buildParams(filters: StatisticsFilterConfig, extra?: Record<string, string>): string {
const params = new URLSearchParams() const params = new URLSearchParams()
filters.libraryIds.forEach((id) => params.append('libraryIds', String(id))) filters.libraryIds.forEach((id) => params.append('libraryIds', String(id)))
@@ -145,9 +158,12 @@ export async function fetchUserReadingSourceDistribution(filters: StatisticsFilt
} }
export async function fetchUserPeakReadingHours(filters: StatisticsFilterConfig): Promise<UserPeakHourStat[]> { export async function fetchUserPeakReadingHours(filters: StatisticsFilterConfig): Promise<UserPeakHourStat[]> {
const res = await api(`/api/v1/user-statistics/peak-hours${buildParams(filters, { days: '365' })}`) const url = `/api/v1/user-statistics/peak-hours${buildParams(filters, { days: '365' })}`
if (!res.ok) throw new Error(`User peak hours request failed: ${res.status}`) return coalesceRequest(url, async () => {
return res.json() as Promise<UserPeakHourStat[]> const res = await api(url)
if (!res.ok) throw new Error(`User peak hours request failed: ${res.status}`)
return res.json() as Promise<UserPeakHourStat[]>
})
} }
export async function fetchUserFavoriteReadingDays(filters: StatisticsFilterConfig): Promise<UserFavoriteDayStat[]> { export async function fetchUserFavoriteReadingDays(filters: StatisticsFilterConfig): Promise<UserFavoriteDayStat[]> {
@@ -157,9 +173,12 @@ export async function fetchUserFavoriteReadingDays(filters: StatisticsFilterConf
} }
export async function fetchUserCompletionTimeline(filters: StatisticsFilterConfig): Promise<UserCompletionTimelinePoint[]> { export async function fetchUserCompletionTimeline(filters: StatisticsFilterConfig): Promise<UserCompletionTimelinePoint[]> {
const res = await api(`/api/v1/user-statistics/completion-timeline${buildParams(filters, { days: '1825' })}`) const url = `/api/v1/user-statistics/completion-timeline${buildParams(filters, { days: '1825' })}`
if (!res.ok) throw new Error(`User completion timeline request failed: ${res.status}`) return coalesceRequest(url, async () => {
return res.json() as Promise<UserCompletionTimelinePoint[]> const res = await api(url)
if (!res.ok) throw new Error(`User completion timeline request failed: ${res.status}`)
return res.json() as Promise<UserCompletionTimelinePoint[]>
})
} }
export async function fetchUserGoalTrajectory(filters: StatisticsFilterConfig, goalBooks = 12): Promise<UserGoalTrajectory> { export async function fetchUserGoalTrajectory(filters: StatisticsFilterConfig, goalBooks = 12): Promise<UserGoalTrajectory> {
@@ -92,6 +92,29 @@ describe('EntityManagerView selection', () => {
vi.clearAllMocks() vi.clearAllMocks()
}) })
it('loads browse mode exactly once on mount', () => {
const entityManager = makeEntityManagerMock()
mocks.useEntityManager.mockReturnValue(entityManager)
shallowMount(EntityManagerView)
expect(entityManager.fetchBrowse).toHaveBeenCalledTimes(1)
})
it('refreshes exactly once when an entity change also resets pagination', async () => {
const entityManager = makeEntityManagerMock()
entityManager.browsePage.value = 3
mocks.useEntityManager.mockReturnValue(entityManager)
shallowMount(EntityManagerView)
entityManager.fetchBrowse.mockClear()
entityManager.entityType.value = 'series'
entityManager.browsePage.value = 1
await nextTick()
expect(entityManager.fetchBrowse).toHaveBeenCalledTimes(1)
})
it('routes ordinary browse selection events to single-item toggle', () => { it('routes ordinary browse selection events to single-item toggle', () => {
const entityManager = makeEntityManagerMock() const entityManager = makeEntityManagerMock()
mocks.useEntityManager.mockReturnValue(entityManager) mocks.useEntityManager.mockReturnValue(entityManager)
@@ -36,11 +36,8 @@ const bulkDeleteDefaultMode = computed<'soft' | 'hard'>(() =>
let searchDebounce: ReturnType<typeof setTimeout> | null = null let searchDebounce: ReturnType<typeof setTimeout> | null = null
function refreshBrowseFromFirstPage(): void { function refreshBrowseFromFirstPage(): void {
if (em.browsePage.value === 1) {
em.fetchBrowse()
return
}
em.browsePage.value = 1 em.browsePage.value = 1
em.fetchBrowse()
} }
watch(em.browseSearch, () => { watch(em.browseSearch, () => {
@@ -50,13 +47,9 @@ watch(em.browseSearch, () => {
}, 300) }, 300)
}) })
watch(em.browsePage, () => {
em.fetchBrowse()
})
watch( watch(
em.mode, [em.mode, em.entityType],
(newMode) => { ([newMode]) => {
if (newMode === 'browse') { if (newMode === 'browse') {
em.fetchBrowse() em.fetchBrowse()
} else if (newMode === 'duplicates') { } else if (newMode === 'duplicates') {
@@ -66,18 +59,6 @@ watch(
{ immediate: true }, { immediate: true },
) )
watch(
em.entityType,
() => {
if (em.mode.value === 'browse') {
em.fetchBrowse()
} else if (em.mode.value === 'duplicates') {
em.fetchScanStatus()
}
},
{ immediate: true },
)
function handleUpdateMode(value: EntityManagerMode): void { function handleUpdateMode(value: EntityManagerMode): void {
em.mode.value = value em.mode.value = value
} }
@@ -114,6 +95,7 @@ function handleUpdateSearch(value: string): void {
function handleUpdatePage(value: number): void { function handleUpdatePage(value: number): void {
em.browsePage.value = value em.browsePage.value = value
em.fetchBrowse()
} }
function handleBrowseSortChange(sortBy: 'name' | 'bookCount', sortOrder: 'asc' | 'desc'): void { function handleBrowseSortChange(sortBy: 'name' | 'bookCount', sortOrder: 'asc' | 'desc'): void {
+8 -2
View File
@@ -1,5 +1,5 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed, onMounted, ref, shallowRef, watch } from 'vue' import { computed, nextTick, onMounted, ref, shallowRef, watch } from 'vue'
import { useRoute, useRouter } from 'vue-router' import { useRoute, useRouter } from 'vue-router'
import { useI18n } from 'vue-i18n' import { useI18n } from 'vue-i18n'
import { formatNumber } from '@/i18n/formatters' import { formatNumber } from '@/i18n/formatters'
@@ -89,6 +89,7 @@ watch(prefs, () => {
const { searchQuery, debouncedQuery, clearSearch } = useViewSearch() const { searchQuery, debouncedQuery, clearSearch } = useViewSearch()
const mainRef = ref<HTMLElement | null>(null) const mainRef = ref<HTMLElement | null>(null)
const scopeQueryReady = ref(false)
const { const {
booksProxy: books, booksProxy: books,
slots, slots,
@@ -125,6 +126,7 @@ const {
railViewport: mainRef, railViewport: mainRef,
collapseEnabled: collapseEnabledRef, collapseEnabled: collapseEnabledRef,
q: debouncedQuery, q: debouncedQuery,
enabled: scopeQueryReady,
}) })
const { setBookContext } = useBookNavigation() const { setBookContext } = useBookNavigation()
useScrollRestoreOnActivate(mainRef) useScrollRestoreOnActivate(mainRef)
@@ -299,8 +301,12 @@ async function handleBulkEditConfirm(fields: BulkEditFields) {
watch( watch(
smartScope, smartScope,
(scope) => { async (scope) => {
scopeQueryReady.value = false
tableSort.value = scope?.defaultSort?.length ? [...scope.defaultSort] : [{ field: 'title', dir: 'asc' }] tableSort.value = scope?.defaultSort?.length ? [...scope.defaultSort] : [{ field: 'title', dir: 'asc' }]
if (!scope) return
await nextTick()
if (smartScope.value?.id === scope.id) scopeQueryReady.value = true
}, },
{ immediate: true }, { immediate: true },
) )
+4
View File
@@ -1,4 +1,5 @@
import { fileURLToPath, URL } from 'node:url' import { fileURLToPath, URL } from 'node:url'
import { Agent } from 'node:http'
import { defineConfig } from 'vite' import { defineConfig } from 'vite'
import vue from '@vitejs/plugin-vue' import vue from '@vitejs/plugin-vue'
@@ -6,6 +7,8 @@ import vueDevTools from 'vite-plugin-vue-devtools'
import tailwindcss from '@tailwindcss/vite' import tailwindcss from '@tailwindcss/vite'
import { VitePWA } from 'vite-plugin-pwa' import { VitePWA } from 'vite-plugin-pwa'
const apiAgent = new Agent({ keepAlive: true })
// https://vite.dev/config/ // https://vite.dev/config/
export default defineConfig({ export default defineConfig({
define: { define: {
@@ -141,6 +144,7 @@ export default defineConfig({
proxy: { proxy: {
'/api': { '/api': {
target: 'http://localhost:3000', target: 'http://localhost:3000',
agent: apiAgent,
configure: (proxy) => { configure: (proxy) => {
proxy.on('proxyReq', (proxyReq, req) => { proxy.on('proxyReq', (proxyReq, req) => {
if (req.headers.host) proxyReq.setHeader('x-forwarded-host', req.headers.host) if (req.headers.host) proxyReq.setHeader('x-forwarded-host', req.headers.host)
+24
View File
@@ -1,3 +1,5 @@
import type { BookCard } from "./book";
export const SCROLLER_TYPE = { export const SCROLLER_TYPE = {
RECENTLY_ADDED: "recently-added", RECENTLY_ADDED: "recently-added",
CONTINUE_READING: "continue-reading", CONTINUE_READING: "continue-reading",
@@ -11,6 +13,28 @@ export const SCROLLER_TYPE = {
export type ScrollerType = (typeof SCROLLER_TYPE)[keyof typeof SCROLLER_TYPE]; export type ScrollerType = (typeof SCROLLER_TYPE)[keyof typeof SCROLLER_TYPE];
export const SCROLLER_TYPES = Object.values(SCROLLER_TYPE) as ReadonlyArray<ScrollerType>; export const SCROLLER_TYPES = Object.values(SCROLLER_TYPE) as ReadonlyArray<ScrollerType>;
export const DASHBOARD_SCROLLER_BATCH_MAX = 8;
export interface DashboardScrollerBatchItem {
id: string;
type: ScrollerType;
limit: number;
smartScopeId?: number;
}
export interface DashboardScrollerBatchRequest {
items: DashboardScrollerBatchItem[];
}
export interface DashboardScrollerBatchResult {
id: string;
books: BookCard[];
failed: boolean;
}
export interface DashboardScrollerBatchResponse {
items: DashboardScrollerBatchResult[];
}
export interface ScrollerConfig { export interface ScrollerConfig {
id: string; id: string;
type: ScrollerType; type: ScrollerType;
+19
View File
@@ -228,6 +228,25 @@ export const E2E_SUITES = Object.freeze({
...SHARED_DB_AND_HELPER_PATHS, ...SHARED_DB_AND_HELPER_PATHS,
], ],
}, },
"dashboard-random": {
id: "dashboard-random",
name: "Dashboard Random Scroller",
timeout: 60,
lane: "full",
description: "Dashboard random scroller repository integration suite",
vitestTarget: "test/dashboard-random.e2e-spec.ts",
junitOutput: `${TEST_RESULTS_DIR}/dashboard-random-e2e-junit.xml`,
prepareDedicatedDatabase: true,
useDedicatedDatabase: true,
changedPaths: [
"server/src/modules/dashboard/**",
"server/test/dashboard-random.e2e-spec.ts",
"server/test/e2e/authorization-matrix/**",
".github/workflows/e2e-runner.yml",
"packages/types/src/dashboard.ts",
...SHARED_DB_AND_HELPER_PATHS,
],
},
"book-api-contract": { "book-api-contract": {
id: "book-api-contract", id: "book-api-contract",
name: "Book API Contract", name: "Book API Contract",
+2 -2
View File
@@ -1,10 +1,10 @@
import { Module, Global } from '@nestjs/common'; import { Module, Global } from '@nestjs/common';
import { ConfigService } from '@nestjs/config'; import { ConfigService } from '@nestjs/config';
import { drizzle } from 'drizzle-orm/node-postgres'; import { drizzle } from 'drizzle-orm/node-postgres';
import { Pool } from 'pg';
import { createPostgresClientConfig } from './postgres-connection-config'; import { createPostgresClientConfig } from './postgres-connection-config';
import * as schema from './schema'; import * as schema from './schema';
import { InstrumentedPgPool } from './instrumented-pg-pool';
export const DB = Symbol('DB'); export const DB = Symbol('DB');
@@ -15,7 +15,7 @@ export const DB = Symbol('DB');
provide: DB, provide: DB,
inject: [ConfigService], inject: [ConfigService],
useFactory: (config: ConfigService) => { useFactory: (config: ConfigService) => {
const pool = new Pool( const pool = new InstrumentedPgPool(
createPostgresClientConfig(config.getOrThrow<string>('db.url'), { createPostgresClientConfig(config.getOrThrow<string>('db.url'), {
max: 20, max: 20,
idleTimeoutMillis: 30_000, idleTimeoutMillis: 30_000,
@@ -0,0 +1,61 @@
import { Logger } from '@nestjs/common';
import { Pool, type PoolClient } from 'pg';
import { InstrumentedPgPool } from './instrumented-pg-pool';
describe('InstrumentedPgPool', () => {
afterEach(() => {
vi.restoreAllMocks();
});
it('logs acquisition time when a query starts with the pool saturated', async () => {
const client = {} as PoolClient;
vi.spyOn(Pool.prototype, 'connect').mockResolvedValue(client);
const debugSpy = vi.spyOn(Logger.prototype, 'debug').mockImplementation(() => undefined);
const pool = new InstrumentedPgPool({ max: 1 });
Object.defineProperty(pool, 'totalCount', { value: 1 });
Object.defineProperty(pool, 'idleCount', { value: 0 });
Object.defineProperty(pool, 'waitingCount', { value: 0 });
await expect(pool.connect()).resolves.toBe(client);
expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining('[db.pool_acquire] [end] acquisitionKind=queued durationMs='));
expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining('acquisitionKind=queued'));
});
it('logs acquisition time when the pool opens a new connection', async () => {
const client = {} as PoolClient;
vi.spyOn(Pool.prototype, 'connect').mockResolvedValue(client);
const debugSpy = vi.spyOn(Logger.prototype, 'debug').mockImplementation(() => undefined);
const pool = new InstrumentedPgPool({ max: 2 });
await expect(pool.connect()).resolves.toBe(client);
expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining('acquisitionKind=new'));
});
it('does not log acquisitions from an idle connection', async () => {
vi.spyOn(Pool.prototype, 'connect').mockResolvedValue({} as PoolClient);
const debugSpy = vi.spyOn(Logger.prototype, 'debug').mockImplementation(() => undefined);
const pool = new InstrumentedPgPool({ max: 2 });
Object.defineProperty(pool, 'totalCount', { value: 1 });
Object.defineProperty(pool, 'idleCount', { value: 1 });
await pool.connect();
expect(debugSpy).not.toHaveBeenCalled();
});
it('logs failures while acquiring an idle connection', async () => {
const error = new Error('connection closed');
vi.spyOn(Pool.prototype, 'connect').mockRejectedValue(error);
const warnSpy = vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
const pool = new InstrumentedPgPool({ max: 2 });
Object.defineProperty(pool, 'totalCount', { value: 1 });
Object.defineProperty(pool, 'idleCount', { value: 1 });
await expect(pool.connect()).rejects.toThrow(error);
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('[db.pool_acquire] [fail] acquisitionKind=idle'));
});
});
+60
View File
@@ -0,0 +1,60 @@
import { Logger } from '@nestjs/common';
import { Pool, type PoolClient } from 'pg';
import { sanitizeLogValue } from '../common/utils/log-sanitize.utils';
type ConnectCallback = (err: Error | undefined, client: PoolClient | undefined, done: (release?: unknown) => void) => void;
type AcquisitionKind = 'idle' | 'new' | 'queued';
export class InstrumentedPgPool extends Pool {
private readonly logger = new Logger(InstrumentedPgPool.name);
override connect(): Promise<PoolClient>;
override connect(callback: ConnectCallback): void;
override connect(callback?: ConnectCallback): Promise<PoolClient> | void {
const startedAt = Date.now();
const acquisitionKind = this.classifyAcquisition();
if (callback) {
super.connect((error, client, done) => {
this.logAcquire(startedAt, acquisitionKind, error);
callback(error, client, done);
});
return;
}
return super.connect().then(
(client) => {
this.logAcquire(startedAt, acquisitionKind);
return client;
},
(error: unknown) => {
this.logAcquire(startedAt, acquisitionKind, error);
throw error;
},
);
}
private classifyAcquisition(): AcquisitionKind {
if (this.idleCount > 0) return 'idle';
if (this.waitingCount > 0 || this.totalCount >= this.options.max) return 'queued';
return 'new';
}
private logAcquire(startedAt: number, acquisitionKind: AcquisitionKind, error?: unknown): void {
if (acquisitionKind === 'idle' && !error) return;
const durationMs = Date.now() - startedAt;
if (error) {
const errorClass = error instanceof Error ? error.constructor.name : typeof error;
const message = sanitizeLogValue(error instanceof Error ? error.message : error);
this.logger.warn(
`[db.pool_acquire] [fail] acquisitionKind=${acquisitionKind} durationMs=${durationMs} totalCount=${this.totalCount} idleCount=${this.idleCount} waitingCount=${this.waitingCount} errorClass=${errorClass} error="${message}" - database connection acquisition failed`,
);
return;
}
this.logger.debug(
`[db.pool_acquire] [end] acquisitionKind=${acquisitionKind} durationMs=${durationMs} totalCount=${this.totalCount} idleCount=${this.idleCount} waitingCount=${this.waitingCount} - database connection acquired`,
);
}
}
@@ -0,0 +1,2 @@
DROP INDEX "books_library_status_idx";--> statement-breakpoint
CREATE INDEX "books_library_status_id_idx" ON "books" USING btree ("library_id","status","id");
File diff suppressed because it is too large Load Diff
@@ -442,6 +442,13 @@
"when": 1785712608966, "when": 1785712608966,
"tag": "0062_add_series_expected_book_count", "tag": "0062_add_series_expected_book_count",
"breakpoints": true "breakpoints": true
},
{
"idx": 63,
"version": "7",
"when": 1785964221973,
"tag": "0063_add_dashboard_random_lookup_index",
"breakpoints": true
} }
] ]
} }
+1 -1
View File
@@ -27,7 +27,7 @@ export const books = pgTable(
uniqueIndex('books_library_id_folder_path_idx').on(t.libraryId, t.folderPath), uniqueIndex('books_library_id_folder_path_idx').on(t.libraryId, t.folderPath),
unique('books_id_library_folder_id_unique').on(t.id, t.libraryFolderId), unique('books_id_library_folder_id_unique').on(t.id, t.libraryFolderId),
index('books_primary_file_id_idx').on(t.primaryFileId), index('books_primary_file_id_idx').on(t.primaryFileId),
index('books_library_status_idx').on(t.libraryId, t.status), index('books_library_status_id_idx').on(t.libraryId, t.status, t.id),
index('books_library_added_at_idx').on(t.libraryId, sql`${t.addedAt} desc`), index('books_library_added_at_idx').on(t.libraryId, sql`${t.addedAt} desc`),
index('books_library_visible_added_id_idx') index('books_library_visible_added_id_idx')
.on(t.libraryId, sql`${t.addedAt} desc`, t.id) .on(t.libraryId, sql`${t.addedAt} desc`, t.id)
@@ -181,6 +181,18 @@ describe('BookRepository', () => {
}); });
}); });
it('findCardIds applies card query pagination without running enrichment queries', async () => {
const chain = makeSelectChain('offset', [{ id: 9 }, { id: 3 }]);
const db = { select: vi.fn().mockReturnValue(chain) };
const repo = new BookRepository(db as never);
await expect(repo.findCardIds({ where: undefined, orderBy: [], limit: 20, offset: 40, userId: 7 })).resolves.toEqual([9, 3]);
expect(db.select).toHaveBeenCalledOnce();
expect(chain.limit).toHaveBeenCalledWith(20);
expect(chain.offset).toHaveBeenCalledWith(40);
});
it('findById returns null when no matching book exists', async () => { it('findById returns null when no matching book exists', async () => {
const db = { const db = {
select: vi.fn().mockReturnValue(makeSelectChain('limit', [])), select: vi.fn().mockReturnValue(makeSelectChain('limit', [])),
@@ -579,6 +579,21 @@ export class BookRepository {
return { rows, ...enrichment, total: Number(total) }; return { rows, ...enrichment, total: Number(total) };
} }
async findCardIds(opts: { where: SQL | undefined; orderBy: SQL[]; limit: number; offset: number; userId: number }): Promise<number[]> {
const rows = await this.db
.select({ id: books.id })
.from(books)
.innerJoin(libraries, eq(libraries.id, books.libraryId))
.leftJoin(bookMetadata, eq(bookMetadata.bookId, books.id))
.leftJoin(userBookRatings, and(eq(userBookRatings.bookId, books.id), eq(userBookRatings.userId, opts.userId)))
.where(this.visibleWhere(opts.where))
.orderBy(...opts.orderBy)
.limit(opts.limit)
.offset(opts.offset);
return rows.map((row) => row.id);
}
private async enrichBookIds(bookRefs: Array<{ id: number; primaryFileId: number | null }>, userId: number) { private async enrichBookIds(bookRefs: Array<{ id: number; primaryFileId: number | null }>, userId: number) {
const bookIds = bookRefs.map((book) => book.id); const bookIds = bookRefs.map((book) => book.id);
const primaryFileIds = bookRefs.map((book) => book.primaryFileId).filter((id): id is number => id != null); const primaryFileIds = bookRefs.map((book) => book.primaryFileId).filter((id): id is number => id != null);
+12
View File
@@ -1141,6 +1141,18 @@ export class BookService {
return result; return result;
} }
async executeBookIdsQuery(userId: number, where: SQL | undefined, query: BookQuery): Promise<number[]> {
const customFieldTypes = await this.resolveCustomSortFieldTypes(query.sort);
const orderBy = this.queryBuilder.buildOrderBy(query.sort, userId, customFieldTypes);
return this.bookRepo.findCardIds({
where,
orderBy,
limit: query.pagination.size,
offset: query.pagination.page * query.pagination.size,
userId,
});
}
async queryJumpBucketsForLibrary(user: RequestUser, libraryId: number, query: JumpBucketsQuery): Promise<JumpBucketsResponse> { async queryJumpBucketsForLibrary(user: RequestUser, libraryId: number, query: JumpBucketsQuery): Promise<JumpBucketsResponse> {
await this.libraryService.verifyUserAccess(user.id, libraryId, this.isSuperuser(user)); await this.libraryService.verifyUserAccess(user.id, libraryId, this.isSuperuser(user));
const timeZone = this.resolveUserTimeZone(user); const timeZone = this.resolveUserTimeZone(user);
@@ -25,6 +25,7 @@ function makeUser(overrides: Partial<RequestUser> = {}): RequestUser {
function makeController() { function makeController() {
const dashboardService = { const dashboardService = {
getScroller: vi.fn(), getScroller: vi.fn(),
getScrollers: vi.fn(),
}; };
const widgetService = { const widgetService = {
getReadingGoal: vi.fn(), getReadingGoal: vi.fn(),
@@ -62,6 +63,19 @@ describe('DashboardController', () => {
expect(dashboardService.getScroller).toHaveBeenCalledWith('up-next-in-series', user, 33, 0); expect(dashboardService.getScroller).toHaveBeenCalledWith('up-next-in-series', user, 33, 0);
expect(result).toEqual(mockResult); expect(result).toEqual(mockResult);
}); });
it('getScrollers delegates the bounded batch to dashboardService', async () => {
const { controller, dashboardService } = makeController();
const user = makeUser({ id: 7 });
const dto = { items: [{ id: 'one', type: 'recently-added' as const, limit: 20 }] };
const mockResult = { items: [{ id: 'one', books: [], failed: false }] };
dashboardService.getScrollers.mockResolvedValue(mockResult);
const result = await controller.getScrollers(user, dto);
expect(dashboardService.getScrollers).toHaveBeenCalledWith(dto, user);
expect(result).toEqual(mockResult);
});
}); });
describe('widget endpoints', () => { describe('widget endpoints', () => {
@@ -1,9 +1,10 @@
import { Controller, DefaultValuePipe, Get, Param, ParseEnumPipe, ParseIntPipe, Query } from '@nestjs/common'; import { Body, Controller, DefaultValuePipe, Get, HttpCode, HttpStatus, Param, ParseEnumPipe, ParseIntPipe, Post, Query } from '@nestjs/common';
import { CurrentUser } from '../../common/decorators/current-user.decorator'; import { CurrentUser } from '../../common/decorators/current-user.decorator';
import type { RequestUser } from '../../common/types/request-user'; import type { RequestUser } from '../../common/types/request-user';
import { DashboardService } from './dashboard.service'; import { DashboardService } from './dashboard.service';
import { DashboardWidgetService } from './dashboard-widget.service'; import { DashboardWidgetService } from './dashboard-widget.service';
import { DashboardScrollerBatchDto } from './dto/dashboard-scroller-batch.dto';
import { ScrollerType } from './dto/scroller-type.enum'; import { ScrollerType } from './dto/scroller-type.enum';
@Controller('dashboard') @Controller('dashboard')
@@ -23,6 +24,12 @@ export class DashboardController {
return this.dashboardService.getScroller(type, user, limit, smartScopeId); return this.dashboardService.getScroller(type, user, limit, smartScopeId);
} }
@Post('scrollers/batch')
@HttpCode(HttpStatus.OK)
getScrollers(@CurrentUser() user: RequestUser, @Body() dto: DashboardScrollerBatchDto) {
return this.dashboardService.getScrollers(dto, user);
}
@Get('widgets/reading-goal') @Get('widgets/reading-goal')
getReadingGoal(@CurrentUser() user: RequestUser) { getReadingGoal(@CurrentUser() user: RequestUser) {
return this.widgetService.getReadingGoal(user); return this.widgetService.getReadingGoal(user);
@@ -1,5 +1,8 @@
import { PgDialect } from 'drizzle-orm/pg-core';
import type { SQL } from 'drizzle-orm';
import { DashboardRepository } from './dashboard.repository'; import { DashboardRepository } from './dashboard.repository';
import { audiobookProgress, bookFiles, readingProgress, userBookStatus } from '../../db/schema'; import { audiobookProgress, bookFiles, books, readingProgress, userBookStatus } from '../../db/schema';
function makeLimitChain<T>(rows: T) { function makeLimitChain<T>(rows: T) {
const chain: Record<string, vi.Mock> = { const chain: Record<string, vi.Mock> = {
@@ -20,6 +23,16 @@ function makeLimitChain<T>(rows: T) {
return chain; return chain;
} }
function makeBoundsChain(minId: number | null, maxId: number | null) {
const chain = {
from: vi.fn(),
where: vi.fn(),
};
chain.from.mockReturnValue(chain);
chain.where.mockResolvedValue([{ minId, maxId }]);
return chain;
}
function collectValues(value: unknown, seen = new WeakSet<object>()): unknown[] { function collectValues(value: unknown, seen = new WeakSet<object>()): unknown[] {
if (value === null || typeof value !== 'object') return [value]; if (value === null || typeof value !== 'object') return [value];
if (seen.has(value)) return []; if (seen.has(value)) return [];
@@ -33,6 +46,13 @@ function collectValues(value: unknown, seen = new WeakSet<object>()): unknown[]
return values; return values;
} }
function compileSql(value: unknown): string {
return new PgDialect()
.sqlToQuery(value as SQL)
.sql.replaceAll(/\s+/g, ' ')
.trim();
}
describe('DashboardRepository', () => { describe('DashboardRepository', () => {
beforeEach(() => { beforeEach(() => {
vi.restoreAllMocks(); vi.restoreAllMocks();
@@ -117,40 +137,109 @@ describe('DashboardRepository', () => {
}); });
it('returns empty random ids when there are no candidates', async () => { it('returns empty random ids when there are no candidates', async () => {
const boundsChain = makeBoundsChain(1, 100);
const listChain = makeLimitChain([]); const listChain = makeLimitChain([]);
const db = { select: vi.fn().mockReturnValue(listChain) }; const db = {
select: vi.fn().mockReturnValueOnce(boundsChain).mockReturnValue(listChain),
execute: vi.fn().mockResolvedValue({ rows: [] }),
};
const repo = new DashboardRepository(db as never); const repo = new DashboardRepository(db as never);
const result = await repo.findRandomBookIds([5], 7, 20); const result = await repo.findRandomBookIds([5], 7, 20);
expect(result).toEqual([]); expect(result).toEqual([]);
expect(db.select).toHaveBeenCalledTimes(1); expect(db.select).toHaveBeenCalledTimes(5);
expect(listChain.leftJoin).toHaveBeenCalledTimes(3); expect(db.execute).toHaveBeenCalledTimes(1);
expect(listChain.leftJoin.mock.calls[0]?.[0]).toBe(bookFiles); expect(listChain.leftJoin).not.toHaveBeenCalled();
expect(listChain.leftJoin.mock.calls[1]?.[0]).toBe(readingProgress); expect(listChain.orderBy).toHaveBeenCalledTimes(2);
expect(listChain.leftJoin.mock.calls[2]?.[0]).toBe(userBookStatus); expect(listChain.limit).toHaveBeenCalledTimes(2);
expect(listChain.orderBy).toHaveBeenCalledTimes(1); expect(listChain.limit).toHaveBeenNthCalledWith(1, 20);
expect(listChain.limit).toHaveBeenCalledWith(20); expect(listChain.limit).toHaveBeenNthCalledWith(2, 20);
}); });
it('maps random rows to id list and excludes active or finished read statuses', async () => { it('samples random rows from independent pivots and excludes active or finished read statuses', async () => {
const listChain = makeLimitChain([{ id: 21 }, { id: 3 }, { id: 15 }]); vi.spyOn(Math, 'random').mockReturnValue(0.5);
const db = { select: vi.fn().mockReturnValue(listChain) }; const boundsChain = makeBoundsChain(1, 100);
const subqueryChain = makeLimitChain([]);
const db = {
select: vi.fn().mockReturnValueOnce(boundsChain).mockReturnValue(subqueryChain),
execute: vi.fn().mockResolvedValue({
rows: [
{ sampleIndex: 0, id: 21 },
{ sampleIndex: 1, id: 3 },
{ sampleIndex: 2, id: 15 },
],
}),
};
const repo = new DashboardRepository(db as never); const repo = new DashboardRepository(db as never);
const result = await repo.findRandomBookIds([5], 7, 3); const result = await repo.findRandomBookIds([5], 7, 3);
const whereArg = listChain.where.mock.calls[0]?.[0]; const queryText = compileSql(db.execute.mock.calls[0]?.[0]);
const whereValues = collectValues(whereArg); const statusWhereValues = collectValues(subqueryChain.where.mock.calls[1]?.[0]);
expect(result).toEqual([21, 3, 15]); expect(result).toEqual([21, 3, 15]);
expect(db.select).toHaveBeenCalledTimes(1); expect(db.select).toHaveBeenCalledTimes(3);
expect(listChain.leftJoin).toHaveBeenCalledTimes(3); expect(db.execute).toHaveBeenCalledTimes(1);
expect(listChain.leftJoin.mock.calls[0]?.[0]).toBe(bookFiles); expect(subqueryChain.offset).toHaveBeenCalledTimes(2);
expect(listChain.leftJoin.mock.calls[1]?.[0]).toBe(readingProgress); expect(subqueryChain.offset).toHaveBeenNthCalledWith(1, 0);
expect(listChain.leftJoin.mock.calls[2]?.[0]).toBe(userBookStatus); expect(subqueryChain.offset).toHaveBeenNthCalledWith(2, 0);
expect(whereValues).toEqual(expect.arrayContaining(['reading', 'rereading', 'on_hold', 'read', 'skimmed', 'abandoned'])); expect(statusWhereValues).toEqual(expect.arrayContaining(['reading', 'rereading', 'on_hold', 'read', 'skimmed', 'abandoned']));
expect(listChain.orderBy).toHaveBeenCalledTimes(1); expect(queryText).toContain('not exists');
expect(listChain.limit).toHaveBeenCalledWith(3); expect(queryText).not.toContain('left join "user_book_status"');
expect(queryText).toContain('where forward_candidate.id is null');
expect(queryText).toContain('order by "books"."id" desc');
expect(Math.random).toHaveBeenCalledTimes(9);
});
it('caps distributed candidate probes for large limits', async () => {
vi.spyOn(Math, 'random').mockReturnValue(0.5);
const boundsChain = makeBoundsChain(1, 1000);
const subqueryChain = makeLimitChain([]);
const db = {
select: vi.fn().mockReturnValueOnce(boundsChain).mockReturnValue(subqueryChain),
execute: vi.fn().mockResolvedValue({
rows: Array.from({ length: 50 }, (_, index) => ({ sampleIndex: index, id: index + 1 })),
}),
};
const repo = new DashboardRepository(db as never);
await expect(repo.findRandomBookIds([5], 7, 50)).resolves.toHaveLength(50);
expect(Math.random).toHaveBeenCalledTimes(60);
});
it('fills a sparse distributed sample from the nearest lower ids', async () => {
vi.spyOn(Math, 'random').mockReturnValue(0.99);
const boundsChain = makeBoundsChain(1, 100);
const subqueryChain = makeLimitChain([]);
const afterPivot = makeLimitChain([]);
const beforePivot = makeLimitChain([{ id: 9 }, { id: 2 }]);
const db = {
select: vi
.fn()
.mockReturnValueOnce(boundsChain)
.mockReturnValueOnce(subqueryChain)
.mockReturnValueOnce(subqueryChain)
.mockReturnValueOnce(afterPivot)
.mockReturnValueOnce(beforePivot),
execute: vi.fn().mockResolvedValue({
rows: [
{ sampleIndex: 0, id: 100 },
{ sampleIndex: 1, id: 100 },
{ sampleIndex: 2, id: 100 },
],
}),
};
const repo = new DashboardRepository(db as never);
const result = await repo.findRandomBookIds([5], 7, 3);
expect(result).toEqual([100, 9, 2]);
expect(afterPivot.limit).toHaveBeenCalledWith(2);
expect(beforePivot.limit).toHaveBeenCalledWith(2);
expect(collectValues(afterPivot.orderBy.mock.calls[0]?.[0])).toContain(' asc');
expect(collectValues(beforePivot.orderBy.mock.calls[0]?.[0])).toContain(' desc');
expect(beforePivot.orderBy.mock.calls[0]?.[0]).toEqual(expect.objectContaining({ queryChunks: expect.arrayContaining([books.id]) }));
}); });
it('returns empty random ids and does not query when limit is zero', async () => { it('returns empty random ids and does not query when limit is zero', async () => {
@@ -1,5 +1,5 @@
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import { and, desc, eq, inArray, isNull, notInArray, or, sql } from 'drizzle-orm'; import { and, asc, desc, eq, gte, inArray, isNull, lt, max, min, notExists, notInArray, or, sql, type SQL } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres'; import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { BOOK_FORMATS, isAudioFormat, type ContentFilterRules, type ReadStatus } from '@bookorbit/types'; import { BOOK_FORMATS, isAudioFormat, type ContentFilterRules, type ReadStatus } from '@bookorbit/types';
@@ -10,12 +10,19 @@ import { buildContentFilterClauses } from '../../common/utils/content-filter-sql
type Db = NodePgDatabase<typeof schema>; type Db = NodePgDatabase<typeof schema>;
type UpNextInSeriesRow = { id: number }; type UpNextInSeriesRow = { id: number };
type RandomCandidateRow = { sampleIndex: number; id: number };
const AUDIO_FORMATS = BOOK_FORMATS.filter(isAudioFormat); const AUDIO_FORMATS = BOOK_FORMATS.filter(isAudioFormat);
const CONTINUE_READING_EXCLUDED_READ_STATUSES = ['unread', 'read', 'skimmed', 'abandoned'] as const satisfies readonly ReadStatus[]; const CONTINUE_READING_EXCLUDED_READ_STATUSES = ['unread', 'read', 'skimmed', 'abandoned'] as const satisfies readonly ReadStatus[];
const DISCOVERY_EXCLUDED_READ_STATUSES = ['reading', 'rereading', 'on_hold', 'read', 'skimmed', 'abandoned'] as const satisfies readonly ReadStatus[]; const DISCOVERY_EXCLUDED_READ_STATUSES = ['reading', 'rereading', 'on_hold', 'read', 'skimmed', 'abandoned'] as const satisfies readonly ReadStatus[];
// Three independent pivots per requested row tolerate moderate collisions from
// sparse eligibility. The ceiling prevents large requests from multiplying probes.
const RANDOM_PIVOT_MULTIPLIER = 3;
const RANDOM_MAX_PIVOTS = 60;
@Injectable() @Injectable()
export class DashboardRepository { export class DashboardRepository {
private readonly randomIdBounds = new Map<string, { minId: number; maxId: number; expiresAt: number }>();
constructor(@Inject(DB) private readonly db: Db) {} constructor(@Inject(DB) private readonly db: Db) {}
async findRecentlyAddedBookIds(accessibleLibraryIds: number[], limit: number, contentFilters?: ContentFilterRules): Promise<number[]> { async findRecentlyAddedBookIds(accessibleLibraryIds: number[], limit: number, contentFilters?: ContentFilterRules): Promise<number[]> {
@@ -229,25 +236,117 @@ export class DashboardRepository {
if (accessibleLibraryIds.length === 0) return []; if (accessibleLibraryIds.length === 0) return [];
if (limit <= 0) return []; if (limit <= 0) return [];
const bounds = await this.findRandomIdBounds(accessibleLibraryIds);
if (!bounds) return [];
const cfClauses = contentFilters ? buildContentFilterClauses(contentFilters, this.db) : []; const cfClauses = contentFilters ? buildContentFilterClauses(contentFilters, this.db) : [];
const rows = await this.db // OFFSET 0 keeps these as correlated index lookups. Without it PostgreSQL
.select({ id: books.id }) // can flatten them into anti-joins that rescan a user's full status set.
.from(books) const hasNoProgress = notExists(
.leftJoin(bookFiles, eq(bookFiles.id, books.primaryFileId)) this.db
.leftJoin(readingProgress, and(eq(readingProgress.bookFileId, bookFiles.id), eq(readingProgress.userId, userId))) .select({ one: sql`1` })
.leftJoin(userBookStatus, and(eq(userBookStatus.bookId, books.id), eq(userBookStatus.userId, userId))) .from(readingProgress)
.where( .where(and(eq(readingProgress.bookFileId, books.primaryFileId), eq(readingProgress.userId, userId), sql`${readingProgress.percentage} <> 0`))
and( .offset(0),
inArray(books.libraryId, accessibleLibraryIds), );
eq(books.status, 'present'), const hasNoExcludedReadStatus = notExists(
or(isNull(readingProgress.bookFileId), eq(readingProgress.percentage, 0)), this.db
or(isNull(userBookStatus.bookId), notInArray(userBookStatus.status, [...DISCOVERY_EXCLUDED_READ_STATUSES])), .select({ one: sql`1` })
...cfClauses, .from(userBookStatus)
), .where(
) and(
.orderBy(sql`random()`) eq(userBookStatus.bookId, books.id),
.limit(limit); eq(userBookStatus.userId, userId),
inArray(userBookStatus.status, [...DISCOVERY_EXCLUDED_READ_STATUSES]),
),
)
.offset(0),
);
const eligibility = and(
inArray(books.libraryId, accessibleLibraryIds),
eq(books.status, 'present'),
hasNoProgress,
hasNoExcludedReadStatus,
...cfClauses,
)!;
return rows.map((row) => row.id); // Independent indexed pivots distribute membership across the id range without
// paying for a full ORDER BY random() or returning one scan-order neighborhood.
const pivotCount = Math.min(limit * RANDOM_PIVOT_MULTIPLIER, RANDOM_MAX_PIVOTS);
const pivots = Array.from({ length: pivotCount }, () => this.randomPivot(bounds));
const sampledIds = [...new Set(await this.findDistributedRandomCandidates(pivots, eligibility))];
if (sampledIds.length >= limit) return sampledIds.slice(0, limit);
const fallbackPivot = this.randomPivot(bounds);
const findFallbackCandidates = (idPredicate: SQL, orderBy: SQL, candidateLimit: number) =>
this.db
.select({ id: books.id })
.from(books)
.where(and(eligibility, sampledIds.length > 0 ? notInArray(books.id, sampledIds) : undefined, idPredicate))
.orderBy(orderBy)
.limit(candidateLimit);
const remaining = limit - sampledIds.length;
const fallbackRows = await findFallbackCandidates(gte(books.id, fallbackPivot), asc(books.id), remaining);
if (fallbackRows.length < remaining) {
fallbackRows.push(...(await findFallbackCandidates(lt(books.id, fallbackPivot), desc(books.id), remaining - fallbackRows.length)));
}
return [...sampledIds, ...fallbackRows.map((row) => row.id)].slice(0, limit);
}
private async findDistributedRandomCandidates(pivots: number[], eligibility: SQL): Promise<number[]> {
const pivotValues = sql.join(
pivots.map((pivot, sampleIndex) => sql`(${sampleIndex}::integer, ${pivot}::integer)`),
sql`, `,
);
const result = await this.db.execute<RandomCandidateRow>(sql`
with pivots(sample_index, pivot_id) as (
values ${pivotValues}
)
select p.sample_index as "sampleIndex", coalesce(forward_candidate.id, wrap_candidate.id) as id
from pivots p
left join lateral (
select ${books.id} as id
from ${books}
where ${eligibility} and ${books.id} >= p.pivot_id
order by ${books.id} asc
limit 1
) forward_candidate on true
left join lateral (
select ${books.id} as id
from ${books}
where forward_candidate.id is null
and ${eligibility}
and ${books.id} < p.pivot_id
order by ${books.id} desc
limit 1
) wrap_candidate on true
where coalesce(forward_candidate.id, wrap_candidate.id) is not null
order by p.sample_index
`);
return result.rows.map((row) => row.id);
}
private randomPivot(bounds: { minId: number; maxId: number }): number {
return bounds.minId + Math.floor(Math.random() * (bounds.maxId - bounds.minId + 1));
}
private async findRandomIdBounds(accessibleLibraryIds: number[]): Promise<{ minId: number; maxId: number } | null> {
const now = Date.now();
const cacheKey = [...accessibleLibraryIds].sort((left, right) => left - right).join(',');
const cached = this.randomIdBounds.get(cacheKey);
if (cached && cached.expiresAt > now) return cached;
const [row] = await this.db
.select({ minId: min(books.id), maxId: max(books.id) })
.from(books)
.where(and(inArray(books.libraryId, accessibleLibraryIds), eq(books.status, 'present')));
if (row?.minId == null || row.maxId == null) return null;
const bounds = { minId: row.minId, maxId: row.maxId, expiresAt: now + 60_000 };
this.randomIdBounds.set(cacheKey, bounds);
if (this.randomIdBounds.size > 100) this.randomIdBounds.delete(this.randomIdBounds.keys().next().value!);
return bounds;
} }
} }
@@ -42,6 +42,7 @@ function makeService() {
}; };
const smartScopeService = { const smartScopeService = {
executeSmartScope: vi.fn(), executeSmartScope: vi.fn(),
executeSmartScopeBookIds: vi.fn(),
}; };
const service = new DashboardService(dashboardRepo as never, bookReadService as never, libraryService as never, smartScopeService as never); const service = new DashboardService(dashboardRepo as never, bookReadService as never, libraryService as never, smartScopeService as never);
@@ -227,4 +228,69 @@ describe('DashboardService', () => {
expect(result).toEqual([]); expect(result).toEqual([]);
expect(bookReadService.findCardsByBookIds).not.toHaveBeenCalled(); expect(bookReadService.findCardsByBookIds).not.toHaveBeenCalled();
}); });
it('batches shelf selection and hydrates overlapping books once', async () => {
const { service, dashboardRepo, bookReadService, libraryService } = makeService();
const user = makeUser({ id: 8 });
libraryService.findAccessibleLibraryIds.mockResolvedValue([10]);
dashboardRepo.findRecentlyAddedBookIds.mockResolvedValue([9, 3]);
dashboardRepo.findWantToReadBookIds.mockResolvedValue([3, 7]);
bookReadService.findCardsByBookIds.mockResolvedValue(makeFindCardsResult([3, 7, 9]));
const result = await service.getScrollers(
{
items: [
{ id: 'recent', type: 'recently-added', limit: 20 },
{ id: 'wanted', type: 'want-to-read', limit: 20 },
],
},
user,
);
expect(libraryService.findAccessibleLibraryIds).toHaveBeenCalledOnce();
expect(bookReadService.findCardsByBookIds).toHaveBeenCalledExactlyOnceWith([9, 3, 7], 8);
expect(result.items.map((item) => ({ id: item.id, ids: item.books.map((book) => book.id), failed: item.failed }))).toEqual([
{ id: 'recent', ids: [9, 3], failed: false },
{ id: 'wanted', ids: [3, 7], failed: false },
]);
});
it('keeps successful shelves when one batched selection fails', async () => {
const { service, dashboardRepo, bookReadService, libraryService, smartScopeService } = makeService();
const user = makeUser({ id: 8 });
libraryService.findAccessibleLibraryIds.mockResolvedValue([10]);
dashboardRepo.findRecentlyAddedBookIds.mockResolvedValue([9]);
smartScopeService.executeSmartScopeBookIds.mockRejectedValue(new Error('scope unavailable'));
bookReadService.findCardsByBookIds.mockResolvedValue(makeFindCardsResult([9]));
const result = await service.getScrollers(
{
items: [
{ id: 'recent', type: 'recently-added', limit: 20 },
{ id: 'scope', type: 'smart-scope', limit: 20, smartScopeId: 7 },
],
},
user,
);
expect(result.items[0]).toMatchObject({ id: 'recent', failed: false });
expect(result.items[0]?.books.map((book) => book.id)).toEqual([9]);
expect(result.items[1]).toEqual({ id: 'scope', books: [], failed: true });
});
it('rejects duplicate batch item ids', async () => {
const { service } = makeService();
await expect(
service.getScrollers(
{
items: [
{ id: 'same', type: 'recently-added', limit: 20 },
{ id: 'same', type: 'random', limit: 20 },
],
},
makeUser(),
),
).rejects.toThrow(BadRequestException);
});
}); });
@@ -1,18 +1,23 @@
import { BadRequestException, Injectable } from '@nestjs/common'; import { BadRequestException, Injectable, Logger } from '@nestjs/common';
import type { BookCard } from '@bookorbit/types'; import type { BookCard, DashboardScrollerBatchResponse } from '@bookorbit/types';
import type { RequestUser } from '../../common/types/request-user'; import type { RequestUser } from '../../common/types/request-user';
import { mapWithConcurrency } from '../../common/utils/batch.utils';
import { sanitizeLogValue } from '../../common/utils/log-sanitize.utils';
import { BookReadService } from '../book/book-read.service'; import { BookReadService } from '../book/book-read.service';
import { assembleBookCards } from '../book/utils/assemble-book-cards'; import { assembleBookCards } from '../book/utils/assemble-book-cards';
import { SmartScopeService } from '../smart-scope/smart-scope.service'; import { SmartScopeService } from '../smart-scope/smart-scope.service';
import { LibraryService } from '../library/library.service'; import { LibraryService } from '../library/library.service';
import { DashboardRepository } from './dashboard.repository'; import { DashboardRepository } from './dashboard.repository';
import { DASHBOARD_SCROLLER_MAX_LIMIT, type DashboardScrollerBatchDto, type DashboardScrollerBatchItemDto } from './dto/dashboard-scroller-batch.dto';
import { ScrollerType } from './dto/scroller-type.enum'; import { ScrollerType } from './dto/scroller-type.enum';
const MAX_LIMIT = 50; const SCROLLER_QUERY_CONCURRENCY = 3;
@Injectable() @Injectable()
export class DashboardService { export class DashboardService {
private readonly logger = new Logger(DashboardService.name);
constructor( constructor(
private readonly dashboardRepo: DashboardRepository, private readonly dashboardRepo: DashboardRepository,
private readonly bookReadService: BookReadService, private readonly bookReadService: BookReadService,
@@ -31,8 +36,83 @@ export class DashboardService {
return bookIds.map((id) => cardsById.get(id)).filter((card): card is BookCard => card != null); return bookIds.map((id) => cardsById.get(id)).filter((card): card is BookCard => card != null);
} }
async getScrollers(dto: DashboardScrollerBatchDto, user: RequestUser): Promise<DashboardScrollerBatchResponse> {
const startedAt = Date.now();
const requestIds = new Set(dto.items.map((item) => item.id));
if (requestIds.size !== dto.items.length) throw new BadRequestException('Scroller batch item IDs must be unique');
this.logger.debug(
`[dashboard.scroller_batch] [start] userId=${user.id} shelfCount=${dto.items.length} concurrency=${SCROLLER_QUERY_CONCURRENCY} - scroller batch started`,
);
const accessibleLibraryIds = await this.libraryService.findAccessibleLibraryIds(user);
const selections = await mapWithConcurrency(dto.items, SCROLLER_QUERY_CONCURRENCY, async (item) => {
const selectionStartedAt = Date.now();
try {
const bookIds = await this.findBatchScrollerBookIds(item, user, accessibleLibraryIds);
return { item, bookIds, failed: false };
} catch (error) {
const errorClass = error instanceof Error ? error.constructor.name : typeof error;
const message = sanitizeLogValue(error instanceof Error ? error.message : error);
this.logger.warn(
`[dashboard.scroller_query] [fail] userId=${user.id} type=${item.type} smartScopeId=${item.smartScopeId ?? 0} durationMs=${Date.now() - selectionStartedAt} errorClass=${errorClass} error="${message}" - scroller selection failed`,
);
return { item, bookIds: [] as number[], failed: true };
}
});
const uniqueBookIds = [...new Set(selections.flatMap((selection) => selection.bookIds))];
const hydrationStartedAt = Date.now();
this.logger.debug(`[dashboard.card_hydration] [start] userId=${user.id} uniqueBookCount=${uniqueBookIds.length} - shared card hydration started`);
let cards: BookCard[];
try {
cards = await this.loadCardsByIds(uniqueBookIds, user.id);
this.logger.debug(
`[dashboard.card_hydration] [end] userId=${user.id} uniqueBookCount=${uniqueBookIds.length} resultCount=${cards.length} durationMs=${Date.now() - hydrationStartedAt} - shared card hydration completed`,
);
} catch (error) {
const errorClass = error instanceof Error ? error.constructor.name : typeof error;
const message = sanitizeLogValue(error instanceof Error ? error.message : error);
this.logger.warn(
`[dashboard.card_hydration] [fail] userId=${user.id} uniqueBookCount=${uniqueBookIds.length} durationMs=${Date.now() - hydrationStartedAt} errorClass=${errorClass} error="${message}" - shared card hydration failed`,
);
throw error;
}
const cardsById = new Map(cards.map((card) => [card.id, card]));
const items = selections.map(({ item, bookIds, failed }) => ({
id: item.id,
books: bookIds.map((id) => cardsById.get(id)).filter((card): card is BookCard => card != null),
failed,
}));
this.logger.debug(
`[dashboard.scroller_batch] [end] userId=${user.id} shelfCount=${items.length} failedCount=${items.filter((item) => item.failed).length} uniqueBookCount=${uniqueBookIds.length} durationMs=${Date.now() - startedAt} - scroller batch completed`,
);
return { items };
}
private async findBatchScrollerBookIds(item: DashboardScrollerBatchItemDto, user: RequestUser, accessibleLibraryIds: number[]): Promise<number[]> {
const startedAt = Date.now();
this.logger.debug(
`[dashboard.scroller_query] [start] userId=${user.id} type=${item.type} smartScopeId=${item.smartScopeId ?? 0} limit=${item.limit} - scroller selection started`,
);
let bookIds: number[];
if (item.type === ScrollerType.SMART_SCOPE) {
const smartScopeId = this.assertSmartScopeId(item.smartScopeId);
bookIds = await this.smartScopeService.executeSmartScopeBookIds(smartScopeId, user, item.limit);
} else {
bookIds = await this.findScrollerBookIdsForLibraries(item.type, user, item.limit, accessibleLibraryIds);
}
this.logger.debug(
`[dashboard.scroller_query] [end] userId=${user.id} type=${item.type} smartScopeId=${item.smartScopeId ?? 0} resultCount=${bookIds.length} durationMs=${Date.now() - startedAt} - scroller selection completed`,
);
return bookIds;
}
async getScroller(type: ScrollerType, user: RequestUser, limit: number, smartScopeId?: number): Promise<BookCard[]> { async getScroller(type: ScrollerType, user: RequestUser, limit: number, smartScopeId?: number): Promise<BookCard[]> {
const clampedLimit = Math.min(Math.max(1, limit), MAX_LIMIT); const clampedLimit = Math.min(Math.max(1, limit), DASHBOARD_SCROLLER_MAX_LIMIT);
if (type === ScrollerType.SMART_SCOPE) { if (type === ScrollerType.SMART_SCOPE) {
const result = await this.smartScopeService.executeSmartScope(this.assertSmartScopeId(smartScopeId), user, 0, clampedLimit); const result = await this.smartScopeService.executeSmartScope(this.assertSmartScopeId(smartScopeId), user, 0, clampedLimit);
@@ -42,11 +122,10 @@ export class DashboardService {
return this.loadCardsByIds(await this.findScrollerBookIds(type, user, clampedLimit), user.id); return this.loadCardsByIds(await this.findScrollerBookIds(type, user, clampedLimit), user.id);
} }
// Book-id selection without the web card assembly, so other modules can shape // Book-id selection without web card assembly lets other clients shape the
// the same rows into their own response type. Smart scopes are excluded: they // same rows. Smart scopes stay separate because they have their own access path.
// resolve through SmartScopeService, which returns assembled cards directly.
async getScrollerBookIds(type: Exclude<ScrollerType, 'smart-scope'>, user: RequestUser, limit: number): Promise<number[]> { async getScrollerBookIds(type: Exclude<ScrollerType, 'smart-scope'>, user: RequestUser, limit: number): Promise<number[]> {
return this.findScrollerBookIds(type, user, Math.min(Math.max(1, limit), MAX_LIMIT)); return this.findScrollerBookIds(type, user, Math.min(Math.max(1, limit), DASHBOARD_SCROLLER_MAX_LIMIT));
} }
async getSmartScopeBookIds(smartScopeId: number | undefined, user: RequestUser, limit: number): Promise<number[]> { async getSmartScopeBookIds(smartScopeId: number | undefined, user: RequestUser, limit: number): Promise<number[]> {
@@ -54,7 +133,7 @@ export class DashboardService {
this.assertSmartScopeId(smartScopeId), this.assertSmartScopeId(smartScopeId),
user, user,
0, 0,
Math.min(Math.max(1, limit), MAX_LIMIT), Math.min(Math.max(1, limit), DASHBOARD_SCROLLER_MAX_LIMIT),
); );
return result.items.map((item) => item.id); return result.items.map((item) => item.id);
} }
@@ -68,6 +147,15 @@ export class DashboardService {
private async findScrollerBookIds(type: Exclude<ScrollerType, 'smart-scope'>, user: RequestUser, clampedLimit: number): Promise<number[]> { private async findScrollerBookIds(type: Exclude<ScrollerType, 'smart-scope'>, user: RequestUser, clampedLimit: number): Promise<number[]> {
const accessibleLibraryIds = await this.libraryService.findAccessibleLibraryIds(user); const accessibleLibraryIds = await this.libraryService.findAccessibleLibraryIds(user);
return this.findScrollerBookIdsForLibraries(type, user, clampedLimit, accessibleLibraryIds);
}
private async findScrollerBookIdsForLibraries(
type: Exclude<ScrollerType, 'smart-scope'>,
user: RequestUser,
clampedLimit: number,
accessibleLibraryIds: number[],
): Promise<number[]> {
if (accessibleLibraryIds.length === 0) return []; if (accessibleLibraryIds.length === 0) return [];
const contentFilters = user.isSuperuser ? undefined : user.contentFilters; const contentFilters = user.isSuperuser ? undefined : user.contentFilters;
@@ -0,0 +1,54 @@
import 'reflect-metadata';
import { plainToInstance } from 'class-transformer';
import { validate, type ValidationError } from 'class-validator';
import { DASHBOARD_SCROLLER_BATCH_MAX } from '@bookorbit/types';
import { DASHBOARD_SCROLLER_MAX_LIMIT, DashboardScrollerBatchDto } from './dashboard-scroller-batch.dto';
function constraintNames(errors: ValidationError[]): string[] {
return errors.flatMap((error) => [...Object.keys(error.constraints ?? {}), ...constraintNames(error.children ?? [])]);
}
describe('DashboardScrollerBatchDto', () => {
it('accepts a bounded valid book shelf batch', async () => {
const dto = plainToInstance(DashboardScrollerBatchDto, {
items: [
{ id: 'recent', type: 'recently-added', limit: 20 },
{ id: 'scope', type: 'smart-scope', limit: 50, smartScopeId: 7 },
],
});
await expect(validate(dto)).resolves.toEqual([]);
});
it('rejects batches above the item-count limit', async () => {
const dto = plainToInstance(DashboardScrollerBatchDto, {
items: Array.from({ length: DASHBOARD_SCROLLER_BATCH_MAX + 1 }, (_, index) => ({
id: String(index),
type: 'recently-added',
limit: 20,
})),
});
const errors = await validate(dto);
expect(constraintNames(errors)).toContain('arrayMaxSize');
});
it('rejects an invalid nested scroller type', async () => {
const dto = plainToInstance(DashboardScrollerBatchDto, {
items: [{ id: 'invalid-type', type: 'continue-podcasts', limit: 20 }],
});
expect(constraintNames(await validate(dto))).toContain('isIn');
});
it('rejects a nested limit above the shared maximum', async () => {
const dto = plainToInstance(DashboardScrollerBatchDto, {
items: [{ id: 'too-large', type: 'random', limit: DASHBOARD_SCROLLER_MAX_LIMIT + 1 }],
});
expect(constraintNames(await validate(dto))).toContain('max');
});
});
@@ -0,0 +1,48 @@
import { Type } from 'class-transformer';
import {
ArrayMaxSize,
ArrayMinSize,
IsArray,
IsIn,
IsInt,
IsOptional,
IsString,
Max,
MaxLength,
Min,
MinLength,
ValidateNested,
} from 'class-validator';
import { DASHBOARD_SCROLLER_BATCH_MAX, SCROLLER_TYPES, type ScrollerType } from '@bookorbit/types';
export const DASHBOARD_SCROLLER_MAX_LIMIT = 50;
export class DashboardScrollerBatchItemDto {
@IsString()
@MinLength(1)
@MaxLength(64)
id!: string;
@IsIn(SCROLLER_TYPES)
type!: ScrollerType;
@IsInt()
@Min(1)
@Max(DASHBOARD_SCROLLER_MAX_LIMIT)
limit!: number;
@IsOptional()
@IsInt()
@Min(1)
smartScopeId?: number;
}
export class DashboardScrollerBatchDto {
@IsArray()
@ArrayMinSize(1)
@ArrayMaxSize(DASHBOARD_SCROLLER_BATCH_MAX)
@ValidateNested({ each: true })
@Type(() => DashboardScrollerBatchItemDto)
items!: DashboardScrollerBatchItemDto[];
}
@@ -69,6 +69,7 @@ function makeService() {
}; };
const bookService = { const bookService = {
executeBooksQuery: vi.fn(), executeBooksQuery: vi.fn(),
executeBookIdsQuery: vi.fn(),
executeJumpBucketsQuery: vi.fn(), executeJumpBucketsQuery: vi.fn(),
}; };
@@ -578,6 +579,28 @@ describe('SmartScopeService', () => {
expect(result).toEqual({ items: [], total: 0, page: 0, size: 25 }); expect(result).toEqual({ items: [], total: 0, page: 0, size: 25 });
}); });
it('executeSmartScopeBookIds applies the saved filter and sort without hydrating cards', async () => {
const { service, smartScopeRepo, libraryService, queryBuilder, bookService } = makeService();
const smartScope = makeSmartScope({
id: 5,
userId: 12,
filter: { type: 'group', join: 'AND', rules: [{ type: 'rule', field: 'title', operator: 'contains', value: 'test' }] },
defaultSort: [{ field: 'title', dir: 'asc' }],
});
smartScopeRepo.findById.mockResolvedValue([smartScope]);
libraryService.findAccessibleLibraryIds.mockResolvedValue([9]);
queryBuilder.buildWhere.mockReturnValue('where');
bookService.executeBookIdsQuery.mockResolvedValue([7, 3]);
await expect(service.executeSmartScopeBookIds(5, makeUser({ id: 12 }), 20)).resolves.toEqual([7, 3]);
expect(bookService.executeBookIdsQuery).toHaveBeenCalledWith(12, 'where', {
filter: smartScope.filter,
sort: smartScope.defaultSort,
pagination: { page: 0, size: 20 },
});
expect(bookService.executeBooksQuery).not.toHaveBeenCalled();
});
it('executeSmartScope seeds sort from the smartScope when the request does not override it', async () => { it('executeSmartScope seeds sort from the smartScope when the request does not override it', async () => {
const { service, smartScopeRepo, libraryService, queryBuilder, bookService } = makeService(); const { service, smartScopeRepo, libraryService, queryBuilder, bookService } = makeService();
const smartScope = makeSmartScope({ const smartScope = makeSmartScope({
@@ -779,6 +802,28 @@ describe('SmartScopeService', () => {
expect(bookService.executeBooksQuery).not.toHaveBeenCalled(); expect(bookService.executeBooksQuery).not.toHaveBeenCalled();
}); });
it('rejects an invalid stored filter before querying books', async () => {
const { service, smartScopeRepo, libraryService, queryBuilder, bookService } = makeService();
smartScopeRepo.findById.mockResolvedValue([
makeSmartScope({
id: 5,
userId: 12,
filter: { sort: 'newest', filter: 'downloaded' } as never,
}),
]);
await expect(
service.queryBooks(5, makeUser({ id: 12 }), {
pagination: { page: 0, size: 50 },
sort: [],
}),
).rejects.toBeInstanceOf(BadRequestException);
expect(libraryService.findAccessibleLibraryIds).not.toHaveBeenCalled();
expect(queryBuilder.buildWhere).not.toHaveBeenCalled();
expect(bookService.executeBooksQuery).not.toHaveBeenCalled();
});
it('builds a combined filter and calls executeBooksQuery with the effective query', async () => { it('builds a combined filter and calls executeBooksQuery with the effective query', async () => {
const { service, smartScopeRepo, libraryService, queryBuilder, bookService } = makeService(); const { service, smartScopeRepo, libraryService, queryBuilder, bookService } = makeService();
const smartScope = makeSmartScope({ const smartScope = makeSmartScope({
@@ -209,6 +209,13 @@ export class SmartScopeService {
}); });
} }
async executeSmartScopeBookIds(id: number, user: RequestUser, size: number): Promise<number[]> {
const query: BookQuery = { sort: [], pagination: { page: 0, size } };
const prepared = await this.prepareBooksQuery(id, user, query);
if (!prepared) return [];
return this.bookService.executeBookIdsQuery(user.id, prepared.where, prepared.effectiveQuery);
}
async queryBooks(id: number, user: RequestUser, query: BookQuery): Promise<BooksPage> { async queryBooks(id: number, user: RequestUser, query: BookQuery): Promise<BooksPage> {
const start = Date.now(); const start = Date.now();
this.logger.debug( this.logger.debug(
@@ -262,11 +269,12 @@ export class SmartScopeService {
const smartScope = await this.getSmartScopeOrThrow(id); const smartScope = await this.getSmartScopeOrThrow(id);
this.assertReadAccess(smartScope, user); this.assertReadAccess(smartScope, user);
if (!smartScope.filter) return null; const scopeFilter = validateGroupRule(smartScope.filter);
if (!scopeFilter) return null;
const accessibleLibraryIds = await this.libraryService.findAccessibleLibraryIds(user); const accessibleLibraryIds = await this.libraryService.findAccessibleLibraryIds(user);
const timeZone = resolveTimeZone((user.settings as { timezone?: unknown } | undefined)?.timezone, 'UTC'); const timeZone = resolveTimeZone((user.settings as { timezone?: unknown } | undefined)?.timezone, 'UTC');
const filter = this.combineFilters(smartScope.filter, query.filter); const filter = this.combineFilters(scopeFilter, query.filter);
const effectiveQuery: T = { const effectiveQuery: T = {
...query, ...query,
filter, filter,
+59 -4
View File
@@ -98,6 +98,10 @@ const routeInventory = loadRouteInventory();
const supportedMethods = new Set<SupportedHttpMethod>(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']); const supportedMethods = new Set<SupportedHttpMethod>(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']);
function isSupportedMethod(method: string): method is SupportedHttpMethod {
return supportedMethods.has(method as SupportedHttpMethod);
}
function loadRouteInventory(): RouteInventory { function loadRouteInventory(): RouteInventory {
const inventoryDir = join(currentDir, 'e2e/authorization-matrix/route-inventory'); const inventoryDir = join(currentDir, 'e2e/authorization-matrix/route-inventory');
const manifest = JSON.parse(readFileSync(join(inventoryDir, 'manifest.json'), 'utf8')) as RouteInventoryManifest; const manifest = JSON.parse(readFileSync(join(inventoryDir, 'manifest.json'), 'utf8')) as RouteInventoryManifest;
@@ -113,6 +117,36 @@ function loadRouteInventory(): RouteInventory {
}; };
} }
function liveRouteLabels(app: NestFastifyApplication): string[] {
const allMethodPaths = new Set(routeInventory.routes.filter((route) => route.httpMethod === 'ALL').map((route) => route.path));
const labels = new Set<string>();
const printedRoutes = app.getHttpAdapter().getInstance().printRoutes({ commonPrefix: false });
const pathAtDepth: string[] = [];
// This test-only parser intentionally follows Fastify's printRoutes tree layout.
// A Fastify formatting change should fail parity and be updated here explicitly.
for (const line of printedRoutes.split('\n')) {
const match = line.match(/^((?:│ {3}| {4})*)[├└]── (.+?)(?: \(([^)]+)\))?$/);
if (!match) continue;
const [, indentation = '', pathPart, rawMethods] = match;
const depth = indentation.length / 4;
pathAtDepth.length = depth;
pathAtDepth.push(pathPart ?? '');
if (!rawMethods) continue;
const prefixedPath = pathAtDepth.join('');
if (!prefixedPath?.startsWith('/api/v1')) continue;
const path = prefixedPath.slice('/api/v1'.length) || '/';
if (allMethodPaths.has(path)) continue;
for (const method of rawMethods?.split(', ') ?? []) {
if (isSupportedMethod(method)) labels.add(`${method} ${path}`);
}
}
return [...labels].sort();
}
describe('Authorization matrix (e2e)', () => { describe('Authorization matrix (e2e)', () => {
let ctx: AuthorizationMatrixE2EContext; let ctx: AuthorizationMatrixE2EContext;
let libraryA: CreatedLibrary; let libraryA: CreatedLibrary;
@@ -440,6 +474,31 @@ describe('Authorization matrix (e2e)', () => {
}); });
describe('guard matrix - jwt/permission/library/default-password', () => { describe('guard matrix - jwt/permission/library/default-password', () => {
it('keeps the dashboard scroller inventory in parity with the live application', () => {
expect(routeInventory.routes).toHaveLength(routeInventory.totalRoutes);
const inventoryLabels = routeInventory.routes
.filter((route) => isSupportedMethod(route.httpMethod) && route.path.startsWith('/dashboard/scrollers'))
.map(routeLabel)
.sort();
const liveLabels = liveRouteLabels(ctx.app).filter((label) => label.includes(' /dashboard/scrollers'));
expect(liveLabels).toEqual(inventoryLabels);
});
it('serves the authenticated random scroller through the batch route', async () => {
const response = await ctx.app.inject({
method: 'POST',
url: '/api/v1/dashboard/scrollers/batch',
headers: authHeader(personas.allPermsUser.accessToken),
payload: { items: [{ id: 'random', type: 'random', limit: 3 }] },
});
expect(response.statusCode).toBe(200);
expect(response.json()).toEqual({
items: [{ id: 'random', books: expect.any(Array), failed: false }],
});
});
it('rejects unauthenticated access to every non-public route', async () => { it('rejects unauthenticated access to every non-public route', async () => {
const failures: MatrixFailure[] = []; const failures: MatrixFailure[] = [];
const protectedRoutes = routeInventory.routes.filter((route) => !route.isPublic && isSupportedMethod(route.httpMethod)); const protectedRoutes = routeInventory.routes.filter((route) => !route.isPublic && isSupportedMethod(route.httpMethod));
@@ -1298,10 +1357,6 @@ describe('Authorization matrix (e2e)', () => {
}); });
}); });
function isSupportedMethod(method: string): method is SupportedHttpMethod {
return supportedMethods.has(method as SupportedHttpMethod);
}
function routeLabel(route: Pick<RouteInventoryRoute, 'httpMethod' | 'path'>): string { function routeLabel(route: Pick<RouteInventoryRoute, 'httpMethod' | 'path'>): string {
return `${route.httpMethod} ${route.path}`; return `${route.httpMethod} ${route.path}`;
} }
+71
View File
@@ -0,0 +1,71 @@
import { randomUUID } from 'crypto';
import * as schema from '../src/db/schema';
import { DashboardRepository } from '../src/modules/dashboard/dashboard.repository';
import {
closeAuthorizationMatrixE2EContext,
createAuthorizationMatrixE2EContext,
createLibraryWithFolder,
createUserAndLogin,
grantLibraryAccess,
type AuthorizationMatrixE2EContext,
} from './e2e/authorization-matrix/authorization-matrix-harness';
const SCENARIO_TIMEOUT_MS = 60_000;
describe('Dashboard random scroller (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () => {
let ctx!: AuthorizationMatrixE2EContext;
let repository!: DashboardRepository;
let libraryId!: number;
let userId!: number;
let bookIds!: number[];
beforeAll(async () => {
ctx = await createAuthorizationMatrixE2EContext();
repository = ctx.app.get(DashboardRepository);
const library = await createLibraryWithFolder(ctx, { name: `dashboard-random-${randomUUID()}` });
const user = await createUserAndLogin(ctx);
libraryId = library.libraryId;
userId = user.userId;
await grantLibraryAccess(ctx, userId, libraryId);
const rows = await ctx.db
.insert(schema.books)
.values(
Array.from({ length: 6 }, (_, index) => ({
libraryId,
libraryFolderId: library.libraryFolderId,
folderPath: `dashboard-random-${index}`,
status: 'present' as const,
})),
)
.returning({ id: schema.books.id });
bookIds = rows.map((row) => row.id);
await ctx.db.insert(schema.userBookStatus).values([
{ userId, bookId: bookIds.at(-1)!, status: 'read', source: 'manual' },
{ userId, bookId: bookIds.at(-2)!, status: 'reading', source: 'manual' },
]);
});
afterEach(() => {
vi.restoreAllMocks();
});
afterAll(async () => {
await closeAuthorizationMatrixE2EContext(ctx);
});
it('uses the nearest eligible lower id when the forward probe has no candidate', async () => {
vi.spyOn(Math, 'random').mockReturnValue(0.99);
await expect(repository.findRandomBookIds([libraryId], userId, 1)).resolves.toEqual([bookIds.at(-3)!]);
});
it('uses the forward candidate without requiring the wrap branch', async () => {
vi.spyOn(Math, 'random').mockReturnValue(0);
await expect(repository.findRandomBookIds([libraryId], userId, 1)).resolves.toEqual([bookIds[0]!]);
});
});
@@ -1,7 +1,7 @@
{ {
"totalRoutes": 370, "totalRoutes": 371,
"byPermission": { "byPermission": {
"(none)": 165, "(none)": 166,
"Permission.ManageMetadataConfig": 32, "Permission.ManageMetadataConfig": 32,
"Permission.EmailSend": 30, "Permission.EmailSend": 30,
"Permission.LibraryEditMetadata": 33, "Permission.LibraryEditMetadata": 33,
@@ -215,6 +215,18 @@
"allowDefault": false, "allowDefault": false,
"useGuards": [] "useGuards": []
}, },
{
"file": "server/src/modules/dashboard/dashboard.controller.ts",
"className": "DashboardController",
"methodName": "getScrollers",
"httpMethod": "POST",
"path": "/dashboard/scrollers/batch",
"permissions": [],
"libraryAccess": [],
"isPublic": false,
"allowDefault": false,
"useGuards": []
},
{ {
"file": "server/src/modules/email/email-admin-log.controller.ts", "file": "server/src/modules/email/email-admin-log.controller.ts",
"className": "EmailAdminLogController", "className": "EmailAdminLogController",