chore: merge origin/main into main

Preserve both divergent commits while reconciling the local and remote histories.
This commit is contained in:
neon
2026-09-10 16:45:53 +00:00
40 changed files with 20969 additions and 20 deletions
@@ -18,6 +18,7 @@ import {
X,
} from '@lucide/vue'
import type { CoverAspectRatio, Library, OrganizationMode } from '@bookorbit/types'
import { useLibraryAddedAt } from '../composables/useLibraryAddedAt'
import { useModal } from '@/composables/useModal'
import { api } from '@/lib/api'
import { useLibraryCreator, type LibraryCreatorSectionId } from '../composables/useLibraryCreator'
@@ -120,6 +121,7 @@ const initializing = ref(true)
const initializationWarning = ref<string | null>(null)
const initialFormSnapshot = ref('')
const nestedModalOpen = ref(false)
const addedAtRecompute = useLibraryAddedAt(editingLibraryId)
const attemptedSections = ref(new Set<LibraryCreatorSectionId>())
const sections = computed(() => (mode.value === 'create' ? ALL_SECTIONS.filter((section) => section.id !== 'access') : ALL_SECTIONS))
@@ -149,6 +151,12 @@ const sectionProps = computed(() => ({
organizationMode: form.organizationMode,
organizationModeLocked: mode.value === 'edit',
allowedFormats: form.allowedFormats,
addedAtSource: form.addedAtSource,
canRecomputeAddedAt: mode.value === 'edit',
storedAddedAtSource: creator.storedAddedAtSource.value,
recomputingAddedAt: addedAtRecompute.running.value,
recomputeJob: addedAtRecompute.job.value,
recomputeErrorKey: addedAtRecompute.errorKey.value,
excludePatterns: form.excludePatterns,
},
metadata: { metadataPrecedence: form.metadataPrecedence, formatPriority: form.formatPriority },
@@ -256,6 +264,14 @@ function handleOrganizationModeUpdate(value: OrganizationMode) {
form.organizationMode = value
}
function handleAddedAtSourceUpdate(value: Library['addedAtSource']) {
form.addedAtSource = value
}
function handleRecomputeAddedAt() {
if (!addedAtRecompute.running.value) void addedAtRecompute.start()
}
function handleNestedModalChange(value: boolean) {
nestedModalOpen.value = value
}
@@ -266,6 +282,8 @@ const sectionListeners = {
'update:coverAspectRatio': handleCoverAspectRatioUpdate,
'update:folders': handleFoldersUpdate,
'update:organizationMode': handleOrganizationModeUpdate,
'update:addedAtSource': handleAddedAtSourceUpdate,
recompute: handleRecomputeAddedAt,
'update:metadataPrecedence': (value: string[]) => (form.metadataPrecedence = value),
'update:formatPriority': (value: string[]) => (form.formatPriority = value),
'update:allowedFormats': (value: string[]) => (form.allowedFormats = value),
@@ -1,8 +1,9 @@
<script setup lang="ts">
import { ref } from 'vue'
import { computed, nextTick, ref } from 'vue'
import { useI18n } from 'vue-i18n'
import { Lock, Plus, X } from '@lucide/vue'
import type { OrganizationMode } from '@bookorbit/types'
import { Lock, Plus, RefreshCw, X } from '@lucide/vue'
import type { AddedAtRecomputeJob, AddedAtSource, OrganizationMode } from '@bookorbit/types'
import { formatNumber } from '@/i18n/formatters'
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'
import { FORMAT_LABELS } from '../composables/useLibraryCreator'
@@ -12,15 +13,69 @@ const props = defineProps<{
organizationMode: OrganizationMode
organizationModeLocked?: boolean
allowedFormats: string[]
addedAtSource: AddedAtSource
canRecomputeAddedAt?: boolean
storedAddedAtSource?: AddedAtSource | null
recomputingAddedAt?: boolean
recomputeJob?: AddedAtRecomputeJob | null
recomputeErrorKey?: string | null
excludePatterns: string[]
}>()
const emit = defineEmits<{
'update:organizationMode': [value: OrganizationMode]
'update:allowedFormats': [value: string[]]
'update:addedAtSource': [value: AddedAtSource]
'update:excludePatterns': [value: string[]]
recompute: []
}>()
const confirmingRecompute = ref(false)
const confirmButton = ref<HTMLButtonElement | null>(null)
const sourceGroup = ref<HTMLElement | null>(null)
const recomputeButton = ref<HTMLButtonElement | null>(null)
const progressText = computed(() => {
const job = props.recomputeJob
if (!job) return ''
const counters = ['processed', 'total', 'updated', 'unchanged', 'skipped', 'failed'] as const
return t('library.creator.scanner.addedAt.progressSummary', Object.fromEntries(counters.map((key) => [key, formatNumber(job[key])])))
})
const ADDED_AT_SOURCES: AddedAtSource[] = ['imported', 'file_modified', 'file_created']
const recomputeDisabled = computed(
() =>
props.recomputingAddedAt ||
!props.storedAddedAtSource ||
props.storedAddedAtSource === 'imported' ||
props.addedAtSource !== props.storedAddedAtSource,
)
const showSaveFirstHint = computed(() => props.canRecomputeAddedAt && props.addedAtSource !== props.storedAddedAtSource)
function selectAddedAtSource(source: AddedAtSource) {
emit('update:addedAtSource', source)
}
async function requestRecompute() {
confirmingRecompute.value = true
await nextTick()
confirmButton.value?.focus()
}
async function cancelRecompute() {
confirmingRecompute.value = false
await nextTick()
recomputeButton.value?.focus()
}
async function confirmRecompute() {
if (recomputeDisabled.value) return
confirmingRecompute.value = false
emit('recompute')
await nextTick()
sourceGroup.value?.querySelector<HTMLInputElement>('input:checked')?.focus()
}
// ── Scan mode ─────────────────────────────────────────────────────────────────
function handleSelectMode(mode: OrganizationMode) {
@@ -156,6 +211,100 @@ function onPatternKeydown(e: KeyboardEvent) {
</div>
</div>
<div>
<p id="added-at-title" class="text-[11px] font-semibold uppercase tracking-widest text-foreground mb-1">
{{ t('library.creator.scanner.addedAt.title') }}
</p>
<p id="added-at-hint" class="text-xs text-muted-foreground mb-3">{{ t('library.creator.scanner.addedAt.hint') }}</p>
<div ref="sourceGroup" role="radiogroup" aria-labelledby="added-at-title" aria-describedby="added-at-hint" class="space-y-2">
<label
v-for="source in ADDED_AT_SOURCES"
:key="source"
class="block w-full cursor-pointer rounded-lg border p-3 text-start transition-colors focus-within:ring-2 focus-within:ring-ring"
:class="addedAtSource === source ? 'border-primary bg-primary/5 ring-1 ring-primary' : 'border-border bg-card hover:border-primary/40'"
>
<span class="mb-1 flex items-center gap-2">
<input
type="radio"
name="added-at-source"
:value="source"
:checked="addedAtSource === source"
class="accent-primary"
@change="selectAddedAtSource(source)"
/>
<span class="text-sm font-semibold text-foreground">{{ t(`library.creator.scanner.addedAt.options.${source}.title`) }}</span>
</span>
<span class="block ps-5.5 text-xs text-muted-foreground leading-relaxed">{{
t(`library.creator.scanner.addedAt.options.${source}.hint`)
}}</span>
</label>
</div>
<div v-if="canRecomputeAddedAt" class="mt-3 flex flex-wrap items-center gap-3">
<button
ref="recomputeButton"
type="button"
class="flex items-center gap-1.5 px-3 py-2 rounded-md border border-border text-sm font-medium text-foreground hover:bg-muted transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-50"
:disabled="recomputeDisabled"
@click="requestRecompute"
>
<RefreshCw :size="13" :class="recomputingAddedAt ? 'motion-safe:animate-spin' : ''" />
{{ recomputingAddedAt ? t('library.creator.scanner.addedAt.recomputing') : t('library.creator.scanner.addedAt.recompute') }}
</button>
<p class="text-xs text-muted-foreground">
{{ showSaveFirstHint ? t('library.creator.scanner.addedAt.recomputeSaveFirst') : t('library.creator.scanner.addedAt.recomputeHint') }}
</p>
</div>
<div
v-if="confirmingRecompute"
role="group"
:aria-label="t('library.creator.scanner.addedAt.recompute')"
class="mt-3 rounded-md border border-border p-3"
>
<p class="text-sm text-muted-foreground">{{ t('library.creator.scanner.addedAt.backgroundConfirm') }}</p>
<div class="mt-3 flex flex-wrap gap-2">
<button
ref="confirmButton"
type="button"
:disabled="recomputeDisabled"
class="rounded-md border border-border px-3 py-2 text-sm text-foreground hover:bg-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring disabled:opacity-50"
@click="confirmRecompute"
>
{{ t('library.creator.scanner.addedAt.recompute') }}
</button>
<button
type="button"
class="rounded-md px-3 py-2 text-sm text-foreground hover:bg-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
@click="cancelRecompute"
>
{{ t('common.cancel') }}
</button>
</div>
</div>
<div v-if="recomputeJob" role="status" class="mt-3 space-y-2 text-sm text-muted-foreground">
<p>{{ t(`library.creator.scanner.addedAt.jobStatus.${recomputeJob.status}`) }}</p>
<progress
v-if="recomputeJob.status === 'running'"
:value="recomputeJob.total ? recomputeJob.processed : undefined"
:max="recomputeJob.total || 1"
:aria-label="t('library.creator.scanner.addedAt.recomputing')"
class="h-2 w-full accent-primary"
/>
<p>{{ progressText }}</p>
<p v-if="recomputeJob.failed > 0">{{ t('library.creator.scanner.addedAt.failedBooksHint') }}</p>
<ul v-if="recomputeJob.failureSamples.length" class="space-y-1">
<li v-for="failure in recomputeJob.failureSamples" :key="failure.bookId">
{{
t('library.creator.scanner.addedAt.failureSample', {
bookId: formatNumber(failure.bookId),
reason: t(`library.creator.scanner.addedAt.failureReasons.${failure.code}`),
})
}}
</li>
</ul>
</div>
<p v-if="recomputeErrorKey" role="alert" class="mt-3 text-sm text-destructive">{{ t(recomputeErrorKey) }}</p>
</div>
<!-- Filtering group -->
<div>
<!-- Allowed formats -->
@@ -1,5 +1,8 @@
import { flushPromises, shallowMount } from '@vue/test-utils'
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { Library } from '@bookorbit/types'
import { useLibraryCreator } from '../../composables/useLibraryCreator'
import LibraryCreatorScanner from '../LibraryCreatorScanner.vue'
import LibraryCreatorDetails from '../LibraryCreatorDetails.vue'
import LibraryCreatorModal from '../LibraryCreatorModal.vue'
@@ -14,6 +17,24 @@ describe('LibraryCreatorModal', () => {
vi.restoreAllMocks()
})
it('uses fetched saved settings for delegated managers whose library list omits the source', async () => {
const full = { ...useLibraryCreator().form, id: 7, name: 'Library', icon: 'BookOpen', folders: [], addedAtSource: 'file_created' }
apiMock.mockImplementation(
async (url) => ({ ok: true, json: async () => (String(url).endsWith('recompute-added-at') ? null : full) }) as Response,
)
const listEntry = { id: 7, name: 'Library', folders: [], coverAspectRatio: '2/3' } as unknown as Library
const wrapper = shallowMount(LibraryCreatorModal, { props: { library: listEntry }, global: { stubs: { teleport: true } } })
await flushPromises()
await wrapper
.findAll('nav button')
.find((button) => button.text().includes('Scanning'))!
.trigger('click')
const scanner = wrapper.getComponent(LibraryCreatorScanner)
expect(scanner.props('addedAtSource')).toBe('file_created')
expect(scanner.props('storedAddedAtSource')).toBe('file_created')
wrapper.unmount()
})
it('renders an accessible responsive dialog with required setup identified', async () => {
const wrapper = shallowMount(LibraryCreatorModal, {
attachTo: document.body,
@@ -15,6 +15,7 @@ function mountScanner(overrides: Partial<InstanceType<typeof LibraryCreatorScann
organizationMode: 'book_per_folder',
organizationModeLocked: false,
allowedFormats: [],
addedAtSource: 'imported',
excludePatterns: [],
...overrides,
},
@@ -45,4 +46,41 @@ describe('LibraryCreatorScanner', () => {
expect(wrapper.emitted('update:organizationMode')).toBeUndefined()
})
it('enables recompute only for saved file sources and requires confirmation', async () => {
const wrapper = mountScanner({ addedAtSource: 'file_created', storedAddedAtSource: 'file_created', canRecomputeAddedAt: true })
const recompute = wrapper.findAll('button').find((button) => button.text() === 'Recompute now')!
expect(recompute.attributes('disabled')).toBeUndefined()
await recompute.trigger('click')
expect(wrapper.emitted('recompute')).toBeUndefined()
await wrapper.get('[role="group"] button').trigger('click')
expect(wrapper.emitted('recompute')).toEqual([[]])
await wrapper.setProps({ addedAtSource: 'file_modified' })
expect(recompute.attributes('disabled')).toBeDefined()
wrapper.unmount()
})
it('uses native radios and exposes partial failures as accessible status', async () => {
const wrapper = mountScanner({
addedAtSource: 'file_created',
recomputeJob: {
id: 'job',
libraryId: 1,
source: 'file_created',
status: 'completed',
total: 2,
processed: 2,
updated: 1,
unchanged: 0,
skipped: 0,
failed: 1,
failureSamples: [],
},
})
const radios = wrapper.findAll('input[type="radio"]')
expect(radios).toHaveLength(3)
await radios[1]!.setValue()
expect(wrapper.emitted('update:addedAtSource')).toEqual([['file_modified']])
expect(wrapper.get('[role="status"]').text()).toContain('1 failed')
wrapper.unmount()
})
})
@@ -20,6 +20,7 @@ function makeLibrary(overrides: Partial<Library> = {}): Library {
formatPriority: [],
allowedFormats: [],
organizationMode: 'book_per_file',
addedAtSource: 'imported',
excludePatterns: [],
readingThreshold: 10,
markAsFinishedPercentComplete: 95,
@@ -0,0 +1,121 @@
import { effectScope, ref } from 'vue'
import { flushPromises } from '@vue/test-utils'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { AddedAtRecomputeJob } from '@bookorbit/types'
import { useLibraryAddedAt } from '../useLibraryAddedAt'
const apiMock = vi.hoisted(() => vi.fn<(input: RequestInfo | URL, init?: RequestInit) => Promise<{ ok: boolean; json: () => Promise<unknown> }>>())
vi.mock('@/lib/api', () => ({ api: apiMock }))
const job: AddedAtRecomputeJob = {
id: 'job',
libraryId: 1,
source: 'file_modified',
status: 'running',
total: 100,
processed: 0,
updated: 0,
unchanged: 0,
skipped: 0,
failed: 0,
failureSamples: [],
}
const response = (value: unknown, ok = true) => ({ ok, json: async () => value })
let scope: ReturnType<typeof effectScope>
beforeEach(() => {
vi.useFakeTimers()
apiMock.mockReset()
scope = effectScope()
})
afterEach(() => {
scope.stop()
vi.useRealTimers()
})
function setup() {
const id = ref<number | null>(1)
const state = scope.run(() => useLibraryAddedAt(id))!
return { ...state, id }
}
describe('useLibraryAddedAt', () => {
it('resumes an active background job and stops polling when complete', async () => {
apiMock.mockResolvedValueOnce(response(job)).mockResolvedValueOnce(response({ ...job, status: 'completed', processed: 100, updated: 100 }))
const state = setup()
await flushPromises()
expect(state.running.value).toBe(true)
await vi.advanceTimersByTimeAsync(1000)
expect(state.job.value).toMatchObject({ status: 'completed', processed: 100 })
expect(state.running.value).toBe(false)
await vi.advanceTimersByTimeAsync(10_000)
expect(apiMock).toHaveBeenCalledTimes(2)
})
it('starts a job and does not submit duplicates while running', async () => {
apiMock.mockResolvedValueOnce(response(null)).mockResolvedValue(response(job))
const state = setup()
await flushPromises()
await state.start()
await state.start()
expect(apiMock).toHaveBeenCalledTimes(2)
expect(apiMock.mock.calls[1][1]).toMatchObject({ method: 'POST' })
expect(state.running.value).toBe(true)
})
it('localizes structured errors without displaying backend text', async () => {
apiMock
.mockResolvedValueOnce(response(null))
.mockResolvedValueOnce(response({ errorCode: 'ADDED_AT_BUSY', message: 'Internal server detail' }, false))
const state = setup()
await flushPromises()
await state.start()
expect(state.errorKey.value).toBe('library.creator.scanner.addedAt.errors.busy')
expect(state.running.value).toBe(false)
})
it('retains progress through a polling failure and reconnects', async () => {
apiMock
.mockResolvedValueOnce(response(job))
.mockRejectedValueOnce(new Error('network'))
.mockResolvedValue(response({ ...job, processed: 50 }))
const state = setup()
await flushPromises()
await vi.advanceTimersByTimeAsync(1000)
expect(state.job.value?.processed).toBe(0)
expect(state.errorKey.value).toBeTruthy()
await vi.advanceTimersByTimeAsync(5000)
expect(state.job.value?.processed).toBe(50)
expect(state.errorKey.value).toBeNull()
})
it('reports lost job state after a server restart instead of claiming success', async () => {
apiMock.mockResolvedValueOnce(response(job)).mockResolvedValueOnce(response(null))
const state = setup()
await flushPromises()
await vi.advanceTimersByTimeAsync(1000)
expect(state.running.value).toBe(false)
expect(state.errorKey.value).toBe('library.creator.scanner.addedAt.errors.interrupted')
})
it('ignores a stale response after switching libraries and aborts on disposal', async () => {
let resolveOld!: (value: ReturnType<typeof response>) => void
apiMock
.mockImplementationOnce(
() =>
new Promise((resolve) => {
resolveOld = resolve
}),
)
.mockResolvedValueOnce(response({ ...job, id: 'new', libraryId: 2 }))
const state = setup()
state.id.value = 2
await flushPromises()
resolveOld(response(job))
await flushPromises()
expect(state.job.value?.libraryId).toBe(2)
scope.stop()
expect(apiMock.mock.calls[1]?.[1]?.signal?.aborted).toBe(true)
await vi.advanceTimersByTimeAsync(10_000)
expect(apiMock).toHaveBeenCalledTimes(2)
})
})
@@ -140,6 +140,7 @@ describe('useLibraryCreator', () => {
formatPriority: ['epub'],
allowedFormats: ['epub'],
organizationMode: 'book_per_folder',
addedAtSource: 'imported',
excludePatterns: [],
readingThreshold: 0.25,
markAsFinishedPercentComplete: 98,
@@ -252,6 +253,7 @@ function makeLibrary(overrides: Partial<Library> = {}): Library {
formatPriority: ['epub'],
allowedFormats: ['epub'],
organizationMode: 'book_per_folder',
addedAtSource: 'imported',
excludePatterns: [],
readingThreshold: 0.25,
markAsFinishedPercentComplete: 98,
@@ -0,0 +1,84 @@
import { computed, onScopeDispose, ref, watch, type Ref } from 'vue'
import type { AddedAtRecomputeJob } from '@bookorbit/types'
import { api } from '@/lib/api'
const ERROR_KEYS: Record<string, string> = {
ADDED_AT_SOURCE_IMPORTED: 'sourceImported',
ADDED_AT_ALREADY_RUNNING: 'alreadyRunning',
ADDED_AT_BUSY: 'busy',
}
const ERROR_PREFIX = 'library.creator.scanner.addedAt.errors.'
export function useLibraryAddedAt(libraryId: Ref<number | null>) {
const job = ref<AddedAtRecomputeJob | null>(null)
const starting = ref(false)
const errorKey = ref<string | null>(null)
const running = computed(() => starting.value || job.value?.status === 'running')
let timer: ReturnType<typeof setTimeout> | undefined
let controller: AbortController | undefined
let generation = 0
let disposed = false
function cancelRequest() {
clearTimeout(timer)
controller?.abort()
generation++
}
async function request(method: 'GET' | 'POST') {
cancelRequest()
const id = libraryId.value
if (id === null || disposed) return
const current = generation
const currentController = new AbortController()
controller = currentController
const signal = currentController.signal
const timeout = setTimeout(() => currentController.abort(), 10_000)
starting.value = method === 'POST'
errorKey.value = null
try {
const response = await api(`/api/v1/libraries/${id}/recompute-added-at`, { method, signal })
if (!response.ok) {
const body = (await response.json().catch(() => ({}))) as { errorCode?: string }
if (current === generation) errorKey.value = ERROR_PREFIX + (ERROR_KEYS[body.errorCode ?? ''] ?? 'requestFailed')
return
}
const result = (await response.json()) as AddedAtRecomputeJob | null
if (current !== generation) return
if (!result && job.value?.status === 'running') errorKey.value = ERROR_PREFIX + 'interrupted'
job.value = result
} catch {
if (current === generation) errorKey.value = ERROR_PREFIX + 'requestFailed'
} finally {
clearTimeout(timeout)
if (current === generation && !disposed) {
starting.value = false
if (job.value?.status === 'running') timer = setTimeout(refresh, errorKey.value ? 5000 : 1000)
}
}
}
function refresh() {
return request('GET')
}
function start() {
if (running.value) return Promise.resolve()
return request('POST')
}
watch(
libraryId,
() => {
job.value = null
void refresh()
},
{ immediate: true },
)
onScopeDispose(() => {
disposed = true
cancelRequest()
})
return { job, running, errorKey, start, refresh }
}
@@ -1,7 +1,7 @@
import { computed, reactive, ref } from 'vue'
import { api } from '@/lib/api'
import { DEFAULT_FORMAT_PRIORITY, FORMAT_LABELS, isFiveFieldCronExpression } from '@bookorbit/types'
import type { CoverAspectRatio, Library, OrganizationMode, PrescanResult } from '@bookorbit/types'
import type { AddedAtSource, CoverAspectRatio, Library, OrganizationMode, PrescanResult } from '@bookorbit/types'
export { DEFAULT_FORMAT_PRIORITY, FORMAT_LABELS }
@@ -30,6 +30,7 @@ function blankForm() {
formatPriority: [...DEFAULT_FORMAT_PRIORITY] as string[],
allowedFormats: [] as string[],
organizationMode: 'book_per_folder' as OrganizationMode,
addedAtSource: 'imported' as AddedAtSource,
excludePatterns: [] as string[],
readingThreshold: 0.25,
markAsFinishedPercentComplete: 98,
@@ -59,6 +60,7 @@ export function useLibraryCreator() {
const prescanLoading = ref(false)
const prescanResult = ref<PrescanResult | null>(null)
const error = ref<string | null>(null)
const storedAddedAtSource = ref<AddedAtSource | null>(null)
const validationErrors = computed<Partial<Record<LibraryCreatorSectionId, string>>>(() => {
const errors: Partial<Record<LibraryCreatorSectionId, string>> = {}
@@ -91,6 +93,7 @@ export function useLibraryCreator() {
})
function initCreate() {
storedAddedAtSource.value = null
Object.assign(form, blankForm())
mode.value = 'create'
editingLibraryId.value = null
@@ -111,6 +114,8 @@ export function useLibraryCreator() {
form.formatPriority = [...library.formatPriority, ...missing]
form.allowedFormats = [...library.allowedFormats]
form.organizationMode = library.organizationMode
form.addedAtSource = library.addedAtSource
storedAddedAtSource.value = library.addedAtSource
form.excludePatterns = [...library.excludePatterns]
form.readingThreshold = library.readingThreshold
form.markAsFinishedPercentComplete = library.markAsFinishedPercentComplete
@@ -206,6 +211,7 @@ export function useLibraryCreator() {
return {
form,
storedAddedAtSource,
mode,
editingLibraryId,
loading,
@@ -76,6 +76,7 @@ function makeLibrary(overrides: Partial<Library> = {}): Library {
formatPriority: [],
allowedFormats: [],
organizationMode: 'book_per_file',
addedAtSource: 'imported',
excludePatterns: [],
readingThreshold: 10,
markAsFinishedPercentComplete: 90,
@@ -17,6 +17,7 @@ function library(overrides: Partial<Library> & Pick<Library, 'id' | 'name'>): Li
formatPriority: [],
allowedFormats: [],
organizationMode: 'book_per_folder',
addedAtSource: 'imported',
excludePatterns: [],
readingThreshold: 10,
markAsFinishedPercentComplete: 90,
@@ -93,6 +93,7 @@ function makeLibrary(): Library {
formatPriority: [],
allowedFormats: [],
organizationMode: 'book_per_file',
addedAtSource: 'imported',
excludePatterns: [],
readingThreshold: 10,
markAsFinishedPercentComplete: 90,
+44
View File
@@ -4127,6 +4127,7 @@
"LibraryAccessRevoke": "Revoke library access",
"LibraryWriteMetadataToFiles": "Write metadata to files",
"LibraryBulkRename": "Rename library files",
"LibraryRecomputeAddedAt": "Recompute date added",
"BookUpload": "Upload book",
"BookMetadataUpdate": "Update book metadata",
"BookMetadataLocksUpdate": "Update metadata locks",
@@ -7975,6 +7976,49 @@
"hint": "Treats every file as an individual book. Best suited for libraries with one format per title. Avoid if your audiobooks span multiple files."
}
},
"addedAt": {
"title": "Date added",
"hint": "Choose what a book's \"date added\" reflects. This applies to books imported from now on.",
"options": {
"imported": {
"title": "When imported",
"hint": "The date BookOrbit first imported the book. This is the default."
},
"file_modified": {
"title": "File modified time",
"hint": "The file's last-modified time (mtime), often preserved when files are copied."
},
"file_created": {
"title": "File creation time",
"hint": "When the file first landed on disk (birthtime). Falls back to modified time on filesystems that don't record creation time."
}
},
"recompute": "Recompute now",
"recomputing": "Recomputing...",
"recomputeHint": "Update the date added for existing books using this setting.",
"recomputeSaveFirst": "Save your changes first, then recompute existing books.",
"backgroundConfirm": "Recompute dates for existing books? This runs in the background, and you can close this editor. Books with unreadable files keep their current date.",
"progressSummary": "Processed {processed} of {total}: {updated} updated, {unchanged} unchanged, {skipped} skipped, {failed} failed.",
"failedBooksHint": "Failed books kept their dates. Check file access, then run recompute again to retry.",
"jobStatus": {
"running": "Recomputing dates in the background.",
"completed": "Date recompute finished.",
"failed": "Recompute stopped before finishing. Completed changes were kept; you can safely run it again."
},
"errors": {
"sourceImported": "Save a file timestamp source before recomputing dates.",
"alreadyRunning": "A recompute is already running for this library.",
"busy": "Other recomputes are running. Try again shortly.",
"requestFailed": "Could not reach the recompute service. Close and reopen this editor to check its status.",
"interrupted": "The job status is no longer available. It may have been interrupted by a server restart. You can safely run recompute again."
},
"failureSample": "Book {bookId}: {reason}",
"failureReasons": {
"file_unavailable": "A content file could not be read.",
"unsafe_path": "A content file is outside its permitted library folder.",
"database_error": "The database update failed."
}
},
"allowedFormats": {
"title": "Allowed formats",
"allowAll": "Allow all",
+1
View File
@@ -42,6 +42,7 @@ export enum AuditAction {
LibraryAccessRevoke = "library.access.revoke",
LibraryWriteMetadataToFiles = "library.write_metadata_to_files",
LibraryBulkRename = "library.bulk_rename",
LibraryRecomputeAddedAt = "library.recompute_added_at",
BookUpload = "book.upload",
BookMetadataUpdate = "book.metadata.update",
+18
View File
@@ -2,6 +2,7 @@ import type { LibraryLastScan } from "./scanner";
export type OrganizationMode = "book_per_file" | "book_per_folder";
export type CoverAspectRatio = "2/3" | "1/1";
export type AddedAtSource = "imported" | "file_modified" | "file_created";
export function normalizeCoverAspectRatio(value: unknown): CoverAspectRatio {
return value === "1/1" ? "1/1" : "2/3";
@@ -64,6 +65,7 @@ export interface Library {
formatPriority: string[];
allowedFormats: string[];
organizationMode: OrganizationMode;
addedAtSource: AddedAtSource;
excludePatterns: string[];
readingThreshold: number;
markAsFinishedPercentComplete: number;
@@ -89,6 +91,22 @@ export interface Library {
updatedAt: string;
}
export type AddedAtRecomputeFailureCode = "file_unavailable" | "unsafe_path" | "database_error";
export interface AddedAtRecomputeJob {
id: string;
libraryId: number;
source: AddedAtSource;
status: "running" | "completed" | "failed";
total: number;
processed: number;
updated: number;
unchanged: number;
skipped: number;
failed: number;
failureSamples: { bookId: number; code: AddedAtRecomputeFailureCode }[];
}
export interface LibraryStats {
totalBooks: number;
totalSizeBytes: number;
@@ -0,0 +1,7 @@
export const MIN_VALID_FILE_TIME_MS = 1000;
export function usableFileTime(date: Date | null | undefined): Date | undefined {
if (!(date instanceof Date)) return undefined;
const ms = date.getTime();
return Number.isFinite(ms) && ms > MIN_VALID_FILE_TIME_MS ? date : undefined;
}
@@ -0,0 +1,2 @@
ALTER TABLE "libraries" ADD COLUMN "added_at_source" varchar(20) DEFAULT 'imported' NOT NULL;--> statement-breakpoint
ALTER TABLE "libraries" ADD CONSTRAINT "libraries_added_at_source_chk" CHECK ("libraries"."added_at_source" in ('imported', 'file_modified', 'file_created'));
File diff suppressed because it is too large Load Diff
@@ -610,6 +610,13 @@
"when": 1788884825211,
"tag": "0086_add_annotation_source_created_at",
"breakpoints": true
},
{
"idx": 87,
"version": "7",
"when": 1789055692244,
"tag": "0087_add_library_added_at_source",
"breakpoints": true
}
]
}
+3 -1
View File
@@ -15,7 +15,7 @@ import {
uniqueIndex,
varchar,
} from 'drizzle-orm/pg-core';
import { FieldPreferenceOverrides, BookMetadataFetchConfigOverride } from '@bookorbit/types';
import { FieldPreferenceOverrides, BookMetadataFetchConfigOverride, AddedAtSource } from '@bookorbit/types';
export const libraries = pgTable(
'libraries',
@@ -41,6 +41,7 @@ export const libraries = pgTable(
.default(['epub', 'pdf', 'cbz', 'cbr', 'cb7', 'mobi', 'azw3', 'azw', 'fb2', 'm4b', 'mp3', 'm4a', 'opus', 'ogg', 'flac']),
allowedFormats: jsonb('allowed_formats').$type<string[]>().notNull().default([]),
organizationMode: varchar('organization_mode', { length: 20 }).notNull().default('book_per_folder'),
addedAtSource: varchar('added_at_source', { length: 20 }).$type<AddedAtSource>().notNull().default('imported'),
excludePatterns: jsonb('exclude_patterns').$type<string[]>().notNull().default([]),
// Reading progress thresholds
@@ -91,6 +92,7 @@ export const libraries = pgTable(
uniqueIndex('libraries_name_lower_uidx').on(sql`lower(${t.name})`),
check('libraries_display_order_nonnegative_chk', sql`${t.displayOrder} >= 0`),
check('libraries_organization_mode_chk', sql`${t.organizationMode} in ('book_per_folder', 'book_per_file')`),
check('libraries_added_at_source_chk', sql`${t.addedAtSource} in ('imported', 'file_modified', 'file_created')`),
check('libraries_reading_threshold_range_chk', sql`${t.readingThreshold} >= 0 and ${t.readingThreshold} <= 100`),
check('libraries_mark_finished_percent_range_chk', sql`${t.markAsFinishedPercentComplete} >= 0 and ${t.markAsFinishedPercentComplete} <= 100`),
check('libraries_scan_mode_chk', sql`${t.scanMode} in ('auto', 'manual')`),
@@ -15,9 +15,10 @@ import {
ValidateIf,
} from 'class-validator';
import { Transform } from 'class-transformer';
import { ICON_VALUE_MAX_LENGTH, type CoverAspectRatio, type OrganizationMode } from '@bookorbit/types';
import { ICON_VALUE_MAX_LENGTH, type AddedAtSource, type CoverAspectRatio, type OrganizationMode } from '@bookorbit/types';
import {
LIBRARY_ADDED_AT_SOURCES,
LIBRARY_COVER_ASPECT_RATIOS,
LIBRARY_FILE_WRITE_MAX_SIZE_MB_MAX,
LIBRARY_FILE_WRITE_MAX_SIZE_MB_MIN,
@@ -90,6 +91,10 @@ export class CreateLibraryDto {
@IsIn(LIBRARY_ORGANIZATION_MODES)
organizationMode?: OrganizationMode;
@ValidateIf((_, value) => value !== undefined)
@IsIn(LIBRARY_ADDED_AT_SOURCES)
addedAtSource?: AddedAtSource;
@IsOptional()
@IsArray()
@IsString({ each: true })
@@ -14,9 +14,10 @@ import {
ValidateIf,
} from 'class-validator';
import { Transform } from 'class-transformer';
import { ICON_VALUE_MAX_LENGTH, type CoverAspectRatio, type OrganizationMode } from '@bookorbit/types';
import { ICON_VALUE_MAX_LENGTH, type AddedAtSource, type CoverAspectRatio, type OrganizationMode } from '@bookorbit/types';
import {
LIBRARY_ADDED_AT_SOURCES,
LIBRARY_COVER_ASPECT_RATIOS,
LIBRARY_FILE_WRITE_MAX_SIZE_MB_MAX,
LIBRARY_FILE_WRITE_MAX_SIZE_MB_MIN,
@@ -91,6 +92,10 @@ export class UpdateLibraryDto {
@IsIn(LIBRARY_ORGANIZATION_MODES)
organizationMode?: OrganizationMode;
@ValidateIf((_, value) => value !== undefined)
@IsIn(LIBRARY_ADDED_AT_SOURCES)
addedAtSource?: AddedAtSource;
@IsOptional()
@IsArray()
@IsString({ each: true })
@@ -0,0 +1,160 @@
import { Test, type TestingModule } from '@nestjs/testing';
import { BadRequestException, ConflictException, ForbiddenException, ServiceUnavailableException } from '@nestjs/common';
import { LibraryAddedAtService } from './library-added-at.service';
import { LibraryRepository } from './library.repository';
import { PathPolicyService } from '../path/path-policy.service';
import type { RequestUser } from '../../common/types/request-user';
import { realpath, stat } from 'fs/promises';
vi.mock('fs/promises', () => ({ realpath: vi.fn(), stat: vi.fn() }));
const user = { id: 7, isSuperuser: false } as RequestUser;
const oldTime = new Date('2020-01-01T00:00:00Z');
const newTime = new Date('2022-01-01T00:00:00Z');
const book = (id: number) => ({ id, addedAt: newTime, previousAddedAt: '2022-01-01 00:00:00.000123+00' });
const file = (id: number, bookId = id) => ({ id, bookId, absolutePath: `/library/${id}.epub`, rootPath: '/library', mtime: oldTime });
describe('LibraryAddedAtService', () => {
let module: TestingModule;
let service: LibraryAddedAtService;
const repo = {
hasUserAccess: vi.fn(),
findById: vi.fn(),
getAddedAtRecomputeBounds: vi.fn(),
findAddedAtBookBatch: vi.fn(),
findAddedAtFileBatch: vi.fn(),
findAddedAtMtimes: vi.fn(),
updateAddedAtBatch: vi.fn(),
};
const pathPolicy = { assertWithinBrowseRoot: vi.fn() };
beforeEach(async () => {
vi.resetAllMocks();
repo.hasUserAccess.mockResolvedValue(true);
repo.findById.mockResolvedValue([{ id: 1, addedAtSource: 'file_modified' }]);
repo.getAddedAtRecomputeBounds.mockResolvedValue({ total: 1, maxId: 1 });
repo.findAddedAtBookBatch.mockResolvedValue([book(1)]);
repo.findAddedAtMtimes.mockResolvedValue([{ bookId: 1, mtime: oldTime }]);
repo.findAddedAtFileBatch.mockImplementation((_lib, _ids, after) => Promise.resolve(after === 0 ? [file(1)] : []));
repo.updateAddedAtBatch.mockImplementation((_lib, changes) => Promise.resolve(changes.map(({ id }: { id: number }) => id)));
pathPolicy.assertWithinBrowseRoot.mockResolvedValue('/library');
vi.mocked(realpath).mockImplementation((path) => Promise.resolve(String(path)));
vi.mocked(stat).mockResolvedValue({ birthtime: oldTime, mtime: newTime, isFile: () => true } as Awaited<ReturnType<typeof stat>>);
module = await Test.createTestingModule({
providers: [LibraryAddedAtService, { provide: LibraryRepository, useValue: repo }, { provide: PathPolicyService, useValue: pathPolicy }],
}).compile();
service = module.get(LibraryAddedAtService);
});
afterEach(async () => {
await module.close();
});
async function complete(libraryId = 1) {
await vi.waitFor(async () => {
expect((await service.get(libraryId, user))?.status).not.toBe('running');
});
return (await service.get(libraryId, user))!;
}
it('returns immediately, updates bounded batches, and retains progress for reopening', async () => {
const all = Array.from({ length: 251 }, (_, i) => book(i + 1));
repo.getAddedAtRecomputeBounds.mockResolvedValue({ total: all.length, maxId: 251 });
repo.findAddedAtBookBatch.mockImplementation((_lib, after, max, limit) =>
Promise.resolve(all.filter(({ id }) => id > after && id <= max).slice(0, limit)),
);
repo.findAddedAtMtimes.mockImplementation((_lib, ids: number[]) => Promise.resolve(ids.map((bookId) => ({ bookId, mtime: oldTime }))));
const initial = await service.start(1, user);
expect(initial.status).toBe('running');
expect(repo.updateAddedAtBatch).not.toHaveBeenCalled();
const result = await complete();
expect(result).toMatchObject({ total: 251, processed: 251, updated: 251, failed: 0, status: 'completed' });
expect(repo.findAddedAtBookBatch.mock.calls.map((args) => args[1])).toEqual([0, 100, 200]);
expect(repo.updateAddedAtBatch.mock.calls.every((args) => args[1].length <= 100)).toBe(true);
expect(repo.findAddedAtFileBatch).not.toHaveBeenCalled();
expect(initial.processed).toBe(0);
});
it('does not change dates when already correct or when no usable timestamp exists', async () => {
repo.getAddedAtRecomputeBounds.mockResolvedValue({ total: 2, maxId: 2 });
repo.findAddedAtBookBatch.mockResolvedValue([book(1), book(2)]);
repo.findAddedAtMtimes.mockResolvedValue([
{ bookId: 1, mtime: newTime },
{ bookId: 2, mtime: new Date(0) },
]);
await service.start(1, user);
expect(await complete()).toMatchObject({ unchanged: 1, skipped: 1, updated: 0, processed: 2 });
expect(repo.updateAddedAtBatch).not.toHaveBeenCalled();
});
it('preserves the entire book if one content file is unreadable', async () => {
repo.findById.mockResolvedValue([{ addedAtSource: 'file_created' }]);
repo.findAddedAtFileBatch.mockImplementation((_lib, _ids, after) => Promise.resolve(after === 0 ? [file(1, 1), file(2, 1)] : []));
vi.mocked(stat)
.mockRejectedValueOnce(new Error('unavailable'))
.mockResolvedValueOnce({ birthtime: oldTime, isFile: () => true } as never);
await service.start(1, user);
expect(await complete()).toMatchObject({ failed: 1, updated: 0, failureSamples: [{ bookId: 1, code: 'file_unavailable' }] });
expect(repo.updateAddedAtBatch).not.toHaveBeenCalled();
});
it('paginates one large book and falls back from birthtime to mtime', async () => {
repo.findById.mockResolvedValue([{ addedAtSource: 'file_created' }]);
const files = Array.from({ length: 451 }, (_, i) => file(i + 1, 1));
repo.findAddedAtFileBatch.mockImplementation((_lib, _ids, after, limit) => Promise.resolve(files.filter(({ id }) => id > after).slice(0, limit)));
vi.mocked(stat).mockResolvedValue({ birthtime: new Date(0), mtime: oldTime, isFile: () => true } as never);
await service.start(1, user);
expect(await complete()).toMatchObject({ updated: 1 });
expect(repo.findAddedAtFileBatch.mock.calls.map((args) => args[2])).toEqual([0, 200, 400, 451]);
expect(repo.updateAddedAtBatch).toHaveBeenCalledWith(1, [{ id: 1, addedAt: oldTime, previousAddedAt: book(1).previousAddedAt }]);
});
it('rejects symlinks that resolve outside the configured library folder', async () => {
repo.findById.mockResolvedValue([{ addedAtSource: 'file_created' }]);
vi.mocked(realpath).mockImplementation((path) => Promise.resolve(String(path).endsWith('.epub') ? '/outside/book.epub' : String(path)));
await service.start(1, user);
expect(await complete()).toMatchObject({ failed: 1, failureSamples: [{ bookId: 1, code: 'unsafe_path' }] });
expect(stat).not.toHaveBeenCalled();
});
it('counts database failures and continues subsequent batches with bounded failure samples', async () => {
repo.getAddedAtRecomputeBounds.mockResolvedValue({ total: 101, maxId: 101 });
repo.findAddedAtBookBatch.mockImplementation((_lib, after) =>
Promise.resolve(after === 0 ? Array.from({ length: 100 }, (_, i) => book(i + 1)) : [book(101)]),
);
repo.findAddedAtMtimes.mockImplementation((_lib, ids: number[]) => Promise.resolve(ids.map((bookId) => ({ bookId, mtime: oldTime }))));
repo.updateAddedAtBatch.mockRejectedValueOnce(new Error('database unavailable'));
await service.start(1, user);
const result = await complete();
expect(result).toMatchObject({ processed: 101, failed: 100, updated: 1, status: 'completed' });
expect(result.failureSamples).toHaveLength(10);
expect(result.failureSamples[0]).toEqual({ bookId: 1, code: 'database_error' });
});
it('reports a terminal failure while retaining counts for already committed batches', async () => {
repo.getAddedAtRecomputeBounds.mockResolvedValue({ total: 2, maxId: 2 });
repo.findAddedAtBookBatch.mockResolvedValueOnce([book(1)]).mockRejectedValueOnce(new Error('connection lost'));
await service.start(1, user);
expect(await complete()).toMatchObject({ status: 'failed', processed: 1, updated: 1, total: 2 });
});
it('checks library access and keeps another user job private', async () => {
repo.hasUserAccess.mockResolvedValueOnce(false);
await expect(service.start(1, user)).rejects.toBeInstanceOf(ForbiddenException);
await service.start(1, user);
expect(await service.get(1, { ...user, id: 8 })).toBeNull();
await complete();
repo.hasUserAccess.mockResolvedValueOnce(false);
await expect(service.get(1, user)).rejects.toBeInstanceOf(ForbiddenException);
});
it('rejects import-time sources, duplicate runs, and excess global concurrency', async () => {
repo.findById.mockResolvedValueOnce([{ addedAtSource: 'imported' }]);
await expect(service.start(1, user)).rejects.toBeInstanceOf(BadRequestException);
await service.start(1, user);
await expect(service.start(1, user)).rejects.toBeInstanceOf(ConflictException);
await service.start(2, user);
await expect(service.start(3, user)).rejects.toBeInstanceOf(ServiceUnavailableException);
await complete(1);
await complete(2);
});
});
@@ -0,0 +1,236 @@
import {
BadRequestException,
ConflictException,
ForbiddenException,
Injectable,
Logger,
NotFoundException,
OnModuleDestroy,
ServiceUnavailableException,
} from '@nestjs/common';
import { randomUUID } from 'crypto';
import { realpath, stat } from 'fs/promises';
import { isAbsolute, relative, sep } from 'path';
import type { AddedAtRecomputeFailureCode, AddedAtRecomputeJob } from '@bookorbit/types';
import type { RequestUser } from '../../common/types/request-user';
import { usableFileTime } from '../../common/utils/file-time.utils';
import { sanitizeLogValue } from '../../common/utils/log-sanitize.utils';
import { PathPolicyService } from '../path/path-policy.service';
import { LibraryRepository } from './library.repository';
const BOOK_BATCH_SIZE = 100;
const FILE_BATCH_SIZE = 200;
const STAT_CONCURRENCY = 10;
const MAX_RUNNING_JOBS = 2;
const MAX_RETAINED_JOBS = 64;
const RESULT_TTL_MS = 60 * 60 * 1000;
const MAX_FAILURE_SAMPLES = 10;
type JobEntry = { job: AddedAtRecomputeJob; userId: number; startedAt: number; finishedAt?: number };
type FileRow = Awaited<ReturnType<LibraryRepository['findAddedAtFileBatch']>>[number];
@Injectable()
export class LibraryAddedAtService implements OnModuleDestroy {
private readonly logger = new Logger(LibraryAddedAtService.name);
private readonly jobs = new Map<number, JobEntry>();
private readonly runs = new Set<Promise<void>>();
private stopping = false;
constructor(
private readonly libraryRepo: LibraryRepository,
private readonly pathPolicy: PathPolicyService,
) {}
async start(libraryId: number, user: RequestUser): Promise<AddedAtRecomputeJob> {
await this.assertAccess(libraryId, user);
const [library] = await this.libraryRepo.findById(libraryId);
if (!library) throw new NotFoundException('Library not found');
if (library.addedAtSource !== 'file_created' && library.addedAtSource !== 'file_modified') {
throw new BadRequestException({ errorCode: 'ADDED_AT_SOURCE_IMPORTED', message: 'Select a file timestamp source before recomputing.' });
}
this.prune();
if (this.jobs.get(libraryId)?.job.status === 'running') {
throw new ConflictException({ errorCode: 'ADDED_AT_ALREADY_RUNNING', message: 'A recompute is already running for this library.' });
}
if (this.stopping || [...this.jobs.values()].filter(({ job }) => job.status === 'running').length >= MAX_RUNNING_JOBS) {
throw new ServiceUnavailableException({ errorCode: 'ADDED_AT_BUSY', message: 'Recompute capacity is busy. Try again later.' });
}
if (!this.jobs.has(libraryId) && this.jobs.size >= MAX_RETAINED_JOBS) {
const oldest = [...this.jobs.entries()]
.filter(([, entry]) => entry.finishedAt !== undefined)
.sort((a, b) => a[1].finishedAt! - b[1].finishedAt!)[0];
if (oldest) this.jobs.delete(oldest[0]);
}
const job: AddedAtRecomputeJob = {
id: randomUUID(),
libraryId,
source: library.addedAtSource,
status: 'running',
total: 0,
processed: 0,
updated: 0,
unchanged: 0,
skipped: 0,
failed: 0,
failureSamples: [],
};
const entry: JobEntry = { job, userId: user.id, startedAt: Date.now() };
this.jobs.set(libraryId, entry);
const run = new Promise<void>((resolve) => setImmediate(resolve)).then(() => this.run(entry));
this.runs.add(run);
void run.finally(() => this.runs.delete(run));
return this.snapshot(job);
}
async get(libraryId: number, user: RequestUser): Promise<AddedAtRecomputeJob | null> {
await this.assertAccess(libraryId, user);
this.prune();
const entry = this.jobs.get(libraryId);
if (!entry || (entry.userId !== user.id && !user.isSuperuser)) return null;
return this.snapshot(entry.job);
}
async onModuleDestroy(): Promise<void> {
this.stopping = true;
await Promise.allSettled(this.runs);
}
private async assertAccess(libraryId: number, user: RequestUser): Promise<void> {
if (!user.isSuperuser && !(await this.libraryRepo.hasUserAccess(user.id, libraryId))) {
throw new ForbiddenException('No access to this library');
}
}
private snapshot(job: AddedAtRecomputeJob): AddedAtRecomputeJob {
return { ...job, failureSamples: job.failureSamples.map((failure) => ({ ...failure })) };
}
private prune(): void {
for (const [libraryId, entry] of this.jobs) {
if (entry.finishedAt !== undefined && Date.now() - entry.finishedAt >= RESULT_TTL_MS) this.jobs.delete(libraryId);
}
}
private async run(entry: JobEntry): Promise<void> {
const { job } = entry;
this.logger.log(
`[library.recompute_added_at] [start] libraryId=${job.libraryId} userId=${entry.userId} jobId=${job.id} source=${job.source} - recompute started`,
);
try {
const { total, maxId } = await this.libraryRepo.getAddedAtRecomputeBounds(job.libraryId);
job.total = total;
let afterId = 0;
while (afterId < maxId) {
if (this.stopping) {
job.status = 'failed';
break;
}
const books = await this.libraryRepo.findAddedAtBookBatch(job.libraryId, afterId, maxId, BOOK_BATCH_SIZE);
if (books.length === 0) break;
await this.processBatch(entry, books);
afterId = books[books.length - 1].id;
}
if (job.status === 'running') {
// Books deleted or moved after the initial count no longer belong to this operation.
job.skipped += Math.max(0, job.total - job.processed);
job.processed = job.total;
job.status = 'completed';
}
} catch (error) {
job.status = 'failed';
this.logFailure(entry, error);
} finally {
entry.finishedAt = Date.now();
this.logger.log(
`[library.recompute_added_at] [end] libraryId=${job.libraryId} userId=${entry.userId} jobId=${job.id} source=${job.source} durationMs=${entry.finishedAt - entry.startedAt} status=${job.status} total=${job.total} processed=${job.processed} updated=${job.updated} unchanged=${job.unchanged} skipped=${job.skipped} failed=${job.failed} - recompute finished`,
);
}
}
private async processBatch(entry: JobEntry, books: Awaited<ReturnType<LibraryRepository['findAddedAtBookBatch']>>): Promise<void> {
const { job } = entry;
const earliest = new Map<number, Date>();
const failures = new Map<number, AddedAtRecomputeFailureCode>();
let afterFileId = 0;
if (job.source === 'file_modified') {
const times = await this.libraryRepo.findAddedAtMtimes(
job.libraryId,
books.map(({ id }) => id),
);
for (const { bookId, mtime } of times) {
const time = usableFileTime(mtime);
if (time) earliest.set(bookId, time);
}
}
while (job.source === 'file_created') {
if (this.stopping) throw new ServiceUnavailableException('Recompute interrupted by shutdown');
const files = await this.libraryRepo.findAddedAtFileBatch(
job.libraryId,
books.map(({ id }) => id),
afterFileId,
FILE_BATCH_SIZE,
);
if (files.length === 0) break;
for (let offset = 0; offset < files.length; offset += STAT_CONCURRENCY) {
await Promise.all(
files.slice(offset, offset + STAT_CONCURRENCY).map(async (file) => {
if (failures.has(file.bookId)) return;
try {
const time = await this.creationTime(file);
const previous = earliest.get(file.bookId);
if (time && (!previous || time < previous)) earliest.set(file.bookId, time);
} catch (error) {
failures.set(file.bookId, error instanceof ForbiddenException ? 'unsafe_path' : 'file_unavailable');
}
}),
);
}
afterFileId = files[files.length - 1].id;
}
const changes: { id: number; addedAt: Date; previousAddedAt: string }[] = [];
for (const book of books) {
const failure = failures.get(book.id);
const addedAt = earliest.get(book.id);
if (failure) this.recordFailure(job, book.id, failure);
else if (!addedAt) job.skipped++;
else if (addedAt.getTime() === book.addedAt.getTime()) job.unchanged++;
else changes.push({ id: book.id, addedAt, previousAddedAt: book.previousAddedAt });
}
if (changes.length > 0) {
try {
const updated = await this.libraryRepo.updateAddedAtBatch(job.libraryId, changes);
job.updated += updated.length;
job.skipped += changes.length - updated.length;
} catch (error) {
for (const { id } of changes) this.recordFailure(job, id, 'database_error');
this.logFailure(entry, error);
}
}
job.processed += books.length;
}
private async creationTime(file: FileRow): Promise<Date | undefined> {
await this.pathPolicy.assertWithinBrowseRoot(file.absolutePath);
const [root, path] = await Promise.all([realpath(file.rootPath), realpath(file.absolutePath)]);
const relativePath = relative(root, path);
if (relativePath === '..' || relativePath.startsWith(`..${sep}`) || isAbsolute(relativePath)) {
throw new ForbiddenException('File is outside its library root');
}
const info = await stat(path);
if (!info.isFile()) throw new BadRequestException('Content path is not a file');
return usableFileTime(info.birthtime) ?? usableFileTime(info.mtime);
}
private recordFailure(job: AddedAtRecomputeJob, bookId: number, code: AddedAtRecomputeFailureCode): void {
job.failed++;
if (job.failureSamples.length < MAX_FAILURE_SAMPLES) job.failureSamples.push({ bookId, code });
}
private logFailure(entry: JobEntry, error: unknown): void {
const errorClass = error instanceof Error ? error.name : 'Error';
const message = error instanceof Error ? error.message : String(error);
this.logger.warn(
`[library.recompute_added_at] [fail] libraryId=${entry.job.libraryId} userId=${entry.userId} jobId=${entry.job.id} source=${entry.job.source} durationMs=${Date.now() - entry.startedAt} errorClass=${errorClass} error="${sanitizeLogValue(message)}" - recompute could not complete all writes`,
);
}
}
@@ -1,10 +1,12 @@
import type { AccessLevel, CoverAspectRatio, OrganizationMode } from '@bookorbit/types';
import type { AccessLevel, AddedAtSource, CoverAspectRatio, OrganizationMode } from '@bookorbit/types';
export const LIBRARY_ACCESS_LEVELS = ['viewer', 'editor', 'owner'] as const satisfies readonly AccessLevel[];
export const LIBRARY_COVER_ASPECT_RATIOS = ['2/3', '1/1'] as const satisfies readonly CoverAspectRatio[];
export const LIBRARY_ORGANIZATION_MODES = ['book_per_file', 'book_per_folder'] as const satisfies readonly OrganizationMode[];
export const LIBRARY_ADDED_AT_SOURCES = ['imported', 'file_modified', 'file_created'] as const satisfies readonly AddedAtSource[];
export const DEFAULT_LIBRARY_ORGANIZATION_MODE: OrganizationMode = 'book_per_folder';
export const DEFAULT_LIBRARY_ADDED_AT_SOURCE: AddedAtSource = 'imported';
export const DEFAULT_LIBRARY_COVER_ASPECT_RATIO: CoverAspectRatio = '2/3';
export const LIBRARY_AUTO_SCAN_CRON_EXPRESSION_ERROR = 'autoScanCronExpression must be a valid 5-field cron expression';
@@ -29,7 +29,8 @@ describe('LibraryController', () => {
execute: vi.fn(),
};
const controller = new LibraryController(libraryService as any, bookService as any, bulkRenameService as any);
const addedAtService = { start: vi.fn(), get: vi.fn() };
const controller = new LibraryController(libraryService as any, bookService as any, bulkRenameService as any, addedAtService as any);
beforeEach(() => {
vi.resetAllMocks();
@@ -20,6 +20,7 @@ import { UpdateLibraryAccessDto } from './dto/update-library-access.dto';
import { UpdateLibraryDto } from './dto/update-library.dto';
import { BulkRenameService } from './bulk-rename.service';
import { LibraryService } from './library.service';
import { LibraryAddedAtService } from './library-added-at.service';
@Controller('libraries')
export class LibraryController {
@@ -27,6 +28,7 @@ export class LibraryController {
private readonly libraryService: LibraryService,
private readonly bookService: BookService,
private readonly bulkRenameService: BulkRenameService,
private readonly addedAtService: LibraryAddedAtService,
) {}
@Get()
@@ -123,6 +125,27 @@ export class LibraryController {
return this.libraryService.getStats(id);
}
@Get(':id/recompute-added-at')
@RequireLibraryAccess('editor')
@RequirePermission(Permission.ManageLibraries)
getAddedAtRecompute(@Param('id', ParseIntPipe) id: number, @CurrentUser() user: RequestUser) {
return this.addedAtService.get(id, user);
}
@Post(':id/recompute-added-at')
@HttpCode(HttpStatus.ACCEPTED)
@RequireLibraryAccess('editor')
@RequirePermission(Permission.ManageLibraries)
@Auditable({
action: AuditAction.LibraryRecomputeAddedAt,
resource: AuditResource.Library,
getResourceId: (req) => parseInt(req.params['id'], 10),
description: (req) => `Started added_at recompute for library #${req.params['id']}`,
})
recomputeAddedAt(@Param('id', ParseIntPipe) id: number, @CurrentUser() user: RequestUser) {
return this.addedAtService.start(id, user);
}
@Post(':id/write-metadata-to-files')
@RequireLibraryAccess('editor')
@RequirePermission(Permission.LibraryEditMetadata)
@@ -33,6 +33,22 @@ describe('Library DTO validation', () => {
).toBe(false);
});
it('CreateLibraryDto accepts every added_at source and rejects unknown ones', async () => {
for (const addedAtSource of ['imported', 'file_modified', 'file_created']) {
expect(await hasErrors(plainToInstance(CreateLibraryDto, { name: 'x', icon: 'BookOpen', folders: ['/a'], addedAtSource }))).toBe(false);
}
expect(await hasErrors(plainToInstance(CreateLibraryDto, { name: 'x', icon: 'BookOpen', folders: ['/a'], addedAtSource: 'bad' }))).toBe(true);
});
it('CreateLibraryDto survives the global whitelist rules with an added_at source', async () => {
// The library creator posts its whole form, and main.ts runs ValidationPipe with
// whitelist and forbidNonWhitelisted, so a field the DTO does not declare fails
// the whole request rather than being quietly dropped.
const dto = plainToInstance(CreateLibraryDto, { name: 'x', icon: 'BookOpen', folders: ['/a'], addedAtSource: 'file_modified' });
const errors = await validate(dto as any, { whitelist: true, forbidNonWhitelisted: true });
expect(errors).toHaveLength(0);
});
it('CreateLibraryDto requires a non-empty icon and UpdateLibraryDto rejects empty icons when provided', async () => {
expect(await hasErrors(plainToInstance(CreateLibraryDto, { name: 'Sci-Fi', folders: ['/books/scifi'] }))).toBe(true);
expect(await hasErrors(plainToInstance(CreateLibraryDto, { name: 'Sci-Fi', icon: ' ', folders: ['/books/scifi'] }))).toBe(true);
@@ -152,6 +168,14 @@ describe('Library DTO validation', () => {
expect(await hasErrors(plainToInstance(CreateLibraryDto, { ...base, fileWriteKindleMaxFileSizeMb: 10001 }))).toBe(true);
});
it('rejects null and invalid date sources before they reach the database', async () => {
for (const addedAtSource of [null, 'unknown', 12]) {
expect(await hasErrors(plainToInstance(CreateLibraryDto, { name: 'x', icon: 'BookOpen', folders: ['/a'], addedAtSource }))).toBe(true);
expect(await hasErrors(plainToInstance(UpdateLibraryDto, { addedAtSource }))).toBe(true);
}
expect(await hasErrors(plainToInstance(UpdateLibraryDto, {}))).toBe(false);
});
it('CreateLibraryDto accepts the Kindle enable flag', async () => {
expect(await hasErrors(plainToInstance(CreateLibraryDto, { ...base, fileWriteKindleEnabled: true }))).toBe(false);
expect(await hasErrors(plainToInstance(CreateLibraryDto, { ...base, fileWriteKindleEnabled: 'yes' }))).toBe(true);
@@ -12,6 +12,7 @@ import { LibraryModule } from './library.module';
import { LibraryRepository } from './library.repository';
import { LibraryScanSchedulerService } from './library-scan-scheduler.service';
import { LibraryService } from './library.service';
import { LibraryAddedAtService } from './library-added-at.service';
import { BulkRenameService } from './bulk-rename.service';
describe('LibraryModule', () => {
@@ -22,6 +23,7 @@ describe('LibraryModule', () => {
LibraryRepository,
LibraryScanSchedulerService,
BulkRenameService,
LibraryAddedAtService,
]);
expect(Reflect.getMetadata('exports', LibraryModule)).toEqual([LibraryService, LibraryRepository]);
});
+2 -1
View File
@@ -12,6 +12,7 @@ import { LibraryController } from './library.controller';
import { LibraryRepository } from './library.repository';
import { LibraryScanSchedulerService } from './library-scan-scheduler.service';
import { LibraryService } from './library.service';
import { LibraryAddedAtService } from './library-added-at.service';
@Module({
imports: [
@@ -24,7 +25,7 @@ import { LibraryService } from './library.service';
PathModule,
],
controllers: [LibraryController],
providers: [LibraryService, LibraryRepository, LibraryScanSchedulerService, BulkRenameService],
providers: [LibraryService, LibraryRepository, LibraryScanSchedulerService, BulkRenameService, LibraryAddedAtService],
exports: [LibraryService, LibraryRepository],
})
export class LibraryModule {}
@@ -1,10 +1,11 @@
import { Inject, Injectable } from '@nestjs/common';
import { and, eq, getTableColumns, inArray, isNotNull, or, sql } from 'drizzle-orm';
import { and, eq, gt, lte, getTableColumns, inArray, isNotNull, or, sql } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import type { AccessLevel, ContentFilterRules, LibraryStats } from '@bookorbit/types';
import { buildContentFilterClauses } from '../../common/utils/content-filter-sql.utils';
import { DB } from '../../db';
import { MIN_VALID_FILE_TIME_MS } from '../../common/utils/file-time.utils';
import * as schema from '../../db/schema';
import { bookFiles, books, libraryFolders, libraries } from '../../db/schema';
import { LIBRARY_BOOK_STATUS_PRESENT } from './library.constants';
@@ -245,6 +246,74 @@ export class LibraryRepository {
.where(and(inArray(schema.users.id, userIds), or(eq(schema.users.isSuperuser, true), isNotNull(schema.userLibraryAccess.userId))));
}
async getAddedAtRecomputeBounds(libraryId: number) {
const [row] = await this.db
.select({
total: sql<number>`count(*)::int`,
maxId: sql<number>`coalesce(max(${books.id}), 0)::int`,
})
.from(books)
.where(eq(books.libraryId, libraryId));
return row ?? { total: 0, maxId: 0 };
}
findAddedAtBookBatch(libraryId: number, afterId: number, maxId: number, limit: number) {
return this.db
.select({ id: books.id, addedAt: books.addedAt, previousAddedAt: sql<string>`${books.addedAt}::text` })
.from(books)
.where(and(eq(books.libraryId, libraryId), gt(books.id, afterId), lte(books.id, maxId)))
.orderBy(books.id)
.limit(limit);
}
findAddedAtMtimes(libraryId: number, bookIds: number[]) {
return this.db
.select({ bookId: bookFiles.bookId, mtime: sql<Date>`min(${bookFiles.mtime})`.mapWith(bookFiles.mtime) })
.from(bookFiles)
.innerJoin(books, eq(books.id, bookFiles.bookId))
.where(
and(
eq(books.libraryId, libraryId),
inArray(books.id, bookIds),
eq(bookFiles.role, 'content'),
gt(bookFiles.mtime, new Date(MIN_VALID_FILE_TIME_MS)),
),
)
.groupBy(bookFiles.bookId);
}
findAddedAtFileBatch(libraryId: number, bookIds: number[], afterId: number, limit: number) {
return this.db
.select({
id: bookFiles.id,
bookId: bookFiles.bookId,
absolutePath: bookFiles.absolutePath,
mtime: bookFiles.mtime,
rootPath: libraryFolders.path,
})
.from(bookFiles)
.innerJoin(books, eq(books.id, bookFiles.bookId))
.innerJoin(libraryFolders, eq(libraryFolders.id, books.libraryFolderId))
.where(and(eq(books.libraryId, libraryId), inArray(books.id, bookIds), eq(bookFiles.role, 'content'), gt(bookFiles.id, afterId)))
.orderBy(bookFiles.id)
.limit(limit);
}
async updateAddedAtBatch(libraryId: number, values: { id: number; addedAt: Date; previousAddedAt: string }[]): Promise<number[]> {
if (values.length === 0) return [];
const rows = sql.join(
values.map(({ id, addedAt, previousAddedAt }) => sql`(${id}::int, ${addedAt.toISOString()}::timestamptz, ${previousAddedAt}::timestamptz)`),
sql`, `,
);
const result = await this.db
.update(books)
.set({ addedAt: sql`dates.added_at`, updatedAt: new Date() })
.from(sql`(values ${rows}) as dates(id, added_at, previous_added_at)`)
.where(and(eq(books.libraryId, libraryId), sql`${books.id} = dates.id`, sql`${books.addedAt} = dates.previous_added_at`))
.returning({ id: books.id });
return result.map(({ id }) => id);
}
async hasUserAccess(userId: number, libraryId: number): Promise<boolean> {
const row = await this.db.query.userLibraryAccess.findFirst({
where: and(eq(schema.userLibraryAccess.userId, userId), eq(schema.userLibraryAccess.libraryId, libraryId)),
@@ -27,7 +27,12 @@ import { GrantLibraryAccessDto } from './dto/grant-library-access.dto';
import { PrescanLibraryDto } from './dto/prescan-library.dto';
import { ReorderLibrariesDto } from './dto/reorder-libraries.dto';
import { UpdateLibraryDto } from './dto/update-library.dto';
import { DEFAULT_LIBRARY_COVER_ASPECT_RATIO, DEFAULT_LIBRARY_ORGANIZATION_MODE, LIBRARY_METADATA_PRECEDENCE_DEFAULT } from './library.constants';
import {
DEFAULT_LIBRARY_ADDED_AT_SOURCE,
DEFAULT_LIBRARY_COVER_ASPECT_RATIO,
DEFAULT_LIBRARY_ORGANIZATION_MODE,
LIBRARY_METADATA_PRECEDENCE_DEFAULT,
} from './library.constants';
import { LibraryRepository } from './library.repository';
import { LibraryScanSchedulerService } from './library-scan-scheduler.service';
@@ -133,6 +138,7 @@ export class LibraryService {
formatPriority: dto.formatPriority ?? [...DEFAULT_FORMAT_PRIORITY],
allowedFormats: dto.allowedFormats ?? [],
organizationMode: dto.organizationMode ?? DEFAULT_LIBRARY_ORGANIZATION_MODE,
addedAtSource: dto.addedAtSource ?? DEFAULT_LIBRARY_ADDED_AT_SOURCE,
excludePatterns: dto.excludePatterns ?? [],
coverAspectRatio: dto.coverAspectRatio ?? DEFAULT_LIBRARY_COVER_ASPECT_RATIO,
readingThreshold: dto.readingThreshold ?? 0.25,
+67 -1
View File
@@ -2,7 +2,8 @@ import { mkdir, mkdtemp, rm, writeFile, chmod } from 'fs/promises';
import { join } from 'path';
import { tmpdir } from 'os';
import { findBookCandidates, buildSingleBookCandidate, findLooseFileCandidates, BookCandidate } from './walk';
import { findBookCandidates, buildSingleBookCandidate, findLooseFileCandidates, earliestContentTime, BookCandidate, FileStat } from './walk';
import type { FileRole } from './classify';
let suiteRoot: string;
let root: string;
@@ -951,3 +952,68 @@ describe('incremental scan — findLooseFileCandidates', () => {
expect(second.unchangedDirs.size).toBeGreaterThan(0);
});
});
// ── earliestContentTime ───────────────────────────────────────────────────────
describe('earliestContentTime', () => {
function fs(role: FileRole, mtime: Date, birthtime: Date): FileStat {
return {
absolutePath: `/lib/${role}-${mtime.getTime()}-${birthtime.getTime()}`,
relPath: `${role}`,
ino: 1n,
sizeBytes: 10,
mtime,
birthtime,
format: 'epub',
role,
};
}
const d = (iso: string) => new Date(iso);
it('file_modified returns the earliest content-file mtime', () => {
const files = [
fs('content', d('2022-05-01T00:00:00Z'), d('2023-01-01T00:00:00Z')),
fs('content', d('2021-03-01T00:00:00Z'), d('2020-01-01T00:00:00Z')),
];
expect(earliestContentTime(files, 'file_modified')).toEqual(d('2021-03-01T00:00:00Z'));
});
it('file_modified ignores non-content files', () => {
const files = [
fs('content', d('2022-05-01T00:00:00Z'), d('2022-05-01T00:00:00Z')),
fs('cover', d('2019-01-01T00:00:00Z'), d('2019-01-01T00:00:00Z')),
fs('metadata', d('2018-01-01T00:00:00Z'), d('2018-01-01T00:00:00Z')),
];
expect(earliestContentTime(files, 'file_modified')).toEqual(d('2022-05-01T00:00:00Z'));
});
it('file_created returns the earliest content-file birthtime', () => {
const files = [
fs('content', d('2022-05-01T00:00:00Z'), d('2020-06-01T00:00:00Z')),
fs('content', d('2021-03-01T00:00:00Z'), d('2019-02-01T00:00:00Z')),
];
expect(earliestContentTime(files, 'file_created')).toEqual(d('2019-02-01T00:00:00Z'));
});
it('file_created falls back to mtime when birthtime is epoch-0/invalid', () => {
const files = [fs('content', d('2021-03-01T00:00:00Z'), new Date(0))];
expect(earliestContentTime(files, 'file_created')).toEqual(d('2021-03-01T00:00:00Z'));
});
it('file_created compares per-file birthtime-or-mtime fallbacks across files', () => {
const files = [
// birthtime invalid -> falls back to its mtime (2021)
fs('content', d('2021-03-01T00:00:00Z'), new Date(0)),
// valid birthtime (2022) later than the other file's fallback mtime
fs('content', d('2024-01-01T00:00:00Z'), d('2022-01-01T00:00:00Z')),
];
expect(earliestContentTime(files, 'file_created')).toEqual(d('2021-03-01T00:00:00Z'));
});
it('returns undefined when no content file yields a usable time', () => {
const files = [fs('cover', d('2021-03-01T00:00:00Z'), d('2021-03-01T00:00:00Z')), fs('content', new Date(0), new Date(0))];
expect(earliestContentTime(files, 'file_modified')).toBeUndefined();
expect(earliestContentTime(files, 'file_created')).toBeUndefined();
});
});
+32
View File
@@ -1,3 +1,4 @@
import { usableFileTime } from '../../../common/utils/file-time.utils';
import { readdir, stat } from 'fs/promises';
import { basename, dirname, join, relative } from 'path';
@@ -10,6 +11,7 @@ export interface FileStat {
ino: bigint;
sizeBytes: number;
mtime: Date;
birthtime: Date;
format: string | null;
role: FileRole;
}
@@ -26,6 +28,34 @@ export interface WalkResult {
dirMtimes: Map<string, number>;
}
/**
* Derive a book's "date added" from the earliest on-disk time of its content
* files. This approximates when the book first landed on disk, rather than when
* BookOrbit happened to import it. Cover/metadata/supplement sidecars are
* excluded because they can be added later without meaning the book is "newer".
*
* For 'file_modified' the earliest valid mtime is used. For 'file_created' the
* earliest valid birthtime is used, falling back to that same file's mtime when
* its birthtime is missing or invalid (some filesystems do not track creation
* time). Returns undefined when no content file yields a usable time, so callers
* can fall back to the DB defaultNow() (import time).
*/
export function earliestContentTime(files: FileStat[], source: 'file_modified' | 'file_created'): Date | undefined {
let earliest: Date | undefined;
for (const file of files) {
if (file.role !== 'content') continue;
let candidate: Date | undefined;
if (source === 'file_modified') {
candidate = usableFileTime(file.mtime);
} else {
candidate = usableFileTime(file.birthtime) ?? usableFileTime(file.mtime);
}
if (candidate === undefined) continue;
if (earliest === undefined || candidate < earliest) earliest = candidate;
}
return earliest;
}
const MAX_PATH_LENGTH = 4096;
const DIR_CONCURRENCY_LIMIT = 50;
@@ -89,6 +119,7 @@ async function statFilesIntoAcc(
ino,
sizeBytes: Number(s.size),
mtime: s.mtime,
birthtime: s.birthtime,
format,
role,
});
@@ -426,6 +457,7 @@ export async function buildSingleBookCandidate(
ino: s.ino,
sizeBytes: Number(s.size),
mtime: s.mtime,
birthtime: s.birthtime,
format,
role,
} satisfies FileStat;
@@ -107,6 +107,7 @@ export class ScannerRepository {
metadataPrecedence: libraries.metadataPrecedence,
excludePatterns: libraries.excludePatterns,
organizationMode: libraries.organizationMode,
addedAtSource: libraries.addedAtSource,
})
.from(libraries)
.where(eq(libraries.id, libraryId));
+38 -1
View File
@@ -6,6 +6,7 @@ import { pathsReferToSameEntry } from '../../common/utils/path-identity.utils';
import { SelfWriteRegistry } from '../../common/services/self-write-registry.service';
import type {
AddedAtSource,
BookMissingEvent,
BookTransferredEvent,
CoverRefreshedEvent,
@@ -30,7 +31,15 @@ import { readdir, stat } from 'fs/promises';
import { classifyFile, DEFAULT_FORMAT_PRIORITY, FileRole, isAudioFormat } from './lib/classify';
import { computeFileHash } from './lib/hash';
import { waitForStability } from './lib/stability';
import { BookCandidate, FileStat, findBookCandidates, findLooseFileCandidates, buildSingleBookCandidate, type WalkResult } from './lib/walk';
import {
BookCandidate,
FileStat,
earliestContentTime,
findBookCandidates,
findLooseFileCandidates,
buildSingleBookCandidate,
type WalkResult,
} from './lib/walk';
import { ScannerRepository } from './scanner.repository';
import { assembleBookCards } from '../book/utils/assemble-book-cards';
import { LIBRARY_METADATA_PRECEDENCE_DEFAULT } from '../library/library.constants';
@@ -75,6 +84,7 @@ interface LibraryScanSettings {
metadataPrecedence: string[];
excludePatterns: string[];
organizationMode: OrganizationMode;
addedAtSource: AddedAtSource;
}
type TargetedScanJob = { type: 'book'; path: string; libraryId: number } | { type: 'directory'; path: string; libraryId: number };
@@ -157,6 +167,10 @@ function normalizeOrganizationMode(mode: string | null | undefined): Organizatio
return mode === 'book_per_file' ? 'book_per_file' : 'book_per_folder';
}
function normalizeAddedAtSource(source: string | null | undefined): AddedAtSource {
return source === 'file_modified' || source === 'file_created' ? source : 'imported';
}
function normalizeMetadataPrecedence(metadataPrecedence: string[] | null | undefined): ScannerMetadataSource[] {
const result: ScannerMetadataSource[] = [];
const configured = metadataPrecedence && metadataPrecedence.length > 0 ? metadataPrecedence : LIBRARY_METADATA_PRECEDENCE_DEFAULT;
@@ -625,6 +639,7 @@ export class ScannerService implements OnApplicationBootstrap {
const metadataPrecedence = settings?.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT];
const excludePatterns = settings?.excludePatterns ?? [];
const organizationMode = normalizeOrganizationMode(settings?.organizationMode);
const addedAtSource = normalizeAddedAtSource(settings?.addedAtSource);
// Invalidate incremental scan cache when scan-affecting settings change.
// On first scan after restart (no stored hash), force full scan to avoid
@@ -653,6 +668,7 @@ export class ScannerService implements OnApplicationBootstrap {
metadataPrecedence,
excludePatterns,
organizationMode,
addedAtSource,
forceFullScan,
).catch((err) => {
const errorClass = err instanceof Error ? err.name : 'Error';
@@ -820,6 +836,7 @@ export class ScannerService implements OnApplicationBootstrap {
metadataPrecedence: rawSettings?.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT],
excludePatterns: rawSettings?.excludePatterns ?? [],
organizationMode: normalizeOrganizationMode(rawSettings?.organizationMode),
addedAtSource: normalizeAddedAtSource(rawSettings?.addedAtSource),
};
if (settings.organizationMode === 'book_per_file') {
@@ -862,6 +879,7 @@ export class ScannerService implements OnApplicationBootstrap {
metadataPrecedence: rawSettings?.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT],
excludePatterns: rawSettings?.excludePatterns ?? [],
organizationMode: normalizeOrganizationMode(rawSettings?.organizationMode),
addedAtSource: normalizeAddedAtSource(rawSettings?.addedAtSource),
};
const walkLogger = (msg: string) =>
@@ -935,6 +953,7 @@ export class ScannerService implements OnApplicationBootstrap {
maps,
settings.formatPriority,
settings.metadataPrecedence,
settings.addedAtSource,
false,
candidateFolderPaths,
);
@@ -1028,6 +1047,7 @@ export class ScannerService implements OnApplicationBootstrap {
ino: fileStat.ino,
sizeBytes: Number(fileStat.size),
mtime: fileStat.mtime,
birthtime: fileStat.birthtime,
format,
role,
},
@@ -1042,6 +1062,7 @@ export class ScannerService implements OnApplicationBootstrap {
maps,
settings.formatPriority,
settings.metadataPrecedence,
settings.addedAtSource,
false,
new Set([candidate.folderPath]),
);
@@ -1097,6 +1118,7 @@ export class ScannerService implements OnApplicationBootstrap {
ino: fileStat.ino,
sizeBytes: Number(fileStat.size),
mtime: fileStat.mtime,
birthtime: fileStat.birthtime,
format,
role,
},
@@ -1111,6 +1133,7 @@ export class ScannerService implements OnApplicationBootstrap {
maps,
settings.formatPriority,
settings.metadataPrecedence,
settings.addedAtSource,
false,
new Set([candidate.folderPath]),
);
@@ -1200,6 +1223,7 @@ export class ScannerService implements OnApplicationBootstrap {
maps,
settings.formatPriority,
settings.metadataPrecedence,
settings.addedAtSource,
false,
new Set([candidate.folderPath]),
);
@@ -1223,6 +1247,7 @@ export class ScannerService implements OnApplicationBootstrap {
metadataPrecedence: string[],
excludePatterns: string[],
organizationMode: OrganizationMode,
addedAtSource: AddedAtSource,
forceFullScan = false,
): Promise<void> {
const event = 'scanner.run_scan';
@@ -1306,6 +1331,7 @@ export class ScannerService implements OnApplicationBootstrap {
jobId,
formatPriority,
metadataPrecedence,
addedAtSource,
skippedDirs,
unchangedDirs,
);
@@ -1391,6 +1417,7 @@ export class ScannerService implements OnApplicationBootstrap {
jobId: number,
formatPriority: string[],
metadataPrecedence: string[],
addedAtSource: AddedAtSource,
skippedDirs: Set<string> = new Set(),
unchangedDirs: Set<string> = new Set(),
): Promise<ScanCounts> {
@@ -1422,6 +1449,7 @@ export class ScannerService implements OnApplicationBootstrap {
maps,
formatPriority,
metadataPrecedence,
addedAtSource,
isFirstScan,
candidateFolderPaths,
);
@@ -1501,6 +1529,7 @@ export class ScannerService implements OnApplicationBootstrap {
maps: ScanLookupMaps,
formatPriority: string[],
metadataPrecedence: string[],
addedAtSource: AddedAtSource,
isFirstScan: boolean,
candidateFolderPaths: Set<string>,
): Promise<ProcessCandidateResult> {
@@ -1519,6 +1548,7 @@ export class ScannerService implements OnApplicationBootstrap {
fileByIno,
fileCounts,
candidateFolderPaths,
addedAtSource,
);
// If the book was transferred from another library, its files exist globally
// but not in our local maps - we need global lookups even on a "first scan"
@@ -1768,6 +1798,7 @@ export class ScannerService implements OnApplicationBootstrap {
fileByIno: Map<bigint, FileByInoEntry>,
counts: ScanCounts,
candidateFolderPaths: Set<string>,
addedAtSource: AddedAtSource,
): Promise<UpsertBookResult> {
const existing = bookByFolderPath.get(candidate.folderPath);
const candidateOwnedBookIds = new Set<number>();
@@ -1824,11 +1855,17 @@ export class ScannerService implements OnApplicationBootstrap {
const transferred = await this.tryTransferMissingBook(candidate, libraryId, libraryFolderId, bookByFolderPath, counts);
if (transferred) return { ...transferred, created: false };
// date added per the library's addedAtSource: 'imported' leaves addedAt unset
// so the DB defaultNow() (import time) applies; on-disk sources derive it from
// the earliest content-file time, still falling back to import time when no
// content file yields a usable time.
const addedAt = addedAtSource === 'imported' ? undefined : earliestContentTime(candidate.files, addedAtSource);
const book = await this.scannerRepo.createBook({
libraryId,
libraryFolderId,
folderPath: candidate.folderPath,
status: 'processing',
addedAt,
});
counts.addedCount++;
const entry = { id: book.id, status: book.status, folderPath: book.folderPath, primaryFileId: book.primaryFileId ?? null };
+6 -1
View File
@@ -570,7 +570,10 @@ describe('Authorization matrix (e2e)', () => {
const live = liveRouteLabels(ctx.app);
const liveLabels = new Set(live);
expect(live.filter((label) => !inventoryLabels.has(label) && !baseline.has(label))).toEqual([]);
// Fastify combines parameter aliases at shared trie nodes, such as :id|:libraryId.
const routeShape = (label: string) => label.replace(/:[\w]+(?:\|:[\w]+)*/g, ':param');
const inventoriedShapes = new Set([...inventoryLabels].map(routeShape));
expect(live.filter((label) => !inventoriedShapes.has(routeShape(label)) && !baseline.has(label))).toEqual([]);
expect([...baseline].filter((label) => inventoryLabels.has(label))).toEqual([]);
expect([...baseline].filter((label) => !liveLabels.has(label))).toEqual([]);
});
@@ -850,6 +853,8 @@ describe('Authorization matrix (e2e)', () => {
'GET /libraries/:id',
'POST /libraries/:id/books',
'GET /libraries/:id/stats',
'GET /libraries/:id/recompute-added-at',
'POST /libraries/:id/recompute-added-at',
'POST /libraries/:id/write-metadata-to-files',
'GET /scanner/libraries/:id/scan-history',
].sort(),
@@ -1,5 +1,5 @@
{
"totalRoutes": 455,
"totalRoutes": 457,
"byPermission": {
"(none)": 168,
"Permission.ManageMetadataConfig": 34,
@@ -7,7 +7,7 @@
"Permission.LibraryEditMetadata": 35,
"Permission.BookDockAccess": 17,
"Permission.ManageAppSettings": 39,
"Permission.ManageLibraries": 17,
"Permission.ManageLibraries": 19,
"Permission.ManageUsers": 13,
"Permission.ManageEmail": 8,
"Permission.KoboSync": 7,
@@ -31,7 +31,7 @@
},
"byLibraryAccess": {
"'viewer'": 4,
"'editor'": 1
"'editor'": 3
},
"routeChunks": [
"routes-01.json",
@@ -587,6 +587,30 @@
"allowDefault": false,
"useGuards": []
},
{
"file": "server/src/modules/library/library.controller.ts",
"className": "LibraryController",
"methodName": "getAddedAtRecompute",
"httpMethod": "GET",
"path": "/libraries/:id/recompute-added-at",
"permissions": ["Permission.ManageLibraries"],
"libraryAccess": ["'editor'"],
"isPublic": false,
"allowDefault": false,
"useGuards": []
},
{
"file": "server/src/modules/library/library.controller.ts",
"className": "LibraryController",
"methodName": "recomputeAddedAt",
"httpMethod": "POST",
"path": "/libraries/:id/recompute-added-at",
"permissions": ["Permission.ManageLibraries"],
"libraryAccess": ["'editor'"],
"isPublic": false,
"allowDefault": false,
"useGuards": []
},
{
"file": "server/src/modules/library/library.controller.ts",
"className": "LibraryController",
+149 -2
View File
@@ -1,9 +1,10 @@
import { randomUUID } from 'crypto';
import { access, mkdir, readFile } from 'fs/promises';
import { access, mkdir, readFile, stat, utimes, writeFile } from 'fs/promises';
import { basename, join } from 'path';
import { and, eq } from 'drizzle-orm';
import { and, eq, sql } from 'drizzle-orm';
import { Permission } from '@bookorbit/types';
import type { AddedAtRecomputeJob } from '@bookorbit/types';
import * as schema from '../src/db/schema';
import { waitForCondition, waitForScanCompletion } from './e2e/app-harness';
@@ -208,6 +209,152 @@ describe('Library admin workflows (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () =
await closeAuthorizationMatrixE2EContext(ctx);
});
describe('date added recompute', () => {
async function finishRecompute(libraryId: number): Promise<AddedAtRecomputeJob> {
let result: AddedAtRecomputeJob;
await waitForCondition(async () => {
const response = await ctx.app.inject({
method: 'GET',
url: `/api/v1/libraries/${libraryId}/recompute-added-at`,
headers: authHeader(manager.accessToken),
});
expect(response.statusCode).toBe(200);
result = response.json() as AddedAtRecomputeJob;
expect(result.status).not.toBe('running');
}, 15_000);
return result!;
}
it('enforces both permission and library access and updates multiple batches with precise database timestamps', async () => {
const library = await createLibraryWithFolder(ctx);
const url = `/api/v1/libraries/${library.libraryId}/recompute-added-at`;
for (const method of ['POST', 'GET'] as const) {
expect((await ctx.app.inject({ method, url, headers: authHeader(manager.accessToken) })).statusCode).toBe(403);
}
await grantLibraryAccess(ctx, manager.userId, library.libraryId, 'viewer');
expect((await ctx.app.inject({ method: 'POST', url, headers: authHeader(manager.accessToken) })).statusCode).toBe(403);
await grantLibraryAccess(ctx, manager.userId, library.libraryId, 'editor');
await grantLibraryAccess(ctx, scopedUser.userId, library.libraryId, 'editor');
for (const method of ['POST', 'GET'] as const) {
expect((await ctx.app.inject({ method, url, headers: authHeader(scopedUser.accessToken) })).statusCode).toBe(403);
}
const imported = await ctx.app.inject({ method: 'POST', url, headers: authHeader(manager.accessToken) });
expect(imported.statusCode).toBe(400);
expect(imported.json().errorCode).toBe('ADDED_AT_SOURCE_IMPORTED');
const update = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(manager.accessToken),
payload: { addedAtSource: 'file_modified', icon: 'BookOpen' },
});
expect(update.statusCode, update.body).toBe(200);
expect(update.json().addedAtSource).toBe('file_modified');
const inserted = await ctx.db
.insert(schema.books)
.values(
Array.from({ length: 105 }, (_, i) => ({
libraryId: library.libraryId,
libraryFolderId: library.libraryFolderId,
folderPath: join(library.folderPath, String(i)),
addedAt: sql`'2026-01-01 00:00:00.000123+00'::timestamptz`,
})),
)
.returning({ id: schema.books.id });
const expectedTime = new Date('2019-01-01T00:00:00Z');
await ctx.db.insert(schema.bookFiles).values(
inserted.slice(0, 104).flatMap(({ id }) => [
{
bookId: id,
libraryFolderId: library.libraryFolderId,
absolutePath: join(library.folderPath, `${id}.epub`),
ino: BigInt(id),
mtime: expectedTime,
role: 'content',
},
{
bookId: id,
libraryFolderId: library.libraryFolderId,
absolutePath: join(library.folderPath, `${id}.pdf`),
ino: BigInt(id + 100000),
mtime: new Date(0),
role: 'content',
},
{
bookId: id,
libraryFolderId: library.libraryFolderId,
absolutePath: join(library.folderPath, `${id}.jpg`),
ino: BigInt(id + 200000),
mtime: new Date('2010-01-01'),
role: 'cover',
},
]),
);
const otherLibrary = await createLibraryWithFolder(ctx);
const [other] = await ctx.db
.insert(schema.books)
.values({ libraryId: otherLibrary.libraryId, libraryFolderId: otherLibrary.libraryFolderId, folderPath: otherLibrary.folderPath })
.returning();
const started = await ctx.app.inject({ method: 'POST', url, headers: authHeader(manager.accessToken) });
expect(started.statusCode).toBe(202);
expect(started.json().status).toBe('running');
expect(await finishRecompute(library.libraryId)).toMatchObject({ total: 105, processed: 105, updated: 104, skipped: 1, failed: 0 });
const [updated] = await ctx.db.select().from(schema.books).where(eq(schema.books.id, inserted[0].id));
expect(updated.addedAt).toEqual(expectedTime);
const [untouched] = await ctx.db.select().from(schema.books).where(eq(schema.books.id, other.id));
expect(untouched.addedAt).toEqual(other.addedAt);
expect((await ctx.app.inject({ method: 'POST', url, headers: authHeader(manager.accessToken) })).statusCode).toBe(202);
expect(await finishRecompute(library.libraryId)).toMatchObject({ updated: 0, unchanged: 104, skipped: 1 });
});
it('uses real birthtimes and preserves dates when any content file is missing', async () => {
const library = await createLibraryWithFolder(ctx);
await grantLibraryAccess(ctx, manager.userId, library.libraryId, 'editor');
await ctx.db.update(schema.libraries).set({ addedAtSource: 'file_created' }).where(eq(schema.libraries.id, library.libraryId));
const original = new Date('2020-01-01T00:00:00Z');
const inserted = await ctx.db
.insert(schema.books)
.values(
[0, 1].map((i) => ({
libraryId: library.libraryId,
libraryFolderId: library.libraryFolderId,
folderPath: join(library.folderPath, String(i)),
addedAt: original,
})),
)
.returning();
const present = join(library.folderPath, 'present.epub');
await writeFile(present, 'timestamp fixture');
const info = await stat(present);
await ctx.db.insert(schema.bookFiles).values([
{ bookId: inserted[0].id, libraryFolderId: library.libraryFolderId, absolutePath: present, ino: 1n },
{ bookId: inserted[1].id, libraryFolderId: library.libraryFolderId, absolutePath: join(library.folderPath, 'missing.epub'), ino: 2n },
]);
const response = await ctx.app.inject({
method: 'POST',
url: `/api/v1/libraries/${library.libraryId}/recompute-added-at`,
headers: authHeader(manager.accessToken),
});
expect(response.statusCode).toBe(202);
const result = await finishRecompute(library.libraryId);
expect(result).toMatchObject({ processed: 2, updated: 1, failed: 1, failureSamples: [{ bookId: inserted[1].id, code: 'file_unavailable' }] });
expect(JSON.stringify(result)).not.toContain(library.folderPath);
const rows = await ctx.db.select().from(schema.books).where(eq(schema.books.libraryId, library.libraryId)).orderBy(schema.books.id);
expect(rows[0].addedAt).toEqual(info.birthtime.getTime() > 1000 ? info.birthtime : info.mtime);
expect(rows[1].addedAt).toEqual(original);
});
it('applies a saved file source to new scanner imports', async () => {
const library = await createLibraryWithFolder(ctx);
await ctx.db.update(schema.libraries).set({ addedAtSource: 'file_modified' }).where(eq(schema.libraries.id, library.libraryId));
const path = await createEpubFixture(library.folderPath, 'dated.epub', { title: 'Dated Book' });
const timestamp = new Date('2018-06-01T00:00:00Z');
await utimes(path, timestamp, timestamp);
await startLibraryScanAs(ctx, ctx.adminToken, library.libraryId);
const [book] = await ctx.db.select().from(schema.books).where(eq(schema.books.libraryId, library.libraryId));
expect(book.addedAt).toEqual(timestamp);
});
});
describe('path and prescan', () => {
it('lists valid directories, hides blocked or hidden entries, and returns prescan details for valid, overlapping, and missing paths', async () => {
const overlapLibraryName = `library-overlap-${randomUUID()}`;