feat(scanner): regex metadata extraction (#1669)

feat(metadata): extract metadata from file paths with regex rules (#1669)

Let libraries define ordered regex rules that read title, authors, series,
series number and dates from each book's path relative to its library folder.
Run rules in a bounded worker pool, honour source precedence and field locks,
and apply them on scans, uploads and watcher moves. Also retry unprobed M4B
tag writes with the mp4 muxer.

Closes #1351

Co-authored-by: neon <18559658+neonsolstice@users.noreply.github.com>
This commit is contained in:
Dennis
2026-10-17 00:24:11 +02:00
committed by GitHub
parent fd10f89586
commit 428aefe8d1
50 changed files with 26474 additions and 74 deletions
@@ -62,7 +62,6 @@ html[data-style='nostalgia'] {
--glow-texture: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 84 84' fill='%23000'%3E%3Crect x='1' y='1' width='19' height='19' fill-opacity='0.044'/%3E%3Crect x='22' y='1' width='19' height='19' fill-opacity='0.058'/%3E%3Crect x='43' y='1' width='19' height='19' fill-opacity='0.042'/%3E%3Crect x='64' y='1' width='19' height='19' fill-opacity='0.035'/%3E%3Crect x='1' y='22' width='19' height='19' fill-opacity='0.059'/%3E%3Crect x='22' y='22' width='19' height='19' fill-opacity='0.02'/%3E%3Crect x='43' y='22' width='19' height='19' fill-opacity='0.054'/%3E%3Crect x='64' y='22' width='19' height='19' fill-opacity='0.03'/%3E%3Crect x='1' y='43' width='19' height='19' fill-opacity='0.024'/%3E%3Crect x='22' y='43' width='19' height='19' fill-opacity='0.023'/%3E%3Crect x='43' y='43' width='19' height='19' fill-opacity='0.031'/%3E%3Crect x='64' y='43' width='19' height='19' fill-opacity='0.052'/%3E%3Crect x='1' y='64' width='19' height='19' fill-opacity='0.026'/%3E%3Crect x='22' y='64' width='19' height='19' fill-opacity='0.042'/%3E%3Crect x='43' y='64' width='19' height='19' fill-opacity='0.045'/%3E%3Crect x='64' y='64' width='19' height='19' fill-opacity='0.034'/%3E%3C/svg%3E");
--glow-texture-size: 107px 107px;
--cover-hover-transform: translateY(-3px);
--format-chip-fill: 18%;
@@ -99,7 +98,6 @@ html[data-style='nostalgia'].dark {
--sidebar-count-foreground: oklch(from var(--primary) max(l, 0.72) c h);
--glow-texture: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 84 84' fill='%23fff'%3E%3Crect x='1' y='1' width='19' height='19' fill-opacity='0.056'/%3E%3Crect x='22' y='1' width='19' height='19' fill-opacity='0.042'/%3E%3Crect x='43' y='1' width='19' height='19' fill-opacity='0.082'/%3E%3Crect x='64' y='1' width='19' height='19' fill-opacity='0.036'/%3E%3Crect x='1' y='22' width='19' height='19' fill-opacity='0.073'/%3E%3Crect x='22' y='22' width='19' height='19' fill-opacity='0.059'/%3E%3Crect x='43' y='22' width='19' height='19' fill-opacity='0.035'/%3E%3Crect x='64' y='22' width='19' height='19' fill-opacity='0.071'/%3E%3Crect x='1' y='43' width='19' height='19' fill-opacity='0.033'/%3E%3Crect x='22' y='43' width='19' height='19' fill-opacity='0.065'/%3E%3Crect x='43' y='43' width='19' height='19' fill-opacity='0.036'/%3E%3Crect x='64' y='43' width='19' height='19' fill-opacity='0.037'/%3E%3Crect x='1' y='64' width='19' height='19' fill-opacity='0.064'/%3E%3Crect x='22' y='64' width='19' height='19' fill-opacity='0.096'/%3E%3Crect x='43' y='64' width='19' height='19' fill-opacity='0.04'/%3E%3Crect x='64' y='64' width='19' height='19' fill-opacity='0.048'/%3E%3C/svg%3E");
/* Solid chips in a deep shade of each format's hue with white ink, as the classic look wore them; at this
lightness every format clears 5.4:1. */
--format-chip-fill: 100%;
@@ -1,7 +1,10 @@
<script setup lang="ts">
import { computed, ref } from 'vue'
import { useI18n } from 'vue-i18n'
import { ArrowDown, ArrowLeftRight, ArrowUp, FileCode, FileText, GripVertical, Headphones, RotateCcw } from '@lucide/vue'
import { ArrowDown, ArrowUp, FileCode, FileText, GripVertical, Headphones, RotateCcw } from '@lucide/vue'
import type { RegexMetadataConfig } from '@bookorbit/types'
import { Button } from '@/components/ui/button'
import RegexMetadataDialog from './RegexMetadataDialog.vue'
import { DEFAULT_FORMAT_PRIORITY, isReadAlongFormatKey, withReadAlongFormatPriority } from '@bookorbit/types'
import { formatNumber } from '@/i18n/formatters'
import { formatColorVar } from '@/features/book/lib/format-colors'
@@ -18,6 +21,8 @@ const PREVIEW_ROWS = 6
const props = withDefaults(
defineProps<{
metadataPrecedence: string[]
regexMetadata?: RegexMetadataConfig | null
libraryId?: number
formatPriority: string[]
allowedFormats?: string[]
formatCounts?: Record<string, number> | null
@@ -26,10 +31,13 @@ const props = withDefaults(
)
const emit = defineEmits<{
'update:regexMetadata': [value: RegexMetadataConfig | null]
'update:regexOpen': [value: boolean]
'update:metadataPrecedence': [value: string[]]
'update:formatPriority': [value: string[]]
}>()
const regexOpen = ref(false)
const showAll = ref(false)
const dragKey = ref<string | null>(null)
const overKey = ref<string | null>(null)
@@ -146,8 +154,47 @@ function onDragEnd() {
overKey.value = null
}
function swapSources() {
emit('update:metadataPrecedence', [...sources.value].reverse())
function moveSourceUp(key: string) {
const next = [...sources.value]
const index = next.indexOf(key)
if (index <= 0) return
next.splice(index, 1)
next.splice(index - 1, 0, key)
emit('update:metadataPrecedence', next)
}
function moveSourceDown(key: string) {
const next = [...sources.value]
const index = next.indexOf(key)
if (index < 0 || index === next.length - 1) return
next.splice(index, 1)
next.splice(index + 1, 0, key)
emit('update:metadataPrecedence', next)
}
function openRegex() {
regexOpen.value = true
emit('update:regexOpen', true)
}
function closeRegex() {
regexOpen.value = false
emit('update:regexOpen', false)
}
function applyRegex(config: RegexMetadataConfig | null) {
emit('update:regexMetadata', config)
closeRegex()
}
function sourceTitle(source: string) {
return t(
source === 'regex'
? 'library.creator.metadata.regex.sourceTitle'
: `library.creator.metadata.source.${source === 'embedded' ? 'embeddedTitle' : 'opfTitle'}`,
)
}
function sourceHint(source: string) {
if (source === 'regex') return t(props.regexMetadata ? 'library.creator.metadata.regex.enabledHint' : 'library.creator.metadata.regex.disabledHint')
return t(`library.creator.metadata.source.${source === 'embedded' ? 'embeddedHint' : 'opfHint'}`)
}
</script>
@@ -246,42 +293,43 @@ function swapSources() {
</LibraryCreatorCard>
<LibraryCreatorCard :label="t('library.creator.metadata.source.title')" label-id="metadata-source-title">
<ol aria-labelledby="metadata-source-title" class="flex flex-col items-stretch gap-2 @lg:flex-row @lg:items-center">
<template v-for="(source, index) in sources" :key="source">
<li
class="flex min-w-0 flex-1 items-center gap-3 rounded-xl border px-3 py-2.5"
:class="index === 0 ? 'border-primary/55 bg-primary/7' : 'border-border bg-background'"
>
<span
class="flex size-5.5 shrink-0 items-center justify-center rounded-full text-[11px] font-bold"
:class="index === 0 ? 'bg-primary text-primary-foreground' : 'bg-muted text-muted-foreground'"
>
{{ formatNumber(index + 1) }}
</span>
<component :is="source === 'embedded' ? FileText : FileCode" :size="16" class="shrink-0 text-foreground" aria-hidden="true" />
<span class="min-w-0">
<span class="block text-[13px] font-semibold text-foreground">
{{ source === 'embedded' ? t('library.creator.metadata.source.embeddedTitle') : t('library.creator.metadata.source.opfTitle') }}
</span>
<span class="block text-xs text-muted-foreground">
{{ source === 'embedded' ? t('library.creator.metadata.source.embeddedHint') : t('library.creator.metadata.source.opfHint') }}
</span>
</span>
</li>
<li v-if="index === 0" class="flex justify-center">
<button
type="button"
class="flex size-8 items-center justify-center rounded-full border border-border bg-card text-foreground transition-colors hover:bg-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
:aria-label="t('library.creator.metadata.source.swap')"
:title="t('library.creator.metadata.source.swap')"
@click="swapSources"
>
<ArrowLeftRight :size="14" class="rotate-90 @lg:rotate-0" aria-hidden="true" />
</button>
</li>
</template>
<ol aria-labelledby="metadata-source-title" class="flex flex-col gap-2">
<li
v-for="(source, index) in sources"
:key="source"
class="flex min-w-0 flex-wrap items-center gap-3 rounded-xl border border-border bg-background px-3 py-2.5"
>
<span class="text-xs font-bold text-muted-foreground">{{ formatNumber(index + 1) }}</span>
<component :is="source === 'embedded' ? FileText : FileCode" :size="16" class="shrink-0" aria-hidden="true" />
<span class="min-w-0 flex-1">
<span class="block text-sm font-semibold">{{ sourceTitle(source) }}</span>
<span class="block text-xs text-muted-foreground">{{ sourceHint(source) }}</span>
</span>
<Button v-if="source === 'regex'" variant="outline" size="sm" @click="openRegex">{{
t('library.creator.metadata.regex.configure')
}}</Button>
<div class="flex gap-1">
<Button
variant="ghost"
size="icon"
:disabled="index === 0"
:aria-label="t('library.creator.metadata.regex.moveSourceUp', { source: sourceTitle(source) })"
@click="moveSourceUp(source)"
><ArrowUp
/></Button>
<Button
variant="ghost"
size="icon"
:disabled="index === sources.length - 1"
:aria-label="t('library.creator.metadata.regex.moveSourceDown', { source: sourceTitle(source) })"
@click="moveSourceDown(source)"
><ArrowDown
/></Button>
</div>
</li>
</ol>
<p class="mt-2.5 text-xs text-muted-foreground">{{ t('library.creator.metadata.source.hint') }}</p>
<p class="mt-2.5 text-xs text-muted-foreground">{{ t('library.creator.metadata.regex.precedenceHint') }}</p>
</LibraryCreatorCard>
<RegexMetadataDialog v-if="regexOpen" :config="regexMetadata ?? null" :library-id="libraryId" @close="closeRegex" @apply="applyRegex" />
</div>
</template>
@@ -183,6 +183,8 @@ function summaryFor(id: LibraryCreatorSectionId): string {
? t('library.creator.summary.scannerAll', { mode: modeLabel.value })
: t('library.creator.summary.scannerSome', { mode: modeLabel.value, count: form.allowedFormats.length })
case 'metadata':
if (form.regexMetadata && form.metadataPrecedence[0] === 'regex')
return t('library.creator.metadata.regex.summary', { format: leadFormat.value })
return form.metadataPrecedence[0] === 'opfFile'
? t('library.creator.summary.metadataOpf', { format: leadFormat.value })
: t('library.creator.summary.metadataFile', { format: leadFormat.value })
@@ -285,6 +287,8 @@ const sectionProps = computed(() => ({
},
metadata: {
metadataPrecedence: form.metadataPrecedence,
regexMetadata: form.regexMetadata ?? null,
libraryId: creator.editingLibraryId.value ?? undefined,
formatPriority: form.formatPriority,
allowedFormats: form.allowedFormats,
formatCounts: stats.value?.formatCounts ?? null,
@@ -495,6 +499,7 @@ const sectionListeners = {
'update:addedAtSource': (value: Library['addedAtSource']) => (form.addedAtSource = value),
recompute: handleRecomputeAddedAt,
check: handleCheckFolders,
'update:regexMetadata': (value: Library['regexMetadata']) => (form.regexMetadata = value),
'update:metadataPrecedence': (value: string[]) => (form.metadataPrecedence = value),
'update:formatPriority': (value: string[]) => (form.formatPriority = value),
'update:allowedFormats': (value: string[]) => (form.allowedFormats = value),
@@ -522,6 +527,7 @@ const sectionListeners = {
'update:fileWriteReadAlongEnabled': (value: boolean) => (form.fileWriteReadAlongEnabled = value),
'update:fileWriteReadAlongMaxFileSizeMb': (value: number) => (form.fileWriteReadAlongMaxFileSizeMb = value),
'update:pickerOpen': handleNestedModalChange,
'update:regexOpen': handleNestedModalChange,
}
useModal({
@@ -0,0 +1,239 @@
<script setup lang="ts">
import { useI18n } from 'vue-i18n'
import { DialogContent, DialogDescription, DialogOverlay, DialogPortal, DialogRoot, DialogTitle } from 'reka-ui'
import { ArrowDown, ArrowUp, Plus, Trash2 } from '@lucide/vue'
import { REGEX_METADATA_GROUPS, REGEX_METADATA_LIMITS, type RegexMetadataConfig } from '@bookorbit/types'
import { Button } from '@/components/ui/button'
import { Input } from '@/components/ui/input'
import ToggleSwitch from '@/components/ui/ToggleSwitch.vue'
import { formatNumber } from '@/i18n/formatters'
import { useRegexMetadataEditor } from '../composables/useRegexMetadataEditor'
const props = defineProps<{ config: RegexMetadataConfig | null; libraryId?: number }>()
const emit = defineEmits<{ close: []; apply: [config: RegexMetadataConfig | null] }>()
const { t } = useI18n()
const {
validation,
validating,
validationError,
ruleErrors,
patternInvalid,
flagsInvalid,
generalDiagnostics,
validRuleCount,
validateRules,
enabled,
rules,
relativePath,
result,
busy,
error,
canApply,
canPreview,
addRule,
removeRule,
moveUp,
moveDown,
value,
preview,
} = useRegexMetadataEditor(props.config, props.libraryId)
const examplePattern = String.raw`(?<author>[^\/]+)\/(?<series>[^\/]+)\/(?<seriesIndex>\d+) - (?<title>.+) - \k<author>`
const examplePath = 'Donna Leon/Commissario Brunetti/01 - Venezianisches Finale - Donna Leon.epub'
const FIELD_LABEL_KEYS: Record<string, string> = {
title: 'title',
author: 'authors',
authors: 'authors',
series: 'series',
seriesName: 'series',
seriesIndex: 'seriesIndex',
publishedYear: 'publishedYear',
publishedDate: 'publishedDate',
}
const diagnosticParams = {
count: formatNumber(REGEX_METADATA_LIMITS.patternLength),
min: formatNumber(1),
max: formatNumber(REGEX_METADATA_LIMITS.rules),
}
function fieldLabel(field: string) {
const key = FIELD_LABEL_KEYS[field]
return key ? t(`library.creator.metadata.regex.fields.${key}`) : field
}
function diagnosticMessage(code: string) {
return t(`library.creator.metadata.regex.diagnostics.${code}`, diagnosticParams)
}
function close() {
emit('close')
}
function handleOpenChange(open: boolean) {
if (!open) close()
}
function apply() {
if (canApply.value) emit('apply', value())
}
</script>
<template>
<DialogRoot :open="true" @update:open="handleOpenChange">
<DialogPortal>
<DialogOverlay class="fixed inset-0 z-[70] bg-scrim" />
<DialogContent
class="fixed left-1/2 top-1/2 z-[70] flex max-h-[90dvh] w-[calc(100%-2rem)] max-w-3xl -translate-x-1/2 -translate-y-1/2 flex-col gap-4 overflow-y-auto rounded-lg border border-border bg-card p-5 shadow-xl"
>
<DialogTitle class="text-lg font-semibold">{{ t('library.creator.metadata.regex.title') }}</DialogTitle>
<DialogDescription class="text-sm text-muted-foreground">{{ t('library.creator.metadata.regex.description') }}</DialogDescription>
<p class="text-xs text-muted-foreground">{{ t('library.creator.metadata.regex.primaryFileHint') }}</p>
<div class="flex items-center gap-2 text-sm font-medium">
<ToggleSwitch v-model="enabled" aria-labelledby="regex-enable-label" />
<span id="regex-enable-label">{{ t('library.creator.metadata.regex.enable') }}</span>
</div>
<p class="text-xs text-muted-foreground">
{{ t('library.creator.metadata.regex.groups') }} <code>{{ REGEX_METADATA_GROUPS.join(', ') }}</code>
</p>
<p class="text-xs text-muted-foreground">{{ t('library.creator.metadata.regex.valuesHint') }}</p>
<section class="flex min-w-0 flex-col gap-2 rounded-md border border-border bg-muted p-3 text-xs">
<h3 class="font-medium">{{ t('library.creator.metadata.regex.exampleTitle') }}</h3>
<dl class="flex min-w-0 flex-col gap-1">
<dt class="text-muted-foreground">{{ t('library.creator.metadata.regex.sourceTitle') }}</dt>
<dd class="select-text break-all font-mono">{{ examplePattern }}</dd>
<dt class="mt-1 text-muted-foreground">{{ t('library.creator.metadata.regex.exampleTestPath') }}</dt>
<dd class="select-text break-all font-mono">{{ examplePath }}</dd>
</dl>
</section>
<fieldset v-if="enabled" class="flex min-w-0 flex-col gap-3">
<legend class="sr-only">{{ t('library.creator.metadata.regex.rules') }}</legend>
<div v-for="(rule, index) in rules" :key="index" class="flex min-w-0 flex-col gap-2 rounded-md border border-border p-3">
<div class="flex items-center justify-between gap-2">
<label :for="`regex-pattern-${index}`" class="text-sm font-medium">{{
t('library.creator.metadata.regex.rule', { number: formatNumber(index + 1) })
}}</label>
<div class="flex gap-1">
<Button
variant="ghost"
size="icon"
:disabled="index === 0"
:aria-label="t('library.creator.metadata.regex.moveUp')"
@click="moveUp(index)"
><ArrowUp
/></Button>
<Button
variant="ghost"
size="icon"
:disabled="index === rules.length - 1"
:aria-label="t('library.creator.metadata.regex.moveDown')"
@click="moveDown(index)"
><ArrowDown
/></Button>
<Button variant="ghost" size="icon" :aria-label="t('library.creator.metadata.regex.remove')" @click="removeRule(index)"
><Trash2
/></Button>
</div>
</div>
<div
class="flex min-w-0 flex-wrap items-center rounded-md border bg-background focus-within:ring-2 focus-within:ring-ring"
:class="patternInvalid[index] ? 'border-destructive' : 'border-input'"
>
<span class="px-2 py-2 text-xs text-muted-foreground">{{ t('library.creator.metadata.regex.basePath') }} /</span>
<input
:id="`regex-pattern-${index}`"
v-model="rule.pattern"
:maxlength="REGEX_METADATA_LIMITS.patternLength"
:aria-describedby="`regex-extension-${index} regex-errors-${index}`"
:aria-invalid="patternInvalid[index] || undefined"
autocomplete="off"
spellcheck="false"
class="min-w-24 flex-1 bg-transparent px-2 py-2 font-mono text-sm outline-none"
/>
<span
:id="`regex-extension-${index}`"
:title="t('library.creator.metadata.regex.extensionHint')"
class="px-2 py-2 font-mono text-xs text-muted-foreground"
>.ext<span class="sr-only">: {{ t('library.creator.metadata.regex.extensionHint') }}</span></span
>
</div>
<label class="flex items-center gap-2 text-xs text-muted-foreground">
{{ t('library.creator.metadata.regex.flags') }}
<Input
v-model="rule.flags"
:aria-describedby="`regex-errors-${index}`"
:aria-invalid="flagsInvalid[index] || undefined"
maxlength="2"
class="w-20 font-mono"
autocomplete="off"
spellcheck="false"
/>
</label>
<div :id="`regex-errors-${index}`" aria-live="polite">
<p v-for="code in ruleErrors[index]" :key="code" class="text-xs text-destructive">
{{ diagnosticMessage(code) }}
</p>
</div>
</div>
<Button variant="outline" :disabled="rules.length >= REGEX_METADATA_LIMITS.rules" class="self-start" @click="addRule"
><Plus />{{ t('library.creator.metadata.regex.add') }}</Button
>
<label class="flex flex-col gap-1 text-sm">
{{ t('library.creator.metadata.regex.examplePath') }}
<Input
v-model="relativePath"
:maxlength="REGEX_METADATA_LIMITS.pathLength"
:placeholder="t('library.creator.metadata.regex.examplePlaceholder')"
/>
</label>
<Button variant="outline" :disabled="!canPreview" class="self-start" @click="preview">{{
t(busy ? 'library.creator.metadata.regex.previewing' : 'library.creator.metadata.regex.preview')
}}</Button>
</fieldset>
<div v-if="enabled" aria-live="polite" class="flex flex-col gap-2 text-sm">
<p v-if="!validRuleCount" class="text-destructive">{{ t('library.creator.metadata.regex.validationRuleCount', diagnosticParams) }}</p>
<p v-else-if="validating" class="text-muted-foreground">{{ t('library.creator.metadata.regex.validating') }}</p>
<p v-else-if="validation?.valid" class="text-muted-foreground">{{ t('library.creator.metadata.regex.validationSuccess') }}</p>
<p v-if="validationError" role="alert" class="text-destructive">{{ t('library.creator.metadata.regex.validationError') }}</p>
<p v-for="diagnostic in generalDiagnostics" :key="diagnostic.code" class="text-destructive">
{{ diagnosticMessage(diagnostic.code) }}
</p>
<Button
v-if="validationError || generalDiagnostics.length"
variant="outline"
class="self-start"
:disabled="validating"
@click="validateRules"
>{{ t('library.creator.metadata.regex.validationRetry') }}</Button
>
</div>
<p v-if="error" role="alert" class="text-sm text-destructive">{{ t('library.creator.metadata.regex.previewError') }}</p>
<div v-if="result" aria-live="polite" class="flex flex-col gap-2 rounded-md border border-border p-3 text-sm">
<p>
{{ t('library.creator.metadata.regex.actualInput') }} <code class="break-all">{{ result.inputPath }}</code>
</p>
<p>
{{
result.matched
? t('library.creator.metadata.regex.matched', { number: formatNumber((result.ruleIndex ?? 0) + 1) })
: t('library.creator.metadata.regex.noMatch')
}}
</p>
<dl v-if="result.matched" class="grid grid-cols-[auto_1fr] gap-x-3 gap-y-1">
<template v-for="(fieldValue, field) in result.metadata" :key="field"
><dt class="text-xs text-muted-foreground">{{ fieldLabel(field) }}</dt>
<dd class="break-all">{{ Array.isArray(fieldValue) ? fieldValue.join('; ') : fieldValue }}</dd></template
>
</dl>
<p v-for="(diagnostic, index) in result.diagnostics" :key="index" class="text-destructive">
{{ diagnosticMessage(diagnostic.code) }} <span v-if="diagnostic.field">({{ fieldLabel(diagnostic.field) }})</span>
</p>
</div>
<p class="text-xs text-muted-foreground">{{ t('library.creator.metadata.regex.scanHint') }}</p>
<div class="flex justify-end gap-2">
<Button variant="outline" @click="close">{{ t('common.cancel') }}</Button>
<Button :disabled="!canApply" @click="apply">{{ t('library.creator.metadata.regex.apply') }}</Button>
</div>
</DialogContent>
</DialogPortal>
</DialogRoot>
</template>
@@ -79,20 +79,20 @@ describe('LibraryCreatorMetadata', () => {
expect(wrapper.findAll('ol[aria-labelledby="primary-file-title"] > li')[1]!.text()).toContain('Not imported')
})
it('swaps which source is read first', async () => {
it('moves a source earlier in the precedence list', async () => {
const wrapper = mountMetadata({ formatPriority: ['epub'] })
await wrapper.get('button[aria-label="Swap the order"]').trigger('click')
await wrapper.get('button[aria-label="Move metadata.opf file up"]').trigger('click')
expect(wrapper.emitted('update:metadataPrecedence')).toEqual([[['opfFile', 'embedded']]])
expect(wrapper.emitted('update:metadataPrecedence')).toEqual([[['opfFile', 'embedded', 'regex']]])
})
it('fills in a source missing from an older saved list before swapping', async () => {
it('fills in a source missing from an older saved list before reordering', async () => {
const wrapper = mountMetadata({ formatPriority: ['epub'], metadataPrecedence: ['embedded'] })
expect(wrapper.text()).toContain('metadata.opf file')
await wrapper.get('button[aria-label="Swap the order"]').trigger('click')
await wrapper.get('button[aria-label="Move metadata.opf file up"]').trigger('click')
expect(wrapper.emitted('update:metadataPrecedence')).toEqual([[['opfFile', 'embedded']]])
expect(wrapper.emitted('update:metadataPrecedence')).toEqual([[['opfFile', 'embedded', 'regex']]])
})
})
@@ -0,0 +1,158 @@
import { effectScope } from 'vue'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { api } from '@/lib/api'
import { useRegexMetadataEditor } from '../useRegexMetadataEditor'
vi.mock('@/lib/api', () => ({ api: vi.fn<typeof import('@/lib/api').api>() }))
describe('useRegexMetadataEditor', () => {
const scopes: ReturnType<typeof effectScope>[] = []
afterEach(() => {
scopes.splice(0).forEach((scope) => scope.stop())
vi.useRealTimers()
vi.resetAllMocks()
})
function editor(config = { rules: [{ pattern: '(?<title>.+)$', flags: '' }] }) {
const scope = effectScope()
scopes.push(scope)
return scope.run(() => useRegexMetadataEditor(config, 42))!
}
it('edits an isolated draft and preserves the ordering on apply', () => {
const config = { rules: [{ pattern: '(?<title>.+)$', flags: '' }] }
const state = editor(config)
state.rules.value[0]!.pattern = '(?<series>.+)'
state.addRule()
state.rules.value[1]!.pattern = '(?<title>.+)'
state.moveUp(1)
expect(config.rules[0]!.pattern).toBe('(?<title>.+)$')
expect(state.value()?.rules.map((rule) => rule.pattern)).toEqual(['(?<title>.+)', '(?<series>.+)'])
state.enabled.value = false
expect(state.value()).toBeNull()
})
it('sends the unchanged pattern and example path with its extension', async () => {
const state = editor()
state.relativePath.value = 'Author/My.Book.epub'
vi.mocked(api).mockResolvedValue(
new Response(
JSON.stringify({
inputPath: 'Author/My.Book',
matched: true,
ruleIndex: 0,
metadata: { title: 'Author/My.Book' },
groups: {},
diagnostics: [],
}),
),
)
await state.preview()
const request = vi.mocked(api).mock.calls[0]!
expect(request[0]).toBe('/api/v1/libraries/regex-metadata/preview')
expect(JSON.parse(request[1]!.body as string)).toEqual({
config: { rules: [{ pattern: '(?<title>.+)$', flags: '' }] },
relativePath: 'Author/My.Book.epub',
libraryId: 42,
})
expect(state.result.value?.inputPath).toBe('Author/My.Book')
})
it('ignores an old response after the input changes', async () => {
const state = editor()
state.relativePath.value = 'Old.epub'
let resolve!: (response: Response) => void
vi.mocked(api).mockReturnValue(
new Promise((done) => {
resolve = done
}),
)
const pending = state.preview()
state.relativePath.value = 'New.epub'
resolve(new Response(JSON.stringify({ matched: true, inputPath: 'Old' })))
await pending
expect(state.result.value).toBeNull()
expect(state.busy.value).toBe(false)
})
it('shows failed requests and clears them when the draft changes', async () => {
const state = editor()
state.relativePath.value = '../outside.epub'
vi.mocked(api).mockResolvedValue(new Response('', { status: 400 }))
await state.preview()
expect(state.error.value).toBe(true)
state.relativePath.value = 'Inside.epub'
expect(state.error.value).toBe(false)
})
it('debounces validation and only allows applying the current validated rules', async () => {
vi.useFakeTimers()
vi.mocked(api).mockResolvedValue(new Response(JSON.stringify({ valid: true, diagnostics: [] })))
const state = editor()
expect(state.canApply.value).toBe(false)
await vi.advanceTimersByTimeAsync(399)
expect(api).not.toHaveBeenCalled()
state.rules.value[0]!.pattern = '(?<author>.+)'
await vi.advanceTimersByTimeAsync(400)
expect(api).toHaveBeenCalledTimes(1)
expect(JSON.parse(vi.mocked(api).mock.calls[0]![1]!.body as string)).toEqual({
config: { rules: [{ pattern: '(?<author>.+)', flags: '' }] },
libraryId: 42,
})
expect(state.canApply.value).toBe(true)
state.relativePath.value = 'No match.epub'
expect(state.canApply.value).toBe(true)
state.rules.value[0]!.pattern = '['
expect(state.canApply.value).toBe(false)
state.enabled.value = false
expect(state.canApply.value).toBe(true)
await vi.advanceTimersByTimeAsync(400)
expect(api).toHaveBeenCalledTimes(1)
})
it('ignores stale validation and remaps errors after rules are reordered', async () => {
vi.useFakeTimers()
let resolve!: (response: Response) => void
vi.mocked(api).mockReturnValueOnce(
new Promise((done) => {
resolve = done
}),
)
const state = editor({
rules: [
{ pattern: '[', flags: '' },
{ pattern: '(?<title>.+)', flags: '' },
],
})
await vi.advanceTimersByTimeAsync(400)
state.moveDown(0)
resolve(new Response(JSON.stringify({ valid: true, diagnostics: [] })))
await Promise.resolve()
expect(state.canApply.value).toBe(false)
vi.mocked(api).mockResolvedValueOnce(new Response(JSON.stringify({ valid: false, diagnostics: [{ code: 'invalid_pattern', ruleIndex: 1 }] })))
await vi.advanceTimersByTimeAsync(400)
expect(state.ruleErrors.value).toEqual([[], ['invalid_pattern']])
expect(state.canApply.value).toBe(false)
state.removeRule(1)
expect(state.ruleErrors.value).toEqual([[]])
expect(state.canApply.value).toBe(false)
})
it('reports local bounds without sending requests and allows retry after network and worker failures', async () => {
vi.useFakeTimers()
const state = editor({ rules: [{ pattern: '', flags: 'gg' }] })
await vi.advanceTimersByTimeAsync(400)
expect(api).not.toHaveBeenCalled()
expect(state.ruleErrors.value[0]).toEqual(['pattern_required', 'invalid_flags'])
state.rules.value = [{ pattern: '(?<title>.+)', flags: '' }]
vi.mocked(api).mockRejectedValueOnce(new Error('offline'))
await vi.advanceTimersByTimeAsync(400)
expect(state.validationError.value).toBe(true)
expect(state.canApply.value).toBe(false)
vi.mocked(api).mockResolvedValueOnce(new Response(JSON.stringify({ valid: false, diagnostics: [{ code: 'worker_failed' }] })))
await state.validateRules()
expect(state.generalDiagnostics.value).toEqual([{ code: 'worker_failed' }])
expect(state.canApply.value).toBe(false)
vi.mocked(api).mockResolvedValueOnce(new Response(JSON.stringify({ valid: true, diagnostics: [] })))
await state.validateRules()
expect(state.canApply.value).toBe(true)
})
})
@@ -12,11 +12,12 @@ import { coveringFolderPath, normalizeFolderPath } from './folder-paths'
export { DEFAULT_FORMAT_PRIORITY }
export const DEFAULT_METADATA_PRECEDENCE = ['embedded', 'opfFile']
export const DEFAULT_METADATA_PRECEDENCE = ['embedded', 'opfFile', 'regex']
export const METADATA_LABELS: Record<string, string> = {
embedded: 'Embedded metadata',
opfFile: 'OPF files',
regex: 'Regex metadata',
}
export type LibraryCreatorSectionId = 'details' | 'folders' | 'scanner' | 'metadata' | 'reading' | 'schedule' | 'fileWrite' | 'access'
@@ -41,6 +42,7 @@ function blankForm() {
watchLocalFolders: true,
autoScanCronExpression: null as string | null,
metadataPrecedence: [...DEFAULT_METADATA_PRECEDENCE],
regexMetadata: null as Library['regexMetadata'],
formatPriority: withReadAlongFormatPriority(DEFAULT_FORMAT_PRIORITY),
allowedFormats: [] as string[],
organizationMode: 'book_per_folder' as OrganizationMode,
@@ -161,6 +163,7 @@ export function useLibraryCreator() {
form.watchLocalFolders = library.watchLocalFolders ?? true
form.autoScanCronExpression = library.autoScanCronExpression ?? null
form.metadataPrecedence = [...library.metadataPrecedence]
form.regexMetadata = library.regexMetadata ? { rules: library.regexMetadata.rules.map((rule) => ({ ...rule })) } : null
const missing = DEFAULT_FORMAT_PRIORITY.filter((f) => !library.formatPriority.includes(f))
form.formatPriority = withReadAlongFormatPriority([...library.formatPriority, ...missing])
form.allowedFormats = [...library.allowedFormats]
@@ -0,0 +1,178 @@
import { computed, onScopeDispose, ref, watch } from 'vue'
import { REGEX_METADATA_LIMITS, type RegexMetadataConfig, type RegexMetadataPreview, type RegexMetadataValidation } from '@bookorbit/types'
import { api } from '@/lib/api'
export function useRegexMetadataEditor(config: RegexMetadataConfig | null, libraryId?: number) {
const enabled = ref(config !== null)
const rules = ref(config?.rules.map((rule) => ({ ...rule })) ?? [{ pattern: '', flags: '' }])
const relativePath = ref('')
const result = ref<RegexMetadataPreview | null>(null)
const busy = ref(false)
const error = ref(false)
let sequence = 0
let controller: AbortController | undefined
function invalidate() {
sequence++
controller?.abort()
busy.value = false
result.value = null
error.value = false
}
watch([rules, relativePath, enabled], invalidate, { deep: true, flush: 'sync' })
onScopeDispose(invalidate)
const validation = ref<RegexMetadataValidation | null>(null)
const validating = ref(false)
const validationError = ref(false)
let validationSequence = 0
let validationController: AbortController | undefined
let validationTimer: ReturnType<typeof setTimeout> | undefined
const localRuleErrors = computed(() =>
rules.value.map((rule) => {
const errors: string[] = []
if (!rule.pattern.length) errors.push('pattern_required')
else if (rule.pattern.length > REGEX_METADATA_LIMITS.patternLength) errors.push('pattern_too_long')
if (!/^(?:i?u?|ui)$/.test(rule.flags)) errors.push('invalid_flags')
return errors
}),
)
const validRuleCount = computed(() => rules.value.length > 0 && rules.value.length <= REGEX_METADATA_LIMITS.rules)
const valid = computed(() => validRuleCount.value && localRuleErrors.value.every((errors) => !errors.length))
const ruleErrors = computed(() =>
localRuleErrors.value.map((errors, index) => [
...errors,
...(validation.value?.diagnostics.filter((diagnostic) => diagnostic.ruleIndex === index).map((diagnostic) => diagnostic.code) ?? []),
]),
)
const patternInvalid = computed(() => ruleErrors.value.map((errors) => errors.some((code) => code !== 'invalid_flags')))
const flagsInvalid = computed(() => ruleErrors.value.map((errors) => errors.includes('invalid_flags')))
const generalDiagnostics = computed(() => validation.value?.diagnostics.filter((diagnostic) => diagnostic.ruleIndex === undefined) ?? [])
const canApply = computed(() => !enabled.value || (valid.value && validation.value?.valid === true && !validating.value && !validationError.value))
const canPreview = computed(() => enabled.value && valid.value && !!relativePath.value && !busy.value)
function invalidateValidation() {
validationSequence++
clearTimeout(validationTimer)
validationController?.abort()
validation.value = null
validationError.value = false
validating.value = false
}
async function validateRules() {
invalidateValidation()
if (!enabled.value || !valid.value) return
const requestSequence = validationSequence
validationController = new AbortController()
validating.value = true
try {
const response = await api('/api/v1/libraries/regex-metadata/validate', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ config: value(), libraryId }),
signal: validationController.signal,
})
if (!response.ok) {
if (requestSequence === validationSequence) validationError.value = true
return
}
const data = (await response.json()) as RegexMetadataValidation
if (requestSequence === validationSequence) validation.value = data
} catch {
if (requestSequence === validationSequence) validationError.value = true
} finally {
if (requestSequence === validationSequence) validating.value = false
}
}
watch(
[rules, enabled],
() => {
invalidateValidation()
if (!enabled.value || !valid.value) return
validating.value = true
validationTimer = setTimeout(validateRules, 400)
},
{ deep: true, flush: 'sync', immediate: true },
)
onScopeDispose(invalidateValidation)
function addRule() {
if (rules.value.length < REGEX_METADATA_LIMITS.rules) rules.value.push({ pattern: '', flags: '' })
}
function removeRule(index: number) {
rules.value.splice(index, 1)
}
function moveRule(index: number, direction: -1 | 1) {
const target = index + direction
if (target < 0 || target >= rules.value.length) return
const next = [...rules.value]
const [rule] = next.splice(index, 1)
next.splice(target, 0, rule!)
rules.value = next
}
function moveUp(index: number) {
moveRule(index, -1)
}
function moveDown(index: number) {
moveRule(index, 1)
}
function value(): RegexMetadataConfig | null {
return enabled.value ? { rules: rules.value.map((rule) => ({ ...rule })) } : null
}
async function preview() {
if (!canPreview.value) return
invalidate()
const requestSequence = sequence
controller = new AbortController()
busy.value = true
try {
const response = await api('/api/v1/libraries/regex-metadata/preview', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ config: value(), relativePath: relativePath.value, libraryId }),
signal: controller.signal,
})
if (!response.ok) {
if (requestSequence === sequence) error.value = true
return
}
const data = (await response.json()) as RegexMetadataPreview
if (requestSequence === sequence) result.value = data
} catch {
if (requestSequence === sequence) error.value = true
} finally {
if (requestSequence === sequence) busy.value = false
}
}
return {
validation,
validating,
validationError,
ruleErrors,
patternInvalid,
flagsInvalid,
generalDiagnostics,
validRuleCount,
validateRules,
enabled,
rules,
relativePath,
result,
busy,
error,
canApply,
canPreview,
addRule,
removeRule,
moveUp,
moveDown,
value,
preview,
}
}
+62
View File
@@ -10535,6 +10535,68 @@
"opfHint": "Saved next to the book by Calibre",
"swap": "Swap the order",
"hint": "BookOrbit uses the first one a book has and ignores the other. Calibre libraries usually want metadata.opf first."
},
"regex": {
"title": "Configure regex metadata",
"sourceTitle": "Regex",
"description": "Extract metadata from the primary file path relative to its library folder. The file extension is removed before matching. The first rule with usable fields wins.",
"enable": "Enable regex metadata",
"primaryFileHint": "For a book made of several files, such as audiobook tracks, the rules read the primary file's path.",
"groups": "Named groups:",
"valuesHint": "Use author for one author or authors separated by semicolons. Dates use YYYY-MM-DD. Flags: i ignores case, u enables Unicode.",
"rules": "Ordered extraction rules",
"rule": "Rule {number}",
"moveUp": "Move rule up",
"moveDown": "Move rule down",
"remove": "Remove rule",
"basePath": "Library folder",
"extensionHint": "The file extension is automatically ignored. The end anchor matches the filename without its extension.",
"flags": "Flags",
"add": "Add rule",
"examplePath": "Example relative file path, including extension",
"examplePlaceholder": "Author/Series/08 - My.Book.epub",
"exampleTitle": "Example with an author backreference",
"exampleTestPath": "Test path (including extension)",
"validating": "Checking rules…",
"validationSuccess": "Rule syntax and groups are valid. Use the preview to check matches for a file path.",
"validationError": "Could not validate the rules. Try again before applying them.",
"validationRetry": "Check again",
"validationRuleCount": "Add between {min} and {max} rules.",
"preview": "Preview",
"previewing": "Testing rules…",
"previewError": "Could not preview the rules. Check the relative path and configuration, then try again.",
"actualInput": "Path tested:",
"matched": "Matched rule {number}",
"noMatch": "No usable match. File metadata will be used.",
"scanHint": "Changes apply to future metadata extraction for new or changed files. Saving does not start a scan or update unchanged books.",
"apply": "Apply to library form",
"configure": "Configure",
"enabledHint": "Extract fields from relative paths without file extensions.",
"disabledHint": "Disabled. Configure rules to enable.",
"moveSourceUp": "Move {source} up",
"moveSourceDown": "Move {source} down",
"diagnostics": {
"pattern_required": "Enter a regular expression.",
"pattern_too_long": "The expression must not exceed {count} characters.",
"invalid_flags": "Only i and u are allowed, each at most once.",
"invalid_pattern": "Invalid regular expression.",
"missing_group": "Include at least one supported named group.",
"conflicting_authors": "Use either author or authors, not both.",
"invalid_value": "An invalid field value was skipped.",
"timeout": "The expression took too long. Simplify the pattern.",
"busy": "All preview workers are busy. Try again shortly.",
"worker_failed": "The expression could not be evaluated. Try again."
},
"precedenceHint": "The first available file source supplies metadata. Regex fields follow this order and missing values are filled from the other source.",
"summary": "{format} first, regex metadata preferred",
"fields": {
"title": "Title",
"authors": "Authors",
"series": "Series",
"seriesIndex": "Series number",
"publishedYear": "Published year",
"publishedDate": "Published date"
}
}
},
"fileWrite": {
+1
View File
@@ -87,6 +87,7 @@ export * from "./cron";
export * from "./tts";
export * from "./watch-download";
export * from "./podcast";
export * from "./regex-metadata";
export * from "./delivery";
export * from "./djvu";
export * from "./email";
+2
View File
@@ -1,3 +1,4 @@
import type { RegexMetadataConfig } from "./regex-metadata";
import type { LibraryLastScan } from "./scanner";
export type OrganizationMode = "book_per_file" | "book_per_folder";
@@ -59,6 +60,7 @@ export interface Library {
watchLocalFolders?: boolean;
autoScanCronExpression?: string | null;
metadataPrecedence: string[];
regexMetadata?: RegexMetadataConfig | null;
formatPriority: string[];
allowedFormats: string[];
organizationMode: OrganizationMode;
+48
View File
@@ -0,0 +1,48 @@
export const REGEX_METADATA_LIMITS = {
rules: 20,
patternLength: 2048,
pathLength: 4096,
} as const;
export const REGEX_METADATA_GROUPS = ["title", "author", "authors", "series", "seriesIndex", "publishedYear", "publishedDate"] as const;
export interface RegexMetadataRule {
pattern: string;
flags: string;
}
export interface RegexMetadataConfig {
rules: RegexMetadataRule[];
}
export interface RegexMetadataFields {
title?: string;
authors?: string[];
seriesName?: string;
seriesIndex?: string;
publishedYear?: number;
publishedDate?: string;
}
export type RegexMetadataDiagnosticCode =
"invalid_pattern" | "missing_group" | "conflicting_authors" | "invalid_value" | "timeout" | "busy" | "worker_failed";
export interface RegexMetadataDiagnostic {
code: RegexMetadataDiagnosticCode;
ruleIndex?: number;
field?: string;
}
export interface RegexMetadataPreview {
inputPath: string;
matched: boolean;
ruleIndex: number | null;
groups: Record<string, string>;
metadata: RegexMetadataFields;
diagnostics: RegexMetadataDiagnostic[];
}
export interface RegexMetadataValidation {
valid: boolean;
diagnostics: RegexMetadataDiagnostic[];
}
@@ -0,0 +1 @@
ALTER TABLE "libraries" ADD COLUMN "regex_metadata" jsonb;
File diff suppressed because it is too large Load Diff
@@ -813,6 +813,13 @@
"when": 1792168359342,
"tag": "0115_add_audit_resource_label",
"breakpoints": true
},
{
"idx": 116,
"version": "7",
"when": 1792185442568,
"tag": "0116_regex_metadata",
"breakpoints": true
}
]
}
+2 -1
View File
@@ -15,7 +15,7 @@ import {
uniqueIndex,
varchar,
} from 'drizzle-orm/pg-core';
import { FieldPreferenceOverrides, BookMetadataFetchConfigOverride, AddedAtSource } from '@bookorbit/types';
import { FieldPreferenceOverrides, BookMetadataFetchConfigOverride, AddedAtSource, RegexMetadataConfig } from '@bookorbit/types';
export const libraries = pgTable(
'libraries',
@@ -38,6 +38,7 @@ export const libraries = pgTable(
.$type<string[]>()
.notNull()
.default(['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar']),
regexMetadata: jsonb('regex_metadata').$type<RegexMetadataConfig>(),
formatPriority: jsonb('format_priority')
.$type<string[]>()
.notNull()
@@ -453,12 +453,50 @@ describe('AudioMetadataEmbedder', () => {
embedder.embedMetadata('/books/audio/book.m4b', 'm4b', { coverBytes: Buffer.from('raw-cover'), metadata: [] }),
).rejects.toBeInstanceOf(FfmpegStalledError);
expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1);
expect(mocks.writeMp4FreeformTags).not.toHaveBeenCalled();
expect(replaceFileAtomicallyMock).not.toHaveBeenCalled();
expect(unlinkMock).toHaveBeenCalledWith('/books/audio/.bookorbit-write-fixed-id.m4b');
expect(unlinkMock).toHaveBeenCalledWith('/books/audio/.bookorbit-cover-fixed-id.jpg');
});
it('retries an unprobed MP4 with the mp4 muxer when the ipod muxer refuses it', async () => {
mockProbeOutput('not json');
mocks.runFfmpeg.mockRejectedValueOnce(new Error("Tag mp4a incompatible with output codec id '86017'")).mockResolvedValueOnce(undefined);
const embedder = new AudioMetadataEmbedder();
await embedder.embedMetadata('/books/audio/merged.m4b', 'm4b', { coverBytes: Buffer.from('raw-cover'), metadata: [] });
expect(mocks.runFfmpeg).toHaveBeenCalledTimes(2);
expect(mocks.runFfmpeg.mock.calls[0]![1]).not.toContain('-f');
expect(mocks.runFfmpeg.mock.calls[1]![1]).toEqual(expect.arrayContaining(['-f', 'mp4', '/books/audio/.bookorbit-write-fixed-id.m4b']));
expect(replaceFileAtomicallyMock).toHaveBeenCalledWith('/books/audio/.bookorbit-write-fixed-id.m4b', '/books/audio/merged.m4b');
});
it('reports the first ffmpeg failure when the mp4 retry fails as well', async () => {
mockProbeOutput('not json');
mocks.runFfmpeg.mockRejectedValueOnce(new Error('first failure')).mockRejectedValueOnce(new Error('retry failure'));
const embedder = new AudioMetadataEmbedder();
await expect(embedder.embedMetadata('/books/audio/merged.m4a', 'm4a', { coverBytes: null, metadata: [] })).rejects.toThrow('first failure');
expect(mocks.runFfmpeg).toHaveBeenCalledTimes(2);
expect(replaceFileAtomicallyMock).not.toHaveBeenCalled();
});
it.each([
['a probed MP4', 'm4b', true],
['an MP3', 'mp3', false],
])('does not retry a failed write of %s', async (_label, format, probed) => {
if (probed) mockProbedStreams([{ codec_type: 'audio', codec_name: 'aac' }]);
mocks.runFfmpeg.mockRejectedValue(new Error('ffmpeg failed'));
const embedder = new AudioMetadataEmbedder();
await expect(embedder.embedMetadata(`/books/audio/book.${format}`, format, { coverBytes: null, metadata: [] })).rejects.toThrow('ffmpeg failed');
expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1);
});
it('preserves the ffmpeg error when cleanup also fails', async () => {
mocks.runFfmpeg.mockRejectedValue(new Error('ffmpeg failed'));
unlinkMock.mockRejectedValue(new Error('cleanup failed'));
@@ -8,13 +8,14 @@ import { promisify } from 'util';
import { FORMAT_M4A, FORMAT_M4B, FORMAT_MP3 } from '../../file-write.constants';
import { replaceFileAtomically } from '../shared/atomic-file-replace';
import { runFfmpeg } from './ffmpeg-runner';
import { FfmpegStalledError, runFfmpeg } from './ffmpeg-runner';
import { MP4_NATIVE_TAG_KEYS, writeMp4FreeformTags, type Mp4FreeformTag } from './mp4-freeform-tags';
const execFile = promisify(execFileCallback);
const FFPROBE_TIMEOUT_MS = 30_000;
/** The only audio FFmpeg's ipod muxer accepts; an MP4 can also carry MP3, MP2, Opus, FLAC and E-AC-3. */
const IPOD_MUXER_AUDIO_CODECS = new Set(['aac', 'alac', 'ac3']);
const MP4_MUXER = 'mp4';
interface ProbedStream {
codecType: string | null;
@@ -53,7 +54,18 @@ export class AudioMetadataEmbedder {
const streams = isMp4Format(format) ? await probeStreams(filePath) : null;
const useMp4MetadataTags = shouldUseMp4MetadataTags(format, coverPath, streams);
const muxer = outputMuxer(format, streams);
await runFfmpeg(resolveFfmpegPath(), buildFfmpegArgs(filePath, coverPath, tempPath, format, input.metadata, { useMp4MetadataTags, muxer }));
const ffmpegArgs = (outputFormat: string | undefined) =>
buildFfmpegArgs(filePath, coverPath, tempPath, format, input.metadata, { useMp4MetadataTags, muxer: outputFormat });
try {
await runFfmpeg(resolveFfmpegPath(), ffmpegArgs(muxer));
} catch (error) {
// Unprobed, the file may hold audio the ipod muxer refuses, such as MP3; the mp4 muxer takes any of
// it. A stalled write is not a muxer refusal, and a second attempt would only double the wait.
if (!isMp4Format(format) || streams || error instanceof FfmpegStalledError) throw error;
await runFfmpeg(resolveFfmpegPath(), ffmpegArgs(MP4_MUXER)).catch(() => {
throw error;
});
}
if (isMp4Format(format) && !useMp4MetadataTags) {
await writeMp4FreeformTags(tempPath, mp4FreeformTags(input.metadata), { carryOverFrom: filePath });
}
@@ -152,7 +164,7 @@ function shouldUseMp4MetadataTags(format: string, coverPath: string | null, stre
function outputMuxer(format: string, streams: ProbedStream[] | null): string | undefined {
if (!isMp4Format(format) || !streams) return undefined;
const ipodRejectsAudio = streams.some((stream) => stream.codecType === 'audio' && !IPOD_MUXER_AUDIO_CODECS.has(stream.codecName ?? ''));
return ipodRejectsAudio ? 'mp4' : undefined;
return ipodRejectsAudio ? MP4_MUXER : undefined;
}
async function probeStreams(filePath: string): Promise<ProbedStream[] | null> {
@@ -405,6 +405,29 @@ describe('audio metadata round-trip', () => {
expect(await probeFormatTags(filePath)).toEqual(expect.objectContaining({ title: 'Tag Only', subtitle: 'Tag Only Subtitle' }));
});
it('takes new tags and a replacement cover even when ffprobe cannot read it', async () => {
const filePath = join(tempDir, `merged.${format}`);
const coverPath = join(tempDir, 'cover.jpg');
await generateCoverFixture(coverPath);
await generateChapteredAudioFixture(filePath, coverPath, 'libmp3lame');
vi.stubEnv('FFPROBE_PATH', join(tempDir, 'missing-ffprobe'));
const writer = new Writer(new AudioMetadataEmbedder());
await expect(
writer.write(
filePath,
{ title: 'Unprobed Title', coverBytes: await readFile(coverPath) },
{ dryRun: false, fieldMask: createBookWriteFieldMask() },
),
).resolves.toEqual(expect.objectContaining({ status: 'success' }));
const streams = await probeStreams(filePath);
expect(streams.find((stream) => stream.codec_type === 'audio')?.codec_name).toBe('mp3');
expect(streams.find((stream) => stream.codec_type === 'video')?.disposition?.attached_pic).toBe(1);
expect((await probeFormatTags(filePath)).title).toBe('Unprobed Title');
expect(await probeChapterTitles(filePath)).toEqual(['One', 'Two']);
});
it('takes new tags when it carries no art at all', async () => {
const filePath = join(tempDir, `bare.${format}`);
await execFile(ffmpegPath, [
@@ -1,3 +1,6 @@
import { RegexMetadataConfigDto } from './regex-metadata.dto';
import { Type } from 'class-transformer';
import { IsObject, ValidateNested } from 'class-validator';
import {
ArrayMinSize,
IsDefined,
@@ -94,6 +97,12 @@ export class CreateLibraryDto {
@IsString({ each: true })
metadataPrecedence?: string[];
@IsOptional()
@IsObject()
@ValidateNested()
@Type(() => RegexMetadataConfigDto)
regexMetadata?: RegexMetadataConfigDto | null;
@IsOptionalNotNull()
@IsArray()
@IsString({ each: true })
@@ -0,0 +1,57 @@
import { Type } from 'class-transformer';
import {
ArrayMaxSize,
ArrayMinSize,
IsArray,
IsDefined,
IsInt,
IsObject,
IsOptional,
IsString,
Matches,
MaxLength,
Min,
MinLength,
ValidateNested,
} from 'class-validator';
import { REGEX_METADATA_LIMITS, type RegexMetadataConfig, type RegexMetadataRule } from '@bookorbit/types';
export class RegexMetadataRuleDto implements RegexMetadataRule {
@IsString()
@MinLength(1)
@MaxLength(REGEX_METADATA_LIMITS.patternLength)
pattern: string;
@IsString()
@Matches(/^(?:i?u?|ui)$/)
flags: string;
}
export class RegexMetadataConfigDto implements RegexMetadataConfig {
@IsArray()
@ArrayMinSize(1)
@ArrayMaxSize(REGEX_METADATA_LIMITS.rules)
@ValidateNested({ each: true })
@Type(() => RegexMetadataRuleDto)
rules: RegexMetadataRuleDto[];
}
export class RegexMetadataValidationDto {
@IsDefined()
@IsObject()
@ValidateNested()
@Type(() => RegexMetadataConfigDto)
config: RegexMetadataConfigDto;
@IsOptional()
@IsInt()
@Min(1)
libraryId?: number;
}
export class RegexMetadataPreviewDto extends RegexMetadataValidationDto {
@IsString()
@MinLength(1)
@MaxLength(REGEX_METADATA_LIMITS.pathLength)
relativePath: string;
}
@@ -1,3 +1,6 @@
import { RegexMetadataConfigDto } from './regex-metadata.dto';
import { Type } from 'class-transformer';
import { IsObject, ValidateNested } from 'class-validator';
import {
ArrayMinSize,
IsArray,
@@ -91,6 +94,12 @@ export class UpdateLibraryDto {
@IsString({ each: true })
metadataPrecedence?: string[];
@IsOptional()
@IsObject()
@ValidateNested()
@Type(() => RegexMetadataConfigDto)
regexMetadata?: RegexMetadataConfigDto | null;
@IsOptionalNotNull()
@IsArray()
@IsString({ each: true })
@@ -18,7 +18,7 @@ export const LIBRARY_MARK_AS_FINISHED_MIN = 90;
export const LIBRARY_MARK_AS_FINISHED_MAX = 100;
export const LIBRARY_BOOK_STATUS_PRESENT = 'present';
export const LIBRARY_METADATA_PRECEDENCE_DEFAULT = ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar'] as const;
export const LIBRARY_METADATA_PRECEDENCE_DEFAULT = ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar', 'regex'] as const;
export const LIBRARY_FILE_WRITE_MAX_SIZE_MB_MIN = 1;
export const LIBRARY_FILE_WRITE_MAX_SIZE_MB_MAX = 10000;
@@ -1,3 +1,4 @@
import { RegexMetadataPreviewDto, RegexMetadataValidationDto } from './dto/regex-metadata.dto';
import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Param, ParseIntPipe, Patch, Post, Query, Res } from '@nestjs/common';
import type { FastifyReply } from 'fastify';
@@ -48,6 +49,20 @@ export class LibraryController {
return this.libraryService.getOverview(user);
}
@Post('regex-metadata/validate')
@HttpCode(HttpStatus.OK)
@RequirePermission(Permission.ManageLibraries)
validateRegexMetadata(@Body() dto: RegexMetadataValidationDto) {
return this.libraryService.validateRegexMetadata(dto);
}
@Post('regex-metadata/preview')
@HttpCode(HttpStatus.OK)
@RequirePermission(Permission.ManageLibraries)
previewRegexMetadata(@Body() dto: RegexMetadataPreviewDto) {
return this.libraryService.previewRegexMetadata(dto);
}
@Get(':id')
@RequireLibraryAccess('viewer')
findOne(@Param('id', ParseIntPipe) id: number) {
@@ -336,3 +336,23 @@ describe('Library DTO validation', () => {
});
});
});
describe('regex metadata DTO validation', () => {
it('accepts a nullable config and validates nested rules with the global whitelist', async () => {
for (const regexMetadata of [null, { rules: [{ pattern: '(?<title>.+)', flags: 'iu' }] }]) {
const dto = plainToInstance(UpdateLibraryDto, { regexMetadata });
expect(await validate(dto, { whitelist: true, forbidNonWhitelisted: true })).toEqual([]);
}
});
it.each([
{ rules: [] },
{ rules: [{ pattern: '(?<title>.+)', flags: 'g' }] },
{ rules: [{ pattern: '(?<title>.+)', flags: '', template: '{title}' }] },
{ rules: [null] },
{ rules: 'invalid' },
[],
])('rejects invalid nested configuration %j', async (regexMetadata) => {
const dto = plainToInstance(UpdateLibraryDto, { regexMetadata });
expect((await validate(dto, { whitelist: true, forbidNonWhitelisted: true })).length).toBeGreaterThan(0);
});
});
@@ -1,3 +1,4 @@
import { MetadataModule } from '../metadata/metadata.module';
import { ReadingAttemptEventsModule } from '../user-book-status/reading-attempt-events.module';
import { Module, forwardRef } from '@nestjs/common';
@@ -19,6 +20,7 @@ import { LibraryAddedAtService } from './library-added-at.service';
imports: [
ReadingAttemptEventsModule,
ScannerModule,
MetadataModule,
AchievementModule,
forwardRef(() => BookModule),
FileWriteModule,
@@ -1,3 +1,6 @@
import 'reflect-metadata';
import { plainToInstance } from 'class-transformer';
import { UpdateLibraryDto } from './dto/update-library.dto';
import { ReadingAttemptEventsService } from '../user-book-status/reading-attempt-events.service';
vi.mock('fs/promises', () => ({
readdir: vi.fn(),
@@ -96,6 +99,7 @@ describe('LibraryService', () => {
removeSchedule: vi.fn(),
};
const regexMetadata = { validate: vi.fn() };
let service: LibraryService;
let readingEvents: ReadingAttemptEventsService;
@@ -112,6 +116,7 @@ describe('LibraryService', () => {
achievementEvents as any,
pathPolicy as any,
scanScheduler as any,
regexMetadata as any,
readingEvents,
);
@@ -202,6 +207,53 @@ describe('LibraryService', () => {
await expect(service.findOne(111)).rejects.toBeInstanceOf(NotFoundException);
});
it('rejects regex validation for podcast libraries', async () => {
libraryRepo.findById.mockResolvedValue([{ id: 2, type: 'podcasts' }]);
libraryRepo.findFoldersByLibrary.mockResolvedValue([]);
await expect(service.validateRegexMetadata({ config: { rules: [{ pattern: '(?<title>.+)', flags: '' }] }, libraryId: 2 })).rejects.toThrow(
'Regex metadata is only available for book libraries',
);
});
describe('regex configuration updates', () => {
const first = { pattern: '(?<title>.+)', flags: 'i' };
const second = { pattern: '(?<series>.+)', flags: '' };
const config = { rules: [first, second] };
beforeEach(() => {
libraryRepo.findById.mockResolvedValue([{ id: 2, type: 'books', icon: 'BookOpen', regexMetadata: config }]);
libraryRepo.update.mockResolvedValue([{ id: 2 }]);
libraryRepo.findFoldersByLibrary.mockResolvedValue([]);
});
it.each([null, { rules: [{ flags: 'i', pattern: first.pattern }, second] }])(
'does not revalidate or rewrite unchanged rules: %j',
async (unchanged) => {
libraryRepo.findById.mockResolvedValue([{ id: 2, type: 'books', icon: 'BookOpen', regexMetadata: unchanged === null ? null : config }]);
await service.update(2, plainToInstance(UpdateLibraryDto, { coverAspectRatio: '1/1', regexMetadata: unchanged }));
expect(regexMetadata.validate).not.toHaveBeenCalled();
expect(libraryRepo.update).toHaveBeenCalledWith(2, { coverAspectRatio: '1/1' });
},
);
it.each([
null,
{ rules: [second, first] },
{ rules: [{ ...first, flags: 'u' }, second] },
{ rules: [{ ...first, pattern: '(?<title>.*)' }, second] },
])('validates and persists an actual change without a library grant: %j', async (changed) => {
await service.update(2, { regexMetadata: changed });
expect(libraryRepo.hasUserAccess).not.toHaveBeenCalled();
expect(regexMetadata.validate).toHaveBeenCalledWith(changed);
expect(libraryRepo.update).toHaveBeenCalledWith(2, { regexMetadata: changed });
});
it('does not persist a change that fails validation', async () => {
regexMetadata.validate.mockRejectedValueOnce(new BadRequestException('Invalid regex metadata configuration'));
await expect(service.update(2, { regexMetadata: null })).rejects.toThrow('Invalid regex metadata configuration');
expect(libraryRepo.update).not.toHaveBeenCalled();
});
});
it('verifyUserAccess bypasses lookup for superusers', async () => {
await service.verifyUserAccess(1, 2, true);
expect(libraryRepo.hasUserAccess).not.toHaveBeenCalled();
@@ -219,6 +271,20 @@ describe('LibraryService', () => {
await expect(service.verifyUserAccessLevel(1, 2, true, 'owner')).resolves.toBeUndefined();
});
it.each([undefined, ['regex', 'embedded']])('preserves regex configuration with precedence %j on create', async (metadataPrecedence) => {
const config = { rules: [{ pattern: '(?<title>.+)', flags: '' }] };
libraryRepo.findByName.mockResolvedValue([]);
libraryRepo.insert.mockResolvedValue([{ id: 5, type: 'books', name: 'Regex', icon: 'BookOpen' }]);
libraryRepo.insertFolders.mockResolvedValue([]);
await service.create({ name: 'Regex', icon: 'BookOpen', folders: ['/a'], regexMetadata: config, metadataPrecedence });
expect(libraryRepo.insert).toHaveBeenCalledWith(
expect.objectContaining({
regexMetadata: config,
metadataPrecedence: metadataPrecedence ?? ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar', 'regex'],
}),
);
});
it('create applies defaults, inserts folders, and starts an async scan', async () => {
libraryRepo.findByName.mockResolvedValue([]);
libraryRepo.insert.mockResolvedValue([{ id: 5, type: 'books', name: 'Sci-Fi', icon: 'BookOpen' }]);
@@ -235,7 +301,7 @@ describe('LibraryService', () => {
icon: 'BookOpen',
displayOrder: 0,
watch: false,
metadataPrecedence: ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar'],
metadataPrecedence: ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar', 'regex'],
formatPriority: [
'epub',
'kepub',
+37 -1
View File
@@ -1,3 +1,6 @@
import { isDeepStrictEqual } from 'node:util';
import { RegexMetadataService } from '../metadata/regex/regex-metadata.service';
import { RegexMetadataPreviewDto, RegexMetadataValidationDto } from './dto/regex-metadata.dto';
import { ReadingAttemptEventsService } from '../user-book-status/reading-attempt-events.service';
import {
BadRequestException,
@@ -13,7 +16,14 @@ import { readdir, realpath, rm, stat } from 'fs/promises';
import { dirname, isAbsolute, join, relative } from 'path';
import { APP_FEATURES, DEFAULT_FORMAT_PRIORITY } from '@bookorbit/types';
import type { AccessLevel, LibraryFileSyncProgressEvent, LibraryOverviewEntry, OrganizationMode, WriteResult } from '@bookorbit/types';
import type {
AccessLevel,
LibraryFileSyncProgressEvent,
LibraryOverviewEntry,
OrganizationMode,
RegexMetadataConfig,
WriteResult,
} from '@bookorbit/types';
import { podcastArtworkDirPath } from '../../common/podcast-artwork-storage';
import { podcastFeedSnapshotPath } from '../../common/podcast-feed-snapshot-storage';
import { sanitizeLogValue } from '../../common/utils/log-sanitize.utils';
@@ -59,6 +69,7 @@ const BOOK_ONLY_LIBRARY_FIELDS = [
'watch',
'autoScanCronExpression',
'metadataPrecedence',
'regexMetadata',
'formatPriority',
'allowedFormats',
'organizationMode',
@@ -102,6 +113,7 @@ export class LibraryService {
private readonly achievementEvents: AchievementEventsService,
private readonly pathPolicy: PathPolicyService,
private readonly scanScheduler: LibraryScanSchedulerService,
private readonly regexMetadata: RegexMetadataService,
private readonly readingAttemptEvents: ReadingAttemptEventsService,
) {
this.appDataPath = this.config.get<string>('storage.appDataPath')!;
@@ -173,6 +185,22 @@ export class LibraryService {
return { ...normalizeLibraryOrganizationMode(library), folders };
}
private async assertRegexMetadataLibrary(libraryId: number | undefined) {
if (libraryId === undefined) return;
const library = await this.findOne(libraryId);
if (library.type !== 'books') throw new BadRequestException('Regex metadata is only available for book libraries');
}
async validateRegexMetadata(dto: RegexMetadataValidationDto) {
await this.assertRegexMetadataLibrary(dto.libraryId);
return this.regexMetadata.check(dto.config);
}
async previewRegexMetadata(dto: RegexMetadataPreviewDto) {
await this.assertRegexMetadataLibrary(dto.libraryId);
return this.regexMetadata.preview(dto.config, dto.relativePath);
}
async create(dto: CreateLibraryDto) {
const libraryType = dto.type ?? 'books';
if (libraryType === 'podcasts' && !APP_FEATURES.podcasts) {
@@ -180,6 +208,7 @@ export class LibraryService {
}
if (libraryType === 'podcasts') this.assertNoBookOnlyFields(dto);
else this.assertNoPodcastOnlyFields(dto);
await this.regexMetadata.validate(dto.regexMetadata);
await this.assertNameAvailable(dto.name);
const folderInputs = resolveLibraryFolderRoles(
libraryType,
@@ -199,6 +228,7 @@ export class LibraryService {
watch: libraryType === 'books' ? (dto.watch ?? false) : false,
watchLocalFolders: libraryType === 'podcasts' ? (dto.watchLocalFolders ?? true) : false,
autoScanCronExpression: libraryType === 'books' ? (dto.autoScanCronExpression ?? null) : null,
regexMetadata: dto.regexMetadata ?? null,
metadataPrecedence: libraryType === 'books' ? (dto.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT]) : [],
formatPriority: libraryType === 'books' ? (dto.formatPriority ?? [...DEFAULT_FORMAT_PRIORITY]) : [],
allowedFormats: libraryType === 'books' ? (dto.allowedFormats ?? []) : [],
@@ -252,6 +282,11 @@ export class LibraryService {
if (existing.type === 'podcasts') this.assertNoBookOnlyFields(dto);
else this.assertNoPodcastOnlyFields(dto);
const regexRules = (config: RegexMetadataConfig | null | undefined) => config?.rules.map(({ pattern, flags }) => [pattern, flags]) ?? null;
const regexMetadataChanged =
dto.regexMetadata !== undefined && !isDeepStrictEqual(regexRules(dto.regexMetadata), regexRules(existing.regexMetadata));
if (regexMetadataChanged) await this.regexMetadata.validate(dto.regexMetadata);
const existingOrganizationMode = normalizeOrganizationMode(existing.organizationMode);
if (dto.organizationMode !== undefined && dto.organizationMode !== existingOrganizationMode) {
throw new BadRequestException(
@@ -264,6 +299,7 @@ export class LibraryService {
}
const { folders: rawFolderPaths, localFolders: rawLocalFolderPaths, ...fields } = dto;
if (!regexMetadataChanged) delete fields.regexMetadata;
let existingFolders: Awaited<ReturnType<LibraryRepository['findFoldersByLibrary']>> | undefined;
let folderInputs: LibraryFolderInput[] | undefined;
if (rawFolderPaths !== undefined || rawLocalFolderPaths !== undefined) {
@@ -1,3 +1,4 @@
import { RegexMetadataService } from './regex/regex-metadata.service';
import { Module } from '@nestjs/common';
import { DjvuDocumentModule } from '../djvu/djvu-document.module';
@@ -14,7 +15,7 @@ import { MetadataService } from './metadata.service';
@Module({
imports: [BookMetadataLockModule, BookCoverStoreModule, EmbeddingModule, MetadataScoreModule, NarratorModule, DjvuDocumentModule],
providers: [MetadataService, MetadataExtractionService, MetadataEventsService, ComicMetadataRepository, CoverSlotReconciler],
exports: [MetadataService, MetadataExtractionService, MetadataEventsService, ComicMetadataRepository, CoverSlotReconciler],
providers: [RegexMetadataService, MetadataService, MetadataExtractionService, MetadataEventsService, ComicMetadataRepository, CoverSlotReconciler],
exports: [RegexMetadataService, MetadataService, MetadataExtractionService, MetadataEventsService, ComicMetadataRepository, CoverSlotReconciler],
})
export class MetadataModule {}
@@ -270,6 +270,202 @@ describe('MetadataService', () => {
mockExtractEpubCover.mockResolvedValueOnce(cover);
}
describe('regex extraction persistence', () => {
describe.each(['epub', 'm4b'])('%s relations', (format) => {
afterEach(() => vi.restoreAllMocks());
it.each([
{ label: 'neither relation', authors: [], genres: [] },
{ label: 'only authors', authors: [{ name: 'File Author', sortName: null }], genres: [] },
{ label: 'only genres', authors: [], genres: ['Mystery'] },
])('preserves omitted relations when the file supplies $label', async ({ authors, genres }) => {
const { db, updateSet } = makeDb();
const service = makeService(db);
vi.spyOn(MetadataExtractionService.prototype, 'extract').mockResolvedValueOnce({ title: 'File title', authors, genres, cover: null });
const replaceAuthors = vi.spyOn(service, 'replaceAuthors').mockResolvedValue(undefined);
const replaceGenres = vi.spyOn(service, 'replaceGenres').mockResolvedValue(undefined);
await service.extractAndSaveWithRegex(
9,
[{ key: 'embedded', absolutePath: `/books/file.${format}`, format }],
{ seriesName: 'Path Series' },
['regex', 'embedded'],
);
expect(updateSet).toHaveBeenCalledWith(expect.objectContaining({ title: 'File title', seriesName: 'Path Series' }));
if (authors.length) expect(replaceAuthors).toHaveBeenCalledWith(9, authors);
else expect(replaceAuthors).not.toHaveBeenCalled();
if (genres.length) expect(replaceGenres).toHaveBeenCalledWith(9, genres);
else expect(replaceGenres).not.toHaveBeenCalled();
});
it.each([
{ fileAuthors: [], precedence: ['embedded', 'regex'], expected: 'Regex Author' },
{ fileAuthors: [{ name: 'File Author', sortName: null }], precedence: ['regex', 'embedded'], expected: 'Regex Author' },
{ fileAuthors: [{ name: 'File Author', sortName: null }], precedence: ['embedded', 'regex'], expected: 'File Author' },
])('merges authors using $precedence with $fileAuthors', async ({ fileAuthors, precedence, expected }) => {
const { db } = makeDb();
const service = makeService(db);
vi.spyOn(MetadataExtractionService.prototype, 'extract').mockResolvedValueOnce({ authors: fileAuthors, genres: [], cover: null });
const replaceAuthors = vi.spyOn(service, 'replaceAuthors').mockResolvedValue(undefined);
const replaceGenres = vi.spyOn(service, 'replaceGenres').mockResolvedValue(undefined);
await service.extractAndSaveWithRegex(
9,
[{ key: 'embedded', absolutePath: `/books/file.${format}`, format }],
{ authors: ['Regex Author'] },
precedence,
);
expect(replaceAuthors).toHaveBeenCalledWith(9, [{ name: expected, sortName: null }]);
expect(replaceGenres).not.toHaveBeenCalled();
});
it('respects relation locks after merging regex and file metadata', async () => {
const { db } = makeDb();
const filter = vi.fn().mockImplementation((_bookId, dto) =>
Promise.resolve({
dto: { ...dto, authors: undefined, genres: undefined },
skippedFields: ['authors', 'genres'],
}),
);
const service = makeService(db, undefined, { bookMetadataLockService: { isFieldLocked: vi.fn(), filterAutomatedBookUpdate: filter } });
vi.spyOn(MetadataExtractionService.prototype, 'extract').mockResolvedValueOnce({ authors: [], genres: ['Mystery'], cover: null });
const replaceAuthors = vi.spyOn(service, 'replaceAuthors');
const replaceGenres = vi.spyOn(service, 'replaceGenres');
await service.extractAndSaveWithRegex(
9,
[{ key: 'embedded', absolutePath: `/books/file.${format}`, format }],
{ authors: ['Regex Author'] },
['regex', 'embedded'],
);
expect(filter).toHaveBeenCalledWith(9, expect.objectContaining({ authors: ['Regex Author'], genres: ['Mystery'] }));
expect(replaceAuthors).not.toHaveBeenCalled();
expect(replaceGenres).not.toHaveBeenCalled();
});
it('retains empty relation replacement for file-only extraction', async () => {
const { db } = makeDb();
const service = makeService(db);
vi.spyOn(MetadataExtractionService.prototype, 'extract').mockResolvedValueOnce({ authors: [], genres: [], cover: null });
const replaceAuthors = vi.spyOn(service, 'replaceAuthors');
const replaceGenres = vi.spyOn(service, 'replaceGenres');
await service.extractAndSave(9, `/books/file.${format}`, format);
expect(replaceAuthors).toHaveBeenCalledWith(9, []);
expect(replaceGenres).toHaveBeenCalledWith(9, []);
expect(db.delete).toHaveBeenCalledWith(bookAuthors);
expect(db.delete).toHaveBeenCalledWith(bookGenres);
});
});
describe('file extraction failures', () => {
const sources = [
{ key: 'embedded', absolutePath: '/books/file.epub', format: 'epub' },
{ key: 'opfFile', absolutePath: '/books/metadata.opf', format: 'opf' },
];
afterEach(() => vi.restoreAllMocks());
it('continues to the next file source and respects its precedence over regex', async () => {
const { db, updateSet } = makeDb();
const service = makeService(db);
const extraction = vi
.spyOn(MetadataExtractionService.prototype, 'extract')
.mockRejectedValueOnce(new Error('unreadable file'))
.mockResolvedValueOnce({ title: 'OPF title', authors: [{ name: 'OPF Author', sortName: null }], genres: [], cover: null });
const replaceAuthors = vi.spyOn(service, 'replaceAuthors').mockResolvedValue(undefined);
await service.extractAndSaveWithRegex(9, sources, { title: 'Regex title', seriesName: 'Path Series' }, ['embedded', 'opfFile', 'regex']);
expect(extraction).toHaveBeenNthCalledWith(1, '/books/file.epub', 'epub');
expect(extraction).toHaveBeenNthCalledWith(2, '/books/metadata.opf', 'opf');
expect(updateSet).toHaveBeenCalledWith(expect.objectContaining({ title: 'OPF title', seriesName: 'Path Series' }));
expect(replaceAuthors).toHaveBeenCalledWith(9, [{ name: 'OPF Author', sortName: null }]);
});
it('saves regex metadata and preserves omitted relations when every file source rejects', async () => {
const { db, updateSet } = makeDb();
const scoreService = { calculateAndSave: vi.fn().mockResolvedValue(undefined) };
const service = makeService(db, undefined, { scoreService });
const extraction = vi.spyOn(MetadataExtractionService.prototype, 'extract').mockRejectedValue(new Error('unreadable file'));
const replaceAuthors = vi.spyOn(service, 'replaceAuthors');
const replaceGenres = vi.spyOn(service, 'replaceGenres');
await service.extractAndSaveWithRegex(9, sources, { title: 'Regex title', seriesName: 'Path Series' }, ['embedded', 'regex', 'opfFile']);
expect(extraction).toHaveBeenCalledTimes(2);
expect(updateSet).toHaveBeenCalledWith(expect.objectContaining({ title: 'Regex title', seriesName: 'Path Series' }));
expect(replaceAuthors).not.toHaveBeenCalled();
expect(replaceGenres).not.toHaveBeenCalled();
expect(scoreService.calculateAndSave).toHaveBeenCalledWith(9);
});
it('propagates persistence failures without trying another file source or retrying the write', async () => {
const { db, updateSet, updateWhere } = makeDb();
updateWhere.mockRejectedValueOnce(new Error('database unavailable'));
const scoreService = { calculateAndSave: vi.fn() };
const service = makeService(db, undefined, { scoreService });
const extraction = vi
.spyOn(MetadataExtractionService.prototype, 'extract')
.mockResolvedValue({ title: 'File title', authors: [], genres: [], cover: null });
await expect(service.extractAndSaveWithRegex(9, sources, { seriesName: 'Path Series' }, ['regex', 'embedded', 'opfFile'])).rejects.toThrow(
'database unavailable',
);
expect(extraction).toHaveBeenCalledTimes(1);
expect(updateSet).toHaveBeenCalledTimes(1);
expect(scoreService.calculateAndSave).not.toHaveBeenCalled();
});
});
it('reports scoring failure after saving regex fields without rolling them back or retrying', async () => {
const { db, updateSet } = makeDb();
const scoreService = { calculateAndSave: vi.fn().mockRejectedValue(new Error('score storage unavailable')) };
const service = makeService(db, undefined, { scoreService });
await expect(service.extractAndSaveWithRegex(9, [], { title: 'Regex title' }, ['regex', 'embedded'])).rejects.toThrow(
'score storage unavailable',
);
expect(updateSet).toHaveBeenCalledWith(expect.objectContaining({ title: 'Regex title' }));
expect(scoreService.calculateAndSave).toHaveBeenCalledTimes(1);
});
it('preserves existing relations when regex is the only source and supplies no authors', async () => {
const { db, updateSet } = makeDb();
const service = makeService(db);
const replaceAuthors = vi.spyOn(service, 'replaceAuthors');
const replaceGenres = vi.spyOn(service, 'replaceGenres');
await service.extractAndSaveWithRegex(9, [], { seriesName: 'Path Series' }, ['regex', 'embedded']);
expect(updateSet).toHaveBeenCalledWith(expect.objectContaining({ seriesName: 'Path Series' }));
expect(replaceAuthors).not.toHaveBeenCalled();
expect(replaceGenres).not.toHaveBeenCalled();
});
it('merges partial regex fields with file authors before enforcing field locks', async () => {
const { db, updateSet } = makeDb();
const filter = vi.fn().mockImplementation((_bookId, dto) => Promise.resolve({ dto: { ...dto, title: undefined }, skippedFields: ['title'] }));
const service = makeService(db, undefined, { bookMetadataLockService: { isFieldLocked: vi.fn(), filterAutomatedBookUpdate: filter } });
const extraction = vi.spyOn(MetadataExtractionService.prototype, 'extract').mockResolvedValueOnce({
title: 'Embedded title',
authors: [{ name: 'Embedded Author', sortName: null }],
genres: [],
cover: null,
});
const replaceAuthors = vi.spyOn(service, 'replaceAuthors').mockResolvedValue(undefined);
vi.spyOn(service, 'replaceGenres').mockResolvedValue(undefined);
await service.extractAndSaveWithRegex(
9,
[{ key: 'embedded', absolutePath: '/books/file.epub', format: 'epub' }],
{ title: 'Regex title', seriesName: 'Path Series' },
['regex', 'embedded'],
);
expect(filter).toHaveBeenCalledWith(
9,
expect.objectContaining({ title: 'Regex title', authors: ['Embedded Author'], seriesName: 'Path Series' }),
);
expect(updateSet).toHaveBeenCalledWith(expect.objectContaining({ seriesName: 'Path Series' }));
expect(updateSet.mock.calls[0][0]).not.toHaveProperty('title');
expect(replaceAuthors).toHaveBeenCalledWith(9, [{ name: 'Embedded Author', sortName: null }]);
extraction.mockRestore();
});
});
describe('downloadAndSaveCover', () => {
async function image(width: number, height: number): Promise<Buffer> {
return sharp({ create: { width, height, channels: 3, background: '#336699' } })
@@ -1,3 +1,5 @@
import type { RegexMetadataFields } from '@bookorbit/types';
import { mergeRegexMetadata } from './regex/merge-regex-metadata';
import { Inject, Injectable, Logger, Optional } from '@nestjs/common';
import { and, eq, inArray, isNull, lte, or, sql } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
@@ -138,6 +140,53 @@ export class MetadataService {
}
}
async extractAndSaveWithRegex(
bookId: number,
sources: { key: string; absolutePath: string; format: string }[],
regex: RegexMetadataFields,
precedence: string[],
): Promise<void> {
const startedAt = Date.now();
this.logger.debug(`[metadata.extract_with_regex] [start] bookId=${bookId} sourceCount=${sources.length} - metadata extraction started`);
try {
let base: ParsedBookData | null = null;
let selected: (typeof sources)[number] | undefined;
for (const source of sources) {
const sourceStartedAt = Date.now();
try {
base = await this.extractionService.extract(source.absolutePath, source.format);
} catch (error) {
this.logger.warn(
`[metadata.extract_source] [fail] bookId=${bookId} source="${sanitizeLogValue(source.key)}" format="${sanitizeLogValue(source.format)}" durationMs=${Date.now() - sourceStartedAt} errorClass=${error instanceof Error ? error.name : 'Error'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - file metadata extraction failed; continuing with remaining sources and regex`,
);
continue;
}
if (base) {
selected = source;
break;
}
}
const regexIndex = precedence.indexOf('regex');
const baseIndex = selected ? precedence.indexOf(selected.key) : -1;
const preferRegex = !selected || (regexIndex !== -1 && (baseIndex === -1 || regexIndex < baseIndex));
const data = mergeRegexMetadata(base, regex, preferRegex);
await this.persistMetadata(bookId, data, selected?.format ?? 'regex', true);
if (selected) {
if (base?.cover) await this.persistSourceCover(bookId, selected.format, base.cover);
await this.persistFixedLayout(bookId, selected.absolutePath, base?.isFixedLayout);
}
await this.scoreService.calculateAndSave(bookId);
this.logger.debug(
`[metadata.extract_with_regex] [end] bookId=${bookId} durationMs=${Date.now() - startedAt} fileSourceFound=${selected !== undefined} - metadata extraction completed`,
);
} catch (error) {
this.logger.warn(
`[metadata.extract_with_regex] [fail] bookId=${bookId} durationMs=${Date.now() - startedAt} errorClass=${error instanceof Error ? error.name : 'Error'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - metadata extraction failed`,
);
throw error;
}
}
// Called when ebook is the winner but audio files are also present.
// Saves audio-specific fields that no ebook format can provide, plus audio provider IDs, and the
// track's art into the audio slot. The leading source owns the ebook slot, and a custom audio
@@ -765,11 +814,11 @@ export class MetadataService {
// ── Persistence ──────────────────────────────────────────────────────────────
private async persistMetadata(bookId: number, data: ParsedBookData, format: string): Promise<void> {
private async persistMetadata(bookId: number, data: ParsedBookData, format: string, preserveAbsentRelations = false): Promise<void> {
if (isAudioFormat(format)) {
await this.persistAudioMetadata(bookId, data);
await this.persistAudioMetadata(bookId, data, preserveAbsentRelations);
} else {
await this.persistBookMetadata(bookId, data, format);
await this.persistBookMetadata(bookId, data, format, preserveAbsentRelations);
}
this.embedder?.embedBook(bookId).catch((error: Error) => {
this.logger.warn(
@@ -778,7 +827,7 @@ export class MetadataService {
});
}
private async persistAudioMetadata(bookId: number, data: ParsedBookData): Promise<void> {
private async persistAudioMetadata(bookId: number, data: ParsedBookData, preserveAbsentRelations = false): Promise<void> {
const { dto: filtered } = await this.bookMetadataLockService.filterAutomatedBookUpdate(bookId, {
title: data.title,
subtitle: data.subtitle,
@@ -789,8 +838,8 @@ export class MetadataService {
language: data.language,
seriesName: normalizeMetadataText(data.seriesName),
seriesIndex: data.seriesIndex,
authors: data.authors.map((author) => author.name),
genres: data.genres,
authors: preserveAbsentRelations && !data.authors.length ? undefined : data.authors.map((author) => author.name),
genres: preserveAbsentRelations && !data.genres.length ? undefined : data.genres,
audibleId: boundProviderId('audibleId', data.audibleId),
librofmId: boundProviderId('librofmId', data.librofmId),
audioMetadata: {
@@ -848,7 +897,7 @@ export class MetadataService {
this.logger.debug(`[metadata.persist_audio] [end] bookId=${bookId} title="${sanitizeLogValue(data.title ?? '')}" - audio metadata persisted`);
}
private async persistBookMetadata(bookId: number, data: ParsedBookData, format: string): Promise<void> {
private async persistBookMetadata(bookId: number, data: ParsedBookData, format: string, preserveAbsentRelations = false): Promise<void> {
const { dto: filtered } = await this.bookMetadataLockService.filterAutomatedBookUpdate(bookId, {
title: data.title,
subtitle: data.subtitle,
@@ -861,8 +910,8 @@ export class MetadataService {
language: data.language,
seriesName: normalizeMetadataText(data.seriesName),
seriesIndex: data.seriesIndex,
authors: data.authors.map((author) => author.name),
genres: data.genres,
authors: preserveAbsentRelations && !data.authors.length ? undefined : data.authors.map((author) => author.name),
genres: preserveAbsentRelations && !data.genres.length ? undefined : data.genres,
tags: data.tags,
rating: normalizeImportedRating(data.rating),
pageCount: data.pageCount,
@@ -0,0 +1,18 @@
import type { RegexMetadataFields } from '@bookorbit/types';
import type { ParsedBookData } from '../extractors/format-extractor.interface';
export function mergeRegexMetadata(base: ParsedBookData | null, regex: RegexMetadataFields, preferRegex: boolean): ParsedBookData {
const result: ParsedBookData = base ? { ...base } : { authors: [], genres: [], cover: null };
for (const key of ['title', 'seriesName', 'seriesIndex'] as const) {
if (regex[key] !== undefined && (preferRegex || !result[key]?.trim())) result[key] = regex[key];
}
if (regex.authors?.length && (preferRegex || !result.authors.length)) {
result.authors = regex.authors.map((name) => ({ name, sortName: null }));
}
// Date and year describe the same value; a higher-priority year must not retain a conflicting date.
if (regex.publishedYear !== undefined && (preferRegex || (!result.publishedYear && !result.publishedDate))) {
result.publishedYear = regex.publishedYear;
result.publishedDate = regex.publishedDate;
}
return result;
}
@@ -0,0 +1,128 @@
import type { RegexMetadataConfig, RegexMetadataDiagnostic, RegexMetadataFields, RegexMetadataPreview } from '@bookorbit/types';
import { isPublishedDateKey, parsePublishedYear } from '../../../common/utils/published-date.utils';
export interface RegexMetadataTask {
config: RegexMetadataConfig;
inputPath?: string;
}
/**
* Shared-package values the worker needs. A worker thread resolves packages without the aliases the
* server process runs under, so it must not import values from @bookorbit/types itself; the
* service hands them over when it spawns the worker.
*/
export interface RegexMetadataWorkerData {
groups: readonly string[];
seriesIndexPattern: RegExp;
seriesIndexMaxLength: number;
}
interface CompiledRules {
key: string;
compiled: RegExp[];
diagnostics: RegexMetadataDiagnostic[];
}
// A scan sends the same rules for every book, so the worker keeps the last compiled set.
let lastCompiled: CompiledRules | undefined;
function compileRules(config: RegexMetadataConfig, shared: RegexMetadataWorkerData): CompiledRules {
const key = JSON.stringify(config.rules.map(({ pattern, flags }) => [pattern, flags]));
if (lastCompiled?.key === key) return lastCompiled;
const compiled: RegExp[] = [];
const diagnostics: RegexMetadataDiagnostic[] = [];
for (const [ruleIndex, rule] of config.rules.entries()) {
try {
const pattern = rule.pattern.normalize('NFC');
const regex = new RegExp(pattern, rule.flags);
// The empty first alternative exposes group names without executing the user's pattern.
const groups = Object.keys(new RegExp(`|(?:${pattern})`, rule.flags).exec('')?.groups ?? {});
if (!groups.some((group) => shared.groups.includes(group))) {
diagnostics.push({ code: 'missing_group', ruleIndex });
}
if (groups.includes('author') && groups.includes('authors')) {
diagnostics.push({ code: 'conflicting_authors', ruleIndex });
}
compiled.push(regex);
} catch {
diagnostics.push({ code: 'invalid_pattern', ruleIndex });
}
}
lastCompiled = { key, compiled, diagnostics };
return lastCompiled;
}
export function evaluateRegexMetadata({ config, inputPath }: RegexMetadataTask, shared: RegexMetadataWorkerData): RegexMetadataPreview {
const { compiled, diagnostics } = compileRules(config, shared);
const result: RegexMetadataPreview = {
inputPath: inputPath ?? '',
matched: false,
ruleIndex: null,
groups: {},
metadata: {},
diagnostics: [...diagnostics],
};
if (result.diagnostics.length || inputPath === undefined) return result;
for (const [ruleIndex, regex] of compiled.entries()) {
const match = regex.exec(inputPath);
if (!match?.groups) continue;
const groups = Object.fromEntries(Object.entries(match.groups).filter((entry): entry is [string, string] => entry[1] !== undefined));
const metadata = normalizeGroups(groups, ruleIndex, result.diagnostics, shared);
if (!Object.keys(metadata).length) continue;
return { ...result, matched: true, ruleIndex, groups, metadata };
}
return result;
}
function normalizeGroups(
groups: Record<string, string>,
ruleIndex: number,
diagnostics: RegexMetadataDiagnostic[],
shared: RegexMetadataWorkerData,
): RegexMetadataFields {
const metadata: RegexMetadataFields = {};
for (const [group, field, limit] of [
['title', 'title', 1000],
['series', 'seriesName', 200],
] as const) {
const value = groups[group]?.trim();
if (!value) continue;
if (value.length > limit) diagnostics.push({ code: 'invalid_value', ruleIndex, field: group });
else metadata[field] = value;
}
const seriesIndex = groups.seriesIndex?.trim();
if (seriesIndex) {
// The column only accepts plain decimals; a decimal comma is the one spelling worth repairing.
const candidate = seriesIndex.replace(/^(\d+),(\d+)$/, '$1.$2');
if (candidate.length > shared.seriesIndexMaxLength || !shared.seriesIndexPattern.test(candidate)) {
diagnostics.push({ code: 'invalid_value', ruleIndex, field: 'seriesIndex' });
} else metadata.seriesIndex = candidate;
}
const author = groups.author?.trim();
const authors = author
? [author]
: groups.authors
?.split(';')
.map((value) => value.trim())
.filter(Boolean);
if (authors?.length) {
if (authors.some((value) => value.length > 200)) diagnostics.push({ code: 'invalid_value', ruleIndex, field: 'authors' });
else metadata.authors = [...new Set(authors)];
}
const year = groups.publishedYear?.trim();
if (year) {
const parsed = /^\d{4}$/.test(year) ? parsePublishedYear(year) : undefined;
if (parsed === undefined) diagnostics.push({ code: 'invalid_value', ruleIndex, field: 'publishedYear' });
else metadata.publishedYear = parsed;
}
const date = groups.publishedDate?.trim();
if (date) {
if (!isPublishedDateKey(date)) diagnostics.push({ code: 'invalid_value', ruleIndex, field: 'publishedDate' });
else {
metadata.publishedDate = date;
metadata.publishedYear = Number(date.slice(0, 4));
}
}
return metadata;
}
@@ -0,0 +1,21 @@
import { BadRequestException } from '@nestjs/common';
import { posix, win32 } from 'node:path';
import { REGEX_METADATA_LIMITS } from '@bookorbit/types';
export function prepareRegexMetadataPath(relativePath: string): string {
// Filesystems hand back either Unicode form, while typed rules are composed.
const path = relativePath.normalize('NFC').replaceAll('\\', '/');
if (
!path ||
path.length > REGEX_METADATA_LIMITS.pathLength ||
path.includes('\0') ||
posix.isAbsolute(path) ||
win32.isAbsolute(path) ||
/^[a-z]:/i.test(path) ||
path.split('/').some((part) => part === '..' || part === '.' || part === '')
) {
throw new BadRequestException('Expected a file path relative to the library folder');
}
const extension = /\.kepub\.epub$/i.test(path) ? path.slice(-11) : posix.extname(path);
return extension ? path.slice(0, -extension.length) : path;
}
@@ -0,0 +1,210 @@
import { Test } from '@nestjs/testing';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { ServiceUnavailableException } from '@nestjs/common';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { RegexMetadataService } from './regex-metadata.service';
import { prepareRegexMetadataPath } from './regex-metadata-path';
import { mergeRegexMetadata } from './merge-regex-metadata';
import type { ParsedBookData } from '../extractors/format-extractor.interface';
describe('RegexMetadataService', () => {
let service: RegexMetadataService;
beforeEach(async () => {
const module = await Test.createTestingModule({ providers: [RegexMetadataService] }).compile();
service = module.get(RegexMetadataService);
});
afterEach(async () => service.onModuleDestroy());
it('validates rules without a path and returns indexed diagnostics', async () => {
expect(await service.check({ rules: [{ pattern: '(?<title>.+)', flags: '' }] })).toEqual({ valid: true, diagnostics: [] });
expect(
await service.check({
rules: [
{ pattern: '[', flags: '' },
{ pattern: '(?<helper>.+)', flags: '' },
{ pattern: '(?<author>.+)(?<authors>.+)', flags: '' },
],
}),
).toEqual({
valid: false,
diagnostics: [
{ code: 'invalid_pattern', ruleIndex: 0 },
{ code: 'missing_group', ruleIndex: 1 },
{ code: 'conflicting_authors', ruleIndex: 2 },
],
});
await service.onModuleDestroy();
expect(await service.check({ rules: [{ pattern: '(?<title>.+)', flags: '' }] })).toEqual({
valid: false,
diagnostics: [{ code: 'worker_failed' }],
});
});
it.each(['epub', 'PDF', 'm4b', 'KEPUB.EPUB'])('matches the relative filename without %s', async (extension) => {
const result = await service.preview(
{ rules: [{ pattern: '^(?<author>[^/]+)/(?<series>[^/]+)/(?<seriesIndex>\\d+) - (?<title>.+)$', flags: 'u' }] },
`Änne/Dr. Who/08 - My.Book.${extension}`,
);
expect(result.inputPath).toBe('Änne/Dr. Who/08 - My.Book');
expect(result.metadata).toEqual({ authors: ['Änne'], seriesName: 'Dr. Who', seriesIndex: '08', title: 'My.Book' });
});
it.each([
['decomposed path, composed rule', 'A\u0308nne/Book.epub', '^(?<author>Änne)/'],
['composed path, decomposed rule', 'Änne/Book.epub', '^(?<author>A\u0308nne)/'],
])('matches across Unicode normalization forms: %s', async (_label, path, pattern) => {
const result = await service.preview({ rules: [{ pattern, flags: 'u' }] }, path);
expect(result.metadata.authors).toEqual(['Änne']);
expect(result.inputPath).toBe('Änne/Book');
});
it('skips unusable matches and selects only the first usable rule', async () => {
const result = await service.preview(
{
rules: [
{ pattern: '^(?<publishedYear>\\d+)', flags: '' },
{ pattern: '^(?<title>.+)$', flags: '' },
{ pattern: '^(?<series>.+)$', flags: '' },
],
},
'999-title.epub',
);
expect(result.ruleIndex).toBe(1);
expect(result.metadata).toEqual({ title: '999-title' });
expect(result.diagnostics).toContainEqual({ code: 'invalid_value', ruleIndex: 0, field: 'publishedYear' });
});
it('normalizes authors and gives a valid date precedence over the captured year', async () => {
const result = await service.preview(
{ rules: [{ pattern: '^(?<authors>[^/]+)/(?<publishedYear>\\d+)/(?<publishedDate>.+)$', flags: '' }] },
'Ann ; Bob; Ann/2020/2024-02-29.epub',
);
expect(result.metadata).toEqual({ authors: ['Ann', 'Bob'], publishedYear: 2024, publishedDate: '2024-02-29' });
});
it('rejects impossible dates without preventing other valid fields', async () => {
const result = await service.preview({ rules: [{ pattern: '^(?<title>[^/]+)/(?<publishedDate>.+)$', flags: '' }] }, 'Book/2023-02-29.epub');
expect(result.metadata).toEqual({ title: 'Book' });
expect(result.diagnostics[0].code).toBe('invalid_value');
});
it.each([
['08', '08'],
['1.5', '1.5'],
['1,5', '1.5'],
])('keeps series index %s as %s', async (captured, expected) => {
const result = await service.preview({ rules: [{ pattern: '^(?<title>[^/]+)/(?<seriesIndex>.+)$', flags: '' }] }, `Book/${captured}.epub`);
expect(result.metadata).toEqual({ title: 'Book', seriesIndex: expected });
});
it.each(['IV', '2a', '1-2', '1,5,2', '.5'])('skips series index %s that the column would reject', async (captured) => {
const result = await service.preview({ rules: [{ pattern: '^(?<title>[^/]+)/(?<seriesIndex>.+)$', flags: '' }] }, `Book/${captured}.epub`);
expect(result.metadata).toEqual({ title: 'Book' });
expect(result.diagnostics).toEqual([{ code: 'invalid_value', ruleIndex: 0, field: 'seriesIndex' }]);
});
it.each([
['[', 'invalid_pattern'],
['(?<helper>.+)', 'missing_group'],
['(?<author>.+)(?<authors>.*)', 'conflicting_authors'],
])('validates pattern %s without saving it', async (pattern, code) => {
const config = { rules: [{ pattern, flags: '' }] };
const result = await service.preview(config, 'Book.epub');
expect(result.diagnostics).toContainEqual({ code, ruleIndex: 0 });
await expect(service.validate(config)).rejects.toThrow('Invalid regex metadata configuration');
});
it('terminates a pathological expression, keeps the event loop responsive, and recovers', async () => {
let timerRan = false;
const timer = setTimeout(() => {
timerRan = true;
}, 20);
const result = await service.preview({ rules: [{ pattern: '^(?<title>(a+)+)$', flags: '' }] }, `${'a'.repeat(200)}!.epub`);
clearTimeout(timer);
expect(result.diagnostics).toEqual([{ code: 'timeout' }]);
expect(timerRan).toBe(true);
const recovered = await service.preview({ rules: [{ pattern: '^(?<title>.+)$', flags: '' }] }, 'Recovered.epub');
expect(recovered.metadata.title).toBe('Recovered');
});
it('bounds queued work and resolves pending requests on shutdown', async () => {
const config = { rules: [{ pattern: '^(?<title>.+)$', flags: '' }] };
const tasks = Array.from({ length: 40 }, () => service.preview(config, 'Book.epub'));
await service.onModuleDestroy();
const results = await Promise.all(tasks);
expect(results.some((result) => result.diagnostics[0]?.code === 'busy')).toBe(true);
expect(results.every((result) => result.diagnostics.length > 0)).toBe(true);
});
it('never refuses scanner evaluation for a full queue', async () => {
const config = { rules: [{ pattern: '^(?<title>.+)$', flags: '' }] };
const results = await Promise.all(Array.from({ length: 40 }, () => service.evaluate(config, 'Book.epub')));
expect(results.every((result) => result.metadata.title === 'Book')).toBe(true);
});
it('retries a pool failure once before reporting it', async () => {
const config = { rules: [{ pattern: '^(?<title>.+)$', flags: '' }] };
const run = vi.spyOn(service as any, 'run').mockResolvedValueOnce({ diagnostics: [{ code: 'timeout' }] });
const result = await service.evaluate(config, 'Book.epub');
expect(run).toHaveBeenCalledTimes(2);
expect(result.metadata).toEqual({ title: 'Book' });
});
it('reports a pool failure on save as unavailable rather than invalid rules', async () => {
vi.spyOn(service as any, 'run').mockResolvedValue({ diagnostics: [{ code: 'worker_failed' }] });
await expect(service.validate({ rules: [{ pattern: '(?<title>.+)', flags: '' }] })).rejects.toBeInstanceOf(ServiceUnavailableException);
});
it('rejects excessive rules, pattern length, and stateful flags', async () => {
for (const config of [
{ rules: Array.from({ length: 21 }, () => ({ pattern: '(?<title>.+)', flags: '' })) },
{ rules: [{ pattern: 'a'.repeat(2049), flags: '' }] },
{ rules: [{ pattern: '(?<title>.+)', flags: 'g' }] },
])
await expect(service.validate(config)).rejects.toThrow();
});
});
describe('regex metadata worker', () => {
// CI runs server tests without building @bookorbit/types, and a worker thread cannot see the test alias.
it.each(['regex-metadata-engine.ts', 'regex-metadata.worker.ts'])('%s imports only types from the shared package', (file) => {
const source = readFileSync(join(__dirname, file), 'utf8');
expect(source).not.toMatch(/^import (?!type )[^;]*'@bookorbit\/types'/m);
});
});
describe('relative regex paths', () => {
it.each(['/books/a.epub', '../a.epub', 'a/../../b.epub', 'C:\\books\\a.epub', 'C:a.epub', '\\\\server\\a.epub', 'a\0.epub', 'a/'.repeat(2050)])(
'rejects unsafe input',
(path) => {
expect(() => prepareRegexMetadataPath(path)).toThrow();
},
);
it('preserves dots in folders and filenames and normalizes separators', () => {
expect(prepareRegexMetadataPath('Dr. Author\\Vol. 1\\My.Book.epub')).toBe('Dr. Author/Vol. 1/My.Book');
expect(prepareRegexMetadataPath('Author/Book')).toBe('Author/Book');
});
});
describe('regex metadata precedence', () => {
const base: ParsedBookData = {
title: 'Embedded',
authors: [{ name: 'Author', sortName: null }],
seriesName: 'Original',
genres: ['Novel'],
cover: Buffer.from('cover'),
};
it('overrides only captured fields when regex has priority', () => {
expect(mergeRegexMetadata(base, { seriesName: 'Regex' }, true)).toEqual({ ...base, seriesName: 'Regex' });
});
it('fills missing values when the file source has priority', () => {
expect(mergeRegexMetadata(base, { title: 'Regex', seriesIndex: '08' }, false)).toEqual({ ...base, seriesIndex: '08' });
});
it('does not keep a lower-priority date when regex overrides the year', () => {
expect(mergeRegexMetadata({ ...base, publishedDate: '2020-01-01', publishedYear: 2020 }, { publishedYear: 2024 }, true)).toMatchObject({
publishedYear: 2024,
publishedDate: undefined,
});
});
});
@@ -0,0 +1,177 @@
import { BadRequestException, Injectable, OnModuleDestroy, ServiceUnavailableException } from '@nestjs/common';
import { existsSync } from 'node:fs';
import { join } from 'node:path';
import { Worker } from 'node:worker_threads';
import {
REGEX_METADATA_GROUPS,
REGEX_METADATA_LIMITS,
SERIES_INDEX_MAX_LENGTH,
SERIES_INDEX_PATTERN,
type RegexMetadataConfig,
type RegexMetadataDiagnosticCode,
type RegexMetadataPreview,
type RegexMetadataValidation,
} from '@bookorbit/types';
import { prepareRegexMetadataPath } from './regex-metadata-path';
import type { RegexMetadataTask, RegexMetadataWorkerData } from './regex-metadata-engine';
const INTERACTIVE_QUEUE_LIMIT = 32;
const TRANSIENT_CODES: ReadonlySet<RegexMetadataDiagnosticCode> = new Set(['timeout', 'busy', 'worker_failed']);
/** Whether the result failed because of the worker pool rather than the rules. */
export function isTransientRegexFailure(code: RegexMetadataDiagnosticCode): boolean {
return TRANSIENT_CODES.has(code);
}
interface Task {
input: RegexMetadataTask;
resolve: (result: RegexMetadataPreview) => void;
}
interface Slot {
worker: Worker;
ready: boolean;
task?: Task;
timer?: ReturnType<typeof setTimeout>;
discarding?: boolean;
}
@Injectable()
export class RegexMetadataService implements OnModuleDestroy {
private readonly slots = new Set<Slot>();
private readonly queue: Task[] = [];
private stopped = false;
/** Rejects rules that cannot be saved; a pool failure is reported as unavailable, never as invalid rules. */
async validate(config: RegexMetadataConfig | null | undefined): Promise<void> {
if (config == null) return;
const { diagnostics } = await this.runWithRetry({ config });
if (diagnostics.some((diagnostic) => isTransientRegexFailure(diagnostic.code)))
throw new ServiceUnavailableException('Regex metadata rules could not be checked right now');
if (diagnostics.length) throw new BadRequestException({ message: 'Invalid regex metadata configuration', diagnostics });
}
async check(config: RegexMetadataConfig): Promise<RegexMetadataValidation> {
const result = await this.run({ config }, true);
return { valid: result.diagnostics.length === 0, diagnostics: result.diagnostics };
}
preview(config: RegexMetadataConfig, relativePath: string): Promise<RegexMetadataPreview> {
return this.run({ config, inputPath: prepareRegexMetadataPath(relativePath) }, true);
}
/** Scanner entry point: never refused for a full queue, and retried once after a pool failure. */
evaluate(config: RegexMetadataConfig, relativePath: string): Promise<RegexMetadataPreview> {
return this.runWithRetry({ config, inputPath: prepareRegexMetadataPath(relativePath) });
}
private async runWithRetry(input: RegexMetadataTask): Promise<RegexMetadataPreview> {
const result = await this.run(input, false);
// The timeout runs on the main thread, so a stalled event loop can trip it; a second attempt tells that apart from a slow expression.
return result.diagnostics.some((diagnostic) => isTransientRegexFailure(diagnostic.code)) ? this.run(input, false) : result;
}
private run(input: RegexMetadataTask, interactive: boolean): Promise<RegexMetadataPreview> {
this.assertBounds(input.config);
if (this.stopped) return Promise.resolve(this.failure(input, 'worker_failed'));
if (interactive && this.queue.length >= INTERACTIVE_QUEUE_LIMIT) return Promise.resolve(this.failure(input, 'busy'));
return new Promise((resolve) => {
this.queue.push({ input, resolve });
this.dispatch();
});
}
private assertBounds(config: RegexMetadataConfig): void {
if (
!Array.isArray(config.rules) ||
!config.rules.length ||
config.rules.length > REGEX_METADATA_LIMITS.rules ||
config.rules.some(
(rule) =>
!rule ||
typeof rule.pattern !== 'string' ||
!rule.pattern.length ||
rule.pattern.length > REGEX_METADATA_LIMITS.patternLength ||
typeof rule.flags !== 'string' ||
!/^(?:i?u?|ui)$/.test(rule.flags),
)
) {
throw new BadRequestException('Invalid regex metadata rules or flags');
}
}
private dispatch(): void {
if (this.stopped) return;
for (const slot of this.slots) {
if (!slot.ready || slot.discarding || slot.task || !this.queue.length) continue;
slot.task = this.queue.shift()!;
slot.timer = setTimeout(() => this.discard(slot, 'timeout'), 100);
slot.worker.postMessage(slot.task.input);
}
if (this.queue.length && this.slots.size < 2) this.spawn();
}
private spawn(): void {
const jsPath = join(__dirname, 'regex-metadata.worker.js');
const workerPath = existsSync(jsPath) ? jsPath : join(__dirname, 'regex-metadata.worker.ts');
const workerData: RegexMetadataWorkerData = {
groups: REGEX_METADATA_GROUPS,
seriesIndexPattern: SERIES_INDEX_PATTERN,
seriesIndexMaxLength: SERIES_INDEX_MAX_LENGTH,
};
const worker = new Worker(workerPath, {
workerData,
execArgv: workerPath.endsWith('.ts') ? ['--import', 'tsx'] : [],
resourceLimits: { maxOldGenerationSizeMb: 64 },
});
const slot: Slot = { worker, ready: false };
this.slots.add(slot);
slot.timer = setTimeout(() => this.discard(slot, 'worker_failed'), 10_000);
worker.on('message', (message: RegexMetadataPreview | 'ready') => {
if (!this.slots.has(slot) || slot.discarding) return;
clearTimeout(slot.timer);
if (message === 'ready') slot.ready = true;
else {
slot.task?.resolve(message);
slot.task = undefined;
}
this.dispatch();
});
worker.on('error', () => this.discard(slot, 'worker_failed'));
worker.on('exit', () => this.discard(slot, 'worker_failed'));
}
private discard(slot: Slot, code: RegexMetadataDiagnosticCode): void {
if (!this.slots.has(slot) || slot.discarding) return;
slot.discarding = true;
clearTimeout(slot.timer);
if (slot.task) slot.task.resolve(this.failure(slot.task.input, code));
else if (!slot.ready) {
// A broken worker entrypoint must not create an endless respawn loop.
for (const task of this.queue.splice(0)) task.resolve(this.failure(task.input, code));
}
slot.task = undefined;
void slot.worker.terminate().finally(() => {
this.slots.delete(slot);
this.dispatch();
});
}
private failure(input: RegexMetadataTask, code: RegexMetadataDiagnosticCode): RegexMetadataPreview {
return { inputPath: input.inputPath ?? '', matched: false, ruleIndex: null, groups: {}, metadata: {}, diagnostics: [{ code }] };
}
async onModuleDestroy(): Promise<void> {
this.stopped = true;
for (const task of this.queue.splice(0)) task.resolve(this.failure(task.input, 'worker_failed'));
const slots = [...this.slots];
this.slots.clear();
await Promise.all(
slots.map((slot) => {
clearTimeout(slot.timer);
if (slot.task) slot.task.resolve(this.failure(slot.task.input, 'worker_failed'));
return slot.worker.terminate();
}),
);
}
}
@@ -0,0 +1,9 @@
import { parentPort, workerData } from 'node:worker_threads';
import { evaluateRegexMetadata, type RegexMetadataTask, type RegexMetadataWorkerData } from './regex-metadata-engine';
const shared = workerData as RegexMetadataWorkerData;
parentPort!.on('message', (task: RegexMetadataTask) => {
parentPort!.postMessage(evaluateRegexMetadata(task, shared));
});
parentPort!.postMessage('ready');
@@ -37,6 +37,7 @@ function makeService(db: any = {}) {
bufferBookMissingEvent: vi.fn(),
bufferBooksUnavailableNotification: vi.fn(),
bufferBooksRestoredNotification: vi.fn(),
refreshRegexMetadataAfterMoveAsync: vi.fn(),
} as unknown as ScannerService;
const service = new FileWatcherService(db, processor, gateway, scannerService, new SelfWriteRegistry());
@@ -33,6 +33,7 @@ function makeService(db: any = {}) {
bufferBooksUnavailableNotification: vi.fn(),
bufferBooksRestoredNotification: vi.fn(),
reconcileCoverSlotsAsync: vi.fn(),
refreshRegexMetadataAfterMoveAsync: vi.fn(),
} as unknown as ScannerService;
const selfWriteRegistry = new SelfWriteRegistry();
@@ -164,6 +165,7 @@ describe('process()', () => {
expect((gateway as any).emitBookMoved).toHaveBeenCalledWith({ libraryId: 1, bookIds: [5] });
expect(scheduleFolderScanSpy).toHaveBeenCalledWith('/books/moved.epub', 1);
expect(scannerService.refreshRegexMetadataAfterMoveAsync).toHaveBeenCalledWith([5]);
expect(scannerService.startScanAsync).not.toHaveBeenCalled();
});
@@ -511,6 +511,8 @@ export class FileWatcherService implements OnApplicationBootstrap, OnModuleDestr
this.scannerService.bufferBooksRestoredNotification(result.libraryId, result.bookIds);
} else if (result.type === 'book-moved') {
this.gateway.emitBookMoved({ libraryId: result.libraryId, bookIds: result.bookIds });
// The new path is already recorded, so the folder scan below cannot see the move regex rules read.
this.scannerService.refreshRegexMetadataAfterMoveAsync(result.bookIds);
// A move updates the file's path and may consolidate virtual-sibling books.
// Schedule a folder scan so upsertBook can drain any remaining siblings.
if (type === 'create') this.scheduleFolderScan(path, libraryId);
@@ -110,6 +110,7 @@ export class ScannerRepository {
allowedFormats: libraries.allowedFormats,
formatPriority: libraries.formatPriority,
metadataPrecedence: libraries.metadataPrecedence,
regexMetadata: libraries.regexMetadata,
excludePatterns: libraries.excludePatterns,
organizationMode: libraries.organizationMode,
addedAtSource: libraries.addedAtSource,
@@ -153,6 +153,7 @@ const mockGateway = {
const mockMetadata = {
extractAndSave: vi.fn().mockResolvedValue(undefined),
extractAndSaveWithRegex: vi.fn().mockResolvedValue(undefined),
refreshCoverForBook: vi.fn().mockResolvedValue(false),
extractAudioFileDuration: vi.fn().mockResolvedValue(undefined),
aggregateAudioDuration: vi.fn().mockResolvedValue(undefined),
@@ -170,6 +171,7 @@ function makeService(
const notificationService = { notify: vi.fn().mockResolvedValue(undefined) };
const achievementEvents = { emit: vi.fn() };
const selfWriteRegistry = new SelfWriteRegistry();
const regexMetadata = { evaluate: vi.fn() };
const service = new ScannerService(
repo as any,
mockMetadata as any,
@@ -178,11 +180,12 @@ function makeService(
notificationService as any,
selfWriteRegistry,
coverStore as any,
regexMetadata as any,
autoFetchOrchestrator as any,
achievementEvents as any,
coverReconciler as any,
);
return { service, jobStore, notificationService, achievementEvents, selfWriteRegistry };
return { service, jobStore, notificationService, achievementEvents, selfWriteRegistry, regexMetadata };
}
/**
@@ -4429,3 +4432,296 @@ describe('cover slot reconcile after a scan', () => {
expect(reconciler.enqueue).toHaveBeenCalledWith([1], { filesChanged: true });
});
});
describe('regex metadata scan integration', () => {
const regexConfig = { rules: [{ pattern: '(?<series>[^/]+)/[^/]+$', flags: '' }] };
function regexRepo(overrides: Record<string, unknown> = {}) {
return makeRepo({
findLibrarySettings: vi.fn().mockResolvedValue({
allowedFormats: [],
formatPriority: DEFAULT_FORMAT_PRIORITY,
metadataPrecedence: ['regex', 'embedded', 'opfFile'],
regexMetadata: regexConfig,
excludePatterns: [],
organizationMode: 'book_per_folder',
}),
...overrides,
});
}
it('passes the primary relative path and precedence to the shared extraction flow', async () => {
mockFindCandidates.mockResolvedValue({
candidates: [makeCandidate('/library/Author/Book', [makeFileStat()])],
skippedDirs: new Set(),
unchangedDirs: new Set(),
dirMtimes: new Map(),
});
const repo = regexRepo();
const done = awaitScan(repo);
const { service, regexMetadata } = makeService(repo);
regexMetadata.evaluate.mockResolvedValue({ matched: true, metadata: { seriesName: 'Book' }, diagnostics: [] });
await service.startScan(1, 'manual');
await done;
expect(regexMetadata.evaluate).toHaveBeenCalledWith(regexConfig, 'Author/Book/book.epub');
expect(mockMetadata.extractAndSaveWithRegex).toHaveBeenCalledWith(
1,
[{ key: 'embedded', absolutePath: '/library/Author/Book/book.epub', format: 'epub' }],
{ seriesName: 'Book' },
['regex', 'embedded', 'opfFile'],
);
expect(mockMetadata.extractAndSave).not.toHaveBeenCalled();
});
it.each(['book_per_folder', 'book_per_file'])('refreshes renamed primary paths in %s mode without content changes', async (organizationMode) => {
const file = makeFileStat({ absolutePath: '/library/Author/Book/renamed.epub', relPath: 'Author/Book/renamed.epub' });
const folderPath = organizationMode === 'book_per_file' ? file.absolutePath : '/library/Author/Book';
const walk = { candidates: [makeCandidate(folderPath, [file])], skippedDirs: new Set(), unchangedDirs: new Set(), dirMtimes: new Map() };
mockFindCandidates.mockResolvedValue(walk);
mockFindLooseCandidates.mockResolvedValue(walk);
mockStat.mockRejectedValue(Object.assign(new Error('missing old path'), { code: 'ENOENT' }));
const repo = regexRepo({
findLibrarySettings: vi.fn().mockResolvedValue({
allowedFormats: [],
formatPriority: DEFAULT_FORMAT_PRIORITY,
metadataPrecedence: ['regex', 'embedded'],
regexMetadata: regexConfig,
excludePatterns: [],
organizationMode,
}),
findBooksByLibraryFolder: vi.fn().mockResolvedValue([
{
id: 1,
status: 'present',
folderPath: organizationMode === 'book_per_file' ? '/library/Author/Book/book.epub' : folderPath,
primaryFileId: 1,
},
]),
findBookFilesByLibraryFolder: vi.fn().mockResolvedValue([makeBookFile()]),
});
const done = awaitScan(repo);
const { service, regexMetadata } = makeService(repo);
regexMetadata.evaluate.mockResolvedValue({ matched: true, metadata: { title: 'renamed' }, diagnostics: [] });
await service.startScan(1, 'manual', true);
await done;
expect(regexMetadata.evaluate).toHaveBeenCalledWith(regexConfig, file.relPath);
expect(mockMetadata.extractAndSaveWithRegex).toHaveBeenCalledWith(1, expect.any(Array), { title: 'renamed' }, ['regex', 'embedded']);
expect(repo.failScanJob).not.toHaveBeenCalled();
});
it.each([false, true])('refreshes a changed primary identity unless a self write is active (%s)', async (suppressed) => {
const file = makeFileStat();
mockFindCandidates.mockResolvedValue({
candidates: [makeCandidate('/library/Author/Book', [file])],
skippedDirs: new Set(),
unchangedDirs: new Set(),
dirMtimes: new Map(),
});
const repo = regexRepo({
findBooksByLibraryFolder: vi.fn().mockResolvedValue([{ id: 1, status: 'present', folderPath: '/library/Author/Book', primaryFileId: 2 }]),
findBookFilesByLibraryFolder: vi.fn().mockResolvedValue([makeBookFile()]),
});
const done = awaitScan(repo);
const { service, regexMetadata, selfWriteRegistry } = makeService(repo);
if (suppressed) selfWriteRegistry.begin([file.absolutePath]);
await service.startScan(1, 'manual', true);
await done;
expect(regexMetadata.evaluate).toHaveBeenCalledTimes(suppressed ? 0 : 1);
selfWriteRegistry.end([file.absolutePath]);
expect(repo.failScanJob).not.toHaveBeenCalled();
});
it.each(['file extraction failed', 'scoring failed after metadata was saved'])('continues later books without retry after %s', async (message) => {
mockFindCandidates.mockResolvedValue({
candidates: [
makeCandidate('/library/Author/Book', [makeFileStat()]),
makeCandidate('/library/Author/Other', [
makeFileStat({ absolutePath: '/library/Author/Other/book.epub', relPath: 'Author/Other/book.epub', ino: 2002n }),
]),
],
skippedDirs: new Set(),
unchangedDirs: new Set(),
dirMtimes: new Map(),
});
const repo = regexRepo();
repo.createBook
.mockResolvedValueOnce({ id: 1, status: 'present', folderPath: '/library/Author/Book' })
.mockResolvedValueOnce({ id: 2, status: 'present', folderPath: '/library/Author/Other' });
const done = awaitScan(repo);
const { service, regexMetadata } = makeService(repo);
regexMetadata.evaluate.mockResolvedValue({ matched: true, metadata: { title: 'Regex title' }, diagnostics: [] });
mockMetadata.extractAndSaveWithRegex.mockRejectedValueOnce(new Error(message));
await service.startScan(1, 'manual');
await done;
expect(mockMetadata.extractAndSaveWithRegex).toHaveBeenCalledTimes(2);
expect(mockMetadata.extractAndSaveWithRegex).toHaveBeenNthCalledWith(2, 2, expect.any(Array), { title: 'Regex title' }, [
'regex',
'embedded',
'opfFile',
]);
expect(mockMetadata.extractAndSave).not.toHaveBeenCalled();
expect(repo.failScanJob).not.toHaveBeenCalled();
expect(repo.completeScanJob).toHaveBeenCalled();
expect(repo.promoteProcessingBookToPresent).toHaveBeenCalledWith(1);
expect(repo.promoteProcessingBookToPresent).toHaveBeenCalledWith(2);
});
it('still uses file metadata when regex evaluation throws before persistence', async () => {
mockFindCandidates.mockResolvedValue({
candidates: [makeCandidate('/library/Author/Book', [makeFileStat()])],
skippedDirs: new Set(),
unchangedDirs: new Set(),
dirMtimes: new Map(),
});
const repo = regexRepo();
const done = awaitScan(repo);
const { service, regexMetadata } = makeService(repo);
regexMetadata.evaluate.mockRejectedValue(new Error('worker unavailable'));
await service.startScan(1, 'manual');
await done;
expect(mockMetadata.extractAndSaveWithRegex).not.toHaveBeenCalled();
expect(mockMetadata.extractAndSave).toHaveBeenCalledTimes(1);
expect(repo.failScanJob).not.toHaveBeenCalled();
});
it('does not re-extract unchanged books on a full scan after changing rules', async () => {
mockFindCandidates.mockResolvedValue({
candidates: [makeCandidate('/library/Author/Book', [makeFileStat()])],
skippedDirs: new Set(),
unchangedDirs: new Set(),
dirMtimes: new Map(),
});
const repo = regexRepo({
findBooksByLibraryFolder: vi.fn().mockResolvedValue([{ id: 1, status: 'present', folderPath: '/library/Author/Book', primaryFileId: 1 }]),
findBookFilesByLibraryFolder: vi.fn().mockResolvedValue([makeBookFile()]),
});
const done = awaitScan(repo);
const { service, regexMetadata } = makeService(repo);
await service.startScan(1, 'manual', true);
await done;
expect(regexMetadata.evaluate).not.toHaveBeenCalled();
expect(mockMetadata.extractAndSaveWithRegex).not.toHaveBeenCalled();
expect(mockMetadata.extractAndSave).not.toHaveBeenCalled();
});
function candidatesFor(count: number) {
return Array.from({ length: count }, (_, index) =>
makeCandidate(`/library/Author/Book${index}`, [
makeFileStat({ absolutePath: `/library/Author/Book${index}/book.epub`, relPath: `Author/Book${index}/book.epub`, ino: BigInt(3000 + index) }),
]),
);
}
it('falls back per book after a pool failure and skips regex once failures reach the limit', async () => {
mockFindCandidates.mockResolvedValue({ candidates: candidatesFor(4), skippedDirs: new Set(), unchangedDirs: new Set(), dirMtimes: new Map() });
const repo = regexRepo();
for (let index = 0; index < 4; index++)
repo.createBook.mockResolvedValueOnce({ id: index + 1, status: 'present', folderPath: `/library/Author/Book${index}` });
const done = awaitScan(repo);
const { service, regexMetadata } = makeService(repo);
regexMetadata.evaluate.mockResolvedValue({ matched: false, metadata: {}, diagnostics: [{ code: 'timeout' }] });
await service.startScan(1, 'manual');
await done;
expect(regexMetadata.evaluate).toHaveBeenCalledTimes(3);
expect(mockMetadata.extractAndSave).toHaveBeenCalledTimes(4);
expect(mockMetadata.extractAndSaveWithRegex).not.toHaveBeenCalled();
});
it('keeps applying regex to later books after one book hits a pool failure', async () => {
mockFindCandidates.mockResolvedValue({ candidates: candidatesFor(2), skippedDirs: new Set(), unchangedDirs: new Set(), dirMtimes: new Map() });
const repo = regexRepo();
repo.createBook
.mockResolvedValueOnce({ id: 1, status: 'present', folderPath: '/library/Author/Book0' })
.mockResolvedValueOnce({ id: 2, status: 'present', folderPath: '/library/Author/Book1' });
const done = awaitScan(repo);
const { service, regexMetadata } = makeService(repo);
regexMetadata.evaluate
.mockResolvedValueOnce({ matched: false, metadata: {}, diagnostics: [{ code: 'busy' }] })
.mockResolvedValueOnce({ matched: true, metadata: { seriesName: 'Book1' }, diagnostics: [] });
await service.startScan(1, 'manual');
await done;
expect(mockMetadata.extractAndSave).toHaveBeenCalledTimes(1);
expect(mockMetadata.extractAndSaveWithRegex).toHaveBeenCalledWith(2, expect.any(Array), { seriesName: 'Book1' }, [
'regex',
'embedded',
'opfFile',
]);
});
it('skips regex for the rest of the scan as soon as the rules themselves fail', async () => {
mockFindCandidates.mockResolvedValue({ candidates: candidatesFor(2), skippedDirs: new Set(), unchangedDirs: new Set(), dirMtimes: new Map() });
const repo = regexRepo();
repo.createBook
.mockResolvedValueOnce({ id: 1, status: 'present', folderPath: '/library/Author/Book0' })
.mockResolvedValueOnce({ id: 2, status: 'present', folderPath: '/library/Author/Book1' });
const done = awaitScan(repo);
const { service, regexMetadata } = makeService(repo);
regexMetadata.evaluate.mockResolvedValue({ matched: false, metadata: {}, diagnostics: [{ code: 'invalid_pattern', ruleIndex: 0 }] });
await service.startScan(1, 'manual');
await done;
expect(regexMetadata.evaluate).toHaveBeenCalledTimes(1);
expect(mockMetadata.extractAndSave).toHaveBeenCalledTimes(2);
});
});
describe('regex metadata outside a scan', () => {
const regexConfig = { rules: [{ pattern: '^(?<series>[^/]+)/', flags: '' }] };
const book = { id: 7, libraryId: 1, primaryFileId: 11, status: 'present', folderPath: '/library/Series/Book' };
const primary = {
id: 11,
bookId: 7,
format: 'epub',
role: 'content',
absolutePath: '/library/Series/Book/book.epub',
relPath: 'Series/Book/book.epub',
sizeBytes: 10,
durationSeconds: null,
mediaOverlayAvailable: false,
sortOrder: 0,
};
function rulesRepo(regexMetadata: unknown) {
return makeRepo({
findBookById: vi.fn().mockResolvedValue(book),
findBookFilesByBookId: vi.fn().mockResolvedValue([primary]),
findLibrarySettings: vi.fn().mockResolvedValue({ metadataPrecedence: ['regex', 'embedded'], regexMetadata }),
});
}
it('leaves books in libraries without rules to the caller', async () => {
const repo = rulesRepo(null);
const { service, regexMetadata } = makeService(repo);
await expect(service.extractMetadataWithLibraryRules(7)).resolves.toBe(false);
expect(repo.findBookFilesByBookId).not.toHaveBeenCalled();
expect(regexMetadata.evaluate).not.toHaveBeenCalled();
});
it('applies the rules to the primary file path with the library precedence', async () => {
const { service, regexMetadata } = makeService(rulesRepo(regexConfig));
regexMetadata.evaluate.mockResolvedValue({ matched: true, metadata: { seriesName: 'Series' }, diagnostics: [] });
await expect(service.extractMetadataWithLibraryRules(7)).resolves.toBe(true);
expect(regexMetadata.evaluate).toHaveBeenCalledWith(regexConfig, 'Series/Book/book.epub');
expect(mockMetadata.extractAndSaveWithRegex).toHaveBeenCalledWith(
7,
[{ key: 'embedded', absolutePath: '/library/Series/Book/book.epub', format: 'epub' }],
{ seriesName: 'Series' },
['regex', 'embedded'],
);
expect(mockMetadata.extractAndSave).not.toHaveBeenCalled();
});
it('falls back to file metadata when no rule matches the path', async () => {
const { service, regexMetadata } = makeService(rulesRepo(regexConfig));
regexMetadata.evaluate.mockResolvedValue({ matched: false, metadata: {}, diagnostics: [] });
await expect(service.extractMetadataWithLibraryRules(7)).resolves.toBe(true);
expect(mockMetadata.extractAndSaveWithRegex).not.toHaveBeenCalled();
expect(mockMetadata.extractAndSave).toHaveBeenCalledWith(7, '/library/Series/Book/book.epub', 'epub');
});
it('does not read a book this instance is writing', async () => {
const { service, regexMetadata, selfWriteRegistry } = makeService(rulesRepo(regexConfig));
selfWriteRegistry.begin([primary.absolutePath]);
await expect(service.extractMetadataWithLibraryRules(7)).resolves.toBe(true);
selfWriteRegistry.end([primary.absolutePath]);
expect(regexMetadata.evaluate).not.toHaveBeenCalled();
expect(mockMetadata.extractAndSave).not.toHaveBeenCalled();
});
});
+155 -3
View File
@@ -1,3 +1,5 @@
import { isTransientRegexFailure, RegexMetadataService } from '../metadata/regex/regex-metadata.service';
import type { RegexMetadataConfig, RegexMetadataPreview } from '@bookorbit/types';
import { ConflictException, Injectable, Logger, NotFoundException, OnApplicationBootstrap, Optional } from '@nestjs/common';
import { createHash } from 'crypto';
import { mapWithConcurrency } from '../../common/utils/batch.utils';
@@ -90,7 +92,21 @@ interface ScanLookupMaps {
fileIdsByBookId: Map<number, Set<number>>;
}
interface RegexScanContext {
config: RegexMetadataConfig | null;
disabled: boolean;
/** Books whose rules could not run even after a retry; past the limit the rest of the scan skips regex. */
failures: number;
}
const REGEX_SCAN_FAILURE_LIMIT = 3;
function createRegexScanContext(config: RegexMetadataConfig | null | undefined): RegexScanContext {
return { config: config ?? null, disabled: false, failures: 0 };
}
interface LibraryScanSettings {
regexContext: RegexScanContext;
allowedFormats: string[];
formatPriority: string[];
metadataPrecedence: string[];
@@ -160,6 +176,7 @@ interface RegisteredFile {
isNew: boolean;
wasReassigned: boolean;
wasChanged: boolean;
wasPathChanged: boolean;
mediaOverlayAvailable: boolean;
/** This book's stored sort order for the file before this scan; null when the file is new to it. */
previousSortOrder: number | null;
@@ -175,6 +192,7 @@ interface ProcessedFileResult {
isNew: boolean;
reassigned: boolean;
changed: boolean;
pathChanged?: boolean;
fileId: number | null;
/** The book that owned the file before this scan moved it here. */
previousBookId?: number;
@@ -254,6 +272,7 @@ export class ScannerService implements OnApplicationBootstrap {
private readonly notificationService: NotificationService,
private readonly selfWriteRegistry: SelfWriteRegistry,
private readonly coverStore: BookCoverStore,
private readonly regexMetadata: RegexMetadataService,
@Optional() private readonly autoFetchOrchestrator?: BookMetadataFetchOrchestratorService,
@Optional() private readonly achievementEvents?: AchievementEventsService,
@Optional() private readonly coverReconciler?: CoverSlotReconciler,
@@ -709,6 +728,7 @@ export class ScannerService implements OnApplicationBootstrap {
const allowedFormats = settings?.allowedFormats ?? [];
const formatPriority = settings?.formatPriority ?? DEFAULT_FORMAT_PRIORITY;
const metadataPrecedence = settings?.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT];
const regexContext = createRegexScanContext(settings?.regexMetadata);
const excludePatterns = settings?.excludePatterns ?? [];
const organizationMode = normalizeOrganizationMode(settings?.organizationMode);
const addedAtSource = normalizeAddedAtSource(settings?.addedAtSource);
@@ -742,6 +762,7 @@ export class ScannerService implements OnApplicationBootstrap {
organizationMode,
addedAtSource,
forceFullScan,
regexContext,
).catch((err) => {
const errorClass = err instanceof Error ? err.name : 'Error';
const errorMessage = sanitizeLogValue(err instanceof Error ? err.message : String(err));
@@ -957,6 +978,7 @@ export class ScannerService implements OnApplicationBootstrap {
const rawSettings = await this.scannerRepo.findLibrarySettings(libraryId);
const settings: LibraryScanSettings = {
regexContext: createRegexScanContext(rawSettings?.regexMetadata),
allowedFormats: rawSettings?.allowedFormats ?? [],
formatPriority: rawSettings?.formatPriority ?? DEFAULT_FORMAT_PRIORITY,
metadataPrecedence: rawSettings?.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT],
@@ -1000,6 +1022,7 @@ export class ScannerService implements OnApplicationBootstrap {
const rawSettings = await this.scannerRepo.findLibrarySettings(libraryId);
const settings: LibraryScanSettings = {
regexContext: createRegexScanContext(rawSettings?.regexMetadata),
allowedFormats: rawSettings?.allowedFormats ?? [],
formatPriority: rawSettings?.formatPriority ?? DEFAULT_FORMAT_PRIORITY,
metadataPrecedence: rawSettings?.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT],
@@ -1083,6 +1106,7 @@ export class ScannerService implements OnApplicationBootstrap {
settings.addedAtSource,
false,
candidateFolderPaths,
settings.regexContext,
);
this.emitTargetedScanResult(libraryId, result);
seenBookIds.add(result.bookId);
@@ -1205,6 +1229,7 @@ export class ScannerService implements OnApplicationBootstrap {
settings.addedAtSource,
false,
new Set([candidate.folderPath]),
settings.regexContext,
);
const pruned = await this.pruneMissingBookFiles(result.bookId, result.retainedFileIds, maps.fileIdsByBookId, maps.fileByPath, maps.fileByIno, {
added: 0,
@@ -1277,6 +1302,7 @@ export class ScannerService implements OnApplicationBootstrap {
settings.addedAtSource,
false,
new Set([candidate.folderPath]),
settings.regexContext,
);
const pruned = await this.pruneMissingBookFiles(result.bookId, result.retainedFileIds, maps.fileIdsByBookId, maps.fileByPath, maps.fileByIno, {
added: 0,
@@ -1368,6 +1394,7 @@ export class ScannerService implements OnApplicationBootstrap {
settings.addedAtSource,
false,
new Set([candidate.folderPath]),
settings.regexContext,
);
const pruned = await this.pruneMissingBookFiles(result.bookId, result.retainedFileIds, maps.fileIdsByBookId, maps.fileByPath, maps.fileByIno, {
added: 0,
@@ -1392,6 +1419,7 @@ export class ScannerService implements OnApplicationBootstrap {
organizationMode: OrganizationMode,
addedAtSource: AddedAtSource,
forceFullScan = false,
regexContext?: RegexScanContext,
): Promise<void> {
const event = 'scanner.run_scan';
const startedAt = Date.now();
@@ -1477,6 +1505,7 @@ export class ScannerService implements OnApplicationBootstrap {
addedAtSource,
skippedDirs,
unchangedDirs,
regexContext,
);
totals.addedCount += counts.addedCount;
totals.updatedCount += counts.updatedCount;
@@ -1564,6 +1593,7 @@ export class ScannerService implements OnApplicationBootstrap {
addedAtSource: AddedAtSource,
skippedDirs: Set<string> = new Set(),
unchangedDirs: Set<string> = new Set(),
regexContext?: RegexScanContext,
): Promise<ScanCounts> {
const event = 'scanner.scan_folder_candidates';
const startedAt = Date.now();
@@ -1597,6 +1627,7 @@ export class ScannerService implements OnApplicationBootstrap {
addedAtSource,
isFirstScan,
candidateFolderPaths,
regexContext,
);
seenBookIds.add(result.bookId);
if (result.created) importedBookIds.push(result.bookId);
@@ -1837,6 +1868,7 @@ export class ScannerService implements OnApplicationBootstrap {
addedAtSource: AddedAtSource,
isFirstScan: boolean,
candidateFolderPaths: Set<string>,
regexContext?: RegexScanContext,
): Promise<ProcessCandidateResult> {
const { bookByFolderPath, booksByParentDir, fileByPath, fileByIno } = maps;
const counts = { added: 0, updated: 0 };
@@ -1924,6 +1956,7 @@ export class ScannerService implements OnApplicationBootstrap {
isNew: processResult.isNew,
wasReassigned: processResult.reassigned,
wasChanged: processResult.changed,
wasPathChanged: processResult.pathChanged === true,
mediaOverlayAvailable: fileByPath.get(fileStat.absolutePath)?.mediaOverlayAvailable === true,
previousSortOrder,
});
@@ -1960,7 +1993,11 @@ export class ScannerService implements OnApplicationBootstrap {
const metadataSources = this.buildMetadataExtractionSources(registeredFiles, winner, metadataPrecedence);
const shouldExtractMetadata =
metadataSources.some((source) => hasMetadataSourceChanged(source.file)) || (book.primaryFileId === null && winner !== null);
metadataSources.some((source) => hasMetadataSourceChanged(source.file)) ||
(regexContext?.config != null &&
winner !== null &&
(hasMetadataSourceChanged(winner) || winner.wasPathChanged || winner.fileId !== book.primaryFileId)) ||
(book.primaryFileId === null && winner !== null);
const audioContentFiles = contentFiles.filter((f) => f.format !== null && isAudioFormat(f.format!));
const changedAudioFiles = audioContentFiles.filter(hasMetadataSourceChanged);
const audioFilesNeedingDuration = audioContentFiles.filter(
@@ -1996,7 +2033,16 @@ export class ScannerService implements OnApplicationBootstrap {
// 3b: Extract shared metadata from the first available configured source.
if (shouldExtractMetadata && !selfWriteInProgress) {
await this.extractFirstAvailableMetadataSource(book.id, metadataSources);
const relativePath = winner ? candidate.files.find((file) => file.absolutePath === winner.absolutePath)?.relPath : undefined;
const startedAt = Date.now();
try {
const regexApplied = await this.extractRegexMetadata(book.id, libraryId, relativePath, metadataSources, metadataPrecedence, regexContext);
if (!regexApplied) await this.extractFirstAvailableMetadataSource(book.id, metadataSources);
} catch (error) {
this.logger.warn(
`[scanner.extract_metadata] [fail] libraryId=${libraryId} bookId=${book.id} durationMs=${Date.now() - startedAt} errorClass=${error instanceof Error ? error.name : 'Error'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - metadata extraction failed; continuing scan without retry`,
);
}
}
// 3c: Write per-file duration for new, changed, or historically unprobed audio files.
@@ -2050,6 +2096,101 @@ export class ScannerService implements OnApplicationBootstrap {
return { bookId: book.id, ...counts, retainedFileIds, becameVisible, created: book.created, coverBookIds };
}
/**
* Shared metadata extraction for one book outside a scan, honouring its library's regex rules.
* An upload registers its own file and the watcher records a move's new path itself, so no later
* scan sees anything to re-read. Resolves false when the library has no rules, leaving the
* caller's plain file extraction in charge.
*/
async extractMetadataWithLibraryRules(bookId: number): Promise<boolean> {
const book = await this.scannerRepo.findBookById(bookId);
if (!book) return false;
const settings = await this.scannerRepo.findLibrarySettings(book.libraryId);
if (!settings?.regexMetadata) return false;
const files = await this.scannerRepo.findBookFilesByBookId(bookId);
if (files.some((file) => this.selfWriteRegistry.isSuppressed(file.absolutePath))) return true;
const registered: RegisteredFile[] = files.map((file) => ({
fileId: file.id,
format: file.format,
role: file.role as FileRole,
absolutePath: file.absolutePath,
sizeBytes: file.sizeBytes ?? 0,
durationSeconds: file.durationSeconds,
isNew: false,
wasReassigned: false,
wasChanged: false,
wasPathChanged: false,
mediaOverlayAvailable: file.mediaOverlayAvailable,
previousSortOrder: file.sortOrder,
}));
const winner = registered.find((file) => file.fileId === book.primaryFileId) ?? null;
const relativePath = files.find((file) => file.id === book.primaryFileId)?.relPath ?? undefined;
const precedence = settings.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT];
const sources = this.buildMetadataExtractionSources(registered, winner, precedence);
const context = createRegexScanContext(settings.regexMetadata);
const regexApplied = await this.extractRegexMetadata(book.id, book.libraryId, relativePath, sources, precedence, context);
if (!regexApplied) await this.extractFirstAvailableMetadataSource(book.id, sources);
return true;
}
/** Re-applies regex rules after the watcher moved these books' files; books in libraries without rules are left alone. */
refreshRegexMetadataAfterMoveAsync(bookIds: number[]): void {
void (async () => {
for (const bookId of bookIds) {
const startedAt = Date.now();
try {
await this.extractMetadataWithLibraryRules(bookId);
} catch (error) {
this.logger.warn(
`[scanner.regex_metadata_after_move] [fail] bookId=${bookId} durationMs=${Date.now() - startedAt} errorClass=${error instanceof Error ? error.name : 'Error'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - regex metadata refresh after move failed`,
);
}
}
})();
}
private async extractRegexMetadata(
bookId: number,
libraryId: number,
relativePath: string | undefined,
sources: MetadataExtractionSource[],
precedence: string[],
context?: RegexScanContext,
): Promise<boolean> {
if (!context?.config || context.disabled || !relativePath) return false;
const startedAt = Date.now();
let result: RegexMetadataPreview;
try {
result = await this.regexMetadata.evaluate(context.config, relativePath);
const failure = result.diagnostics.find((entry) => entry.code !== 'invalid_value');
if (failure) {
// Rules that cannot compile fail every book at once; a pool failure that outlived a retry may be this book alone.
const transient = isTransientRegexFailure(failure.code);
if (transient) context.failures++;
if (!transient || context.failures >= REGEX_SCAN_FAILURE_LIMIT) context.disabled = true;
this.logger.warn(
`[scanner.regex_metadata] [fail] libraryId=${libraryId} bookId=${bookId} durationMs=${Date.now() - startedAt} errorClass=RegexEvaluationError error="${sanitizeLogValue(failure.code)}" regexDisabled=${context.disabled} - regex skipped; file metadata used`,
);
return false;
}
if (!result.matched) return false;
} catch (error) {
this.logger.warn(
`[scanner.regex_metadata] [fail] libraryId=${libraryId} bookId=${bookId} durationMs=${Date.now() - startedAt} errorClass=${error instanceof Error ? error.name : 'Error'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - regex evaluation failed`,
);
return false;
}
// Persistence may already have written metadata before failing; never retry with different precedence.
await this.metadataService.extractAndSaveWithRegex(
bookId,
sources.map((source) => ({ key: source.key, absolutePath: source.file.absolutePath, format: source.format })),
result.metadata,
precedence,
);
return true;
}
private buildMetadataExtractionSources(
registeredFiles: RegisteredFile[],
winner: RegisteredFile | null,
@@ -2618,7 +2759,14 @@ export class ScannerService implements OnApplicationBootstrap {
mtime: fileStat.mtime,
});
}
return { isNew: false, reassigned, changed: !sizeUnchanged || !mtimeUnchanged || !roleUnchanged, fileId: byPath.id, previousBookId };
return {
isNew: false,
reassigned,
changed: !sizeUnchanged || !mtimeUnchanged || !roleUnchanged,
pathChanged: !relPathUnchanged,
fileId: byPath.id,
previousBookId,
};
}
private async resolveByLocalIno(
@@ -2674,6 +2822,7 @@ export class ScannerService implements OnApplicationBootstrap {
isNew: false,
reassigned: byIno.bookId !== bookId,
changed: !sizeUnchanged || !mtimeUnchanged,
pathChanged: oldPathEntry?.relPath !== fileStat.relPath,
fileId: byIno.id,
previousBookId: byIno.bookId !== bookId ? byIno.bookId : undefined,
};
@@ -2757,6 +2906,7 @@ export class ScannerService implements OnApplicationBootstrap {
isNew: false,
reassigned: globalByIno.file.bookId !== bookId,
changed: !sizeUnchanged || !mtimeUnchanged,
pathChanged: globalByIno.file.relPath !== fileStat.relPath,
fileId: globalByIno.file.id,
previousBookId: globalByIno.file.bookId !== bookId ? globalByIno.file.bookId : undefined,
};
@@ -2840,6 +2990,7 @@ export class ScannerService implements OnApplicationBootstrap {
isNew: false,
reassigned: byHash.bookId !== bookId,
changed: false,
pathChanged: byHash.relPath !== fileStat.relPath,
fileId: byHash.id,
previousBookId: byHash.bookId !== bookId ? byHash.bookId : undefined,
};
@@ -2911,6 +3062,7 @@ export class ScannerService implements OnApplicationBootstrap {
isNew: false,
reassigned: globalByHash.file.bookId !== bookId,
changed: false,
pathChanged: globalByHash.file.relPath !== fileStat.relPath,
fileId: globalByHash.file.id,
previousBookId: globalByHash.file.bookId !== bookId ? globalByHash.file.bookId : undefined,
};
@@ -780,4 +780,25 @@ describe('UploadProcessorService', () => {
expect(coverReconciler.enqueue).toHaveBeenCalledWith([7], { filesChanged: true });
expect(coverReconciler.enqueue.mock.invocationCallOrder[0]).toBeGreaterThan(metadataService.extractAndSave.mock.invocationCallOrder[0]!);
});
it.each([
[true, 0],
[false, 1],
])('lets the library regex rules own extraction when they apply (%s)', async (handled, plainCalls) => {
const scanner = { extractMetadataWithLibraryRules: vi.fn().mockResolvedValue(handled) };
const withRules = new UploadProcessorService(
db as any,
metadataService as any,
coverStore as any,
orchestrator as any,
undefined,
scanner as any,
);
metadataService.extractAndSave.mockResolvedValue(undefined);
await withRules.extractMetadata(7, '/tmp/book.epub', 'epub');
expect(scanner.extractMetadataWithLibraryRules).toHaveBeenCalledWith(7);
expect(metadataService.extractAndSave).toHaveBeenCalledTimes(plainCalls);
});
});
@@ -19,6 +19,7 @@ import { BookMetadataFetchOrchestratorService } from '../book-metadata-fetch/boo
import { BookCoverStore } from '../book-cover-store/book-cover-store.service';
import { CoverSlotReconciler } from '../metadata/cover-slot-reconciler.service';
import { MetadataService } from '../metadata/metadata.service';
import { ScannerService } from '../scanner/scanner.service';
import { METADATA_AUDIO_FORMATS } from '../metadata/metadata-extraction.service';
import { computeFileHash } from '../scanner/lib/hash';
import { inspectEpubMediaOverlayFields } from '../reader/epub/epub-media-overlay-capability';
@@ -104,6 +105,7 @@ export class UploadProcessorService {
private readonly coverStore: BookCoverStore,
@Optional() private readonly autoFetchOrchestrator?: BookMetadataFetchOrchestratorService,
@Optional() private readonly coverReconciler?: CoverSlotReconciler,
@Optional() private readonly scanner?: ScannerService,
) {}
/** A file joined or left these books, so their cover slots are brought back in line with it. */
@@ -404,7 +406,9 @@ export class UploadProcessorService {
);
return;
}
await this.metadataService.extractAndSave(bookId, absolutePath, format);
// An upload records its own file, so no later scan would apply the library's regex rules to it.
const extractedWithRules = (await this.scanner?.extractMetadataWithLibraryRules(bookId)) ?? false;
if (!extractedWithRules) await this.metadataService.extractAndSave(bookId, absolutePath, format);
// The embedded extractor writes the aggregate duration to book_metadata, but the
// per-file book_files.durationSeconds the player sums is only populated here.
if (isAudioFormat(format)) {
+11 -1
View File
@@ -7,6 +7,7 @@ import { FileWriteModule } from '../file-write/file-write.module';
import { LibraryModule } from '../library/library.module';
import { MetadataModule } from '../metadata/metadata.module';
import { PathModule } from '../path/path.module';
import { ScannerModule } from '../scanner/scanner.module';
import { BookFileUploadController } from './book-file-upload.controller';
import { UploadController } from './upload.controller';
import { UploadProcessorService } from './upload-processor.service';
@@ -18,7 +19,16 @@ import { UploadSessionRepository } from './upload-session.repository';
import { UploadSessionService } from './upload-session.service';
@Module({
imports: [AppSettingsModule, BookCoverStoreModule, LibraryModule, MetadataModule, BookMetadataFetchModule, FileWriteModule, PathModule],
imports: [
AppSettingsModule,
BookCoverStoreModule,
LibraryModule,
MetadataModule,
BookMetadataFetchModule,
FileWriteModule,
PathModule,
ScannerModule,
],
controllers: [UploadController, BookFileUploadController, UploadSessionController],
providers: [UploadService, UploadValidatorService, UploadStorageService, UploadProcessorService, UploadSessionRepository, UploadSessionService],
exports: [UploadValidatorService, UploadStorageService, UploadProcessorService, UploadService],
@@ -1,5 +1,5 @@
{
"totalRoutes": 660,
"totalRoutes": 662,
"byPermission": {
"(none)": 286,
"Permission.ManageMetadataConfig": 41,
@@ -7,7 +7,7 @@
"Permission.LibraryEditMetadata": 35,
"Permission.BookDockAccess": 17,
"Permission.ManageAppSettings": 62,
"Permission.ManageLibraries": 22,
"Permission.ManageLibraries": 24,
"Permission.ManageUsers": 13,
"Permission.ManageEmail": 8,
"Permission.KoboSync": 13,
@@ -503,6 +503,30 @@
"allowDefault": false,
"useGuards": []
},
{
"file": "server/src/modules/library/library.controller.ts",
"className": "LibraryController",
"methodName": "validateRegexMetadata",
"httpMethod": "POST",
"path": "/libraries/regex-metadata/validate",
"permissions": ["Permission.ManageLibraries"],
"libraryAccess": [],
"isPublic": false,
"allowDefault": false,
"useGuards": []
},
{
"file": "server/src/modules/library/library.controller.ts",
"className": "LibraryController",
"methodName": "previewRegexMetadata",
"httpMethod": "POST",
"path": "/libraries/regex-metadata/preview",
"permissions": ["Permission.ManageLibraries"],
"libraryAccess": [],
"isPublic": false,
"allowDefault": false,
"useGuards": []
},
{
"file": "server/src/modules/library/library.controller.ts",
"className": "LibraryController",
@@ -4,7 +4,7 @@ import { basename, join } from 'path';
import { and, eq, sql } from 'drizzle-orm';
import { Permission } from '@bookorbit/types';
import type { AddedAtRecomputeJob } from '@bookorbit/types';
import type { AddedAtRecomputeJob, RegexMetadataConfig } from '@bookorbit/types';
import * as schema from '../src/db/schema';
import { waitForCondition, waitForScanCompletion } from './e2e/app-harness';
@@ -140,6 +140,8 @@ async function createLibraryViaApi(
readingThreshold: number;
markAsFinishedPercentComplete: number;
fileWriteEnabled: boolean;
regexMetadata: RegexMetadataConfig;
metadataPrecedence: string[];
}>,
): Promise<{
response: InjectResponse;
@@ -167,6 +169,8 @@ async function createLibraryViaApi(
readingThreshold: input?.readingThreshold ?? 0.4,
markAsFinishedPercentComplete: input?.markAsFinishedPercentComplete ?? 95,
fileWriteEnabled: input?.fileWriteEnabled ?? false,
regexMetadata: input?.regexMetadata,
metadataPrecedence: input?.metadataPrecedence,
},
});
expect(response.statusCode).toBe(201);
@@ -210,6 +214,66 @@ describe('Library admin workflows (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () =
await closeAuthorizationMatrixE2EContext(ctx);
});
it('protects regex preview and validates its nested request contract', async () => {
const payload = { config: { rules: [{ pattern: '^(?<title>.+)$', flags: '' }] }, relativePath: 'Book.epub' };
for (const [token, status] of [
[manager.accessToken, 200],
[noPermissionUser.accessToken, 403],
] as const) {
const response = await ctx.app.inject({ method: 'POST', url: '/api/v1/libraries/regex-metadata/preview', headers: authHeader(token), payload });
expect(response.statusCode).toBe(status);
if (status === 200) expect(response.json()).toMatchObject({ inputPath: 'Book', metadata: { title: 'Book' } });
}
for (const bad of [
{ ...payload, relativePath: '/private/Book.epub' },
{ ...payload, relativePath: '../Book.epub' },
{ ...payload, config: { rules: [{ ...payload.config.rules[0], template: '{title}' }] } },
]) {
const response = await ctx.app.inject({
method: 'POST',
url: '/api/v1/libraries/regex-metadata/preview',
headers: authHeader(manager.accessToken),
payload: bad,
});
expect(response.statusCode).toBe(400);
}
// Like every other library setting, regex rules need ManageLibraries and no per-library grant.
const library = await createLibraryWithFolder(ctx);
const scoped = await ctx.app.inject({
method: 'POST',
url: '/api/v1/libraries/regex-metadata/preview',
headers: authHeader(manager.accessToken),
payload: { ...payload, libraryId: library.libraryId },
});
expect(scoped.statusCode).toBe(200);
const update = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(manager.accessToken),
payload: { icon: 'BookOpen', regexMetadata: payload.config },
});
expect(update.statusCode, update.body).toBe(200);
expect(update.json().regexMetadata).toEqual(payload.config);
});
it('validates regex rules without a path and enforces access and DTO boundaries', async () => {
const config = { rules: [{ pattern: '(?<title>.+)', flags: '' }] };
const request = (payload: object, token = manager.accessToken) =>
ctx.app.inject({ method: 'POST', url: '/api/v1/libraries/regex-metadata/validate', headers: authHeader(token), payload });
const valid = await request({ config });
expect(valid.statusCode).toBe(200);
expect(valid.json()).toEqual({ valid: true, diagnostics: [] });
const invalid = await request({ config: { rules: [{ pattern: '[', flags: '' }] } });
expect(invalid.statusCode).toBe(200);
expect(invalid.json()).toEqual({ valid: false, diagnostics: [{ code: 'invalid_pattern', ruleIndex: 0 }] });
expect((await request({ config }, noPermissionUser.accessToken)).statusCode).toBe(403);
expect((await request({ config, relativePath: 'unexpected.epub' })).statusCode).toBe(400);
expect((await request({ config: { rules: [{ pattern: '(?<title>.+)', flags: 'g' }] } })).statusCode).toBe(400);
const library = await createLibraryWithFolder(ctx);
expect((await request({ config, libraryId: library.libraryId })).statusCode).toBe(200);
expect((await request({ config, libraryId: 2_147_483_000 })).statusCode).toBe(404);
});
it.each([
{ organizationMode: 'book_per_file', autoScanCronExpression: '0 2 * * *', watch: true, fileWriteEnabled: true },
{ organizationMode: 'book_per_folder', autoScanCronExpression: null, watch: false, fileWriteEnabled: false },
@@ -302,6 +366,32 @@ describe('Library admin workflows (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () =
expect(storedLibrary).toMatchObject(savedSettings);
});
it.each([null, { rules: [{ pattern: '(?<title>.+)', flags: '' }] }])(
'lets managers save regex rules and unrelated settings without a library grant: %j',
async (regexMetadata) => {
const library = await createLibraryWithFolder(ctx);
await ctx.db.update(schema.libraries).set({ regexMetadata }).where(eq(schema.libraries.id, library.libraryId));
const response = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(manager.accessToken),
payload: { name: `Renamed library ${library.libraryId}`, icon: 'BookOpen', regexMetadata },
});
expect(response.statusCode, response.body).toBe(200);
expect(response.json()).toMatchObject({ name: `Renamed library ${library.libraryId}`, regexMetadata });
const nextRegexMetadata = regexMetadata === null ? { rules: [{ pattern: '(?<title>.+)', flags: '' }] } : null;
const changed = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(manager.accessToken),
payload: { icon: 'BookOpen', regexMetadata: nextRegexMetadata },
});
expect(changed.statusCode, changed.body).toBe(200);
const [stored] = await ctx.db.select().from(schema.libraries).where(eq(schema.libraries.id, library.libraryId));
expect(stored.regexMetadata).toEqual(nextRegexMetadata);
},
);
describe('date added recompute', () => {
async function finishRecompute(libraryId: number): Promise<AddedAtRecomputeJob> {
let result: AddedAtRecomputeJob;
@@ -578,6 +668,14 @@ describe('Library admin workflows (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () =
});
describe('create library and manage access', () => {
it.each([undefined, ['regex', 'embedded']])('creates regex rules with precedence %j', async (metadataPrecedence) => {
const regexMetadata = { rules: [{ pattern: '(?<title>.+)', flags: '' }] };
const { body: library } = await createLibraryViaApi(ctx, manager.accessToken, { regexMetadata, metadataPrecedence });
expect(library.regexMetadata).toEqual(regexMetadata);
expect(library.metadataPrecedence).toEqual(metadataPrecedence ?? ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar', 'regex']);
await waitForNoRunningScans(ctx, library.id);
});
it('persists fractional finished thresholds through create and update', async () => {
const { body: createdLibrary } = await createLibraryViaApi(ctx, manager.accessToken, {
markAsFinishedPercentComplete: 99.95,
+92
View File
@@ -1,3 +1,6 @@
import { ScannerService } from '../src/modules/scanner/scanner.service';
import { eq } from 'drizzle-orm';
import { bookMetadata, bookAuthors, bookGenres } from '../src/db/schema';
import { randomUUID } from 'crypto';
import { readFile, rename } from 'fs/promises';
import { join } from 'path';
@@ -22,6 +25,7 @@ import {
triggerAndWaitForLibraryScan,
uploadBookCover,
waitForCondition,
waitForScanCompletion,
waitForMetadataFetchIdle,
type MetadataLockE2EContext,
} from './e2e/metadata-lock/metadata-lock-harness';
@@ -48,6 +52,94 @@ describe('metadata-lock e2e', { timeout: 30_000 }, () => {
await closeMetadataLockE2EContext(ctx);
});
it.each(['book_per_file', 'book_per_folder'] as const)(
'merges relative regex metadata and preserves unchanged and locked fields (%s)',
async (mode) => {
const library = await createLibraryWithFolder(ctx, { mode });
const fullScan = async () => {
const { jobId } = await ctx.app.get(ScannerService).startScan(library.libraryId, 'manual', true);
await waitForScanCompletion(ctx.db, jobId);
};
const relPath = 'My.Series/08 - File.Name.epub';
await createEpubFixture(library.folderPath, relPath, { title: 'Embedded title' });
const config = { rules: [{ pattern: '^(?<series>[^/]+)/', flags: '' }] };
const configure = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(ctx.adminToken),
payload: { icon: 'BookOpen', regexMetadata: config, metadataPrecedence: ['regex', 'embedded', 'opfFile'] },
});
expect(configure.statusCode).toBe(200);
expect(configure.json().regexMetadata).toEqual(config);
const preview = await ctx.app.inject({
method: 'POST',
url: '/api/v1/libraries/regex-metadata/preview',
headers: authHeader(ctx.adminToken),
payload: { libraryId: library.libraryId, config, relativePath: relPath },
});
expect(preview.statusCode).toBe(200);
expect(preview.json()).toMatchObject({ inputPath: 'My.Series/08 - File.Name', metadata: { seriesName: 'My.Series' } });
await fullScan();
const located = await locateBookFileByRelPath(ctx, library.libraryId, relPath);
const read = async () => (await ctx.db.select().from(bookMetadata).where(eq(bookMetadata.bookId, located.bookId)))[0];
expect(await read()).toMatchObject({ title: 'Embedded title', seriesName: 'My.Series' });
const originalAuthors = await ctx.db.select().from(bookAuthors).where(eq(bookAuthors.bookId, located.bookId));
const changed = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(ctx.adminToken),
payload: { regexMetadata: { rules: [{ pattern: '^(?<series>[^/]+)/(?<seriesIndex>[^ ]+) - (?<title>.+)$', flags: '' }] } },
});
expect(changed.statusCode).toBe(200);
await fullScan();
expect(await read()).toMatchObject({ title: 'Embedded title', seriesIndex: null });
expect((await patchLocks(located.bookId, ['title'])).statusCode).toBe(200);
await pause(1100);
await createEpubFixture(library.folderPath, relPath, { title: 'Changed embedded title with a different size' });
await fullScan();
expect(await read()).toMatchObject({ title: 'Embedded title', seriesName: 'My.Series', seriesIndex: '08' });
expect(await ctx.db.select().from(bookAuthors).where(eq(bookAuthors.bookId, located.bookId))).toEqual(originalAuthors);
},
);
it.each(['book_per_file', 'book_per_folder'] as const)('preserves unlocked relations on a partial regex rescan (%s)', async (mode) => {
const library = await createLibraryWithFolder(ctx, { mode });
const fullScan = async () => {
const { jobId } = await ctx.app.get(ScannerService).startScan(library.libraryId, 'manual', true);
await waitForScanCompletion(ctx.db, jobId);
};
const relPath = 'Path Series/book.epub';
await createEpubWithCoverFixture(library.folderPath, relPath, { title: 'Original title' });
await fullScan();
const { bookId } = await locateBookFileByRelPath(ctx, library.libraryId, relPath);
expect((await patchMetadata(bookId, { authors: ['Existing Author'], genres: ['Existing Genre'] })).statusCode).toBe(200);
expect((await patchLocks(bookId, [])).statusCode).toBe(200);
const originalAuthors = await ctx.db.select().from(bookAuthors).where(eq(bookAuthors.bookId, bookId));
const originalGenres = await ctx.db.select().from(bookGenres).where(eq(bookGenres.bookId, bookId));
expect(originalAuthors).toHaveLength(1);
expect(originalGenres).toHaveLength(1);
const configure = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(ctx.adminToken),
payload: {
icon: 'BookOpen',
regexMetadata: { rules: [{ pattern: '^(?<series>[^/]+)/', flags: '' }] },
metadataPrecedence: ['regex', 'embedded', 'opfFile'],
},
});
expect(configure.statusCode).toBe(200);
await pause(1100);
await createEpubWithCoverFixture(library.folderPath, relPath, { title: 'Changed embedded title with a different size' });
await fullScan();
expect((await locateBookFileByRelPath(ctx, library.libraryId, relPath)).bookId).toBe(bookId);
const [metadata] = await ctx.db.select().from(bookMetadata).where(eq(bookMetadata.bookId, bookId));
expect(metadata).toMatchObject({ title: 'Changed embedded title with a different size', seriesName: 'Path Series', lockedFields: [] });
expect(await ctx.db.select().from(bookAuthors).where(eq(bookAuthors.bookId, bookId))).toEqual(originalAuthors);
expect(await ctx.db.select().from(bookGenres).where(eq(bookGenres.bookId, bookId))).toEqual(originalGenres);
});
it('normalizes and persists lock updates and rejects duplicate lock fields', async () => {
const scanned = await scanSingleEpub('lock-contract', { title: 'Lock Contract Base' });