mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
feat(scanner): regex metadata extraction (#1669)
feat(metadata): extract metadata from file paths with regex rules (#1669) Let libraries define ordered regex rules that read title, authors, series, series number and dates from each book's path relative to its library folder. Run rules in a bounded worker pool, honour source precedence and field locks, and apply them on scans, uploads and watcher moves. Also retry unprobed M4B tag writes with the mp4 muxer. Closes #1351 Co-authored-by: neon <18559658+neonsolstice@users.noreply.github.com>
This commit is contained in:
@@ -62,7 +62,6 @@ html[data-style='nostalgia'] {
|
||||
--glow-texture: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 84 84' fill='%23000'%3E%3Crect x='1' y='1' width='19' height='19' fill-opacity='0.044'/%3E%3Crect x='22' y='1' width='19' height='19' fill-opacity='0.058'/%3E%3Crect x='43' y='1' width='19' height='19' fill-opacity='0.042'/%3E%3Crect x='64' y='1' width='19' height='19' fill-opacity='0.035'/%3E%3Crect x='1' y='22' width='19' height='19' fill-opacity='0.059'/%3E%3Crect x='22' y='22' width='19' height='19' fill-opacity='0.02'/%3E%3Crect x='43' y='22' width='19' height='19' fill-opacity='0.054'/%3E%3Crect x='64' y='22' width='19' height='19' fill-opacity='0.03'/%3E%3Crect x='1' y='43' width='19' height='19' fill-opacity='0.024'/%3E%3Crect x='22' y='43' width='19' height='19' fill-opacity='0.023'/%3E%3Crect x='43' y='43' width='19' height='19' fill-opacity='0.031'/%3E%3Crect x='64' y='43' width='19' height='19' fill-opacity='0.052'/%3E%3Crect x='1' y='64' width='19' height='19' fill-opacity='0.026'/%3E%3Crect x='22' y='64' width='19' height='19' fill-opacity='0.042'/%3E%3Crect x='43' y='64' width='19' height='19' fill-opacity='0.045'/%3E%3Crect x='64' y='64' width='19' height='19' fill-opacity='0.034'/%3E%3C/svg%3E");
|
||||
--glow-texture-size: 107px 107px;
|
||||
|
||||
|
||||
--cover-hover-transform: translateY(-3px);
|
||||
|
||||
--format-chip-fill: 18%;
|
||||
@@ -99,7 +98,6 @@ html[data-style='nostalgia'].dark {
|
||||
--sidebar-count-foreground: oklch(from var(--primary) max(l, 0.72) c h);
|
||||
--glow-texture: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 84 84' fill='%23fff'%3E%3Crect x='1' y='1' width='19' height='19' fill-opacity='0.056'/%3E%3Crect x='22' y='1' width='19' height='19' fill-opacity='0.042'/%3E%3Crect x='43' y='1' width='19' height='19' fill-opacity='0.082'/%3E%3Crect x='64' y='1' width='19' height='19' fill-opacity='0.036'/%3E%3Crect x='1' y='22' width='19' height='19' fill-opacity='0.073'/%3E%3Crect x='22' y='22' width='19' height='19' fill-opacity='0.059'/%3E%3Crect x='43' y='22' width='19' height='19' fill-opacity='0.035'/%3E%3Crect x='64' y='22' width='19' height='19' fill-opacity='0.071'/%3E%3Crect x='1' y='43' width='19' height='19' fill-opacity='0.033'/%3E%3Crect x='22' y='43' width='19' height='19' fill-opacity='0.065'/%3E%3Crect x='43' y='43' width='19' height='19' fill-opacity='0.036'/%3E%3Crect x='64' y='43' width='19' height='19' fill-opacity='0.037'/%3E%3Crect x='1' y='64' width='19' height='19' fill-opacity='0.064'/%3E%3Crect x='22' y='64' width='19' height='19' fill-opacity='0.096'/%3E%3Crect x='43' y='64' width='19' height='19' fill-opacity='0.04'/%3E%3Crect x='64' y='64' width='19' height='19' fill-opacity='0.048'/%3E%3C/svg%3E");
|
||||
|
||||
|
||||
/* Solid chips in a deep shade of each format's hue with white ink, as the classic look wore them; at this
|
||||
lightness every format clears 5.4:1. */
|
||||
--format-chip-fill: 100%;
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, ref } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { ArrowDown, ArrowLeftRight, ArrowUp, FileCode, FileText, GripVertical, Headphones, RotateCcw } from '@lucide/vue'
|
||||
import { ArrowDown, ArrowUp, FileCode, FileText, GripVertical, Headphones, RotateCcw } from '@lucide/vue'
|
||||
import type { RegexMetadataConfig } from '@bookorbit/types'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import RegexMetadataDialog from './RegexMetadataDialog.vue'
|
||||
import { DEFAULT_FORMAT_PRIORITY, isReadAlongFormatKey, withReadAlongFormatPriority } from '@bookorbit/types'
|
||||
import { formatNumber } from '@/i18n/formatters'
|
||||
import { formatColorVar } from '@/features/book/lib/format-colors'
|
||||
@@ -18,6 +21,8 @@ const PREVIEW_ROWS = 6
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
metadataPrecedence: string[]
|
||||
regexMetadata?: RegexMetadataConfig | null
|
||||
libraryId?: number
|
||||
formatPriority: string[]
|
||||
allowedFormats?: string[]
|
||||
formatCounts?: Record<string, number> | null
|
||||
@@ -26,10 +31,13 @@ const props = withDefaults(
|
||||
)
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:regexMetadata': [value: RegexMetadataConfig | null]
|
||||
'update:regexOpen': [value: boolean]
|
||||
'update:metadataPrecedence': [value: string[]]
|
||||
'update:formatPriority': [value: string[]]
|
||||
}>()
|
||||
|
||||
const regexOpen = ref(false)
|
||||
const showAll = ref(false)
|
||||
const dragKey = ref<string | null>(null)
|
||||
const overKey = ref<string | null>(null)
|
||||
@@ -146,8 +154,47 @@ function onDragEnd() {
|
||||
overKey.value = null
|
||||
}
|
||||
|
||||
function swapSources() {
|
||||
emit('update:metadataPrecedence', [...sources.value].reverse())
|
||||
function moveSourceUp(key: string) {
|
||||
const next = [...sources.value]
|
||||
const index = next.indexOf(key)
|
||||
if (index <= 0) return
|
||||
next.splice(index, 1)
|
||||
next.splice(index - 1, 0, key)
|
||||
emit('update:metadataPrecedence', next)
|
||||
}
|
||||
|
||||
function moveSourceDown(key: string) {
|
||||
const next = [...sources.value]
|
||||
const index = next.indexOf(key)
|
||||
if (index < 0 || index === next.length - 1) return
|
||||
next.splice(index, 1)
|
||||
next.splice(index + 1, 0, key)
|
||||
emit('update:metadataPrecedence', next)
|
||||
}
|
||||
|
||||
function openRegex() {
|
||||
regexOpen.value = true
|
||||
emit('update:regexOpen', true)
|
||||
}
|
||||
function closeRegex() {
|
||||
regexOpen.value = false
|
||||
emit('update:regexOpen', false)
|
||||
}
|
||||
function applyRegex(config: RegexMetadataConfig | null) {
|
||||
emit('update:regexMetadata', config)
|
||||
closeRegex()
|
||||
}
|
||||
|
||||
function sourceTitle(source: string) {
|
||||
return t(
|
||||
source === 'regex'
|
||||
? 'library.creator.metadata.regex.sourceTitle'
|
||||
: `library.creator.metadata.source.${source === 'embedded' ? 'embeddedTitle' : 'opfTitle'}`,
|
||||
)
|
||||
}
|
||||
function sourceHint(source: string) {
|
||||
if (source === 'regex') return t(props.regexMetadata ? 'library.creator.metadata.regex.enabledHint' : 'library.creator.metadata.regex.disabledHint')
|
||||
return t(`library.creator.metadata.source.${source === 'embedded' ? 'embeddedHint' : 'opfHint'}`)
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -246,42 +293,43 @@ function swapSources() {
|
||||
</LibraryCreatorCard>
|
||||
|
||||
<LibraryCreatorCard :label="t('library.creator.metadata.source.title')" label-id="metadata-source-title">
|
||||
<ol aria-labelledby="metadata-source-title" class="flex flex-col items-stretch gap-2 @lg:flex-row @lg:items-center">
|
||||
<template v-for="(source, index) in sources" :key="source">
|
||||
<li
|
||||
class="flex min-w-0 flex-1 items-center gap-3 rounded-xl border px-3 py-2.5"
|
||||
:class="index === 0 ? 'border-primary/55 bg-primary/7' : 'border-border bg-background'"
|
||||
>
|
||||
<span
|
||||
class="flex size-5.5 shrink-0 items-center justify-center rounded-full text-[11px] font-bold"
|
||||
:class="index === 0 ? 'bg-primary text-primary-foreground' : 'bg-muted text-muted-foreground'"
|
||||
>
|
||||
{{ formatNumber(index + 1) }}
|
||||
</span>
|
||||
<component :is="source === 'embedded' ? FileText : FileCode" :size="16" class="shrink-0 text-foreground" aria-hidden="true" />
|
||||
<span class="min-w-0">
|
||||
<span class="block text-[13px] font-semibold text-foreground">
|
||||
{{ source === 'embedded' ? t('library.creator.metadata.source.embeddedTitle') : t('library.creator.metadata.source.opfTitle') }}
|
||||
</span>
|
||||
<span class="block text-xs text-muted-foreground">
|
||||
{{ source === 'embedded' ? t('library.creator.metadata.source.embeddedHint') : t('library.creator.metadata.source.opfHint') }}
|
||||
</span>
|
||||
</span>
|
||||
</li>
|
||||
<li v-if="index === 0" class="flex justify-center">
|
||||
<button
|
||||
type="button"
|
||||
class="flex size-8 items-center justify-center rounded-full border border-border bg-card text-foreground transition-colors hover:bg-muted focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
|
||||
:aria-label="t('library.creator.metadata.source.swap')"
|
||||
:title="t('library.creator.metadata.source.swap')"
|
||||
@click="swapSources"
|
||||
>
|
||||
<ArrowLeftRight :size="14" class="rotate-90 @lg:rotate-0" aria-hidden="true" />
|
||||
</button>
|
||||
</li>
|
||||
</template>
|
||||
<ol aria-labelledby="metadata-source-title" class="flex flex-col gap-2">
|
||||
<li
|
||||
v-for="(source, index) in sources"
|
||||
:key="source"
|
||||
class="flex min-w-0 flex-wrap items-center gap-3 rounded-xl border border-border bg-background px-3 py-2.5"
|
||||
>
|
||||
<span class="text-xs font-bold text-muted-foreground">{{ formatNumber(index + 1) }}</span>
|
||||
<component :is="source === 'embedded' ? FileText : FileCode" :size="16" class="shrink-0" aria-hidden="true" />
|
||||
<span class="min-w-0 flex-1">
|
||||
<span class="block text-sm font-semibold">{{ sourceTitle(source) }}</span>
|
||||
<span class="block text-xs text-muted-foreground">{{ sourceHint(source) }}</span>
|
||||
</span>
|
||||
<Button v-if="source === 'regex'" variant="outline" size="sm" @click="openRegex">{{
|
||||
t('library.creator.metadata.regex.configure')
|
||||
}}</Button>
|
||||
<div class="flex gap-1">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
:disabled="index === 0"
|
||||
:aria-label="t('library.creator.metadata.regex.moveSourceUp', { source: sourceTitle(source) })"
|
||||
@click="moveSourceUp(source)"
|
||||
><ArrowUp
|
||||
/></Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
:disabled="index === sources.length - 1"
|
||||
:aria-label="t('library.creator.metadata.regex.moveSourceDown', { source: sourceTitle(source) })"
|
||||
@click="moveSourceDown(source)"
|
||||
><ArrowDown
|
||||
/></Button>
|
||||
</div>
|
||||
</li>
|
||||
</ol>
|
||||
<p class="mt-2.5 text-xs text-muted-foreground">{{ t('library.creator.metadata.source.hint') }}</p>
|
||||
<p class="mt-2.5 text-xs text-muted-foreground">{{ t('library.creator.metadata.regex.precedenceHint') }}</p>
|
||||
</LibraryCreatorCard>
|
||||
<RegexMetadataDialog v-if="regexOpen" :config="regexMetadata ?? null" :library-id="libraryId" @close="closeRegex" @apply="applyRegex" />
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -183,6 +183,8 @@ function summaryFor(id: LibraryCreatorSectionId): string {
|
||||
? t('library.creator.summary.scannerAll', { mode: modeLabel.value })
|
||||
: t('library.creator.summary.scannerSome', { mode: modeLabel.value, count: form.allowedFormats.length })
|
||||
case 'metadata':
|
||||
if (form.regexMetadata && form.metadataPrecedence[0] === 'regex')
|
||||
return t('library.creator.metadata.regex.summary', { format: leadFormat.value })
|
||||
return form.metadataPrecedence[0] === 'opfFile'
|
||||
? t('library.creator.summary.metadataOpf', { format: leadFormat.value })
|
||||
: t('library.creator.summary.metadataFile', { format: leadFormat.value })
|
||||
@@ -285,6 +287,8 @@ const sectionProps = computed(() => ({
|
||||
},
|
||||
metadata: {
|
||||
metadataPrecedence: form.metadataPrecedence,
|
||||
regexMetadata: form.regexMetadata ?? null,
|
||||
libraryId: creator.editingLibraryId.value ?? undefined,
|
||||
formatPriority: form.formatPriority,
|
||||
allowedFormats: form.allowedFormats,
|
||||
formatCounts: stats.value?.formatCounts ?? null,
|
||||
@@ -495,6 +499,7 @@ const sectionListeners = {
|
||||
'update:addedAtSource': (value: Library['addedAtSource']) => (form.addedAtSource = value),
|
||||
recompute: handleRecomputeAddedAt,
|
||||
check: handleCheckFolders,
|
||||
'update:regexMetadata': (value: Library['regexMetadata']) => (form.regexMetadata = value),
|
||||
'update:metadataPrecedence': (value: string[]) => (form.metadataPrecedence = value),
|
||||
'update:formatPriority': (value: string[]) => (form.formatPriority = value),
|
||||
'update:allowedFormats': (value: string[]) => (form.allowedFormats = value),
|
||||
@@ -522,6 +527,7 @@ const sectionListeners = {
|
||||
'update:fileWriteReadAlongEnabled': (value: boolean) => (form.fileWriteReadAlongEnabled = value),
|
||||
'update:fileWriteReadAlongMaxFileSizeMb': (value: number) => (form.fileWriteReadAlongMaxFileSizeMb = value),
|
||||
'update:pickerOpen': handleNestedModalChange,
|
||||
'update:regexOpen': handleNestedModalChange,
|
||||
}
|
||||
|
||||
useModal({
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
<script setup lang="ts">
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { DialogContent, DialogDescription, DialogOverlay, DialogPortal, DialogRoot, DialogTitle } from 'reka-ui'
|
||||
import { ArrowDown, ArrowUp, Plus, Trash2 } from '@lucide/vue'
|
||||
import { REGEX_METADATA_GROUPS, REGEX_METADATA_LIMITS, type RegexMetadataConfig } from '@bookorbit/types'
|
||||
import { Button } from '@/components/ui/button'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import ToggleSwitch from '@/components/ui/ToggleSwitch.vue'
|
||||
import { formatNumber } from '@/i18n/formatters'
|
||||
import { useRegexMetadataEditor } from '../composables/useRegexMetadataEditor'
|
||||
|
||||
const props = defineProps<{ config: RegexMetadataConfig | null; libraryId?: number }>()
|
||||
const emit = defineEmits<{ close: []; apply: [config: RegexMetadataConfig | null] }>()
|
||||
const { t } = useI18n()
|
||||
const {
|
||||
validation,
|
||||
validating,
|
||||
validationError,
|
||||
ruleErrors,
|
||||
patternInvalid,
|
||||
flagsInvalid,
|
||||
generalDiagnostics,
|
||||
validRuleCount,
|
||||
validateRules,
|
||||
enabled,
|
||||
rules,
|
||||
relativePath,
|
||||
result,
|
||||
busy,
|
||||
error,
|
||||
canApply,
|
||||
canPreview,
|
||||
addRule,
|
||||
removeRule,
|
||||
moveUp,
|
||||
moveDown,
|
||||
value,
|
||||
preview,
|
||||
} = useRegexMetadataEditor(props.config, props.libraryId)
|
||||
const examplePattern = String.raw`(?<author>[^\/]+)\/(?<series>[^\/]+)\/(?<seriesIndex>\d+) - (?<title>.+) - \k<author>`
|
||||
const examplePath = 'Donna Leon/Commissario Brunetti/01 - Venezianisches Finale - Donna Leon.epub'
|
||||
|
||||
const FIELD_LABEL_KEYS: Record<string, string> = {
|
||||
title: 'title',
|
||||
author: 'authors',
|
||||
authors: 'authors',
|
||||
series: 'series',
|
||||
seriesName: 'series',
|
||||
seriesIndex: 'seriesIndex',
|
||||
publishedYear: 'publishedYear',
|
||||
publishedDate: 'publishedDate',
|
||||
}
|
||||
const diagnosticParams = {
|
||||
count: formatNumber(REGEX_METADATA_LIMITS.patternLength),
|
||||
min: formatNumber(1),
|
||||
max: formatNumber(REGEX_METADATA_LIMITS.rules),
|
||||
}
|
||||
|
||||
function fieldLabel(field: string) {
|
||||
const key = FIELD_LABEL_KEYS[field]
|
||||
return key ? t(`library.creator.metadata.regex.fields.${key}`) : field
|
||||
}
|
||||
function diagnosticMessage(code: string) {
|
||||
return t(`library.creator.metadata.regex.diagnostics.${code}`, diagnosticParams)
|
||||
}
|
||||
function close() {
|
||||
emit('close')
|
||||
}
|
||||
function handleOpenChange(open: boolean) {
|
||||
if (!open) close()
|
||||
}
|
||||
function apply() {
|
||||
if (canApply.value) emit('apply', value())
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<DialogRoot :open="true" @update:open="handleOpenChange">
|
||||
<DialogPortal>
|
||||
<DialogOverlay class="fixed inset-0 z-[70] bg-scrim" />
|
||||
<DialogContent
|
||||
class="fixed left-1/2 top-1/2 z-[70] flex max-h-[90dvh] w-[calc(100%-2rem)] max-w-3xl -translate-x-1/2 -translate-y-1/2 flex-col gap-4 overflow-y-auto rounded-lg border border-border bg-card p-5 shadow-xl"
|
||||
>
|
||||
<DialogTitle class="text-lg font-semibold">{{ t('library.creator.metadata.regex.title') }}</DialogTitle>
|
||||
<DialogDescription class="text-sm text-muted-foreground">{{ t('library.creator.metadata.regex.description') }}</DialogDescription>
|
||||
<p class="text-xs text-muted-foreground">{{ t('library.creator.metadata.regex.primaryFileHint') }}</p>
|
||||
<div class="flex items-center gap-2 text-sm font-medium">
|
||||
<ToggleSwitch v-model="enabled" aria-labelledby="regex-enable-label" />
|
||||
<span id="regex-enable-label">{{ t('library.creator.metadata.regex.enable') }}</span>
|
||||
</div>
|
||||
<p class="text-xs text-muted-foreground">
|
||||
{{ t('library.creator.metadata.regex.groups') }} <code>{{ REGEX_METADATA_GROUPS.join(', ') }}</code>
|
||||
</p>
|
||||
<p class="text-xs text-muted-foreground">{{ t('library.creator.metadata.regex.valuesHint') }}</p>
|
||||
|
||||
<section class="flex min-w-0 flex-col gap-2 rounded-md border border-border bg-muted p-3 text-xs">
|
||||
<h3 class="font-medium">{{ t('library.creator.metadata.regex.exampleTitle') }}</h3>
|
||||
<dl class="flex min-w-0 flex-col gap-1">
|
||||
<dt class="text-muted-foreground">{{ t('library.creator.metadata.regex.sourceTitle') }}</dt>
|
||||
<dd class="select-text break-all font-mono">{{ examplePattern }}</dd>
|
||||
<dt class="mt-1 text-muted-foreground">{{ t('library.creator.metadata.regex.exampleTestPath') }}</dt>
|
||||
<dd class="select-text break-all font-mono">{{ examplePath }}</dd>
|
||||
</dl>
|
||||
</section>
|
||||
|
||||
<fieldset v-if="enabled" class="flex min-w-0 flex-col gap-3">
|
||||
<legend class="sr-only">{{ t('library.creator.metadata.regex.rules') }}</legend>
|
||||
<div v-for="(rule, index) in rules" :key="index" class="flex min-w-0 flex-col gap-2 rounded-md border border-border p-3">
|
||||
<div class="flex items-center justify-between gap-2">
|
||||
<label :for="`regex-pattern-${index}`" class="text-sm font-medium">{{
|
||||
t('library.creator.metadata.regex.rule', { number: formatNumber(index + 1) })
|
||||
}}</label>
|
||||
<div class="flex gap-1">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
:disabled="index === 0"
|
||||
:aria-label="t('library.creator.metadata.regex.moveUp')"
|
||||
@click="moveUp(index)"
|
||||
><ArrowUp
|
||||
/></Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
:disabled="index === rules.length - 1"
|
||||
:aria-label="t('library.creator.metadata.regex.moveDown')"
|
||||
@click="moveDown(index)"
|
||||
><ArrowDown
|
||||
/></Button>
|
||||
<Button variant="ghost" size="icon" :aria-label="t('library.creator.metadata.regex.remove')" @click="removeRule(index)"
|
||||
><Trash2
|
||||
/></Button>
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
class="flex min-w-0 flex-wrap items-center rounded-md border bg-background focus-within:ring-2 focus-within:ring-ring"
|
||||
:class="patternInvalid[index] ? 'border-destructive' : 'border-input'"
|
||||
>
|
||||
<span class="px-2 py-2 text-xs text-muted-foreground">{{ t('library.creator.metadata.regex.basePath') }} /</span>
|
||||
<input
|
||||
:id="`regex-pattern-${index}`"
|
||||
v-model="rule.pattern"
|
||||
:maxlength="REGEX_METADATA_LIMITS.patternLength"
|
||||
:aria-describedby="`regex-extension-${index} regex-errors-${index}`"
|
||||
:aria-invalid="patternInvalid[index] || undefined"
|
||||
autocomplete="off"
|
||||
spellcheck="false"
|
||||
class="min-w-24 flex-1 bg-transparent px-2 py-2 font-mono text-sm outline-none"
|
||||
/>
|
||||
<span
|
||||
:id="`regex-extension-${index}`"
|
||||
:title="t('library.creator.metadata.regex.extensionHint')"
|
||||
class="px-2 py-2 font-mono text-xs text-muted-foreground"
|
||||
>.ext<span class="sr-only">: {{ t('library.creator.metadata.regex.extensionHint') }}</span></span
|
||||
>
|
||||
</div>
|
||||
<label class="flex items-center gap-2 text-xs text-muted-foreground">
|
||||
{{ t('library.creator.metadata.regex.flags') }}
|
||||
<Input
|
||||
v-model="rule.flags"
|
||||
:aria-describedby="`regex-errors-${index}`"
|
||||
:aria-invalid="flagsInvalid[index] || undefined"
|
||||
maxlength="2"
|
||||
class="w-20 font-mono"
|
||||
autocomplete="off"
|
||||
spellcheck="false"
|
||||
/>
|
||||
</label>
|
||||
<div :id="`regex-errors-${index}`" aria-live="polite">
|
||||
<p v-for="code in ruleErrors[index]" :key="code" class="text-xs text-destructive">
|
||||
{{ diagnosticMessage(code) }}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<Button variant="outline" :disabled="rules.length >= REGEX_METADATA_LIMITS.rules" class="self-start" @click="addRule"
|
||||
><Plus />{{ t('library.creator.metadata.regex.add') }}</Button
|
||||
>
|
||||
<label class="flex flex-col gap-1 text-sm">
|
||||
{{ t('library.creator.metadata.regex.examplePath') }}
|
||||
<Input
|
||||
v-model="relativePath"
|
||||
:maxlength="REGEX_METADATA_LIMITS.pathLength"
|
||||
:placeholder="t('library.creator.metadata.regex.examplePlaceholder')"
|
||||
/>
|
||||
</label>
|
||||
<Button variant="outline" :disabled="!canPreview" class="self-start" @click="preview">{{
|
||||
t(busy ? 'library.creator.metadata.regex.previewing' : 'library.creator.metadata.regex.preview')
|
||||
}}</Button>
|
||||
</fieldset>
|
||||
|
||||
<div v-if="enabled" aria-live="polite" class="flex flex-col gap-2 text-sm">
|
||||
<p v-if="!validRuleCount" class="text-destructive">{{ t('library.creator.metadata.regex.validationRuleCount', diagnosticParams) }}</p>
|
||||
<p v-else-if="validating" class="text-muted-foreground">{{ t('library.creator.metadata.regex.validating') }}</p>
|
||||
<p v-else-if="validation?.valid" class="text-muted-foreground">{{ t('library.creator.metadata.regex.validationSuccess') }}</p>
|
||||
<p v-if="validationError" role="alert" class="text-destructive">{{ t('library.creator.metadata.regex.validationError') }}</p>
|
||||
<p v-for="diagnostic in generalDiagnostics" :key="diagnostic.code" class="text-destructive">
|
||||
{{ diagnosticMessage(diagnostic.code) }}
|
||||
</p>
|
||||
<Button
|
||||
v-if="validationError || generalDiagnostics.length"
|
||||
variant="outline"
|
||||
class="self-start"
|
||||
:disabled="validating"
|
||||
@click="validateRules"
|
||||
>{{ t('library.creator.metadata.regex.validationRetry') }}</Button
|
||||
>
|
||||
</div>
|
||||
|
||||
<p v-if="error" role="alert" class="text-sm text-destructive">{{ t('library.creator.metadata.regex.previewError') }}</p>
|
||||
<div v-if="result" aria-live="polite" class="flex flex-col gap-2 rounded-md border border-border p-3 text-sm">
|
||||
<p>
|
||||
{{ t('library.creator.metadata.regex.actualInput') }} <code class="break-all">{{ result.inputPath }}</code>
|
||||
</p>
|
||||
<p>
|
||||
{{
|
||||
result.matched
|
||||
? t('library.creator.metadata.regex.matched', { number: formatNumber((result.ruleIndex ?? 0) + 1) })
|
||||
: t('library.creator.metadata.regex.noMatch')
|
||||
}}
|
||||
</p>
|
||||
<dl v-if="result.matched" class="grid grid-cols-[auto_1fr] gap-x-3 gap-y-1">
|
||||
<template v-for="(fieldValue, field) in result.metadata" :key="field"
|
||||
><dt class="text-xs text-muted-foreground">{{ fieldLabel(field) }}</dt>
|
||||
<dd class="break-all">{{ Array.isArray(fieldValue) ? fieldValue.join('; ') : fieldValue }}</dd></template
|
||||
>
|
||||
</dl>
|
||||
<p v-for="(diagnostic, index) in result.diagnostics" :key="index" class="text-destructive">
|
||||
{{ diagnosticMessage(diagnostic.code) }} <span v-if="diagnostic.field">({{ fieldLabel(diagnostic.field) }})</span>
|
||||
</p>
|
||||
</div>
|
||||
<p class="text-xs text-muted-foreground">{{ t('library.creator.metadata.regex.scanHint') }}</p>
|
||||
<div class="flex justify-end gap-2">
|
||||
<Button variant="outline" @click="close">{{ t('common.cancel') }}</Button>
|
||||
<Button :disabled="!canApply" @click="apply">{{ t('library.creator.metadata.regex.apply') }}</Button>
|
||||
</div>
|
||||
</DialogContent>
|
||||
</DialogPortal>
|
||||
</DialogRoot>
|
||||
</template>
|
||||
@@ -79,20 +79,20 @@ describe('LibraryCreatorMetadata', () => {
|
||||
expect(wrapper.findAll('ol[aria-labelledby="primary-file-title"] > li')[1]!.text()).toContain('Not imported')
|
||||
})
|
||||
|
||||
it('swaps which source is read first', async () => {
|
||||
it('moves a source earlier in the precedence list', async () => {
|
||||
const wrapper = mountMetadata({ formatPriority: ['epub'] })
|
||||
|
||||
await wrapper.get('button[aria-label="Swap the order"]').trigger('click')
|
||||
await wrapper.get('button[aria-label="Move metadata.opf file up"]').trigger('click')
|
||||
|
||||
expect(wrapper.emitted('update:metadataPrecedence')).toEqual([[['opfFile', 'embedded']]])
|
||||
expect(wrapper.emitted('update:metadataPrecedence')).toEqual([[['opfFile', 'embedded', 'regex']]])
|
||||
})
|
||||
|
||||
it('fills in a source missing from an older saved list before swapping', async () => {
|
||||
it('fills in a source missing from an older saved list before reordering', async () => {
|
||||
const wrapper = mountMetadata({ formatPriority: ['epub'], metadataPrecedence: ['embedded'] })
|
||||
|
||||
expect(wrapper.text()).toContain('metadata.opf file')
|
||||
await wrapper.get('button[aria-label="Swap the order"]').trigger('click')
|
||||
await wrapper.get('button[aria-label="Move metadata.opf file up"]').trigger('click')
|
||||
|
||||
expect(wrapper.emitted('update:metadataPrecedence')).toEqual([[['opfFile', 'embedded']]])
|
||||
expect(wrapper.emitted('update:metadataPrecedence')).toEqual([[['opfFile', 'embedded', 'regex']]])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
import { effectScope } from 'vue'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { api } from '@/lib/api'
|
||||
import { useRegexMetadataEditor } from '../useRegexMetadataEditor'
|
||||
|
||||
vi.mock('@/lib/api', () => ({ api: vi.fn<typeof import('@/lib/api').api>() }))
|
||||
|
||||
describe('useRegexMetadataEditor', () => {
|
||||
const scopes: ReturnType<typeof effectScope>[] = []
|
||||
afterEach(() => {
|
||||
scopes.splice(0).forEach((scope) => scope.stop())
|
||||
vi.useRealTimers()
|
||||
vi.resetAllMocks()
|
||||
})
|
||||
function editor(config = { rules: [{ pattern: '(?<title>.+)$', flags: '' }] }) {
|
||||
const scope = effectScope()
|
||||
scopes.push(scope)
|
||||
return scope.run(() => useRegexMetadataEditor(config, 42))!
|
||||
}
|
||||
|
||||
it('edits an isolated draft and preserves the ordering on apply', () => {
|
||||
const config = { rules: [{ pattern: '(?<title>.+)$', flags: '' }] }
|
||||
const state = editor(config)
|
||||
state.rules.value[0]!.pattern = '(?<series>.+)'
|
||||
state.addRule()
|
||||
state.rules.value[1]!.pattern = '(?<title>.+)'
|
||||
state.moveUp(1)
|
||||
expect(config.rules[0]!.pattern).toBe('(?<title>.+)$')
|
||||
expect(state.value()?.rules.map((rule) => rule.pattern)).toEqual(['(?<title>.+)', '(?<series>.+)'])
|
||||
state.enabled.value = false
|
||||
expect(state.value()).toBeNull()
|
||||
})
|
||||
|
||||
it('sends the unchanged pattern and example path with its extension', async () => {
|
||||
const state = editor()
|
||||
state.relativePath.value = 'Author/My.Book.epub'
|
||||
vi.mocked(api).mockResolvedValue(
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
inputPath: 'Author/My.Book',
|
||||
matched: true,
|
||||
ruleIndex: 0,
|
||||
metadata: { title: 'Author/My.Book' },
|
||||
groups: {},
|
||||
diagnostics: [],
|
||||
}),
|
||||
),
|
||||
)
|
||||
await state.preview()
|
||||
const request = vi.mocked(api).mock.calls[0]!
|
||||
expect(request[0]).toBe('/api/v1/libraries/regex-metadata/preview')
|
||||
expect(JSON.parse(request[1]!.body as string)).toEqual({
|
||||
config: { rules: [{ pattern: '(?<title>.+)$', flags: '' }] },
|
||||
relativePath: 'Author/My.Book.epub',
|
||||
libraryId: 42,
|
||||
})
|
||||
expect(state.result.value?.inputPath).toBe('Author/My.Book')
|
||||
})
|
||||
|
||||
it('ignores an old response after the input changes', async () => {
|
||||
const state = editor()
|
||||
state.relativePath.value = 'Old.epub'
|
||||
let resolve!: (response: Response) => void
|
||||
vi.mocked(api).mockReturnValue(
|
||||
new Promise((done) => {
|
||||
resolve = done
|
||||
}),
|
||||
)
|
||||
const pending = state.preview()
|
||||
state.relativePath.value = 'New.epub'
|
||||
resolve(new Response(JSON.stringify({ matched: true, inputPath: 'Old' })))
|
||||
await pending
|
||||
expect(state.result.value).toBeNull()
|
||||
expect(state.busy.value).toBe(false)
|
||||
})
|
||||
|
||||
it('shows failed requests and clears them when the draft changes', async () => {
|
||||
const state = editor()
|
||||
state.relativePath.value = '../outside.epub'
|
||||
vi.mocked(api).mockResolvedValue(new Response('', { status: 400 }))
|
||||
await state.preview()
|
||||
expect(state.error.value).toBe(true)
|
||||
state.relativePath.value = 'Inside.epub'
|
||||
expect(state.error.value).toBe(false)
|
||||
})
|
||||
it('debounces validation and only allows applying the current validated rules', async () => {
|
||||
vi.useFakeTimers()
|
||||
vi.mocked(api).mockResolvedValue(new Response(JSON.stringify({ valid: true, diagnostics: [] })))
|
||||
const state = editor()
|
||||
expect(state.canApply.value).toBe(false)
|
||||
await vi.advanceTimersByTimeAsync(399)
|
||||
expect(api).not.toHaveBeenCalled()
|
||||
state.rules.value[0]!.pattern = '(?<author>.+)'
|
||||
await vi.advanceTimersByTimeAsync(400)
|
||||
expect(api).toHaveBeenCalledTimes(1)
|
||||
expect(JSON.parse(vi.mocked(api).mock.calls[0]![1]!.body as string)).toEqual({
|
||||
config: { rules: [{ pattern: '(?<author>.+)', flags: '' }] },
|
||||
libraryId: 42,
|
||||
})
|
||||
expect(state.canApply.value).toBe(true)
|
||||
state.relativePath.value = 'No match.epub'
|
||||
expect(state.canApply.value).toBe(true)
|
||||
state.rules.value[0]!.pattern = '['
|
||||
expect(state.canApply.value).toBe(false)
|
||||
state.enabled.value = false
|
||||
expect(state.canApply.value).toBe(true)
|
||||
await vi.advanceTimersByTimeAsync(400)
|
||||
expect(api).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('ignores stale validation and remaps errors after rules are reordered', async () => {
|
||||
vi.useFakeTimers()
|
||||
let resolve!: (response: Response) => void
|
||||
vi.mocked(api).mockReturnValueOnce(
|
||||
new Promise((done) => {
|
||||
resolve = done
|
||||
}),
|
||||
)
|
||||
const state = editor({
|
||||
rules: [
|
||||
{ pattern: '[', flags: '' },
|
||||
{ pattern: '(?<title>.+)', flags: '' },
|
||||
],
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(400)
|
||||
state.moveDown(0)
|
||||
resolve(new Response(JSON.stringify({ valid: true, diagnostics: [] })))
|
||||
await Promise.resolve()
|
||||
expect(state.canApply.value).toBe(false)
|
||||
vi.mocked(api).mockResolvedValueOnce(new Response(JSON.stringify({ valid: false, diagnostics: [{ code: 'invalid_pattern', ruleIndex: 1 }] })))
|
||||
await vi.advanceTimersByTimeAsync(400)
|
||||
expect(state.ruleErrors.value).toEqual([[], ['invalid_pattern']])
|
||||
expect(state.canApply.value).toBe(false)
|
||||
state.removeRule(1)
|
||||
expect(state.ruleErrors.value).toEqual([[]])
|
||||
expect(state.canApply.value).toBe(false)
|
||||
})
|
||||
|
||||
it('reports local bounds without sending requests and allows retry after network and worker failures', async () => {
|
||||
vi.useFakeTimers()
|
||||
const state = editor({ rules: [{ pattern: '', flags: 'gg' }] })
|
||||
await vi.advanceTimersByTimeAsync(400)
|
||||
expect(api).not.toHaveBeenCalled()
|
||||
expect(state.ruleErrors.value[0]).toEqual(['pattern_required', 'invalid_flags'])
|
||||
state.rules.value = [{ pattern: '(?<title>.+)', flags: '' }]
|
||||
vi.mocked(api).mockRejectedValueOnce(new Error('offline'))
|
||||
await vi.advanceTimersByTimeAsync(400)
|
||||
expect(state.validationError.value).toBe(true)
|
||||
expect(state.canApply.value).toBe(false)
|
||||
vi.mocked(api).mockResolvedValueOnce(new Response(JSON.stringify({ valid: false, diagnostics: [{ code: 'worker_failed' }] })))
|
||||
await state.validateRules()
|
||||
expect(state.generalDiagnostics.value).toEqual([{ code: 'worker_failed' }])
|
||||
expect(state.canApply.value).toBe(false)
|
||||
vi.mocked(api).mockResolvedValueOnce(new Response(JSON.stringify({ valid: true, diagnostics: [] })))
|
||||
await state.validateRules()
|
||||
expect(state.canApply.value).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -12,11 +12,12 @@ import { coveringFolderPath, normalizeFolderPath } from './folder-paths'
|
||||
|
||||
export { DEFAULT_FORMAT_PRIORITY }
|
||||
|
||||
export const DEFAULT_METADATA_PRECEDENCE = ['embedded', 'opfFile']
|
||||
export const DEFAULT_METADATA_PRECEDENCE = ['embedded', 'opfFile', 'regex']
|
||||
|
||||
export const METADATA_LABELS: Record<string, string> = {
|
||||
embedded: 'Embedded metadata',
|
||||
opfFile: 'OPF files',
|
||||
regex: 'Regex metadata',
|
||||
}
|
||||
|
||||
export type LibraryCreatorSectionId = 'details' | 'folders' | 'scanner' | 'metadata' | 'reading' | 'schedule' | 'fileWrite' | 'access'
|
||||
@@ -41,6 +42,7 @@ function blankForm() {
|
||||
watchLocalFolders: true,
|
||||
autoScanCronExpression: null as string | null,
|
||||
metadataPrecedence: [...DEFAULT_METADATA_PRECEDENCE],
|
||||
regexMetadata: null as Library['regexMetadata'],
|
||||
formatPriority: withReadAlongFormatPriority(DEFAULT_FORMAT_PRIORITY),
|
||||
allowedFormats: [] as string[],
|
||||
organizationMode: 'book_per_folder' as OrganizationMode,
|
||||
@@ -161,6 +163,7 @@ export function useLibraryCreator() {
|
||||
form.watchLocalFolders = library.watchLocalFolders ?? true
|
||||
form.autoScanCronExpression = library.autoScanCronExpression ?? null
|
||||
form.metadataPrecedence = [...library.metadataPrecedence]
|
||||
form.regexMetadata = library.regexMetadata ? { rules: library.regexMetadata.rules.map((rule) => ({ ...rule })) } : null
|
||||
const missing = DEFAULT_FORMAT_PRIORITY.filter((f) => !library.formatPriority.includes(f))
|
||||
form.formatPriority = withReadAlongFormatPriority([...library.formatPriority, ...missing])
|
||||
form.allowedFormats = [...library.allowedFormats]
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
import { computed, onScopeDispose, ref, watch } from 'vue'
|
||||
import { REGEX_METADATA_LIMITS, type RegexMetadataConfig, type RegexMetadataPreview, type RegexMetadataValidation } from '@bookorbit/types'
|
||||
import { api } from '@/lib/api'
|
||||
|
||||
export function useRegexMetadataEditor(config: RegexMetadataConfig | null, libraryId?: number) {
|
||||
const enabled = ref(config !== null)
|
||||
const rules = ref(config?.rules.map((rule) => ({ ...rule })) ?? [{ pattern: '', flags: '' }])
|
||||
const relativePath = ref('')
|
||||
const result = ref<RegexMetadataPreview | null>(null)
|
||||
const busy = ref(false)
|
||||
const error = ref(false)
|
||||
let sequence = 0
|
||||
let controller: AbortController | undefined
|
||||
|
||||
function invalidate() {
|
||||
sequence++
|
||||
controller?.abort()
|
||||
busy.value = false
|
||||
result.value = null
|
||||
error.value = false
|
||||
}
|
||||
watch([rules, relativePath, enabled], invalidate, { deep: true, flush: 'sync' })
|
||||
onScopeDispose(invalidate)
|
||||
|
||||
const validation = ref<RegexMetadataValidation | null>(null)
|
||||
const validating = ref(false)
|
||||
const validationError = ref(false)
|
||||
let validationSequence = 0
|
||||
let validationController: AbortController | undefined
|
||||
let validationTimer: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
const localRuleErrors = computed(() =>
|
||||
rules.value.map((rule) => {
|
||||
const errors: string[] = []
|
||||
if (!rule.pattern.length) errors.push('pattern_required')
|
||||
else if (rule.pattern.length > REGEX_METADATA_LIMITS.patternLength) errors.push('pattern_too_long')
|
||||
if (!/^(?:i?u?|ui)$/.test(rule.flags)) errors.push('invalid_flags')
|
||||
return errors
|
||||
}),
|
||||
)
|
||||
const validRuleCount = computed(() => rules.value.length > 0 && rules.value.length <= REGEX_METADATA_LIMITS.rules)
|
||||
const valid = computed(() => validRuleCount.value && localRuleErrors.value.every((errors) => !errors.length))
|
||||
const ruleErrors = computed(() =>
|
||||
localRuleErrors.value.map((errors, index) => [
|
||||
...errors,
|
||||
...(validation.value?.diagnostics.filter((diagnostic) => diagnostic.ruleIndex === index).map((diagnostic) => diagnostic.code) ?? []),
|
||||
]),
|
||||
)
|
||||
const patternInvalid = computed(() => ruleErrors.value.map((errors) => errors.some((code) => code !== 'invalid_flags')))
|
||||
const flagsInvalid = computed(() => ruleErrors.value.map((errors) => errors.includes('invalid_flags')))
|
||||
const generalDiagnostics = computed(() => validation.value?.diagnostics.filter((diagnostic) => diagnostic.ruleIndex === undefined) ?? [])
|
||||
const canApply = computed(() => !enabled.value || (valid.value && validation.value?.valid === true && !validating.value && !validationError.value))
|
||||
const canPreview = computed(() => enabled.value && valid.value && !!relativePath.value && !busy.value)
|
||||
|
||||
function invalidateValidation() {
|
||||
validationSequence++
|
||||
clearTimeout(validationTimer)
|
||||
validationController?.abort()
|
||||
validation.value = null
|
||||
validationError.value = false
|
||||
validating.value = false
|
||||
}
|
||||
|
||||
async function validateRules() {
|
||||
invalidateValidation()
|
||||
if (!enabled.value || !valid.value) return
|
||||
const requestSequence = validationSequence
|
||||
validationController = new AbortController()
|
||||
validating.value = true
|
||||
try {
|
||||
const response = await api('/api/v1/libraries/regex-metadata/validate', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ config: value(), libraryId }),
|
||||
signal: validationController.signal,
|
||||
})
|
||||
if (!response.ok) {
|
||||
if (requestSequence === validationSequence) validationError.value = true
|
||||
return
|
||||
}
|
||||
const data = (await response.json()) as RegexMetadataValidation
|
||||
if (requestSequence === validationSequence) validation.value = data
|
||||
} catch {
|
||||
if (requestSequence === validationSequence) validationError.value = true
|
||||
} finally {
|
||||
if (requestSequence === validationSequence) validating.value = false
|
||||
}
|
||||
}
|
||||
|
||||
watch(
|
||||
[rules, enabled],
|
||||
() => {
|
||||
invalidateValidation()
|
||||
if (!enabled.value || !valid.value) return
|
||||
validating.value = true
|
||||
validationTimer = setTimeout(validateRules, 400)
|
||||
},
|
||||
{ deep: true, flush: 'sync', immediate: true },
|
||||
)
|
||||
onScopeDispose(invalidateValidation)
|
||||
|
||||
function addRule() {
|
||||
if (rules.value.length < REGEX_METADATA_LIMITS.rules) rules.value.push({ pattern: '', flags: '' })
|
||||
}
|
||||
function removeRule(index: number) {
|
||||
rules.value.splice(index, 1)
|
||||
}
|
||||
function moveRule(index: number, direction: -1 | 1) {
|
||||
const target = index + direction
|
||||
if (target < 0 || target >= rules.value.length) return
|
||||
const next = [...rules.value]
|
||||
const [rule] = next.splice(index, 1)
|
||||
next.splice(target, 0, rule!)
|
||||
rules.value = next
|
||||
}
|
||||
function moveUp(index: number) {
|
||||
moveRule(index, -1)
|
||||
}
|
||||
function moveDown(index: number) {
|
||||
moveRule(index, 1)
|
||||
}
|
||||
|
||||
function value(): RegexMetadataConfig | null {
|
||||
return enabled.value ? { rules: rules.value.map((rule) => ({ ...rule })) } : null
|
||||
}
|
||||
|
||||
async function preview() {
|
||||
if (!canPreview.value) return
|
||||
invalidate()
|
||||
const requestSequence = sequence
|
||||
controller = new AbortController()
|
||||
busy.value = true
|
||||
try {
|
||||
const response = await api('/api/v1/libraries/regex-metadata/preview', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ config: value(), relativePath: relativePath.value, libraryId }),
|
||||
signal: controller.signal,
|
||||
})
|
||||
if (!response.ok) {
|
||||
if (requestSequence === sequence) error.value = true
|
||||
return
|
||||
}
|
||||
const data = (await response.json()) as RegexMetadataPreview
|
||||
if (requestSequence === sequence) result.value = data
|
||||
} catch {
|
||||
if (requestSequence === sequence) error.value = true
|
||||
} finally {
|
||||
if (requestSequence === sequence) busy.value = false
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
validation,
|
||||
validating,
|
||||
validationError,
|
||||
ruleErrors,
|
||||
patternInvalid,
|
||||
flagsInvalid,
|
||||
generalDiagnostics,
|
||||
validRuleCount,
|
||||
validateRules,
|
||||
enabled,
|
||||
rules,
|
||||
relativePath,
|
||||
result,
|
||||
busy,
|
||||
error,
|
||||
canApply,
|
||||
canPreview,
|
||||
addRule,
|
||||
removeRule,
|
||||
moveUp,
|
||||
moveDown,
|
||||
value,
|
||||
preview,
|
||||
}
|
||||
}
|
||||
@@ -10535,6 +10535,68 @@
|
||||
"opfHint": "Saved next to the book by Calibre",
|
||||
"swap": "Swap the order",
|
||||
"hint": "BookOrbit uses the first one a book has and ignores the other. Calibre libraries usually want metadata.opf first."
|
||||
},
|
||||
"regex": {
|
||||
"title": "Configure regex metadata",
|
||||
"sourceTitle": "Regex",
|
||||
"description": "Extract metadata from the primary file path relative to its library folder. The file extension is removed before matching. The first rule with usable fields wins.",
|
||||
"enable": "Enable regex metadata",
|
||||
"primaryFileHint": "For a book made of several files, such as audiobook tracks, the rules read the primary file's path.",
|
||||
"groups": "Named groups:",
|
||||
"valuesHint": "Use author for one author or authors separated by semicolons. Dates use YYYY-MM-DD. Flags: i ignores case, u enables Unicode.",
|
||||
"rules": "Ordered extraction rules",
|
||||
"rule": "Rule {number}",
|
||||
"moveUp": "Move rule up",
|
||||
"moveDown": "Move rule down",
|
||||
"remove": "Remove rule",
|
||||
"basePath": "Library folder",
|
||||
"extensionHint": "The file extension is automatically ignored. The end anchor matches the filename without its extension.",
|
||||
"flags": "Flags",
|
||||
"add": "Add rule",
|
||||
"examplePath": "Example relative file path, including extension",
|
||||
"examplePlaceholder": "Author/Series/08 - My.Book.epub",
|
||||
"exampleTitle": "Example with an author backreference",
|
||||
"exampleTestPath": "Test path (including extension)",
|
||||
"validating": "Checking rules…",
|
||||
"validationSuccess": "Rule syntax and groups are valid. Use the preview to check matches for a file path.",
|
||||
"validationError": "Could not validate the rules. Try again before applying them.",
|
||||
"validationRetry": "Check again",
|
||||
"validationRuleCount": "Add between {min} and {max} rules.",
|
||||
"preview": "Preview",
|
||||
"previewing": "Testing rules…",
|
||||
"previewError": "Could not preview the rules. Check the relative path and configuration, then try again.",
|
||||
"actualInput": "Path tested:",
|
||||
"matched": "Matched rule {number}",
|
||||
"noMatch": "No usable match. File metadata will be used.",
|
||||
"scanHint": "Changes apply to future metadata extraction for new or changed files. Saving does not start a scan or update unchanged books.",
|
||||
"apply": "Apply to library form",
|
||||
"configure": "Configure",
|
||||
"enabledHint": "Extract fields from relative paths without file extensions.",
|
||||
"disabledHint": "Disabled. Configure rules to enable.",
|
||||
"moveSourceUp": "Move {source} up",
|
||||
"moveSourceDown": "Move {source} down",
|
||||
"diagnostics": {
|
||||
"pattern_required": "Enter a regular expression.",
|
||||
"pattern_too_long": "The expression must not exceed {count} characters.",
|
||||
"invalid_flags": "Only i and u are allowed, each at most once.",
|
||||
"invalid_pattern": "Invalid regular expression.",
|
||||
"missing_group": "Include at least one supported named group.",
|
||||
"conflicting_authors": "Use either author or authors, not both.",
|
||||
"invalid_value": "An invalid field value was skipped.",
|
||||
"timeout": "The expression took too long. Simplify the pattern.",
|
||||
"busy": "All preview workers are busy. Try again shortly.",
|
||||
"worker_failed": "The expression could not be evaluated. Try again."
|
||||
},
|
||||
"precedenceHint": "The first available file source supplies metadata. Regex fields follow this order and missing values are filled from the other source.",
|
||||
"summary": "{format} first, regex metadata preferred",
|
||||
"fields": {
|
||||
"title": "Title",
|
||||
"authors": "Authors",
|
||||
"series": "Series",
|
||||
"seriesIndex": "Series number",
|
||||
"publishedYear": "Published year",
|
||||
"publishedDate": "Published date"
|
||||
}
|
||||
}
|
||||
},
|
||||
"fileWrite": {
|
||||
|
||||
@@ -87,6 +87,7 @@ export * from "./cron";
|
||||
export * from "./tts";
|
||||
export * from "./watch-download";
|
||||
export * from "./podcast";
|
||||
export * from "./regex-metadata";
|
||||
export * from "./delivery";
|
||||
export * from "./djvu";
|
||||
export * from "./email";
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { RegexMetadataConfig } from "./regex-metadata";
|
||||
import type { LibraryLastScan } from "./scanner";
|
||||
|
||||
export type OrganizationMode = "book_per_file" | "book_per_folder";
|
||||
@@ -59,6 +60,7 @@ export interface Library {
|
||||
watchLocalFolders?: boolean;
|
||||
autoScanCronExpression?: string | null;
|
||||
metadataPrecedence: string[];
|
||||
regexMetadata?: RegexMetadataConfig | null;
|
||||
formatPriority: string[];
|
||||
allowedFormats: string[];
|
||||
organizationMode: OrganizationMode;
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
export const REGEX_METADATA_LIMITS = {
|
||||
rules: 20,
|
||||
patternLength: 2048,
|
||||
pathLength: 4096,
|
||||
} as const;
|
||||
|
||||
export const REGEX_METADATA_GROUPS = ["title", "author", "authors", "series", "seriesIndex", "publishedYear", "publishedDate"] as const;
|
||||
|
||||
export interface RegexMetadataRule {
|
||||
pattern: string;
|
||||
flags: string;
|
||||
}
|
||||
|
||||
export interface RegexMetadataConfig {
|
||||
rules: RegexMetadataRule[];
|
||||
}
|
||||
|
||||
export interface RegexMetadataFields {
|
||||
title?: string;
|
||||
authors?: string[];
|
||||
seriesName?: string;
|
||||
seriesIndex?: string;
|
||||
publishedYear?: number;
|
||||
publishedDate?: string;
|
||||
}
|
||||
|
||||
export type RegexMetadataDiagnosticCode =
|
||||
"invalid_pattern" | "missing_group" | "conflicting_authors" | "invalid_value" | "timeout" | "busy" | "worker_failed";
|
||||
|
||||
export interface RegexMetadataDiagnostic {
|
||||
code: RegexMetadataDiagnosticCode;
|
||||
ruleIndex?: number;
|
||||
field?: string;
|
||||
}
|
||||
|
||||
export interface RegexMetadataPreview {
|
||||
inputPath: string;
|
||||
matched: boolean;
|
||||
ruleIndex: number | null;
|
||||
groups: Record<string, string>;
|
||||
metadata: RegexMetadataFields;
|
||||
diagnostics: RegexMetadataDiagnostic[];
|
||||
}
|
||||
|
||||
export interface RegexMetadataValidation {
|
||||
valid: boolean;
|
||||
diagnostics: RegexMetadataDiagnostic[];
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE "libraries" ADD COLUMN "regex_metadata" jsonb;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -813,6 +813,13 @@
|
||||
"when": 1792168359342,
|
||||
"tag": "0115_add_audit_resource_label",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 116,
|
||||
"version": "7",
|
||||
"when": 1792185442568,
|
||||
"tag": "0116_regex_metadata",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
uniqueIndex,
|
||||
varchar,
|
||||
} from 'drizzle-orm/pg-core';
|
||||
import { FieldPreferenceOverrides, BookMetadataFetchConfigOverride, AddedAtSource } from '@bookorbit/types';
|
||||
import { FieldPreferenceOverrides, BookMetadataFetchConfigOverride, AddedAtSource, RegexMetadataConfig } from '@bookorbit/types';
|
||||
|
||||
export const libraries = pgTable(
|
||||
'libraries',
|
||||
@@ -38,6 +38,7 @@ export const libraries = pgTable(
|
||||
.$type<string[]>()
|
||||
.notNull()
|
||||
.default(['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar']),
|
||||
regexMetadata: jsonb('regex_metadata').$type<RegexMetadataConfig>(),
|
||||
formatPriority: jsonb('format_priority')
|
||||
.$type<string[]>()
|
||||
.notNull()
|
||||
|
||||
@@ -453,12 +453,50 @@ describe('AudioMetadataEmbedder', () => {
|
||||
embedder.embedMetadata('/books/audio/book.m4b', 'm4b', { coverBytes: Buffer.from('raw-cover'), metadata: [] }),
|
||||
).rejects.toBeInstanceOf(FfmpegStalledError);
|
||||
|
||||
expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1);
|
||||
expect(mocks.writeMp4FreeformTags).not.toHaveBeenCalled();
|
||||
expect(replaceFileAtomicallyMock).not.toHaveBeenCalled();
|
||||
expect(unlinkMock).toHaveBeenCalledWith('/books/audio/.bookorbit-write-fixed-id.m4b');
|
||||
expect(unlinkMock).toHaveBeenCalledWith('/books/audio/.bookorbit-cover-fixed-id.jpg');
|
||||
});
|
||||
|
||||
it('retries an unprobed MP4 with the mp4 muxer when the ipod muxer refuses it', async () => {
|
||||
mockProbeOutput('not json');
|
||||
mocks.runFfmpeg.mockRejectedValueOnce(new Error("Tag mp4a incompatible with output codec id '86017'")).mockResolvedValueOnce(undefined);
|
||||
const embedder = new AudioMetadataEmbedder();
|
||||
|
||||
await embedder.embedMetadata('/books/audio/merged.m4b', 'm4b', { coverBytes: Buffer.from('raw-cover'), metadata: [] });
|
||||
|
||||
expect(mocks.runFfmpeg).toHaveBeenCalledTimes(2);
|
||||
expect(mocks.runFfmpeg.mock.calls[0]![1]).not.toContain('-f');
|
||||
expect(mocks.runFfmpeg.mock.calls[1]![1]).toEqual(expect.arrayContaining(['-f', 'mp4', '/books/audio/.bookorbit-write-fixed-id.m4b']));
|
||||
expect(replaceFileAtomicallyMock).toHaveBeenCalledWith('/books/audio/.bookorbit-write-fixed-id.m4b', '/books/audio/merged.m4b');
|
||||
});
|
||||
|
||||
it('reports the first ffmpeg failure when the mp4 retry fails as well', async () => {
|
||||
mockProbeOutput('not json');
|
||||
mocks.runFfmpeg.mockRejectedValueOnce(new Error('first failure')).mockRejectedValueOnce(new Error('retry failure'));
|
||||
const embedder = new AudioMetadataEmbedder();
|
||||
|
||||
await expect(embedder.embedMetadata('/books/audio/merged.m4a', 'm4a', { coverBytes: null, metadata: [] })).rejects.toThrow('first failure');
|
||||
|
||||
expect(mocks.runFfmpeg).toHaveBeenCalledTimes(2);
|
||||
expect(replaceFileAtomicallyMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['a probed MP4', 'm4b', true],
|
||||
['an MP3', 'mp3', false],
|
||||
])('does not retry a failed write of %s', async (_label, format, probed) => {
|
||||
if (probed) mockProbedStreams([{ codec_type: 'audio', codec_name: 'aac' }]);
|
||||
mocks.runFfmpeg.mockRejectedValue(new Error('ffmpeg failed'));
|
||||
const embedder = new AudioMetadataEmbedder();
|
||||
|
||||
await expect(embedder.embedMetadata(`/books/audio/book.${format}`, format, { coverBytes: null, metadata: [] })).rejects.toThrow('ffmpeg failed');
|
||||
|
||||
expect(mocks.runFfmpeg).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('preserves the ffmpeg error when cleanup also fails', async () => {
|
||||
mocks.runFfmpeg.mockRejectedValue(new Error('ffmpeg failed'));
|
||||
unlinkMock.mockRejectedValue(new Error('cleanup failed'));
|
||||
|
||||
@@ -8,13 +8,14 @@ import { promisify } from 'util';
|
||||
|
||||
import { FORMAT_M4A, FORMAT_M4B, FORMAT_MP3 } from '../../file-write.constants';
|
||||
import { replaceFileAtomically } from '../shared/atomic-file-replace';
|
||||
import { runFfmpeg } from './ffmpeg-runner';
|
||||
import { FfmpegStalledError, runFfmpeg } from './ffmpeg-runner';
|
||||
import { MP4_NATIVE_TAG_KEYS, writeMp4FreeformTags, type Mp4FreeformTag } from './mp4-freeform-tags';
|
||||
|
||||
const execFile = promisify(execFileCallback);
|
||||
const FFPROBE_TIMEOUT_MS = 30_000;
|
||||
/** The only audio FFmpeg's ipod muxer accepts; an MP4 can also carry MP3, MP2, Opus, FLAC and E-AC-3. */
|
||||
const IPOD_MUXER_AUDIO_CODECS = new Set(['aac', 'alac', 'ac3']);
|
||||
const MP4_MUXER = 'mp4';
|
||||
|
||||
interface ProbedStream {
|
||||
codecType: string | null;
|
||||
@@ -53,7 +54,18 @@ export class AudioMetadataEmbedder {
|
||||
const streams = isMp4Format(format) ? await probeStreams(filePath) : null;
|
||||
const useMp4MetadataTags = shouldUseMp4MetadataTags(format, coverPath, streams);
|
||||
const muxer = outputMuxer(format, streams);
|
||||
await runFfmpeg(resolveFfmpegPath(), buildFfmpegArgs(filePath, coverPath, tempPath, format, input.metadata, { useMp4MetadataTags, muxer }));
|
||||
const ffmpegArgs = (outputFormat: string | undefined) =>
|
||||
buildFfmpegArgs(filePath, coverPath, tempPath, format, input.metadata, { useMp4MetadataTags, muxer: outputFormat });
|
||||
try {
|
||||
await runFfmpeg(resolveFfmpegPath(), ffmpegArgs(muxer));
|
||||
} catch (error) {
|
||||
// Unprobed, the file may hold audio the ipod muxer refuses, such as MP3; the mp4 muxer takes any of
|
||||
// it. A stalled write is not a muxer refusal, and a second attempt would only double the wait.
|
||||
if (!isMp4Format(format) || streams || error instanceof FfmpegStalledError) throw error;
|
||||
await runFfmpeg(resolveFfmpegPath(), ffmpegArgs(MP4_MUXER)).catch(() => {
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
if (isMp4Format(format) && !useMp4MetadataTags) {
|
||||
await writeMp4FreeformTags(tempPath, mp4FreeformTags(input.metadata), { carryOverFrom: filePath });
|
||||
}
|
||||
@@ -152,7 +164,7 @@ function shouldUseMp4MetadataTags(format: string, coverPath: string | null, stre
|
||||
function outputMuxer(format: string, streams: ProbedStream[] | null): string | undefined {
|
||||
if (!isMp4Format(format) || !streams) return undefined;
|
||||
const ipodRejectsAudio = streams.some((stream) => stream.codecType === 'audio' && !IPOD_MUXER_AUDIO_CODECS.has(stream.codecName ?? ''));
|
||||
return ipodRejectsAudio ? 'mp4' : undefined;
|
||||
return ipodRejectsAudio ? MP4_MUXER : undefined;
|
||||
}
|
||||
|
||||
async function probeStreams(filePath: string): Promise<ProbedStream[] | null> {
|
||||
|
||||
@@ -405,6 +405,29 @@ describe('audio metadata round-trip', () => {
|
||||
expect(await probeFormatTags(filePath)).toEqual(expect.objectContaining({ title: 'Tag Only', subtitle: 'Tag Only Subtitle' }));
|
||||
});
|
||||
|
||||
it('takes new tags and a replacement cover even when ffprobe cannot read it', async () => {
|
||||
const filePath = join(tempDir, `merged.${format}`);
|
||||
const coverPath = join(tempDir, 'cover.jpg');
|
||||
await generateCoverFixture(coverPath);
|
||||
await generateChapteredAudioFixture(filePath, coverPath, 'libmp3lame');
|
||||
vi.stubEnv('FFPROBE_PATH', join(tempDir, 'missing-ffprobe'));
|
||||
|
||||
const writer = new Writer(new AudioMetadataEmbedder());
|
||||
await expect(
|
||||
writer.write(
|
||||
filePath,
|
||||
{ title: 'Unprobed Title', coverBytes: await readFile(coverPath) },
|
||||
{ dryRun: false, fieldMask: createBookWriteFieldMask() },
|
||||
),
|
||||
).resolves.toEqual(expect.objectContaining({ status: 'success' }));
|
||||
|
||||
const streams = await probeStreams(filePath);
|
||||
expect(streams.find((stream) => stream.codec_type === 'audio')?.codec_name).toBe('mp3');
|
||||
expect(streams.find((stream) => stream.codec_type === 'video')?.disposition?.attached_pic).toBe(1);
|
||||
expect((await probeFormatTags(filePath)).title).toBe('Unprobed Title');
|
||||
expect(await probeChapterTitles(filePath)).toEqual(['One', 'Two']);
|
||||
});
|
||||
|
||||
it('takes new tags when it carries no art at all', async () => {
|
||||
const filePath = join(tempDir, `bare.${format}`);
|
||||
await execFile(ffmpegPath, [
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { RegexMetadataConfigDto } from './regex-metadata.dto';
|
||||
import { Type } from 'class-transformer';
|
||||
import { IsObject, ValidateNested } from 'class-validator';
|
||||
import {
|
||||
ArrayMinSize,
|
||||
IsDefined,
|
||||
@@ -94,6 +97,12 @@ export class CreateLibraryDto {
|
||||
@IsString({ each: true })
|
||||
metadataPrecedence?: string[];
|
||||
|
||||
@IsOptional()
|
||||
@IsObject()
|
||||
@ValidateNested()
|
||||
@Type(() => RegexMetadataConfigDto)
|
||||
regexMetadata?: RegexMetadataConfigDto | null;
|
||||
|
||||
@IsOptionalNotNull()
|
||||
@IsArray()
|
||||
@IsString({ each: true })
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import { Type } from 'class-transformer';
|
||||
import {
|
||||
ArrayMaxSize,
|
||||
ArrayMinSize,
|
||||
IsArray,
|
||||
IsDefined,
|
||||
IsInt,
|
||||
IsObject,
|
||||
IsOptional,
|
||||
IsString,
|
||||
Matches,
|
||||
MaxLength,
|
||||
Min,
|
||||
MinLength,
|
||||
ValidateNested,
|
||||
} from 'class-validator';
|
||||
import { REGEX_METADATA_LIMITS, type RegexMetadataConfig, type RegexMetadataRule } from '@bookorbit/types';
|
||||
|
||||
export class RegexMetadataRuleDto implements RegexMetadataRule {
|
||||
@IsString()
|
||||
@MinLength(1)
|
||||
@MaxLength(REGEX_METADATA_LIMITS.patternLength)
|
||||
pattern: string;
|
||||
|
||||
@IsString()
|
||||
@Matches(/^(?:i?u?|ui)$/)
|
||||
flags: string;
|
||||
}
|
||||
|
||||
export class RegexMetadataConfigDto implements RegexMetadataConfig {
|
||||
@IsArray()
|
||||
@ArrayMinSize(1)
|
||||
@ArrayMaxSize(REGEX_METADATA_LIMITS.rules)
|
||||
@ValidateNested({ each: true })
|
||||
@Type(() => RegexMetadataRuleDto)
|
||||
rules: RegexMetadataRuleDto[];
|
||||
}
|
||||
|
||||
export class RegexMetadataValidationDto {
|
||||
@IsDefined()
|
||||
@IsObject()
|
||||
@ValidateNested()
|
||||
@Type(() => RegexMetadataConfigDto)
|
||||
config: RegexMetadataConfigDto;
|
||||
|
||||
@IsOptional()
|
||||
@IsInt()
|
||||
@Min(1)
|
||||
libraryId?: number;
|
||||
}
|
||||
|
||||
export class RegexMetadataPreviewDto extends RegexMetadataValidationDto {
|
||||
@IsString()
|
||||
@MinLength(1)
|
||||
@MaxLength(REGEX_METADATA_LIMITS.pathLength)
|
||||
relativePath: string;
|
||||
}
|
||||
@@ -1,3 +1,6 @@
|
||||
import { RegexMetadataConfigDto } from './regex-metadata.dto';
|
||||
import { Type } from 'class-transformer';
|
||||
import { IsObject, ValidateNested } from 'class-validator';
|
||||
import {
|
||||
ArrayMinSize,
|
||||
IsArray,
|
||||
@@ -91,6 +94,12 @@ export class UpdateLibraryDto {
|
||||
@IsString({ each: true })
|
||||
metadataPrecedence?: string[];
|
||||
|
||||
@IsOptional()
|
||||
@IsObject()
|
||||
@ValidateNested()
|
||||
@Type(() => RegexMetadataConfigDto)
|
||||
regexMetadata?: RegexMetadataConfigDto | null;
|
||||
|
||||
@IsOptionalNotNull()
|
||||
@IsArray()
|
||||
@IsString({ each: true })
|
||||
|
||||
@@ -18,7 +18,7 @@ export const LIBRARY_MARK_AS_FINISHED_MIN = 90;
|
||||
export const LIBRARY_MARK_AS_FINISHED_MAX = 100;
|
||||
|
||||
export const LIBRARY_BOOK_STATUS_PRESENT = 'present';
|
||||
export const LIBRARY_METADATA_PRECEDENCE_DEFAULT = ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar'] as const;
|
||||
export const LIBRARY_METADATA_PRECEDENCE_DEFAULT = ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar', 'regex'] as const;
|
||||
|
||||
export const LIBRARY_FILE_WRITE_MAX_SIZE_MB_MIN = 1;
|
||||
export const LIBRARY_FILE_WRITE_MAX_SIZE_MB_MAX = 10000;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { RegexMetadataPreviewDto, RegexMetadataValidationDto } from './dto/regex-metadata.dto';
|
||||
import { Body, Controller, Delete, Get, HttpCode, HttpStatus, Param, ParseIntPipe, Patch, Post, Query, Res } from '@nestjs/common';
|
||||
import type { FastifyReply } from 'fastify';
|
||||
|
||||
@@ -48,6 +49,20 @@ export class LibraryController {
|
||||
return this.libraryService.getOverview(user);
|
||||
}
|
||||
|
||||
@Post('regex-metadata/validate')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@RequirePermission(Permission.ManageLibraries)
|
||||
validateRegexMetadata(@Body() dto: RegexMetadataValidationDto) {
|
||||
return this.libraryService.validateRegexMetadata(dto);
|
||||
}
|
||||
|
||||
@Post('regex-metadata/preview')
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@RequirePermission(Permission.ManageLibraries)
|
||||
previewRegexMetadata(@Body() dto: RegexMetadataPreviewDto) {
|
||||
return this.libraryService.previewRegexMetadata(dto);
|
||||
}
|
||||
|
||||
@Get(':id')
|
||||
@RequireLibraryAccess('viewer')
|
||||
findOne(@Param('id', ParseIntPipe) id: number) {
|
||||
|
||||
@@ -336,3 +336,23 @@ describe('Library DTO validation', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('regex metadata DTO validation', () => {
|
||||
it('accepts a nullable config and validates nested rules with the global whitelist', async () => {
|
||||
for (const regexMetadata of [null, { rules: [{ pattern: '(?<title>.+)', flags: 'iu' }] }]) {
|
||||
const dto = plainToInstance(UpdateLibraryDto, { regexMetadata });
|
||||
expect(await validate(dto, { whitelist: true, forbidNonWhitelisted: true })).toEqual([]);
|
||||
}
|
||||
});
|
||||
it.each([
|
||||
{ rules: [] },
|
||||
{ rules: [{ pattern: '(?<title>.+)', flags: 'g' }] },
|
||||
{ rules: [{ pattern: '(?<title>.+)', flags: '', template: '{title}' }] },
|
||||
{ rules: [null] },
|
||||
{ rules: 'invalid' },
|
||||
[],
|
||||
])('rejects invalid nested configuration %j', async (regexMetadata) => {
|
||||
const dto = plainToInstance(UpdateLibraryDto, { regexMetadata });
|
||||
expect((await validate(dto, { whitelist: true, forbidNonWhitelisted: true })).length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { MetadataModule } from '../metadata/metadata.module';
|
||||
import { ReadingAttemptEventsModule } from '../user-book-status/reading-attempt-events.module';
|
||||
import { Module, forwardRef } from '@nestjs/common';
|
||||
|
||||
@@ -19,6 +20,7 @@ import { LibraryAddedAtService } from './library-added-at.service';
|
||||
imports: [
|
||||
ReadingAttemptEventsModule,
|
||||
ScannerModule,
|
||||
MetadataModule,
|
||||
AchievementModule,
|
||||
forwardRef(() => BookModule),
|
||||
FileWriteModule,
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import 'reflect-metadata';
|
||||
import { plainToInstance } from 'class-transformer';
|
||||
import { UpdateLibraryDto } from './dto/update-library.dto';
|
||||
import { ReadingAttemptEventsService } from '../user-book-status/reading-attempt-events.service';
|
||||
vi.mock('fs/promises', () => ({
|
||||
readdir: vi.fn(),
|
||||
@@ -96,6 +99,7 @@ describe('LibraryService', () => {
|
||||
removeSchedule: vi.fn(),
|
||||
};
|
||||
|
||||
const regexMetadata = { validate: vi.fn() };
|
||||
let service: LibraryService;
|
||||
let readingEvents: ReadingAttemptEventsService;
|
||||
|
||||
@@ -112,6 +116,7 @@ describe('LibraryService', () => {
|
||||
achievementEvents as any,
|
||||
pathPolicy as any,
|
||||
scanScheduler as any,
|
||||
regexMetadata as any,
|
||||
readingEvents,
|
||||
);
|
||||
|
||||
@@ -202,6 +207,53 @@ describe('LibraryService', () => {
|
||||
await expect(service.findOne(111)).rejects.toBeInstanceOf(NotFoundException);
|
||||
});
|
||||
|
||||
it('rejects regex validation for podcast libraries', async () => {
|
||||
libraryRepo.findById.mockResolvedValue([{ id: 2, type: 'podcasts' }]);
|
||||
libraryRepo.findFoldersByLibrary.mockResolvedValue([]);
|
||||
await expect(service.validateRegexMetadata({ config: { rules: [{ pattern: '(?<title>.+)', flags: '' }] }, libraryId: 2 })).rejects.toThrow(
|
||||
'Regex metadata is only available for book libraries',
|
||||
);
|
||||
});
|
||||
|
||||
describe('regex configuration updates', () => {
|
||||
const first = { pattern: '(?<title>.+)', flags: 'i' };
|
||||
const second = { pattern: '(?<series>.+)', flags: '' };
|
||||
const config = { rules: [first, second] };
|
||||
beforeEach(() => {
|
||||
libraryRepo.findById.mockResolvedValue([{ id: 2, type: 'books', icon: 'BookOpen', regexMetadata: config }]);
|
||||
libraryRepo.update.mockResolvedValue([{ id: 2 }]);
|
||||
libraryRepo.findFoldersByLibrary.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
it.each([null, { rules: [{ flags: 'i', pattern: first.pattern }, second] }])(
|
||||
'does not revalidate or rewrite unchanged rules: %j',
|
||||
async (unchanged) => {
|
||||
libraryRepo.findById.mockResolvedValue([{ id: 2, type: 'books', icon: 'BookOpen', regexMetadata: unchanged === null ? null : config }]);
|
||||
await service.update(2, plainToInstance(UpdateLibraryDto, { coverAspectRatio: '1/1', regexMetadata: unchanged }));
|
||||
expect(regexMetadata.validate).not.toHaveBeenCalled();
|
||||
expect(libraryRepo.update).toHaveBeenCalledWith(2, { coverAspectRatio: '1/1' });
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
null,
|
||||
{ rules: [second, first] },
|
||||
{ rules: [{ ...first, flags: 'u' }, second] },
|
||||
{ rules: [{ ...first, pattern: '(?<title>.*)' }, second] },
|
||||
])('validates and persists an actual change without a library grant: %j', async (changed) => {
|
||||
await service.update(2, { regexMetadata: changed });
|
||||
expect(libraryRepo.hasUserAccess).not.toHaveBeenCalled();
|
||||
expect(regexMetadata.validate).toHaveBeenCalledWith(changed);
|
||||
expect(libraryRepo.update).toHaveBeenCalledWith(2, { regexMetadata: changed });
|
||||
});
|
||||
|
||||
it('does not persist a change that fails validation', async () => {
|
||||
regexMetadata.validate.mockRejectedValueOnce(new BadRequestException('Invalid regex metadata configuration'));
|
||||
await expect(service.update(2, { regexMetadata: null })).rejects.toThrow('Invalid regex metadata configuration');
|
||||
expect(libraryRepo.update).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it('verifyUserAccess bypasses lookup for superusers', async () => {
|
||||
await service.verifyUserAccess(1, 2, true);
|
||||
expect(libraryRepo.hasUserAccess).not.toHaveBeenCalled();
|
||||
@@ -219,6 +271,20 @@ describe('LibraryService', () => {
|
||||
await expect(service.verifyUserAccessLevel(1, 2, true, 'owner')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([undefined, ['regex', 'embedded']])('preserves regex configuration with precedence %j on create', async (metadataPrecedence) => {
|
||||
const config = { rules: [{ pattern: '(?<title>.+)', flags: '' }] };
|
||||
libraryRepo.findByName.mockResolvedValue([]);
|
||||
libraryRepo.insert.mockResolvedValue([{ id: 5, type: 'books', name: 'Regex', icon: 'BookOpen' }]);
|
||||
libraryRepo.insertFolders.mockResolvedValue([]);
|
||||
await service.create({ name: 'Regex', icon: 'BookOpen', folders: ['/a'], regexMetadata: config, metadataPrecedence });
|
||||
expect(libraryRepo.insert).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
regexMetadata: config,
|
||||
metadataPrecedence: metadataPrecedence ?? ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar', 'regex'],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('create applies defaults, inserts folders, and starts an async scan', async () => {
|
||||
libraryRepo.findByName.mockResolvedValue([]);
|
||||
libraryRepo.insert.mockResolvedValue([{ id: 5, type: 'books', name: 'Sci-Fi', icon: 'BookOpen' }]);
|
||||
@@ -235,7 +301,7 @@ describe('LibraryService', () => {
|
||||
icon: 'BookOpen',
|
||||
displayOrder: 0,
|
||||
watch: false,
|
||||
metadataPrecedence: ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar'],
|
||||
metadataPrecedence: ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar', 'regex'],
|
||||
formatPriority: [
|
||||
'epub',
|
||||
'kepub',
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { isDeepStrictEqual } from 'node:util';
|
||||
import { RegexMetadataService } from '../metadata/regex/regex-metadata.service';
|
||||
import { RegexMetadataPreviewDto, RegexMetadataValidationDto } from './dto/regex-metadata.dto';
|
||||
import { ReadingAttemptEventsService } from '../user-book-status/reading-attempt-events.service';
|
||||
import {
|
||||
BadRequestException,
|
||||
@@ -13,7 +16,14 @@ import { readdir, realpath, rm, stat } from 'fs/promises';
|
||||
import { dirname, isAbsolute, join, relative } from 'path';
|
||||
|
||||
import { APP_FEATURES, DEFAULT_FORMAT_PRIORITY } from '@bookorbit/types';
|
||||
import type { AccessLevel, LibraryFileSyncProgressEvent, LibraryOverviewEntry, OrganizationMode, WriteResult } from '@bookorbit/types';
|
||||
import type {
|
||||
AccessLevel,
|
||||
LibraryFileSyncProgressEvent,
|
||||
LibraryOverviewEntry,
|
||||
OrganizationMode,
|
||||
RegexMetadataConfig,
|
||||
WriteResult,
|
||||
} from '@bookorbit/types';
|
||||
import { podcastArtworkDirPath } from '../../common/podcast-artwork-storage';
|
||||
import { podcastFeedSnapshotPath } from '../../common/podcast-feed-snapshot-storage';
|
||||
import { sanitizeLogValue } from '../../common/utils/log-sanitize.utils';
|
||||
@@ -59,6 +69,7 @@ const BOOK_ONLY_LIBRARY_FIELDS = [
|
||||
'watch',
|
||||
'autoScanCronExpression',
|
||||
'metadataPrecedence',
|
||||
'regexMetadata',
|
||||
'formatPriority',
|
||||
'allowedFormats',
|
||||
'organizationMode',
|
||||
@@ -102,6 +113,7 @@ export class LibraryService {
|
||||
private readonly achievementEvents: AchievementEventsService,
|
||||
private readonly pathPolicy: PathPolicyService,
|
||||
private readonly scanScheduler: LibraryScanSchedulerService,
|
||||
private readonly regexMetadata: RegexMetadataService,
|
||||
private readonly readingAttemptEvents: ReadingAttemptEventsService,
|
||||
) {
|
||||
this.appDataPath = this.config.get<string>('storage.appDataPath')!;
|
||||
@@ -173,6 +185,22 @@ export class LibraryService {
|
||||
return { ...normalizeLibraryOrganizationMode(library), folders };
|
||||
}
|
||||
|
||||
private async assertRegexMetadataLibrary(libraryId: number | undefined) {
|
||||
if (libraryId === undefined) return;
|
||||
const library = await this.findOne(libraryId);
|
||||
if (library.type !== 'books') throw new BadRequestException('Regex metadata is only available for book libraries');
|
||||
}
|
||||
|
||||
async validateRegexMetadata(dto: RegexMetadataValidationDto) {
|
||||
await this.assertRegexMetadataLibrary(dto.libraryId);
|
||||
return this.regexMetadata.check(dto.config);
|
||||
}
|
||||
|
||||
async previewRegexMetadata(dto: RegexMetadataPreviewDto) {
|
||||
await this.assertRegexMetadataLibrary(dto.libraryId);
|
||||
return this.regexMetadata.preview(dto.config, dto.relativePath);
|
||||
}
|
||||
|
||||
async create(dto: CreateLibraryDto) {
|
||||
const libraryType = dto.type ?? 'books';
|
||||
if (libraryType === 'podcasts' && !APP_FEATURES.podcasts) {
|
||||
@@ -180,6 +208,7 @@ export class LibraryService {
|
||||
}
|
||||
if (libraryType === 'podcasts') this.assertNoBookOnlyFields(dto);
|
||||
else this.assertNoPodcastOnlyFields(dto);
|
||||
await this.regexMetadata.validate(dto.regexMetadata);
|
||||
await this.assertNameAvailable(dto.name);
|
||||
const folderInputs = resolveLibraryFolderRoles(
|
||||
libraryType,
|
||||
@@ -199,6 +228,7 @@ export class LibraryService {
|
||||
watch: libraryType === 'books' ? (dto.watch ?? false) : false,
|
||||
watchLocalFolders: libraryType === 'podcasts' ? (dto.watchLocalFolders ?? true) : false,
|
||||
autoScanCronExpression: libraryType === 'books' ? (dto.autoScanCronExpression ?? null) : null,
|
||||
regexMetadata: dto.regexMetadata ?? null,
|
||||
metadataPrecedence: libraryType === 'books' ? (dto.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT]) : [],
|
||||
formatPriority: libraryType === 'books' ? (dto.formatPriority ?? [...DEFAULT_FORMAT_PRIORITY]) : [],
|
||||
allowedFormats: libraryType === 'books' ? (dto.allowedFormats ?? []) : [],
|
||||
@@ -252,6 +282,11 @@ export class LibraryService {
|
||||
if (existing.type === 'podcasts') this.assertNoBookOnlyFields(dto);
|
||||
else this.assertNoPodcastOnlyFields(dto);
|
||||
|
||||
const regexRules = (config: RegexMetadataConfig | null | undefined) => config?.rules.map(({ pattern, flags }) => [pattern, flags]) ?? null;
|
||||
const regexMetadataChanged =
|
||||
dto.regexMetadata !== undefined && !isDeepStrictEqual(regexRules(dto.regexMetadata), regexRules(existing.regexMetadata));
|
||||
if (regexMetadataChanged) await this.regexMetadata.validate(dto.regexMetadata);
|
||||
|
||||
const existingOrganizationMode = normalizeOrganizationMode(existing.organizationMode);
|
||||
if (dto.organizationMode !== undefined && dto.organizationMode !== existingOrganizationMode) {
|
||||
throw new BadRequestException(
|
||||
@@ -264,6 +299,7 @@ export class LibraryService {
|
||||
}
|
||||
|
||||
const { folders: rawFolderPaths, localFolders: rawLocalFolderPaths, ...fields } = dto;
|
||||
if (!regexMetadataChanged) delete fields.regexMetadata;
|
||||
let existingFolders: Awaited<ReturnType<LibraryRepository['findFoldersByLibrary']>> | undefined;
|
||||
let folderInputs: LibraryFolderInput[] | undefined;
|
||||
if (rawFolderPaths !== undefined || rawLocalFolderPaths !== undefined) {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { RegexMetadataService } from './regex/regex-metadata.service';
|
||||
import { Module } from '@nestjs/common';
|
||||
import { DjvuDocumentModule } from '../djvu/djvu-document.module';
|
||||
|
||||
@@ -14,7 +15,7 @@ import { MetadataService } from './metadata.service';
|
||||
|
||||
@Module({
|
||||
imports: [BookMetadataLockModule, BookCoverStoreModule, EmbeddingModule, MetadataScoreModule, NarratorModule, DjvuDocumentModule],
|
||||
providers: [MetadataService, MetadataExtractionService, MetadataEventsService, ComicMetadataRepository, CoverSlotReconciler],
|
||||
exports: [MetadataService, MetadataExtractionService, MetadataEventsService, ComicMetadataRepository, CoverSlotReconciler],
|
||||
providers: [RegexMetadataService, MetadataService, MetadataExtractionService, MetadataEventsService, ComicMetadataRepository, CoverSlotReconciler],
|
||||
exports: [RegexMetadataService, MetadataService, MetadataExtractionService, MetadataEventsService, ComicMetadataRepository, CoverSlotReconciler],
|
||||
})
|
||||
export class MetadataModule {}
|
||||
|
||||
@@ -270,6 +270,202 @@ describe('MetadataService', () => {
|
||||
mockExtractEpubCover.mockResolvedValueOnce(cover);
|
||||
}
|
||||
|
||||
describe('regex extraction persistence', () => {
|
||||
describe.each(['epub', 'm4b'])('%s relations', (format) => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it.each([
|
||||
{ label: 'neither relation', authors: [], genres: [] },
|
||||
{ label: 'only authors', authors: [{ name: 'File Author', sortName: null }], genres: [] },
|
||||
{ label: 'only genres', authors: [], genres: ['Mystery'] },
|
||||
])('preserves omitted relations when the file supplies $label', async ({ authors, genres }) => {
|
||||
const { db, updateSet } = makeDb();
|
||||
const service = makeService(db);
|
||||
vi.spyOn(MetadataExtractionService.prototype, 'extract').mockResolvedValueOnce({ title: 'File title', authors, genres, cover: null });
|
||||
const replaceAuthors = vi.spyOn(service, 'replaceAuthors').mockResolvedValue(undefined);
|
||||
const replaceGenres = vi.spyOn(service, 'replaceGenres').mockResolvedValue(undefined);
|
||||
await service.extractAndSaveWithRegex(
|
||||
9,
|
||||
[{ key: 'embedded', absolutePath: `/books/file.${format}`, format }],
|
||||
{ seriesName: 'Path Series' },
|
||||
['regex', 'embedded'],
|
||||
);
|
||||
expect(updateSet).toHaveBeenCalledWith(expect.objectContaining({ title: 'File title', seriesName: 'Path Series' }));
|
||||
if (authors.length) expect(replaceAuthors).toHaveBeenCalledWith(9, authors);
|
||||
else expect(replaceAuthors).not.toHaveBeenCalled();
|
||||
if (genres.length) expect(replaceGenres).toHaveBeenCalledWith(9, genres);
|
||||
else expect(replaceGenres).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ fileAuthors: [], precedence: ['embedded', 'regex'], expected: 'Regex Author' },
|
||||
{ fileAuthors: [{ name: 'File Author', sortName: null }], precedence: ['regex', 'embedded'], expected: 'Regex Author' },
|
||||
{ fileAuthors: [{ name: 'File Author', sortName: null }], precedence: ['embedded', 'regex'], expected: 'File Author' },
|
||||
])('merges authors using $precedence with $fileAuthors', async ({ fileAuthors, precedence, expected }) => {
|
||||
const { db } = makeDb();
|
||||
const service = makeService(db);
|
||||
vi.spyOn(MetadataExtractionService.prototype, 'extract').mockResolvedValueOnce({ authors: fileAuthors, genres: [], cover: null });
|
||||
const replaceAuthors = vi.spyOn(service, 'replaceAuthors').mockResolvedValue(undefined);
|
||||
const replaceGenres = vi.spyOn(service, 'replaceGenres').mockResolvedValue(undefined);
|
||||
await service.extractAndSaveWithRegex(
|
||||
9,
|
||||
[{ key: 'embedded', absolutePath: `/books/file.${format}`, format }],
|
||||
{ authors: ['Regex Author'] },
|
||||
precedence,
|
||||
);
|
||||
expect(replaceAuthors).toHaveBeenCalledWith(9, [{ name: expected, sortName: null }]);
|
||||
expect(replaceGenres).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('respects relation locks after merging regex and file metadata', async () => {
|
||||
const { db } = makeDb();
|
||||
const filter = vi.fn().mockImplementation((_bookId, dto) =>
|
||||
Promise.resolve({
|
||||
dto: { ...dto, authors: undefined, genres: undefined },
|
||||
skippedFields: ['authors', 'genres'],
|
||||
}),
|
||||
);
|
||||
const service = makeService(db, undefined, { bookMetadataLockService: { isFieldLocked: vi.fn(), filterAutomatedBookUpdate: filter } });
|
||||
vi.spyOn(MetadataExtractionService.prototype, 'extract').mockResolvedValueOnce({ authors: [], genres: ['Mystery'], cover: null });
|
||||
const replaceAuthors = vi.spyOn(service, 'replaceAuthors');
|
||||
const replaceGenres = vi.spyOn(service, 'replaceGenres');
|
||||
await service.extractAndSaveWithRegex(
|
||||
9,
|
||||
[{ key: 'embedded', absolutePath: `/books/file.${format}`, format }],
|
||||
{ authors: ['Regex Author'] },
|
||||
['regex', 'embedded'],
|
||||
);
|
||||
expect(filter).toHaveBeenCalledWith(9, expect.objectContaining({ authors: ['Regex Author'], genres: ['Mystery'] }));
|
||||
expect(replaceAuthors).not.toHaveBeenCalled();
|
||||
expect(replaceGenres).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('retains empty relation replacement for file-only extraction', async () => {
|
||||
const { db } = makeDb();
|
||||
const service = makeService(db);
|
||||
vi.spyOn(MetadataExtractionService.prototype, 'extract').mockResolvedValueOnce({ authors: [], genres: [], cover: null });
|
||||
const replaceAuthors = vi.spyOn(service, 'replaceAuthors');
|
||||
const replaceGenres = vi.spyOn(service, 'replaceGenres');
|
||||
await service.extractAndSave(9, `/books/file.${format}`, format);
|
||||
expect(replaceAuthors).toHaveBeenCalledWith(9, []);
|
||||
expect(replaceGenres).toHaveBeenCalledWith(9, []);
|
||||
expect(db.delete).toHaveBeenCalledWith(bookAuthors);
|
||||
expect(db.delete).toHaveBeenCalledWith(bookGenres);
|
||||
});
|
||||
});
|
||||
|
||||
describe('file extraction failures', () => {
|
||||
const sources = [
|
||||
{ key: 'embedded', absolutePath: '/books/file.epub', format: 'epub' },
|
||||
{ key: 'opfFile', absolutePath: '/books/metadata.opf', format: 'opf' },
|
||||
];
|
||||
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it('continues to the next file source and respects its precedence over regex', async () => {
|
||||
const { db, updateSet } = makeDb();
|
||||
const service = makeService(db);
|
||||
const extraction = vi
|
||||
.spyOn(MetadataExtractionService.prototype, 'extract')
|
||||
.mockRejectedValueOnce(new Error('unreadable file'))
|
||||
.mockResolvedValueOnce({ title: 'OPF title', authors: [{ name: 'OPF Author', sortName: null }], genres: [], cover: null });
|
||||
const replaceAuthors = vi.spyOn(service, 'replaceAuthors').mockResolvedValue(undefined);
|
||||
|
||||
await service.extractAndSaveWithRegex(9, sources, { title: 'Regex title', seriesName: 'Path Series' }, ['embedded', 'opfFile', 'regex']);
|
||||
|
||||
expect(extraction).toHaveBeenNthCalledWith(1, '/books/file.epub', 'epub');
|
||||
expect(extraction).toHaveBeenNthCalledWith(2, '/books/metadata.opf', 'opf');
|
||||
expect(updateSet).toHaveBeenCalledWith(expect.objectContaining({ title: 'OPF title', seriesName: 'Path Series' }));
|
||||
expect(replaceAuthors).toHaveBeenCalledWith(9, [{ name: 'OPF Author', sortName: null }]);
|
||||
});
|
||||
|
||||
it('saves regex metadata and preserves omitted relations when every file source rejects', async () => {
|
||||
const { db, updateSet } = makeDb();
|
||||
const scoreService = { calculateAndSave: vi.fn().mockResolvedValue(undefined) };
|
||||
const service = makeService(db, undefined, { scoreService });
|
||||
const extraction = vi.spyOn(MetadataExtractionService.prototype, 'extract').mockRejectedValue(new Error('unreadable file'));
|
||||
const replaceAuthors = vi.spyOn(service, 'replaceAuthors');
|
||||
const replaceGenres = vi.spyOn(service, 'replaceGenres');
|
||||
|
||||
await service.extractAndSaveWithRegex(9, sources, { title: 'Regex title', seriesName: 'Path Series' }, ['embedded', 'regex', 'opfFile']);
|
||||
|
||||
expect(extraction).toHaveBeenCalledTimes(2);
|
||||
expect(updateSet).toHaveBeenCalledWith(expect.objectContaining({ title: 'Regex title', seriesName: 'Path Series' }));
|
||||
expect(replaceAuthors).not.toHaveBeenCalled();
|
||||
expect(replaceGenres).not.toHaveBeenCalled();
|
||||
expect(scoreService.calculateAndSave).toHaveBeenCalledWith(9);
|
||||
});
|
||||
|
||||
it('propagates persistence failures without trying another file source or retrying the write', async () => {
|
||||
const { db, updateSet, updateWhere } = makeDb();
|
||||
updateWhere.mockRejectedValueOnce(new Error('database unavailable'));
|
||||
const scoreService = { calculateAndSave: vi.fn() };
|
||||
const service = makeService(db, undefined, { scoreService });
|
||||
const extraction = vi
|
||||
.spyOn(MetadataExtractionService.prototype, 'extract')
|
||||
.mockResolvedValue({ title: 'File title', authors: [], genres: [], cover: null });
|
||||
|
||||
await expect(service.extractAndSaveWithRegex(9, sources, { seriesName: 'Path Series' }, ['regex', 'embedded', 'opfFile'])).rejects.toThrow(
|
||||
'database unavailable',
|
||||
);
|
||||
|
||||
expect(extraction).toHaveBeenCalledTimes(1);
|
||||
expect(updateSet).toHaveBeenCalledTimes(1);
|
||||
expect(scoreService.calculateAndSave).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it('reports scoring failure after saving regex fields without rolling them back or retrying', async () => {
|
||||
const { db, updateSet } = makeDb();
|
||||
const scoreService = { calculateAndSave: vi.fn().mockRejectedValue(new Error('score storage unavailable')) };
|
||||
const service = makeService(db, undefined, { scoreService });
|
||||
await expect(service.extractAndSaveWithRegex(9, [], { title: 'Regex title' }, ['regex', 'embedded'])).rejects.toThrow(
|
||||
'score storage unavailable',
|
||||
);
|
||||
expect(updateSet).toHaveBeenCalledWith(expect.objectContaining({ title: 'Regex title' }));
|
||||
expect(scoreService.calculateAndSave).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('preserves existing relations when regex is the only source and supplies no authors', async () => {
|
||||
const { db, updateSet } = makeDb();
|
||||
const service = makeService(db);
|
||||
const replaceAuthors = vi.spyOn(service, 'replaceAuthors');
|
||||
const replaceGenres = vi.spyOn(service, 'replaceGenres');
|
||||
await service.extractAndSaveWithRegex(9, [], { seriesName: 'Path Series' }, ['regex', 'embedded']);
|
||||
expect(updateSet).toHaveBeenCalledWith(expect.objectContaining({ seriesName: 'Path Series' }));
|
||||
expect(replaceAuthors).not.toHaveBeenCalled();
|
||||
expect(replaceGenres).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('merges partial regex fields with file authors before enforcing field locks', async () => {
|
||||
const { db, updateSet } = makeDb();
|
||||
const filter = vi.fn().mockImplementation((_bookId, dto) => Promise.resolve({ dto: { ...dto, title: undefined }, skippedFields: ['title'] }));
|
||||
const service = makeService(db, undefined, { bookMetadataLockService: { isFieldLocked: vi.fn(), filterAutomatedBookUpdate: filter } });
|
||||
const extraction = vi.spyOn(MetadataExtractionService.prototype, 'extract').mockResolvedValueOnce({
|
||||
title: 'Embedded title',
|
||||
authors: [{ name: 'Embedded Author', sortName: null }],
|
||||
genres: [],
|
||||
cover: null,
|
||||
});
|
||||
const replaceAuthors = vi.spyOn(service, 'replaceAuthors').mockResolvedValue(undefined);
|
||||
vi.spyOn(service, 'replaceGenres').mockResolvedValue(undefined);
|
||||
await service.extractAndSaveWithRegex(
|
||||
9,
|
||||
[{ key: 'embedded', absolutePath: '/books/file.epub', format: 'epub' }],
|
||||
{ title: 'Regex title', seriesName: 'Path Series' },
|
||||
['regex', 'embedded'],
|
||||
);
|
||||
expect(filter).toHaveBeenCalledWith(
|
||||
9,
|
||||
expect.objectContaining({ title: 'Regex title', authors: ['Embedded Author'], seriesName: 'Path Series' }),
|
||||
);
|
||||
expect(updateSet).toHaveBeenCalledWith(expect.objectContaining({ seriesName: 'Path Series' }));
|
||||
expect(updateSet.mock.calls[0][0]).not.toHaveProperty('title');
|
||||
expect(replaceAuthors).toHaveBeenCalledWith(9, [{ name: 'Embedded Author', sortName: null }]);
|
||||
extraction.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('downloadAndSaveCover', () => {
|
||||
async function image(width: number, height: number): Promise<Buffer> {
|
||||
return sharp({ create: { width, height, channels: 3, background: '#336699' } })
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { RegexMetadataFields } from '@bookorbit/types';
|
||||
import { mergeRegexMetadata } from './regex/merge-regex-metadata';
|
||||
import { Inject, Injectable, Logger, Optional } from '@nestjs/common';
|
||||
import { and, eq, inArray, isNull, lte, or, sql } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
@@ -138,6 +140,53 @@ export class MetadataService {
|
||||
}
|
||||
}
|
||||
|
||||
async extractAndSaveWithRegex(
|
||||
bookId: number,
|
||||
sources: { key: string; absolutePath: string; format: string }[],
|
||||
regex: RegexMetadataFields,
|
||||
precedence: string[],
|
||||
): Promise<void> {
|
||||
const startedAt = Date.now();
|
||||
this.logger.debug(`[metadata.extract_with_regex] [start] bookId=${bookId} sourceCount=${sources.length} - metadata extraction started`);
|
||||
try {
|
||||
let base: ParsedBookData | null = null;
|
||||
let selected: (typeof sources)[number] | undefined;
|
||||
for (const source of sources) {
|
||||
const sourceStartedAt = Date.now();
|
||||
try {
|
||||
base = await this.extractionService.extract(source.absolutePath, source.format);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`[metadata.extract_source] [fail] bookId=${bookId} source="${sanitizeLogValue(source.key)}" format="${sanitizeLogValue(source.format)}" durationMs=${Date.now() - sourceStartedAt} errorClass=${error instanceof Error ? error.name : 'Error'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - file metadata extraction failed; continuing with remaining sources and regex`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (base) {
|
||||
selected = source;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const regexIndex = precedence.indexOf('regex');
|
||||
const baseIndex = selected ? precedence.indexOf(selected.key) : -1;
|
||||
const preferRegex = !selected || (regexIndex !== -1 && (baseIndex === -1 || regexIndex < baseIndex));
|
||||
const data = mergeRegexMetadata(base, regex, preferRegex);
|
||||
await this.persistMetadata(bookId, data, selected?.format ?? 'regex', true);
|
||||
if (selected) {
|
||||
if (base?.cover) await this.persistSourceCover(bookId, selected.format, base.cover);
|
||||
await this.persistFixedLayout(bookId, selected.absolutePath, base?.isFixedLayout);
|
||||
}
|
||||
await this.scoreService.calculateAndSave(bookId);
|
||||
this.logger.debug(
|
||||
`[metadata.extract_with_regex] [end] bookId=${bookId} durationMs=${Date.now() - startedAt} fileSourceFound=${selected !== undefined} - metadata extraction completed`,
|
||||
);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`[metadata.extract_with_regex] [fail] bookId=${bookId} durationMs=${Date.now() - startedAt} errorClass=${error instanceof Error ? error.name : 'Error'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - metadata extraction failed`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Called when ebook is the winner but audio files are also present.
|
||||
// Saves audio-specific fields that no ebook format can provide, plus audio provider IDs, and the
|
||||
// track's art into the audio slot. The leading source owns the ebook slot, and a custom audio
|
||||
@@ -765,11 +814,11 @@ export class MetadataService {
|
||||
|
||||
// ── Persistence ──────────────────────────────────────────────────────────────
|
||||
|
||||
private async persistMetadata(bookId: number, data: ParsedBookData, format: string): Promise<void> {
|
||||
private async persistMetadata(bookId: number, data: ParsedBookData, format: string, preserveAbsentRelations = false): Promise<void> {
|
||||
if (isAudioFormat(format)) {
|
||||
await this.persistAudioMetadata(bookId, data);
|
||||
await this.persistAudioMetadata(bookId, data, preserveAbsentRelations);
|
||||
} else {
|
||||
await this.persistBookMetadata(bookId, data, format);
|
||||
await this.persistBookMetadata(bookId, data, format, preserveAbsentRelations);
|
||||
}
|
||||
this.embedder?.embedBook(bookId).catch((error: Error) => {
|
||||
this.logger.warn(
|
||||
@@ -778,7 +827,7 @@ export class MetadataService {
|
||||
});
|
||||
}
|
||||
|
||||
private async persistAudioMetadata(bookId: number, data: ParsedBookData): Promise<void> {
|
||||
private async persistAudioMetadata(bookId: number, data: ParsedBookData, preserveAbsentRelations = false): Promise<void> {
|
||||
const { dto: filtered } = await this.bookMetadataLockService.filterAutomatedBookUpdate(bookId, {
|
||||
title: data.title,
|
||||
subtitle: data.subtitle,
|
||||
@@ -789,8 +838,8 @@ export class MetadataService {
|
||||
language: data.language,
|
||||
seriesName: normalizeMetadataText(data.seriesName),
|
||||
seriesIndex: data.seriesIndex,
|
||||
authors: data.authors.map((author) => author.name),
|
||||
genres: data.genres,
|
||||
authors: preserveAbsentRelations && !data.authors.length ? undefined : data.authors.map((author) => author.name),
|
||||
genres: preserveAbsentRelations && !data.genres.length ? undefined : data.genres,
|
||||
audibleId: boundProviderId('audibleId', data.audibleId),
|
||||
librofmId: boundProviderId('librofmId', data.librofmId),
|
||||
audioMetadata: {
|
||||
@@ -848,7 +897,7 @@ export class MetadataService {
|
||||
this.logger.debug(`[metadata.persist_audio] [end] bookId=${bookId} title="${sanitizeLogValue(data.title ?? '')}" - audio metadata persisted`);
|
||||
}
|
||||
|
||||
private async persistBookMetadata(bookId: number, data: ParsedBookData, format: string): Promise<void> {
|
||||
private async persistBookMetadata(bookId: number, data: ParsedBookData, format: string, preserveAbsentRelations = false): Promise<void> {
|
||||
const { dto: filtered } = await this.bookMetadataLockService.filterAutomatedBookUpdate(bookId, {
|
||||
title: data.title,
|
||||
subtitle: data.subtitle,
|
||||
@@ -861,8 +910,8 @@ export class MetadataService {
|
||||
language: data.language,
|
||||
seriesName: normalizeMetadataText(data.seriesName),
|
||||
seriesIndex: data.seriesIndex,
|
||||
authors: data.authors.map((author) => author.name),
|
||||
genres: data.genres,
|
||||
authors: preserveAbsentRelations && !data.authors.length ? undefined : data.authors.map((author) => author.name),
|
||||
genres: preserveAbsentRelations && !data.genres.length ? undefined : data.genres,
|
||||
tags: data.tags,
|
||||
rating: normalizeImportedRating(data.rating),
|
||||
pageCount: data.pageCount,
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { RegexMetadataFields } from '@bookorbit/types';
|
||||
import type { ParsedBookData } from '../extractors/format-extractor.interface';
|
||||
|
||||
export function mergeRegexMetadata(base: ParsedBookData | null, regex: RegexMetadataFields, preferRegex: boolean): ParsedBookData {
|
||||
const result: ParsedBookData = base ? { ...base } : { authors: [], genres: [], cover: null };
|
||||
for (const key of ['title', 'seriesName', 'seriesIndex'] as const) {
|
||||
if (regex[key] !== undefined && (preferRegex || !result[key]?.trim())) result[key] = regex[key];
|
||||
}
|
||||
if (regex.authors?.length && (preferRegex || !result.authors.length)) {
|
||||
result.authors = regex.authors.map((name) => ({ name, sortName: null }));
|
||||
}
|
||||
// Date and year describe the same value; a higher-priority year must not retain a conflicting date.
|
||||
if (regex.publishedYear !== undefined && (preferRegex || (!result.publishedYear && !result.publishedDate))) {
|
||||
result.publishedYear = regex.publishedYear;
|
||||
result.publishedDate = regex.publishedDate;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import type { RegexMetadataConfig, RegexMetadataDiagnostic, RegexMetadataFields, RegexMetadataPreview } from '@bookorbit/types';
|
||||
import { isPublishedDateKey, parsePublishedYear } from '../../../common/utils/published-date.utils';
|
||||
|
||||
export interface RegexMetadataTask {
|
||||
config: RegexMetadataConfig;
|
||||
inputPath?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared-package values the worker needs. A worker thread resolves packages without the aliases the
|
||||
* server process runs under, so it must not import values from @bookorbit/types itself; the
|
||||
* service hands them over when it spawns the worker.
|
||||
*/
|
||||
export interface RegexMetadataWorkerData {
|
||||
groups: readonly string[];
|
||||
seriesIndexPattern: RegExp;
|
||||
seriesIndexMaxLength: number;
|
||||
}
|
||||
|
||||
interface CompiledRules {
|
||||
key: string;
|
||||
compiled: RegExp[];
|
||||
diagnostics: RegexMetadataDiagnostic[];
|
||||
}
|
||||
|
||||
// A scan sends the same rules for every book, so the worker keeps the last compiled set.
|
||||
let lastCompiled: CompiledRules | undefined;
|
||||
|
||||
function compileRules(config: RegexMetadataConfig, shared: RegexMetadataWorkerData): CompiledRules {
|
||||
const key = JSON.stringify(config.rules.map(({ pattern, flags }) => [pattern, flags]));
|
||||
if (lastCompiled?.key === key) return lastCompiled;
|
||||
const compiled: RegExp[] = [];
|
||||
const diagnostics: RegexMetadataDiagnostic[] = [];
|
||||
for (const [ruleIndex, rule] of config.rules.entries()) {
|
||||
try {
|
||||
const pattern = rule.pattern.normalize('NFC');
|
||||
const regex = new RegExp(pattern, rule.flags);
|
||||
// The empty first alternative exposes group names without executing the user's pattern.
|
||||
const groups = Object.keys(new RegExp(`|(?:${pattern})`, rule.flags).exec('')?.groups ?? {});
|
||||
if (!groups.some((group) => shared.groups.includes(group))) {
|
||||
diagnostics.push({ code: 'missing_group', ruleIndex });
|
||||
}
|
||||
if (groups.includes('author') && groups.includes('authors')) {
|
||||
diagnostics.push({ code: 'conflicting_authors', ruleIndex });
|
||||
}
|
||||
compiled.push(regex);
|
||||
} catch {
|
||||
diagnostics.push({ code: 'invalid_pattern', ruleIndex });
|
||||
}
|
||||
}
|
||||
lastCompiled = { key, compiled, diagnostics };
|
||||
return lastCompiled;
|
||||
}
|
||||
|
||||
export function evaluateRegexMetadata({ config, inputPath }: RegexMetadataTask, shared: RegexMetadataWorkerData): RegexMetadataPreview {
|
||||
const { compiled, diagnostics } = compileRules(config, shared);
|
||||
const result: RegexMetadataPreview = {
|
||||
inputPath: inputPath ?? '',
|
||||
matched: false,
|
||||
ruleIndex: null,
|
||||
groups: {},
|
||||
metadata: {},
|
||||
diagnostics: [...diagnostics],
|
||||
};
|
||||
if (result.diagnostics.length || inputPath === undefined) return result;
|
||||
|
||||
for (const [ruleIndex, regex] of compiled.entries()) {
|
||||
const match = regex.exec(inputPath);
|
||||
if (!match?.groups) continue;
|
||||
const groups = Object.fromEntries(Object.entries(match.groups).filter((entry): entry is [string, string] => entry[1] !== undefined));
|
||||
const metadata = normalizeGroups(groups, ruleIndex, result.diagnostics, shared);
|
||||
if (!Object.keys(metadata).length) continue;
|
||||
return { ...result, matched: true, ruleIndex, groups, metadata };
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeGroups(
|
||||
groups: Record<string, string>,
|
||||
ruleIndex: number,
|
||||
diagnostics: RegexMetadataDiagnostic[],
|
||||
shared: RegexMetadataWorkerData,
|
||||
): RegexMetadataFields {
|
||||
const metadata: RegexMetadataFields = {};
|
||||
for (const [group, field, limit] of [
|
||||
['title', 'title', 1000],
|
||||
['series', 'seriesName', 200],
|
||||
] as const) {
|
||||
const value = groups[group]?.trim();
|
||||
if (!value) continue;
|
||||
if (value.length > limit) diagnostics.push({ code: 'invalid_value', ruleIndex, field: group });
|
||||
else metadata[field] = value;
|
||||
}
|
||||
const seriesIndex = groups.seriesIndex?.trim();
|
||||
if (seriesIndex) {
|
||||
// The column only accepts plain decimals; a decimal comma is the one spelling worth repairing.
|
||||
const candidate = seriesIndex.replace(/^(\d+),(\d+)$/, '$1.$2');
|
||||
if (candidate.length > shared.seriesIndexMaxLength || !shared.seriesIndexPattern.test(candidate)) {
|
||||
diagnostics.push({ code: 'invalid_value', ruleIndex, field: 'seriesIndex' });
|
||||
} else metadata.seriesIndex = candidate;
|
||||
}
|
||||
const author = groups.author?.trim();
|
||||
const authors = author
|
||||
? [author]
|
||||
: groups.authors
|
||||
?.split(';')
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
if (authors?.length) {
|
||||
if (authors.some((value) => value.length > 200)) diagnostics.push({ code: 'invalid_value', ruleIndex, field: 'authors' });
|
||||
else metadata.authors = [...new Set(authors)];
|
||||
}
|
||||
const year = groups.publishedYear?.trim();
|
||||
if (year) {
|
||||
const parsed = /^\d{4}$/.test(year) ? parsePublishedYear(year) : undefined;
|
||||
if (parsed === undefined) diagnostics.push({ code: 'invalid_value', ruleIndex, field: 'publishedYear' });
|
||||
else metadata.publishedYear = parsed;
|
||||
}
|
||||
const date = groups.publishedDate?.trim();
|
||||
if (date) {
|
||||
if (!isPublishedDateKey(date)) diagnostics.push({ code: 'invalid_value', ruleIndex, field: 'publishedDate' });
|
||||
else {
|
||||
metadata.publishedDate = date;
|
||||
metadata.publishedYear = Number(date.slice(0, 4));
|
||||
}
|
||||
}
|
||||
return metadata;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { posix, win32 } from 'node:path';
|
||||
import { REGEX_METADATA_LIMITS } from '@bookorbit/types';
|
||||
|
||||
export function prepareRegexMetadataPath(relativePath: string): string {
|
||||
// Filesystems hand back either Unicode form, while typed rules are composed.
|
||||
const path = relativePath.normalize('NFC').replaceAll('\\', '/');
|
||||
if (
|
||||
!path ||
|
||||
path.length > REGEX_METADATA_LIMITS.pathLength ||
|
||||
path.includes('\0') ||
|
||||
posix.isAbsolute(path) ||
|
||||
win32.isAbsolute(path) ||
|
||||
/^[a-z]:/i.test(path) ||
|
||||
path.split('/').some((part) => part === '..' || part === '.' || part === '')
|
||||
) {
|
||||
throw new BadRequestException('Expected a file path relative to the library folder');
|
||||
}
|
||||
const extension = /\.kepub\.epub$/i.test(path) ? path.slice(-11) : posix.extname(path);
|
||||
return extension ? path.slice(0, -extension.length) : path;
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { ServiceUnavailableException } from '@nestjs/common';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { RegexMetadataService } from './regex-metadata.service';
|
||||
import { prepareRegexMetadataPath } from './regex-metadata-path';
|
||||
import { mergeRegexMetadata } from './merge-regex-metadata';
|
||||
import type { ParsedBookData } from '../extractors/format-extractor.interface';
|
||||
|
||||
describe('RegexMetadataService', () => {
|
||||
let service: RegexMetadataService;
|
||||
beforeEach(async () => {
|
||||
const module = await Test.createTestingModule({ providers: [RegexMetadataService] }).compile();
|
||||
service = module.get(RegexMetadataService);
|
||||
});
|
||||
afterEach(async () => service.onModuleDestroy());
|
||||
|
||||
it('validates rules without a path and returns indexed diagnostics', async () => {
|
||||
expect(await service.check({ rules: [{ pattern: '(?<title>.+)', flags: '' }] })).toEqual({ valid: true, diagnostics: [] });
|
||||
expect(
|
||||
await service.check({
|
||||
rules: [
|
||||
{ pattern: '[', flags: '' },
|
||||
{ pattern: '(?<helper>.+)', flags: '' },
|
||||
{ pattern: '(?<author>.+)(?<authors>.+)', flags: '' },
|
||||
],
|
||||
}),
|
||||
).toEqual({
|
||||
valid: false,
|
||||
diagnostics: [
|
||||
{ code: 'invalid_pattern', ruleIndex: 0 },
|
||||
{ code: 'missing_group', ruleIndex: 1 },
|
||||
{ code: 'conflicting_authors', ruleIndex: 2 },
|
||||
],
|
||||
});
|
||||
await service.onModuleDestroy();
|
||||
expect(await service.check({ rules: [{ pattern: '(?<title>.+)', flags: '' }] })).toEqual({
|
||||
valid: false,
|
||||
diagnostics: [{ code: 'worker_failed' }],
|
||||
});
|
||||
});
|
||||
|
||||
it.each(['epub', 'PDF', 'm4b', 'KEPUB.EPUB'])('matches the relative filename without %s', async (extension) => {
|
||||
const result = await service.preview(
|
||||
{ rules: [{ pattern: '^(?<author>[^/]+)/(?<series>[^/]+)/(?<seriesIndex>\\d+) - (?<title>.+)$', flags: 'u' }] },
|
||||
`Änne/Dr. Who/08 - My.Book.${extension}`,
|
||||
);
|
||||
expect(result.inputPath).toBe('Änne/Dr. Who/08 - My.Book');
|
||||
expect(result.metadata).toEqual({ authors: ['Änne'], seriesName: 'Dr. Who', seriesIndex: '08', title: 'My.Book' });
|
||||
});
|
||||
|
||||
it.each([
|
||||
['decomposed path, composed rule', 'A\u0308nne/Book.epub', '^(?<author>Änne)/'],
|
||||
['composed path, decomposed rule', 'Änne/Book.epub', '^(?<author>A\u0308nne)/'],
|
||||
])('matches across Unicode normalization forms: %s', async (_label, path, pattern) => {
|
||||
const result = await service.preview({ rules: [{ pattern, flags: 'u' }] }, path);
|
||||
expect(result.metadata.authors).toEqual(['Änne']);
|
||||
expect(result.inputPath).toBe('Änne/Book');
|
||||
});
|
||||
|
||||
it('skips unusable matches and selects only the first usable rule', async () => {
|
||||
const result = await service.preview(
|
||||
{
|
||||
rules: [
|
||||
{ pattern: '^(?<publishedYear>\\d+)', flags: '' },
|
||||
{ pattern: '^(?<title>.+)$', flags: '' },
|
||||
{ pattern: '^(?<series>.+)$', flags: '' },
|
||||
],
|
||||
},
|
||||
'999-title.epub',
|
||||
);
|
||||
expect(result.ruleIndex).toBe(1);
|
||||
expect(result.metadata).toEqual({ title: '999-title' });
|
||||
expect(result.diagnostics).toContainEqual({ code: 'invalid_value', ruleIndex: 0, field: 'publishedYear' });
|
||||
});
|
||||
|
||||
it('normalizes authors and gives a valid date precedence over the captured year', async () => {
|
||||
const result = await service.preview(
|
||||
{ rules: [{ pattern: '^(?<authors>[^/]+)/(?<publishedYear>\\d+)/(?<publishedDate>.+)$', flags: '' }] },
|
||||
'Ann ; Bob; Ann/2020/2024-02-29.epub',
|
||||
);
|
||||
expect(result.metadata).toEqual({ authors: ['Ann', 'Bob'], publishedYear: 2024, publishedDate: '2024-02-29' });
|
||||
});
|
||||
|
||||
it('rejects impossible dates without preventing other valid fields', async () => {
|
||||
const result = await service.preview({ rules: [{ pattern: '^(?<title>[^/]+)/(?<publishedDate>.+)$', flags: '' }] }, 'Book/2023-02-29.epub');
|
||||
expect(result.metadata).toEqual({ title: 'Book' });
|
||||
expect(result.diagnostics[0].code).toBe('invalid_value');
|
||||
});
|
||||
|
||||
it.each([
|
||||
['08', '08'],
|
||||
['1.5', '1.5'],
|
||||
['1,5', '1.5'],
|
||||
])('keeps series index %s as %s', async (captured, expected) => {
|
||||
const result = await service.preview({ rules: [{ pattern: '^(?<title>[^/]+)/(?<seriesIndex>.+)$', flags: '' }] }, `Book/${captured}.epub`);
|
||||
expect(result.metadata).toEqual({ title: 'Book', seriesIndex: expected });
|
||||
});
|
||||
|
||||
it.each(['IV', '2a', '1-2', '1,5,2', '.5'])('skips series index %s that the column would reject', async (captured) => {
|
||||
const result = await service.preview({ rules: [{ pattern: '^(?<title>[^/]+)/(?<seriesIndex>.+)$', flags: '' }] }, `Book/${captured}.epub`);
|
||||
expect(result.metadata).toEqual({ title: 'Book' });
|
||||
expect(result.diagnostics).toEqual([{ code: 'invalid_value', ruleIndex: 0, field: 'seriesIndex' }]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['[', 'invalid_pattern'],
|
||||
['(?<helper>.+)', 'missing_group'],
|
||||
['(?<author>.+)(?<authors>.*)', 'conflicting_authors'],
|
||||
])('validates pattern %s without saving it', async (pattern, code) => {
|
||||
const config = { rules: [{ pattern, flags: '' }] };
|
||||
const result = await service.preview(config, 'Book.epub');
|
||||
expect(result.diagnostics).toContainEqual({ code, ruleIndex: 0 });
|
||||
await expect(service.validate(config)).rejects.toThrow('Invalid regex metadata configuration');
|
||||
});
|
||||
|
||||
it('terminates a pathological expression, keeps the event loop responsive, and recovers', async () => {
|
||||
let timerRan = false;
|
||||
const timer = setTimeout(() => {
|
||||
timerRan = true;
|
||||
}, 20);
|
||||
const result = await service.preview({ rules: [{ pattern: '^(?<title>(a+)+)$', flags: '' }] }, `${'a'.repeat(200)}!.epub`);
|
||||
clearTimeout(timer);
|
||||
expect(result.diagnostics).toEqual([{ code: 'timeout' }]);
|
||||
expect(timerRan).toBe(true);
|
||||
const recovered = await service.preview({ rules: [{ pattern: '^(?<title>.+)$', flags: '' }] }, 'Recovered.epub');
|
||||
expect(recovered.metadata.title).toBe('Recovered');
|
||||
});
|
||||
|
||||
it('bounds queued work and resolves pending requests on shutdown', async () => {
|
||||
const config = { rules: [{ pattern: '^(?<title>.+)$', flags: '' }] };
|
||||
const tasks = Array.from({ length: 40 }, () => service.preview(config, 'Book.epub'));
|
||||
await service.onModuleDestroy();
|
||||
const results = await Promise.all(tasks);
|
||||
expect(results.some((result) => result.diagnostics[0]?.code === 'busy')).toBe(true);
|
||||
expect(results.every((result) => result.diagnostics.length > 0)).toBe(true);
|
||||
});
|
||||
|
||||
it('never refuses scanner evaluation for a full queue', async () => {
|
||||
const config = { rules: [{ pattern: '^(?<title>.+)$', flags: '' }] };
|
||||
const results = await Promise.all(Array.from({ length: 40 }, () => service.evaluate(config, 'Book.epub')));
|
||||
expect(results.every((result) => result.metadata.title === 'Book')).toBe(true);
|
||||
});
|
||||
|
||||
it('retries a pool failure once before reporting it', async () => {
|
||||
const config = { rules: [{ pattern: '^(?<title>.+)$', flags: '' }] };
|
||||
const run = vi.spyOn(service as any, 'run').mockResolvedValueOnce({ diagnostics: [{ code: 'timeout' }] });
|
||||
const result = await service.evaluate(config, 'Book.epub');
|
||||
expect(run).toHaveBeenCalledTimes(2);
|
||||
expect(result.metadata).toEqual({ title: 'Book' });
|
||||
});
|
||||
|
||||
it('reports a pool failure on save as unavailable rather than invalid rules', async () => {
|
||||
vi.spyOn(service as any, 'run').mockResolvedValue({ diagnostics: [{ code: 'worker_failed' }] });
|
||||
await expect(service.validate({ rules: [{ pattern: '(?<title>.+)', flags: '' }] })).rejects.toBeInstanceOf(ServiceUnavailableException);
|
||||
});
|
||||
|
||||
it('rejects excessive rules, pattern length, and stateful flags', async () => {
|
||||
for (const config of [
|
||||
{ rules: Array.from({ length: 21 }, () => ({ pattern: '(?<title>.+)', flags: '' })) },
|
||||
{ rules: [{ pattern: 'a'.repeat(2049), flags: '' }] },
|
||||
{ rules: [{ pattern: '(?<title>.+)', flags: 'g' }] },
|
||||
])
|
||||
await expect(service.validate(config)).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('regex metadata worker', () => {
|
||||
// CI runs server tests without building @bookorbit/types, and a worker thread cannot see the test alias.
|
||||
it.each(['regex-metadata-engine.ts', 'regex-metadata.worker.ts'])('%s imports only types from the shared package', (file) => {
|
||||
const source = readFileSync(join(__dirname, file), 'utf8');
|
||||
expect(source).not.toMatch(/^import (?!type )[^;]*'@bookorbit\/types'/m);
|
||||
});
|
||||
});
|
||||
|
||||
describe('relative regex paths', () => {
|
||||
it.each(['/books/a.epub', '../a.epub', 'a/../../b.epub', 'C:\\books\\a.epub', 'C:a.epub', '\\\\server\\a.epub', 'a\0.epub', 'a/'.repeat(2050)])(
|
||||
'rejects unsafe input',
|
||||
(path) => {
|
||||
expect(() => prepareRegexMetadataPath(path)).toThrow();
|
||||
},
|
||||
);
|
||||
it('preserves dots in folders and filenames and normalizes separators', () => {
|
||||
expect(prepareRegexMetadataPath('Dr. Author\\Vol. 1\\My.Book.epub')).toBe('Dr. Author/Vol. 1/My.Book');
|
||||
expect(prepareRegexMetadataPath('Author/Book')).toBe('Author/Book');
|
||||
});
|
||||
});
|
||||
|
||||
describe('regex metadata precedence', () => {
|
||||
const base: ParsedBookData = {
|
||||
title: 'Embedded',
|
||||
authors: [{ name: 'Author', sortName: null }],
|
||||
seriesName: 'Original',
|
||||
genres: ['Novel'],
|
||||
cover: Buffer.from('cover'),
|
||||
};
|
||||
it('overrides only captured fields when regex has priority', () => {
|
||||
expect(mergeRegexMetadata(base, { seriesName: 'Regex' }, true)).toEqual({ ...base, seriesName: 'Regex' });
|
||||
});
|
||||
it('fills missing values when the file source has priority', () => {
|
||||
expect(mergeRegexMetadata(base, { title: 'Regex', seriesIndex: '08' }, false)).toEqual({ ...base, seriesIndex: '08' });
|
||||
});
|
||||
it('does not keep a lower-priority date when regex overrides the year', () => {
|
||||
expect(mergeRegexMetadata({ ...base, publishedDate: '2020-01-01', publishedYear: 2020 }, { publishedYear: 2024 }, true)).toMatchObject({
|
||||
publishedYear: 2024,
|
||||
publishedDate: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,177 @@
|
||||
import { BadRequestException, Injectable, OnModuleDestroy, ServiceUnavailableException } from '@nestjs/common';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { Worker } from 'node:worker_threads';
|
||||
import {
|
||||
REGEX_METADATA_GROUPS,
|
||||
REGEX_METADATA_LIMITS,
|
||||
SERIES_INDEX_MAX_LENGTH,
|
||||
SERIES_INDEX_PATTERN,
|
||||
type RegexMetadataConfig,
|
||||
type RegexMetadataDiagnosticCode,
|
||||
type RegexMetadataPreview,
|
||||
type RegexMetadataValidation,
|
||||
} from '@bookorbit/types';
|
||||
import { prepareRegexMetadataPath } from './regex-metadata-path';
|
||||
import type { RegexMetadataTask, RegexMetadataWorkerData } from './regex-metadata-engine';
|
||||
|
||||
const INTERACTIVE_QUEUE_LIMIT = 32;
|
||||
const TRANSIENT_CODES: ReadonlySet<RegexMetadataDiagnosticCode> = new Set(['timeout', 'busy', 'worker_failed']);
|
||||
|
||||
/** Whether the result failed because of the worker pool rather than the rules. */
|
||||
export function isTransientRegexFailure(code: RegexMetadataDiagnosticCode): boolean {
|
||||
return TRANSIENT_CODES.has(code);
|
||||
}
|
||||
|
||||
interface Task {
|
||||
input: RegexMetadataTask;
|
||||
resolve: (result: RegexMetadataPreview) => void;
|
||||
}
|
||||
|
||||
interface Slot {
|
||||
worker: Worker;
|
||||
ready: boolean;
|
||||
task?: Task;
|
||||
timer?: ReturnType<typeof setTimeout>;
|
||||
discarding?: boolean;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class RegexMetadataService implements OnModuleDestroy {
|
||||
private readonly slots = new Set<Slot>();
|
||||
private readonly queue: Task[] = [];
|
||||
private stopped = false;
|
||||
|
||||
/** Rejects rules that cannot be saved; a pool failure is reported as unavailable, never as invalid rules. */
|
||||
async validate(config: RegexMetadataConfig | null | undefined): Promise<void> {
|
||||
if (config == null) return;
|
||||
const { diagnostics } = await this.runWithRetry({ config });
|
||||
if (diagnostics.some((diagnostic) => isTransientRegexFailure(diagnostic.code)))
|
||||
throw new ServiceUnavailableException('Regex metadata rules could not be checked right now');
|
||||
if (diagnostics.length) throw new BadRequestException({ message: 'Invalid regex metadata configuration', diagnostics });
|
||||
}
|
||||
|
||||
async check(config: RegexMetadataConfig): Promise<RegexMetadataValidation> {
|
||||
const result = await this.run({ config }, true);
|
||||
return { valid: result.diagnostics.length === 0, diagnostics: result.diagnostics };
|
||||
}
|
||||
|
||||
preview(config: RegexMetadataConfig, relativePath: string): Promise<RegexMetadataPreview> {
|
||||
return this.run({ config, inputPath: prepareRegexMetadataPath(relativePath) }, true);
|
||||
}
|
||||
|
||||
/** Scanner entry point: never refused for a full queue, and retried once after a pool failure. */
|
||||
evaluate(config: RegexMetadataConfig, relativePath: string): Promise<RegexMetadataPreview> {
|
||||
return this.runWithRetry({ config, inputPath: prepareRegexMetadataPath(relativePath) });
|
||||
}
|
||||
|
||||
private async runWithRetry(input: RegexMetadataTask): Promise<RegexMetadataPreview> {
|
||||
const result = await this.run(input, false);
|
||||
// The timeout runs on the main thread, so a stalled event loop can trip it; a second attempt tells that apart from a slow expression.
|
||||
return result.diagnostics.some((diagnostic) => isTransientRegexFailure(diagnostic.code)) ? this.run(input, false) : result;
|
||||
}
|
||||
|
||||
private run(input: RegexMetadataTask, interactive: boolean): Promise<RegexMetadataPreview> {
|
||||
this.assertBounds(input.config);
|
||||
if (this.stopped) return Promise.resolve(this.failure(input, 'worker_failed'));
|
||||
if (interactive && this.queue.length >= INTERACTIVE_QUEUE_LIMIT) return Promise.resolve(this.failure(input, 'busy'));
|
||||
return new Promise((resolve) => {
|
||||
this.queue.push({ input, resolve });
|
||||
this.dispatch();
|
||||
});
|
||||
}
|
||||
|
||||
private assertBounds(config: RegexMetadataConfig): void {
|
||||
if (
|
||||
!Array.isArray(config.rules) ||
|
||||
!config.rules.length ||
|
||||
config.rules.length > REGEX_METADATA_LIMITS.rules ||
|
||||
config.rules.some(
|
||||
(rule) =>
|
||||
!rule ||
|
||||
typeof rule.pattern !== 'string' ||
|
||||
!rule.pattern.length ||
|
||||
rule.pattern.length > REGEX_METADATA_LIMITS.patternLength ||
|
||||
typeof rule.flags !== 'string' ||
|
||||
!/^(?:i?u?|ui)$/.test(rule.flags),
|
||||
)
|
||||
) {
|
||||
throw new BadRequestException('Invalid regex metadata rules or flags');
|
||||
}
|
||||
}
|
||||
|
||||
private dispatch(): void {
|
||||
if (this.stopped) return;
|
||||
for (const slot of this.slots) {
|
||||
if (!slot.ready || slot.discarding || slot.task || !this.queue.length) continue;
|
||||
slot.task = this.queue.shift()!;
|
||||
slot.timer = setTimeout(() => this.discard(slot, 'timeout'), 100);
|
||||
slot.worker.postMessage(slot.task.input);
|
||||
}
|
||||
if (this.queue.length && this.slots.size < 2) this.spawn();
|
||||
}
|
||||
|
||||
private spawn(): void {
|
||||
const jsPath = join(__dirname, 'regex-metadata.worker.js');
|
||||
const workerPath = existsSync(jsPath) ? jsPath : join(__dirname, 'regex-metadata.worker.ts');
|
||||
const workerData: RegexMetadataWorkerData = {
|
||||
groups: REGEX_METADATA_GROUPS,
|
||||
seriesIndexPattern: SERIES_INDEX_PATTERN,
|
||||
seriesIndexMaxLength: SERIES_INDEX_MAX_LENGTH,
|
||||
};
|
||||
const worker = new Worker(workerPath, {
|
||||
workerData,
|
||||
execArgv: workerPath.endsWith('.ts') ? ['--import', 'tsx'] : [],
|
||||
resourceLimits: { maxOldGenerationSizeMb: 64 },
|
||||
});
|
||||
const slot: Slot = { worker, ready: false };
|
||||
this.slots.add(slot);
|
||||
slot.timer = setTimeout(() => this.discard(slot, 'worker_failed'), 10_000);
|
||||
worker.on('message', (message: RegexMetadataPreview | 'ready') => {
|
||||
if (!this.slots.has(slot) || slot.discarding) return;
|
||||
clearTimeout(slot.timer);
|
||||
if (message === 'ready') slot.ready = true;
|
||||
else {
|
||||
slot.task?.resolve(message);
|
||||
slot.task = undefined;
|
||||
}
|
||||
this.dispatch();
|
||||
});
|
||||
worker.on('error', () => this.discard(slot, 'worker_failed'));
|
||||
worker.on('exit', () => this.discard(slot, 'worker_failed'));
|
||||
}
|
||||
|
||||
private discard(slot: Slot, code: RegexMetadataDiagnosticCode): void {
|
||||
if (!this.slots.has(slot) || slot.discarding) return;
|
||||
slot.discarding = true;
|
||||
clearTimeout(slot.timer);
|
||||
if (slot.task) slot.task.resolve(this.failure(slot.task.input, code));
|
||||
else if (!slot.ready) {
|
||||
// A broken worker entrypoint must not create an endless respawn loop.
|
||||
for (const task of this.queue.splice(0)) task.resolve(this.failure(task.input, code));
|
||||
}
|
||||
slot.task = undefined;
|
||||
void slot.worker.terminate().finally(() => {
|
||||
this.slots.delete(slot);
|
||||
this.dispatch();
|
||||
});
|
||||
}
|
||||
|
||||
private failure(input: RegexMetadataTask, code: RegexMetadataDiagnosticCode): RegexMetadataPreview {
|
||||
return { inputPath: input.inputPath ?? '', matched: false, ruleIndex: null, groups: {}, metadata: {}, diagnostics: [{ code }] };
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
this.stopped = true;
|
||||
for (const task of this.queue.splice(0)) task.resolve(this.failure(task.input, 'worker_failed'));
|
||||
const slots = [...this.slots];
|
||||
this.slots.clear();
|
||||
await Promise.all(
|
||||
slots.map((slot) => {
|
||||
clearTimeout(slot.timer);
|
||||
if (slot.task) slot.task.resolve(this.failure(slot.task.input, 'worker_failed'));
|
||||
return slot.worker.terminate();
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { parentPort, workerData } from 'node:worker_threads';
|
||||
import { evaluateRegexMetadata, type RegexMetadataTask, type RegexMetadataWorkerData } from './regex-metadata-engine';
|
||||
|
||||
const shared = workerData as RegexMetadataWorkerData;
|
||||
|
||||
parentPort!.on('message', (task: RegexMetadataTask) => {
|
||||
parentPort!.postMessage(evaluateRegexMetadata(task, shared));
|
||||
});
|
||||
parentPort!.postMessage('ready');
|
||||
@@ -37,6 +37,7 @@ function makeService(db: any = {}) {
|
||||
bufferBookMissingEvent: vi.fn(),
|
||||
bufferBooksUnavailableNotification: vi.fn(),
|
||||
bufferBooksRestoredNotification: vi.fn(),
|
||||
refreshRegexMetadataAfterMoveAsync: vi.fn(),
|
||||
} as unknown as ScannerService;
|
||||
|
||||
const service = new FileWatcherService(db, processor, gateway, scannerService, new SelfWriteRegistry());
|
||||
|
||||
@@ -33,6 +33,7 @@ function makeService(db: any = {}) {
|
||||
bufferBooksUnavailableNotification: vi.fn(),
|
||||
bufferBooksRestoredNotification: vi.fn(),
|
||||
reconcileCoverSlotsAsync: vi.fn(),
|
||||
refreshRegexMetadataAfterMoveAsync: vi.fn(),
|
||||
} as unknown as ScannerService;
|
||||
|
||||
const selfWriteRegistry = new SelfWriteRegistry();
|
||||
@@ -164,6 +165,7 @@ describe('process()', () => {
|
||||
|
||||
expect((gateway as any).emitBookMoved).toHaveBeenCalledWith({ libraryId: 1, bookIds: [5] });
|
||||
expect(scheduleFolderScanSpy).toHaveBeenCalledWith('/books/moved.epub', 1);
|
||||
expect(scannerService.refreshRegexMetadataAfterMoveAsync).toHaveBeenCalledWith([5]);
|
||||
expect(scannerService.startScanAsync).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
||||
@@ -511,6 +511,8 @@ export class FileWatcherService implements OnApplicationBootstrap, OnModuleDestr
|
||||
this.scannerService.bufferBooksRestoredNotification(result.libraryId, result.bookIds);
|
||||
} else if (result.type === 'book-moved') {
|
||||
this.gateway.emitBookMoved({ libraryId: result.libraryId, bookIds: result.bookIds });
|
||||
// The new path is already recorded, so the folder scan below cannot see the move regex rules read.
|
||||
this.scannerService.refreshRegexMetadataAfterMoveAsync(result.bookIds);
|
||||
// A move updates the file's path and may consolidate virtual-sibling books.
|
||||
// Schedule a folder scan so upsertBook can drain any remaining siblings.
|
||||
if (type === 'create') this.scheduleFolderScan(path, libraryId);
|
||||
|
||||
@@ -110,6 +110,7 @@ export class ScannerRepository {
|
||||
allowedFormats: libraries.allowedFormats,
|
||||
formatPriority: libraries.formatPriority,
|
||||
metadataPrecedence: libraries.metadataPrecedence,
|
||||
regexMetadata: libraries.regexMetadata,
|
||||
excludePatterns: libraries.excludePatterns,
|
||||
organizationMode: libraries.organizationMode,
|
||||
addedAtSource: libraries.addedAtSource,
|
||||
|
||||
@@ -153,6 +153,7 @@ const mockGateway = {
|
||||
|
||||
const mockMetadata = {
|
||||
extractAndSave: vi.fn().mockResolvedValue(undefined),
|
||||
extractAndSaveWithRegex: vi.fn().mockResolvedValue(undefined),
|
||||
refreshCoverForBook: vi.fn().mockResolvedValue(false),
|
||||
extractAudioFileDuration: vi.fn().mockResolvedValue(undefined),
|
||||
aggregateAudioDuration: vi.fn().mockResolvedValue(undefined),
|
||||
@@ -170,6 +171,7 @@ function makeService(
|
||||
const notificationService = { notify: vi.fn().mockResolvedValue(undefined) };
|
||||
const achievementEvents = { emit: vi.fn() };
|
||||
const selfWriteRegistry = new SelfWriteRegistry();
|
||||
const regexMetadata = { evaluate: vi.fn() };
|
||||
const service = new ScannerService(
|
||||
repo as any,
|
||||
mockMetadata as any,
|
||||
@@ -178,11 +180,12 @@ function makeService(
|
||||
notificationService as any,
|
||||
selfWriteRegistry,
|
||||
coverStore as any,
|
||||
regexMetadata as any,
|
||||
autoFetchOrchestrator as any,
|
||||
achievementEvents as any,
|
||||
coverReconciler as any,
|
||||
);
|
||||
return { service, jobStore, notificationService, achievementEvents, selfWriteRegistry };
|
||||
return { service, jobStore, notificationService, achievementEvents, selfWriteRegistry, regexMetadata };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -4429,3 +4432,296 @@ describe('cover slot reconcile after a scan', () => {
|
||||
expect(reconciler.enqueue).toHaveBeenCalledWith([1], { filesChanged: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe('regex metadata scan integration', () => {
|
||||
const regexConfig = { rules: [{ pattern: '(?<series>[^/]+)/[^/]+$', flags: '' }] };
|
||||
function regexRepo(overrides: Record<string, unknown> = {}) {
|
||||
return makeRepo({
|
||||
findLibrarySettings: vi.fn().mockResolvedValue({
|
||||
allowedFormats: [],
|
||||
formatPriority: DEFAULT_FORMAT_PRIORITY,
|
||||
metadataPrecedence: ['regex', 'embedded', 'opfFile'],
|
||||
regexMetadata: regexConfig,
|
||||
excludePatterns: [],
|
||||
organizationMode: 'book_per_folder',
|
||||
}),
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
it('passes the primary relative path and precedence to the shared extraction flow', async () => {
|
||||
mockFindCandidates.mockResolvedValue({
|
||||
candidates: [makeCandidate('/library/Author/Book', [makeFileStat()])],
|
||||
skippedDirs: new Set(),
|
||||
unchangedDirs: new Set(),
|
||||
dirMtimes: new Map(),
|
||||
});
|
||||
const repo = regexRepo();
|
||||
const done = awaitScan(repo);
|
||||
const { service, regexMetadata } = makeService(repo);
|
||||
regexMetadata.evaluate.mockResolvedValue({ matched: true, metadata: { seriesName: 'Book' }, diagnostics: [] });
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
expect(regexMetadata.evaluate).toHaveBeenCalledWith(regexConfig, 'Author/Book/book.epub');
|
||||
expect(mockMetadata.extractAndSaveWithRegex).toHaveBeenCalledWith(
|
||||
1,
|
||||
[{ key: 'embedded', absolutePath: '/library/Author/Book/book.epub', format: 'epub' }],
|
||||
{ seriesName: 'Book' },
|
||||
['regex', 'embedded', 'opfFile'],
|
||||
);
|
||||
expect(mockMetadata.extractAndSave).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['book_per_folder', 'book_per_file'])('refreshes renamed primary paths in %s mode without content changes', async (organizationMode) => {
|
||||
const file = makeFileStat({ absolutePath: '/library/Author/Book/renamed.epub', relPath: 'Author/Book/renamed.epub' });
|
||||
const folderPath = organizationMode === 'book_per_file' ? file.absolutePath : '/library/Author/Book';
|
||||
const walk = { candidates: [makeCandidate(folderPath, [file])], skippedDirs: new Set(), unchangedDirs: new Set(), dirMtimes: new Map() };
|
||||
mockFindCandidates.mockResolvedValue(walk);
|
||||
mockFindLooseCandidates.mockResolvedValue(walk);
|
||||
mockStat.mockRejectedValue(Object.assign(new Error('missing old path'), { code: 'ENOENT' }));
|
||||
const repo = regexRepo({
|
||||
findLibrarySettings: vi.fn().mockResolvedValue({
|
||||
allowedFormats: [],
|
||||
formatPriority: DEFAULT_FORMAT_PRIORITY,
|
||||
metadataPrecedence: ['regex', 'embedded'],
|
||||
regexMetadata: regexConfig,
|
||||
excludePatterns: [],
|
||||
organizationMode,
|
||||
}),
|
||||
findBooksByLibraryFolder: vi.fn().mockResolvedValue([
|
||||
{
|
||||
id: 1,
|
||||
status: 'present',
|
||||
folderPath: organizationMode === 'book_per_file' ? '/library/Author/Book/book.epub' : folderPath,
|
||||
primaryFileId: 1,
|
||||
},
|
||||
]),
|
||||
findBookFilesByLibraryFolder: vi.fn().mockResolvedValue([makeBookFile()]),
|
||||
});
|
||||
const done = awaitScan(repo);
|
||||
const { service, regexMetadata } = makeService(repo);
|
||||
regexMetadata.evaluate.mockResolvedValue({ matched: true, metadata: { title: 'renamed' }, diagnostics: [] });
|
||||
await service.startScan(1, 'manual', true);
|
||||
await done;
|
||||
expect(regexMetadata.evaluate).toHaveBeenCalledWith(regexConfig, file.relPath);
|
||||
expect(mockMetadata.extractAndSaveWithRegex).toHaveBeenCalledWith(1, expect.any(Array), { title: 'renamed' }, ['regex', 'embedded']);
|
||||
expect(repo.failScanJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([false, true])('refreshes a changed primary identity unless a self write is active (%s)', async (suppressed) => {
|
||||
const file = makeFileStat();
|
||||
mockFindCandidates.mockResolvedValue({
|
||||
candidates: [makeCandidate('/library/Author/Book', [file])],
|
||||
skippedDirs: new Set(),
|
||||
unchangedDirs: new Set(),
|
||||
dirMtimes: new Map(),
|
||||
});
|
||||
const repo = regexRepo({
|
||||
findBooksByLibraryFolder: vi.fn().mockResolvedValue([{ id: 1, status: 'present', folderPath: '/library/Author/Book', primaryFileId: 2 }]),
|
||||
findBookFilesByLibraryFolder: vi.fn().mockResolvedValue([makeBookFile()]),
|
||||
});
|
||||
const done = awaitScan(repo);
|
||||
const { service, regexMetadata, selfWriteRegistry } = makeService(repo);
|
||||
if (suppressed) selfWriteRegistry.begin([file.absolutePath]);
|
||||
await service.startScan(1, 'manual', true);
|
||||
await done;
|
||||
expect(regexMetadata.evaluate).toHaveBeenCalledTimes(suppressed ? 0 : 1);
|
||||
selfWriteRegistry.end([file.absolutePath]);
|
||||
expect(repo.failScanJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['file extraction failed', 'scoring failed after metadata was saved'])('continues later books without retry after %s', async (message) => {
|
||||
mockFindCandidates.mockResolvedValue({
|
||||
candidates: [
|
||||
makeCandidate('/library/Author/Book', [makeFileStat()]),
|
||||
makeCandidate('/library/Author/Other', [
|
||||
makeFileStat({ absolutePath: '/library/Author/Other/book.epub', relPath: 'Author/Other/book.epub', ino: 2002n }),
|
||||
]),
|
||||
],
|
||||
skippedDirs: new Set(),
|
||||
unchangedDirs: new Set(),
|
||||
dirMtimes: new Map(),
|
||||
});
|
||||
const repo = regexRepo();
|
||||
repo.createBook
|
||||
.mockResolvedValueOnce({ id: 1, status: 'present', folderPath: '/library/Author/Book' })
|
||||
.mockResolvedValueOnce({ id: 2, status: 'present', folderPath: '/library/Author/Other' });
|
||||
const done = awaitScan(repo);
|
||||
const { service, regexMetadata } = makeService(repo);
|
||||
regexMetadata.evaluate.mockResolvedValue({ matched: true, metadata: { title: 'Regex title' }, diagnostics: [] });
|
||||
mockMetadata.extractAndSaveWithRegex.mockRejectedValueOnce(new Error(message));
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
expect(mockMetadata.extractAndSaveWithRegex).toHaveBeenCalledTimes(2);
|
||||
expect(mockMetadata.extractAndSaveWithRegex).toHaveBeenNthCalledWith(2, 2, expect.any(Array), { title: 'Regex title' }, [
|
||||
'regex',
|
||||
'embedded',
|
||||
'opfFile',
|
||||
]);
|
||||
expect(mockMetadata.extractAndSave).not.toHaveBeenCalled();
|
||||
expect(repo.failScanJob).not.toHaveBeenCalled();
|
||||
expect(repo.completeScanJob).toHaveBeenCalled();
|
||||
expect(repo.promoteProcessingBookToPresent).toHaveBeenCalledWith(1);
|
||||
expect(repo.promoteProcessingBookToPresent).toHaveBeenCalledWith(2);
|
||||
});
|
||||
|
||||
it('still uses file metadata when regex evaluation throws before persistence', async () => {
|
||||
mockFindCandidates.mockResolvedValue({
|
||||
candidates: [makeCandidate('/library/Author/Book', [makeFileStat()])],
|
||||
skippedDirs: new Set(),
|
||||
unchangedDirs: new Set(),
|
||||
dirMtimes: new Map(),
|
||||
});
|
||||
const repo = regexRepo();
|
||||
const done = awaitScan(repo);
|
||||
const { service, regexMetadata } = makeService(repo);
|
||||
regexMetadata.evaluate.mockRejectedValue(new Error('worker unavailable'));
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
expect(mockMetadata.extractAndSaveWithRegex).not.toHaveBeenCalled();
|
||||
expect(mockMetadata.extractAndSave).toHaveBeenCalledTimes(1);
|
||||
expect(repo.failScanJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not re-extract unchanged books on a full scan after changing rules', async () => {
|
||||
mockFindCandidates.mockResolvedValue({
|
||||
candidates: [makeCandidate('/library/Author/Book', [makeFileStat()])],
|
||||
skippedDirs: new Set(),
|
||||
unchangedDirs: new Set(),
|
||||
dirMtimes: new Map(),
|
||||
});
|
||||
const repo = regexRepo({
|
||||
findBooksByLibraryFolder: vi.fn().mockResolvedValue([{ id: 1, status: 'present', folderPath: '/library/Author/Book', primaryFileId: 1 }]),
|
||||
findBookFilesByLibraryFolder: vi.fn().mockResolvedValue([makeBookFile()]),
|
||||
});
|
||||
const done = awaitScan(repo);
|
||||
const { service, regexMetadata } = makeService(repo);
|
||||
await service.startScan(1, 'manual', true);
|
||||
await done;
|
||||
expect(regexMetadata.evaluate).not.toHaveBeenCalled();
|
||||
expect(mockMetadata.extractAndSaveWithRegex).not.toHaveBeenCalled();
|
||||
expect(mockMetadata.extractAndSave).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
function candidatesFor(count: number) {
|
||||
return Array.from({ length: count }, (_, index) =>
|
||||
makeCandidate(`/library/Author/Book${index}`, [
|
||||
makeFileStat({ absolutePath: `/library/Author/Book${index}/book.epub`, relPath: `Author/Book${index}/book.epub`, ino: BigInt(3000 + index) }),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
it('falls back per book after a pool failure and skips regex once failures reach the limit', async () => {
|
||||
mockFindCandidates.mockResolvedValue({ candidates: candidatesFor(4), skippedDirs: new Set(), unchangedDirs: new Set(), dirMtimes: new Map() });
|
||||
const repo = regexRepo();
|
||||
for (let index = 0; index < 4; index++)
|
||||
repo.createBook.mockResolvedValueOnce({ id: index + 1, status: 'present', folderPath: `/library/Author/Book${index}` });
|
||||
const done = awaitScan(repo);
|
||||
const { service, regexMetadata } = makeService(repo);
|
||||
regexMetadata.evaluate.mockResolvedValue({ matched: false, metadata: {}, diagnostics: [{ code: 'timeout' }] });
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
expect(regexMetadata.evaluate).toHaveBeenCalledTimes(3);
|
||||
expect(mockMetadata.extractAndSave).toHaveBeenCalledTimes(4);
|
||||
expect(mockMetadata.extractAndSaveWithRegex).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps applying regex to later books after one book hits a pool failure', async () => {
|
||||
mockFindCandidates.mockResolvedValue({ candidates: candidatesFor(2), skippedDirs: new Set(), unchangedDirs: new Set(), dirMtimes: new Map() });
|
||||
const repo = regexRepo();
|
||||
repo.createBook
|
||||
.mockResolvedValueOnce({ id: 1, status: 'present', folderPath: '/library/Author/Book0' })
|
||||
.mockResolvedValueOnce({ id: 2, status: 'present', folderPath: '/library/Author/Book1' });
|
||||
const done = awaitScan(repo);
|
||||
const { service, regexMetadata } = makeService(repo);
|
||||
regexMetadata.evaluate
|
||||
.mockResolvedValueOnce({ matched: false, metadata: {}, diagnostics: [{ code: 'busy' }] })
|
||||
.mockResolvedValueOnce({ matched: true, metadata: { seriesName: 'Book1' }, diagnostics: [] });
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
expect(mockMetadata.extractAndSave).toHaveBeenCalledTimes(1);
|
||||
expect(mockMetadata.extractAndSaveWithRegex).toHaveBeenCalledWith(2, expect.any(Array), { seriesName: 'Book1' }, [
|
||||
'regex',
|
||||
'embedded',
|
||||
'opfFile',
|
||||
]);
|
||||
});
|
||||
|
||||
it('skips regex for the rest of the scan as soon as the rules themselves fail', async () => {
|
||||
mockFindCandidates.mockResolvedValue({ candidates: candidatesFor(2), skippedDirs: new Set(), unchangedDirs: new Set(), dirMtimes: new Map() });
|
||||
const repo = regexRepo();
|
||||
repo.createBook
|
||||
.mockResolvedValueOnce({ id: 1, status: 'present', folderPath: '/library/Author/Book0' })
|
||||
.mockResolvedValueOnce({ id: 2, status: 'present', folderPath: '/library/Author/Book1' });
|
||||
const done = awaitScan(repo);
|
||||
const { service, regexMetadata } = makeService(repo);
|
||||
regexMetadata.evaluate.mockResolvedValue({ matched: false, metadata: {}, diagnostics: [{ code: 'invalid_pattern', ruleIndex: 0 }] });
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
expect(regexMetadata.evaluate).toHaveBeenCalledTimes(1);
|
||||
expect(mockMetadata.extractAndSave).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('regex metadata outside a scan', () => {
|
||||
const regexConfig = { rules: [{ pattern: '^(?<series>[^/]+)/', flags: '' }] };
|
||||
const book = { id: 7, libraryId: 1, primaryFileId: 11, status: 'present', folderPath: '/library/Series/Book' };
|
||||
const primary = {
|
||||
id: 11,
|
||||
bookId: 7,
|
||||
format: 'epub',
|
||||
role: 'content',
|
||||
absolutePath: '/library/Series/Book/book.epub',
|
||||
relPath: 'Series/Book/book.epub',
|
||||
sizeBytes: 10,
|
||||
durationSeconds: null,
|
||||
mediaOverlayAvailable: false,
|
||||
sortOrder: 0,
|
||||
};
|
||||
function rulesRepo(regexMetadata: unknown) {
|
||||
return makeRepo({
|
||||
findBookById: vi.fn().mockResolvedValue(book),
|
||||
findBookFilesByBookId: vi.fn().mockResolvedValue([primary]),
|
||||
findLibrarySettings: vi.fn().mockResolvedValue({ metadataPrecedence: ['regex', 'embedded'], regexMetadata }),
|
||||
});
|
||||
}
|
||||
|
||||
it('leaves books in libraries without rules to the caller', async () => {
|
||||
const repo = rulesRepo(null);
|
||||
const { service, regexMetadata } = makeService(repo);
|
||||
await expect(service.extractMetadataWithLibraryRules(7)).resolves.toBe(false);
|
||||
expect(repo.findBookFilesByBookId).not.toHaveBeenCalled();
|
||||
expect(regexMetadata.evaluate).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('applies the rules to the primary file path with the library precedence', async () => {
|
||||
const { service, regexMetadata } = makeService(rulesRepo(regexConfig));
|
||||
regexMetadata.evaluate.mockResolvedValue({ matched: true, metadata: { seriesName: 'Series' }, diagnostics: [] });
|
||||
await expect(service.extractMetadataWithLibraryRules(7)).resolves.toBe(true);
|
||||
expect(regexMetadata.evaluate).toHaveBeenCalledWith(regexConfig, 'Series/Book/book.epub');
|
||||
expect(mockMetadata.extractAndSaveWithRegex).toHaveBeenCalledWith(
|
||||
7,
|
||||
[{ key: 'embedded', absolutePath: '/library/Series/Book/book.epub', format: 'epub' }],
|
||||
{ seriesName: 'Series' },
|
||||
['regex', 'embedded'],
|
||||
);
|
||||
expect(mockMetadata.extractAndSave).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('falls back to file metadata when no rule matches the path', async () => {
|
||||
const { service, regexMetadata } = makeService(rulesRepo(regexConfig));
|
||||
regexMetadata.evaluate.mockResolvedValue({ matched: false, metadata: {}, diagnostics: [] });
|
||||
await expect(service.extractMetadataWithLibraryRules(7)).resolves.toBe(true);
|
||||
expect(mockMetadata.extractAndSaveWithRegex).not.toHaveBeenCalled();
|
||||
expect(mockMetadata.extractAndSave).toHaveBeenCalledWith(7, '/library/Series/Book/book.epub', 'epub');
|
||||
});
|
||||
|
||||
it('does not read a book this instance is writing', async () => {
|
||||
const { service, regexMetadata, selfWriteRegistry } = makeService(rulesRepo(regexConfig));
|
||||
selfWriteRegistry.begin([primary.absolutePath]);
|
||||
await expect(service.extractMetadataWithLibraryRules(7)).resolves.toBe(true);
|
||||
selfWriteRegistry.end([primary.absolutePath]);
|
||||
expect(regexMetadata.evaluate).not.toHaveBeenCalled();
|
||||
expect(mockMetadata.extractAndSave).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { isTransientRegexFailure, RegexMetadataService } from '../metadata/regex/regex-metadata.service';
|
||||
import type { RegexMetadataConfig, RegexMetadataPreview } from '@bookorbit/types';
|
||||
import { ConflictException, Injectable, Logger, NotFoundException, OnApplicationBootstrap, Optional } from '@nestjs/common';
|
||||
import { createHash } from 'crypto';
|
||||
import { mapWithConcurrency } from '../../common/utils/batch.utils';
|
||||
@@ -90,7 +92,21 @@ interface ScanLookupMaps {
|
||||
fileIdsByBookId: Map<number, Set<number>>;
|
||||
}
|
||||
|
||||
interface RegexScanContext {
|
||||
config: RegexMetadataConfig | null;
|
||||
disabled: boolean;
|
||||
/** Books whose rules could not run even after a retry; past the limit the rest of the scan skips regex. */
|
||||
failures: number;
|
||||
}
|
||||
|
||||
const REGEX_SCAN_FAILURE_LIMIT = 3;
|
||||
|
||||
function createRegexScanContext(config: RegexMetadataConfig | null | undefined): RegexScanContext {
|
||||
return { config: config ?? null, disabled: false, failures: 0 };
|
||||
}
|
||||
|
||||
interface LibraryScanSettings {
|
||||
regexContext: RegexScanContext;
|
||||
allowedFormats: string[];
|
||||
formatPriority: string[];
|
||||
metadataPrecedence: string[];
|
||||
@@ -160,6 +176,7 @@ interface RegisteredFile {
|
||||
isNew: boolean;
|
||||
wasReassigned: boolean;
|
||||
wasChanged: boolean;
|
||||
wasPathChanged: boolean;
|
||||
mediaOverlayAvailable: boolean;
|
||||
/** This book's stored sort order for the file before this scan; null when the file is new to it. */
|
||||
previousSortOrder: number | null;
|
||||
@@ -175,6 +192,7 @@ interface ProcessedFileResult {
|
||||
isNew: boolean;
|
||||
reassigned: boolean;
|
||||
changed: boolean;
|
||||
pathChanged?: boolean;
|
||||
fileId: number | null;
|
||||
/** The book that owned the file before this scan moved it here. */
|
||||
previousBookId?: number;
|
||||
@@ -254,6 +272,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
private readonly notificationService: NotificationService,
|
||||
private readonly selfWriteRegistry: SelfWriteRegistry,
|
||||
private readonly coverStore: BookCoverStore,
|
||||
private readonly regexMetadata: RegexMetadataService,
|
||||
@Optional() private readonly autoFetchOrchestrator?: BookMetadataFetchOrchestratorService,
|
||||
@Optional() private readonly achievementEvents?: AchievementEventsService,
|
||||
@Optional() private readonly coverReconciler?: CoverSlotReconciler,
|
||||
@@ -709,6 +728,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
const allowedFormats = settings?.allowedFormats ?? [];
|
||||
const formatPriority = settings?.formatPriority ?? DEFAULT_FORMAT_PRIORITY;
|
||||
const metadataPrecedence = settings?.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT];
|
||||
const regexContext = createRegexScanContext(settings?.regexMetadata);
|
||||
const excludePatterns = settings?.excludePatterns ?? [];
|
||||
const organizationMode = normalizeOrganizationMode(settings?.organizationMode);
|
||||
const addedAtSource = normalizeAddedAtSource(settings?.addedAtSource);
|
||||
@@ -742,6 +762,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
organizationMode,
|
||||
addedAtSource,
|
||||
forceFullScan,
|
||||
regexContext,
|
||||
).catch((err) => {
|
||||
const errorClass = err instanceof Error ? err.name : 'Error';
|
||||
const errorMessage = sanitizeLogValue(err instanceof Error ? err.message : String(err));
|
||||
@@ -957,6 +978,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
|
||||
const rawSettings = await this.scannerRepo.findLibrarySettings(libraryId);
|
||||
const settings: LibraryScanSettings = {
|
||||
regexContext: createRegexScanContext(rawSettings?.regexMetadata),
|
||||
allowedFormats: rawSettings?.allowedFormats ?? [],
|
||||
formatPriority: rawSettings?.formatPriority ?? DEFAULT_FORMAT_PRIORITY,
|
||||
metadataPrecedence: rawSettings?.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT],
|
||||
@@ -1000,6 +1022,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
|
||||
const rawSettings = await this.scannerRepo.findLibrarySettings(libraryId);
|
||||
const settings: LibraryScanSettings = {
|
||||
regexContext: createRegexScanContext(rawSettings?.regexMetadata),
|
||||
allowedFormats: rawSettings?.allowedFormats ?? [],
|
||||
formatPriority: rawSettings?.formatPriority ?? DEFAULT_FORMAT_PRIORITY,
|
||||
metadataPrecedence: rawSettings?.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT],
|
||||
@@ -1083,6 +1106,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
settings.addedAtSource,
|
||||
false,
|
||||
candidateFolderPaths,
|
||||
settings.regexContext,
|
||||
);
|
||||
this.emitTargetedScanResult(libraryId, result);
|
||||
seenBookIds.add(result.bookId);
|
||||
@@ -1205,6 +1229,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
settings.addedAtSource,
|
||||
false,
|
||||
new Set([candidate.folderPath]),
|
||||
settings.regexContext,
|
||||
);
|
||||
const pruned = await this.pruneMissingBookFiles(result.bookId, result.retainedFileIds, maps.fileIdsByBookId, maps.fileByPath, maps.fileByIno, {
|
||||
added: 0,
|
||||
@@ -1277,6 +1302,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
settings.addedAtSource,
|
||||
false,
|
||||
new Set([candidate.folderPath]),
|
||||
settings.regexContext,
|
||||
);
|
||||
const pruned = await this.pruneMissingBookFiles(result.bookId, result.retainedFileIds, maps.fileIdsByBookId, maps.fileByPath, maps.fileByIno, {
|
||||
added: 0,
|
||||
@@ -1368,6 +1394,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
settings.addedAtSource,
|
||||
false,
|
||||
new Set([candidate.folderPath]),
|
||||
settings.regexContext,
|
||||
);
|
||||
const pruned = await this.pruneMissingBookFiles(result.bookId, result.retainedFileIds, maps.fileIdsByBookId, maps.fileByPath, maps.fileByIno, {
|
||||
added: 0,
|
||||
@@ -1392,6 +1419,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
organizationMode: OrganizationMode,
|
||||
addedAtSource: AddedAtSource,
|
||||
forceFullScan = false,
|
||||
regexContext?: RegexScanContext,
|
||||
): Promise<void> {
|
||||
const event = 'scanner.run_scan';
|
||||
const startedAt = Date.now();
|
||||
@@ -1477,6 +1505,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
addedAtSource,
|
||||
skippedDirs,
|
||||
unchangedDirs,
|
||||
regexContext,
|
||||
);
|
||||
totals.addedCount += counts.addedCount;
|
||||
totals.updatedCount += counts.updatedCount;
|
||||
@@ -1564,6 +1593,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
addedAtSource: AddedAtSource,
|
||||
skippedDirs: Set<string> = new Set(),
|
||||
unchangedDirs: Set<string> = new Set(),
|
||||
regexContext?: RegexScanContext,
|
||||
): Promise<ScanCounts> {
|
||||
const event = 'scanner.scan_folder_candidates';
|
||||
const startedAt = Date.now();
|
||||
@@ -1597,6 +1627,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
addedAtSource,
|
||||
isFirstScan,
|
||||
candidateFolderPaths,
|
||||
regexContext,
|
||||
);
|
||||
seenBookIds.add(result.bookId);
|
||||
if (result.created) importedBookIds.push(result.bookId);
|
||||
@@ -1837,6 +1868,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
addedAtSource: AddedAtSource,
|
||||
isFirstScan: boolean,
|
||||
candidateFolderPaths: Set<string>,
|
||||
regexContext?: RegexScanContext,
|
||||
): Promise<ProcessCandidateResult> {
|
||||
const { bookByFolderPath, booksByParentDir, fileByPath, fileByIno } = maps;
|
||||
const counts = { added: 0, updated: 0 };
|
||||
@@ -1924,6 +1956,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
isNew: processResult.isNew,
|
||||
wasReassigned: processResult.reassigned,
|
||||
wasChanged: processResult.changed,
|
||||
wasPathChanged: processResult.pathChanged === true,
|
||||
mediaOverlayAvailable: fileByPath.get(fileStat.absolutePath)?.mediaOverlayAvailable === true,
|
||||
previousSortOrder,
|
||||
});
|
||||
@@ -1960,7 +1993,11 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
|
||||
const metadataSources = this.buildMetadataExtractionSources(registeredFiles, winner, metadataPrecedence);
|
||||
const shouldExtractMetadata =
|
||||
metadataSources.some((source) => hasMetadataSourceChanged(source.file)) || (book.primaryFileId === null && winner !== null);
|
||||
metadataSources.some((source) => hasMetadataSourceChanged(source.file)) ||
|
||||
(regexContext?.config != null &&
|
||||
winner !== null &&
|
||||
(hasMetadataSourceChanged(winner) || winner.wasPathChanged || winner.fileId !== book.primaryFileId)) ||
|
||||
(book.primaryFileId === null && winner !== null);
|
||||
const audioContentFiles = contentFiles.filter((f) => f.format !== null && isAudioFormat(f.format!));
|
||||
const changedAudioFiles = audioContentFiles.filter(hasMetadataSourceChanged);
|
||||
const audioFilesNeedingDuration = audioContentFiles.filter(
|
||||
@@ -1996,7 +2033,16 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
|
||||
// 3b: Extract shared metadata from the first available configured source.
|
||||
if (shouldExtractMetadata && !selfWriteInProgress) {
|
||||
await this.extractFirstAvailableMetadataSource(book.id, metadataSources);
|
||||
const relativePath = winner ? candidate.files.find((file) => file.absolutePath === winner.absolutePath)?.relPath : undefined;
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
const regexApplied = await this.extractRegexMetadata(book.id, libraryId, relativePath, metadataSources, metadataPrecedence, regexContext);
|
||||
if (!regexApplied) await this.extractFirstAvailableMetadataSource(book.id, metadataSources);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`[scanner.extract_metadata] [fail] libraryId=${libraryId} bookId=${book.id} durationMs=${Date.now() - startedAt} errorClass=${error instanceof Error ? error.name : 'Error'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - metadata extraction failed; continuing scan without retry`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 3c: Write per-file duration for new, changed, or historically unprobed audio files.
|
||||
@@ -2050,6 +2096,101 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
return { bookId: book.id, ...counts, retainedFileIds, becameVisible, created: book.created, coverBookIds };
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared metadata extraction for one book outside a scan, honouring its library's regex rules.
|
||||
* An upload registers its own file and the watcher records a move's new path itself, so no later
|
||||
* scan sees anything to re-read. Resolves false when the library has no rules, leaving the
|
||||
* caller's plain file extraction in charge.
|
||||
*/
|
||||
async extractMetadataWithLibraryRules(bookId: number): Promise<boolean> {
|
||||
const book = await this.scannerRepo.findBookById(bookId);
|
||||
if (!book) return false;
|
||||
const settings = await this.scannerRepo.findLibrarySettings(book.libraryId);
|
||||
if (!settings?.regexMetadata) return false;
|
||||
const files = await this.scannerRepo.findBookFilesByBookId(bookId);
|
||||
if (files.some((file) => this.selfWriteRegistry.isSuppressed(file.absolutePath))) return true;
|
||||
|
||||
const registered: RegisteredFile[] = files.map((file) => ({
|
||||
fileId: file.id,
|
||||
format: file.format,
|
||||
role: file.role as FileRole,
|
||||
absolutePath: file.absolutePath,
|
||||
sizeBytes: file.sizeBytes ?? 0,
|
||||
durationSeconds: file.durationSeconds,
|
||||
isNew: false,
|
||||
wasReassigned: false,
|
||||
wasChanged: false,
|
||||
wasPathChanged: false,
|
||||
mediaOverlayAvailable: file.mediaOverlayAvailable,
|
||||
previousSortOrder: file.sortOrder,
|
||||
}));
|
||||
const winner = registered.find((file) => file.fileId === book.primaryFileId) ?? null;
|
||||
const relativePath = files.find((file) => file.id === book.primaryFileId)?.relPath ?? undefined;
|
||||
const precedence = settings.metadataPrecedence ?? [...LIBRARY_METADATA_PRECEDENCE_DEFAULT];
|
||||
const sources = this.buildMetadataExtractionSources(registered, winner, precedence);
|
||||
const context = createRegexScanContext(settings.regexMetadata);
|
||||
const regexApplied = await this.extractRegexMetadata(book.id, book.libraryId, relativePath, sources, precedence, context);
|
||||
if (!regexApplied) await this.extractFirstAvailableMetadataSource(book.id, sources);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Re-applies regex rules after the watcher moved these books' files; books in libraries without rules are left alone. */
|
||||
refreshRegexMetadataAfterMoveAsync(bookIds: number[]): void {
|
||||
void (async () => {
|
||||
for (const bookId of bookIds) {
|
||||
const startedAt = Date.now();
|
||||
try {
|
||||
await this.extractMetadataWithLibraryRules(bookId);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`[scanner.regex_metadata_after_move] [fail] bookId=${bookId} durationMs=${Date.now() - startedAt} errorClass=${error instanceof Error ? error.name : 'Error'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - regex metadata refresh after move failed`,
|
||||
);
|
||||
}
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
private async extractRegexMetadata(
|
||||
bookId: number,
|
||||
libraryId: number,
|
||||
relativePath: string | undefined,
|
||||
sources: MetadataExtractionSource[],
|
||||
precedence: string[],
|
||||
context?: RegexScanContext,
|
||||
): Promise<boolean> {
|
||||
if (!context?.config || context.disabled || !relativePath) return false;
|
||||
const startedAt = Date.now();
|
||||
let result: RegexMetadataPreview;
|
||||
try {
|
||||
result = await this.regexMetadata.evaluate(context.config, relativePath);
|
||||
const failure = result.diagnostics.find((entry) => entry.code !== 'invalid_value');
|
||||
if (failure) {
|
||||
// Rules that cannot compile fail every book at once; a pool failure that outlived a retry may be this book alone.
|
||||
const transient = isTransientRegexFailure(failure.code);
|
||||
if (transient) context.failures++;
|
||||
if (!transient || context.failures >= REGEX_SCAN_FAILURE_LIMIT) context.disabled = true;
|
||||
this.logger.warn(
|
||||
`[scanner.regex_metadata] [fail] libraryId=${libraryId} bookId=${bookId} durationMs=${Date.now() - startedAt} errorClass=RegexEvaluationError error="${sanitizeLogValue(failure.code)}" regexDisabled=${context.disabled} - regex skipped; file metadata used`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
if (!result.matched) return false;
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`[scanner.regex_metadata] [fail] libraryId=${libraryId} bookId=${bookId} durationMs=${Date.now() - startedAt} errorClass=${error instanceof Error ? error.name : 'Error'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - regex evaluation failed`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
// Persistence may already have written metadata before failing; never retry with different precedence.
|
||||
await this.metadataService.extractAndSaveWithRegex(
|
||||
bookId,
|
||||
sources.map((source) => ({ key: source.key, absolutePath: source.file.absolutePath, format: source.format })),
|
||||
result.metadata,
|
||||
precedence,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
private buildMetadataExtractionSources(
|
||||
registeredFiles: RegisteredFile[],
|
||||
winner: RegisteredFile | null,
|
||||
@@ -2618,7 +2759,14 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
mtime: fileStat.mtime,
|
||||
});
|
||||
}
|
||||
return { isNew: false, reassigned, changed: !sizeUnchanged || !mtimeUnchanged || !roleUnchanged, fileId: byPath.id, previousBookId };
|
||||
return {
|
||||
isNew: false,
|
||||
reassigned,
|
||||
changed: !sizeUnchanged || !mtimeUnchanged || !roleUnchanged,
|
||||
pathChanged: !relPathUnchanged,
|
||||
fileId: byPath.id,
|
||||
previousBookId,
|
||||
};
|
||||
}
|
||||
|
||||
private async resolveByLocalIno(
|
||||
@@ -2674,6 +2822,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
isNew: false,
|
||||
reassigned: byIno.bookId !== bookId,
|
||||
changed: !sizeUnchanged || !mtimeUnchanged,
|
||||
pathChanged: oldPathEntry?.relPath !== fileStat.relPath,
|
||||
fileId: byIno.id,
|
||||
previousBookId: byIno.bookId !== bookId ? byIno.bookId : undefined,
|
||||
};
|
||||
@@ -2757,6 +2906,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
isNew: false,
|
||||
reassigned: globalByIno.file.bookId !== bookId,
|
||||
changed: !sizeUnchanged || !mtimeUnchanged,
|
||||
pathChanged: globalByIno.file.relPath !== fileStat.relPath,
|
||||
fileId: globalByIno.file.id,
|
||||
previousBookId: globalByIno.file.bookId !== bookId ? globalByIno.file.bookId : undefined,
|
||||
};
|
||||
@@ -2840,6 +2990,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
isNew: false,
|
||||
reassigned: byHash.bookId !== bookId,
|
||||
changed: false,
|
||||
pathChanged: byHash.relPath !== fileStat.relPath,
|
||||
fileId: byHash.id,
|
||||
previousBookId: byHash.bookId !== bookId ? byHash.bookId : undefined,
|
||||
};
|
||||
@@ -2911,6 +3062,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
isNew: false,
|
||||
reassigned: globalByHash.file.bookId !== bookId,
|
||||
changed: false,
|
||||
pathChanged: globalByHash.file.relPath !== fileStat.relPath,
|
||||
fileId: globalByHash.file.id,
|
||||
previousBookId: globalByHash.file.bookId !== bookId ? globalByHash.file.bookId : undefined,
|
||||
};
|
||||
|
||||
@@ -780,4 +780,25 @@ describe('UploadProcessorService', () => {
|
||||
expect(coverReconciler.enqueue).toHaveBeenCalledWith([7], { filesChanged: true });
|
||||
expect(coverReconciler.enqueue.mock.invocationCallOrder[0]).toBeGreaterThan(metadataService.extractAndSave.mock.invocationCallOrder[0]!);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[true, 0],
|
||||
[false, 1],
|
||||
])('lets the library regex rules own extraction when they apply (%s)', async (handled, plainCalls) => {
|
||||
const scanner = { extractMetadataWithLibraryRules: vi.fn().mockResolvedValue(handled) };
|
||||
const withRules = new UploadProcessorService(
|
||||
db as any,
|
||||
metadataService as any,
|
||||
coverStore as any,
|
||||
orchestrator as any,
|
||||
undefined,
|
||||
scanner as any,
|
||||
);
|
||||
metadataService.extractAndSave.mockResolvedValue(undefined);
|
||||
|
||||
await withRules.extractMetadata(7, '/tmp/book.epub', 'epub');
|
||||
|
||||
expect(scanner.extractMetadataWithLibraryRules).toHaveBeenCalledWith(7);
|
||||
expect(metadataService.extractAndSave).toHaveBeenCalledTimes(plainCalls);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -19,6 +19,7 @@ import { BookMetadataFetchOrchestratorService } from '../book-metadata-fetch/boo
|
||||
import { BookCoverStore } from '../book-cover-store/book-cover-store.service';
|
||||
import { CoverSlotReconciler } from '../metadata/cover-slot-reconciler.service';
|
||||
import { MetadataService } from '../metadata/metadata.service';
|
||||
import { ScannerService } from '../scanner/scanner.service';
|
||||
import { METADATA_AUDIO_FORMATS } from '../metadata/metadata-extraction.service';
|
||||
import { computeFileHash } from '../scanner/lib/hash';
|
||||
import { inspectEpubMediaOverlayFields } from '../reader/epub/epub-media-overlay-capability';
|
||||
@@ -104,6 +105,7 @@ export class UploadProcessorService {
|
||||
private readonly coverStore: BookCoverStore,
|
||||
@Optional() private readonly autoFetchOrchestrator?: BookMetadataFetchOrchestratorService,
|
||||
@Optional() private readonly coverReconciler?: CoverSlotReconciler,
|
||||
@Optional() private readonly scanner?: ScannerService,
|
||||
) {}
|
||||
|
||||
/** A file joined or left these books, so their cover slots are brought back in line with it. */
|
||||
@@ -404,7 +406,9 @@ export class UploadProcessorService {
|
||||
);
|
||||
return;
|
||||
}
|
||||
await this.metadataService.extractAndSave(bookId, absolutePath, format);
|
||||
// An upload records its own file, so no later scan would apply the library's regex rules to it.
|
||||
const extractedWithRules = (await this.scanner?.extractMetadataWithLibraryRules(bookId)) ?? false;
|
||||
if (!extractedWithRules) await this.metadataService.extractAndSave(bookId, absolutePath, format);
|
||||
// The embedded extractor writes the aggregate duration to book_metadata, but the
|
||||
// per-file book_files.durationSeconds the player sums is only populated here.
|
||||
if (isAudioFormat(format)) {
|
||||
|
||||
@@ -7,6 +7,7 @@ import { FileWriteModule } from '../file-write/file-write.module';
|
||||
import { LibraryModule } from '../library/library.module';
|
||||
import { MetadataModule } from '../metadata/metadata.module';
|
||||
import { PathModule } from '../path/path.module';
|
||||
import { ScannerModule } from '../scanner/scanner.module';
|
||||
import { BookFileUploadController } from './book-file-upload.controller';
|
||||
import { UploadController } from './upload.controller';
|
||||
import { UploadProcessorService } from './upload-processor.service';
|
||||
@@ -18,7 +19,16 @@ import { UploadSessionRepository } from './upload-session.repository';
|
||||
import { UploadSessionService } from './upload-session.service';
|
||||
|
||||
@Module({
|
||||
imports: [AppSettingsModule, BookCoverStoreModule, LibraryModule, MetadataModule, BookMetadataFetchModule, FileWriteModule, PathModule],
|
||||
imports: [
|
||||
AppSettingsModule,
|
||||
BookCoverStoreModule,
|
||||
LibraryModule,
|
||||
MetadataModule,
|
||||
BookMetadataFetchModule,
|
||||
FileWriteModule,
|
||||
PathModule,
|
||||
ScannerModule,
|
||||
],
|
||||
controllers: [UploadController, BookFileUploadController, UploadSessionController],
|
||||
providers: [UploadService, UploadValidatorService, UploadStorageService, UploadProcessorService, UploadSessionRepository, UploadSessionService],
|
||||
exports: [UploadValidatorService, UploadStorageService, UploadProcessorService, UploadService],
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"totalRoutes": 660,
|
||||
"totalRoutes": 662,
|
||||
"byPermission": {
|
||||
"(none)": 286,
|
||||
"Permission.ManageMetadataConfig": 41,
|
||||
@@ -7,7 +7,7 @@
|
||||
"Permission.LibraryEditMetadata": 35,
|
||||
"Permission.BookDockAccess": 17,
|
||||
"Permission.ManageAppSettings": 62,
|
||||
"Permission.ManageLibraries": 22,
|
||||
"Permission.ManageLibraries": 24,
|
||||
"Permission.ManageUsers": 13,
|
||||
"Permission.ManageEmail": 8,
|
||||
"Permission.KoboSync": 13,
|
||||
|
||||
@@ -503,6 +503,30 @@
|
||||
"allowDefault": false,
|
||||
"useGuards": []
|
||||
},
|
||||
{
|
||||
"file": "server/src/modules/library/library.controller.ts",
|
||||
"className": "LibraryController",
|
||||
"methodName": "validateRegexMetadata",
|
||||
"httpMethod": "POST",
|
||||
"path": "/libraries/regex-metadata/validate",
|
||||
"permissions": ["Permission.ManageLibraries"],
|
||||
"libraryAccess": [],
|
||||
"isPublic": false,
|
||||
"allowDefault": false,
|
||||
"useGuards": []
|
||||
},
|
||||
{
|
||||
"file": "server/src/modules/library/library.controller.ts",
|
||||
"className": "LibraryController",
|
||||
"methodName": "previewRegexMetadata",
|
||||
"httpMethod": "POST",
|
||||
"path": "/libraries/regex-metadata/preview",
|
||||
"permissions": ["Permission.ManageLibraries"],
|
||||
"libraryAccess": [],
|
||||
"isPublic": false,
|
||||
"allowDefault": false,
|
||||
"useGuards": []
|
||||
},
|
||||
{
|
||||
"file": "server/src/modules/library/library.controller.ts",
|
||||
"className": "LibraryController",
|
||||
|
||||
@@ -4,7 +4,7 @@ import { basename, join } from 'path';
|
||||
|
||||
import { and, eq, sql } from 'drizzle-orm';
|
||||
import { Permission } from '@bookorbit/types';
|
||||
import type { AddedAtRecomputeJob } from '@bookorbit/types';
|
||||
import type { AddedAtRecomputeJob, RegexMetadataConfig } from '@bookorbit/types';
|
||||
|
||||
import * as schema from '../src/db/schema';
|
||||
import { waitForCondition, waitForScanCompletion } from './e2e/app-harness';
|
||||
@@ -140,6 +140,8 @@ async function createLibraryViaApi(
|
||||
readingThreshold: number;
|
||||
markAsFinishedPercentComplete: number;
|
||||
fileWriteEnabled: boolean;
|
||||
regexMetadata: RegexMetadataConfig;
|
||||
metadataPrecedence: string[];
|
||||
}>,
|
||||
): Promise<{
|
||||
response: InjectResponse;
|
||||
@@ -167,6 +169,8 @@ async function createLibraryViaApi(
|
||||
readingThreshold: input?.readingThreshold ?? 0.4,
|
||||
markAsFinishedPercentComplete: input?.markAsFinishedPercentComplete ?? 95,
|
||||
fileWriteEnabled: input?.fileWriteEnabled ?? false,
|
||||
regexMetadata: input?.regexMetadata,
|
||||
metadataPrecedence: input?.metadataPrecedence,
|
||||
},
|
||||
});
|
||||
expect(response.statusCode).toBe(201);
|
||||
@@ -210,6 +214,66 @@ describe('Library admin workflows (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () =
|
||||
await closeAuthorizationMatrixE2EContext(ctx);
|
||||
});
|
||||
|
||||
it('protects regex preview and validates its nested request contract', async () => {
|
||||
const payload = { config: { rules: [{ pattern: '^(?<title>.+)$', flags: '' }] }, relativePath: 'Book.epub' };
|
||||
for (const [token, status] of [
|
||||
[manager.accessToken, 200],
|
||||
[noPermissionUser.accessToken, 403],
|
||||
] as const) {
|
||||
const response = await ctx.app.inject({ method: 'POST', url: '/api/v1/libraries/regex-metadata/preview', headers: authHeader(token), payload });
|
||||
expect(response.statusCode).toBe(status);
|
||||
if (status === 200) expect(response.json()).toMatchObject({ inputPath: 'Book', metadata: { title: 'Book' } });
|
||||
}
|
||||
for (const bad of [
|
||||
{ ...payload, relativePath: '/private/Book.epub' },
|
||||
{ ...payload, relativePath: '../Book.epub' },
|
||||
{ ...payload, config: { rules: [{ ...payload.config.rules[0], template: '{title}' }] } },
|
||||
]) {
|
||||
const response = await ctx.app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/v1/libraries/regex-metadata/preview',
|
||||
headers: authHeader(manager.accessToken),
|
||||
payload: bad,
|
||||
});
|
||||
expect(response.statusCode).toBe(400);
|
||||
}
|
||||
// Like every other library setting, regex rules need ManageLibraries and no per-library grant.
|
||||
const library = await createLibraryWithFolder(ctx);
|
||||
const scoped = await ctx.app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/v1/libraries/regex-metadata/preview',
|
||||
headers: authHeader(manager.accessToken),
|
||||
payload: { ...payload, libraryId: library.libraryId },
|
||||
});
|
||||
expect(scoped.statusCode).toBe(200);
|
||||
const update = await ctx.app.inject({
|
||||
method: 'PATCH',
|
||||
url: `/api/v1/libraries/${library.libraryId}`,
|
||||
headers: authHeader(manager.accessToken),
|
||||
payload: { icon: 'BookOpen', regexMetadata: payload.config },
|
||||
});
|
||||
expect(update.statusCode, update.body).toBe(200);
|
||||
expect(update.json().regexMetadata).toEqual(payload.config);
|
||||
});
|
||||
|
||||
it('validates regex rules without a path and enforces access and DTO boundaries', async () => {
|
||||
const config = { rules: [{ pattern: '(?<title>.+)', flags: '' }] };
|
||||
const request = (payload: object, token = manager.accessToken) =>
|
||||
ctx.app.inject({ method: 'POST', url: '/api/v1/libraries/regex-metadata/validate', headers: authHeader(token), payload });
|
||||
const valid = await request({ config });
|
||||
expect(valid.statusCode).toBe(200);
|
||||
expect(valid.json()).toEqual({ valid: true, diagnostics: [] });
|
||||
const invalid = await request({ config: { rules: [{ pattern: '[', flags: '' }] } });
|
||||
expect(invalid.statusCode).toBe(200);
|
||||
expect(invalid.json()).toEqual({ valid: false, diagnostics: [{ code: 'invalid_pattern', ruleIndex: 0 }] });
|
||||
expect((await request({ config }, noPermissionUser.accessToken)).statusCode).toBe(403);
|
||||
expect((await request({ config, relativePath: 'unexpected.epub' })).statusCode).toBe(400);
|
||||
expect((await request({ config: { rules: [{ pattern: '(?<title>.+)', flags: 'g' }] } })).statusCode).toBe(400);
|
||||
const library = await createLibraryWithFolder(ctx);
|
||||
expect((await request({ config, libraryId: library.libraryId })).statusCode).toBe(200);
|
||||
expect((await request({ config, libraryId: 2_147_483_000 })).statusCode).toBe(404);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ organizationMode: 'book_per_file', autoScanCronExpression: '0 2 * * *', watch: true, fileWriteEnabled: true },
|
||||
{ organizationMode: 'book_per_folder', autoScanCronExpression: null, watch: false, fileWriteEnabled: false },
|
||||
@@ -302,6 +366,32 @@ describe('Library admin workflows (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () =
|
||||
expect(storedLibrary).toMatchObject(savedSettings);
|
||||
});
|
||||
|
||||
it.each([null, { rules: [{ pattern: '(?<title>.+)', flags: '' }] }])(
|
||||
'lets managers save regex rules and unrelated settings without a library grant: %j',
|
||||
async (regexMetadata) => {
|
||||
const library = await createLibraryWithFolder(ctx);
|
||||
await ctx.db.update(schema.libraries).set({ regexMetadata }).where(eq(schema.libraries.id, library.libraryId));
|
||||
const response = await ctx.app.inject({
|
||||
method: 'PATCH',
|
||||
url: `/api/v1/libraries/${library.libraryId}`,
|
||||
headers: authHeader(manager.accessToken),
|
||||
payload: { name: `Renamed library ${library.libraryId}`, icon: 'BookOpen', regexMetadata },
|
||||
});
|
||||
expect(response.statusCode, response.body).toBe(200);
|
||||
expect(response.json()).toMatchObject({ name: `Renamed library ${library.libraryId}`, regexMetadata });
|
||||
const nextRegexMetadata = regexMetadata === null ? { rules: [{ pattern: '(?<title>.+)', flags: '' }] } : null;
|
||||
const changed = await ctx.app.inject({
|
||||
method: 'PATCH',
|
||||
url: `/api/v1/libraries/${library.libraryId}`,
|
||||
headers: authHeader(manager.accessToken),
|
||||
payload: { icon: 'BookOpen', regexMetadata: nextRegexMetadata },
|
||||
});
|
||||
expect(changed.statusCode, changed.body).toBe(200);
|
||||
const [stored] = await ctx.db.select().from(schema.libraries).where(eq(schema.libraries.id, library.libraryId));
|
||||
expect(stored.regexMetadata).toEqual(nextRegexMetadata);
|
||||
},
|
||||
);
|
||||
|
||||
describe('date added recompute', () => {
|
||||
async function finishRecompute(libraryId: number): Promise<AddedAtRecomputeJob> {
|
||||
let result: AddedAtRecomputeJob;
|
||||
@@ -578,6 +668,14 @@ describe('Library admin workflows (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () =
|
||||
});
|
||||
|
||||
describe('create library and manage access', () => {
|
||||
it.each([undefined, ['regex', 'embedded']])('creates regex rules with precedence %j', async (metadataPrecedence) => {
|
||||
const regexMetadata = { rules: [{ pattern: '(?<title>.+)', flags: '' }] };
|
||||
const { body: library } = await createLibraryViaApi(ctx, manager.accessToken, { regexMetadata, metadataPrecedence });
|
||||
expect(library.regexMetadata).toEqual(regexMetadata);
|
||||
expect(library.metadataPrecedence).toEqual(metadataPrecedence ?? ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar', 'regex']);
|
||||
await waitForNoRunningScans(ctx, library.id);
|
||||
});
|
||||
|
||||
it('persists fractional finished thresholds through create and update', async () => {
|
||||
const { body: createdLibrary } = await createLibraryViaApi(ctx, manager.accessToken, {
|
||||
markAsFinishedPercentComplete: 99.95,
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { ScannerService } from '../src/modules/scanner/scanner.service';
|
||||
import { eq } from 'drizzle-orm';
|
||||
import { bookMetadata, bookAuthors, bookGenres } from '../src/db/schema';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { readFile, rename } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
@@ -22,6 +25,7 @@ import {
|
||||
triggerAndWaitForLibraryScan,
|
||||
uploadBookCover,
|
||||
waitForCondition,
|
||||
waitForScanCompletion,
|
||||
waitForMetadataFetchIdle,
|
||||
type MetadataLockE2EContext,
|
||||
} from './e2e/metadata-lock/metadata-lock-harness';
|
||||
@@ -48,6 +52,94 @@ describe('metadata-lock e2e', { timeout: 30_000 }, () => {
|
||||
await closeMetadataLockE2EContext(ctx);
|
||||
});
|
||||
|
||||
it.each(['book_per_file', 'book_per_folder'] as const)(
|
||||
'merges relative regex metadata and preserves unchanged and locked fields (%s)',
|
||||
async (mode) => {
|
||||
const library = await createLibraryWithFolder(ctx, { mode });
|
||||
const fullScan = async () => {
|
||||
const { jobId } = await ctx.app.get(ScannerService).startScan(library.libraryId, 'manual', true);
|
||||
await waitForScanCompletion(ctx.db, jobId);
|
||||
};
|
||||
const relPath = 'My.Series/08 - File.Name.epub';
|
||||
await createEpubFixture(library.folderPath, relPath, { title: 'Embedded title' });
|
||||
const config = { rules: [{ pattern: '^(?<series>[^/]+)/', flags: '' }] };
|
||||
const configure = await ctx.app.inject({
|
||||
method: 'PATCH',
|
||||
url: `/api/v1/libraries/${library.libraryId}`,
|
||||
headers: authHeader(ctx.adminToken),
|
||||
payload: { icon: 'BookOpen', regexMetadata: config, metadataPrecedence: ['regex', 'embedded', 'opfFile'] },
|
||||
});
|
||||
expect(configure.statusCode).toBe(200);
|
||||
expect(configure.json().regexMetadata).toEqual(config);
|
||||
const preview = await ctx.app.inject({
|
||||
method: 'POST',
|
||||
url: '/api/v1/libraries/regex-metadata/preview',
|
||||
headers: authHeader(ctx.adminToken),
|
||||
payload: { libraryId: library.libraryId, config, relativePath: relPath },
|
||||
});
|
||||
expect(preview.statusCode).toBe(200);
|
||||
expect(preview.json()).toMatchObject({ inputPath: 'My.Series/08 - File.Name', metadata: { seriesName: 'My.Series' } });
|
||||
await fullScan();
|
||||
const located = await locateBookFileByRelPath(ctx, library.libraryId, relPath);
|
||||
const read = async () => (await ctx.db.select().from(bookMetadata).where(eq(bookMetadata.bookId, located.bookId)))[0];
|
||||
expect(await read()).toMatchObject({ title: 'Embedded title', seriesName: 'My.Series' });
|
||||
const originalAuthors = await ctx.db.select().from(bookAuthors).where(eq(bookAuthors.bookId, located.bookId));
|
||||
const changed = await ctx.app.inject({
|
||||
method: 'PATCH',
|
||||
url: `/api/v1/libraries/${library.libraryId}`,
|
||||
headers: authHeader(ctx.adminToken),
|
||||
payload: { regexMetadata: { rules: [{ pattern: '^(?<series>[^/]+)/(?<seriesIndex>[^ ]+) - (?<title>.+)$', flags: '' }] } },
|
||||
});
|
||||
expect(changed.statusCode).toBe(200);
|
||||
await fullScan();
|
||||
expect(await read()).toMatchObject({ title: 'Embedded title', seriesIndex: null });
|
||||
expect((await patchLocks(located.bookId, ['title'])).statusCode).toBe(200);
|
||||
await pause(1100);
|
||||
await createEpubFixture(library.folderPath, relPath, { title: 'Changed embedded title with a different size' });
|
||||
await fullScan();
|
||||
expect(await read()).toMatchObject({ title: 'Embedded title', seriesName: 'My.Series', seriesIndex: '08' });
|
||||
expect(await ctx.db.select().from(bookAuthors).where(eq(bookAuthors.bookId, located.bookId))).toEqual(originalAuthors);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(['book_per_file', 'book_per_folder'] as const)('preserves unlocked relations on a partial regex rescan (%s)', async (mode) => {
|
||||
const library = await createLibraryWithFolder(ctx, { mode });
|
||||
const fullScan = async () => {
|
||||
const { jobId } = await ctx.app.get(ScannerService).startScan(library.libraryId, 'manual', true);
|
||||
await waitForScanCompletion(ctx.db, jobId);
|
||||
};
|
||||
const relPath = 'Path Series/book.epub';
|
||||
await createEpubWithCoverFixture(library.folderPath, relPath, { title: 'Original title' });
|
||||
await fullScan();
|
||||
const { bookId } = await locateBookFileByRelPath(ctx, library.libraryId, relPath);
|
||||
expect((await patchMetadata(bookId, { authors: ['Existing Author'], genres: ['Existing Genre'] })).statusCode).toBe(200);
|
||||
expect((await patchLocks(bookId, [])).statusCode).toBe(200);
|
||||
const originalAuthors = await ctx.db.select().from(bookAuthors).where(eq(bookAuthors.bookId, bookId));
|
||||
const originalGenres = await ctx.db.select().from(bookGenres).where(eq(bookGenres.bookId, bookId));
|
||||
expect(originalAuthors).toHaveLength(1);
|
||||
expect(originalGenres).toHaveLength(1);
|
||||
const configure = await ctx.app.inject({
|
||||
method: 'PATCH',
|
||||
url: `/api/v1/libraries/${library.libraryId}`,
|
||||
headers: authHeader(ctx.adminToken),
|
||||
payload: {
|
||||
icon: 'BookOpen',
|
||||
regexMetadata: { rules: [{ pattern: '^(?<series>[^/]+)/', flags: '' }] },
|
||||
metadataPrecedence: ['regex', 'embedded', 'opfFile'],
|
||||
},
|
||||
});
|
||||
expect(configure.statusCode).toBe(200);
|
||||
await pause(1100);
|
||||
await createEpubWithCoverFixture(library.folderPath, relPath, { title: 'Changed embedded title with a different size' });
|
||||
await fullScan();
|
||||
|
||||
expect((await locateBookFileByRelPath(ctx, library.libraryId, relPath)).bookId).toBe(bookId);
|
||||
const [metadata] = await ctx.db.select().from(bookMetadata).where(eq(bookMetadata.bookId, bookId));
|
||||
expect(metadata).toMatchObject({ title: 'Changed embedded title with a different size', seriesName: 'Path Series', lockedFields: [] });
|
||||
expect(await ctx.db.select().from(bookAuthors).where(eq(bookAuthors.bookId, bookId))).toEqual(originalAuthors);
|
||||
expect(await ctx.db.select().from(bookGenres).where(eq(bookGenres.bookId, bookId))).toEqual(originalGenres);
|
||||
});
|
||||
|
||||
it('normalizes and persists lock updates and rejects duplicate lock fields', async () => {
|
||||
const scanned = await scanSingleEpub('lock-contract', { title: 'Lock Contract Base' });
|
||||
|
||||
|
||||
Reference in New Issue
Block a user