Migrations/bookorbit
1
0
Fork 0
mirror of https://github.com/bookorbit/bookorbit.git synced 2026-11-03 00:11:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

feat(scanner): regex metadata extraction (#1669)

Browse Source 
feat(metadata): extract metadata from file paths with regex rules (#1669)

Let libraries define ordered regex rules that read title, authors, series,
series number and dates from each book's path relative to its library folder.
Run rules in a bounded worker pool, honour source precedence and field locks,
and apply them on scans, uploads and watcher moves. Also retry unprobed M4B
tag writes with the mp4 muxer.

Closes #1351

Co-authored-by: neon <18559658+neonsolstice@users.noreply.github.com>
This commit is contained in:
Dennis
2026-10-17 00:24:11 +02:00
committed by GitHub
parent fd10f89586
commit 428aefe8d1
50 changed files with 26474 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/test/e2e/authorization-matrix/route-inventory/manifest.json
+2 -2
View File
@@ -1,5 +1,5 @@
{
"totalRoutes": 660,
"totalRoutes": 662,
"byPermission": {
"(none)": 286,
"Permission.ManageMetadataConfig": 41,
@@ -7,7 +7,7 @@
"Permission.LibraryEditMetadata": 35,
"Permission.BookDockAccess": 17,
"Permission.ManageAppSettings": 62,
"Permission.ManageLibraries": 22,
"Permission.ManageLibraries": 24,
"Permission.ManageUsers": 13,
"Permission.ManageEmail": 8,
"Permission.KoboSync": 13,
server/test/e2e/authorization-matrix/route-inventory/routes-04.json
+24
View File
@@ -503,6 +503,30 @@
"allowDefault": false,
"useGuards": []
},
{
"file": "server/src/modules/library/library.controller.ts",
"className": "LibraryController",
"methodName": "validateRegexMetadata",
"httpMethod": "POST",
"path": "/libraries/regex-metadata/validate",
"permissions": ["Permission.ManageLibraries"],
"libraryAccess": [],
"isPublic": false,
"allowDefault": false,
"useGuards": []
},
{
"file": "server/src/modules/library/library.controller.ts",
"className": "LibraryController",
"methodName": "previewRegexMetadata",
"httpMethod": "POST",
"path": "/libraries/regex-metadata/preview",
"permissions": ["Permission.ManageLibraries"],
"libraryAccess": [],
"isPublic": false,
"allowDefault": false,
"useGuards": []
},
{
"file": "server/src/modules/library/library.controller.ts",
"className": "LibraryController",
server/test/library-admin-workflows.e2e-spec.ts
+99 -1
View File
@@ -4,7 +4,7 @@ import { basename, join } from 'path';
import { and, eq, sql } from 'drizzle-orm';
import { Permission } from '@bookorbit/types';
import type { AddedAtRecomputeJob } from '@bookorbit/types';
import type { AddedAtRecomputeJob, RegexMetadataConfig } from '@bookorbit/types';
import * as schema from '../src/db/schema';
import { waitForCondition, waitForScanCompletion } from './e2e/app-harness';
@@ -140,6 +140,8 @@ async function createLibraryViaApi(
readingThreshold: number;
markAsFinishedPercentComplete: number;
fileWriteEnabled: boolean;
regexMetadata: RegexMetadataConfig;
metadataPrecedence: string[];
}>,
): Promise<{
response: InjectResponse;
@@ -167,6 +169,8 @@ async function createLibraryViaApi(
readingThreshold: input?.readingThreshold ?? 0.4,
markAsFinishedPercentComplete: input?.markAsFinishedPercentComplete ?? 95,
fileWriteEnabled: input?.fileWriteEnabled ?? false,
regexMetadata: input?.regexMetadata,
metadataPrecedence: input?.metadataPrecedence,
},
});
expect(response.statusCode).toBe(201);
@@ -210,6 +214,66 @@ describe('Library admin workflows (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () =
await closeAuthorizationMatrixE2EContext(ctx);
});
it('protects regex preview and validates its nested request contract', async () => {
const payload = { config: { rules: [{ pattern: '^(?<title>.+)$', flags: '' }] }, relativePath: 'Book.epub' };
for (const [token, status] of [
[manager.accessToken, 200],
[noPermissionUser.accessToken, 403],
] as const) {
const response = await ctx.app.inject({ method: 'POST', url: '/api/v1/libraries/regex-metadata/preview', headers: authHeader(token), payload });
expect(response.statusCode).toBe(status);
if (status === 200) expect(response.json()).toMatchObject({ inputPath: 'Book', metadata: { title: 'Book' } });
}
for (const bad of [
{ ...payload, relativePath: '/private/Book.epub' },
{ ...payload, relativePath: '../Book.epub' },
{ ...payload, config: { rules: [{ ...payload.config.rules[0], template: '{title}' }] } },
]) {
const response = await ctx.app.inject({
method: 'POST',
url: '/api/v1/libraries/regex-metadata/preview',
headers: authHeader(manager.accessToken),
payload: bad,
});
expect(response.statusCode).toBe(400);
}
// Like every other library setting, regex rules need ManageLibraries and no per-library grant.
const library = await createLibraryWithFolder(ctx);
const scoped = await ctx.app.inject({
method: 'POST',
url: '/api/v1/libraries/regex-metadata/preview',
headers: authHeader(manager.accessToken),
payload: { ...payload, libraryId: library.libraryId },
});
expect(scoped.statusCode).toBe(200);
const update = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(manager.accessToken),
payload: { icon: 'BookOpen', regexMetadata: payload.config },
});
expect(update.statusCode, update.body).toBe(200);
expect(update.json().regexMetadata).toEqual(payload.config);
});
it('validates regex rules without a path and enforces access and DTO boundaries', async () => {
const config = { rules: [{ pattern: '(?<title>.+)', flags: '' }] };
const request = (payload: object, token = manager.accessToken) =>
ctx.app.inject({ method: 'POST', url: '/api/v1/libraries/regex-metadata/validate', headers: authHeader(token), payload });
const valid = await request({ config });
expect(valid.statusCode).toBe(200);
expect(valid.json()).toEqual({ valid: true, diagnostics: [] });
const invalid = await request({ config: { rules: [{ pattern: '[', flags: '' }] } });
expect(invalid.statusCode).toBe(200);
expect(invalid.json()).toEqual({ valid: false, diagnostics: [{ code: 'invalid_pattern', ruleIndex: 0 }] });
expect((await request({ config }, noPermissionUser.accessToken)).statusCode).toBe(403);
expect((await request({ config, relativePath: 'unexpected.epub' })).statusCode).toBe(400);
expect((await request({ config: { rules: [{ pattern: '(?<title>.+)', flags: 'g' }] } })).statusCode).toBe(400);
const library = await createLibraryWithFolder(ctx);
expect((await request({ config, libraryId: library.libraryId })).statusCode).toBe(200);
expect((await request({ config, libraryId: 2_147_483_000 })).statusCode).toBe(404);
});
it.each([
{ organizationMode: 'book_per_file', autoScanCronExpression: '0 2 * * *', watch: true, fileWriteEnabled: true },
{ organizationMode: 'book_per_folder', autoScanCronExpression: null, watch: false, fileWriteEnabled: false },
@@ -302,6 +366,32 @@ describe('Library admin workflows (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () =
expect(storedLibrary).toMatchObject(savedSettings);
});
it.each([null, { rules: [{ pattern: '(?<title>.+)', flags: '' }] }])(
'lets managers save regex rules and unrelated settings without a library grant: %j',
async (regexMetadata) => {
const library = await createLibraryWithFolder(ctx);
await ctx.db.update(schema.libraries).set({ regexMetadata }).where(eq(schema.libraries.id, library.libraryId));
const response = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(manager.accessToken),
payload: { name: `Renamed library ${library.libraryId}`, icon: 'BookOpen', regexMetadata },
});
expect(response.statusCode, response.body).toBe(200);
expect(response.json()).toMatchObject({ name: `Renamed library ${library.libraryId}`, regexMetadata });
const nextRegexMetadata = regexMetadata === null ? { rules: [{ pattern: '(?<title>.+)', flags: '' }] } : null;
const changed = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(manager.accessToken),
payload: { icon: 'BookOpen', regexMetadata: nextRegexMetadata },
});
expect(changed.statusCode, changed.body).toBe(200);
const [stored] = await ctx.db.select().from(schema.libraries).where(eq(schema.libraries.id, library.libraryId));
expect(stored.regexMetadata).toEqual(nextRegexMetadata);
},
);
describe('date added recompute', () => {
async function finishRecompute(libraryId: number): Promise<AddedAtRecomputeJob> {
let result: AddedAtRecomputeJob;
@@ -578,6 +668,14 @@ describe('Library admin workflows (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () =
});
describe('create library and manage access', () => {
it.each([undefined, ['regex', 'embedded']])('creates regex rules with precedence %j', async (metadataPrecedence) => {
const regexMetadata = { rules: [{ pattern: '(?<title>.+)', flags: '' }] };
const { body: library } = await createLibraryViaApi(ctx, manager.accessToken, { regexMetadata, metadataPrecedence });
expect(library.regexMetadata).toEqual(regexMetadata);
expect(library.metadataPrecedence).toEqual(metadataPrecedence ?? ['folderStructure', 'embedded', 'nfoFile', 'opfFile', 'sidecar', 'regex']);
await waitForNoRunningScans(ctx, library.id);
});
it('persists fractional finished thresholds through create and update', async () => {
const { body: createdLibrary } = await createLibraryViaApi(ctx, manager.accessToken, {
markAsFinishedPercentComplete: 99.95,
server/test/metadata-lock.e2e-spec.ts
+92
View File
@@ -1,3 +1,6 @@
import { ScannerService } from '../src/modules/scanner/scanner.service';
import { eq } from 'drizzle-orm';
import { bookMetadata, bookAuthors, bookGenres } from '../src/db/schema';
import { randomUUID } from 'crypto';
import { readFile, rename } from 'fs/promises';
import { join } from 'path';
@@ -22,6 +25,7 @@ import {
triggerAndWaitForLibraryScan,
uploadBookCover,
waitForCondition,
waitForScanCompletion,
waitForMetadataFetchIdle,
type MetadataLockE2EContext,
} from './e2e/metadata-lock/metadata-lock-harness';
@@ -48,6 +52,94 @@ describe('metadata-lock e2e', { timeout: 30_000 }, () => {
await closeMetadataLockE2EContext(ctx);
});
it.each(['book_per_file', 'book_per_folder'] as const)(
'merges relative regex metadata and preserves unchanged and locked fields (%s)',
async (mode) => {
const library = await createLibraryWithFolder(ctx, { mode });
const fullScan = async () => {
const { jobId } = await ctx.app.get(ScannerService).startScan(library.libraryId, 'manual', true);
await waitForScanCompletion(ctx.db, jobId);
};
const relPath = 'My.Series/08 - File.Name.epub';
await createEpubFixture(library.folderPath, relPath, { title: 'Embedded title' });
const config = { rules: [{ pattern: '^(?<series>[^/]+)/', flags: '' }] };
const configure = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(ctx.adminToken),
payload: { icon: 'BookOpen', regexMetadata: config, metadataPrecedence: ['regex', 'embedded', 'opfFile'] },
});
expect(configure.statusCode).toBe(200);
expect(configure.json().regexMetadata).toEqual(config);
const preview = await ctx.app.inject({
method: 'POST',
url: '/api/v1/libraries/regex-metadata/preview',
headers: authHeader(ctx.adminToken),
payload: { libraryId: library.libraryId, config, relativePath: relPath },
});
expect(preview.statusCode).toBe(200);
expect(preview.json()).toMatchObject({ inputPath: 'My.Series/08 - File.Name', metadata: { seriesName: 'My.Series' } });
await fullScan();
const located = await locateBookFileByRelPath(ctx, library.libraryId, relPath);
const read = async () => (await ctx.db.select().from(bookMetadata).where(eq(bookMetadata.bookId, located.bookId)))[0];
expect(await read()).toMatchObject({ title: 'Embedded title', seriesName: 'My.Series' });
const originalAuthors = await ctx.db.select().from(bookAuthors).where(eq(bookAuthors.bookId, located.bookId));
const changed = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(ctx.adminToken),
payload: { regexMetadata: { rules: [{ pattern: '^(?<series>[^/]+)/(?<seriesIndex>[^ ]+) - (?<title>.+)$', flags: '' }] } },
});
expect(changed.statusCode).toBe(200);
await fullScan();
expect(await read()).toMatchObject({ title: 'Embedded title', seriesIndex: null });
expect((await patchLocks(located.bookId, ['title'])).statusCode).toBe(200);
await pause(1100);
await createEpubFixture(library.folderPath, relPath, { title: 'Changed embedded title with a different size' });
await fullScan();
expect(await read()).toMatchObject({ title: 'Embedded title', seriesName: 'My.Series', seriesIndex: '08' });
expect(await ctx.db.select().from(bookAuthors).where(eq(bookAuthors.bookId, located.bookId))).toEqual(originalAuthors);
},
);
it.each(['book_per_file', 'book_per_folder'] as const)('preserves unlocked relations on a partial regex rescan (%s)', async (mode) => {
const library = await createLibraryWithFolder(ctx, { mode });
const fullScan = async () => {
const { jobId } = await ctx.app.get(ScannerService).startScan(library.libraryId, 'manual', true);
await waitForScanCompletion(ctx.db, jobId);
};
const relPath = 'Path Series/book.epub';
await createEpubWithCoverFixture(library.folderPath, relPath, { title: 'Original title' });
await fullScan();
const { bookId } = await locateBookFileByRelPath(ctx, library.libraryId, relPath);
expect((await patchMetadata(bookId, { authors: ['Existing Author'], genres: ['Existing Genre'] })).statusCode).toBe(200);
expect((await patchLocks(bookId, [])).statusCode).toBe(200);
const originalAuthors = await ctx.db.select().from(bookAuthors).where(eq(bookAuthors.bookId, bookId));
const originalGenres = await ctx.db.select().from(bookGenres).where(eq(bookGenres.bookId, bookId));
expect(originalAuthors).toHaveLength(1);
expect(originalGenres).toHaveLength(1);
const configure = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/libraries/${library.libraryId}`,
headers: authHeader(ctx.adminToken),
payload: {
icon: 'BookOpen',
regexMetadata: { rules: [{ pattern: '^(?<series>[^/]+)/', flags: '' }] },
metadataPrecedence: ['regex', 'embedded', 'opfFile'],
},
});
expect(configure.statusCode).toBe(200);
await pause(1100);
await createEpubWithCoverFixture(library.folderPath, relPath, { title: 'Changed embedded title with a different size' });
await fullScan();
expect((await locateBookFileByRelPath(ctx, library.libraryId, relPath)).bookId).toBe(bookId);
const [metadata] = await ctx.db.select().from(bookMetadata).where(eq(bookMetadata.bookId, bookId));
expect(metadata).toMatchObject({ title: 'Changed embedded title with a different size', seriesName: 'Path Series', lockedFields: [] });
expect(await ctx.db.select().from(bookAuthors).where(eq(bookAuthors.bookId, bookId))).toEqual(originalAuthors);
expect(await ctx.db.select().from(bookGenres).where(eq(bookGenres.bookId, bookId))).toEqual(originalGenres);
});
it('normalizes and persists lock updates and rejects duplicate lock fields', async () => {
const scanned = await scanSingleEpub('lock-contract', { title: 'Lock Contract Base' });