fix(metadata): offload large pdf parsing to worker thread

Offload large PDF parsing to a worker thread to prevent the Node.js event loop from being blocked. This ensures the application remains responsive during heavy metadata extraction tasks.

Closes #371
This commit is contained in:
neon
2026-06-20 21:42:10 +00:00
parent f1a7ba6070
commit 21f15c3116
11 changed files with 799 additions and 133 deletions
@@ -1,16 +1,18 @@
import { readFile, rename, unlink, writeFile } from 'fs/promises'; import { readFile, rename, stat, unlink, writeFile } from 'fs/promises';
import type { MockedFunction } from 'vitest'; import type { MockedFunction } from 'vitest';
import { randomUUID } from 'crypto'; import { randomUUID } from 'crypto';
import { PDFDocument, PDFName } from 'pdf-lib'; import { PDFDocument, PDFName } from 'pdf-lib';
import { PdfFormatWriter } from './pdf-format-writer'; import { PDF_WORKER_WRITE_THRESHOLD_BYTES, PdfFormatWriter } from './pdf-format-writer';
import { buildXmp } from './pdf-xmp-builder'; import { buildXmp } from './pdf-xmp-builder';
import { writePdfMetadataInWorker } from './pdf-worker-runner';
vi.mock('fs/promises', async () => { vi.mock('fs/promises', async () => {
const actual = await vi.importActual('fs/promises'); const actual = await vi.importActual('fs/promises');
return { return {
...actual, ...actual,
readFile: vi.fn(), readFile: vi.fn(),
stat: vi.fn(),
writeFile: vi.fn(), writeFile: vi.fn(),
rename: vi.fn(), rename: vi.fn(),
unlink: vi.fn(), unlink: vi.fn(),
@@ -38,7 +40,12 @@ vi.mock('./pdf-xmp-builder', () => ({
buildXmp: vi.fn(), buildXmp: vi.fn(),
})); }));
vi.mock('./pdf-worker-runner', () => ({
writePdfMetadataInWorker: vi.fn(),
}));
const mockReadFile = readFile as MockedFunction<typeof readFile>; const mockReadFile = readFile as MockedFunction<typeof readFile>;
const mockStat = stat as MockedFunction<typeof stat>;
const mockWriteFile = writeFile as MockedFunction<typeof writeFile>; const mockWriteFile = writeFile as MockedFunction<typeof writeFile>;
const mockRename = rename as MockedFunction<typeof rename>; const mockRename = rename as MockedFunction<typeof rename>;
const mockUnlink = unlink as MockedFunction<typeof unlink>; const mockUnlink = unlink as MockedFunction<typeof unlink>;
@@ -46,6 +53,7 @@ const mockRandomUuid = randomUUID as MockedFunction<typeof randomUUID>;
const mockPdfLoad = PDFDocument.load as MockedFunction<typeof PDFDocument.load>; const mockPdfLoad = PDFDocument.load as MockedFunction<typeof PDFDocument.load>;
const mockPdfNameOf = PDFName.of as MockedFunction<typeof PDFName.of>; const mockPdfNameOf = PDFName.of as MockedFunction<typeof PDFName.of>;
const mockBuildXmp = buildXmp as MockedFunction<typeof buildXmp>; const mockBuildXmp = buildXmp as MockedFunction<typeof buildXmp>;
const mockWritePdfMetadataInWorker = writePdfMetadataInWorker as MockedFunction<typeof writePdfMetadataInWorker>;
describe('PdfFormatWriter', () => { describe('PdfFormatWriter', () => {
function makePdfDoc(overrides: { isEncrypted?: boolean } = {}) { function makePdfDoc(overrides: { isEncrypted?: boolean } = {}) {
@@ -73,11 +81,13 @@ describe('PdfFormatWriter', () => {
beforeEach(() => { beforeEach(() => {
vi.clearAllMocks(); vi.clearAllMocks();
mockReadFile.mockResolvedValue(Buffer.from('pdf-bytes') as never); mockReadFile.mockResolvedValue(Buffer.from('pdf-bytes') as never);
mockStat.mockResolvedValue({ size: PDF_WORKER_WRITE_THRESHOLD_BYTES - 1 } as never);
mockWriteFile.mockResolvedValue(undefined); mockWriteFile.mockResolvedValue(undefined);
mockRename.mockResolvedValue(undefined); mockRename.mockResolvedValue(undefined);
mockUnlink.mockResolvedValue(undefined); mockUnlink.mockResolvedValue(undefined);
mockRandomUuid.mockReturnValue('abc-uuid'); mockRandomUuid.mockReturnValue('abc-uuid');
mockBuildXmp.mockReturnValue('<xmp />'); mockBuildXmp.mockReturnValue('<xmp />');
mockWritePdfMetadataInWorker.mockResolvedValue({ status: 'success', fieldsWritten: ['title'], durationMs: 200 });
}); });
it('returns dry-run result without touching filesystem/pdf-lib', async () => { it('returns dry-run result without touching filesystem/pdf-lib', async () => {
@@ -87,6 +97,7 @@ describe('PdfFormatWriter', () => {
expect(result.status).toBe('skipped'); expect(result.status).toBe('skipped');
expect(result.reason).toBe('dry-run'); expect(result.reason).toBe('dry-run');
expect(mockStat).not.toHaveBeenCalled();
expect(mockReadFile).not.toHaveBeenCalled(); expect(mockReadFile).not.toHaveBeenCalled();
expect(mockPdfLoad).not.toHaveBeenCalled(); expect(mockPdfLoad).not.toHaveBeenCalled();
}); });
@@ -133,6 +144,48 @@ describe('PdfFormatWriter', () => {
expect(result.status).toBe('success'); expect(result.status).toBe('success');
}); });
it('delegates large PDFs to a worker without loading or saving on the main thread', async () => {
mockStat.mockResolvedValue({ size: PDF_WORKER_WRITE_THRESHOLD_BYTES } as never);
const writer = new PdfFormatWriter();
const payload = {
title: 'Dune',
authors: [{ name: 'Frank Herbert', sortName: null }],
coverBytes: Buffer.from('cover'),
};
const result = await writer.write('/books/large.pdf', payload, {
fieldMask: new Set(['title', 'authors', 'coverBytes']),
dryRun: false,
});
expect(mockWritePdfMetadataInWorker).toHaveBeenCalledWith({
filePath: '/books/large.pdf',
payload,
fieldMask: ['title', 'authors'],
});
expect(mockReadFile).not.toHaveBeenCalled();
expect(mockPdfLoad).not.toHaveBeenCalled();
expect(mockWriteFile).not.toHaveBeenCalled();
expect(mockRename).not.toHaveBeenCalled();
expect(result).toMatchObject({ status: 'success', fieldsWritten: ['title'] });
expect(typeof result.durationMs).toBe('number');
});
it('propagates worker failures for large PDFs', async () => {
mockStat.mockResolvedValue({ size: PDF_WORKER_WRITE_THRESHOLD_BYTES + 1 } as never);
mockWritePdfMetadataInWorker.mockRejectedValue(new Error('worker failed'));
const writer = new PdfFormatWriter();
await expect(writer.write('/books/large.pdf', { title: 'Dune' }, { fieldMask: new Set(['title']), dryRun: false })).rejects.toThrow(
'worker failed',
);
expect(mockReadFile).not.toHaveBeenCalled();
expect(mockPdfLoad).not.toHaveBeenCalled();
});
it('deletes temp file when rename fails', async () => { it('deletes temp file when rename fails', async () => {
const pdfDoc = makePdfDoc(); const pdfDoc = makePdfDoc();
mockPdfLoad.mockResolvedValue(pdfDoc as never); mockPdfLoad.mockResolvedValue(pdfDoc as never);
@@ -1,19 +1,14 @@
import { Injectable } from '@nestjs/common'; import { Injectable } from '@nestjs/common';
import { readFile, writeFile } from 'fs/promises'; import { stat } from 'fs/promises';
import { dirname, join } from 'path';
import { randomUUID } from 'crypto';
import { PDFDocument, PDFName } from 'pdf-lib';
import { PDF_BOOK_FILE_WRITE_FIELDS, type WriteResult } from '@bookorbit/types'; import type { WriteResult } from '@bookorbit/types';
import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface'; import type { BookWritePayload } from '../../interfaces/book-write-payload.interface';
import type { FormatWriter } from '../../interfaces/format-writer.interface'; import type { FormatWriter } from '../../interfaces/format-writer.interface';
import type { FormatWriteOptions } from '../../interfaces/format-write-options.interface'; import type { FormatWriteOptions } from '../../interfaces/format-write-options.interface';
import { replaceFileAtomically } from '../shared/atomic-file-replace'; import { resolvePdfFieldsWritten, writePdfMetadataInProcess } from './pdf-write-core';
import { BOOKORBIT_NS_PREFIX } from '../shared/bookorbit-ns'; import { writePdfMetadataInWorker } from './pdf-worker-runner';
import { resolveFieldsWritten } from '../shared/resolve-fields-written';
import { buildXmp } from './pdf-xmp-builder';
const PDF_WRITABLE_FIELDS = new Set<BookWritePayloadKey>(PDF_BOOK_FILE_WRITE_FIELDS); export const PDF_WORKER_WRITE_THRESHOLD_BYTES = 25 * 1024 * 1024;
@Injectable() @Injectable()
export class PdfFormatWriter implements FormatWriter { export class PdfFormatWriter implements FormatWriter {
@@ -22,65 +17,22 @@ export class PdfFormatWriter implements FormatWriter {
async write(filePath: string, payload: BookWritePayload, options: FormatWriteOptions): Promise<WriteResult> { async write(filePath: string, payload: BookWritePayload, options: FormatWriteOptions): Promise<WriteResult> {
const start = Date.now(); const start = Date.now();
const { fieldMask, dryRun } = options; const { fieldMask, dryRun } = options;
const pdfFieldMask = new Set([...fieldMask].filter((key) => PDF_WRITABLE_FIELDS.has(key))); const { fieldsWritten, pdfFieldMask } = resolvePdfFieldsWritten(payload, fieldMask);
const fieldsWritten = resolveFieldsWritten(payload, pdfFieldMask);
if (dryRun) { if (dryRun) {
return { status: 'skipped', reason: 'dry-run', fieldsWritten, durationMs: Date.now() - start }; return { status: 'skipped', reason: 'dry-run', fieldsWritten, durationMs: Date.now() - start };
} }
const originalBytes = await readFile(filePath); const fileStats = await stat(filePath);
const pdfDoc = await PDFDocument.load(originalBytes, { ignoreEncryption: true }); if (fileStats.size >= PDF_WORKER_WRITE_THRESHOLD_BYTES) {
const result = await writePdfMetadataInWorker({
// pdf-lib cannot re-encrypt, so saving an encrypted PDF would strip its filePath,
// encryption and write back still-encrypted byte streams, silently corrupting payload,
// the file. Skip the file write (DB metadata is persisted separately). fieldMask: [...pdfFieldMask],
if (pdfDoc.isEncrypted) { });
return { status: 'skipped', reason: 'encrypted-pdf', fieldsWritten: [], durationMs: Date.now() - start }; return { ...result, durationMs: Date.now() - start };
} }
applyInfoDict(pdfDoc, payload, pdfFieldMask); return writePdfMetadataInProcess(filePath, payload, pdfFieldMask, start);
embedXmp(pdfDoc, buildXmp(payload, pdfFieldMask));
const savedBytes = await pdfDoc.save();
const tempPath = join(dirname(filePath), `.tmp-${randomUUID()}.pdf`);
await writeFile(tempPath, savedBytes);
await replaceFileAtomically(tempPath, filePath);
return { status: 'success', fieldsWritten, durationMs: Date.now() - start };
} }
} }
function applyInfoDict(pdfDoc: PDFDocument, payload: BookWritePayload, fieldMask: Set<BookWritePayloadKey>): void {
if (fieldMask.has('title') && payload.title != null) {
pdfDoc.setTitle(payload.title);
}
if (fieldMask.has('authors') && payload.authors?.length) {
pdfDoc.setAuthor(payload.authors.map((a) => a.name).join(', '));
}
if (fieldMask.has('description') && payload.description != null) {
pdfDoc.setSubject(payload.description);
}
if (fieldMask.has('publishedYear') && payload.publishedYear != null) {
pdfDoc.setCreationDate(new Date(payload.publishedYear, 0, 1));
}
pdfDoc.setCreator(BOOKORBIT_NS_PREFIX);
const keywords: string[] = [];
if (fieldMask.has('genres') && payload.genres?.length) keywords.push(...payload.genres);
if (fieldMask.has('tags') && payload.tags?.length) keywords.push(...payload.tags);
if (keywords.length) {
pdfDoc.setKeywords(keywords);
}
}
function embedXmp(pdfDoc: PDFDocument, xmpXml: string): void {
const xmpBytes = Buffer.from(xmpXml, 'utf-8');
const xmpStream = pdfDoc.context.stream(xmpBytes, {
Type: 'Metadata',
Subtype: 'XML',
});
pdfDoc.catalog.set(PDFName.of('Metadata'), pdfDoc.context.register(xmpStream));
}
@@ -0,0 +1,77 @@
import { EventEmitter } from 'events';
import type { WriteResult } from '@bookorbit/types';
import type { PdfWorkerFactory, PdfWriteWorkerData } from './pdf-worker-runner';
import { writePdfMetadataInWorker } from './pdf-worker-runner';
describe('writePdfMetadataInWorker', () => {
const workerData: PdfWriteWorkerData = {
filePath: '/books/large.pdf',
payload: { title: 'Dune' },
fieldMask: ['title'],
};
function makeWorkerHarness() {
const worker = new EventEmitter();
const createWorker = vi.fn<PdfWorkerFactory>().mockReturnValue(worker as never);
return { worker, createWorker };
}
it('resolves with the worker result message', async () => {
const { worker, createWorker } = makeWorkerHarness();
const result: WriteResult = { status: 'success', fieldsWritten: ['title'], durationMs: 50 };
const promise = writePdfMetadataInWorker(workerData, createWorker);
worker.emit('message', { type: 'result', result });
await expect(promise).resolves.toEqual(result);
expect(createWorker).toHaveBeenCalledWith(workerData);
});
it('rejects with worker error messages and preserves the error class', async () => {
const { worker, createWorker } = makeWorkerHarness();
const promise = writePdfMetadataInWorker(workerData, createWorker);
worker.emit('message', { type: 'error', errorClass: 'BadPdfError', errorMessage: 'bad pdf' });
await expect(promise).rejects.toMatchObject({ name: 'BadPdfError', message: 'bad pdf' });
});
it('rejects invalid worker messages', async () => {
const { worker, createWorker } = makeWorkerHarness();
const promise = writePdfMetadataInWorker(workerData, createWorker);
worker.emit('message', { nope: true });
await expect(promise).rejects.toThrow('PDF write worker returned an invalid response');
});
it('rejects when the worker emits an error event', async () => {
const { worker, createWorker } = makeWorkerHarness();
const promise = writePdfMetadataInWorker(workerData, createWorker);
worker.emit('error', new Error('thread crashed'));
await expect(promise).rejects.toThrow('thread crashed');
});
it('rejects when the worker exits non-zero before returning a result', async () => {
const { worker, createWorker } = makeWorkerHarness();
const promise = writePdfMetadataInWorker(workerData, createWorker);
worker.emit('exit', 1);
await expect(promise).rejects.toThrow('PDF write worker exited with code 1');
});
it('ignores exit events after a result has settled the worker', async () => {
const { worker, createWorker } = makeWorkerHarness();
const result: WriteResult = { status: 'skipped', fieldsWritten: [], durationMs: 12, reason: 'encrypted-pdf' };
const promise = writePdfMetadataInWorker(workerData, createWorker);
worker.emit('message', { type: 'result', result });
worker.emit('exit', 1);
await expect(promise).resolves.toEqual(result);
});
});
@@ -0,0 +1,93 @@
import { existsSync } from 'fs';
import { join } from 'path';
import { Worker } from 'worker_threads';
import type { WriteResult } from '@bookorbit/types';
import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
export interface PdfWriteWorkerData {
filePath: string;
payload: BookWritePayload;
fieldMask: BookWritePayloadKey[];
}
export type PdfWriteWorkerMessage =
| { type: 'result'; result: WriteResult }
| { type: 'error'; errorClass: string; errorMessage: string; stack?: string };
interface PdfWorkerProcess {
once(event: 'message', listener: (message: unknown) => void): this;
once(event: 'error', listener: (error: Error) => void): this;
once(event: 'exit', listener: (code: number) => void): this;
}
export type PdfWorkerFactory = (data: PdfWriteWorkerData) => PdfWorkerProcess;
export function createPdfWriteWorker(data: PdfWriteWorkerData): PdfWorkerProcess {
const workerPath = resolvePdfWorkerPath();
return new Worker(workerPath, {
workerData: data,
execArgv: workerPath.endsWith('.ts') ? ['--import', 'tsx'] : undefined,
});
}
export function writePdfMetadataInWorker(data: PdfWriteWorkerData, createWorker: PdfWorkerFactory = createPdfWriteWorker): Promise<WriteResult> {
return new Promise((resolve, reject) => {
const worker = createWorker(data);
let settled = false;
const settle = (callback: () => void): void => {
if (settled) return;
settled = true;
callback();
};
worker.once('message', (message) => {
settle(() => {
if (isWorkerResultMessage(message)) {
resolve(message.result);
return;
}
if (isWorkerErrorMessage(message)) {
reject(toWorkerError(message));
return;
}
reject(new Error('PDF write worker returned an invalid response'));
});
});
worker.once('error', (error) => {
settle(() => reject(error));
});
worker.once('exit', (code) => {
if (code === 0) return;
settle(() => reject(new Error(`PDF write worker exited with code ${code}`)));
});
});
}
function resolvePdfWorkerPath(): string {
const jsPath = join(__dirname, 'pdf-write.worker.js');
if (existsSync(jsPath)) return jsPath;
const tsPath = join(__dirname, 'pdf-write.worker.ts');
if (existsSync(tsPath)) return tsPath;
return jsPath;
}
function isWorkerResultMessage(message: unknown): message is Extract<PdfWriteWorkerMessage, { type: 'result' }> {
return typeof message === 'object' && message !== null && (message as PdfWriteWorkerMessage).type === 'result' && 'result' in message;
}
function isWorkerErrorMessage(message: unknown): message is Extract<PdfWriteWorkerMessage, { type: 'error' }> {
return typeof message === 'object' && message !== null && (message as PdfWriteWorkerMessage).type === 'error' && 'errorMessage' in message;
}
function toWorkerError(message: Extract<PdfWriteWorkerMessage, { type: 'error' }>): Error {
const error = new Error(message.errorMessage);
error.name = message.errorClass || 'Error';
if (message.stack) error.stack = message.stack;
return error;
}
@@ -0,0 +1,89 @@
import { randomUUID } from 'crypto';
import { readFile, writeFile } from 'fs/promises';
import { dirname, join } from 'path';
import { PDFDocument, PDFName } from 'pdf-lib';
import { PDF_BOOK_FILE_WRITE_FIELDS, type WriteResult } from '@bookorbit/types';
import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
import { replaceFileAtomically } from '../shared/atomic-file-replace';
import { BOOKORBIT_NS_PREFIX } from '../shared/bookorbit-ns';
import { resolveFieldsWritten } from '../shared/resolve-fields-written';
import { buildXmp } from './pdf-xmp-builder';
const PDF_WRITABLE_FIELDS = new Set<BookWritePayloadKey>(PDF_BOOK_FILE_WRITE_FIELDS);
export function resolvePdfFieldsWritten(
payload: BookWritePayload,
fieldMask: Set<BookWritePayloadKey>,
): {
fieldsWritten: string[];
pdfFieldMask: Set<BookWritePayloadKey>;
} {
const pdfFieldMask = new Set([...fieldMask].filter((key) => PDF_WRITABLE_FIELDS.has(key)));
return {
fieldsWritten: resolveFieldsWritten(payload, pdfFieldMask),
pdfFieldMask,
};
}
export async function writePdfMetadataInProcess(
filePath: string,
payload: BookWritePayload,
pdfFieldMask: Set<BookWritePayloadKey>,
startedAt = Date.now(),
): Promise<WriteResult> {
const fieldsWritten = resolveFieldsWritten(payload, pdfFieldMask);
const originalBytes = await readFile(filePath);
const pdfDoc = await PDFDocument.load(originalBytes, { ignoreEncryption: true });
// pdf-lib cannot re-encrypt, so saving an encrypted PDF would strip its
// encryption and write back still-encrypted byte streams, silently corrupting
// the file. Skip the file write (DB metadata is persisted separately).
if (pdfDoc.isEncrypted) {
return { status: 'skipped', reason: 'encrypted-pdf', fieldsWritten: [], durationMs: Date.now() - startedAt };
}
applyInfoDict(pdfDoc, payload, pdfFieldMask);
embedXmp(pdfDoc, buildXmp(payload, pdfFieldMask));
const savedBytes = await pdfDoc.save();
const tempPath = join(dirname(filePath), `.tmp-${randomUUID()}.pdf`);
await writeFile(tempPath, savedBytes);
await replaceFileAtomically(tempPath, filePath);
return { status: 'success', fieldsWritten, durationMs: Date.now() - startedAt };
}
function applyInfoDict(pdfDoc: PDFDocument, payload: BookWritePayload, fieldMask: Set<BookWritePayloadKey>): void {
if (fieldMask.has('title') && payload.title != null) {
pdfDoc.setTitle(payload.title);
}
if (fieldMask.has('authors') && payload.authors?.length) {
pdfDoc.setAuthor(payload.authors.map((a) => a.name).join(', '));
}
if (fieldMask.has('description') && payload.description != null) {
pdfDoc.setSubject(payload.description);
}
if (fieldMask.has('publishedYear') && payload.publishedYear != null) {
pdfDoc.setCreationDate(new Date(payload.publishedYear, 0, 1));
}
pdfDoc.setCreator(BOOKORBIT_NS_PREFIX);
const keywords: string[] = [];
if (fieldMask.has('genres') && payload.genres?.length) keywords.push(...payload.genres);
if (fieldMask.has('tags') && payload.tags?.length) keywords.push(...payload.tags);
if (keywords.length) {
pdfDoc.setKeywords(keywords);
}
}
function embedXmp(pdfDoc: PDFDocument, xmpXml: string): void {
const xmpBytes = Buffer.from(xmpXml, 'utf-8');
const xmpStream = pdfDoc.context.stream(xmpBytes, {
Type: 'Metadata',
Subtype: 'XML',
});
pdfDoc.catalog.set(PDFName.of('Metadata'), pdfDoc.context.register(xmpStream));
}
@@ -0,0 +1,31 @@
import { parentPort, workerData } from 'worker_threads';
import type { BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
import { writePdfMetadataInProcess } from './pdf-write-core';
import type { PdfWriteWorkerData, PdfWriteWorkerMessage } from './pdf-worker-runner';
async function run(): Promise<void> {
if (!parentPort) {
throw new Error('PDF write worker requires a parent port');
}
const data = workerData as PdfWriteWorkerData;
try {
const result = await writePdfMetadataInProcess(data.filePath, data.payload, new Set<BookWritePayloadKey>(data.fieldMask));
postMessage({ type: 'result', result });
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error));
postMessage({
type: 'error',
errorClass: err.name,
errorMessage: err.message,
stack: err.stack,
});
}
}
function postMessage(message: PdfWriteWorkerMessage): void {
parentPort!.postMessage(message);
}
void run();
@@ -0,0 +1,109 @@
import { EventEmitter } from 'events';
import type { PdfParsed } from './pdf-parser';
import type { PdfParseWorkerFactory, PdfParseWorkerData } from './pdf-parse-worker-runner';
import { parsePdfFileInWorker } from './pdf-parse-worker-runner';
describe('parsePdfFileInWorker', () => {
const workerData: PdfParseWorkerData = {
absolutePath: '/books/large.pdf',
extractCover: true,
};
function makeWorkerHarness() {
const worker = new EventEmitter();
const createWorker = vi.fn<PdfParseWorkerFactory>().mockReturnValue(worker as never);
return { worker, createWorker };
}
function makeParsed(overrides: Partial<PdfParsed> = {}): PdfParsed {
return {
title: 'Worker Title',
subtitle: null,
authors: [],
description: null,
publisher: null,
publishedYear: null,
language: null,
genres: [],
tags: [],
isbn10: null,
isbn13: null,
seriesName: null,
seriesIndex: null,
rating: null,
pageCount: null,
googleBooksId: null,
goodreadsId: null,
amazonId: null,
hardcoverId: null,
openLibraryId: null,
ranobedbId: null,
koboId: null,
lubimyczytacId: null,
aladinId: null,
itunesId: null,
coverBuffer: null,
...overrides,
};
}
it('resolves with the worker result message', async () => {
const { worker, createWorker } = makeWorkerHarness();
const result = { parsed: makeParsed(), warnings: [] };
const promise = parsePdfFileInWorker(workerData, createWorker);
worker.emit('message', { type: 'result', result });
await expect(promise).resolves.toEqual(result);
expect(createWorker).toHaveBeenCalledWith(workerData);
});
it('normalizes cloned cover bytes back to Buffer', async () => {
const { worker, createWorker } = makeWorkerHarness();
const coverBytes = new Uint8Array([0xff, 0xd8, 0xff]);
const promise = parsePdfFileInWorker(workerData, createWorker);
worker.emit('message', { type: 'result', result: { parsed: makeParsed({ coverBuffer: coverBytes as never }), warnings: [] } });
const result = await promise;
expect(Buffer.isBuffer(result.parsed?.coverBuffer)).toBe(true);
expect(result.parsed?.coverBuffer).toEqual(Buffer.from(coverBytes));
});
it('rejects with worker error messages and preserves the error class', async () => {
const { worker, createWorker } = makeWorkerHarness();
const promise = parsePdfFileInWorker(workerData, createWorker);
worker.emit('message', { type: 'error', errorClass: 'BadPdfError', errorMessage: 'bad pdf' });
await expect(promise).rejects.toMatchObject({ name: 'BadPdfError', message: 'bad pdf' });
});
it('rejects invalid worker messages', async () => {
const { worker, createWorker } = makeWorkerHarness();
const promise = parsePdfFileInWorker(workerData, createWorker);
worker.emit('message', { nope: true });
await expect(promise).rejects.toThrow('PDF parse worker returned an invalid response');
});
it('rejects when the worker emits an error event', async () => {
const { worker, createWorker } = makeWorkerHarness();
const promise = parsePdfFileInWorker(workerData, createWorker);
worker.emit('error', new Error('thread crashed'));
await expect(promise).rejects.toThrow('thread crashed');
});
it('rejects when the worker exits non-zero before returning a result', async () => {
const { worker, createWorker } = makeWorkerHarness();
const promise = parsePdfFileInWorker(workerData, createWorker);
worker.emit('exit', 1);
await expect(promise).rejects.toThrow('PDF parse worker exited with code 1');
});
});
@@ -0,0 +1,113 @@
import { existsSync } from 'fs';
import { join } from 'path';
import { Worker } from 'worker_threads';
import type { PdfParsed, PdfParseWarning } from './pdf-parser';
export interface PdfParseWorkerData {
absolutePath: string;
extractCover: boolean;
}
export interface PdfParseWorkerResult {
parsed: PdfParsed | null;
warnings: PdfParseWarning[];
}
export type PdfParseWorkerMessage =
| { type: 'result'; result: PdfParseWorkerResult }
| { type: 'error'; errorClass: string; errorMessage: string; stack?: string };
interface PdfWorkerProcess {
once(event: 'message', listener: (message: unknown) => void): this;
once(event: 'error', listener: (error: Error) => void): this;
once(event: 'exit', listener: (code: number) => void): this;
}
export type PdfParseWorkerFactory = (data: PdfParseWorkerData) => PdfWorkerProcess;
export function createPdfParseWorker(data: PdfParseWorkerData): PdfWorkerProcess {
const workerPath = resolvePdfWorkerPath();
return new Worker(workerPath, {
workerData: data,
execArgv: workerPath.endsWith('.ts') ? ['--import', 'tsx'] : undefined,
});
}
export function parsePdfFileInWorker(
data: PdfParseWorkerData,
createWorker: PdfParseWorkerFactory = createPdfParseWorker,
): Promise<PdfParseWorkerResult> {
return new Promise((resolve, reject) => {
const worker = createWorker(data);
let settled = false;
const settle = (callback: () => void): void => {
if (settled) return;
settled = true;
callback();
};
worker.once('message', (message) => {
settle(() => {
if (isWorkerResultMessage(message)) {
resolve(normalizeWorkerResult(message.result));
return;
}
if (isWorkerErrorMessage(message)) {
reject(toWorkerError(message));
return;
}
reject(new Error('PDF parse worker returned an invalid response'));
});
});
worker.once('error', (error) => {
settle(() => reject(error));
});
worker.once('exit', (code) => {
if (code === 0) return;
settle(() => reject(new Error(`PDF parse worker exited with code ${code}`)));
});
});
}
function resolvePdfWorkerPath(): string {
const jsPath = join(__dirname, 'pdf-parse.worker.js');
if (existsSync(jsPath)) return jsPath;
const tsPath = join(__dirname, 'pdf-parse.worker.ts');
if (existsSync(tsPath)) return tsPath;
return jsPath;
}
function isWorkerResultMessage(message: unknown): message is Extract<PdfParseWorkerMessage, { type: 'result' }> {
return typeof message === 'object' && message !== null && (message as PdfParseWorkerMessage).type === 'result' && 'result' in message;
}
function isWorkerErrorMessage(message: unknown): message is Extract<PdfParseWorkerMessage, { type: 'error' }> {
return typeof message === 'object' && message !== null && (message as PdfParseWorkerMessage).type === 'error' && 'errorMessage' in message;
}
function normalizeWorkerResult(result: PdfParseWorkerResult): PdfParseWorkerResult {
if (!result.parsed?.coverBuffer || Buffer.isBuffer(result.parsed.coverBuffer)) {
return result;
}
return {
...result,
parsed: {
...result.parsed,
coverBuffer: Buffer.from(result.parsed.coverBuffer),
},
};
}
function toWorkerError(message: Extract<PdfParseWorkerMessage, { type: 'error' }>): Error {
const error = new Error(message.errorMessage);
error.name = message.errorClass || 'Error';
if (message.stack) error.stack = message.stack;
return error;
}
@@ -0,0 +1,37 @@
import { readFile } from 'fs/promises';
import { parentPort, workerData } from 'worker_threads';
import { parsePdfBuffer, type PdfParseWarning } from './pdf-parser';
import type { PdfParseWorkerData, PdfParseWorkerMessage } from './pdf-parse-worker-runner';
async function run(): Promise<void> {
if (!parentPort) {
throw new Error('PDF parse worker requires a parent port');
}
const data = workerData as PdfParseWorkerData;
const warnings: PdfParseWarning[] = [];
try {
const buffer = await readFile(data.absolutePath);
const parsed = await parsePdfBuffer(data.absolutePath, buffer, {
extractCover: data.extractCover,
onWarning: (warning) => warnings.push(warning),
});
postMessage({ type: 'result', result: { parsed, warnings } });
} catch (error) {
const err = error instanceof Error ? error : new Error(String(error));
postMessage({
type: 'error',
errorClass: err.name,
errorMessage: err.message,
stack: err.stack,
});
}
}
function postMessage(message: PdfParseWorkerMessage): void {
parentPort!.postMessage(message);
}
void run();
@@ -1,5 +1,6 @@
vi.mock('fs/promises', () => ({ vi.mock('fs/promises', () => ({
readFile: vi.fn(), readFile: vi.fn(),
stat: vi.fn(),
})); }));
vi.mock('pdf-lib', () => ({ vi.mock('pdf-lib', () => ({
@@ -17,19 +18,26 @@ vi.mock('./pdf-cover', () => ({
extractPdfCover: vi.fn(), extractPdfCover: vi.fn(),
})); }));
import { readFile } from 'fs/promises'; vi.mock('./pdf-parse-worker-runner', () => ({
parsePdfFileInWorker: vi.fn(),
}));
import { readFile, stat } from 'fs/promises';
import type { MockedFunction } from 'vitest'; import type { MockedFunction } from 'vitest';
import { PDFDocument } from 'pdf-lib'; import { PDFDocument } from 'pdf-lib';
import { extractPdfCover } from './pdf-cover'; import { extractPdfCover } from './pdf-cover';
import { parsePdfFile, PDF_BUFFER_WARNING_BYTES } from './pdf-parser'; import { parsePdfFile, PDF_BUFFER_WARNING_BYTES, type PdfParsed } from './pdf-parser';
import { parsePdfFileInWorker } from './pdf-parse-worker-runner';
import { extractXmpXml, parseXmp } from './pdf-xmp-reader'; import { extractXmpXml, parseXmp } from './pdf-xmp-reader';
const mockReadFile = readFile as MockedFunction<typeof readFile>; const mockReadFile = readFile as MockedFunction<typeof readFile>;
const mockStat = stat as MockedFunction<typeof stat>;
const mockPdfLoad = PDFDocument.load as MockedFunction<typeof PDFDocument.load>; const mockPdfLoad = PDFDocument.load as MockedFunction<typeof PDFDocument.load>;
const mockExtractXmpXml = extractXmpXml as MockedFunction<typeof extractXmpXml>; const mockExtractXmpXml = extractXmpXml as MockedFunction<typeof extractXmpXml>;
const mockParseXmp = parseXmp as MockedFunction<typeof parseXmp>; const mockParseXmp = parseXmp as MockedFunction<typeof parseXmp>;
const mockExtractPdfCover = extractPdfCover as MockedFunction<typeof extractPdfCover>; const mockExtractPdfCover = extractPdfCover as MockedFunction<typeof extractPdfCover>;
const mockParsePdfFileInWorker = parsePdfFileInWorker as MockedFunction<typeof parsePdfFileInWorker>;
function makePdfDoc(overrides: Partial<Record<string, unknown>> = {}) { function makePdfDoc(overrides: Partial<Record<string, unknown>> = {}) {
return { return {
@@ -44,15 +52,49 @@ function makePdfDoc(overrides: Partial<Record<string, unknown>> = {}) {
}; };
} }
function makeParsed(overrides: Partial<PdfParsed> = {}): PdfParsed {
return {
title: 'Worker Title',
subtitle: null,
authors: [{ name: 'Worker Author', sortName: null }],
description: null,
publisher: null,
publishedYear: null,
language: null,
genres: [],
tags: [],
isbn10: null,
isbn13: null,
seriesName: null,
seriesIndex: null,
rating: null,
pageCount: 100,
googleBooksId: null,
goodreadsId: null,
amazonId: null,
hardcoverId: null,
openLibraryId: null,
ranobedbId: null,
koboId: null,
lubimyczytacId: null,
aladinId: null,
itunesId: null,
coverBuffer: null,
...overrides,
};
}
describe('parsePdfFile', () => { describe('parsePdfFile', () => {
beforeEach(() => { beforeEach(() => {
vi.resetAllMocks(); vi.resetAllMocks();
mockStat.mockResolvedValue({ size: PDF_BUFFER_WARNING_BYTES - 1 } as never);
mockReadFile.mockResolvedValue(Buffer.from('%PDF-1.7') as never); mockReadFile.mockResolvedValue(Buffer.from('%PDF-1.7') as never);
mockPdfLoad.mockResolvedValue(makePdfDoc() as never); mockPdfLoad.mockResolvedValue(makePdfDoc() as never);
mockExtractXmpXml.mockReturnValue(null); mockExtractXmpXml.mockReturnValue(null);
mockParseXmp.mockReturnValue(null); mockParseXmp.mockReturnValue(null);
mockExtractPdfCover.mockResolvedValue(Buffer.from([0xff, 0xd8, 0xff])); mockExtractPdfCover.mockResolvedValue(Buffer.from([0xff, 0xd8, 0xff]));
mockParsePdfFileInWorker.mockResolvedValue({ parsed: makeParsed(), warnings: [] });
}); });
it('prefers XMP metadata and includes extracted cover when requested', async () => { it('prefers XMP metadata and includes extracted cover when requested', async () => {
@@ -143,11 +185,12 @@ describe('parsePdfFile', () => {
}); });
it('emits a warning when a large PDF must be buffered in memory', async () => { it('emits a warning when a large PDF must be buffered in memory', async () => {
mockReadFile.mockResolvedValue(Buffer.alloc(PDF_BUFFER_WARNING_BYTES) as never); mockStat.mockResolvedValue({ size: PDF_BUFFER_WARNING_BYTES } as never);
const onWarning = vi.fn(); const onWarning = vi.fn();
await parsePdfFile('/books/large.pdf', { onWarning }); const parsed = await parsePdfFile('/books/large.pdf', { onWarning });
expect(parsed).toEqual(expect.objectContaining({ title: 'Worker Title' }));
expect(onWarning).toHaveBeenCalledWith( expect(onWarning).toHaveBeenCalledWith(
expect.objectContaining({ expect.objectContaining({
code: 'buffered-large-pdf', code: 'buffered-large-pdf',
@@ -156,6 +199,60 @@ describe('parsePdfFile', () => {
thresholdBytes: PDF_BUFFER_WARNING_BYTES, thresholdBytes: PDF_BUFFER_WARNING_BYTES,
}), }),
); );
expect(mockParsePdfFileInWorker).toHaveBeenCalledWith({
absolutePath: '/books/large.pdf',
extractCover: false,
});
expect(mockReadFile).not.toHaveBeenCalled();
expect(mockPdfLoad).not.toHaveBeenCalled();
});
it('replays large PDF worker warnings through the caller warning handler', async () => {
mockStat.mockResolvedValue({ size: PDF_BUFFER_WARNING_BYTES + 1 } as never);
mockParsePdfFileInWorker.mockResolvedValue({
parsed: makeParsed(),
warnings: [
{
code: 'cover-extraction-failed',
absolutePath: '/books/large.pdf',
errorClass: 'Error',
errorMessage: 'pdftoppm missing',
},
],
});
const onWarning = vi.fn();
await parsePdfFile('/books/large.pdf', { extractCover: true, onWarning });
expect(mockParsePdfFileInWorker).toHaveBeenCalledWith({
absolutePath: '/books/large.pdf',
extractCover: true,
});
expect(onWarning).toHaveBeenCalledWith(expect.objectContaining({ code: 'buffered-large-pdf' }));
expect(onWarning).toHaveBeenCalledWith(
expect.objectContaining({
code: 'cover-extraction-failed',
errorMessage: 'pdftoppm missing',
}),
);
});
it('returns null and emits a parse warning when the large PDF worker fails', async () => {
mockStat.mockResolvedValue({ size: PDF_BUFFER_WARNING_BYTES + 1 } as never);
mockParsePdfFileInWorker.mockRejectedValue(new Error('worker failed'));
const onWarning = vi.fn();
await expect(parsePdfFile('/books/large.pdf', { onWarning })).resolves.toBeNull();
expect(onWarning).toHaveBeenCalledWith(
expect.objectContaining({
code: 'parse-failed',
absolutePath: '/books/large.pdf',
errorClass: 'Error',
errorMessage: 'worker failed',
}),
);
expect(mockReadFile).not.toHaveBeenCalled();
expect(mockPdfLoad).not.toHaveBeenCalled();
}); });
it('falls back to the native PDF page count when XMP omits bookorbit:pageCount', async () => { it('falls back to the native PDF page count when XMP omits bookorbit:pageCount', async () => {
+78 -63
View File
@@ -1,8 +1,9 @@
import { readFile } from 'fs/promises'; import { readFile, stat } from 'fs/promises';
import { PDFDocument } from 'pdf-lib'; import { PDFDocument } from 'pdf-lib';
import { BOOKORBIT_NS_PREFIX } from '../../../common/bookorbit-ns'; import { BOOKORBIT_NS_PREFIX } from '../../../common/bookorbit-ns';
import { sanitizeLogValue } from '../../../common/utils/log-sanitize.utils'; import { sanitizeLogValue } from '../../../common/utils/log-sanitize.utils';
import { parsePdfFileInWorker } from './pdf-parse-worker-runner';
import { extractPdfCover } from './pdf-cover'; import { extractPdfCover } from './pdf-cover';
import { extractXmpXml, parseXmp, type XmpParsed } from './pdf-xmp-reader'; import { extractXmpXml, parseXmp, type XmpParsed } from './pdf-xmp-reader';
@@ -82,73 +83,87 @@ function createLargeBufferWarning(absolutePath: string, sizeBytes: number): PdfP
export async function parsePdfFile(absolutePath: string, options: PdfParseOptions = {}): Promise<PdfParsed | null> { export async function parsePdfFile(absolutePath: string, options: PdfParseOptions = {}): Promise<PdfParsed | null> {
try { try {
const buf = await readFile(absolutePath); const fileStats = await stat(absolutePath);
if (buf.length >= PDF_BUFFER_WARNING_BYTES) { if (fileStats.size >= PDF_BUFFER_WARNING_BYTES) {
options.onWarning?.(createLargeBufferWarning(absolutePath, buf.length)); options.onWarning?.(createLargeBufferWarning(absolutePath, fileStats.size));
} const result = await parsePdfFileInWorker({
const doc = await PDFDocument.load(buf, { ignoreEncryption: true }); absolutePath,
extractCover: options.extractCover === true,
// XMP is the authoritative source; Info Dictionary is fallback-only. });
const xmpXml = extractXmpXml(doc); for (const warning of result.warnings) {
const xmp: XmpParsed | null = xmpXml ? parseXmp(xmpXml) : null; options.onWarning?.(warning);
const hasXmp = xmp !== null;
const infoTitle = clean(doc.getTitle());
const infoAuthorRaw = clean(doc.getAuthor());
const infoCreator = clean(doc.getCreator());
const infoProducer = clean(doc.getProducer());
const infoSubject = clean(doc.getSubject());
const infoKeywords = splitCommaList(clean(doc.getKeywords()));
const isBookorbitInfo = infoCreator === BOOKORBIT_NS_PREFIX;
const infoAuthors = infoAuthorRaw
? infoAuthorRaw
.split(/[,;]/)
.map((s) => s.trim())
.filter(Boolean)
.map((name) => ({ name, sortName: null }))
: [];
let coverBuffer: Buffer | null = null;
if (options.extractCover === true) {
try {
coverBuffer = await extractPdfCover(absolutePath);
} catch (error) {
options.onWarning?.(createWarning('cover-extraction-failed', absolutePath, error));
} }
return result.parsed;
} }
return { const buf = await readFile(absolutePath);
title: hasXmp ? xmp.title : infoTitle, return await parsePdfBuffer(absolutePath, buf, options);
subtitle: xmp?.subtitle ?? null,
authors: hasXmp ? xmp.authors : infoAuthors,
description: hasXmp ? xmp.description : infoSubject,
publisher: hasXmp ? (xmp.publisher ?? null) : isBookorbitInfo ? infoProducer : null,
publishedYear: xmp?.publishedYear ?? null,
language: xmp?.language ?? null,
genres: xmp?.genres?.length ? xmp.genres : [],
// Info Dict keywords are genres+tags mixed; only use as tags when XMP is absent.
tags: hasXmp ? xmp.tags : infoKeywords,
isbn10: xmp?.isbn10 ?? null,
isbn13: xmp?.isbn13 ?? null,
seriesName: xmp?.seriesName ?? null,
seriesIndex: xmp?.seriesIndex ?? null,
rating: xmp?.rating ?? null,
pageCount: hasXmp ? (xmp.pageCount ?? (isBookorbitInfo ? null : doc.getPageCount())) : doc.getPageCount(),
googleBooksId: xmp?.googleBooksId ?? null,
goodreadsId: xmp?.goodreadsId ?? null,
amazonId: xmp?.amazonId ?? null,
hardcoverId: xmp?.hardcoverId ?? null,
openLibraryId: xmp?.openLibraryId ?? null,
ranobedbId: xmp?.ranobedbId ?? null,
koboId: xmp?.koboId ?? null,
lubimyczytacId: xmp?.lubimyczytacId ?? null,
aladinId: xmp?.aladinId ?? null,
itunesId: xmp?.itunesId ?? null,
coverBuffer,
};
} catch (error) { } catch (error) {
options.onWarning?.(createWarning('parse-failed', absolutePath, error)); options.onWarning?.(createWarning('parse-failed', absolutePath, error));
return null; return null;
} }
} }
export async function parsePdfBuffer(absolutePath: string, buf: Buffer, options: PdfParseOptions = {}): Promise<PdfParsed | null> {
const doc = await PDFDocument.load(buf, { ignoreEncryption: true });
// XMP is the authoritative source; Info Dictionary is fallback-only.
const xmpXml = extractXmpXml(doc);
const xmp: XmpParsed | null = xmpXml ? parseXmp(xmpXml) : null;
const hasXmp = xmp !== null;
const infoTitle = clean(doc.getTitle());
const infoAuthorRaw = clean(doc.getAuthor());
const infoCreator = clean(doc.getCreator());
const infoProducer = clean(doc.getProducer());
const infoSubject = clean(doc.getSubject());
const infoKeywords = splitCommaList(clean(doc.getKeywords()));
const isBookorbitInfo = infoCreator === BOOKORBIT_NS_PREFIX;
const infoAuthors = infoAuthorRaw
? infoAuthorRaw
.split(/[,;]/)
.map((s) => s.trim())
.filter(Boolean)
.map((name) => ({ name, sortName: null }))
: [];
let coverBuffer: Buffer | null = null;
if (options.extractCover === true) {
try {
coverBuffer = await extractPdfCover(absolutePath);
} catch (error) {
options.onWarning?.(createWarning('cover-extraction-failed', absolutePath, error));
}
}
return {
title: hasXmp ? xmp.title : infoTitle,
subtitle: xmp?.subtitle ?? null,
authors: hasXmp ? xmp.authors : infoAuthors,
description: hasXmp ? xmp.description : infoSubject,
publisher: hasXmp ? (xmp.publisher ?? null) : isBookorbitInfo ? infoProducer : null,
publishedYear: xmp?.publishedYear ?? null,
language: xmp?.language ?? null,
genres: xmp?.genres?.length ? xmp.genres : [],
// Info Dict keywords are genres+tags mixed; only use as tags when XMP is absent.
tags: hasXmp ? xmp.tags : infoKeywords,
isbn10: xmp?.isbn10 ?? null,
isbn13: xmp?.isbn13 ?? null,
seriesName: xmp?.seriesName ?? null,
seriesIndex: xmp?.seriesIndex ?? null,
rating: xmp?.rating ?? null,
pageCount: hasXmp ? (xmp.pageCount ?? (isBookorbitInfo ? null : doc.getPageCount())) : doc.getPageCount(),
googleBooksId: xmp?.googleBooksId ?? null,
goodreadsId: xmp?.goodreadsId ?? null,
amazonId: xmp?.amazonId ?? null,
hardcoverId: xmp?.hardcoverId ?? null,
openLibraryId: xmp?.openLibraryId ?? null,
ranobedbId: xmp?.ranobedbId ?? null,
koboId: xmp?.koboId ?? null,
lubimyczytacId: xmp?.lubimyczytacId ?? null,
aladinId: xmp?.aladinId ?? null,
itunesId: xmp?.itunesId ?? null,
coverBuffer,
};
}