mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 08:21:39 -06:00
fix(metadata): offload large pdf parsing to worker thread
Offload large PDF parsing to a worker thread to prevent the Node.js event loop from being blocked. This ensures the application remains responsive during heavy metadata extraction tasks. Closes #371
This commit is contained in:
@@ -1,16 +1,18 @@
|
|||||||
import { readFile, rename, unlink, writeFile } from 'fs/promises';
|
import { readFile, rename, stat, unlink, writeFile } from 'fs/promises';
|
||||||
import type { MockedFunction } from 'vitest';
|
import type { MockedFunction } from 'vitest';
|
||||||
import { randomUUID } from 'crypto';
|
import { randomUUID } from 'crypto';
|
||||||
import { PDFDocument, PDFName } from 'pdf-lib';
|
import { PDFDocument, PDFName } from 'pdf-lib';
|
||||||
|
|
||||||
import { PdfFormatWriter } from './pdf-format-writer';
|
import { PDF_WORKER_WRITE_THRESHOLD_BYTES, PdfFormatWriter } from './pdf-format-writer';
|
||||||
import { buildXmp } from './pdf-xmp-builder';
|
import { buildXmp } from './pdf-xmp-builder';
|
||||||
|
import { writePdfMetadataInWorker } from './pdf-worker-runner';
|
||||||
|
|
||||||
vi.mock('fs/promises', async () => {
|
vi.mock('fs/promises', async () => {
|
||||||
const actual = await vi.importActual('fs/promises');
|
const actual = await vi.importActual('fs/promises');
|
||||||
return {
|
return {
|
||||||
...actual,
|
...actual,
|
||||||
readFile: vi.fn(),
|
readFile: vi.fn(),
|
||||||
|
stat: vi.fn(),
|
||||||
writeFile: vi.fn(),
|
writeFile: vi.fn(),
|
||||||
rename: vi.fn(),
|
rename: vi.fn(),
|
||||||
unlink: vi.fn(),
|
unlink: vi.fn(),
|
||||||
@@ -38,7 +40,12 @@ vi.mock('./pdf-xmp-builder', () => ({
|
|||||||
buildXmp: vi.fn(),
|
buildXmp: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock('./pdf-worker-runner', () => ({
|
||||||
|
writePdfMetadataInWorker: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
const mockReadFile = readFile as MockedFunction<typeof readFile>;
|
const mockReadFile = readFile as MockedFunction<typeof readFile>;
|
||||||
|
const mockStat = stat as MockedFunction<typeof stat>;
|
||||||
const mockWriteFile = writeFile as MockedFunction<typeof writeFile>;
|
const mockWriteFile = writeFile as MockedFunction<typeof writeFile>;
|
||||||
const mockRename = rename as MockedFunction<typeof rename>;
|
const mockRename = rename as MockedFunction<typeof rename>;
|
||||||
const mockUnlink = unlink as MockedFunction<typeof unlink>;
|
const mockUnlink = unlink as MockedFunction<typeof unlink>;
|
||||||
@@ -46,6 +53,7 @@ const mockRandomUuid = randomUUID as MockedFunction<typeof randomUUID>;
|
|||||||
const mockPdfLoad = PDFDocument.load as MockedFunction<typeof PDFDocument.load>;
|
const mockPdfLoad = PDFDocument.load as MockedFunction<typeof PDFDocument.load>;
|
||||||
const mockPdfNameOf = PDFName.of as MockedFunction<typeof PDFName.of>;
|
const mockPdfNameOf = PDFName.of as MockedFunction<typeof PDFName.of>;
|
||||||
const mockBuildXmp = buildXmp as MockedFunction<typeof buildXmp>;
|
const mockBuildXmp = buildXmp as MockedFunction<typeof buildXmp>;
|
||||||
|
const mockWritePdfMetadataInWorker = writePdfMetadataInWorker as MockedFunction<typeof writePdfMetadataInWorker>;
|
||||||
|
|
||||||
describe('PdfFormatWriter', () => {
|
describe('PdfFormatWriter', () => {
|
||||||
function makePdfDoc(overrides: { isEncrypted?: boolean } = {}) {
|
function makePdfDoc(overrides: { isEncrypted?: boolean } = {}) {
|
||||||
@@ -73,11 +81,13 @@ describe('PdfFormatWriter', () => {
|
|||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
mockReadFile.mockResolvedValue(Buffer.from('pdf-bytes') as never);
|
mockReadFile.mockResolvedValue(Buffer.from('pdf-bytes') as never);
|
||||||
|
mockStat.mockResolvedValue({ size: PDF_WORKER_WRITE_THRESHOLD_BYTES - 1 } as never);
|
||||||
mockWriteFile.mockResolvedValue(undefined);
|
mockWriteFile.mockResolvedValue(undefined);
|
||||||
mockRename.mockResolvedValue(undefined);
|
mockRename.mockResolvedValue(undefined);
|
||||||
mockUnlink.mockResolvedValue(undefined);
|
mockUnlink.mockResolvedValue(undefined);
|
||||||
mockRandomUuid.mockReturnValue('abc-uuid');
|
mockRandomUuid.mockReturnValue('abc-uuid');
|
||||||
mockBuildXmp.mockReturnValue('<xmp />');
|
mockBuildXmp.mockReturnValue('<xmp />');
|
||||||
|
mockWritePdfMetadataInWorker.mockResolvedValue({ status: 'success', fieldsWritten: ['title'], durationMs: 200 });
|
||||||
});
|
});
|
||||||
|
|
||||||
it('returns dry-run result without touching filesystem/pdf-lib', async () => {
|
it('returns dry-run result without touching filesystem/pdf-lib', async () => {
|
||||||
@@ -87,6 +97,7 @@ describe('PdfFormatWriter', () => {
|
|||||||
|
|
||||||
expect(result.status).toBe('skipped');
|
expect(result.status).toBe('skipped');
|
||||||
expect(result.reason).toBe('dry-run');
|
expect(result.reason).toBe('dry-run');
|
||||||
|
expect(mockStat).not.toHaveBeenCalled();
|
||||||
expect(mockReadFile).not.toHaveBeenCalled();
|
expect(mockReadFile).not.toHaveBeenCalled();
|
||||||
expect(mockPdfLoad).not.toHaveBeenCalled();
|
expect(mockPdfLoad).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
@@ -133,6 +144,48 @@ describe('PdfFormatWriter', () => {
|
|||||||
expect(result.status).toBe('success');
|
expect(result.status).toBe('success');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('delegates large PDFs to a worker without loading or saving on the main thread', async () => {
|
||||||
|
mockStat.mockResolvedValue({ size: PDF_WORKER_WRITE_THRESHOLD_BYTES } as never);
|
||||||
|
|
||||||
|
const writer = new PdfFormatWriter();
|
||||||
|
const payload = {
|
||||||
|
title: 'Dune',
|
||||||
|
authors: [{ name: 'Frank Herbert', sortName: null }],
|
||||||
|
coverBytes: Buffer.from('cover'),
|
||||||
|
};
|
||||||
|
|
||||||
|
const result = await writer.write('/books/large.pdf', payload, {
|
||||||
|
fieldMask: new Set(['title', 'authors', 'coverBytes']),
|
||||||
|
dryRun: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(mockWritePdfMetadataInWorker).toHaveBeenCalledWith({
|
||||||
|
filePath: '/books/large.pdf',
|
||||||
|
payload,
|
||||||
|
fieldMask: ['title', 'authors'],
|
||||||
|
});
|
||||||
|
expect(mockReadFile).not.toHaveBeenCalled();
|
||||||
|
expect(mockPdfLoad).not.toHaveBeenCalled();
|
||||||
|
expect(mockWriteFile).not.toHaveBeenCalled();
|
||||||
|
expect(mockRename).not.toHaveBeenCalled();
|
||||||
|
expect(result).toMatchObject({ status: 'success', fieldsWritten: ['title'] });
|
||||||
|
expect(typeof result.durationMs).toBe('number');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('propagates worker failures for large PDFs', async () => {
|
||||||
|
mockStat.mockResolvedValue({ size: PDF_WORKER_WRITE_THRESHOLD_BYTES + 1 } as never);
|
||||||
|
mockWritePdfMetadataInWorker.mockRejectedValue(new Error('worker failed'));
|
||||||
|
|
||||||
|
const writer = new PdfFormatWriter();
|
||||||
|
|
||||||
|
await expect(writer.write('/books/large.pdf', { title: 'Dune' }, { fieldMask: new Set(['title']), dryRun: false })).rejects.toThrow(
|
||||||
|
'worker failed',
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(mockReadFile).not.toHaveBeenCalled();
|
||||||
|
expect(mockPdfLoad).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
it('deletes temp file when rename fails', async () => {
|
it('deletes temp file when rename fails', async () => {
|
||||||
const pdfDoc = makePdfDoc();
|
const pdfDoc = makePdfDoc();
|
||||||
mockPdfLoad.mockResolvedValue(pdfDoc as never);
|
mockPdfLoad.mockResolvedValue(pdfDoc as never);
|
||||||
|
|||||||
@@ -1,19 +1,14 @@
|
|||||||
import { Injectable } from '@nestjs/common';
|
import { Injectable } from '@nestjs/common';
|
||||||
import { readFile, writeFile } from 'fs/promises';
|
import { stat } from 'fs/promises';
|
||||||
import { dirname, join } from 'path';
|
|
||||||
import { randomUUID } from 'crypto';
|
|
||||||
import { PDFDocument, PDFName } from 'pdf-lib';
|
|
||||||
|
|
||||||
import { PDF_BOOK_FILE_WRITE_FIELDS, type WriteResult } from '@bookorbit/types';
|
import type { WriteResult } from '@bookorbit/types';
|
||||||
import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
|
import type { BookWritePayload } from '../../interfaces/book-write-payload.interface';
|
||||||
import type { FormatWriter } from '../../interfaces/format-writer.interface';
|
import type { FormatWriter } from '../../interfaces/format-writer.interface';
|
||||||
import type { FormatWriteOptions } from '../../interfaces/format-write-options.interface';
|
import type { FormatWriteOptions } from '../../interfaces/format-write-options.interface';
|
||||||
import { replaceFileAtomically } from '../shared/atomic-file-replace';
|
import { resolvePdfFieldsWritten, writePdfMetadataInProcess } from './pdf-write-core';
|
||||||
import { BOOKORBIT_NS_PREFIX } from '../shared/bookorbit-ns';
|
import { writePdfMetadataInWorker } from './pdf-worker-runner';
|
||||||
import { resolveFieldsWritten } from '../shared/resolve-fields-written';
|
|
||||||
import { buildXmp } from './pdf-xmp-builder';
|
|
||||||
|
|
||||||
const PDF_WRITABLE_FIELDS = new Set<BookWritePayloadKey>(PDF_BOOK_FILE_WRITE_FIELDS);
|
export const PDF_WORKER_WRITE_THRESHOLD_BYTES = 25 * 1024 * 1024;
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class PdfFormatWriter implements FormatWriter {
|
export class PdfFormatWriter implements FormatWriter {
|
||||||
@@ -22,65 +17,22 @@ export class PdfFormatWriter implements FormatWriter {
|
|||||||
async write(filePath: string, payload: BookWritePayload, options: FormatWriteOptions): Promise<WriteResult> {
|
async write(filePath: string, payload: BookWritePayload, options: FormatWriteOptions): Promise<WriteResult> {
|
||||||
const start = Date.now();
|
const start = Date.now();
|
||||||
const { fieldMask, dryRun } = options;
|
const { fieldMask, dryRun } = options;
|
||||||
const pdfFieldMask = new Set([...fieldMask].filter((key) => PDF_WRITABLE_FIELDS.has(key)));
|
const { fieldsWritten, pdfFieldMask } = resolvePdfFieldsWritten(payload, fieldMask);
|
||||||
const fieldsWritten = resolveFieldsWritten(payload, pdfFieldMask);
|
|
||||||
|
|
||||||
if (dryRun) {
|
if (dryRun) {
|
||||||
return { status: 'skipped', reason: 'dry-run', fieldsWritten, durationMs: Date.now() - start };
|
return { status: 'skipped', reason: 'dry-run', fieldsWritten, durationMs: Date.now() - start };
|
||||||
}
|
}
|
||||||
|
|
||||||
const originalBytes = await readFile(filePath);
|
const fileStats = await stat(filePath);
|
||||||
const pdfDoc = await PDFDocument.load(originalBytes, { ignoreEncryption: true });
|
if (fileStats.size >= PDF_WORKER_WRITE_THRESHOLD_BYTES) {
|
||||||
|
const result = await writePdfMetadataInWorker({
|
||||||
// pdf-lib cannot re-encrypt, so saving an encrypted PDF would strip its
|
filePath,
|
||||||
// encryption and write back still-encrypted byte streams, silently corrupting
|
payload,
|
||||||
// the file. Skip the file write (DB metadata is persisted separately).
|
fieldMask: [...pdfFieldMask],
|
||||||
if (pdfDoc.isEncrypted) {
|
});
|
||||||
return { status: 'skipped', reason: 'encrypted-pdf', fieldsWritten: [], durationMs: Date.now() - start };
|
return { ...result, durationMs: Date.now() - start };
|
||||||
}
|
}
|
||||||
|
|
||||||
applyInfoDict(pdfDoc, payload, pdfFieldMask);
|
return writePdfMetadataInProcess(filePath, payload, pdfFieldMask, start);
|
||||||
embedXmp(pdfDoc, buildXmp(payload, pdfFieldMask));
|
|
||||||
|
|
||||||
const savedBytes = await pdfDoc.save();
|
|
||||||
|
|
||||||
const tempPath = join(dirname(filePath), `.tmp-${randomUUID()}.pdf`);
|
|
||||||
await writeFile(tempPath, savedBytes);
|
|
||||||
await replaceFileAtomically(tempPath, filePath);
|
|
||||||
|
|
||||||
return { status: 'success', fieldsWritten, durationMs: Date.now() - start };
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function applyInfoDict(pdfDoc: PDFDocument, payload: BookWritePayload, fieldMask: Set<BookWritePayloadKey>): void {
|
|
||||||
if (fieldMask.has('title') && payload.title != null) {
|
|
||||||
pdfDoc.setTitle(payload.title);
|
|
||||||
}
|
|
||||||
if (fieldMask.has('authors') && payload.authors?.length) {
|
|
||||||
pdfDoc.setAuthor(payload.authors.map((a) => a.name).join(', '));
|
|
||||||
}
|
|
||||||
if (fieldMask.has('description') && payload.description != null) {
|
|
||||||
pdfDoc.setSubject(payload.description);
|
|
||||||
}
|
|
||||||
if (fieldMask.has('publishedYear') && payload.publishedYear != null) {
|
|
||||||
pdfDoc.setCreationDate(new Date(payload.publishedYear, 0, 1));
|
|
||||||
}
|
|
||||||
|
|
||||||
pdfDoc.setCreator(BOOKORBIT_NS_PREFIX);
|
|
||||||
|
|
||||||
const keywords: string[] = [];
|
|
||||||
if (fieldMask.has('genres') && payload.genres?.length) keywords.push(...payload.genres);
|
|
||||||
if (fieldMask.has('tags') && payload.tags?.length) keywords.push(...payload.tags);
|
|
||||||
if (keywords.length) {
|
|
||||||
pdfDoc.setKeywords(keywords);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function embedXmp(pdfDoc: PDFDocument, xmpXml: string): void {
|
|
||||||
const xmpBytes = Buffer.from(xmpXml, 'utf-8');
|
|
||||||
const xmpStream = pdfDoc.context.stream(xmpBytes, {
|
|
||||||
Type: 'Metadata',
|
|
||||||
Subtype: 'XML',
|
|
||||||
});
|
|
||||||
pdfDoc.catalog.set(PDFName.of('Metadata'), pdfDoc.context.register(xmpStream));
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
import { EventEmitter } from 'events';
|
||||||
|
|
||||||
|
import type { WriteResult } from '@bookorbit/types';
|
||||||
|
import type { PdfWorkerFactory, PdfWriteWorkerData } from './pdf-worker-runner';
|
||||||
|
import { writePdfMetadataInWorker } from './pdf-worker-runner';
|
||||||
|
|
||||||
|
describe('writePdfMetadataInWorker', () => {
|
||||||
|
const workerData: PdfWriteWorkerData = {
|
||||||
|
filePath: '/books/large.pdf',
|
||||||
|
payload: { title: 'Dune' },
|
||||||
|
fieldMask: ['title'],
|
||||||
|
};
|
||||||
|
|
||||||
|
function makeWorkerHarness() {
|
||||||
|
const worker = new EventEmitter();
|
||||||
|
const createWorker = vi.fn<PdfWorkerFactory>().mockReturnValue(worker as never);
|
||||||
|
return { worker, createWorker };
|
||||||
|
}
|
||||||
|
|
||||||
|
it('resolves with the worker result message', async () => {
|
||||||
|
const { worker, createWorker } = makeWorkerHarness();
|
||||||
|
const result: WriteResult = { status: 'success', fieldsWritten: ['title'], durationMs: 50 };
|
||||||
|
|
||||||
|
const promise = writePdfMetadataInWorker(workerData, createWorker);
|
||||||
|
worker.emit('message', { type: 'result', result });
|
||||||
|
|
||||||
|
await expect(promise).resolves.toEqual(result);
|
||||||
|
expect(createWorker).toHaveBeenCalledWith(workerData);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects with worker error messages and preserves the error class', async () => {
|
||||||
|
const { worker, createWorker } = makeWorkerHarness();
|
||||||
|
|
||||||
|
const promise = writePdfMetadataInWorker(workerData, createWorker);
|
||||||
|
worker.emit('message', { type: 'error', errorClass: 'BadPdfError', errorMessage: 'bad pdf' });
|
||||||
|
|
||||||
|
await expect(promise).rejects.toMatchObject({ name: 'BadPdfError', message: 'bad pdf' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects invalid worker messages', async () => {
|
||||||
|
const { worker, createWorker } = makeWorkerHarness();
|
||||||
|
|
||||||
|
const promise = writePdfMetadataInWorker(workerData, createWorker);
|
||||||
|
worker.emit('message', { nope: true });
|
||||||
|
|
||||||
|
await expect(promise).rejects.toThrow('PDF write worker returned an invalid response');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects when the worker emits an error event', async () => {
|
||||||
|
const { worker, createWorker } = makeWorkerHarness();
|
||||||
|
|
||||||
|
const promise = writePdfMetadataInWorker(workerData, createWorker);
|
||||||
|
worker.emit('error', new Error('thread crashed'));
|
||||||
|
|
||||||
|
await expect(promise).rejects.toThrow('thread crashed');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects when the worker exits non-zero before returning a result', async () => {
|
||||||
|
const { worker, createWorker } = makeWorkerHarness();
|
||||||
|
|
||||||
|
const promise = writePdfMetadataInWorker(workerData, createWorker);
|
||||||
|
worker.emit('exit', 1);
|
||||||
|
|
||||||
|
await expect(promise).rejects.toThrow('PDF write worker exited with code 1');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('ignores exit events after a result has settled the worker', async () => {
|
||||||
|
const { worker, createWorker } = makeWorkerHarness();
|
||||||
|
const result: WriteResult = { status: 'skipped', fieldsWritten: [], durationMs: 12, reason: 'encrypted-pdf' };
|
||||||
|
|
||||||
|
const promise = writePdfMetadataInWorker(workerData, createWorker);
|
||||||
|
worker.emit('message', { type: 'result', result });
|
||||||
|
worker.emit('exit', 1);
|
||||||
|
|
||||||
|
await expect(promise).resolves.toEqual(result);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
import { existsSync } from 'fs';
|
||||||
|
import { join } from 'path';
|
||||||
|
import { Worker } from 'worker_threads';
|
||||||
|
|
||||||
|
import type { WriteResult } from '@bookorbit/types';
|
||||||
|
import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
|
||||||
|
|
||||||
|
export interface PdfWriteWorkerData {
|
||||||
|
filePath: string;
|
||||||
|
payload: BookWritePayload;
|
||||||
|
fieldMask: BookWritePayloadKey[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type PdfWriteWorkerMessage =
|
||||||
|
| { type: 'result'; result: WriteResult }
|
||||||
|
| { type: 'error'; errorClass: string; errorMessage: string; stack?: string };
|
||||||
|
|
||||||
|
interface PdfWorkerProcess {
|
||||||
|
once(event: 'message', listener: (message: unknown) => void): this;
|
||||||
|
once(event: 'error', listener: (error: Error) => void): this;
|
||||||
|
once(event: 'exit', listener: (code: number) => void): this;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type PdfWorkerFactory = (data: PdfWriteWorkerData) => PdfWorkerProcess;
|
||||||
|
|
||||||
|
export function createPdfWriteWorker(data: PdfWriteWorkerData): PdfWorkerProcess {
|
||||||
|
const workerPath = resolvePdfWorkerPath();
|
||||||
|
return new Worker(workerPath, {
|
||||||
|
workerData: data,
|
||||||
|
execArgv: workerPath.endsWith('.ts') ? ['--import', 'tsx'] : undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function writePdfMetadataInWorker(data: PdfWriteWorkerData, createWorker: PdfWorkerFactory = createPdfWriteWorker): Promise<WriteResult> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const worker = createWorker(data);
|
||||||
|
let settled = false;
|
||||||
|
|
||||||
|
const settle = (callback: () => void): void => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
callback();
|
||||||
|
};
|
||||||
|
|
||||||
|
worker.once('message', (message) => {
|
||||||
|
settle(() => {
|
||||||
|
if (isWorkerResultMessage(message)) {
|
||||||
|
resolve(message.result);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (isWorkerErrorMessage(message)) {
|
||||||
|
reject(toWorkerError(message));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
reject(new Error('PDF write worker returned an invalid response'));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
worker.once('error', (error) => {
|
||||||
|
settle(() => reject(error));
|
||||||
|
});
|
||||||
|
|
||||||
|
worker.once('exit', (code) => {
|
||||||
|
if (code === 0) return;
|
||||||
|
settle(() => reject(new Error(`PDF write worker exited with code ${code}`)));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolvePdfWorkerPath(): string {
|
||||||
|
const jsPath = join(__dirname, 'pdf-write.worker.js');
|
||||||
|
if (existsSync(jsPath)) return jsPath;
|
||||||
|
|
||||||
|
const tsPath = join(__dirname, 'pdf-write.worker.ts');
|
||||||
|
if (existsSync(tsPath)) return tsPath;
|
||||||
|
|
||||||
|
return jsPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isWorkerResultMessage(message: unknown): message is Extract<PdfWriteWorkerMessage, { type: 'result' }> {
|
||||||
|
return typeof message === 'object' && message !== null && (message as PdfWriteWorkerMessage).type === 'result' && 'result' in message;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isWorkerErrorMessage(message: unknown): message is Extract<PdfWriteWorkerMessage, { type: 'error' }> {
|
||||||
|
return typeof message === 'object' && message !== null && (message as PdfWriteWorkerMessage).type === 'error' && 'errorMessage' in message;
|
||||||
|
}
|
||||||
|
|
||||||
|
function toWorkerError(message: Extract<PdfWriteWorkerMessage, { type: 'error' }>): Error {
|
||||||
|
const error = new Error(message.errorMessage);
|
||||||
|
error.name = message.errorClass || 'Error';
|
||||||
|
if (message.stack) error.stack = message.stack;
|
||||||
|
return error;
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
import { randomUUID } from 'crypto';
|
||||||
|
import { readFile, writeFile } from 'fs/promises';
|
||||||
|
import { dirname, join } from 'path';
|
||||||
|
import { PDFDocument, PDFName } from 'pdf-lib';
|
||||||
|
|
||||||
|
import { PDF_BOOK_FILE_WRITE_FIELDS, type WriteResult } from '@bookorbit/types';
|
||||||
|
import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
|
||||||
|
import { replaceFileAtomically } from '../shared/atomic-file-replace';
|
||||||
|
import { BOOKORBIT_NS_PREFIX } from '../shared/bookorbit-ns';
|
||||||
|
import { resolveFieldsWritten } from '../shared/resolve-fields-written';
|
||||||
|
import { buildXmp } from './pdf-xmp-builder';
|
||||||
|
|
||||||
|
const PDF_WRITABLE_FIELDS = new Set<BookWritePayloadKey>(PDF_BOOK_FILE_WRITE_FIELDS);
|
||||||
|
|
||||||
|
export function resolvePdfFieldsWritten(
|
||||||
|
payload: BookWritePayload,
|
||||||
|
fieldMask: Set<BookWritePayloadKey>,
|
||||||
|
): {
|
||||||
|
fieldsWritten: string[];
|
||||||
|
pdfFieldMask: Set<BookWritePayloadKey>;
|
||||||
|
} {
|
||||||
|
const pdfFieldMask = new Set([...fieldMask].filter((key) => PDF_WRITABLE_FIELDS.has(key)));
|
||||||
|
return {
|
||||||
|
fieldsWritten: resolveFieldsWritten(payload, pdfFieldMask),
|
||||||
|
pdfFieldMask,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function writePdfMetadataInProcess(
|
||||||
|
filePath: string,
|
||||||
|
payload: BookWritePayload,
|
||||||
|
pdfFieldMask: Set<BookWritePayloadKey>,
|
||||||
|
startedAt = Date.now(),
|
||||||
|
): Promise<WriteResult> {
|
||||||
|
const fieldsWritten = resolveFieldsWritten(payload, pdfFieldMask);
|
||||||
|
const originalBytes = await readFile(filePath);
|
||||||
|
const pdfDoc = await PDFDocument.load(originalBytes, { ignoreEncryption: true });
|
||||||
|
|
||||||
|
// pdf-lib cannot re-encrypt, so saving an encrypted PDF would strip its
|
||||||
|
// encryption and write back still-encrypted byte streams, silently corrupting
|
||||||
|
// the file. Skip the file write (DB metadata is persisted separately).
|
||||||
|
if (pdfDoc.isEncrypted) {
|
||||||
|
return { status: 'skipped', reason: 'encrypted-pdf', fieldsWritten: [], durationMs: Date.now() - startedAt };
|
||||||
|
}
|
||||||
|
|
||||||
|
applyInfoDict(pdfDoc, payload, pdfFieldMask);
|
||||||
|
embedXmp(pdfDoc, buildXmp(payload, pdfFieldMask));
|
||||||
|
|
||||||
|
const savedBytes = await pdfDoc.save();
|
||||||
|
|
||||||
|
const tempPath = join(dirname(filePath), `.tmp-${randomUUID()}.pdf`);
|
||||||
|
await writeFile(tempPath, savedBytes);
|
||||||
|
await replaceFileAtomically(tempPath, filePath);
|
||||||
|
|
||||||
|
return { status: 'success', fieldsWritten, durationMs: Date.now() - startedAt };
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyInfoDict(pdfDoc: PDFDocument, payload: BookWritePayload, fieldMask: Set<BookWritePayloadKey>): void {
|
||||||
|
if (fieldMask.has('title') && payload.title != null) {
|
||||||
|
pdfDoc.setTitle(payload.title);
|
||||||
|
}
|
||||||
|
if (fieldMask.has('authors') && payload.authors?.length) {
|
||||||
|
pdfDoc.setAuthor(payload.authors.map((a) => a.name).join(', '));
|
||||||
|
}
|
||||||
|
if (fieldMask.has('description') && payload.description != null) {
|
||||||
|
pdfDoc.setSubject(payload.description);
|
||||||
|
}
|
||||||
|
if (fieldMask.has('publishedYear') && payload.publishedYear != null) {
|
||||||
|
pdfDoc.setCreationDate(new Date(payload.publishedYear, 0, 1));
|
||||||
|
}
|
||||||
|
|
||||||
|
pdfDoc.setCreator(BOOKORBIT_NS_PREFIX);
|
||||||
|
|
||||||
|
const keywords: string[] = [];
|
||||||
|
if (fieldMask.has('genres') && payload.genres?.length) keywords.push(...payload.genres);
|
||||||
|
if (fieldMask.has('tags') && payload.tags?.length) keywords.push(...payload.tags);
|
||||||
|
if (keywords.length) {
|
||||||
|
pdfDoc.setKeywords(keywords);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function embedXmp(pdfDoc: PDFDocument, xmpXml: string): void {
|
||||||
|
const xmpBytes = Buffer.from(xmpXml, 'utf-8');
|
||||||
|
const xmpStream = pdfDoc.context.stream(xmpBytes, {
|
||||||
|
Type: 'Metadata',
|
||||||
|
Subtype: 'XML',
|
||||||
|
});
|
||||||
|
pdfDoc.catalog.set(PDFName.of('Metadata'), pdfDoc.context.register(xmpStream));
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
import { parentPort, workerData } from 'worker_threads';
|
||||||
|
|
||||||
|
import type { BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
|
||||||
|
import { writePdfMetadataInProcess } from './pdf-write-core';
|
||||||
|
import type { PdfWriteWorkerData, PdfWriteWorkerMessage } from './pdf-worker-runner';
|
||||||
|
|
||||||
|
async function run(): Promise<void> {
|
||||||
|
if (!parentPort) {
|
||||||
|
throw new Error('PDF write worker requires a parent port');
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = workerData as PdfWriteWorkerData;
|
||||||
|
try {
|
||||||
|
const result = await writePdfMetadataInProcess(data.filePath, data.payload, new Set<BookWritePayloadKey>(data.fieldMask));
|
||||||
|
postMessage({ type: 'result', result });
|
||||||
|
} catch (error) {
|
||||||
|
const err = error instanceof Error ? error : new Error(String(error));
|
||||||
|
postMessage({
|
||||||
|
type: 'error',
|
||||||
|
errorClass: err.name,
|
||||||
|
errorMessage: err.message,
|
||||||
|
stack: err.stack,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function postMessage(message: PdfWriteWorkerMessage): void {
|
||||||
|
parentPort!.postMessage(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
void run();
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
import { EventEmitter } from 'events';
|
||||||
|
|
||||||
|
import type { PdfParsed } from './pdf-parser';
|
||||||
|
import type { PdfParseWorkerFactory, PdfParseWorkerData } from './pdf-parse-worker-runner';
|
||||||
|
import { parsePdfFileInWorker } from './pdf-parse-worker-runner';
|
||||||
|
|
||||||
|
describe('parsePdfFileInWorker', () => {
|
||||||
|
const workerData: PdfParseWorkerData = {
|
||||||
|
absolutePath: '/books/large.pdf',
|
||||||
|
extractCover: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
function makeWorkerHarness() {
|
||||||
|
const worker = new EventEmitter();
|
||||||
|
const createWorker = vi.fn<PdfParseWorkerFactory>().mockReturnValue(worker as never);
|
||||||
|
return { worker, createWorker };
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeParsed(overrides: Partial<PdfParsed> = {}): PdfParsed {
|
||||||
|
return {
|
||||||
|
title: 'Worker Title',
|
||||||
|
subtitle: null,
|
||||||
|
authors: [],
|
||||||
|
description: null,
|
||||||
|
publisher: null,
|
||||||
|
publishedYear: null,
|
||||||
|
language: null,
|
||||||
|
genres: [],
|
||||||
|
tags: [],
|
||||||
|
isbn10: null,
|
||||||
|
isbn13: null,
|
||||||
|
seriesName: null,
|
||||||
|
seriesIndex: null,
|
||||||
|
rating: null,
|
||||||
|
pageCount: null,
|
||||||
|
googleBooksId: null,
|
||||||
|
goodreadsId: null,
|
||||||
|
amazonId: null,
|
||||||
|
hardcoverId: null,
|
||||||
|
openLibraryId: null,
|
||||||
|
ranobedbId: null,
|
||||||
|
koboId: null,
|
||||||
|
lubimyczytacId: null,
|
||||||
|
aladinId: null,
|
||||||
|
itunesId: null,
|
||||||
|
coverBuffer: null,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
it('resolves with the worker result message', async () => {
|
||||||
|
const { worker, createWorker } = makeWorkerHarness();
|
||||||
|
const result = { parsed: makeParsed(), warnings: [] };
|
||||||
|
|
||||||
|
const promise = parsePdfFileInWorker(workerData, createWorker);
|
||||||
|
worker.emit('message', { type: 'result', result });
|
||||||
|
|
||||||
|
await expect(promise).resolves.toEqual(result);
|
||||||
|
expect(createWorker).toHaveBeenCalledWith(workerData);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('normalizes cloned cover bytes back to Buffer', async () => {
|
||||||
|
const { worker, createWorker } = makeWorkerHarness();
|
||||||
|
const coverBytes = new Uint8Array([0xff, 0xd8, 0xff]);
|
||||||
|
|
||||||
|
const promise = parsePdfFileInWorker(workerData, createWorker);
|
||||||
|
worker.emit('message', { type: 'result', result: { parsed: makeParsed({ coverBuffer: coverBytes as never }), warnings: [] } });
|
||||||
|
|
||||||
|
const result = await promise;
|
||||||
|
expect(Buffer.isBuffer(result.parsed?.coverBuffer)).toBe(true);
|
||||||
|
expect(result.parsed?.coverBuffer).toEqual(Buffer.from(coverBytes));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects with worker error messages and preserves the error class', async () => {
|
||||||
|
const { worker, createWorker } = makeWorkerHarness();
|
||||||
|
|
||||||
|
const promise = parsePdfFileInWorker(workerData, createWorker);
|
||||||
|
worker.emit('message', { type: 'error', errorClass: 'BadPdfError', errorMessage: 'bad pdf' });
|
||||||
|
|
||||||
|
await expect(promise).rejects.toMatchObject({ name: 'BadPdfError', message: 'bad pdf' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects invalid worker messages', async () => {
|
||||||
|
const { worker, createWorker } = makeWorkerHarness();
|
||||||
|
|
||||||
|
const promise = parsePdfFileInWorker(workerData, createWorker);
|
||||||
|
worker.emit('message', { nope: true });
|
||||||
|
|
||||||
|
await expect(promise).rejects.toThrow('PDF parse worker returned an invalid response');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects when the worker emits an error event', async () => {
|
||||||
|
const { worker, createWorker } = makeWorkerHarness();
|
||||||
|
|
||||||
|
const promise = parsePdfFileInWorker(workerData, createWorker);
|
||||||
|
worker.emit('error', new Error('thread crashed'));
|
||||||
|
|
||||||
|
await expect(promise).rejects.toThrow('thread crashed');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects when the worker exits non-zero before returning a result', async () => {
|
||||||
|
const { worker, createWorker } = makeWorkerHarness();
|
||||||
|
|
||||||
|
const promise = parsePdfFileInWorker(workerData, createWorker);
|
||||||
|
worker.emit('exit', 1);
|
||||||
|
|
||||||
|
await expect(promise).rejects.toThrow('PDF parse worker exited with code 1');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
import { existsSync } from 'fs';
|
||||||
|
import { join } from 'path';
|
||||||
|
import { Worker } from 'worker_threads';
|
||||||
|
|
||||||
|
import type { PdfParsed, PdfParseWarning } from './pdf-parser';
|
||||||
|
|
||||||
|
export interface PdfParseWorkerData {
|
||||||
|
absolutePath: string;
|
||||||
|
extractCover: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PdfParseWorkerResult {
|
||||||
|
parsed: PdfParsed | null;
|
||||||
|
warnings: PdfParseWarning[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type PdfParseWorkerMessage =
|
||||||
|
| { type: 'result'; result: PdfParseWorkerResult }
|
||||||
|
| { type: 'error'; errorClass: string; errorMessage: string; stack?: string };
|
||||||
|
|
||||||
|
interface PdfWorkerProcess {
|
||||||
|
once(event: 'message', listener: (message: unknown) => void): this;
|
||||||
|
once(event: 'error', listener: (error: Error) => void): this;
|
||||||
|
once(event: 'exit', listener: (code: number) => void): this;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type PdfParseWorkerFactory = (data: PdfParseWorkerData) => PdfWorkerProcess;
|
||||||
|
|
||||||
|
export function createPdfParseWorker(data: PdfParseWorkerData): PdfWorkerProcess {
|
||||||
|
const workerPath = resolvePdfWorkerPath();
|
||||||
|
return new Worker(workerPath, {
|
||||||
|
workerData: data,
|
||||||
|
execArgv: workerPath.endsWith('.ts') ? ['--import', 'tsx'] : undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parsePdfFileInWorker(
|
||||||
|
data: PdfParseWorkerData,
|
||||||
|
createWorker: PdfParseWorkerFactory = createPdfParseWorker,
|
||||||
|
): Promise<PdfParseWorkerResult> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const worker = createWorker(data);
|
||||||
|
let settled = false;
|
||||||
|
|
||||||
|
const settle = (callback: () => void): void => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
callback();
|
||||||
|
};
|
||||||
|
|
||||||
|
worker.once('message', (message) => {
|
||||||
|
settle(() => {
|
||||||
|
if (isWorkerResultMessage(message)) {
|
||||||
|
resolve(normalizeWorkerResult(message.result));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (isWorkerErrorMessage(message)) {
|
||||||
|
reject(toWorkerError(message));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
reject(new Error('PDF parse worker returned an invalid response'));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
worker.once('error', (error) => {
|
||||||
|
settle(() => reject(error));
|
||||||
|
});
|
||||||
|
|
||||||
|
worker.once('exit', (code) => {
|
||||||
|
if (code === 0) return;
|
||||||
|
settle(() => reject(new Error(`PDF parse worker exited with code ${code}`)));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolvePdfWorkerPath(): string {
|
||||||
|
const jsPath = join(__dirname, 'pdf-parse.worker.js');
|
||||||
|
if (existsSync(jsPath)) return jsPath;
|
||||||
|
|
||||||
|
const tsPath = join(__dirname, 'pdf-parse.worker.ts');
|
||||||
|
if (existsSync(tsPath)) return tsPath;
|
||||||
|
|
||||||
|
return jsPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isWorkerResultMessage(message: unknown): message is Extract<PdfParseWorkerMessage, { type: 'result' }> {
|
||||||
|
return typeof message === 'object' && message !== null && (message as PdfParseWorkerMessage).type === 'result' && 'result' in message;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isWorkerErrorMessage(message: unknown): message is Extract<PdfParseWorkerMessage, { type: 'error' }> {
|
||||||
|
return typeof message === 'object' && message !== null && (message as PdfParseWorkerMessage).type === 'error' && 'errorMessage' in message;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeWorkerResult(result: PdfParseWorkerResult): PdfParseWorkerResult {
|
||||||
|
if (!result.parsed?.coverBuffer || Buffer.isBuffer(result.parsed.coverBuffer)) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
...result,
|
||||||
|
parsed: {
|
||||||
|
...result.parsed,
|
||||||
|
coverBuffer: Buffer.from(result.parsed.coverBuffer),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function toWorkerError(message: Extract<PdfParseWorkerMessage, { type: 'error' }>): Error {
|
||||||
|
const error = new Error(message.errorMessage);
|
||||||
|
error.name = message.errorClass || 'Error';
|
||||||
|
if (message.stack) error.stack = message.stack;
|
||||||
|
return error;
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
import { readFile } from 'fs/promises';
|
||||||
|
import { parentPort, workerData } from 'worker_threads';
|
||||||
|
|
||||||
|
import { parsePdfBuffer, type PdfParseWarning } from './pdf-parser';
|
||||||
|
import type { PdfParseWorkerData, PdfParseWorkerMessage } from './pdf-parse-worker-runner';
|
||||||
|
|
||||||
|
async function run(): Promise<void> {
|
||||||
|
if (!parentPort) {
|
||||||
|
throw new Error('PDF parse worker requires a parent port');
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = workerData as PdfParseWorkerData;
|
||||||
|
const warnings: PdfParseWarning[] = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
const buffer = await readFile(data.absolutePath);
|
||||||
|
const parsed = await parsePdfBuffer(data.absolutePath, buffer, {
|
||||||
|
extractCover: data.extractCover,
|
||||||
|
onWarning: (warning) => warnings.push(warning),
|
||||||
|
});
|
||||||
|
postMessage({ type: 'result', result: { parsed, warnings } });
|
||||||
|
} catch (error) {
|
||||||
|
const err = error instanceof Error ? error : new Error(String(error));
|
||||||
|
postMessage({
|
||||||
|
type: 'error',
|
||||||
|
errorClass: err.name,
|
||||||
|
errorMessage: err.message,
|
||||||
|
stack: err.stack,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function postMessage(message: PdfParseWorkerMessage): void {
|
||||||
|
parentPort!.postMessage(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
void run();
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
vi.mock('fs/promises', () => ({
|
vi.mock('fs/promises', () => ({
|
||||||
readFile: vi.fn(),
|
readFile: vi.fn(),
|
||||||
|
stat: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('pdf-lib', () => ({
|
vi.mock('pdf-lib', () => ({
|
||||||
@@ -17,19 +18,26 @@ vi.mock('./pdf-cover', () => ({
|
|||||||
extractPdfCover: vi.fn(),
|
extractPdfCover: vi.fn(),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
import { readFile } from 'fs/promises';
|
vi.mock('./pdf-parse-worker-runner', () => ({
|
||||||
|
parsePdfFileInWorker: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
import { readFile, stat } from 'fs/promises';
|
||||||
import type { MockedFunction } from 'vitest';
|
import type { MockedFunction } from 'vitest';
|
||||||
import { PDFDocument } from 'pdf-lib';
|
import { PDFDocument } from 'pdf-lib';
|
||||||
|
|
||||||
import { extractPdfCover } from './pdf-cover';
|
import { extractPdfCover } from './pdf-cover';
|
||||||
import { parsePdfFile, PDF_BUFFER_WARNING_BYTES } from './pdf-parser';
|
import { parsePdfFile, PDF_BUFFER_WARNING_BYTES, type PdfParsed } from './pdf-parser';
|
||||||
|
import { parsePdfFileInWorker } from './pdf-parse-worker-runner';
|
||||||
import { extractXmpXml, parseXmp } from './pdf-xmp-reader';
|
import { extractXmpXml, parseXmp } from './pdf-xmp-reader';
|
||||||
|
|
||||||
const mockReadFile = readFile as MockedFunction<typeof readFile>;
|
const mockReadFile = readFile as MockedFunction<typeof readFile>;
|
||||||
|
const mockStat = stat as MockedFunction<typeof stat>;
|
||||||
const mockPdfLoad = PDFDocument.load as MockedFunction<typeof PDFDocument.load>;
|
const mockPdfLoad = PDFDocument.load as MockedFunction<typeof PDFDocument.load>;
|
||||||
const mockExtractXmpXml = extractXmpXml as MockedFunction<typeof extractXmpXml>;
|
const mockExtractXmpXml = extractXmpXml as MockedFunction<typeof extractXmpXml>;
|
||||||
const mockParseXmp = parseXmp as MockedFunction<typeof parseXmp>;
|
const mockParseXmp = parseXmp as MockedFunction<typeof parseXmp>;
|
||||||
const mockExtractPdfCover = extractPdfCover as MockedFunction<typeof extractPdfCover>;
|
const mockExtractPdfCover = extractPdfCover as MockedFunction<typeof extractPdfCover>;
|
||||||
|
const mockParsePdfFileInWorker = parsePdfFileInWorker as MockedFunction<typeof parsePdfFileInWorker>;
|
||||||
|
|
||||||
function makePdfDoc(overrides: Partial<Record<string, unknown>> = {}) {
|
function makePdfDoc(overrides: Partial<Record<string, unknown>> = {}) {
|
||||||
return {
|
return {
|
||||||
@@ -44,15 +52,49 @@ function makePdfDoc(overrides: Partial<Record<string, unknown>> = {}) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function makeParsed(overrides: Partial<PdfParsed> = {}): PdfParsed {
|
||||||
|
return {
|
||||||
|
title: 'Worker Title',
|
||||||
|
subtitle: null,
|
||||||
|
authors: [{ name: 'Worker Author', sortName: null }],
|
||||||
|
description: null,
|
||||||
|
publisher: null,
|
||||||
|
publishedYear: null,
|
||||||
|
language: null,
|
||||||
|
genres: [],
|
||||||
|
tags: [],
|
||||||
|
isbn10: null,
|
||||||
|
isbn13: null,
|
||||||
|
seriesName: null,
|
||||||
|
seriesIndex: null,
|
||||||
|
rating: null,
|
||||||
|
pageCount: 100,
|
||||||
|
googleBooksId: null,
|
||||||
|
goodreadsId: null,
|
||||||
|
amazonId: null,
|
||||||
|
hardcoverId: null,
|
||||||
|
openLibraryId: null,
|
||||||
|
ranobedbId: null,
|
||||||
|
koboId: null,
|
||||||
|
lubimyczytacId: null,
|
||||||
|
aladinId: null,
|
||||||
|
itunesId: null,
|
||||||
|
coverBuffer: null,
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
describe('parsePdfFile', () => {
|
describe('parsePdfFile', () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.resetAllMocks();
|
vi.resetAllMocks();
|
||||||
|
|
||||||
|
mockStat.mockResolvedValue({ size: PDF_BUFFER_WARNING_BYTES - 1 } as never);
|
||||||
mockReadFile.mockResolvedValue(Buffer.from('%PDF-1.7') as never);
|
mockReadFile.mockResolvedValue(Buffer.from('%PDF-1.7') as never);
|
||||||
mockPdfLoad.mockResolvedValue(makePdfDoc() as never);
|
mockPdfLoad.mockResolvedValue(makePdfDoc() as never);
|
||||||
mockExtractXmpXml.mockReturnValue(null);
|
mockExtractXmpXml.mockReturnValue(null);
|
||||||
mockParseXmp.mockReturnValue(null);
|
mockParseXmp.mockReturnValue(null);
|
||||||
mockExtractPdfCover.mockResolvedValue(Buffer.from([0xff, 0xd8, 0xff]));
|
mockExtractPdfCover.mockResolvedValue(Buffer.from([0xff, 0xd8, 0xff]));
|
||||||
|
mockParsePdfFileInWorker.mockResolvedValue({ parsed: makeParsed(), warnings: [] });
|
||||||
});
|
});
|
||||||
|
|
||||||
it('prefers XMP metadata and includes extracted cover when requested', async () => {
|
it('prefers XMP metadata and includes extracted cover when requested', async () => {
|
||||||
@@ -143,11 +185,12 @@ describe('parsePdfFile', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('emits a warning when a large PDF must be buffered in memory', async () => {
|
it('emits a warning when a large PDF must be buffered in memory', async () => {
|
||||||
mockReadFile.mockResolvedValue(Buffer.alloc(PDF_BUFFER_WARNING_BYTES) as never);
|
mockStat.mockResolvedValue({ size: PDF_BUFFER_WARNING_BYTES } as never);
|
||||||
const onWarning = vi.fn();
|
const onWarning = vi.fn();
|
||||||
|
|
||||||
await parsePdfFile('/books/large.pdf', { onWarning });
|
const parsed = await parsePdfFile('/books/large.pdf', { onWarning });
|
||||||
|
|
||||||
|
expect(parsed).toEqual(expect.objectContaining({ title: 'Worker Title' }));
|
||||||
expect(onWarning).toHaveBeenCalledWith(
|
expect(onWarning).toHaveBeenCalledWith(
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
code: 'buffered-large-pdf',
|
code: 'buffered-large-pdf',
|
||||||
@@ -156,6 +199,60 @@ describe('parsePdfFile', () => {
|
|||||||
thresholdBytes: PDF_BUFFER_WARNING_BYTES,
|
thresholdBytes: PDF_BUFFER_WARNING_BYTES,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
expect(mockParsePdfFileInWorker).toHaveBeenCalledWith({
|
||||||
|
absolutePath: '/books/large.pdf',
|
||||||
|
extractCover: false,
|
||||||
|
});
|
||||||
|
expect(mockReadFile).not.toHaveBeenCalled();
|
||||||
|
expect(mockPdfLoad).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('replays large PDF worker warnings through the caller warning handler', async () => {
|
||||||
|
mockStat.mockResolvedValue({ size: PDF_BUFFER_WARNING_BYTES + 1 } as never);
|
||||||
|
mockParsePdfFileInWorker.mockResolvedValue({
|
||||||
|
parsed: makeParsed(),
|
||||||
|
warnings: [
|
||||||
|
{
|
||||||
|
code: 'cover-extraction-failed',
|
||||||
|
absolutePath: '/books/large.pdf',
|
||||||
|
errorClass: 'Error',
|
||||||
|
errorMessage: 'pdftoppm missing',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
const onWarning = vi.fn();
|
||||||
|
|
||||||
|
await parsePdfFile('/books/large.pdf', { extractCover: true, onWarning });
|
||||||
|
|
||||||
|
expect(mockParsePdfFileInWorker).toHaveBeenCalledWith({
|
||||||
|
absolutePath: '/books/large.pdf',
|
||||||
|
extractCover: true,
|
||||||
|
});
|
||||||
|
expect(onWarning).toHaveBeenCalledWith(expect.objectContaining({ code: 'buffered-large-pdf' }));
|
||||||
|
expect(onWarning).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
code: 'cover-extraction-failed',
|
||||||
|
errorMessage: 'pdftoppm missing',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns null and emits a parse warning when the large PDF worker fails', async () => {
|
||||||
|
mockStat.mockResolvedValue({ size: PDF_BUFFER_WARNING_BYTES + 1 } as never);
|
||||||
|
mockParsePdfFileInWorker.mockRejectedValue(new Error('worker failed'));
|
||||||
|
const onWarning = vi.fn();
|
||||||
|
|
||||||
|
await expect(parsePdfFile('/books/large.pdf', { onWarning })).resolves.toBeNull();
|
||||||
|
expect(onWarning).toHaveBeenCalledWith(
|
||||||
|
expect.objectContaining({
|
||||||
|
code: 'parse-failed',
|
||||||
|
absolutePath: '/books/large.pdf',
|
||||||
|
errorClass: 'Error',
|
||||||
|
errorMessage: 'worker failed',
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(mockReadFile).not.toHaveBeenCalled();
|
||||||
|
expect(mockPdfLoad).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('falls back to the native PDF page count when XMP omits bookorbit:pageCount', async () => {
|
it('falls back to the native PDF page count when XMP omits bookorbit:pageCount', async () => {
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
import { readFile } from 'fs/promises';
|
import { readFile, stat } from 'fs/promises';
|
||||||
import { PDFDocument } from 'pdf-lib';
|
import { PDFDocument } from 'pdf-lib';
|
||||||
|
|
||||||
import { BOOKORBIT_NS_PREFIX } from '../../../common/bookorbit-ns';
|
import { BOOKORBIT_NS_PREFIX } from '../../../common/bookorbit-ns';
|
||||||
import { sanitizeLogValue } from '../../../common/utils/log-sanitize.utils';
|
import { sanitizeLogValue } from '../../../common/utils/log-sanitize.utils';
|
||||||
|
import { parsePdfFileInWorker } from './pdf-parse-worker-runner';
|
||||||
import { extractPdfCover } from './pdf-cover';
|
import { extractPdfCover } from './pdf-cover';
|
||||||
import { extractXmpXml, parseXmp, type XmpParsed } from './pdf-xmp-reader';
|
import { extractXmpXml, parseXmp, type XmpParsed } from './pdf-xmp-reader';
|
||||||
|
|
||||||
@@ -82,73 +83,87 @@ function createLargeBufferWarning(absolutePath: string, sizeBytes: number): PdfP
|
|||||||
|
|
||||||
export async function parsePdfFile(absolutePath: string, options: PdfParseOptions = {}): Promise<PdfParsed | null> {
|
export async function parsePdfFile(absolutePath: string, options: PdfParseOptions = {}): Promise<PdfParsed | null> {
|
||||||
try {
|
try {
|
||||||
const buf = await readFile(absolutePath);
|
const fileStats = await stat(absolutePath);
|
||||||
if (buf.length >= PDF_BUFFER_WARNING_BYTES) {
|
if (fileStats.size >= PDF_BUFFER_WARNING_BYTES) {
|
||||||
options.onWarning?.(createLargeBufferWarning(absolutePath, buf.length));
|
options.onWarning?.(createLargeBufferWarning(absolutePath, fileStats.size));
|
||||||
}
|
const result = await parsePdfFileInWorker({
|
||||||
const doc = await PDFDocument.load(buf, { ignoreEncryption: true });
|
absolutePath,
|
||||||
|
extractCover: options.extractCover === true,
|
||||||
// XMP is the authoritative source; Info Dictionary is fallback-only.
|
});
|
||||||
const xmpXml = extractXmpXml(doc);
|
for (const warning of result.warnings) {
|
||||||
const xmp: XmpParsed | null = xmpXml ? parseXmp(xmpXml) : null;
|
options.onWarning?.(warning);
|
||||||
const hasXmp = xmp !== null;
|
|
||||||
|
|
||||||
const infoTitle = clean(doc.getTitle());
|
|
||||||
const infoAuthorRaw = clean(doc.getAuthor());
|
|
||||||
const infoCreator = clean(doc.getCreator());
|
|
||||||
const infoProducer = clean(doc.getProducer());
|
|
||||||
const infoSubject = clean(doc.getSubject());
|
|
||||||
const infoKeywords = splitCommaList(clean(doc.getKeywords()));
|
|
||||||
const isBookorbitInfo = infoCreator === BOOKORBIT_NS_PREFIX;
|
|
||||||
|
|
||||||
const infoAuthors = infoAuthorRaw
|
|
||||||
? infoAuthorRaw
|
|
||||||
.split(/[,;]/)
|
|
||||||
.map((s) => s.trim())
|
|
||||||
.filter(Boolean)
|
|
||||||
.map((name) => ({ name, sortName: null }))
|
|
||||||
: [];
|
|
||||||
|
|
||||||
let coverBuffer: Buffer | null = null;
|
|
||||||
if (options.extractCover === true) {
|
|
||||||
try {
|
|
||||||
coverBuffer = await extractPdfCover(absolutePath);
|
|
||||||
} catch (error) {
|
|
||||||
options.onWarning?.(createWarning('cover-extraction-failed', absolutePath, error));
|
|
||||||
}
|
}
|
||||||
|
return result.parsed;
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
const buf = await readFile(absolutePath);
|
||||||
title: hasXmp ? xmp.title : infoTitle,
|
return await parsePdfBuffer(absolutePath, buf, options);
|
||||||
subtitle: xmp?.subtitle ?? null,
|
|
||||||
authors: hasXmp ? xmp.authors : infoAuthors,
|
|
||||||
description: hasXmp ? xmp.description : infoSubject,
|
|
||||||
publisher: hasXmp ? (xmp.publisher ?? null) : isBookorbitInfo ? infoProducer : null,
|
|
||||||
publishedYear: xmp?.publishedYear ?? null,
|
|
||||||
language: xmp?.language ?? null,
|
|
||||||
genres: xmp?.genres?.length ? xmp.genres : [],
|
|
||||||
// Info Dict keywords are genres+tags mixed; only use as tags when XMP is absent.
|
|
||||||
tags: hasXmp ? xmp.tags : infoKeywords,
|
|
||||||
isbn10: xmp?.isbn10 ?? null,
|
|
||||||
isbn13: xmp?.isbn13 ?? null,
|
|
||||||
seriesName: xmp?.seriesName ?? null,
|
|
||||||
seriesIndex: xmp?.seriesIndex ?? null,
|
|
||||||
rating: xmp?.rating ?? null,
|
|
||||||
pageCount: hasXmp ? (xmp.pageCount ?? (isBookorbitInfo ? null : doc.getPageCount())) : doc.getPageCount(),
|
|
||||||
googleBooksId: xmp?.googleBooksId ?? null,
|
|
||||||
goodreadsId: xmp?.goodreadsId ?? null,
|
|
||||||
amazonId: xmp?.amazonId ?? null,
|
|
||||||
hardcoverId: xmp?.hardcoverId ?? null,
|
|
||||||
openLibraryId: xmp?.openLibraryId ?? null,
|
|
||||||
ranobedbId: xmp?.ranobedbId ?? null,
|
|
||||||
koboId: xmp?.koboId ?? null,
|
|
||||||
lubimyczytacId: xmp?.lubimyczytacId ?? null,
|
|
||||||
aladinId: xmp?.aladinId ?? null,
|
|
||||||
itunesId: xmp?.itunesId ?? null,
|
|
||||||
coverBuffer,
|
|
||||||
};
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
options.onWarning?.(createWarning('parse-failed', absolutePath, error));
|
options.onWarning?.(createWarning('parse-failed', absolutePath, error));
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function parsePdfBuffer(absolutePath: string, buf: Buffer, options: PdfParseOptions = {}): Promise<PdfParsed | null> {
|
||||||
|
const doc = await PDFDocument.load(buf, { ignoreEncryption: true });
|
||||||
|
|
||||||
|
// XMP is the authoritative source; Info Dictionary is fallback-only.
|
||||||
|
const xmpXml = extractXmpXml(doc);
|
||||||
|
const xmp: XmpParsed | null = xmpXml ? parseXmp(xmpXml) : null;
|
||||||
|
const hasXmp = xmp !== null;
|
||||||
|
|
||||||
|
const infoTitle = clean(doc.getTitle());
|
||||||
|
const infoAuthorRaw = clean(doc.getAuthor());
|
||||||
|
const infoCreator = clean(doc.getCreator());
|
||||||
|
const infoProducer = clean(doc.getProducer());
|
||||||
|
const infoSubject = clean(doc.getSubject());
|
||||||
|
const infoKeywords = splitCommaList(clean(doc.getKeywords()));
|
||||||
|
const isBookorbitInfo = infoCreator === BOOKORBIT_NS_PREFIX;
|
||||||
|
|
||||||
|
const infoAuthors = infoAuthorRaw
|
||||||
|
? infoAuthorRaw
|
||||||
|
.split(/[,;]/)
|
||||||
|
.map((s) => s.trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
.map((name) => ({ name, sortName: null }))
|
||||||
|
: [];
|
||||||
|
|
||||||
|
let coverBuffer: Buffer | null = null;
|
||||||
|
if (options.extractCover === true) {
|
||||||
|
try {
|
||||||
|
coverBuffer = await extractPdfCover(absolutePath);
|
||||||
|
} catch (error) {
|
||||||
|
options.onWarning?.(createWarning('cover-extraction-failed', absolutePath, error));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
title: hasXmp ? xmp.title : infoTitle,
|
||||||
|
subtitle: xmp?.subtitle ?? null,
|
||||||
|
authors: hasXmp ? xmp.authors : infoAuthors,
|
||||||
|
description: hasXmp ? xmp.description : infoSubject,
|
||||||
|
publisher: hasXmp ? (xmp.publisher ?? null) : isBookorbitInfo ? infoProducer : null,
|
||||||
|
publishedYear: xmp?.publishedYear ?? null,
|
||||||
|
language: xmp?.language ?? null,
|
||||||
|
genres: xmp?.genres?.length ? xmp.genres : [],
|
||||||
|
// Info Dict keywords are genres+tags mixed; only use as tags when XMP is absent.
|
||||||
|
tags: hasXmp ? xmp.tags : infoKeywords,
|
||||||
|
isbn10: xmp?.isbn10 ?? null,
|
||||||
|
isbn13: xmp?.isbn13 ?? null,
|
||||||
|
seriesName: xmp?.seriesName ?? null,
|
||||||
|
seriesIndex: xmp?.seriesIndex ?? null,
|
||||||
|
rating: xmp?.rating ?? null,
|
||||||
|
pageCount: hasXmp ? (xmp.pageCount ?? (isBookorbitInfo ? null : doc.getPageCount())) : doc.getPageCount(),
|
||||||
|
googleBooksId: xmp?.googleBooksId ?? null,
|
||||||
|
goodreadsId: xmp?.goodreadsId ?? null,
|
||||||
|
amazonId: xmp?.amazonId ?? null,
|
||||||
|
hardcoverId: xmp?.hardcoverId ?? null,
|
||||||
|
openLibraryId: xmp?.openLibraryId ?? null,
|
||||||
|
ranobedbId: xmp?.ranobedbId ?? null,
|
||||||
|
koboId: xmp?.koboId ?? null,
|
||||||
|
lubimyczytacId: xmp?.lubimyczytacId ?? null,
|
||||||
|
aladinId: xmp?.aladinId ?? null,
|
||||||
|
itunesId: xmp?.itunesId ?? null,
|
||||||
|
coverBuffer,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user