mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
fix(metadata): offload large pdf parsing to worker thread
Offload large PDF parsing to a worker thread to prevent the Node.js event loop from being blocked. This ensures the application remains responsive during heavy metadata extraction tasks. Closes #371
This commit is contained in:
@@ -1,16 +1,18 @@
|
||||
import { readFile, rename, unlink, writeFile } from 'fs/promises';
|
||||
import { readFile, rename, stat, unlink, writeFile } from 'fs/promises';
|
||||
import type { MockedFunction } from 'vitest';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { PDFDocument, PDFName } from 'pdf-lib';
|
||||
|
||||
import { PdfFormatWriter } from './pdf-format-writer';
|
||||
import { PDF_WORKER_WRITE_THRESHOLD_BYTES, PdfFormatWriter } from './pdf-format-writer';
|
||||
import { buildXmp } from './pdf-xmp-builder';
|
||||
import { writePdfMetadataInWorker } from './pdf-worker-runner';
|
||||
|
||||
vi.mock('fs/promises', async () => {
|
||||
const actual = await vi.importActual('fs/promises');
|
||||
return {
|
||||
...actual,
|
||||
readFile: vi.fn(),
|
||||
stat: vi.fn(),
|
||||
writeFile: vi.fn(),
|
||||
rename: vi.fn(),
|
||||
unlink: vi.fn(),
|
||||
@@ -38,7 +40,12 @@ vi.mock('./pdf-xmp-builder', () => ({
|
||||
buildXmp: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('./pdf-worker-runner', () => ({
|
||||
writePdfMetadataInWorker: vi.fn(),
|
||||
}));
|
||||
|
||||
const mockReadFile = readFile as MockedFunction<typeof readFile>;
|
||||
const mockStat = stat as MockedFunction<typeof stat>;
|
||||
const mockWriteFile = writeFile as MockedFunction<typeof writeFile>;
|
||||
const mockRename = rename as MockedFunction<typeof rename>;
|
||||
const mockUnlink = unlink as MockedFunction<typeof unlink>;
|
||||
@@ -46,6 +53,7 @@ const mockRandomUuid = randomUUID as MockedFunction<typeof randomUUID>;
|
||||
const mockPdfLoad = PDFDocument.load as MockedFunction<typeof PDFDocument.load>;
|
||||
const mockPdfNameOf = PDFName.of as MockedFunction<typeof PDFName.of>;
|
||||
const mockBuildXmp = buildXmp as MockedFunction<typeof buildXmp>;
|
||||
const mockWritePdfMetadataInWorker = writePdfMetadataInWorker as MockedFunction<typeof writePdfMetadataInWorker>;
|
||||
|
||||
describe('PdfFormatWriter', () => {
|
||||
function makePdfDoc(overrides: { isEncrypted?: boolean } = {}) {
|
||||
@@ -73,11 +81,13 @@ describe('PdfFormatWriter', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockReadFile.mockResolvedValue(Buffer.from('pdf-bytes') as never);
|
||||
mockStat.mockResolvedValue({ size: PDF_WORKER_WRITE_THRESHOLD_BYTES - 1 } as never);
|
||||
mockWriteFile.mockResolvedValue(undefined);
|
||||
mockRename.mockResolvedValue(undefined);
|
||||
mockUnlink.mockResolvedValue(undefined);
|
||||
mockRandomUuid.mockReturnValue('abc-uuid');
|
||||
mockBuildXmp.mockReturnValue('<xmp />');
|
||||
mockWritePdfMetadataInWorker.mockResolvedValue({ status: 'success', fieldsWritten: ['title'], durationMs: 200 });
|
||||
});
|
||||
|
||||
it('returns dry-run result without touching filesystem/pdf-lib', async () => {
|
||||
@@ -87,6 +97,7 @@ describe('PdfFormatWriter', () => {
|
||||
|
||||
expect(result.status).toBe('skipped');
|
||||
expect(result.reason).toBe('dry-run');
|
||||
expect(mockStat).not.toHaveBeenCalled();
|
||||
expect(mockReadFile).not.toHaveBeenCalled();
|
||||
expect(mockPdfLoad).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -133,6 +144,48 @@ describe('PdfFormatWriter', () => {
|
||||
expect(result.status).toBe('success');
|
||||
});
|
||||
|
||||
it('delegates large PDFs to a worker without loading or saving on the main thread', async () => {
|
||||
mockStat.mockResolvedValue({ size: PDF_WORKER_WRITE_THRESHOLD_BYTES } as never);
|
||||
|
||||
const writer = new PdfFormatWriter();
|
||||
const payload = {
|
||||
title: 'Dune',
|
||||
authors: [{ name: 'Frank Herbert', sortName: null }],
|
||||
coverBytes: Buffer.from('cover'),
|
||||
};
|
||||
|
||||
const result = await writer.write('/books/large.pdf', payload, {
|
||||
fieldMask: new Set(['title', 'authors', 'coverBytes']),
|
||||
dryRun: false,
|
||||
});
|
||||
|
||||
expect(mockWritePdfMetadataInWorker).toHaveBeenCalledWith({
|
||||
filePath: '/books/large.pdf',
|
||||
payload,
|
||||
fieldMask: ['title', 'authors'],
|
||||
});
|
||||
expect(mockReadFile).not.toHaveBeenCalled();
|
||||
expect(mockPdfLoad).not.toHaveBeenCalled();
|
||||
expect(mockWriteFile).not.toHaveBeenCalled();
|
||||
expect(mockRename).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({ status: 'success', fieldsWritten: ['title'] });
|
||||
expect(typeof result.durationMs).toBe('number');
|
||||
});
|
||||
|
||||
it('propagates worker failures for large PDFs', async () => {
|
||||
mockStat.mockResolvedValue({ size: PDF_WORKER_WRITE_THRESHOLD_BYTES + 1 } as never);
|
||||
mockWritePdfMetadataInWorker.mockRejectedValue(new Error('worker failed'));
|
||||
|
||||
const writer = new PdfFormatWriter();
|
||||
|
||||
await expect(writer.write('/books/large.pdf', { title: 'Dune' }, { fieldMask: new Set(['title']), dryRun: false })).rejects.toThrow(
|
||||
'worker failed',
|
||||
);
|
||||
|
||||
expect(mockReadFile).not.toHaveBeenCalled();
|
||||
expect(mockPdfLoad).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('deletes temp file when rename fails', async () => {
|
||||
const pdfDoc = makePdfDoc();
|
||||
mockPdfLoad.mockResolvedValue(pdfDoc as never);
|
||||
|
||||
@@ -1,19 +1,14 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { readFile, writeFile } from 'fs/promises';
|
||||
import { dirname, join } from 'path';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { PDFDocument, PDFName } from 'pdf-lib';
|
||||
import { stat } from 'fs/promises';
|
||||
|
||||
import { PDF_BOOK_FILE_WRITE_FIELDS, type WriteResult } from '@bookorbit/types';
|
||||
import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
|
||||
import type { WriteResult } from '@bookorbit/types';
|
||||
import type { BookWritePayload } from '../../interfaces/book-write-payload.interface';
|
||||
import type { FormatWriter } from '../../interfaces/format-writer.interface';
|
||||
import type { FormatWriteOptions } from '../../interfaces/format-write-options.interface';
|
||||
import { replaceFileAtomically } from '../shared/atomic-file-replace';
|
||||
import { BOOKORBIT_NS_PREFIX } from '../shared/bookorbit-ns';
|
||||
import { resolveFieldsWritten } from '../shared/resolve-fields-written';
|
||||
import { buildXmp } from './pdf-xmp-builder';
|
||||
import { resolvePdfFieldsWritten, writePdfMetadataInProcess } from './pdf-write-core';
|
||||
import { writePdfMetadataInWorker } from './pdf-worker-runner';
|
||||
|
||||
const PDF_WRITABLE_FIELDS = new Set<BookWritePayloadKey>(PDF_BOOK_FILE_WRITE_FIELDS);
|
||||
export const PDF_WORKER_WRITE_THRESHOLD_BYTES = 25 * 1024 * 1024;
|
||||
|
||||
@Injectable()
|
||||
export class PdfFormatWriter implements FormatWriter {
|
||||
@@ -22,65 +17,22 @@ export class PdfFormatWriter implements FormatWriter {
|
||||
async write(filePath: string, payload: BookWritePayload, options: FormatWriteOptions): Promise<WriteResult> {
|
||||
const start = Date.now();
|
||||
const { fieldMask, dryRun } = options;
|
||||
const pdfFieldMask = new Set([...fieldMask].filter((key) => PDF_WRITABLE_FIELDS.has(key)));
|
||||
const fieldsWritten = resolveFieldsWritten(payload, pdfFieldMask);
|
||||
const { fieldsWritten, pdfFieldMask } = resolvePdfFieldsWritten(payload, fieldMask);
|
||||
|
||||
if (dryRun) {
|
||||
return { status: 'skipped', reason: 'dry-run', fieldsWritten, durationMs: Date.now() - start };
|
||||
}
|
||||
|
||||
const originalBytes = await readFile(filePath);
|
||||
const pdfDoc = await PDFDocument.load(originalBytes, { ignoreEncryption: true });
|
||||
|
||||
// pdf-lib cannot re-encrypt, so saving an encrypted PDF would strip its
|
||||
// encryption and write back still-encrypted byte streams, silently corrupting
|
||||
// the file. Skip the file write (DB metadata is persisted separately).
|
||||
if (pdfDoc.isEncrypted) {
|
||||
return { status: 'skipped', reason: 'encrypted-pdf', fieldsWritten: [], durationMs: Date.now() - start };
|
||||
const fileStats = await stat(filePath);
|
||||
if (fileStats.size >= PDF_WORKER_WRITE_THRESHOLD_BYTES) {
|
||||
const result = await writePdfMetadataInWorker({
|
||||
filePath,
|
||||
payload,
|
||||
fieldMask: [...pdfFieldMask],
|
||||
});
|
||||
return { ...result, durationMs: Date.now() - start };
|
||||
}
|
||||
|
||||
applyInfoDict(pdfDoc, payload, pdfFieldMask);
|
||||
embedXmp(pdfDoc, buildXmp(payload, pdfFieldMask));
|
||||
|
||||
const savedBytes = await pdfDoc.save();
|
||||
|
||||
const tempPath = join(dirname(filePath), `.tmp-${randomUUID()}.pdf`);
|
||||
await writeFile(tempPath, savedBytes);
|
||||
await replaceFileAtomically(tempPath, filePath);
|
||||
|
||||
return { status: 'success', fieldsWritten, durationMs: Date.now() - start };
|
||||
return writePdfMetadataInProcess(filePath, payload, pdfFieldMask, start);
|
||||
}
|
||||
}
|
||||
|
||||
function applyInfoDict(pdfDoc: PDFDocument, payload: BookWritePayload, fieldMask: Set<BookWritePayloadKey>): void {
|
||||
if (fieldMask.has('title') && payload.title != null) {
|
||||
pdfDoc.setTitle(payload.title);
|
||||
}
|
||||
if (fieldMask.has('authors') && payload.authors?.length) {
|
||||
pdfDoc.setAuthor(payload.authors.map((a) => a.name).join(', '));
|
||||
}
|
||||
if (fieldMask.has('description') && payload.description != null) {
|
||||
pdfDoc.setSubject(payload.description);
|
||||
}
|
||||
if (fieldMask.has('publishedYear') && payload.publishedYear != null) {
|
||||
pdfDoc.setCreationDate(new Date(payload.publishedYear, 0, 1));
|
||||
}
|
||||
|
||||
pdfDoc.setCreator(BOOKORBIT_NS_PREFIX);
|
||||
|
||||
const keywords: string[] = [];
|
||||
if (fieldMask.has('genres') && payload.genres?.length) keywords.push(...payload.genres);
|
||||
if (fieldMask.has('tags') && payload.tags?.length) keywords.push(...payload.tags);
|
||||
if (keywords.length) {
|
||||
pdfDoc.setKeywords(keywords);
|
||||
}
|
||||
}
|
||||
|
||||
function embedXmp(pdfDoc: PDFDocument, xmpXml: string): void {
|
||||
const xmpBytes = Buffer.from(xmpXml, 'utf-8');
|
||||
const xmpStream = pdfDoc.context.stream(xmpBytes, {
|
||||
Type: 'Metadata',
|
||||
Subtype: 'XML',
|
||||
});
|
||||
pdfDoc.catalog.set(PDFName.of('Metadata'), pdfDoc.context.register(xmpStream));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { EventEmitter } from 'events';
|
||||
|
||||
import type { WriteResult } from '@bookorbit/types';
|
||||
import type { PdfWorkerFactory, PdfWriteWorkerData } from './pdf-worker-runner';
|
||||
import { writePdfMetadataInWorker } from './pdf-worker-runner';
|
||||
|
||||
describe('writePdfMetadataInWorker', () => {
|
||||
const workerData: PdfWriteWorkerData = {
|
||||
filePath: '/books/large.pdf',
|
||||
payload: { title: 'Dune' },
|
||||
fieldMask: ['title'],
|
||||
};
|
||||
|
||||
function makeWorkerHarness() {
|
||||
const worker = new EventEmitter();
|
||||
const createWorker = vi.fn<PdfWorkerFactory>().mockReturnValue(worker as never);
|
||||
return { worker, createWorker };
|
||||
}
|
||||
|
||||
it('resolves with the worker result message', async () => {
|
||||
const { worker, createWorker } = makeWorkerHarness();
|
||||
const result: WriteResult = { status: 'success', fieldsWritten: ['title'], durationMs: 50 };
|
||||
|
||||
const promise = writePdfMetadataInWorker(workerData, createWorker);
|
||||
worker.emit('message', { type: 'result', result });
|
||||
|
||||
await expect(promise).resolves.toEqual(result);
|
||||
expect(createWorker).toHaveBeenCalledWith(workerData);
|
||||
});
|
||||
|
||||
it('rejects with worker error messages and preserves the error class', async () => {
|
||||
const { worker, createWorker } = makeWorkerHarness();
|
||||
|
||||
const promise = writePdfMetadataInWorker(workerData, createWorker);
|
||||
worker.emit('message', { type: 'error', errorClass: 'BadPdfError', errorMessage: 'bad pdf' });
|
||||
|
||||
await expect(promise).rejects.toMatchObject({ name: 'BadPdfError', message: 'bad pdf' });
|
||||
});
|
||||
|
||||
it('rejects invalid worker messages', async () => {
|
||||
const { worker, createWorker } = makeWorkerHarness();
|
||||
|
||||
const promise = writePdfMetadataInWorker(workerData, createWorker);
|
||||
worker.emit('message', { nope: true });
|
||||
|
||||
await expect(promise).rejects.toThrow('PDF write worker returned an invalid response');
|
||||
});
|
||||
|
||||
it('rejects when the worker emits an error event', async () => {
|
||||
const { worker, createWorker } = makeWorkerHarness();
|
||||
|
||||
const promise = writePdfMetadataInWorker(workerData, createWorker);
|
||||
worker.emit('error', new Error('thread crashed'));
|
||||
|
||||
await expect(promise).rejects.toThrow('thread crashed');
|
||||
});
|
||||
|
||||
it('rejects when the worker exits non-zero before returning a result', async () => {
|
||||
const { worker, createWorker } = makeWorkerHarness();
|
||||
|
||||
const promise = writePdfMetadataInWorker(workerData, createWorker);
|
||||
worker.emit('exit', 1);
|
||||
|
||||
await expect(promise).rejects.toThrow('PDF write worker exited with code 1');
|
||||
});
|
||||
|
||||
it('ignores exit events after a result has settled the worker', async () => {
|
||||
const { worker, createWorker } = makeWorkerHarness();
|
||||
const result: WriteResult = { status: 'skipped', fieldsWritten: [], durationMs: 12, reason: 'encrypted-pdf' };
|
||||
|
||||
const promise = writePdfMetadataInWorker(workerData, createWorker);
|
||||
worker.emit('message', { type: 'result', result });
|
||||
worker.emit('exit', 1);
|
||||
|
||||
await expect(promise).resolves.toEqual(result);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,93 @@
|
||||
import { existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { Worker } from 'worker_threads';
|
||||
|
||||
import type { WriteResult } from '@bookorbit/types';
|
||||
import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
|
||||
|
||||
export interface PdfWriteWorkerData {
|
||||
filePath: string;
|
||||
payload: BookWritePayload;
|
||||
fieldMask: BookWritePayloadKey[];
|
||||
}
|
||||
|
||||
export type PdfWriteWorkerMessage =
|
||||
| { type: 'result'; result: WriteResult }
|
||||
| { type: 'error'; errorClass: string; errorMessage: string; stack?: string };
|
||||
|
||||
interface PdfWorkerProcess {
|
||||
once(event: 'message', listener: (message: unknown) => void): this;
|
||||
once(event: 'error', listener: (error: Error) => void): this;
|
||||
once(event: 'exit', listener: (code: number) => void): this;
|
||||
}
|
||||
|
||||
export type PdfWorkerFactory = (data: PdfWriteWorkerData) => PdfWorkerProcess;
|
||||
|
||||
export function createPdfWriteWorker(data: PdfWriteWorkerData): PdfWorkerProcess {
|
||||
const workerPath = resolvePdfWorkerPath();
|
||||
return new Worker(workerPath, {
|
||||
workerData: data,
|
||||
execArgv: workerPath.endsWith('.ts') ? ['--import', 'tsx'] : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
export function writePdfMetadataInWorker(data: PdfWriteWorkerData, createWorker: PdfWorkerFactory = createPdfWriteWorker): Promise<WriteResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const worker = createWorker(data);
|
||||
let settled = false;
|
||||
|
||||
const settle = (callback: () => void): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
callback();
|
||||
};
|
||||
|
||||
worker.once('message', (message) => {
|
||||
settle(() => {
|
||||
if (isWorkerResultMessage(message)) {
|
||||
resolve(message.result);
|
||||
return;
|
||||
}
|
||||
if (isWorkerErrorMessage(message)) {
|
||||
reject(toWorkerError(message));
|
||||
return;
|
||||
}
|
||||
reject(new Error('PDF write worker returned an invalid response'));
|
||||
});
|
||||
});
|
||||
|
||||
worker.once('error', (error) => {
|
||||
settle(() => reject(error));
|
||||
});
|
||||
|
||||
worker.once('exit', (code) => {
|
||||
if (code === 0) return;
|
||||
settle(() => reject(new Error(`PDF write worker exited with code ${code}`)));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function resolvePdfWorkerPath(): string {
|
||||
const jsPath = join(__dirname, 'pdf-write.worker.js');
|
||||
if (existsSync(jsPath)) return jsPath;
|
||||
|
||||
const tsPath = join(__dirname, 'pdf-write.worker.ts');
|
||||
if (existsSync(tsPath)) return tsPath;
|
||||
|
||||
return jsPath;
|
||||
}
|
||||
|
||||
function isWorkerResultMessage(message: unknown): message is Extract<PdfWriteWorkerMessage, { type: 'result' }> {
|
||||
return typeof message === 'object' && message !== null && (message as PdfWriteWorkerMessage).type === 'result' && 'result' in message;
|
||||
}
|
||||
|
||||
function isWorkerErrorMessage(message: unknown): message is Extract<PdfWriteWorkerMessage, { type: 'error' }> {
|
||||
return typeof message === 'object' && message !== null && (message as PdfWriteWorkerMessage).type === 'error' && 'errorMessage' in message;
|
||||
}
|
||||
|
||||
function toWorkerError(message: Extract<PdfWriteWorkerMessage, { type: 'error' }>): Error {
|
||||
const error = new Error(message.errorMessage);
|
||||
error.name = message.errorClass || 'Error';
|
||||
if (message.stack) error.stack = message.stack;
|
||||
return error;
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { readFile, writeFile } from 'fs/promises';
|
||||
import { dirname, join } from 'path';
|
||||
import { PDFDocument, PDFName } from 'pdf-lib';
|
||||
|
||||
import { PDF_BOOK_FILE_WRITE_FIELDS, type WriteResult } from '@bookorbit/types';
|
||||
import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
|
||||
import { replaceFileAtomically } from '../shared/atomic-file-replace';
|
||||
import { BOOKORBIT_NS_PREFIX } from '../shared/bookorbit-ns';
|
||||
import { resolveFieldsWritten } from '../shared/resolve-fields-written';
|
||||
import { buildXmp } from './pdf-xmp-builder';
|
||||
|
||||
const PDF_WRITABLE_FIELDS = new Set<BookWritePayloadKey>(PDF_BOOK_FILE_WRITE_FIELDS);
|
||||
|
||||
export function resolvePdfFieldsWritten(
|
||||
payload: BookWritePayload,
|
||||
fieldMask: Set<BookWritePayloadKey>,
|
||||
): {
|
||||
fieldsWritten: string[];
|
||||
pdfFieldMask: Set<BookWritePayloadKey>;
|
||||
} {
|
||||
const pdfFieldMask = new Set([...fieldMask].filter((key) => PDF_WRITABLE_FIELDS.has(key)));
|
||||
return {
|
||||
fieldsWritten: resolveFieldsWritten(payload, pdfFieldMask),
|
||||
pdfFieldMask,
|
||||
};
|
||||
}
|
||||
|
||||
export async function writePdfMetadataInProcess(
|
||||
filePath: string,
|
||||
payload: BookWritePayload,
|
||||
pdfFieldMask: Set<BookWritePayloadKey>,
|
||||
startedAt = Date.now(),
|
||||
): Promise<WriteResult> {
|
||||
const fieldsWritten = resolveFieldsWritten(payload, pdfFieldMask);
|
||||
const originalBytes = await readFile(filePath);
|
||||
const pdfDoc = await PDFDocument.load(originalBytes, { ignoreEncryption: true });
|
||||
|
||||
// pdf-lib cannot re-encrypt, so saving an encrypted PDF would strip its
|
||||
// encryption and write back still-encrypted byte streams, silently corrupting
|
||||
// the file. Skip the file write (DB metadata is persisted separately).
|
||||
if (pdfDoc.isEncrypted) {
|
||||
return { status: 'skipped', reason: 'encrypted-pdf', fieldsWritten: [], durationMs: Date.now() - startedAt };
|
||||
}
|
||||
|
||||
applyInfoDict(pdfDoc, payload, pdfFieldMask);
|
||||
embedXmp(pdfDoc, buildXmp(payload, pdfFieldMask));
|
||||
|
||||
const savedBytes = await pdfDoc.save();
|
||||
|
||||
const tempPath = join(dirname(filePath), `.tmp-${randomUUID()}.pdf`);
|
||||
await writeFile(tempPath, savedBytes);
|
||||
await replaceFileAtomically(tempPath, filePath);
|
||||
|
||||
return { status: 'success', fieldsWritten, durationMs: Date.now() - startedAt };
|
||||
}
|
||||
|
||||
function applyInfoDict(pdfDoc: PDFDocument, payload: BookWritePayload, fieldMask: Set<BookWritePayloadKey>): void {
|
||||
if (fieldMask.has('title') && payload.title != null) {
|
||||
pdfDoc.setTitle(payload.title);
|
||||
}
|
||||
if (fieldMask.has('authors') && payload.authors?.length) {
|
||||
pdfDoc.setAuthor(payload.authors.map((a) => a.name).join(', '));
|
||||
}
|
||||
if (fieldMask.has('description') && payload.description != null) {
|
||||
pdfDoc.setSubject(payload.description);
|
||||
}
|
||||
if (fieldMask.has('publishedYear') && payload.publishedYear != null) {
|
||||
pdfDoc.setCreationDate(new Date(payload.publishedYear, 0, 1));
|
||||
}
|
||||
|
||||
pdfDoc.setCreator(BOOKORBIT_NS_PREFIX);
|
||||
|
||||
const keywords: string[] = [];
|
||||
if (fieldMask.has('genres') && payload.genres?.length) keywords.push(...payload.genres);
|
||||
if (fieldMask.has('tags') && payload.tags?.length) keywords.push(...payload.tags);
|
||||
if (keywords.length) {
|
||||
pdfDoc.setKeywords(keywords);
|
||||
}
|
||||
}
|
||||
|
||||
function embedXmp(pdfDoc: PDFDocument, xmpXml: string): void {
|
||||
const xmpBytes = Buffer.from(xmpXml, 'utf-8');
|
||||
const xmpStream = pdfDoc.context.stream(xmpBytes, {
|
||||
Type: 'Metadata',
|
||||
Subtype: 'XML',
|
||||
});
|
||||
pdfDoc.catalog.set(PDFName.of('Metadata'), pdfDoc.context.register(xmpStream));
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { parentPort, workerData } from 'worker_threads';
|
||||
|
||||
import type { BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
|
||||
import { writePdfMetadataInProcess } from './pdf-write-core';
|
||||
import type { PdfWriteWorkerData, PdfWriteWorkerMessage } from './pdf-worker-runner';
|
||||
|
||||
async function run(): Promise<void> {
|
||||
if (!parentPort) {
|
||||
throw new Error('PDF write worker requires a parent port');
|
||||
}
|
||||
|
||||
const data = workerData as PdfWriteWorkerData;
|
||||
try {
|
||||
const result = await writePdfMetadataInProcess(data.filePath, data.payload, new Set<BookWritePayloadKey>(data.fieldMask));
|
||||
postMessage({ type: 'result', result });
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
postMessage({
|
||||
type: 'error',
|
||||
errorClass: err.name,
|
||||
errorMessage: err.message,
|
||||
stack: err.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function postMessage(message: PdfWriteWorkerMessage): void {
|
||||
parentPort!.postMessage(message);
|
||||
}
|
||||
|
||||
void run();
|
||||
@@ -0,0 +1,109 @@
|
||||
import { EventEmitter } from 'events';
|
||||
|
||||
import type { PdfParsed } from './pdf-parser';
|
||||
import type { PdfParseWorkerFactory, PdfParseWorkerData } from './pdf-parse-worker-runner';
|
||||
import { parsePdfFileInWorker } from './pdf-parse-worker-runner';
|
||||
|
||||
describe('parsePdfFileInWorker', () => {
|
||||
const workerData: PdfParseWorkerData = {
|
||||
absolutePath: '/books/large.pdf',
|
||||
extractCover: true,
|
||||
};
|
||||
|
||||
function makeWorkerHarness() {
|
||||
const worker = new EventEmitter();
|
||||
const createWorker = vi.fn<PdfParseWorkerFactory>().mockReturnValue(worker as never);
|
||||
return { worker, createWorker };
|
||||
}
|
||||
|
||||
function makeParsed(overrides: Partial<PdfParsed> = {}): PdfParsed {
|
||||
return {
|
||||
title: 'Worker Title',
|
||||
subtitle: null,
|
||||
authors: [],
|
||||
description: null,
|
||||
publisher: null,
|
||||
publishedYear: null,
|
||||
language: null,
|
||||
genres: [],
|
||||
tags: [],
|
||||
isbn10: null,
|
||||
isbn13: null,
|
||||
seriesName: null,
|
||||
seriesIndex: null,
|
||||
rating: null,
|
||||
pageCount: null,
|
||||
googleBooksId: null,
|
||||
goodreadsId: null,
|
||||
amazonId: null,
|
||||
hardcoverId: null,
|
||||
openLibraryId: null,
|
||||
ranobedbId: null,
|
||||
koboId: null,
|
||||
lubimyczytacId: null,
|
||||
aladinId: null,
|
||||
itunesId: null,
|
||||
coverBuffer: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
it('resolves with the worker result message', async () => {
|
||||
const { worker, createWorker } = makeWorkerHarness();
|
||||
const result = { parsed: makeParsed(), warnings: [] };
|
||||
|
||||
const promise = parsePdfFileInWorker(workerData, createWorker);
|
||||
worker.emit('message', { type: 'result', result });
|
||||
|
||||
await expect(promise).resolves.toEqual(result);
|
||||
expect(createWorker).toHaveBeenCalledWith(workerData);
|
||||
});
|
||||
|
||||
it('normalizes cloned cover bytes back to Buffer', async () => {
|
||||
const { worker, createWorker } = makeWorkerHarness();
|
||||
const coverBytes = new Uint8Array([0xff, 0xd8, 0xff]);
|
||||
|
||||
const promise = parsePdfFileInWorker(workerData, createWorker);
|
||||
worker.emit('message', { type: 'result', result: { parsed: makeParsed({ coverBuffer: coverBytes as never }), warnings: [] } });
|
||||
|
||||
const result = await promise;
|
||||
expect(Buffer.isBuffer(result.parsed?.coverBuffer)).toBe(true);
|
||||
expect(result.parsed?.coverBuffer).toEqual(Buffer.from(coverBytes));
|
||||
});
|
||||
|
||||
it('rejects with worker error messages and preserves the error class', async () => {
|
||||
const { worker, createWorker } = makeWorkerHarness();
|
||||
|
||||
const promise = parsePdfFileInWorker(workerData, createWorker);
|
||||
worker.emit('message', { type: 'error', errorClass: 'BadPdfError', errorMessage: 'bad pdf' });
|
||||
|
||||
await expect(promise).rejects.toMatchObject({ name: 'BadPdfError', message: 'bad pdf' });
|
||||
});
|
||||
|
||||
it('rejects invalid worker messages', async () => {
|
||||
const { worker, createWorker } = makeWorkerHarness();
|
||||
|
||||
const promise = parsePdfFileInWorker(workerData, createWorker);
|
||||
worker.emit('message', { nope: true });
|
||||
|
||||
await expect(promise).rejects.toThrow('PDF parse worker returned an invalid response');
|
||||
});
|
||||
|
||||
it('rejects when the worker emits an error event', async () => {
|
||||
const { worker, createWorker } = makeWorkerHarness();
|
||||
|
||||
const promise = parsePdfFileInWorker(workerData, createWorker);
|
||||
worker.emit('error', new Error('thread crashed'));
|
||||
|
||||
await expect(promise).rejects.toThrow('thread crashed');
|
||||
});
|
||||
|
||||
it('rejects when the worker exits non-zero before returning a result', async () => {
|
||||
const { worker, createWorker } = makeWorkerHarness();
|
||||
|
||||
const promise = parsePdfFileInWorker(workerData, createWorker);
|
||||
worker.emit('exit', 1);
|
||||
|
||||
await expect(promise).rejects.toThrow('PDF parse worker exited with code 1');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,113 @@
|
||||
import { existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { Worker } from 'worker_threads';
|
||||
|
||||
import type { PdfParsed, PdfParseWarning } from './pdf-parser';
|
||||
|
||||
export interface PdfParseWorkerData {
|
||||
absolutePath: string;
|
||||
extractCover: boolean;
|
||||
}
|
||||
|
||||
export interface PdfParseWorkerResult {
|
||||
parsed: PdfParsed | null;
|
||||
warnings: PdfParseWarning[];
|
||||
}
|
||||
|
||||
export type PdfParseWorkerMessage =
|
||||
| { type: 'result'; result: PdfParseWorkerResult }
|
||||
| { type: 'error'; errorClass: string; errorMessage: string; stack?: string };
|
||||
|
||||
interface PdfWorkerProcess {
|
||||
once(event: 'message', listener: (message: unknown) => void): this;
|
||||
once(event: 'error', listener: (error: Error) => void): this;
|
||||
once(event: 'exit', listener: (code: number) => void): this;
|
||||
}
|
||||
|
||||
export type PdfParseWorkerFactory = (data: PdfParseWorkerData) => PdfWorkerProcess;
|
||||
|
||||
export function createPdfParseWorker(data: PdfParseWorkerData): PdfWorkerProcess {
|
||||
const workerPath = resolvePdfWorkerPath();
|
||||
return new Worker(workerPath, {
|
||||
workerData: data,
|
||||
execArgv: workerPath.endsWith('.ts') ? ['--import', 'tsx'] : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
export function parsePdfFileInWorker(
|
||||
data: PdfParseWorkerData,
|
||||
createWorker: PdfParseWorkerFactory = createPdfParseWorker,
|
||||
): Promise<PdfParseWorkerResult> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const worker = createWorker(data);
|
||||
let settled = false;
|
||||
|
||||
const settle = (callback: () => void): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
callback();
|
||||
};
|
||||
|
||||
worker.once('message', (message) => {
|
||||
settle(() => {
|
||||
if (isWorkerResultMessage(message)) {
|
||||
resolve(normalizeWorkerResult(message.result));
|
||||
return;
|
||||
}
|
||||
if (isWorkerErrorMessage(message)) {
|
||||
reject(toWorkerError(message));
|
||||
return;
|
||||
}
|
||||
reject(new Error('PDF parse worker returned an invalid response'));
|
||||
});
|
||||
});
|
||||
|
||||
worker.once('error', (error) => {
|
||||
settle(() => reject(error));
|
||||
});
|
||||
|
||||
worker.once('exit', (code) => {
|
||||
if (code === 0) return;
|
||||
settle(() => reject(new Error(`PDF parse worker exited with code ${code}`)));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function resolvePdfWorkerPath(): string {
|
||||
const jsPath = join(__dirname, 'pdf-parse.worker.js');
|
||||
if (existsSync(jsPath)) return jsPath;
|
||||
|
||||
const tsPath = join(__dirname, 'pdf-parse.worker.ts');
|
||||
if (existsSync(tsPath)) return tsPath;
|
||||
|
||||
return jsPath;
|
||||
}
|
||||
|
||||
function isWorkerResultMessage(message: unknown): message is Extract<PdfParseWorkerMessage, { type: 'result' }> {
|
||||
return typeof message === 'object' && message !== null && (message as PdfParseWorkerMessage).type === 'result' && 'result' in message;
|
||||
}
|
||||
|
||||
function isWorkerErrorMessage(message: unknown): message is Extract<PdfParseWorkerMessage, { type: 'error' }> {
|
||||
return typeof message === 'object' && message !== null && (message as PdfParseWorkerMessage).type === 'error' && 'errorMessage' in message;
|
||||
}
|
||||
|
||||
function normalizeWorkerResult(result: PdfParseWorkerResult): PdfParseWorkerResult {
|
||||
if (!result.parsed?.coverBuffer || Buffer.isBuffer(result.parsed.coverBuffer)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
return {
|
||||
...result,
|
||||
parsed: {
|
||||
...result.parsed,
|
||||
coverBuffer: Buffer.from(result.parsed.coverBuffer),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function toWorkerError(message: Extract<PdfParseWorkerMessage, { type: 'error' }>): Error {
|
||||
const error = new Error(message.errorMessage);
|
||||
error.name = message.errorClass || 'Error';
|
||||
if (message.stack) error.stack = message.stack;
|
||||
return error;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
import { readFile } from 'fs/promises';
|
||||
import { parentPort, workerData } from 'worker_threads';
|
||||
|
||||
import { parsePdfBuffer, type PdfParseWarning } from './pdf-parser';
|
||||
import type { PdfParseWorkerData, PdfParseWorkerMessage } from './pdf-parse-worker-runner';
|
||||
|
||||
async function run(): Promise<void> {
|
||||
if (!parentPort) {
|
||||
throw new Error('PDF parse worker requires a parent port');
|
||||
}
|
||||
|
||||
const data = workerData as PdfParseWorkerData;
|
||||
const warnings: PdfParseWarning[] = [];
|
||||
|
||||
try {
|
||||
const buffer = await readFile(data.absolutePath);
|
||||
const parsed = await parsePdfBuffer(data.absolutePath, buffer, {
|
||||
extractCover: data.extractCover,
|
||||
onWarning: (warning) => warnings.push(warning),
|
||||
});
|
||||
postMessage({ type: 'result', result: { parsed, warnings } });
|
||||
} catch (error) {
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
postMessage({
|
||||
type: 'error',
|
||||
errorClass: err.name,
|
||||
errorMessage: err.message,
|
||||
stack: err.stack,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function postMessage(message: PdfParseWorkerMessage): void {
|
||||
parentPort!.postMessage(message);
|
||||
}
|
||||
|
||||
void run();
|
||||
@@ -1,5 +1,6 @@
|
||||
vi.mock('fs/promises', () => ({
|
||||
readFile: vi.fn(),
|
||||
stat: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('pdf-lib', () => ({
|
||||
@@ -17,19 +18,26 @@ vi.mock('./pdf-cover', () => ({
|
||||
extractPdfCover: vi.fn(),
|
||||
}));
|
||||
|
||||
import { readFile } from 'fs/promises';
|
||||
vi.mock('./pdf-parse-worker-runner', () => ({
|
||||
parsePdfFileInWorker: vi.fn(),
|
||||
}));
|
||||
|
||||
import { readFile, stat } from 'fs/promises';
|
||||
import type { MockedFunction } from 'vitest';
|
||||
import { PDFDocument } from 'pdf-lib';
|
||||
|
||||
import { extractPdfCover } from './pdf-cover';
|
||||
import { parsePdfFile, PDF_BUFFER_WARNING_BYTES } from './pdf-parser';
|
||||
import { parsePdfFile, PDF_BUFFER_WARNING_BYTES, type PdfParsed } from './pdf-parser';
|
||||
import { parsePdfFileInWorker } from './pdf-parse-worker-runner';
|
||||
import { extractXmpXml, parseXmp } from './pdf-xmp-reader';
|
||||
|
||||
const mockReadFile = readFile as MockedFunction<typeof readFile>;
|
||||
const mockStat = stat as MockedFunction<typeof stat>;
|
||||
const mockPdfLoad = PDFDocument.load as MockedFunction<typeof PDFDocument.load>;
|
||||
const mockExtractXmpXml = extractXmpXml as MockedFunction<typeof extractXmpXml>;
|
||||
const mockParseXmp = parseXmp as MockedFunction<typeof parseXmp>;
|
||||
const mockExtractPdfCover = extractPdfCover as MockedFunction<typeof extractPdfCover>;
|
||||
const mockParsePdfFileInWorker = parsePdfFileInWorker as MockedFunction<typeof parsePdfFileInWorker>;
|
||||
|
||||
function makePdfDoc(overrides: Partial<Record<string, unknown>> = {}) {
|
||||
return {
|
||||
@@ -44,15 +52,49 @@ function makePdfDoc(overrides: Partial<Record<string, unknown>> = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function makeParsed(overrides: Partial<PdfParsed> = {}): PdfParsed {
|
||||
return {
|
||||
title: 'Worker Title',
|
||||
subtitle: null,
|
||||
authors: [{ name: 'Worker Author', sortName: null }],
|
||||
description: null,
|
||||
publisher: null,
|
||||
publishedYear: null,
|
||||
language: null,
|
||||
genres: [],
|
||||
tags: [],
|
||||
isbn10: null,
|
||||
isbn13: null,
|
||||
seriesName: null,
|
||||
seriesIndex: null,
|
||||
rating: null,
|
||||
pageCount: 100,
|
||||
googleBooksId: null,
|
||||
goodreadsId: null,
|
||||
amazonId: null,
|
||||
hardcoverId: null,
|
||||
openLibraryId: null,
|
||||
ranobedbId: null,
|
||||
koboId: null,
|
||||
lubimyczytacId: null,
|
||||
aladinId: null,
|
||||
itunesId: null,
|
||||
coverBuffer: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('parsePdfFile', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
|
||||
mockStat.mockResolvedValue({ size: PDF_BUFFER_WARNING_BYTES - 1 } as never);
|
||||
mockReadFile.mockResolvedValue(Buffer.from('%PDF-1.7') as never);
|
||||
mockPdfLoad.mockResolvedValue(makePdfDoc() as never);
|
||||
mockExtractXmpXml.mockReturnValue(null);
|
||||
mockParseXmp.mockReturnValue(null);
|
||||
mockExtractPdfCover.mockResolvedValue(Buffer.from([0xff, 0xd8, 0xff]));
|
||||
mockParsePdfFileInWorker.mockResolvedValue({ parsed: makeParsed(), warnings: [] });
|
||||
});
|
||||
|
||||
it('prefers XMP metadata and includes extracted cover when requested', async () => {
|
||||
@@ -143,11 +185,12 @@ describe('parsePdfFile', () => {
|
||||
});
|
||||
|
||||
it('emits a warning when a large PDF must be buffered in memory', async () => {
|
||||
mockReadFile.mockResolvedValue(Buffer.alloc(PDF_BUFFER_WARNING_BYTES) as never);
|
||||
mockStat.mockResolvedValue({ size: PDF_BUFFER_WARNING_BYTES } as never);
|
||||
const onWarning = vi.fn();
|
||||
|
||||
await parsePdfFile('/books/large.pdf', { onWarning });
|
||||
const parsed = await parsePdfFile('/books/large.pdf', { onWarning });
|
||||
|
||||
expect(parsed).toEqual(expect.objectContaining({ title: 'Worker Title' }));
|
||||
expect(onWarning).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
code: 'buffered-large-pdf',
|
||||
@@ -156,6 +199,60 @@ describe('parsePdfFile', () => {
|
||||
thresholdBytes: PDF_BUFFER_WARNING_BYTES,
|
||||
}),
|
||||
);
|
||||
expect(mockParsePdfFileInWorker).toHaveBeenCalledWith({
|
||||
absolutePath: '/books/large.pdf',
|
||||
extractCover: false,
|
||||
});
|
||||
expect(mockReadFile).not.toHaveBeenCalled();
|
||||
expect(mockPdfLoad).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('replays large PDF worker warnings through the caller warning handler', async () => {
|
||||
mockStat.mockResolvedValue({ size: PDF_BUFFER_WARNING_BYTES + 1 } as never);
|
||||
mockParsePdfFileInWorker.mockResolvedValue({
|
||||
parsed: makeParsed(),
|
||||
warnings: [
|
||||
{
|
||||
code: 'cover-extraction-failed',
|
||||
absolutePath: '/books/large.pdf',
|
||||
errorClass: 'Error',
|
||||
errorMessage: 'pdftoppm missing',
|
||||
},
|
||||
],
|
||||
});
|
||||
const onWarning = vi.fn();
|
||||
|
||||
await parsePdfFile('/books/large.pdf', { extractCover: true, onWarning });
|
||||
|
||||
expect(mockParsePdfFileInWorker).toHaveBeenCalledWith({
|
||||
absolutePath: '/books/large.pdf',
|
||||
extractCover: true,
|
||||
});
|
||||
expect(onWarning).toHaveBeenCalledWith(expect.objectContaining({ code: 'buffered-large-pdf' }));
|
||||
expect(onWarning).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
code: 'cover-extraction-failed',
|
||||
errorMessage: 'pdftoppm missing',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns null and emits a parse warning when the large PDF worker fails', async () => {
|
||||
mockStat.mockResolvedValue({ size: PDF_BUFFER_WARNING_BYTES + 1 } as never);
|
||||
mockParsePdfFileInWorker.mockRejectedValue(new Error('worker failed'));
|
||||
const onWarning = vi.fn();
|
||||
|
||||
await expect(parsePdfFile('/books/large.pdf', { onWarning })).resolves.toBeNull();
|
||||
expect(onWarning).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
code: 'parse-failed',
|
||||
absolutePath: '/books/large.pdf',
|
||||
errorClass: 'Error',
|
||||
errorMessage: 'worker failed',
|
||||
}),
|
||||
);
|
||||
expect(mockReadFile).not.toHaveBeenCalled();
|
||||
expect(mockPdfLoad).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('falls back to the native PDF page count when XMP omits bookorbit:pageCount', async () => {
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { readFile } from 'fs/promises';
|
||||
import { readFile, stat } from 'fs/promises';
|
||||
import { PDFDocument } from 'pdf-lib';
|
||||
|
||||
import { BOOKORBIT_NS_PREFIX } from '../../../common/bookorbit-ns';
|
||||
import { sanitizeLogValue } from '../../../common/utils/log-sanitize.utils';
|
||||
import { parsePdfFileInWorker } from './pdf-parse-worker-runner';
|
||||
import { extractPdfCover } from './pdf-cover';
|
||||
import { extractXmpXml, parseXmp, type XmpParsed } from './pdf-xmp-reader';
|
||||
|
||||
@@ -82,73 +83,87 @@ function createLargeBufferWarning(absolutePath: string, sizeBytes: number): PdfP
|
||||
|
||||
export async function parsePdfFile(absolutePath: string, options: PdfParseOptions = {}): Promise<PdfParsed | null> {
|
||||
try {
|
||||
const buf = await readFile(absolutePath);
|
||||
if (buf.length >= PDF_BUFFER_WARNING_BYTES) {
|
||||
options.onWarning?.(createLargeBufferWarning(absolutePath, buf.length));
|
||||
}
|
||||
const doc = await PDFDocument.load(buf, { ignoreEncryption: true });
|
||||
|
||||
// XMP is the authoritative source; Info Dictionary is fallback-only.
|
||||
const xmpXml = extractXmpXml(doc);
|
||||
const xmp: XmpParsed | null = xmpXml ? parseXmp(xmpXml) : null;
|
||||
const hasXmp = xmp !== null;
|
||||
|
||||
const infoTitle = clean(doc.getTitle());
|
||||
const infoAuthorRaw = clean(doc.getAuthor());
|
||||
const infoCreator = clean(doc.getCreator());
|
||||
const infoProducer = clean(doc.getProducer());
|
||||
const infoSubject = clean(doc.getSubject());
|
||||
const infoKeywords = splitCommaList(clean(doc.getKeywords()));
|
||||
const isBookorbitInfo = infoCreator === BOOKORBIT_NS_PREFIX;
|
||||
|
||||
const infoAuthors = infoAuthorRaw
|
||||
? infoAuthorRaw
|
||||
.split(/[,;]/)
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
.map((name) => ({ name, sortName: null }))
|
||||
: [];
|
||||
|
||||
let coverBuffer: Buffer | null = null;
|
||||
if (options.extractCover === true) {
|
||||
try {
|
||||
coverBuffer = await extractPdfCover(absolutePath);
|
||||
} catch (error) {
|
||||
options.onWarning?.(createWarning('cover-extraction-failed', absolutePath, error));
|
||||
const fileStats = await stat(absolutePath);
|
||||
if (fileStats.size >= PDF_BUFFER_WARNING_BYTES) {
|
||||
options.onWarning?.(createLargeBufferWarning(absolutePath, fileStats.size));
|
||||
const result = await parsePdfFileInWorker({
|
||||
absolutePath,
|
||||
extractCover: options.extractCover === true,
|
||||
});
|
||||
for (const warning of result.warnings) {
|
||||
options.onWarning?.(warning);
|
||||
}
|
||||
return result.parsed;
|
||||
}
|
||||
|
||||
return {
|
||||
title: hasXmp ? xmp.title : infoTitle,
|
||||
subtitle: xmp?.subtitle ?? null,
|
||||
authors: hasXmp ? xmp.authors : infoAuthors,
|
||||
description: hasXmp ? xmp.description : infoSubject,
|
||||
publisher: hasXmp ? (xmp.publisher ?? null) : isBookorbitInfo ? infoProducer : null,
|
||||
publishedYear: xmp?.publishedYear ?? null,
|
||||
language: xmp?.language ?? null,
|
||||
genres: xmp?.genres?.length ? xmp.genres : [],
|
||||
// Info Dict keywords are genres+tags mixed; only use as tags when XMP is absent.
|
||||
tags: hasXmp ? xmp.tags : infoKeywords,
|
||||
isbn10: xmp?.isbn10 ?? null,
|
||||
isbn13: xmp?.isbn13 ?? null,
|
||||
seriesName: xmp?.seriesName ?? null,
|
||||
seriesIndex: xmp?.seriesIndex ?? null,
|
||||
rating: xmp?.rating ?? null,
|
||||
pageCount: hasXmp ? (xmp.pageCount ?? (isBookorbitInfo ? null : doc.getPageCount())) : doc.getPageCount(),
|
||||
googleBooksId: xmp?.googleBooksId ?? null,
|
||||
goodreadsId: xmp?.goodreadsId ?? null,
|
||||
amazonId: xmp?.amazonId ?? null,
|
||||
hardcoverId: xmp?.hardcoverId ?? null,
|
||||
openLibraryId: xmp?.openLibraryId ?? null,
|
||||
ranobedbId: xmp?.ranobedbId ?? null,
|
||||
koboId: xmp?.koboId ?? null,
|
||||
lubimyczytacId: xmp?.lubimyczytacId ?? null,
|
||||
aladinId: xmp?.aladinId ?? null,
|
||||
itunesId: xmp?.itunesId ?? null,
|
||||
coverBuffer,
|
||||
};
|
||||
const buf = await readFile(absolutePath);
|
||||
return await parsePdfBuffer(absolutePath, buf, options);
|
||||
} catch (error) {
|
||||
options.onWarning?.(createWarning('parse-failed', absolutePath, error));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function parsePdfBuffer(absolutePath: string, buf: Buffer, options: PdfParseOptions = {}): Promise<PdfParsed | null> {
|
||||
const doc = await PDFDocument.load(buf, { ignoreEncryption: true });
|
||||
|
||||
// XMP is the authoritative source; Info Dictionary is fallback-only.
|
||||
const xmpXml = extractXmpXml(doc);
|
||||
const xmp: XmpParsed | null = xmpXml ? parseXmp(xmpXml) : null;
|
||||
const hasXmp = xmp !== null;
|
||||
|
||||
const infoTitle = clean(doc.getTitle());
|
||||
const infoAuthorRaw = clean(doc.getAuthor());
|
||||
const infoCreator = clean(doc.getCreator());
|
||||
const infoProducer = clean(doc.getProducer());
|
||||
const infoSubject = clean(doc.getSubject());
|
||||
const infoKeywords = splitCommaList(clean(doc.getKeywords()));
|
||||
const isBookorbitInfo = infoCreator === BOOKORBIT_NS_PREFIX;
|
||||
|
||||
const infoAuthors = infoAuthorRaw
|
||||
? infoAuthorRaw
|
||||
.split(/[,;]/)
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
.map((name) => ({ name, sortName: null }))
|
||||
: [];
|
||||
|
||||
let coverBuffer: Buffer | null = null;
|
||||
if (options.extractCover === true) {
|
||||
try {
|
||||
coverBuffer = await extractPdfCover(absolutePath);
|
||||
} catch (error) {
|
||||
options.onWarning?.(createWarning('cover-extraction-failed', absolutePath, error));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title: hasXmp ? xmp.title : infoTitle,
|
||||
subtitle: xmp?.subtitle ?? null,
|
||||
authors: hasXmp ? xmp.authors : infoAuthors,
|
||||
description: hasXmp ? xmp.description : infoSubject,
|
||||
publisher: hasXmp ? (xmp.publisher ?? null) : isBookorbitInfo ? infoProducer : null,
|
||||
publishedYear: xmp?.publishedYear ?? null,
|
||||
language: xmp?.language ?? null,
|
||||
genres: xmp?.genres?.length ? xmp.genres : [],
|
||||
// Info Dict keywords are genres+tags mixed; only use as tags when XMP is absent.
|
||||
tags: hasXmp ? xmp.tags : infoKeywords,
|
||||
isbn10: xmp?.isbn10 ?? null,
|
||||
isbn13: xmp?.isbn13 ?? null,
|
||||
seriesName: xmp?.seriesName ?? null,
|
||||
seriesIndex: xmp?.seriesIndex ?? null,
|
||||
rating: xmp?.rating ?? null,
|
||||
pageCount: hasXmp ? (xmp.pageCount ?? (isBookorbitInfo ? null : doc.getPageCount())) : doc.getPageCount(),
|
||||
googleBooksId: xmp?.googleBooksId ?? null,
|
||||
goodreadsId: xmp?.goodreadsId ?? null,
|
||||
amazonId: xmp?.amazonId ?? null,
|
||||
hardcoverId: xmp?.hardcoverId ?? null,
|
||||
openLibraryId: xmp?.openLibraryId ?? null,
|
||||
ranobedbId: xmp?.ranobedbId ?? null,
|
||||
koboId: xmp?.koboId ?? null,
|
||||
lubimyczytacId: xmp?.lubimyczytacId ?? null,
|
||||
aladinId: xmp?.aladinId ?? null,
|
||||
itunesId: xmp?.itunesId ?? null,
|
||||
coverBuffer,
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user