feat: enhance PDF metadata extraction and refactor book cover storage

This commit is contained in:
neon
2026-04-08 20:32:38 -06:00
parent a8b203ef4a
commit 9f6b7d4f2b
31 changed files with 792 additions and 155 deletions
@@ -0,0 +1,35 @@
import { join } from 'path';
import {
bookCoverDirPath,
bookThumbnailPath,
findExtractedBookCoverFileName,
findPreferredBookCoverFileName,
isCustomBookCoverFileName,
isExtractedBookCoverFileName,
} from './book-cover-storage';
describe('book-cover-storage', () => {
it('builds cover and thumbnail paths under the book cover directory', () => {
expect(bookCoverDirPath('/books', 42)).toBe(join('/books', 'covers', '42'));
expect(bookThumbnailPath('/books', 42)).toBe(join('/books', 'covers', '42', 'thumbnail.jpg'));
});
it('detects custom and extracted cover file names', () => {
expect(isCustomBookCoverFileName('cover_custom.png')).toBe(true);
expect(isCustomBookCoverFileName('cover_extracted.png')).toBe(false);
expect(isExtractedBookCoverFileName('cover_extracted.jpg')).toBe(true);
expect(isExtractedBookCoverFileName('cover_custom.jpg')).toBe(false);
});
it('prefers custom covers over extracted covers', () => {
expect(findPreferredBookCoverFileName(['thumbnail.jpg', 'cover_extracted.jpg', 'cover_custom.png'])).toBe('cover_custom.png');
expect(findPreferredBookCoverFileName(['thumbnail.jpg', 'cover_extracted.jpg'])).toBe('cover_extracted.jpg');
expect(findPreferredBookCoverFileName(['thumbnail.jpg'])).toBeNull();
});
it('finds extracted cover files when present', () => {
expect(findExtractedBookCoverFileName(['cover_custom.png', 'cover_extracted.jpg'])).toBe('cover_extracted.jpg');
expect(findExtractedBookCoverFileName(['cover_custom.png'])).toBeNull();
});
});
+29
View File
@@ -0,0 +1,29 @@
import { join } from 'path';
export const COVER_CUSTOM_FILE_PREFIX = 'cover_custom.';
export const COVER_EXTRACTED_FILE_PREFIX = 'cover_extracted.';
export const COVER_THUMBNAIL_FILE_NAME = 'thumbnail.jpg';
export function bookCoverDirPath(booksPath: string, bookId: number): string {
return join(booksPath, 'covers', String(bookId));
}
export function bookThumbnailPath(booksPath: string, bookId: number): string {
return join(bookCoverDirPath(booksPath, bookId), COVER_THUMBNAIL_FILE_NAME);
}
export function isCustomBookCoverFileName(fileName: string): boolean {
return fileName.startsWith(COVER_CUSTOM_FILE_PREFIX);
}
export function isExtractedBookCoverFileName(fileName: string): boolean {
return fileName.startsWith(COVER_EXTRACTED_FILE_PREFIX);
}
export function findPreferredBookCoverFileName(files: readonly string[]): string | null {
return files.find(isCustomBookCoverFileName) ?? files.find(isExtractedBookCoverFileName) ?? null;
}
export function findExtractedBookCoverFileName(files: readonly string[]): string | null {
return files.find(isExtractedBookCoverFileName) ?? null;
}
+2
View File
@@ -0,0 +1,2 @@
export const PROJECTX_NS_PREFIX = 'projectx';
export const PROJECTX_NS_URI = 'https://projectx.app/metadata/1.0/';
@@ -0,0 +1,145 @@
vi.mock('fs/promises', () => ({
mkdir: vi.fn(),
writeFile: vi.fn(),
}));
vi.mock('../metadata/lib/pdf-parser', () => ({
parsePdfFile: vi.fn(),
}));
vi.mock('../metadata/lib/cover', () => ({
extractCover: vi.fn(),
generateThumbnail: vi.fn(),
imageExt: vi.fn(),
}));
import { mkdir, writeFile } from 'fs/promises';
import type { MockedFunction } from 'vitest';
import { extractCover, generateThumbnail, imageExt } from '../metadata/lib/cover';
import { parsePdfFile } from '../metadata/lib/pdf-parser';
import { BookBucketMetadataService } from './book-bucket-metadata.service';
const mockMkdir = mkdir as MockedFunction<typeof mkdir>;
const mockWriteFile = writeFile as MockedFunction<typeof writeFile>;
const mockExtractCover = extractCover as MockedFunction<typeof extractCover>;
const mockGenerateThumbnail = generateThumbnail as MockedFunction<typeof generateThumbnail>;
const mockImageExt = imageExt as MockedFunction<typeof imageExt>;
const mockParsePdfFile = parsePdfFile as MockedFunction<typeof parsePdfFile>;
describe('BookBucketMetadataService', () => {
const repo = {
update: vi.fn().mockResolvedValue(undefined),
};
beforeEach(() => {
vi.resetAllMocks();
repo.update.mockResolvedValue(undefined);
mockMkdir.mockResolvedValue(undefined);
mockWriteFile.mockResolvedValue(undefined);
mockExtractCover.mockResolvedValue(null);
mockGenerateThumbnail.mockResolvedValue(Buffer.from('thumb'));
mockImageExt.mockReturnValue('png');
mockParsePdfFile.mockResolvedValue(null);
});
it('extractAndSave(pdf) reuses a single parse result for metadata and cover persistence', async () => {
mockParsePdfFile.mockResolvedValue({
title: 'PDF Title',
subtitle: null,
description: null,
publisher: 'PDF Publisher',
publishedYear: null,
language: null,
authors: [{ name: 'Author A', sortName: null }],
genres: ['Genre A'],
tags: [],
isbn10: null,
isbn13: null,
seriesName: null,
seriesIndex: null,
rating: null,
pageCount: 200,
googleBooksId: null,
goodreadsId: null,
amazonId: null,
hardcoverId: null,
openLibraryId: null,
itunesId: null,
coverBuffer: Buffer.from('cover'),
});
const service = new BookBucketMetadataService(repo as never);
await service.extractAndSave(5, '/tmp/book.pdf', 'pdf', '/tmp/covers');
expect(mockParsePdfFile).toHaveBeenCalledTimes(1);
expect(mockParsePdfFile).toHaveBeenCalledWith(
'/tmp/book.pdf',
expect.objectContaining({
extractCover: true,
onWarning: expect.any(Function),
}),
);
expect(mockExtractCover).not.toHaveBeenCalled();
expect(repo.update).toHaveBeenNthCalledWith(1, 5, { status: 'extracting' });
expect(repo.update).toHaveBeenNthCalledWith(2, 5, {
embeddedMetadata: {
title: 'PDF Title',
publisher: 'PDF Publisher',
pageCount: 200,
authors: ['Author A'],
genres: ['Genre A'],
},
coverPath: '/tmp/covers/5.png',
status: 'ready',
});
expect(mockMkdir).toHaveBeenCalledWith('/tmp/covers', { recursive: true });
expect(mockWriteFile).toHaveBeenCalledWith('/tmp/covers/5.png', Buffer.from('cover'));
expect(mockWriteFile).toHaveBeenCalledWith('/tmp/covers/5_thumb.jpg', Buffer.from('thumb'));
});
it('extractAndSave(pdf) leaves coverPath null when no cover bytes are available', async () => {
mockParsePdfFile.mockResolvedValue({
title: 'PDF Title',
subtitle: null,
description: null,
publisher: null,
publishedYear: null,
language: null,
authors: [],
genres: [],
tags: [],
isbn10: null,
isbn13: null,
seriesName: null,
seriesIndex: null,
rating: null,
pageCount: 12,
googleBooksId: null,
goodreadsId: null,
amazonId: null,
hardcoverId: null,
openLibraryId: null,
itunesId: null,
coverBuffer: null,
});
const service = new BookBucketMetadataService(repo as never);
await service.extractAndSave(6, '/tmp/book.pdf', 'pdf', '/tmp/covers');
expect(repo.update).toHaveBeenNthCalledWith(2, 6, {
embeddedMetadata: {
title: 'PDF Title',
publisher: undefined,
pageCount: 12,
authors: undefined,
genres: undefined,
},
coverPath: null,
status: 'ready',
});
expect(mockWriteFile).not.toHaveBeenCalled();
});
});
@@ -7,7 +7,7 @@ import { extractEpubMetadata } from '../metadata/lib/epub';
import { extractCbzMetadata, extractCbrMetadata, extractCb7Metadata } from '../metadata/lib/cbz-metadata';
import { parseFb2File } from '../metadata/lib/fb2-parser';
import { parseMobiFile } from '../metadata/lib/mobi-parser';
import { parsePdfFile } from '../metadata/lib/pdf-parser';
import { parsePdfFile, type PdfParsed, type PdfParseWarning } from '../metadata/lib/pdf-parser';
import { extractCover, generateThumbnail, imageExt } from '../metadata/lib/cover';
import { BookBucketRepository } from './book-bucket.repository';
@@ -18,9 +18,19 @@ export class BookBucketMetadataService {
constructor(private readonly repo: BookBucketRepository) {}
async extractAndSave(fileId: number, absolutePath: string, format: string, coversDir: string): Promise<void> {
const startedAt = Date.now();
await this.repo.update(fileId, { status: 'extracting' });
try {
const [metadata, coverBytes] = await Promise.all([this.extractMetadata(absolutePath, format), extractCover(absolutePath, format)]);
let metadata: BookBucketMetadata;
let coverBytes: Buffer | null;
if (format === 'pdf') {
const parsedPdf = await this.extractPdfData(absolutePath, true);
metadata = this.toPdfMetadata(parsedPdf);
coverBytes = parsedPdf?.coverBuffer ?? null;
} else {
[metadata, coverBytes] = await Promise.all([this.extractMetadata(absolutePath, format), extractCover(absolutePath, format)]);
}
let coverPath: string | null = null;
if (coverBytes && coverBytes.length > 0) {
@@ -33,7 +43,11 @@ export class BookBucketMetadataService {
status: 'ready',
});
} catch (err) {
this.logger.warn(`Metadata extraction failed for Book Bucket file ${fileId}: ${err instanceof Error ? err.message : String(err)}`);
const errorClass = err instanceof Error ? err.name : 'Error';
const errorMessage = (err instanceof Error ? err.message : String(err)).replace(/"/g, '\\"');
this.logger.warn(
`[book_bucket.extract_metadata] [fail] fileId=${fileId} format=${format} durationMs=${Date.now() - startedAt} errorClass=${errorClass} error="${errorMessage}" - metadata extraction failed`,
);
await this.repo.update(fileId, {
status: 'error',
errorMessage: err instanceof Error ? err.message : 'Metadata extraction failed',
@@ -84,15 +98,7 @@ export class BookBucketMetadataService {
}
private async fromPdf(absolutePath: string): Promise<BookBucketMetadata> {
const parsed = await parsePdfFile(absolutePath);
if (!parsed) return {};
return {
title: parsed.title ?? undefined,
publisher: parsed.publisher ?? undefined,
pageCount: parsed.pageCount ?? undefined,
authors: parsed.authors.length > 0 ? parsed.authors.map((a) => a.name) : undefined,
genres: parsed.genres.length > 0 ? parsed.genres : undefined,
};
return this.toPdfMetadata(await this.extractPdfData(absolutePath, false));
}
private async fromMobi(absolutePath: string): Promise<BookBucketMetadata> {
@@ -155,4 +161,34 @@ export class BookBucketMetadataService {
return coverPath;
}
private async extractPdfData(absolutePath: string, extractCover: boolean): Promise<PdfParsed | null> {
return parsePdfFile(absolutePath, {
extractCover,
onWarning: (warning) => this.logPdfParseWarning(warning),
});
}
private toPdfMetadata(parsed: PdfParsed | null): BookBucketMetadata {
if (!parsed) return {};
return {
title: parsed.title ?? undefined,
publisher: parsed.publisher ?? undefined,
pageCount: parsed.pageCount ?? undefined,
authors: parsed.authors.length > 0 ? parsed.authors.map((a) => a.name) : undefined,
genres: parsed.genres.length > 0 ? parsed.genres : undefined,
};
}
private logPdfParseWarning(warning: PdfParseWarning): void {
if (warning.code === 'buffered-large-pdf') {
this.logger.warn(
`[book_bucket.pdf_parse] [end] path="${warning.absolutePath}" code=${warning.code} sizeBytes=${warning.sizeBytes ?? 0} thresholdBytes=${warning.thresholdBytes ?? 0} - large pdf buffered in memory`,
);
return;
}
this.logger.warn(
`[book_bucket.pdf_parse] [fail] path="${warning.absolutePath}" code=${warning.code} errorClass=${warning.errorClass} error="${warning.errorMessage}" - pdf parse warning emitted`,
);
}
}
+21 -5
View File
@@ -2,11 +2,12 @@ import { BadRequestException, Injectable, InternalServerErrorException, Logger,
import { ConfigService } from '@nestjs/config';
import { access, readdir, rm, stat } from 'fs/promises';
import { bookCoverDirPath, bookThumbnailPath, findPreferredBookCoverFileName } from '../../common/book-cover-storage';
import { extractEpubMetadata } from '../metadata/lib/epub';
import { extractCbzMetadata, extractCbrMetadata, extractCb7Metadata } from '../metadata/lib/cbz-metadata';
import { parseFb2File } from '../metadata/lib/fb2-parser';
import { parseMobiFile } from '../metadata/lib/mobi-parser';
import { parsePdfFile } from '../metadata/lib/pdf-parser';
import { parsePdfFile, type PdfParseWarning } from '../metadata/lib/pdf-parser';
import { basename, extname, join } from 'path';
import { DEFAULT_DOWNLOAD_PATTERN, MetadataProviderKey, Permission, isAudioFormat, resolveUploadPath } from '@projectx/types';
@@ -185,10 +186,10 @@ export class BookService {
async getCoverPath(id: number, user: RequestUser): Promise<string | null> {
const event = 'book.get_cover_path';
await this.verifyBookAccess(id, user);
const dir = join(this.booksPath, 'covers', String(id));
const dir = bookCoverDirPath(this.booksPath, id);
try {
const files = await readdir(dir);
const cover = files.find((f) => f.startsWith('cover_custom.')) ?? files.find((f) => f.startsWith('cover_extracted.'));
const cover = findPreferredBookCoverFileName(files);
return cover ? join(dir, cover) : null;
} catch (err) {
if (this.isMissingFilesystemEntry(err)) return null;
@@ -204,7 +205,7 @@ export class BookService {
async getThumbnailPath(id: number, user: RequestUser): Promise<string | null> {
const event = 'book.get_thumbnail_path';
await this.verifyBookAccess(id, user);
const path = join(this.booksPath, 'covers', String(id), 'thumbnail.jpg');
const path = bookThumbnailPath(this.booksPath, id);
try {
await access(path);
return path;
@@ -1117,7 +1118,10 @@ export class BookService {
};
}
case 'pdf': {
const parsed = await parsePdfFile(absolutePath);
const parsed = await parsePdfFile(absolutePath, {
extractCover: false,
onWarning: (warning) => this.logPdfFileMetadataWarning(warning),
});
if (!parsed) return {};
return {
title: parsed.title,
@@ -1186,6 +1190,18 @@ export class BookService {
}
}
private logPdfFileMetadataWarning(warning: PdfParseWarning): void {
if (warning.code === 'buffered-large-pdf') {
this.logger.warn(
`[book.file_metadata_pdf] [end] path="${warning.absolutePath}" code=${warning.code} sizeBytes=${warning.sizeBytes ?? 0} thresholdBytes=${warning.thresholdBytes ?? 0} - large pdf buffered in memory`,
);
return;
}
this.logger.warn(
`[book.file_metadata_pdf] [fail] path="${warning.absolutePath}" code=${warning.code} errorClass=${warning.errorClass} error="${warning.errorMessage}" - pdf file metadata warning emitted`,
);
}
private resolveChapters(
stored: AudiobookChapter[] | null | undefined,
fileRows: { absolutePath: string; format: string | null; durationSeconds: number | null }[],
+2 -4
View File
@@ -1,3 +1,5 @@
export { COVER_CUSTOM_FILE_PREFIX, COVER_EXTRACTED_FILE_PREFIX, COVER_THUMBNAIL_FILE_NAME } from '../../common/book-cover-storage';
export const COVER_PROVIDERS = Symbol('COVER_PROVIDERS');
export const COVER_PROXY_USER_AGENT =
@@ -6,7 +8,3 @@ export const COVER_PROXY_USER_AGENT =
export const COVER_PROXY_TIMEOUT_MS = 10_000;
export const COVER_PROXY_MAX_IMAGE_BYTES = 20 * 1024 * 1024;
export const COVER_PROXY_MAX_REDIRECTS = 5;
export const COVER_CUSTOM_FILE_PREFIX = 'cover_custom.';
export const COVER_EXTRACTED_FILE_PREFIX = 'cover_extracted.';
export const COVER_THUMBNAIL_FILE_NAME = 'thumbnail.jpg';
+9 -10
View File
@@ -7,6 +7,7 @@ import { mkdir, readdir, readFile, unlink, writeFile } from 'fs/promises';
import { isIP } from 'net';
import { join } from 'path';
import { bookCoverDirPath, bookThumbnailPath, findExtractedBookCoverFileName } from '../../common/book-cover-storage';
import type { RequestUser } from '../../common/types/request-user';
import { DB } from '../../db';
import * as schema from '../../db/schema';
@@ -14,15 +15,13 @@ import { bookMetadata } from '../../db/schema';
import { BookReadService } from '../book/book-read.service';
import { BookMetadataLockService } from '../book-metadata-lock/book-metadata-lock.service';
import { LibraryService } from '../library/library.service';
import { coverDirPath, generateThumbnail, imageExt } from '../metadata/lib/cover';
import { generateThumbnail, imageExt } from '../metadata/lib/cover';
import {
COVER_CUSTOM_FILE_PREFIX,
COVER_EXTRACTED_FILE_PREFIX,
COVER_PROXY_MAX_IMAGE_BYTES,
COVER_PROXY_MAX_REDIRECTS,
COVER_PROXY_TIMEOUT_MS,
COVER_PROXY_USER_AGENT,
COVER_THUMBNAIL_FILE_NAME,
} from './constants';
import { CoverProviderRegistry } from './provider-registry';
import {
@@ -153,12 +152,12 @@ export class CoverService {
try {
await this.verifyAccess(bookId, user);
await this.bookMetadataLockService.assertFieldsUnlocked(bookId, ['cover']);
const dir = coverDirPath(this.booksPath, bookId);
const dir = bookCoverDirPath(this.booksPath, bookId);
await this.deleteFilesByPrefix(dir, COVER_CUSTOM_FILE_PREFIX);
const extractedPath = await this.findExtractedCover(bookId);
if (!extractedPath) {
await this.removeFileIfPresent(join(dir, COVER_THUMBNAIL_FILE_NAME));
await this.removeFileIfPresent(bookThumbnailPath(this.booksPath, bookId));
await this.setCoverSource(bookId, null);
this.logger.log(
`[cover.delete] [end] bookId=${bookId} userId=${user.id} durationMs=${Date.now() - startedAt} coverSource=null - cover deletion completed`,
@@ -168,7 +167,7 @@ export class CoverService {
const bytes = await readFile(extractedPath);
const thumb = await generateThumbnail(bytes);
await writeFile(join(dir, COVER_THUMBNAIL_FILE_NAME), thumb);
await writeFile(bookThumbnailPath(this.booksPath, bookId), thumb);
await this.setCoverSource(bookId, 'extracted');
this.logger.log(
`[cover.delete] [end] bookId=${bookId} userId=${user.id} durationMs=${Date.now() - startedAt} coverSource=extracted - cover deletion completed`,
@@ -231,20 +230,20 @@ export class CoverService {
}
private async saveCustomCover(bookId: number, buffer: Buffer): Promise<void> {
const dir = coverDirPath(this.booksPath, bookId);
const dir = bookCoverDirPath(this.booksPath, bookId);
await mkdir(dir, { recursive: true });
await this.deleteFilesByPrefix(dir, COVER_CUSTOM_FILE_PREFIX);
const ext = imageExt(buffer);
await writeFile(join(dir, `${COVER_CUSTOM_FILE_PREFIX}${ext}`), buffer);
const thumb = await generateThumbnail(buffer);
await writeFile(join(dir, COVER_THUMBNAIL_FILE_NAME), thumb);
await writeFile(bookThumbnailPath(this.booksPath, bookId), thumb);
}
private async findExtractedCover(bookId: number): Promise<string | null> {
const dir = coverDirPath(this.booksPath, bookId);
const dir = bookCoverDirPath(this.booksPath, bookId);
const files = await this.readDirIfExists(dir);
const found = files.find((fileName) => fileName.startsWith(COVER_EXTRACTED_FILE_PREFIX));
const found = findExtractedBookCoverFileName(files);
return found ? join(dir, found) : null;
}
@@ -4,6 +4,7 @@ import { readdir, readFile } from 'fs/promises';
import { join } from 'path';
import type { WriteResult } from '@projectx/types';
import { bookCoverDirPath, findPreferredBookCoverFileName } from '../../common/book-cover-storage';
import { FORMAT_CB7, FORMAT_CBZ, FORMAT_EPUB, FORMAT_PDF, createBookWriteFieldMask } from './file-write.constants';
import { FileLockService } from './file-lock.service';
import { FileWriteRepository } from './file-write.repository';
@@ -201,10 +202,10 @@ export class FileWriteService implements OnModuleDestroy {
private async loadCoverBytes(bookId: number): Promise<Buffer | null> {
const startedAt = Date.now();
const dir = join(this.booksPath, 'covers', String(bookId));
const dir = bookCoverDirPath(this.booksPath, bookId);
try {
const files = await readdir(dir);
const cover = files.find((f) => f.startsWith('cover_custom.')) ?? files.find((f) => f.startsWith('cover_extracted.'));
const cover = findPreferredBookCoverFileName(files);
if (!cover) return null;
return readFile(join(dir, cover));
} catch (error) {
@@ -115,7 +115,6 @@ describe('PdfFormatWriter', () => {
expect(pdfDoc.setTitle).toHaveBeenCalledWith('Dune');
expect(pdfDoc.setAuthor).toHaveBeenCalledWith('Frank Herbert');
expect(pdfDoc.setSubject).toHaveBeenCalledWith('Sci-fi classic');
expect(pdfDoc.setProducer).toHaveBeenCalledWith('Ace');
expect(pdfDoc.setCreationDate).toHaveBeenCalledWith(new Date(1965, 0, 1));
expect(pdfDoc.setCreator).toHaveBeenCalledWith('projectx');
expect(pdfDoc.setKeywords).toHaveBeenCalledWith(['Sci-Fi', 'Classic']);
@@ -52,9 +52,6 @@ function applyInfoDict(pdfDoc: PDFDocument, payload: BookWritePayload, fieldMask
if (fieldMask.has('description') && payload.description != null) {
pdfDoc.setSubject(payload.description);
}
if (fieldMask.has('publisher') && payload.publisher != null) {
pdfDoc.setProducer(payload.publisher);
}
if (fieldMask.has('publishedYear') && payload.publishedYear != null) {
pdfDoc.setCreationDate(new Date(payload.publishedYear, 0, 1));
}
@@ -1,2 +1 @@
export const PROJECTX_NS_PREFIX = 'projectx';
export const PROJECTX_NS_URI = 'https://projectx.app/metadata/1.0/';
export { PROJECTX_NS_PREFIX, PROJECTX_NS_URI } from '../../../../common/projectx-ns';
@@ -4,6 +4,7 @@ import { readdir, stat } from 'fs/promises';
import { createReadStream } from 'fs';
import { join } from 'path';
import type { FastifyReply } from 'fastify';
import { bookCoverDirPath, bookThumbnailPath, findPreferredBookCoverFileName } from '../../../common/book-cover-storage';
import { KoboBookAccessService } from './kobo-book-access.service';
@Injectable()
@@ -20,7 +21,7 @@ export class KoboThumbnailService {
async serveThumbnail(userId: number, bookId: number, ifNoneMatch: string | undefined, reply: FastifyReply) {
await this.bookAccessService.assertBookAccessible(userId, bookId);
const thumbnailPath = join(this.booksPath, 'covers', String(bookId), 'thumbnail.jpg');
const thumbnailPath = bookThumbnailPath(this.booksPath, bookId);
try {
const { mtimeMs } = await stat(thumbnailPath);
const etag = `"${Math.floor(mtimeMs)}"`;
@@ -38,10 +39,10 @@ export class KoboThumbnailService {
}
async serveCover(bookId: number, ifNoneMatch: string | undefined, reply: FastifyReply) {
const dir = join(this.booksPath, 'covers', String(bookId));
const dir = bookCoverDirPath(this.booksPath, bookId);
try {
const files = await readdir(dir);
const cover = files.find((f) => f.startsWith('cover.'));
const cover = findPreferredBookCoverFileName(files);
if (!cover) throw new NotFoundException('No cover');
const coverPath = join(dir, cover);
const { mtimeMs } = await stat(coverPath);
@@ -13,6 +13,14 @@ export interface ParsedBookData {
seriesIndex?: number | null;
authors: { name: string; sortName: string | null }[];
genres: string[];
tags?: string[];
rating?: number | null;
googleBooksId?: string | null;
goodreadsId?: string | null;
amazonId?: string | null;
hardcoverId?: string | null;
openLibraryId?: string | null;
itunesId?: string | null;
cover: Buffer | null;
// audio-specific
narrators?: string[];
@@ -1,10 +1,17 @@
import { parseBookFilename } from '../lib/filename-parser';
import { parsePdfFile } from '../lib/pdf-parser';
import { parsePdfFile, type PdfParseOptions } from '../lib/pdf-parser';
import type { FormatExtractor, ParsedBookData } from './format-extractor.interface';
type PdfFormatExtractorOptions = Pick<PdfParseOptions, 'extractCover' | 'onWarning'>;
export class PdfFormatExtractor implements FormatExtractor {
constructor(private readonly options: PdfFormatExtractorOptions = {}) {}
async extract(absolutePath: string): Promise<ParsedBookData | null> {
const pdf = await parsePdfFile(absolutePath);
const pdf = await parsePdfFile(absolutePath, {
extractCover: this.options.extractCover === true,
onWarning: this.options.onWarning,
});
if (!pdf) return null;
const fb = !pdf.title ? parseBookFilename(absolutePath) : null;
@@ -21,6 +28,14 @@ export class PdfFormatExtractor implements FormatExtractor {
seriesIndex: pdf.seriesIndex,
authors: pdf.authors,
genres: pdf.genres,
tags: pdf.tags,
rating: pdf.rating,
googleBooksId: pdf.googleBooksId,
goodreadsId: pdf.goodreadsId,
amazonId: pdf.amazonId,
hardcoverId: pdf.hardcoverId,
openLibraryId: pdf.openLibraryId,
itunesId: pdf.itunesId,
cover: pdf.coverBuffer,
pageCount: pdf.pageCount,
};
+11 -1
View File
@@ -4,6 +4,7 @@ vi.mock('./cover-cbr', () => ({ extractCbrCover: vi.fn() }));
vi.mock('./cover-cb7', () => ({ extractCb7Cover: vi.fn() }));
vi.mock('./cover-fb2', () => ({ extractFb2Cover: vi.fn() }));
vi.mock('./mobi-parser', () => ({ extractMobiCover: vi.fn() }));
vi.mock('./pdf-cover', () => ({ extractPdfCover: vi.fn() }));
vi.mock('sharp', () => {
const chain = {
resize: vi.fn(),
@@ -24,6 +25,7 @@ import { extractCbzCover } from './cover-cbz';
import { extractEpubCover } from './cover-epub';
import { extractFb2Cover } from './cover-fb2';
import { extractMobiCover } from './mobi-parser';
import { extractPdfCover } from './pdf-cover';
import { coverDirPath, extractCover, generateThumbnail, imageExt } from './cover';
const mockEpub = extractEpubCover as MockedFunction<typeof extractEpubCover>;
const mockMobi = extractMobiCover as MockedFunction<typeof extractMobiCover>;
@@ -31,6 +33,7 @@ const mockCbz = extractCbzCover as MockedFunction<typeof extractCbzCover>;
const mockCbr = extractCbrCover as MockedFunction<typeof extractCbrCover>;
const mockCb7 = extractCb7Cover as MockedFunction<typeof extractCb7Cover>;
const mockFb2 = extractFb2Cover as MockedFunction<typeof extractFb2Cover>;
const mockPdf = extractPdfCover as MockedFunction<typeof extractPdfCover>;
const JPEG_BYTES = Buffer.from([0xff, 0xd8, 0xff, 0xe0]);
@@ -43,6 +46,7 @@ beforeEach(() => {
mockCbr.mockResolvedValue(null);
mockCb7.mockResolvedValue(null);
mockFb2.mockResolvedValue(null);
mockPdf.mockResolvedValue(null);
});
// ── imageExt ──────────────────────────────────────────────────────────────────
@@ -150,8 +154,14 @@ describe('extractCover', () => {
expect(mockFb2).toHaveBeenCalledWith('/book.fb2');
});
it('routes pdf to extractPdfCover', async () => {
mockPdf.mockResolvedValue(JPEG_BYTES);
await extractCover('/book.pdf', 'pdf');
expect(mockPdf).toHaveBeenCalledWith('/book.pdf');
});
it('returns null for unsupported format', async () => {
expect(await extractCover('/book.pdf', 'pdf')).toBeNull();
expect(await extractCover('/book.txt', 'txt')).toBeNull();
});
it('format matching is case-insensitive', async () => {
+5 -2
View File
@@ -1,12 +1,13 @@
import { join } from 'path';
import sharp from 'sharp';
import { bookCoverDirPath } from '../../../common/book-cover-storage';
import { extractCb7Cover } from './cover-cb7';
import { extractCbrCover } from './cover-cbr';
import { extractCbzCover } from './cover-cbz';
import { extractEpubCover } from './cover-epub';
import { extractFb2Cover } from './cover-fb2';
import { extractMobiCover } from './mobi-parser';
import { extractPdfCover } from './pdf-cover';
const THUMBNAIL_WIDTH = 400;
const THUMBNAIL_HEIGHT = 600;
@@ -41,6 +42,8 @@ export async function extractCover(absolutePath: string, format: string): Promis
return extractCb7Cover(absolutePath);
case 'fb2':
return extractFb2Cover(absolutePath);
case 'pdf':
return extractPdfCover(absolutePath).catch(() => null);
default:
return null;
}
@@ -48,5 +51,5 @@ export async function extractCover(absolutePath: string, format: string): Promis
/** Resolve the cover directory for a book on disk. */
export function coverDirPath(booksPath: string, bookId: number): string {
return join(booksPath, 'covers', String(bookId));
return bookCoverDirPath(booksPath, bookId);
}
@@ -0,0 +1,19 @@
import { execFile } from 'child_process';
import { mkdtemp, readFile, rm } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { promisify } from 'util';
const execFileAsync = promisify(execFile);
export async function extractPdfCover(absolutePath: string): Promise<Buffer | null> {
const tmpDir = await mkdtemp(join(tmpdir(), 'pdf-cover-'));
const outPrefix = join(tmpDir, 'cover');
try {
await execFileAsync('pdftoppm', ['-jpeg', '-singlefile', '-r', '150', '-f', '1', '-l', '1', absolutePath, outPrefix]);
return await readFile(`${outPrefix}.jpg`);
} finally {
await rm(tmpDir, { recursive: true, force: true });
}
}
@@ -1,12 +1,7 @@
vi.mock('fs/promises', () => ({
mkdtemp: vi.fn(),
readFile: vi.fn(),
readdir: vi.fn(),
rm: vi.fn(),
}));
vi.mock('child_process', () => ({ execFile: vi.fn() }));
vi.mock('pdf-lib', () => ({
PDFDocument: {
load: vi.fn(),
@@ -18,51 +13,49 @@ vi.mock('./pdf-xmp-reader', () => ({
parseXmp: vi.fn(),
}));
import { execFile } from 'child_process';
import { mkdtemp, readFile, readdir, rm } from 'fs/promises';
vi.mock('./pdf-cover', () => ({
extractPdfCover: vi.fn(),
}));
import { readFile } from 'fs/promises';
import type { MockedFunction } from 'vitest';
import { PDFDocument } from 'pdf-lib';
import { extractPdfCover } from './pdf-cover';
import { parsePdfFile, PDF_BUFFER_WARNING_BYTES } from './pdf-parser';
import { extractXmpXml, parseXmp } from './pdf-xmp-reader';
import { parsePdfFile } from './pdf-parser';
const mockExecFile = execFile as unknown as vi.Mock;
const mockMkdtemp = mkdtemp as MockedFunction<typeof mkdtemp>;
const mockReadFile = readFile as MockedFunction<typeof readFile>;
const mockReaddir = readdir as MockedFunction<typeof readdir>;
const mockRm = rm as MockedFunction<typeof rm>;
const mockPdfLoad = PDFDocument.load as vi.Mock;
const mockPdfLoad = PDFDocument.load as MockedFunction<typeof PDFDocument.load>;
const mockExtractXmpXml = extractXmpXml as MockedFunction<typeof extractXmpXml>;
const mockParseXmp = parseXmp as MockedFunction<typeof parseXmp>;
const mockExtractPdfCover = extractPdfCover as MockedFunction<typeof extractPdfCover>;
function mockExecSuccess() {
mockExecFile.mockImplementation((_file: string, _args: string[], cb: (err: Error | null, stdout?: string, stderr?: string) => void) => {
cb(null, '', '');
});
function makePdfDoc(overrides: Partial<Record<string, unknown>> = {}) {
return {
getTitle: () => 'Info Title',
getAuthor: () => 'Author One; Author Two',
getCreator: () => null,
getProducer: () => null,
getSubject: () => 'Info Subject',
getKeywords: () => 'tag1, tag2',
getPageCount: () => 123,
...overrides,
};
}
describe('parsePdfFile', () => {
beforeEach(() => {
vi.resetAllMocks();
mockMkdtemp.mockResolvedValue('/tmp/pdf-cover-1');
mockReaddir.mockResolvedValue(['cover-1.jpg'] as unknown as Awaited<ReturnType<typeof readdir>>);
mockRm.mockResolvedValue(undefined);
mockReadFile
.mockResolvedValueOnce(Buffer.from('%PDF-1.7') as unknown as Awaited<ReturnType<typeof readFile>>)
.mockResolvedValueOnce(Buffer.from([0xff, 0xd8, 0xff]) as unknown as Awaited<ReturnType<typeof readFile>>);
mockPdfLoad.mockResolvedValue({
getTitle: () => 'Info Title',
getAuthor: () => 'Author One; Author Two',
getSubject: () => 'Info Subject',
getKeywords: () => 'tag1, tag2',
getPageCount: () => 123,
});
mockExecSuccess();
mockReadFile.mockResolvedValue(Buffer.from('%PDF-1.7') as never);
mockPdfLoad.mockResolvedValue(makePdfDoc() as never);
mockExtractXmpXml.mockReturnValue(null);
mockParseXmp.mockReturnValue(null);
mockExtractPdfCover.mockResolvedValue(Buffer.from([0xff, 0xd8, 0xff]));
});
it('prefers XMP metadata when available and still includes extracted cover/pageCount', async () => {
it('prefers XMP metadata and includes extracted cover when requested', async () => {
mockExtractXmpXml.mockReturnValue('<xmp/>');
mockParseXmp.mockReturnValue({
title: 'XMP Title',
@@ -85,9 +78,10 @@ describe('parsePdfFile', () => {
amazonId: 'a1',
hardcoverId: 'h1',
openLibraryId: 'ol1',
itunesId: 'it1',
});
const parsed = await parsePdfFile('/books/book.pdf');
const parsed = await parsePdfFile('/books/book.pdf', { extractCover: true });
expect(parsed).toEqual(
expect.objectContaining({
@@ -98,11 +92,10 @@ describe('parsePdfFile', () => {
coverBuffer: Buffer.from([0xff, 0xd8, 0xff]),
}),
);
expect(mockExtractPdfCover).toHaveBeenCalledWith('/books/book.pdf');
});
it('falls back to Info dictionary fields when no XMP is present', async () => {
mockExtractXmpXml.mockReturnValue(null);
const parsed = await parsePdfFile('/books/book.pdf');
expect(parsed).toEqual(
@@ -114,33 +107,111 @@ describe('parsePdfFile', () => {
],
description: 'Info Subject',
tags: ['tag1', 'tag2'],
coverBuffer: null,
}),
);
});
it('can skip cover extraction when only metadata is needed', async () => {
mockExtractXmpXml.mockReturnValue(null);
const parsed = await parsePdfFile('/books/book.pdf', { extractCover: false });
expect(parsed).toEqual(expect.objectContaining({ title: 'Info Title', coverBuffer: null }));
expect(mockExecFile).not.toHaveBeenCalled();
expect(mockReadFile).toHaveBeenCalledTimes(1);
});
it('returns null coverBuffer when pdftoppm command fails, but still returns metadata', async () => {
mockExtractXmpXml.mockReturnValue(null);
mockExecFile.mockImplementation((_file: string, _args: string[], cb: (err: Error | null) => void) => cb(new Error('pdftoppm missing')));
it('skips cover extraction by default for metadata-only callers', async () => {
const parsed = await parsePdfFile('/books/book.pdf');
expect(parsed).toEqual(expect.objectContaining({ title: 'Info Title', coverBuffer: null }));
expect(mockRm).toHaveBeenCalledWith('/tmp/pdf-cover-1', { recursive: true, force: true });
expect(mockExtractPdfCover).not.toHaveBeenCalled();
expect(mockReadFile).toHaveBeenCalledTimes(1);
});
it('returns null when PDF cannot be parsed', async () => {
mockPdfLoad.mockRejectedValue(new Error('invalid pdf'));
it('returns null coverBuffer and emits a warning when cover extraction fails', async () => {
mockExtractPdfCover.mockRejectedValue(new Error('pdftoppm missing'));
const onWarning = vi.fn();
await expect(parsePdfFile('/books/bad.pdf')).resolves.toBeNull();
const parsed = await parsePdfFile('/books/book.pdf', {
extractCover: true,
onWarning,
});
expect(parsed).toEqual(expect.objectContaining({ title: 'Info Title', coverBuffer: null }));
expect(onWarning).toHaveBeenCalledWith(
expect.objectContaining({
code: 'cover-extraction-failed',
absolutePath: '/books/book.pdf',
errorClass: 'Error',
errorMessage: 'pdftoppm missing',
}),
);
});
it('emits a warning when a large PDF must be buffered in memory', async () => {
mockReadFile.mockResolvedValue(Buffer.alloc(PDF_BUFFER_WARNING_BYTES) as never);
const onWarning = vi.fn();
await parsePdfFile('/books/large.pdf', { onWarning });
expect(onWarning).toHaveBeenCalledWith(
expect.objectContaining({
code: 'buffered-large-pdf',
absolutePath: '/books/large.pdf',
sizeBytes: PDF_BUFFER_WARNING_BYTES,
thresholdBytes: PDF_BUFFER_WARNING_BYTES,
}),
);
});
it('falls back to the native PDF page count when XMP omits projectx:pageCount', async () => {
mockExtractXmpXml.mockReturnValue('<xmp/>');
mockParseXmp.mockReturnValue({
title: 'XMP Title',
subtitle: null,
description: null,
publisher: null,
publishedYear: null,
language: null,
authors: [],
genres: [],
tags: [],
isbn10: null,
isbn13: null,
seriesName: null,
seriesIndex: null,
rating: null,
pageCount: null,
googleBooksId: null,
goodreadsId: null,
amazonId: null,
hardcoverId: null,
openLibraryId: null,
itunesId: null,
});
const parsed = await parsePdfFile('/books/book.pdf');
expect(parsed?.pageCount).toBe(123);
});
it('falls back to legacy ProjectX producer metadata when XMP publisher is absent', async () => {
mockPdfLoad.mockResolvedValue(
makePdfDoc({
getCreator: () => 'projectx',
getProducer: () => 'Legacy Publisher',
}) as never,
);
const parsed = await parsePdfFile('/books/book.pdf');
expect(parsed?.publisher).toBe('Legacy Publisher');
});
it('returns null and emits a parse warning when the PDF cannot be read', async () => {
mockPdfLoad.mockRejectedValue(new Error('invalid pdf'));
const onWarning = vi.fn();
await expect(parsePdfFile('/books/bad.pdf', { onWarning })).resolves.toBeNull();
expect(onWarning).toHaveBeenCalledWith(
expect.objectContaining({
code: 'parse-failed',
absolutePath: '/books/bad.pdf',
errorClass: 'Error',
errorMessage: 'invalid pdf',
}),
);
});
});
+47 -27
View File
@@ -1,14 +1,10 @@
import { execFile } from 'child_process';
import { mkdtemp, readFile, readdir, rm } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { promisify } from 'util';
import { readFile } from 'fs/promises';
import { PDFDocument } from 'pdf-lib';
import { PROJECTX_NS_PREFIX } from '../../../common/projectx-ns';
import { extractPdfCover } from './pdf-cover';
import { extractXmpXml, parseXmp, type XmpParsed } from './pdf-xmp-reader';
const execFileAsync = promisify(execFile);
export interface PdfParsed {
title: string | null;
subtitle: string | null;
@@ -34,10 +30,22 @@ export interface PdfParsed {
coverBuffer: Buffer | null;
}
interface PdfParseOptions {
extractCover?: boolean;
export interface PdfParseWarning {
code: 'buffered-large-pdf' | 'cover-extraction-failed' | 'parse-failed';
absolutePath: string;
errorClass?: string;
errorMessage?: string;
sizeBytes?: number;
thresholdBytes?: number;
}
export interface PdfParseOptions {
extractCover?: boolean;
onWarning?: (warning: PdfParseWarning) => void;
}
export const PDF_BUFFER_WARNING_BYTES = 25 * 1024 * 1024;
function clean(value: string | undefined): string | null {
if (!value) return null;
const s = value.trim();
@@ -52,25 +60,27 @@ function splitCommaList(value: string | null): string[] {
.filter(Boolean);
}
async function extractPdfCover(absolutePath: string): Promise<Buffer | null> {
const tmpDir = await mkdtemp(join(tmpdir(), 'pdf-cover-'));
try {
const outPrefix = join(tmpDir, 'cover');
await execFileAsync('pdftoppm', ['-jpeg', '-r', '150', '-f', '1', '-l', '1', absolutePath, outPrefix]);
const files = await readdir(tmpDir);
const coverFile = files.find((f) => f.endsWith('.jpg'));
if (!coverFile) return null;
return await readFile(join(tmpDir, coverFile));
} catch {
return null;
} finally {
await rm(tmpDir, { recursive: true, force: true });
}
function createWarning(code: PdfParseWarning['code'], absolutePath: string, error: unknown): PdfParseWarning {
const errorClass = error instanceof Error ? error.name : 'Error';
const errorMessage = (error instanceof Error ? error.message : String(error)).replace(/"/g, '\\"');
return { code, absolutePath, errorClass, errorMessage };
}
function createLargeBufferWarning(absolutePath: string, sizeBytes: number): PdfParseWarning {
return {
code: 'buffered-large-pdf',
absolutePath,
sizeBytes,
thresholdBytes: PDF_BUFFER_WARNING_BYTES,
};
}
export async function parsePdfFile(absolutePath: string, options: PdfParseOptions = {}): Promise<PdfParsed | null> {
try {
const buf = await readFile(absolutePath);
if (buf.length >= PDF_BUFFER_WARNING_BYTES) {
options.onWarning?.(createLargeBufferWarning(absolutePath, buf.length));
}
const doc = await PDFDocument.load(buf, { ignoreEncryption: true });
// XMP is the authoritative source — richer and semantically correct.
@@ -81,6 +91,8 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
const infoTitle = clean(doc.getTitle());
const infoAuthorRaw = clean(doc.getAuthor());
const infoCreator = clean(doc.getCreator());
const infoProducer = clean(doc.getProducer());
const infoSubject = clean(doc.getSubject());
const infoKeywords = splitCommaList(clean(doc.getKeywords()));
@@ -92,14 +104,21 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
.map((name) => ({ name, sortName: null }))
: [];
const coverBuffer = options.extractCover === false ? null : await extractPdfCover(absolutePath);
let coverBuffer: Buffer | null = null;
if (options.extractCover === true) {
try {
coverBuffer = await extractPdfCover(absolutePath);
} catch (error) {
options.onWarning?.(createWarning('cover-extraction-failed', absolutePath, error));
}
}
return {
title: hasXmp ? xmp.title : infoTitle,
subtitle: xmp?.subtitle ?? null,
authors: hasXmp ? xmp.authors : infoAuthors,
description: hasXmp ? xmp.description : infoSubject,
publisher: xmp?.publisher ?? null,
publisher: xmp?.publisher ?? (infoCreator === PROJECTX_NS_PREFIX ? infoProducer : null),
publishedYear: xmp?.publishedYear ?? null,
language: xmp?.language ?? null,
genres: xmp?.genres?.length ? xmp.genres : [],
@@ -110,7 +129,7 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
seriesName: xmp?.seriesName ?? null,
seriesIndex: xmp?.seriesIndex ?? null,
rating: xmp?.rating ?? null,
pageCount: hasXmp ? xmp.pageCount : doc.getPageCount(),
pageCount: xmp?.pageCount ?? doc.getPageCount(),
googleBooksId: xmp?.googleBooksId ?? null,
goodreadsId: xmp?.goodreadsId ?? null,
amazonId: xmp?.amazonId ?? null,
@@ -119,7 +138,8 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
itunesId: xmp?.itunesId ?? null,
coverBuffer,
};
} catch {
} catch (error) {
options.onWarning?.(createWarning('parse-failed', absolutePath, error));
return null;
}
}
@@ -1,3 +1,4 @@
import { PROJECTX_NS_PREFIX, PROJECTX_NS_URI } from '../../../common/projectx-ns';
import { parseXmp } from './pdf-xmp-reader';
// Wraps XMP content in standard RDF envelope
@@ -7,7 +8,7 @@ function xmpDoc(body: string): string {
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<rdf:Description rdf:about=""
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:projectx="https://projectx.app/xmp/1.0/">
xmlns:${PROJECTX_NS_PREFIX}="${PROJECTX_NS_URI}">
${body}
</rdf:Description>
</rdf:RDF>
@@ -195,7 +196,7 @@ describe('parseXmp', () => {
<rdf:Description rdf:about="" xmlns:dc="http://purl.org/dc/elements/1.1/">
<dc:title>Merged Title</dc:title>
</rdf:Description>
<rdf:Description rdf:about="" xmlns:projectx="https://projectx.app/xmp/1.0/">
<rdf:Description rdf:about="" xmlns:${PROJECTX_NS_PREFIX}="${PROJECTX_NS_URI}">
<projectx:seriesName>Merged Series</projectx:seriesName>
</rdf:Description>
</rdf:RDF>
@@ -1,6 +1,8 @@
import { XMLParser } from 'fast-xml-parser';
import { PDFDocument, PDFName, PDFRawStream, PDFRef } from 'pdf-lib';
import { PROJECTX_NS_PREFIX } from '../../../common/projectx-ns';
export interface XmpParsed {
title: string | null;
subtitle: string | null;
@@ -125,7 +127,7 @@ export function parseXmp(xmpXml: string): XmpParsed | null {
}
}
const px = 'projectx';
const px = PROJECTX_NS_PREFIX;
return {
title: str(merged['dc:title']),
@@ -77,6 +77,7 @@ vi.mock('./extractors/audio.extractor', () => ({
}));
import { mkdir, readdir, rm, writeFile } from 'fs/promises';
import { Logger } from '@nestjs/common';
import { authors, bookAuthors, bookGenres, bookMetadata, bookTags, genres, tags } from '../../db/schema';
import { generateThumbnail, imageExt } from './lib/cover';
@@ -235,11 +236,57 @@ describe('MetadataService', () => {
await expect(service.extractAndSave(15, '/books/a.pdf', 'pdf')).rejects.toThrow('bad metadata');
});
it('extractAndSave(pdf) surfaces parser warnings in logs', async () => {
const { db } = makeDb();
const service = makeService(db);
const warnSpy = vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
mockParsePdfFile.mockImplementation((_absolutePath, options) => {
options?.onWarning?.({
code: 'cover-extraction-failed',
absolutePath: '/tmp/warn.pdf',
errorClass: 'Error',
errorMessage: 'pdftoppm missing',
});
return Promise.resolve({
title: 'Warn PDF',
subtitle: null,
description: null,
isbn10: null,
isbn13: null,
publisher: null,
publishedYear: null,
language: null,
seriesName: null,
seriesIndex: null,
authors: [],
genres: [],
tags: [],
rating: null,
pageCount: null,
googleBooksId: null,
goodreadsId: null,
amazonId: null,
hardcoverId: null,
openLibraryId: null,
itunesId: null,
coverBuffer: null,
});
});
await service.extractAndSave(21, '/tmp/warn.pdf', 'pdf');
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('[metadata.pdf_parse] [fail]'));
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('pdftoppm missing'));
});
it('extractAndSave(pdf) persists fallback title/year, page count, and extracted cover bytes', async () => {
const { db, updateSet } = makeDb();
const service = makeService(db);
const replaceAuthorsSpy = vi.spyOn(service, 'replaceAuthors').mockResolvedValue(undefined);
const replaceGenresSpy = vi.spyOn(service, 'replaceGenres').mockResolvedValue(undefined);
const replaceTagsSpy = vi.spyOn(service, 'replaceTags').mockResolvedValue(undefined);
mockParsePdfFile.mockResolvedValue({
title: null,
@@ -254,14 +301,29 @@ describe('MetadataService', () => {
seriesIndex: 2,
authors: [{ name: 'Author A', sortName: null }],
genres: ['Fantasy'],
tags: [],
tags: ['Shelf'],
rating: 4.6,
pageCount: 321,
googleBooksId: 'google-1',
goodreadsId: 'goodreads-1',
amazonId: 'amazon-1',
hardcoverId: 'hardcover-1',
openLibraryId: 'open-library-1',
itunesId: 'itunes-1',
coverBuffer: Buffer.from('jpeg-bytes'),
});
mockParseBookFilename.mockReturnValue({ title: 'Title From Filename', publishedYear: 1999 });
await service.extractAndSave(22, '/tmp/book.pdf', 'pdf');
expect(mockParsePdfFile).toHaveBeenCalledWith(
'/tmp/book.pdf',
expect.objectContaining({
extractCover: true,
onWarning: expect.any(Function),
}),
);
expect(updateSet).toHaveBeenCalledWith(
expect.objectContaining({
title: 'Title From Filename',
@@ -269,12 +331,20 @@ describe('MetadataService', () => {
subtitle: 'Subtitle',
description: 'Description',
pageCount: 321,
rating: 5,
googleBooksId: 'google-1',
goodreadsId: 'goodreads-1',
amazonId: 'amazon-1',
hardcoverId: 'hardcover-1',
openLibraryId: 'open-library-1',
itunesId: 'itunes-1',
updatedAt: expect.any(Date),
}),
);
expect(mockWriteFile).toHaveBeenCalledWith('/books/covers/22/cover_extracted.png', Buffer.from('jpeg-bytes'));
expect(replaceAuthorsSpy).toHaveBeenCalledWith(22, [{ name: 'Author A', sortName: null }]);
expect(replaceGenresSpy).toHaveBeenCalledWith(22, ['Fantasy']);
expect(replaceTagsSpy).toHaveBeenCalledWith(22, ['Shelf']);
expect(embedder.embedBook).toHaveBeenCalledWith(22);
});
@@ -7,6 +7,13 @@ import { join } from 'path';
import { DB } from '../../db';
import * as schema from '../../db/schema';
import {
COVER_EXTRACTED_FILE_PREFIX,
bookCoverDirPath,
bookThumbnailPath,
isCustomBookCoverFileName,
isExtractedBookCoverFileName,
} from '../../common/book-cover-storage';
import { BookEmbedderService } from '../embedding/book-embedder.service';
import { BookMetadataLockService } from '../book-metadata-lock/book-metadata-lock.service';
import { ComicMetadataRepository } from './comic-metadata.repository';
@@ -23,6 +30,7 @@ import { MobiFormatExtractor } from './extractors/mobi-format.extractor';
import { PdfFormatExtractor } from './extractors/pdf-format.extractor';
import type { FormatExtractor, ParsedBookData } from './extractors/format-extractor.interface';
import { generateThumbnail, imageExt } from './lib/cover';
import type { PdfParseWarning } from './lib/pdf-parser';
import { MetadataEventsService, METADATA_AUTHORS_REPLACED } from './metadata-events.service';
type Db = NodePgDatabase<typeof schema>;
@@ -35,9 +43,6 @@ interface RelationMutationOptions {
const AUDIO_FORMATS = ['m4b', 'mp3', 'm4a', 'opus', 'ogg', 'flac'] as const;
const MAX_RELATION_NAME_LENGTH = 200;
const CUSTOM_COVER_FILE_PREFIX = 'cover_custom.';
const EXTRACTED_COVER_FILE_PREFIX = 'cover_extracted.';
const THUMBNAIL_FILE_NAME = 'thumbnail.jpg';
const EXTRACTED_COVER_SOURCE = 'extracted';
@Injectable()
@@ -63,7 +68,7 @@ export class MetadataService {
this.extractorMap = new Map<string, FormatExtractor>([
['epub', epub],
['kepub', epub],
['pdf', new PdfFormatExtractor()],
['pdf', new PdfFormatExtractor({ extractCover: true, onWarning: (warning) => this.logPdfParseWarning(warning) })],
['mobi', mobi],
['azw3', mobi],
['azw', mobi],
@@ -468,7 +473,15 @@ export class MetadataService {
seriesIndex: data.seriesIndex,
authors: data.authors.map((author) => author.name),
genres: data.genres,
tags: data.tags,
rating: normalizeImportedRating(data.rating),
pageCount: data.pageCount,
googleBooksId: data.googleBooksId,
goodreadsId: data.goodreadsId,
amazonId: data.amazonId,
hardcoverId: data.hardcoverId,
openLibraryId: data.openLibraryId,
itunesId: data.itunesId,
comicMetadata: data.comicMetadata ?? undefined,
});
@@ -483,7 +496,14 @@ export class MetadataService {
if (filtered.language !== undefined) scalarFields.language = filtered.language;
if (filtered.seriesName !== undefined) scalarFields.seriesName = filtered.seriesName;
if (filtered.seriesIndex !== undefined) scalarFields.seriesIndex = filtered.seriesIndex;
if (filtered.rating !== undefined) scalarFields.rating = filtered.rating;
if (filtered.pageCount !== undefined) scalarFields.pageCount = filtered.pageCount;
if (filtered.googleBooksId !== undefined) scalarFields.googleBooksId = filtered.googleBooksId;
if (filtered.goodreadsId !== undefined) scalarFields.goodreadsId = filtered.goodreadsId;
if (filtered.amazonId !== undefined) scalarFields.amazonId = filtered.amazonId;
if (filtered.hardcoverId !== undefined) scalarFields.hardcoverId = filtered.hardcoverId;
if (filtered.openLibraryId !== undefined) scalarFields.openLibraryId = filtered.openLibraryId;
if (filtered.itunesId !== undefined) scalarFields.itunesId = filtered.itunesId;
if (Object.keys(scalarFields).length > 0) {
scalarFields.updatedAt = new Date();
await this.db.update(bookMetadata).set(scalarFields).where(eq(bookMetadata.bookId, bookId));
@@ -495,6 +515,9 @@ export class MetadataService {
if (filtered.genres !== undefined) {
await this.replaceGenres(bookId, filtered.genres);
}
if (filtered.tags !== undefined) {
await this.replaceTags(bookId, filtered.tags);
}
if (filtered.comicMetadata) {
await this.comicMetadataRepository.upsert(bookId, filtered.comicMetadata);
@@ -517,20 +540,20 @@ export class MetadataService {
private async persistCover(bookId: number, bytes: Buffer, overwrite: boolean): Promise<void> {
if (await this.bookMetadataLockService.isFieldLocked(bookId, 'cover')) return;
const ext = imageExt(bytes);
const dir = join(this.booksPath, 'covers', String(bookId));
const dir = bookCoverDirPath(this.booksPath, bookId);
await mkdir(dir, { recursive: true });
const files = await readdir(dir).catch(() => [] as string[]);
const hasCustom = files.some((fileName) => fileName.startsWith(CUSTOM_COVER_FILE_PREFIX));
const hasCustom = files.some(isCustomBookCoverFileName);
const staleExtractedFiles = files.filter((fileName) => fileName.startsWith(EXTRACTED_COVER_FILE_PREFIX));
const staleExtractedFiles = files.filter(isExtractedBookCoverFileName);
await Promise.all(staleExtractedFiles.map((fileName) => rm(join(dir, fileName), { force: true })));
await writeFile(join(dir, `${EXTRACTED_COVER_FILE_PREFIX}${ext}`), bytes);
await writeFile(join(dir, `${COVER_EXTRACTED_FILE_PREFIX}${ext}`), bytes);
if (!hasCustom) {
const thumbnail = await generateThumbnail(bytes);
await writeFile(join(dir, THUMBNAIL_FILE_NAME), thumbnail);
await writeFile(bookThumbnailPath(this.booksPath, bookId), thumbnail);
}
if (overwrite) {
@@ -543,7 +566,27 @@ export class MetadataService {
}
}
private logPdfParseWarning(warning: PdfParseWarning): void {
if (warning.code === 'buffered-large-pdf') {
this.logger.warn(
`[metadata.pdf_parse] [end] path="${warning.absolutePath}" code=${warning.code} sizeBytes=${warning.sizeBytes ?? 0} thresholdBytes=${warning.thresholdBytes ?? 0} - large pdf buffered in memory`,
);
return;
}
this.logger.warn(
`[metadata.pdf_parse] [fail] path="${warning.absolutePath}" code=${warning.code} errorClass=${warning.errorClass} error="${warning.errorMessage}" - pdf parse warning emitted`,
);
}
private normalizeUniqueRelationNames(values: string[]): string[] {
return [...new Set(values.map((value) => value.trim().substring(0, MAX_RELATION_NAME_LENGTH)).filter(Boolean))];
}
}
function normalizeImportedRating(value: number | null | undefined): number | null | undefined {
if (value === undefined) return undefined;
if (value === null) return null;
if (!Number.isFinite(value)) return null;
const normalized = Math.round(value);
return normalized >= 1 && normalized <= 10 ? normalized : null;
}
@@ -4,7 +4,8 @@ import { access, mkdir, readdir, readFile, unlink, writeFile } from 'fs/promises
import { join } from 'path';
import type { CoverRefreshedEvent } from '@projectx/types';
import { coverDirPath, generateThumbnail, imageExt } from '../../metadata/lib/cover';
import { COVER_CUSTOM_FILE_PREFIX, COVER_THUMBNAIL_FILE_NAME, bookCoverDirPath } from '../../../common/book-cover-storage';
import { generateThumbnail, imageExt } from '../../metadata/lib/cover';
import { MigrationRepository } from '../migration.repository';
import { MigrationImportRepository } from './migration-import.repository';
import { ScanGateway } from '../../scanner/scan.gateway';
@@ -92,16 +93,16 @@ export class CoverImporter {
}
private async importSingleCover(targetBookId: number, booksPath: string, coverBytes: Buffer, sourceThumbnailPath: string): Promise<void> {
const targetCoverDir = coverDirPath(booksPath, targetBookId);
const targetCoverDir = bookCoverDirPath(booksPath, targetBookId);
await mkdir(targetCoverDir, { recursive: true });
await this.deleteFilesByPrefix(targetCoverDir, 'cover_custom.');
await this.deleteFilesByPrefix(targetCoverDir, COVER_CUSTOM_FILE_PREFIX);
const coverExt = imageExt(coverBytes);
await writeFile(join(targetCoverDir, `cover_custom.${coverExt}`), coverBytes);
await writeFile(join(targetCoverDir, `${COVER_CUSTOM_FILE_PREFIX}${coverExt}`), coverBytes);
const sourceThumbnailBytes = await this.readOptionalFile(sourceThumbnailPath);
const thumbnailBytes = sourceThumbnailBytes ?? (await generateThumbnail(coverBytes));
await writeFile(join(targetCoverDir, 'thumbnail.jpg'), thumbnailBytes);
await writeFile(join(targetCoverDir, COVER_THUMBNAIL_FILE_NAME), thumbnailBytes);
await this.importRepo.markCoverAsCustom(targetBookId);
}
@@ -0,0 +1,72 @@
vi.mock('fs', () => ({
createReadStream: vi.fn(),
}));
vi.mock('fs/promises', () => ({
readdir: vi.fn(),
stat: vi.fn(),
}));
import { createReadStream } from 'fs';
import { readdir, stat } from 'fs/promises';
import type { MockedFunction } from 'vitest';
import { OpdsController } from './opds.controller';
const mockCreateReadStream = createReadStream as MockedFunction<typeof createReadStream>;
const mockReaddir = readdir as MockedFunction<typeof readdir>;
const mockStat = stat as MockedFunction<typeof stat>;
function makeController() {
const opdsService = {} as never;
const opdsBookService = {
validateBookAccess: vi.fn().mockResolvedValue(undefined),
} as never;
const config = {
get: vi.fn().mockReturnValue('/books'),
} as never;
return {
controller: new OpdsController(opdsService, opdsBookService, config),
opdsBookService,
};
}
function makeReply() {
const reply = {
header: vi.fn(),
type: vi.fn(),
status: vi.fn(),
send: vi.fn(),
};
reply.header.mockReturnValue(reply);
reply.type.mockReturnValue(reply);
reply.status.mockReturnValue(reply);
return reply as never;
}
describe('OpdsController', () => {
beforeEach(() => {
vi.resetAllMocks();
});
it('serves the preferred stored cover file for OPDS clients', async () => {
const { controller, opdsBookService } = makeController();
const reply = makeReply();
const stream = { kind: 'stream' };
mockReaddir.mockResolvedValue(['cover_extracted.jpg', 'cover_custom.png'] as never);
mockStat.mockResolvedValue({ mtimeMs: 1234 } as never);
mockCreateReadStream.mockReturnValue(stream as never);
await controller.cover(42, { userId: 7, isSuperuser: false } as never, reply);
expect(opdsBookService.validateBookAccess).toHaveBeenCalledWith(42, 7, false);
expect(mockCreateReadStream).toHaveBeenCalledWith('/books/covers/42/cover_custom.png');
expect(reply.header).toHaveBeenCalledWith('ETag', '"1234"');
expect(reply.type).toHaveBeenCalledWith('image/png');
expect(reply.send).toHaveBeenCalledWith(stream);
});
});
+4 -3
View File
@@ -17,6 +17,7 @@ import { join } from 'path';
import { ConfigService } from '@nestjs/config';
import type { FastifyReply } from 'fastify';
import { bookCoverDirPath, bookThumbnailPath, findPreferredBookCoverFileName } from '../../common/book-cover-storage';
import { Public } from '../../common/decorators/public.decorator';
import { OPDS_MIME_ACQ, OPDS_MIME_NAV, OPDS_MIME_SEARCH, fileMimeType } from './opds-xml.helpers';
import { OpdsAuthGuard } from './opds-auth.guard';
@@ -193,10 +194,10 @@ export class OpdsController {
@Headers('if-none-match') ifNoneMatch?: string,
) {
await this.opdsBookService.validateBookAccess(bookId, user.userId, user.isSuperuser);
const dir = join(this.booksPath, 'covers', String(bookId));
const dir = bookCoverDirPath(this.booksPath, bookId);
try {
const files = await readdir(dir);
const cover = files.find((f) => f.startsWith('cover.'));
const cover = findPreferredBookCoverFileName(files);
if (!cover) throw new NotFoundException('No cover');
const coverPath = join(dir, cover);
const { mtimeMs } = await stat(coverPath);
@@ -223,7 +224,7 @@ export class OpdsController {
@Headers('if-none-match') ifNoneMatch?: string,
) {
await this.opdsBookService.validateBookAccess(bookId, user.userId, user.isSuperuser);
const thumbnailPath = join(this.booksPath, 'covers', String(bookId), 'thumbnail.jpg');
const thumbnailPath = bookThumbnailPath(this.booksPath, bookId);
try {
const { mtimeMs } = await stat(thumbnailPath);
const etag = `"${Math.floor(mtimeMs)}"`;
@@ -17,8 +17,8 @@ import { assembleBookCards } from '../book/utils/assemble-book-cards';
const METADATA_FORMATS = new Set(['epub', 'mobi', 'azw3', 'azw', 'cbz', 'cbr', 'cb7', 'fb2', 'pdf', 'm4b', 'mp3', 'm4a', 'opus', 'ogg', 'flac']);
const BATCH_SIZE = 3;
const BOOK_EMIT_BUFFER_SIZE = 30;
const BOOK_EMIT_FLUSH_INTERVAL_MS = 1500;
const BOOK_EMIT_BUFFER_SIZE = 20;
const BOOK_EMIT_FLUSH_INTERVAL_MS = 1000;
type OrganizationMode = 'book_per_file' | 'book_per_folder';
interface ScanCounts {
@@ -340,6 +340,13 @@ describe('UploadService', () => {
expect(result).toEqual({ bookId: 99, filename: 'Research Paper.pdf', format: 'pdf', sizeBytes: 456 });
expect(storage.moveToPath).toHaveBeenCalledWith('/tmp/upload.bin', '/library/Research Paper.pdf');
expect(mockParsePdfFile).toHaveBeenCalledWith(
'/tmp/upload.bin',
expect.objectContaining({
extractCover: false,
onWarning: expect.any(Function),
}),
);
});
it('cbz metadata populates tokens', async () => {
+17 -2
View File
@@ -19,7 +19,7 @@ import type { UploadResult } from '@projectx/types';
import { extractEpubMetadata } from '../metadata/lib/epub';
import { extractCbzMetadata, extractCbrMetadata, extractCb7Metadata } from '../metadata/lib/cbz-metadata';
import { parseMobiFile } from '../metadata/lib/mobi-parser';
import { parsePdfFile } from '../metadata/lib/pdf-parser';
import { parsePdfFile, type PdfParseWarning } from '../metadata/lib/pdf-parser';
type Db = NodePgDatabase<typeof schema>;
@@ -180,7 +180,10 @@ export class UploadService {
};
}
} else if (format === 'pdf') {
const pdf = await parsePdfFile(tempPath);
const pdf = await parsePdfFile(tempPath, {
extractCover: false,
onWarning: (warning) => this.logPdfPatternTokenWarning(warning),
});
if (pdf) {
parsed = { title: pdf.title, publisher: pdf.publisher, authors: pdf.authors, seriesName: null, seriesIndex: null };
}
@@ -220,6 +223,18 @@ export class UploadService {
return { errorClass, errorMessage };
}
private logPdfPatternTokenWarning(warning: PdfParseWarning): void {
if (warning.code === 'buffered-large-pdf') {
this.logger.warn(
`[upload.pattern_tokens_pdf] [end] path="${warning.absolutePath}" code=${warning.code} sizeBytes=${warning.sizeBytes ?? 0} thresholdBytes=${warning.thresholdBytes ?? 0} - large pdf buffered in memory`,
);
return;
}
this.logger.warn(
`[upload.pattern_tokens_pdf] [fail] path="${warning.absolutePath}" code=${warning.code} errorClass=${warning.errorClass} error="${warning.errorMessage}" - pdf token extraction warning emitted`,
);
}
private async destinationExists(absolutePath: string): Promise<boolean> {
try {
await fsAccess(absolutePath);
@@ -1,4 +1,4 @@
import { mkdir, writeFile } from 'fs/promises';
import { mkdir, readFile, writeFile } from 'fs/promises';
import { dirname, join } from 'path';
import { afterEach } from 'vitest';
@@ -7,6 +7,7 @@ import { Permission, type BookBucketFile, type BookBucketFinalizeResult } from '
import * as schema from '../src/db/schema';
import { buildFb2Fixture } from './e2e/book-bucket/book-bucket-fixture-builder';
import { createPdfFixture } from './e2e/metadata-write/metadata-write-fixture-builder';
import {
authHeader,
closeBookBucketE2EContext,
@@ -141,6 +142,27 @@ describe('Book Bucket ingest + finalize (e2e)', () => {
expect(summaryResponse.json()).toMatchObject({ total: 1, ready: 1 });
});
it('ingests pdf uploads and persists extracted cover files', async () => {
const fixturePath = await createPdfFixture(context.fixture.rootPath, 'fixtures/book-bucket-cover.pdf', 'Book Bucket PDF Title');
const pdfBytes = await readFile(fixturePath);
const uploadResponse = await uploadBookBucketFile(context, {
token: context.adminToken,
fileName: 'book-bucket-cover.pdf',
content: pdfBytes,
contentType: 'application/pdf',
});
expect(uploadResponse.statusCode).toBe(201);
const uploaded = uploadResponse.json() as BookBucketFile;
const ready = await waitForBookBucketStatus(context, uploaded.id, ['ready']);
expect(ready.embeddedMetadata?.title).toBe('Book Bucket PDF Title');
expect(ready.coverPath).toEqual(expect.stringContaining('.jpg'));
expect(ready.coverPath).not.toBeNull();
expect(await fileExists(ready.coverPath!)).toBe(true);
});
it('bulk set-target selectAll with status=pending includes pending, extracting, and fetching', async () => {
const destination = await createLibraryWithFolder(context);
const pendingRow = await createBookBucketRow(context, { fileName: 'pending-target.fb2', status: 'pending' });