mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 08:21:39 -06:00
feat: enhance PDF metadata extraction and refactor book cover storage
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
import { join } from 'path';
|
||||
|
||||
import {
|
||||
bookCoverDirPath,
|
||||
bookThumbnailPath,
|
||||
findExtractedBookCoverFileName,
|
||||
findPreferredBookCoverFileName,
|
||||
isCustomBookCoverFileName,
|
||||
isExtractedBookCoverFileName,
|
||||
} from './book-cover-storage';
|
||||
|
||||
describe('book-cover-storage', () => {
|
||||
it('builds cover and thumbnail paths under the book cover directory', () => {
|
||||
expect(bookCoverDirPath('/books', 42)).toBe(join('/books', 'covers', '42'));
|
||||
expect(bookThumbnailPath('/books', 42)).toBe(join('/books', 'covers', '42', 'thumbnail.jpg'));
|
||||
});
|
||||
|
||||
it('detects custom and extracted cover file names', () => {
|
||||
expect(isCustomBookCoverFileName('cover_custom.png')).toBe(true);
|
||||
expect(isCustomBookCoverFileName('cover_extracted.png')).toBe(false);
|
||||
expect(isExtractedBookCoverFileName('cover_extracted.jpg')).toBe(true);
|
||||
expect(isExtractedBookCoverFileName('cover_custom.jpg')).toBe(false);
|
||||
});
|
||||
|
||||
it('prefers custom covers over extracted covers', () => {
|
||||
expect(findPreferredBookCoverFileName(['thumbnail.jpg', 'cover_extracted.jpg', 'cover_custom.png'])).toBe('cover_custom.png');
|
||||
expect(findPreferredBookCoverFileName(['thumbnail.jpg', 'cover_extracted.jpg'])).toBe('cover_extracted.jpg');
|
||||
expect(findPreferredBookCoverFileName(['thumbnail.jpg'])).toBeNull();
|
||||
});
|
||||
|
||||
it('finds extracted cover files when present', () => {
|
||||
expect(findExtractedBookCoverFileName(['cover_custom.png', 'cover_extracted.jpg'])).toBe('cover_extracted.jpg');
|
||||
expect(findExtractedBookCoverFileName(['cover_custom.png'])).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
import { join } from 'path';
|
||||
|
||||
export const COVER_CUSTOM_FILE_PREFIX = 'cover_custom.';
|
||||
export const COVER_EXTRACTED_FILE_PREFIX = 'cover_extracted.';
|
||||
export const COVER_THUMBNAIL_FILE_NAME = 'thumbnail.jpg';
|
||||
|
||||
export function bookCoverDirPath(booksPath: string, bookId: number): string {
|
||||
return join(booksPath, 'covers', String(bookId));
|
||||
}
|
||||
|
||||
export function bookThumbnailPath(booksPath: string, bookId: number): string {
|
||||
return join(bookCoverDirPath(booksPath, bookId), COVER_THUMBNAIL_FILE_NAME);
|
||||
}
|
||||
|
||||
export function isCustomBookCoverFileName(fileName: string): boolean {
|
||||
return fileName.startsWith(COVER_CUSTOM_FILE_PREFIX);
|
||||
}
|
||||
|
||||
export function isExtractedBookCoverFileName(fileName: string): boolean {
|
||||
return fileName.startsWith(COVER_EXTRACTED_FILE_PREFIX);
|
||||
}
|
||||
|
||||
export function findPreferredBookCoverFileName(files: readonly string[]): string | null {
|
||||
return files.find(isCustomBookCoverFileName) ?? files.find(isExtractedBookCoverFileName) ?? null;
|
||||
}
|
||||
|
||||
export function findExtractedBookCoverFileName(files: readonly string[]): string | null {
|
||||
return files.find(isExtractedBookCoverFileName) ?? null;
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export const PROJECTX_NS_PREFIX = 'projectx';
|
||||
export const PROJECTX_NS_URI = 'https://projectx.app/metadata/1.0/';
|
||||
@@ -0,0 +1,145 @@
|
||||
vi.mock('fs/promises', () => ({
|
||||
mkdir: vi.fn(),
|
||||
writeFile: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../metadata/lib/pdf-parser', () => ({
|
||||
parsePdfFile: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../metadata/lib/cover', () => ({
|
||||
extractCover: vi.fn(),
|
||||
generateThumbnail: vi.fn(),
|
||||
imageExt: vi.fn(),
|
||||
}));
|
||||
|
||||
import { mkdir, writeFile } from 'fs/promises';
|
||||
import type { MockedFunction } from 'vitest';
|
||||
|
||||
import { extractCover, generateThumbnail, imageExt } from '../metadata/lib/cover';
|
||||
import { parsePdfFile } from '../metadata/lib/pdf-parser';
|
||||
import { BookBucketMetadataService } from './book-bucket-metadata.service';
|
||||
|
||||
const mockMkdir = mkdir as MockedFunction<typeof mkdir>;
|
||||
const mockWriteFile = writeFile as MockedFunction<typeof writeFile>;
|
||||
const mockExtractCover = extractCover as MockedFunction<typeof extractCover>;
|
||||
const mockGenerateThumbnail = generateThumbnail as MockedFunction<typeof generateThumbnail>;
|
||||
const mockImageExt = imageExt as MockedFunction<typeof imageExt>;
|
||||
const mockParsePdfFile = parsePdfFile as MockedFunction<typeof parsePdfFile>;
|
||||
|
||||
describe('BookBucketMetadataService', () => {
|
||||
const repo = {
|
||||
update: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
repo.update.mockResolvedValue(undefined);
|
||||
mockMkdir.mockResolvedValue(undefined);
|
||||
mockWriteFile.mockResolvedValue(undefined);
|
||||
mockExtractCover.mockResolvedValue(null);
|
||||
mockGenerateThumbnail.mockResolvedValue(Buffer.from('thumb'));
|
||||
mockImageExt.mockReturnValue('png');
|
||||
mockParsePdfFile.mockResolvedValue(null);
|
||||
});
|
||||
|
||||
it('extractAndSave(pdf) reuses a single parse result for metadata and cover persistence', async () => {
|
||||
mockParsePdfFile.mockResolvedValue({
|
||||
title: 'PDF Title',
|
||||
subtitle: null,
|
||||
description: null,
|
||||
publisher: 'PDF Publisher',
|
||||
publishedYear: null,
|
||||
language: null,
|
||||
authors: [{ name: 'Author A', sortName: null }],
|
||||
genres: ['Genre A'],
|
||||
tags: [],
|
||||
isbn10: null,
|
||||
isbn13: null,
|
||||
seriesName: null,
|
||||
seriesIndex: null,
|
||||
rating: null,
|
||||
pageCount: 200,
|
||||
googleBooksId: null,
|
||||
goodreadsId: null,
|
||||
amazonId: null,
|
||||
hardcoverId: null,
|
||||
openLibraryId: null,
|
||||
itunesId: null,
|
||||
coverBuffer: Buffer.from('cover'),
|
||||
});
|
||||
|
||||
const service = new BookBucketMetadataService(repo as never);
|
||||
|
||||
await service.extractAndSave(5, '/tmp/book.pdf', 'pdf', '/tmp/covers');
|
||||
|
||||
expect(mockParsePdfFile).toHaveBeenCalledTimes(1);
|
||||
expect(mockParsePdfFile).toHaveBeenCalledWith(
|
||||
'/tmp/book.pdf',
|
||||
expect.objectContaining({
|
||||
extractCover: true,
|
||||
onWarning: expect.any(Function),
|
||||
}),
|
||||
);
|
||||
expect(mockExtractCover).not.toHaveBeenCalled();
|
||||
expect(repo.update).toHaveBeenNthCalledWith(1, 5, { status: 'extracting' });
|
||||
expect(repo.update).toHaveBeenNthCalledWith(2, 5, {
|
||||
embeddedMetadata: {
|
||||
title: 'PDF Title',
|
||||
publisher: 'PDF Publisher',
|
||||
pageCount: 200,
|
||||
authors: ['Author A'],
|
||||
genres: ['Genre A'],
|
||||
},
|
||||
coverPath: '/tmp/covers/5.png',
|
||||
status: 'ready',
|
||||
});
|
||||
expect(mockMkdir).toHaveBeenCalledWith('/tmp/covers', { recursive: true });
|
||||
expect(mockWriteFile).toHaveBeenCalledWith('/tmp/covers/5.png', Buffer.from('cover'));
|
||||
expect(mockWriteFile).toHaveBeenCalledWith('/tmp/covers/5_thumb.jpg', Buffer.from('thumb'));
|
||||
});
|
||||
|
||||
it('extractAndSave(pdf) leaves coverPath null when no cover bytes are available', async () => {
|
||||
mockParsePdfFile.mockResolvedValue({
|
||||
title: 'PDF Title',
|
||||
subtitle: null,
|
||||
description: null,
|
||||
publisher: null,
|
||||
publishedYear: null,
|
||||
language: null,
|
||||
authors: [],
|
||||
genres: [],
|
||||
tags: [],
|
||||
isbn10: null,
|
||||
isbn13: null,
|
||||
seriesName: null,
|
||||
seriesIndex: null,
|
||||
rating: null,
|
||||
pageCount: 12,
|
||||
googleBooksId: null,
|
||||
goodreadsId: null,
|
||||
amazonId: null,
|
||||
hardcoverId: null,
|
||||
openLibraryId: null,
|
||||
itunesId: null,
|
||||
coverBuffer: null,
|
||||
});
|
||||
|
||||
const service = new BookBucketMetadataService(repo as never);
|
||||
|
||||
await service.extractAndSave(6, '/tmp/book.pdf', 'pdf', '/tmp/covers');
|
||||
|
||||
expect(repo.update).toHaveBeenNthCalledWith(2, 6, {
|
||||
embeddedMetadata: {
|
||||
title: 'PDF Title',
|
||||
publisher: undefined,
|
||||
pageCount: 12,
|
||||
authors: undefined,
|
||||
genres: undefined,
|
||||
},
|
||||
coverPath: null,
|
||||
status: 'ready',
|
||||
});
|
||||
expect(mockWriteFile).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -7,7 +7,7 @@ import { extractEpubMetadata } from '../metadata/lib/epub';
|
||||
import { extractCbzMetadata, extractCbrMetadata, extractCb7Metadata } from '../metadata/lib/cbz-metadata';
|
||||
import { parseFb2File } from '../metadata/lib/fb2-parser';
|
||||
import { parseMobiFile } from '../metadata/lib/mobi-parser';
|
||||
import { parsePdfFile } from '../metadata/lib/pdf-parser';
|
||||
import { parsePdfFile, type PdfParsed, type PdfParseWarning } from '../metadata/lib/pdf-parser';
|
||||
import { extractCover, generateThumbnail, imageExt } from '../metadata/lib/cover';
|
||||
import { BookBucketRepository } from './book-bucket.repository';
|
||||
|
||||
@@ -18,9 +18,19 @@ export class BookBucketMetadataService {
|
||||
constructor(private readonly repo: BookBucketRepository) {}
|
||||
|
||||
async extractAndSave(fileId: number, absolutePath: string, format: string, coversDir: string): Promise<void> {
|
||||
const startedAt = Date.now();
|
||||
await this.repo.update(fileId, { status: 'extracting' });
|
||||
try {
|
||||
const [metadata, coverBytes] = await Promise.all([this.extractMetadata(absolutePath, format), extractCover(absolutePath, format)]);
|
||||
let metadata: BookBucketMetadata;
|
||||
let coverBytes: Buffer | null;
|
||||
|
||||
if (format === 'pdf') {
|
||||
const parsedPdf = await this.extractPdfData(absolutePath, true);
|
||||
metadata = this.toPdfMetadata(parsedPdf);
|
||||
coverBytes = parsedPdf?.coverBuffer ?? null;
|
||||
} else {
|
||||
[metadata, coverBytes] = await Promise.all([this.extractMetadata(absolutePath, format), extractCover(absolutePath, format)]);
|
||||
}
|
||||
|
||||
let coverPath: string | null = null;
|
||||
if (coverBytes && coverBytes.length > 0) {
|
||||
@@ -33,7 +43,11 @@ export class BookBucketMetadataService {
|
||||
status: 'ready',
|
||||
});
|
||||
} catch (err) {
|
||||
this.logger.warn(`Metadata extraction failed for Book Bucket file ${fileId}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
const errorClass = err instanceof Error ? err.name : 'Error';
|
||||
const errorMessage = (err instanceof Error ? err.message : String(err)).replace(/"/g, '\\"');
|
||||
this.logger.warn(
|
||||
`[book_bucket.extract_metadata] [fail] fileId=${fileId} format=${format} durationMs=${Date.now() - startedAt} errorClass=${errorClass} error="${errorMessage}" - metadata extraction failed`,
|
||||
);
|
||||
await this.repo.update(fileId, {
|
||||
status: 'error',
|
||||
errorMessage: err instanceof Error ? err.message : 'Metadata extraction failed',
|
||||
@@ -84,15 +98,7 @@ export class BookBucketMetadataService {
|
||||
}
|
||||
|
||||
private async fromPdf(absolutePath: string): Promise<BookBucketMetadata> {
|
||||
const parsed = await parsePdfFile(absolutePath);
|
||||
if (!parsed) return {};
|
||||
return {
|
||||
title: parsed.title ?? undefined,
|
||||
publisher: parsed.publisher ?? undefined,
|
||||
pageCount: parsed.pageCount ?? undefined,
|
||||
authors: parsed.authors.length > 0 ? parsed.authors.map((a) => a.name) : undefined,
|
||||
genres: parsed.genres.length > 0 ? parsed.genres : undefined,
|
||||
};
|
||||
return this.toPdfMetadata(await this.extractPdfData(absolutePath, false));
|
||||
}
|
||||
|
||||
private async fromMobi(absolutePath: string): Promise<BookBucketMetadata> {
|
||||
@@ -155,4 +161,34 @@ export class BookBucketMetadataService {
|
||||
|
||||
return coverPath;
|
||||
}
|
||||
|
||||
private async extractPdfData(absolutePath: string, extractCover: boolean): Promise<PdfParsed | null> {
|
||||
return parsePdfFile(absolutePath, {
|
||||
extractCover,
|
||||
onWarning: (warning) => this.logPdfParseWarning(warning),
|
||||
});
|
||||
}
|
||||
|
||||
private toPdfMetadata(parsed: PdfParsed | null): BookBucketMetadata {
|
||||
if (!parsed) return {};
|
||||
return {
|
||||
title: parsed.title ?? undefined,
|
||||
publisher: parsed.publisher ?? undefined,
|
||||
pageCount: parsed.pageCount ?? undefined,
|
||||
authors: parsed.authors.length > 0 ? parsed.authors.map((a) => a.name) : undefined,
|
||||
genres: parsed.genres.length > 0 ? parsed.genres : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
private logPdfParseWarning(warning: PdfParseWarning): void {
|
||||
if (warning.code === 'buffered-large-pdf') {
|
||||
this.logger.warn(
|
||||
`[book_bucket.pdf_parse] [end] path="${warning.absolutePath}" code=${warning.code} sizeBytes=${warning.sizeBytes ?? 0} thresholdBytes=${warning.thresholdBytes ?? 0} - large pdf buffered in memory`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.logger.warn(
|
||||
`[book_bucket.pdf_parse] [fail] path="${warning.absolutePath}" code=${warning.code} errorClass=${warning.errorClass} error="${warning.errorMessage}" - pdf parse warning emitted`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,11 +2,12 @@ import { BadRequestException, Injectable, InternalServerErrorException, Logger,
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { access, readdir, rm, stat } from 'fs/promises';
|
||||
|
||||
import { bookCoverDirPath, bookThumbnailPath, findPreferredBookCoverFileName } from '../../common/book-cover-storage';
|
||||
import { extractEpubMetadata } from '../metadata/lib/epub';
|
||||
import { extractCbzMetadata, extractCbrMetadata, extractCb7Metadata } from '../metadata/lib/cbz-metadata';
|
||||
import { parseFb2File } from '../metadata/lib/fb2-parser';
|
||||
import { parseMobiFile } from '../metadata/lib/mobi-parser';
|
||||
import { parsePdfFile } from '../metadata/lib/pdf-parser';
|
||||
import { parsePdfFile, type PdfParseWarning } from '../metadata/lib/pdf-parser';
|
||||
import { basename, extname, join } from 'path';
|
||||
|
||||
import { DEFAULT_DOWNLOAD_PATTERN, MetadataProviderKey, Permission, isAudioFormat, resolveUploadPath } from '@projectx/types';
|
||||
@@ -185,10 +186,10 @@ export class BookService {
|
||||
async getCoverPath(id: number, user: RequestUser): Promise<string | null> {
|
||||
const event = 'book.get_cover_path';
|
||||
await this.verifyBookAccess(id, user);
|
||||
const dir = join(this.booksPath, 'covers', String(id));
|
||||
const dir = bookCoverDirPath(this.booksPath, id);
|
||||
try {
|
||||
const files = await readdir(dir);
|
||||
const cover = files.find((f) => f.startsWith('cover_custom.')) ?? files.find((f) => f.startsWith('cover_extracted.'));
|
||||
const cover = findPreferredBookCoverFileName(files);
|
||||
return cover ? join(dir, cover) : null;
|
||||
} catch (err) {
|
||||
if (this.isMissingFilesystemEntry(err)) return null;
|
||||
@@ -204,7 +205,7 @@ export class BookService {
|
||||
async getThumbnailPath(id: number, user: RequestUser): Promise<string | null> {
|
||||
const event = 'book.get_thumbnail_path';
|
||||
await this.verifyBookAccess(id, user);
|
||||
const path = join(this.booksPath, 'covers', String(id), 'thumbnail.jpg');
|
||||
const path = bookThumbnailPath(this.booksPath, id);
|
||||
try {
|
||||
await access(path);
|
||||
return path;
|
||||
@@ -1117,7 +1118,10 @@ export class BookService {
|
||||
};
|
||||
}
|
||||
case 'pdf': {
|
||||
const parsed = await parsePdfFile(absolutePath);
|
||||
const parsed = await parsePdfFile(absolutePath, {
|
||||
extractCover: false,
|
||||
onWarning: (warning) => this.logPdfFileMetadataWarning(warning),
|
||||
});
|
||||
if (!parsed) return {};
|
||||
return {
|
||||
title: parsed.title,
|
||||
@@ -1186,6 +1190,18 @@ export class BookService {
|
||||
}
|
||||
}
|
||||
|
||||
private logPdfFileMetadataWarning(warning: PdfParseWarning): void {
|
||||
if (warning.code === 'buffered-large-pdf') {
|
||||
this.logger.warn(
|
||||
`[book.file_metadata_pdf] [end] path="${warning.absolutePath}" code=${warning.code} sizeBytes=${warning.sizeBytes ?? 0} thresholdBytes=${warning.thresholdBytes ?? 0} - large pdf buffered in memory`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.logger.warn(
|
||||
`[book.file_metadata_pdf] [fail] path="${warning.absolutePath}" code=${warning.code} errorClass=${warning.errorClass} error="${warning.errorMessage}" - pdf file metadata warning emitted`,
|
||||
);
|
||||
}
|
||||
|
||||
private resolveChapters(
|
||||
stored: AudiobookChapter[] | null | undefined,
|
||||
fileRows: { absolutePath: string; format: string | null; durationSeconds: number | null }[],
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
export { COVER_CUSTOM_FILE_PREFIX, COVER_EXTRACTED_FILE_PREFIX, COVER_THUMBNAIL_FILE_NAME } from '../../common/book-cover-storage';
|
||||
|
||||
export const COVER_PROVIDERS = Symbol('COVER_PROVIDERS');
|
||||
|
||||
export const COVER_PROXY_USER_AGENT =
|
||||
@@ -6,7 +8,3 @@ export const COVER_PROXY_USER_AGENT =
|
||||
export const COVER_PROXY_TIMEOUT_MS = 10_000;
|
||||
export const COVER_PROXY_MAX_IMAGE_BYTES = 20 * 1024 * 1024;
|
||||
export const COVER_PROXY_MAX_REDIRECTS = 5;
|
||||
|
||||
export const COVER_CUSTOM_FILE_PREFIX = 'cover_custom.';
|
||||
export const COVER_EXTRACTED_FILE_PREFIX = 'cover_extracted.';
|
||||
export const COVER_THUMBNAIL_FILE_NAME = 'thumbnail.jpg';
|
||||
|
||||
@@ -7,6 +7,7 @@ import { mkdir, readdir, readFile, unlink, writeFile } from 'fs/promises';
|
||||
import { isIP } from 'net';
|
||||
import { join } from 'path';
|
||||
|
||||
import { bookCoverDirPath, bookThumbnailPath, findExtractedBookCoverFileName } from '../../common/book-cover-storage';
|
||||
import type { RequestUser } from '../../common/types/request-user';
|
||||
import { DB } from '../../db';
|
||||
import * as schema from '../../db/schema';
|
||||
@@ -14,15 +15,13 @@ import { bookMetadata } from '../../db/schema';
|
||||
import { BookReadService } from '../book/book-read.service';
|
||||
import { BookMetadataLockService } from '../book-metadata-lock/book-metadata-lock.service';
|
||||
import { LibraryService } from '../library/library.service';
|
||||
import { coverDirPath, generateThumbnail, imageExt } from '../metadata/lib/cover';
|
||||
import { generateThumbnail, imageExt } from '../metadata/lib/cover';
|
||||
import {
|
||||
COVER_CUSTOM_FILE_PREFIX,
|
||||
COVER_EXTRACTED_FILE_PREFIX,
|
||||
COVER_PROXY_MAX_IMAGE_BYTES,
|
||||
COVER_PROXY_MAX_REDIRECTS,
|
||||
COVER_PROXY_TIMEOUT_MS,
|
||||
COVER_PROXY_USER_AGENT,
|
||||
COVER_THUMBNAIL_FILE_NAME,
|
||||
} from './constants';
|
||||
import { CoverProviderRegistry } from './provider-registry';
|
||||
import {
|
||||
@@ -153,12 +152,12 @@ export class CoverService {
|
||||
try {
|
||||
await this.verifyAccess(bookId, user);
|
||||
await this.bookMetadataLockService.assertFieldsUnlocked(bookId, ['cover']);
|
||||
const dir = coverDirPath(this.booksPath, bookId);
|
||||
const dir = bookCoverDirPath(this.booksPath, bookId);
|
||||
await this.deleteFilesByPrefix(dir, COVER_CUSTOM_FILE_PREFIX);
|
||||
|
||||
const extractedPath = await this.findExtractedCover(bookId);
|
||||
if (!extractedPath) {
|
||||
await this.removeFileIfPresent(join(dir, COVER_THUMBNAIL_FILE_NAME));
|
||||
await this.removeFileIfPresent(bookThumbnailPath(this.booksPath, bookId));
|
||||
await this.setCoverSource(bookId, null);
|
||||
this.logger.log(
|
||||
`[cover.delete] [end] bookId=${bookId} userId=${user.id} durationMs=${Date.now() - startedAt} coverSource=null - cover deletion completed`,
|
||||
@@ -168,7 +167,7 @@ export class CoverService {
|
||||
|
||||
const bytes = await readFile(extractedPath);
|
||||
const thumb = await generateThumbnail(bytes);
|
||||
await writeFile(join(dir, COVER_THUMBNAIL_FILE_NAME), thumb);
|
||||
await writeFile(bookThumbnailPath(this.booksPath, bookId), thumb);
|
||||
await this.setCoverSource(bookId, 'extracted');
|
||||
this.logger.log(
|
||||
`[cover.delete] [end] bookId=${bookId} userId=${user.id} durationMs=${Date.now() - startedAt} coverSource=extracted - cover deletion completed`,
|
||||
@@ -231,20 +230,20 @@ export class CoverService {
|
||||
}
|
||||
|
||||
private async saveCustomCover(bookId: number, buffer: Buffer): Promise<void> {
|
||||
const dir = coverDirPath(this.booksPath, bookId);
|
||||
const dir = bookCoverDirPath(this.booksPath, bookId);
|
||||
await mkdir(dir, { recursive: true });
|
||||
await this.deleteFilesByPrefix(dir, COVER_CUSTOM_FILE_PREFIX);
|
||||
|
||||
const ext = imageExt(buffer);
|
||||
await writeFile(join(dir, `${COVER_CUSTOM_FILE_PREFIX}${ext}`), buffer);
|
||||
const thumb = await generateThumbnail(buffer);
|
||||
await writeFile(join(dir, COVER_THUMBNAIL_FILE_NAME), thumb);
|
||||
await writeFile(bookThumbnailPath(this.booksPath, bookId), thumb);
|
||||
}
|
||||
|
||||
private async findExtractedCover(bookId: number): Promise<string | null> {
|
||||
const dir = coverDirPath(this.booksPath, bookId);
|
||||
const dir = bookCoverDirPath(this.booksPath, bookId);
|
||||
const files = await this.readDirIfExists(dir);
|
||||
const found = files.find((fileName) => fileName.startsWith(COVER_EXTRACTED_FILE_PREFIX));
|
||||
const found = findExtractedBookCoverFileName(files);
|
||||
return found ? join(dir, found) : null;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import { readdir, readFile } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
|
||||
import type { WriteResult } from '@projectx/types';
|
||||
import { bookCoverDirPath, findPreferredBookCoverFileName } from '../../common/book-cover-storage';
|
||||
import { FORMAT_CB7, FORMAT_CBZ, FORMAT_EPUB, FORMAT_PDF, createBookWriteFieldMask } from './file-write.constants';
|
||||
import { FileLockService } from './file-lock.service';
|
||||
import { FileWriteRepository } from './file-write.repository';
|
||||
@@ -201,10 +202,10 @@ export class FileWriteService implements OnModuleDestroy {
|
||||
|
||||
private async loadCoverBytes(bookId: number): Promise<Buffer | null> {
|
||||
const startedAt = Date.now();
|
||||
const dir = join(this.booksPath, 'covers', String(bookId));
|
||||
const dir = bookCoverDirPath(this.booksPath, bookId);
|
||||
try {
|
||||
const files = await readdir(dir);
|
||||
const cover = files.find((f) => f.startsWith('cover_custom.')) ?? files.find((f) => f.startsWith('cover_extracted.'));
|
||||
const cover = findPreferredBookCoverFileName(files);
|
||||
if (!cover) return null;
|
||||
return readFile(join(dir, cover));
|
||||
} catch (error) {
|
||||
|
||||
@@ -115,7 +115,6 @@ describe('PdfFormatWriter', () => {
|
||||
expect(pdfDoc.setTitle).toHaveBeenCalledWith('Dune');
|
||||
expect(pdfDoc.setAuthor).toHaveBeenCalledWith('Frank Herbert');
|
||||
expect(pdfDoc.setSubject).toHaveBeenCalledWith('Sci-fi classic');
|
||||
expect(pdfDoc.setProducer).toHaveBeenCalledWith('Ace');
|
||||
expect(pdfDoc.setCreationDate).toHaveBeenCalledWith(new Date(1965, 0, 1));
|
||||
expect(pdfDoc.setCreator).toHaveBeenCalledWith('projectx');
|
||||
expect(pdfDoc.setKeywords).toHaveBeenCalledWith(['Sci-Fi', 'Classic']);
|
||||
|
||||
@@ -52,9 +52,6 @@ function applyInfoDict(pdfDoc: PDFDocument, payload: BookWritePayload, fieldMask
|
||||
if (fieldMask.has('description') && payload.description != null) {
|
||||
pdfDoc.setSubject(payload.description);
|
||||
}
|
||||
if (fieldMask.has('publisher') && payload.publisher != null) {
|
||||
pdfDoc.setProducer(payload.publisher);
|
||||
}
|
||||
if (fieldMask.has('publishedYear') && payload.publishedYear != null) {
|
||||
pdfDoc.setCreationDate(new Date(payload.publishedYear, 0, 1));
|
||||
}
|
||||
|
||||
@@ -1,2 +1 @@
|
||||
export const PROJECTX_NS_PREFIX = 'projectx';
|
||||
export const PROJECTX_NS_URI = 'https://projectx.app/metadata/1.0/';
|
||||
export { PROJECTX_NS_PREFIX, PROJECTX_NS_URI } from '../../../../common/projectx-ns';
|
||||
|
||||
@@ -4,6 +4,7 @@ import { readdir, stat } from 'fs/promises';
|
||||
import { createReadStream } from 'fs';
|
||||
import { join } from 'path';
|
||||
import type { FastifyReply } from 'fastify';
|
||||
import { bookCoverDirPath, bookThumbnailPath, findPreferredBookCoverFileName } from '../../../common/book-cover-storage';
|
||||
import { KoboBookAccessService } from './kobo-book-access.service';
|
||||
|
||||
@Injectable()
|
||||
@@ -20,7 +21,7 @@ export class KoboThumbnailService {
|
||||
async serveThumbnail(userId: number, bookId: number, ifNoneMatch: string | undefined, reply: FastifyReply) {
|
||||
await this.bookAccessService.assertBookAccessible(userId, bookId);
|
||||
|
||||
const thumbnailPath = join(this.booksPath, 'covers', String(bookId), 'thumbnail.jpg');
|
||||
const thumbnailPath = bookThumbnailPath(this.booksPath, bookId);
|
||||
try {
|
||||
const { mtimeMs } = await stat(thumbnailPath);
|
||||
const etag = `"${Math.floor(mtimeMs)}"`;
|
||||
@@ -38,10 +39,10 @@ export class KoboThumbnailService {
|
||||
}
|
||||
|
||||
async serveCover(bookId: number, ifNoneMatch: string | undefined, reply: FastifyReply) {
|
||||
const dir = join(this.booksPath, 'covers', String(bookId));
|
||||
const dir = bookCoverDirPath(this.booksPath, bookId);
|
||||
try {
|
||||
const files = await readdir(dir);
|
||||
const cover = files.find((f) => f.startsWith('cover.'));
|
||||
const cover = findPreferredBookCoverFileName(files);
|
||||
if (!cover) throw new NotFoundException('No cover');
|
||||
const coverPath = join(dir, cover);
|
||||
const { mtimeMs } = await stat(coverPath);
|
||||
|
||||
@@ -13,6 +13,14 @@ export interface ParsedBookData {
|
||||
seriesIndex?: number | null;
|
||||
authors: { name: string; sortName: string | null }[];
|
||||
genres: string[];
|
||||
tags?: string[];
|
||||
rating?: number | null;
|
||||
googleBooksId?: string | null;
|
||||
goodreadsId?: string | null;
|
||||
amazonId?: string | null;
|
||||
hardcoverId?: string | null;
|
||||
openLibraryId?: string | null;
|
||||
itunesId?: string | null;
|
||||
cover: Buffer | null;
|
||||
// audio-specific
|
||||
narrators?: string[];
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
import { parseBookFilename } from '../lib/filename-parser';
|
||||
import { parsePdfFile } from '../lib/pdf-parser';
|
||||
import { parsePdfFile, type PdfParseOptions } from '../lib/pdf-parser';
|
||||
import type { FormatExtractor, ParsedBookData } from './format-extractor.interface';
|
||||
|
||||
type PdfFormatExtractorOptions = Pick<PdfParseOptions, 'extractCover' | 'onWarning'>;
|
||||
|
||||
export class PdfFormatExtractor implements FormatExtractor {
|
||||
constructor(private readonly options: PdfFormatExtractorOptions = {}) {}
|
||||
|
||||
async extract(absolutePath: string): Promise<ParsedBookData | null> {
|
||||
const pdf = await parsePdfFile(absolutePath);
|
||||
const pdf = await parsePdfFile(absolutePath, {
|
||||
extractCover: this.options.extractCover === true,
|
||||
onWarning: this.options.onWarning,
|
||||
});
|
||||
if (!pdf) return null;
|
||||
|
||||
const fb = !pdf.title ? parseBookFilename(absolutePath) : null;
|
||||
@@ -21,6 +28,14 @@ export class PdfFormatExtractor implements FormatExtractor {
|
||||
seriesIndex: pdf.seriesIndex,
|
||||
authors: pdf.authors,
|
||||
genres: pdf.genres,
|
||||
tags: pdf.tags,
|
||||
rating: pdf.rating,
|
||||
googleBooksId: pdf.googleBooksId,
|
||||
goodreadsId: pdf.goodreadsId,
|
||||
amazonId: pdf.amazonId,
|
||||
hardcoverId: pdf.hardcoverId,
|
||||
openLibraryId: pdf.openLibraryId,
|
||||
itunesId: pdf.itunesId,
|
||||
cover: pdf.coverBuffer,
|
||||
pageCount: pdf.pageCount,
|
||||
};
|
||||
|
||||
@@ -4,6 +4,7 @@ vi.mock('./cover-cbr', () => ({ extractCbrCover: vi.fn() }));
|
||||
vi.mock('./cover-cb7', () => ({ extractCb7Cover: vi.fn() }));
|
||||
vi.mock('./cover-fb2', () => ({ extractFb2Cover: vi.fn() }));
|
||||
vi.mock('./mobi-parser', () => ({ extractMobiCover: vi.fn() }));
|
||||
vi.mock('./pdf-cover', () => ({ extractPdfCover: vi.fn() }));
|
||||
vi.mock('sharp', () => {
|
||||
const chain = {
|
||||
resize: vi.fn(),
|
||||
@@ -24,6 +25,7 @@ import { extractCbzCover } from './cover-cbz';
|
||||
import { extractEpubCover } from './cover-epub';
|
||||
import { extractFb2Cover } from './cover-fb2';
|
||||
import { extractMobiCover } from './mobi-parser';
|
||||
import { extractPdfCover } from './pdf-cover';
|
||||
import { coverDirPath, extractCover, generateThumbnail, imageExt } from './cover';
|
||||
const mockEpub = extractEpubCover as MockedFunction<typeof extractEpubCover>;
|
||||
const mockMobi = extractMobiCover as MockedFunction<typeof extractMobiCover>;
|
||||
@@ -31,6 +33,7 @@ const mockCbz = extractCbzCover as MockedFunction<typeof extractCbzCover>;
|
||||
const mockCbr = extractCbrCover as MockedFunction<typeof extractCbrCover>;
|
||||
const mockCb7 = extractCb7Cover as MockedFunction<typeof extractCb7Cover>;
|
||||
const mockFb2 = extractFb2Cover as MockedFunction<typeof extractFb2Cover>;
|
||||
const mockPdf = extractPdfCover as MockedFunction<typeof extractPdfCover>;
|
||||
|
||||
const JPEG_BYTES = Buffer.from([0xff, 0xd8, 0xff, 0xe0]);
|
||||
|
||||
@@ -43,6 +46,7 @@ beforeEach(() => {
|
||||
mockCbr.mockResolvedValue(null);
|
||||
mockCb7.mockResolvedValue(null);
|
||||
mockFb2.mockResolvedValue(null);
|
||||
mockPdf.mockResolvedValue(null);
|
||||
});
|
||||
|
||||
// ── imageExt ──────────────────────────────────────────────────────────────────
|
||||
@@ -150,8 +154,14 @@ describe('extractCover', () => {
|
||||
expect(mockFb2).toHaveBeenCalledWith('/book.fb2');
|
||||
});
|
||||
|
||||
it('routes pdf to extractPdfCover', async () => {
|
||||
mockPdf.mockResolvedValue(JPEG_BYTES);
|
||||
await extractCover('/book.pdf', 'pdf');
|
||||
expect(mockPdf).toHaveBeenCalledWith('/book.pdf');
|
||||
});
|
||||
|
||||
it('returns null for unsupported format', async () => {
|
||||
expect(await extractCover('/book.pdf', 'pdf')).toBeNull();
|
||||
expect(await extractCover('/book.txt', 'txt')).toBeNull();
|
||||
});
|
||||
|
||||
it('format matching is case-insensitive', async () => {
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import { join } from 'path';
|
||||
import sharp from 'sharp';
|
||||
|
||||
import { bookCoverDirPath } from '../../../common/book-cover-storage';
|
||||
import { extractCb7Cover } from './cover-cb7';
|
||||
import { extractCbrCover } from './cover-cbr';
|
||||
import { extractCbzCover } from './cover-cbz';
|
||||
import { extractEpubCover } from './cover-epub';
|
||||
import { extractFb2Cover } from './cover-fb2';
|
||||
import { extractMobiCover } from './mobi-parser';
|
||||
import { extractPdfCover } from './pdf-cover';
|
||||
|
||||
const THUMBNAIL_WIDTH = 400;
|
||||
const THUMBNAIL_HEIGHT = 600;
|
||||
@@ -41,6 +42,8 @@ export async function extractCover(absolutePath: string, format: string): Promis
|
||||
return extractCb7Cover(absolutePath);
|
||||
case 'fb2':
|
||||
return extractFb2Cover(absolutePath);
|
||||
case 'pdf':
|
||||
return extractPdfCover(absolutePath).catch(() => null);
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
@@ -48,5 +51,5 @@ export async function extractCover(absolutePath: string, format: string): Promis
|
||||
|
||||
/** Resolve the cover directory for a book on disk. */
|
||||
export function coverDirPath(booksPath: string, bookId: number): string {
|
||||
return join(booksPath, 'covers', String(bookId));
|
||||
return bookCoverDirPath(booksPath, bookId);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import { execFile } from 'child_process';
|
||||
import { mkdtemp, readFile, rm } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { promisify } from 'util';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export async function extractPdfCover(absolutePath: string): Promise<Buffer | null> {
|
||||
const tmpDir = await mkdtemp(join(tmpdir(), 'pdf-cover-'));
|
||||
const outPrefix = join(tmpDir, 'cover');
|
||||
|
||||
try {
|
||||
await execFileAsync('pdftoppm', ['-jpeg', '-singlefile', '-r', '150', '-f', '1', '-l', '1', absolutePath, outPrefix]);
|
||||
return await readFile(`${outPrefix}.jpg`);
|
||||
} finally {
|
||||
await rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,7 @@
|
||||
vi.mock('fs/promises', () => ({
|
||||
mkdtemp: vi.fn(),
|
||||
readFile: vi.fn(),
|
||||
readdir: vi.fn(),
|
||||
rm: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('child_process', () => ({ execFile: vi.fn() }));
|
||||
|
||||
vi.mock('pdf-lib', () => ({
|
||||
PDFDocument: {
|
||||
load: vi.fn(),
|
||||
@@ -18,51 +13,49 @@ vi.mock('./pdf-xmp-reader', () => ({
|
||||
parseXmp: vi.fn(),
|
||||
}));
|
||||
|
||||
import { execFile } from 'child_process';
|
||||
import { mkdtemp, readFile, readdir, rm } from 'fs/promises';
|
||||
vi.mock('./pdf-cover', () => ({
|
||||
extractPdfCover: vi.fn(),
|
||||
}));
|
||||
|
||||
import { readFile } from 'fs/promises';
|
||||
import type { MockedFunction } from 'vitest';
|
||||
import { PDFDocument } from 'pdf-lib';
|
||||
|
||||
import { extractPdfCover } from './pdf-cover';
|
||||
import { parsePdfFile, PDF_BUFFER_WARNING_BYTES } from './pdf-parser';
|
||||
import { extractXmpXml, parseXmp } from './pdf-xmp-reader';
|
||||
import { parsePdfFile } from './pdf-parser';
|
||||
|
||||
const mockExecFile = execFile as unknown as vi.Mock;
|
||||
const mockMkdtemp = mkdtemp as MockedFunction<typeof mkdtemp>;
|
||||
const mockReadFile = readFile as MockedFunction<typeof readFile>;
|
||||
const mockReaddir = readdir as MockedFunction<typeof readdir>;
|
||||
const mockRm = rm as MockedFunction<typeof rm>;
|
||||
const mockPdfLoad = PDFDocument.load as vi.Mock;
|
||||
const mockPdfLoad = PDFDocument.load as MockedFunction<typeof PDFDocument.load>;
|
||||
const mockExtractXmpXml = extractXmpXml as MockedFunction<typeof extractXmpXml>;
|
||||
const mockParseXmp = parseXmp as MockedFunction<typeof parseXmp>;
|
||||
const mockExtractPdfCover = extractPdfCover as MockedFunction<typeof extractPdfCover>;
|
||||
|
||||
function mockExecSuccess() {
|
||||
mockExecFile.mockImplementation((_file: string, _args: string[], cb: (err: Error | null, stdout?: string, stderr?: string) => void) => {
|
||||
cb(null, '', '');
|
||||
});
|
||||
function makePdfDoc(overrides: Partial<Record<string, unknown>> = {}) {
|
||||
return {
|
||||
getTitle: () => 'Info Title',
|
||||
getAuthor: () => 'Author One; Author Two',
|
||||
getCreator: () => null,
|
||||
getProducer: () => null,
|
||||
getSubject: () => 'Info Subject',
|
||||
getKeywords: () => 'tag1, tag2',
|
||||
getPageCount: () => 123,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('parsePdfFile', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
|
||||
mockMkdtemp.mockResolvedValue('/tmp/pdf-cover-1');
|
||||
mockReaddir.mockResolvedValue(['cover-1.jpg'] as unknown as Awaited<ReturnType<typeof readdir>>);
|
||||
mockRm.mockResolvedValue(undefined);
|
||||
mockReadFile
|
||||
.mockResolvedValueOnce(Buffer.from('%PDF-1.7') as unknown as Awaited<ReturnType<typeof readFile>>)
|
||||
.mockResolvedValueOnce(Buffer.from([0xff, 0xd8, 0xff]) as unknown as Awaited<ReturnType<typeof readFile>>);
|
||||
|
||||
mockPdfLoad.mockResolvedValue({
|
||||
getTitle: () => 'Info Title',
|
||||
getAuthor: () => 'Author One; Author Two',
|
||||
getSubject: () => 'Info Subject',
|
||||
getKeywords: () => 'tag1, tag2',
|
||||
getPageCount: () => 123,
|
||||
});
|
||||
|
||||
mockExecSuccess();
|
||||
mockReadFile.mockResolvedValue(Buffer.from('%PDF-1.7') as never);
|
||||
mockPdfLoad.mockResolvedValue(makePdfDoc() as never);
|
||||
mockExtractXmpXml.mockReturnValue(null);
|
||||
mockParseXmp.mockReturnValue(null);
|
||||
mockExtractPdfCover.mockResolvedValue(Buffer.from([0xff, 0xd8, 0xff]));
|
||||
});
|
||||
|
||||
it('prefers XMP metadata when available and still includes extracted cover/pageCount', async () => {
|
||||
it('prefers XMP metadata and includes extracted cover when requested', async () => {
|
||||
mockExtractXmpXml.mockReturnValue('<xmp/>');
|
||||
mockParseXmp.mockReturnValue({
|
||||
title: 'XMP Title',
|
||||
@@ -85,9 +78,10 @@ describe('parsePdfFile', () => {
|
||||
amazonId: 'a1',
|
||||
hardcoverId: 'h1',
|
||||
openLibraryId: 'ol1',
|
||||
itunesId: 'it1',
|
||||
});
|
||||
|
||||
const parsed = await parsePdfFile('/books/book.pdf');
|
||||
const parsed = await parsePdfFile('/books/book.pdf', { extractCover: true });
|
||||
|
||||
expect(parsed).toEqual(
|
||||
expect.objectContaining({
|
||||
@@ -98,11 +92,10 @@ describe('parsePdfFile', () => {
|
||||
coverBuffer: Buffer.from([0xff, 0xd8, 0xff]),
|
||||
}),
|
||||
);
|
||||
expect(mockExtractPdfCover).toHaveBeenCalledWith('/books/book.pdf');
|
||||
});
|
||||
|
||||
it('falls back to Info dictionary fields when no XMP is present', async () => {
|
||||
mockExtractXmpXml.mockReturnValue(null);
|
||||
|
||||
const parsed = await parsePdfFile('/books/book.pdf');
|
||||
|
||||
expect(parsed).toEqual(
|
||||
@@ -114,33 +107,111 @@ describe('parsePdfFile', () => {
|
||||
],
|
||||
description: 'Info Subject',
|
||||
tags: ['tag1', 'tag2'],
|
||||
coverBuffer: null,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('can skip cover extraction when only metadata is needed', async () => {
|
||||
mockExtractXmpXml.mockReturnValue(null);
|
||||
|
||||
const parsed = await parsePdfFile('/books/book.pdf', { extractCover: false });
|
||||
|
||||
expect(parsed).toEqual(expect.objectContaining({ title: 'Info Title', coverBuffer: null }));
|
||||
expect(mockExecFile).not.toHaveBeenCalled();
|
||||
expect(mockReadFile).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('returns null coverBuffer when pdftoppm command fails, but still returns metadata', async () => {
|
||||
mockExtractXmpXml.mockReturnValue(null);
|
||||
mockExecFile.mockImplementation((_file: string, _args: string[], cb: (err: Error | null) => void) => cb(new Error('pdftoppm missing')));
|
||||
|
||||
it('skips cover extraction by default for metadata-only callers', async () => {
|
||||
const parsed = await parsePdfFile('/books/book.pdf');
|
||||
|
||||
expect(parsed).toEqual(expect.objectContaining({ title: 'Info Title', coverBuffer: null }));
|
||||
expect(mockRm).toHaveBeenCalledWith('/tmp/pdf-cover-1', { recursive: true, force: true });
|
||||
expect(mockExtractPdfCover).not.toHaveBeenCalled();
|
||||
expect(mockReadFile).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('returns null when PDF cannot be parsed', async () => {
|
||||
mockPdfLoad.mockRejectedValue(new Error('invalid pdf'));
|
||||
it('returns null coverBuffer and emits a warning when cover extraction fails', async () => {
|
||||
mockExtractPdfCover.mockRejectedValue(new Error('pdftoppm missing'));
|
||||
const onWarning = vi.fn();
|
||||
|
||||
await expect(parsePdfFile('/books/bad.pdf')).resolves.toBeNull();
|
||||
const parsed = await parsePdfFile('/books/book.pdf', {
|
||||
extractCover: true,
|
||||
onWarning,
|
||||
});
|
||||
|
||||
expect(parsed).toEqual(expect.objectContaining({ title: 'Info Title', coverBuffer: null }));
|
||||
expect(onWarning).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
code: 'cover-extraction-failed',
|
||||
absolutePath: '/books/book.pdf',
|
||||
errorClass: 'Error',
|
||||
errorMessage: 'pdftoppm missing',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('emits a warning when a large PDF must be buffered in memory', async () => {
|
||||
mockReadFile.mockResolvedValue(Buffer.alloc(PDF_BUFFER_WARNING_BYTES) as never);
|
||||
const onWarning = vi.fn();
|
||||
|
||||
await parsePdfFile('/books/large.pdf', { onWarning });
|
||||
|
||||
expect(onWarning).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
code: 'buffered-large-pdf',
|
||||
absolutePath: '/books/large.pdf',
|
||||
sizeBytes: PDF_BUFFER_WARNING_BYTES,
|
||||
thresholdBytes: PDF_BUFFER_WARNING_BYTES,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to the native PDF page count when XMP omits projectx:pageCount', async () => {
|
||||
mockExtractXmpXml.mockReturnValue('<xmp/>');
|
||||
mockParseXmp.mockReturnValue({
|
||||
title: 'XMP Title',
|
||||
subtitle: null,
|
||||
description: null,
|
||||
publisher: null,
|
||||
publishedYear: null,
|
||||
language: null,
|
||||
authors: [],
|
||||
genres: [],
|
||||
tags: [],
|
||||
isbn10: null,
|
||||
isbn13: null,
|
||||
seriesName: null,
|
||||
seriesIndex: null,
|
||||
rating: null,
|
||||
pageCount: null,
|
||||
googleBooksId: null,
|
||||
goodreadsId: null,
|
||||
amazonId: null,
|
||||
hardcoverId: null,
|
||||
openLibraryId: null,
|
||||
itunesId: null,
|
||||
});
|
||||
|
||||
const parsed = await parsePdfFile('/books/book.pdf');
|
||||
|
||||
expect(parsed?.pageCount).toBe(123);
|
||||
});
|
||||
|
||||
it('falls back to legacy ProjectX producer metadata when XMP publisher is absent', async () => {
|
||||
mockPdfLoad.mockResolvedValue(
|
||||
makePdfDoc({
|
||||
getCreator: () => 'projectx',
|
||||
getProducer: () => 'Legacy Publisher',
|
||||
}) as never,
|
||||
);
|
||||
|
||||
const parsed = await parsePdfFile('/books/book.pdf');
|
||||
|
||||
expect(parsed?.publisher).toBe('Legacy Publisher');
|
||||
});
|
||||
|
||||
it('returns null and emits a parse warning when the PDF cannot be read', async () => {
|
||||
mockPdfLoad.mockRejectedValue(new Error('invalid pdf'));
|
||||
const onWarning = vi.fn();
|
||||
|
||||
await expect(parsePdfFile('/books/bad.pdf', { onWarning })).resolves.toBeNull();
|
||||
expect(onWarning).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
code: 'parse-failed',
|
||||
absolutePath: '/books/bad.pdf',
|
||||
errorClass: 'Error',
|
||||
errorMessage: 'invalid pdf',
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,14 +1,10 @@
|
||||
import { execFile } from 'child_process';
|
||||
import { mkdtemp, readFile, readdir, rm } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { promisify } from 'util';
|
||||
import { readFile } from 'fs/promises';
|
||||
import { PDFDocument } from 'pdf-lib';
|
||||
|
||||
import { PROJECTX_NS_PREFIX } from '../../../common/projectx-ns';
|
||||
import { extractPdfCover } from './pdf-cover';
|
||||
import { extractXmpXml, parseXmp, type XmpParsed } from './pdf-xmp-reader';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export interface PdfParsed {
|
||||
title: string | null;
|
||||
subtitle: string | null;
|
||||
@@ -34,10 +30,22 @@ export interface PdfParsed {
|
||||
coverBuffer: Buffer | null;
|
||||
}
|
||||
|
||||
interface PdfParseOptions {
|
||||
extractCover?: boolean;
|
||||
export interface PdfParseWarning {
|
||||
code: 'buffered-large-pdf' | 'cover-extraction-failed' | 'parse-failed';
|
||||
absolutePath: string;
|
||||
errorClass?: string;
|
||||
errorMessage?: string;
|
||||
sizeBytes?: number;
|
||||
thresholdBytes?: number;
|
||||
}
|
||||
|
||||
export interface PdfParseOptions {
|
||||
extractCover?: boolean;
|
||||
onWarning?: (warning: PdfParseWarning) => void;
|
||||
}
|
||||
|
||||
export const PDF_BUFFER_WARNING_BYTES = 25 * 1024 * 1024;
|
||||
|
||||
function clean(value: string | undefined): string | null {
|
||||
if (!value) return null;
|
||||
const s = value.trim();
|
||||
@@ -52,25 +60,27 @@ function splitCommaList(value: string | null): string[] {
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
async function extractPdfCover(absolutePath: string): Promise<Buffer | null> {
|
||||
const tmpDir = await mkdtemp(join(tmpdir(), 'pdf-cover-'));
|
||||
try {
|
||||
const outPrefix = join(tmpDir, 'cover');
|
||||
await execFileAsync('pdftoppm', ['-jpeg', '-r', '150', '-f', '1', '-l', '1', absolutePath, outPrefix]);
|
||||
const files = await readdir(tmpDir);
|
||||
const coverFile = files.find((f) => f.endsWith('.jpg'));
|
||||
if (!coverFile) return null;
|
||||
return await readFile(join(tmpDir, coverFile));
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
await rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
function createWarning(code: PdfParseWarning['code'], absolutePath: string, error: unknown): PdfParseWarning {
|
||||
const errorClass = error instanceof Error ? error.name : 'Error';
|
||||
const errorMessage = (error instanceof Error ? error.message : String(error)).replace(/"/g, '\\"');
|
||||
return { code, absolutePath, errorClass, errorMessage };
|
||||
}
|
||||
|
||||
function createLargeBufferWarning(absolutePath: string, sizeBytes: number): PdfParseWarning {
|
||||
return {
|
||||
code: 'buffered-large-pdf',
|
||||
absolutePath,
|
||||
sizeBytes,
|
||||
thresholdBytes: PDF_BUFFER_WARNING_BYTES,
|
||||
};
|
||||
}
|
||||
|
||||
export async function parsePdfFile(absolutePath: string, options: PdfParseOptions = {}): Promise<PdfParsed | null> {
|
||||
try {
|
||||
const buf = await readFile(absolutePath);
|
||||
if (buf.length >= PDF_BUFFER_WARNING_BYTES) {
|
||||
options.onWarning?.(createLargeBufferWarning(absolutePath, buf.length));
|
||||
}
|
||||
const doc = await PDFDocument.load(buf, { ignoreEncryption: true });
|
||||
|
||||
// XMP is the authoritative source — richer and semantically correct.
|
||||
@@ -81,6 +91,8 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
|
||||
|
||||
const infoTitle = clean(doc.getTitle());
|
||||
const infoAuthorRaw = clean(doc.getAuthor());
|
||||
const infoCreator = clean(doc.getCreator());
|
||||
const infoProducer = clean(doc.getProducer());
|
||||
const infoSubject = clean(doc.getSubject());
|
||||
const infoKeywords = splitCommaList(clean(doc.getKeywords()));
|
||||
|
||||
@@ -92,14 +104,21 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
|
||||
.map((name) => ({ name, sortName: null }))
|
||||
: [];
|
||||
|
||||
const coverBuffer = options.extractCover === false ? null : await extractPdfCover(absolutePath);
|
||||
let coverBuffer: Buffer | null = null;
|
||||
if (options.extractCover === true) {
|
||||
try {
|
||||
coverBuffer = await extractPdfCover(absolutePath);
|
||||
} catch (error) {
|
||||
options.onWarning?.(createWarning('cover-extraction-failed', absolutePath, error));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title: hasXmp ? xmp.title : infoTitle,
|
||||
subtitle: xmp?.subtitle ?? null,
|
||||
authors: hasXmp ? xmp.authors : infoAuthors,
|
||||
description: hasXmp ? xmp.description : infoSubject,
|
||||
publisher: xmp?.publisher ?? null,
|
||||
publisher: xmp?.publisher ?? (infoCreator === PROJECTX_NS_PREFIX ? infoProducer : null),
|
||||
publishedYear: xmp?.publishedYear ?? null,
|
||||
language: xmp?.language ?? null,
|
||||
genres: xmp?.genres?.length ? xmp.genres : [],
|
||||
@@ -110,7 +129,7 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
|
||||
seriesName: xmp?.seriesName ?? null,
|
||||
seriesIndex: xmp?.seriesIndex ?? null,
|
||||
rating: xmp?.rating ?? null,
|
||||
pageCount: hasXmp ? xmp.pageCount : doc.getPageCount(),
|
||||
pageCount: xmp?.pageCount ?? doc.getPageCount(),
|
||||
googleBooksId: xmp?.googleBooksId ?? null,
|
||||
goodreadsId: xmp?.goodreadsId ?? null,
|
||||
amazonId: xmp?.amazonId ?? null,
|
||||
@@ -119,7 +138,8 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
|
||||
itunesId: xmp?.itunesId ?? null,
|
||||
coverBuffer,
|
||||
};
|
||||
} catch {
|
||||
} catch (error) {
|
||||
options.onWarning?.(createWarning('parse-failed', absolutePath, error));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { PROJECTX_NS_PREFIX, PROJECTX_NS_URI } from '../../../common/projectx-ns';
|
||||
import { parseXmp } from './pdf-xmp-reader';
|
||||
|
||||
// Wraps XMP content in standard RDF envelope
|
||||
@@ -7,7 +8,7 @@ function xmpDoc(body: string): string {
|
||||
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
|
||||
<rdf:Description rdf:about=""
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:projectx="https://projectx.app/xmp/1.0/">
|
||||
xmlns:${PROJECTX_NS_PREFIX}="${PROJECTX_NS_URI}">
|
||||
${body}
|
||||
</rdf:Description>
|
||||
</rdf:RDF>
|
||||
@@ -195,7 +196,7 @@ describe('parseXmp', () => {
|
||||
<rdf:Description rdf:about="" xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:title>Merged Title</dc:title>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about="" xmlns:projectx="https://projectx.app/xmp/1.0/">
|
||||
<rdf:Description rdf:about="" xmlns:${PROJECTX_NS_PREFIX}="${PROJECTX_NS_URI}">
|
||||
<projectx:seriesName>Merged Series</projectx:seriesName>
|
||||
</rdf:Description>
|
||||
</rdf:RDF>
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
import { PDFDocument, PDFName, PDFRawStream, PDFRef } from 'pdf-lib';
|
||||
|
||||
import { PROJECTX_NS_PREFIX } from '../../../common/projectx-ns';
|
||||
|
||||
export interface XmpParsed {
|
||||
title: string | null;
|
||||
subtitle: string | null;
|
||||
@@ -125,7 +127,7 @@ export function parseXmp(xmpXml: string): XmpParsed | null {
|
||||
}
|
||||
}
|
||||
|
||||
const px = 'projectx';
|
||||
const px = PROJECTX_NS_PREFIX;
|
||||
|
||||
return {
|
||||
title: str(merged['dc:title']),
|
||||
|
||||
@@ -77,6 +77,7 @@ vi.mock('./extractors/audio.extractor', () => ({
|
||||
}));
|
||||
|
||||
import { mkdir, readdir, rm, writeFile } from 'fs/promises';
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
import { authors, bookAuthors, bookGenres, bookMetadata, bookTags, genres, tags } from '../../db/schema';
|
||||
import { generateThumbnail, imageExt } from './lib/cover';
|
||||
@@ -235,11 +236,57 @@ describe('MetadataService', () => {
|
||||
await expect(service.extractAndSave(15, '/books/a.pdf', 'pdf')).rejects.toThrow('bad metadata');
|
||||
});
|
||||
|
||||
it('extractAndSave(pdf) surfaces parser warnings in logs', async () => {
|
||||
const { db } = makeDb();
|
||||
const service = makeService(db);
|
||||
const warnSpy = vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
|
||||
|
||||
mockParsePdfFile.mockImplementation((_absolutePath, options) => {
|
||||
options?.onWarning?.({
|
||||
code: 'cover-extraction-failed',
|
||||
absolutePath: '/tmp/warn.pdf',
|
||||
errorClass: 'Error',
|
||||
errorMessage: 'pdftoppm missing',
|
||||
});
|
||||
|
||||
return Promise.resolve({
|
||||
title: 'Warn PDF',
|
||||
subtitle: null,
|
||||
description: null,
|
||||
isbn10: null,
|
||||
isbn13: null,
|
||||
publisher: null,
|
||||
publishedYear: null,
|
||||
language: null,
|
||||
seriesName: null,
|
||||
seriesIndex: null,
|
||||
authors: [],
|
||||
genres: [],
|
||||
tags: [],
|
||||
rating: null,
|
||||
pageCount: null,
|
||||
googleBooksId: null,
|
||||
goodreadsId: null,
|
||||
amazonId: null,
|
||||
hardcoverId: null,
|
||||
openLibraryId: null,
|
||||
itunesId: null,
|
||||
coverBuffer: null,
|
||||
});
|
||||
});
|
||||
|
||||
await service.extractAndSave(21, '/tmp/warn.pdf', 'pdf');
|
||||
|
||||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('[metadata.pdf_parse] [fail]'));
|
||||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('pdftoppm missing'));
|
||||
});
|
||||
|
||||
it('extractAndSave(pdf) persists fallback title/year, page count, and extracted cover bytes', async () => {
|
||||
const { db, updateSet } = makeDb();
|
||||
const service = makeService(db);
|
||||
const replaceAuthorsSpy = vi.spyOn(service, 'replaceAuthors').mockResolvedValue(undefined);
|
||||
const replaceGenresSpy = vi.spyOn(service, 'replaceGenres').mockResolvedValue(undefined);
|
||||
const replaceTagsSpy = vi.spyOn(service, 'replaceTags').mockResolvedValue(undefined);
|
||||
|
||||
mockParsePdfFile.mockResolvedValue({
|
||||
title: null,
|
||||
@@ -254,14 +301,29 @@ describe('MetadataService', () => {
|
||||
seriesIndex: 2,
|
||||
authors: [{ name: 'Author A', sortName: null }],
|
||||
genres: ['Fantasy'],
|
||||
tags: [],
|
||||
tags: ['Shelf'],
|
||||
rating: 4.6,
|
||||
pageCount: 321,
|
||||
googleBooksId: 'google-1',
|
||||
goodreadsId: 'goodreads-1',
|
||||
amazonId: 'amazon-1',
|
||||
hardcoverId: 'hardcover-1',
|
||||
openLibraryId: 'open-library-1',
|
||||
itunesId: 'itunes-1',
|
||||
coverBuffer: Buffer.from('jpeg-bytes'),
|
||||
});
|
||||
mockParseBookFilename.mockReturnValue({ title: 'Title From Filename', publishedYear: 1999 });
|
||||
|
||||
await service.extractAndSave(22, '/tmp/book.pdf', 'pdf');
|
||||
|
||||
expect(mockParsePdfFile).toHaveBeenCalledWith(
|
||||
'/tmp/book.pdf',
|
||||
expect.objectContaining({
|
||||
extractCover: true,
|
||||
onWarning: expect.any(Function),
|
||||
}),
|
||||
);
|
||||
|
||||
expect(updateSet).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
title: 'Title From Filename',
|
||||
@@ -269,12 +331,20 @@ describe('MetadataService', () => {
|
||||
subtitle: 'Subtitle',
|
||||
description: 'Description',
|
||||
pageCount: 321,
|
||||
rating: 5,
|
||||
googleBooksId: 'google-1',
|
||||
goodreadsId: 'goodreads-1',
|
||||
amazonId: 'amazon-1',
|
||||
hardcoverId: 'hardcover-1',
|
||||
openLibraryId: 'open-library-1',
|
||||
itunesId: 'itunes-1',
|
||||
updatedAt: expect.any(Date),
|
||||
}),
|
||||
);
|
||||
expect(mockWriteFile).toHaveBeenCalledWith('/books/covers/22/cover_extracted.png', Buffer.from('jpeg-bytes'));
|
||||
expect(replaceAuthorsSpy).toHaveBeenCalledWith(22, [{ name: 'Author A', sortName: null }]);
|
||||
expect(replaceGenresSpy).toHaveBeenCalledWith(22, ['Fantasy']);
|
||||
expect(replaceTagsSpy).toHaveBeenCalledWith(22, ['Shelf']);
|
||||
expect(embedder.embedBook).toHaveBeenCalledWith(22);
|
||||
});
|
||||
|
||||
|
||||
@@ -7,6 +7,13 @@ import { join } from 'path';
|
||||
|
||||
import { DB } from '../../db';
|
||||
import * as schema from '../../db/schema';
|
||||
import {
|
||||
COVER_EXTRACTED_FILE_PREFIX,
|
||||
bookCoverDirPath,
|
||||
bookThumbnailPath,
|
||||
isCustomBookCoverFileName,
|
||||
isExtractedBookCoverFileName,
|
||||
} from '../../common/book-cover-storage';
|
||||
import { BookEmbedderService } from '../embedding/book-embedder.service';
|
||||
import { BookMetadataLockService } from '../book-metadata-lock/book-metadata-lock.service';
|
||||
import { ComicMetadataRepository } from './comic-metadata.repository';
|
||||
@@ -23,6 +30,7 @@ import { MobiFormatExtractor } from './extractors/mobi-format.extractor';
|
||||
import { PdfFormatExtractor } from './extractors/pdf-format.extractor';
|
||||
import type { FormatExtractor, ParsedBookData } from './extractors/format-extractor.interface';
|
||||
import { generateThumbnail, imageExt } from './lib/cover';
|
||||
import type { PdfParseWarning } from './lib/pdf-parser';
|
||||
import { MetadataEventsService, METADATA_AUTHORS_REPLACED } from './metadata-events.service';
|
||||
|
||||
type Db = NodePgDatabase<typeof schema>;
|
||||
@@ -35,9 +43,6 @@ interface RelationMutationOptions {
|
||||
|
||||
const AUDIO_FORMATS = ['m4b', 'mp3', 'm4a', 'opus', 'ogg', 'flac'] as const;
|
||||
const MAX_RELATION_NAME_LENGTH = 200;
|
||||
const CUSTOM_COVER_FILE_PREFIX = 'cover_custom.';
|
||||
const EXTRACTED_COVER_FILE_PREFIX = 'cover_extracted.';
|
||||
const THUMBNAIL_FILE_NAME = 'thumbnail.jpg';
|
||||
const EXTRACTED_COVER_SOURCE = 'extracted';
|
||||
|
||||
@Injectable()
|
||||
@@ -63,7 +68,7 @@ export class MetadataService {
|
||||
this.extractorMap = new Map<string, FormatExtractor>([
|
||||
['epub', epub],
|
||||
['kepub', epub],
|
||||
['pdf', new PdfFormatExtractor()],
|
||||
['pdf', new PdfFormatExtractor({ extractCover: true, onWarning: (warning) => this.logPdfParseWarning(warning) })],
|
||||
['mobi', mobi],
|
||||
['azw3', mobi],
|
||||
['azw', mobi],
|
||||
@@ -468,7 +473,15 @@ export class MetadataService {
|
||||
seriesIndex: data.seriesIndex,
|
||||
authors: data.authors.map((author) => author.name),
|
||||
genres: data.genres,
|
||||
tags: data.tags,
|
||||
rating: normalizeImportedRating(data.rating),
|
||||
pageCount: data.pageCount,
|
||||
googleBooksId: data.googleBooksId,
|
||||
goodreadsId: data.goodreadsId,
|
||||
amazonId: data.amazonId,
|
||||
hardcoverId: data.hardcoverId,
|
||||
openLibraryId: data.openLibraryId,
|
||||
itunesId: data.itunesId,
|
||||
comicMetadata: data.comicMetadata ?? undefined,
|
||||
});
|
||||
|
||||
@@ -483,7 +496,14 @@ export class MetadataService {
|
||||
if (filtered.language !== undefined) scalarFields.language = filtered.language;
|
||||
if (filtered.seriesName !== undefined) scalarFields.seriesName = filtered.seriesName;
|
||||
if (filtered.seriesIndex !== undefined) scalarFields.seriesIndex = filtered.seriesIndex;
|
||||
if (filtered.rating !== undefined) scalarFields.rating = filtered.rating;
|
||||
if (filtered.pageCount !== undefined) scalarFields.pageCount = filtered.pageCount;
|
||||
if (filtered.googleBooksId !== undefined) scalarFields.googleBooksId = filtered.googleBooksId;
|
||||
if (filtered.goodreadsId !== undefined) scalarFields.goodreadsId = filtered.goodreadsId;
|
||||
if (filtered.amazonId !== undefined) scalarFields.amazonId = filtered.amazonId;
|
||||
if (filtered.hardcoverId !== undefined) scalarFields.hardcoverId = filtered.hardcoverId;
|
||||
if (filtered.openLibraryId !== undefined) scalarFields.openLibraryId = filtered.openLibraryId;
|
||||
if (filtered.itunesId !== undefined) scalarFields.itunesId = filtered.itunesId;
|
||||
if (Object.keys(scalarFields).length > 0) {
|
||||
scalarFields.updatedAt = new Date();
|
||||
await this.db.update(bookMetadata).set(scalarFields).where(eq(bookMetadata.bookId, bookId));
|
||||
@@ -495,6 +515,9 @@ export class MetadataService {
|
||||
if (filtered.genres !== undefined) {
|
||||
await this.replaceGenres(bookId, filtered.genres);
|
||||
}
|
||||
if (filtered.tags !== undefined) {
|
||||
await this.replaceTags(bookId, filtered.tags);
|
||||
}
|
||||
|
||||
if (filtered.comicMetadata) {
|
||||
await this.comicMetadataRepository.upsert(bookId, filtered.comicMetadata);
|
||||
@@ -517,20 +540,20 @@ export class MetadataService {
|
||||
private async persistCover(bookId: number, bytes: Buffer, overwrite: boolean): Promise<void> {
|
||||
if (await this.bookMetadataLockService.isFieldLocked(bookId, 'cover')) return;
|
||||
const ext = imageExt(bytes);
|
||||
const dir = join(this.booksPath, 'covers', String(bookId));
|
||||
const dir = bookCoverDirPath(this.booksPath, bookId);
|
||||
await mkdir(dir, { recursive: true });
|
||||
|
||||
const files = await readdir(dir).catch(() => [] as string[]);
|
||||
const hasCustom = files.some((fileName) => fileName.startsWith(CUSTOM_COVER_FILE_PREFIX));
|
||||
const hasCustom = files.some(isCustomBookCoverFileName);
|
||||
|
||||
const staleExtractedFiles = files.filter((fileName) => fileName.startsWith(EXTRACTED_COVER_FILE_PREFIX));
|
||||
const staleExtractedFiles = files.filter(isExtractedBookCoverFileName);
|
||||
await Promise.all(staleExtractedFiles.map((fileName) => rm(join(dir, fileName), { force: true })));
|
||||
|
||||
await writeFile(join(dir, `${EXTRACTED_COVER_FILE_PREFIX}${ext}`), bytes);
|
||||
await writeFile(join(dir, `${COVER_EXTRACTED_FILE_PREFIX}${ext}`), bytes);
|
||||
|
||||
if (!hasCustom) {
|
||||
const thumbnail = await generateThumbnail(bytes);
|
||||
await writeFile(join(dir, THUMBNAIL_FILE_NAME), thumbnail);
|
||||
await writeFile(bookThumbnailPath(this.booksPath, bookId), thumbnail);
|
||||
}
|
||||
|
||||
if (overwrite) {
|
||||
@@ -543,7 +566,27 @@ export class MetadataService {
|
||||
}
|
||||
}
|
||||
|
||||
private logPdfParseWarning(warning: PdfParseWarning): void {
|
||||
if (warning.code === 'buffered-large-pdf') {
|
||||
this.logger.warn(
|
||||
`[metadata.pdf_parse] [end] path="${warning.absolutePath}" code=${warning.code} sizeBytes=${warning.sizeBytes ?? 0} thresholdBytes=${warning.thresholdBytes ?? 0} - large pdf buffered in memory`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.logger.warn(
|
||||
`[metadata.pdf_parse] [fail] path="${warning.absolutePath}" code=${warning.code} errorClass=${warning.errorClass} error="${warning.errorMessage}" - pdf parse warning emitted`,
|
||||
);
|
||||
}
|
||||
|
||||
private normalizeUniqueRelationNames(values: string[]): string[] {
|
||||
return [...new Set(values.map((value) => value.trim().substring(0, MAX_RELATION_NAME_LENGTH)).filter(Boolean))];
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeImportedRating(value: number | null | undefined): number | null | undefined {
|
||||
if (value === undefined) return undefined;
|
||||
if (value === null) return null;
|
||||
if (!Number.isFinite(value)) return null;
|
||||
const normalized = Math.round(value);
|
||||
return normalized >= 1 && normalized <= 10 ? normalized : null;
|
||||
}
|
||||
|
||||
@@ -4,7 +4,8 @@ import { access, mkdir, readdir, readFile, unlink, writeFile } from 'fs/promises
|
||||
import { join } from 'path';
|
||||
|
||||
import type { CoverRefreshedEvent } from '@projectx/types';
|
||||
import { coverDirPath, generateThumbnail, imageExt } from '../../metadata/lib/cover';
|
||||
import { COVER_CUSTOM_FILE_PREFIX, COVER_THUMBNAIL_FILE_NAME, bookCoverDirPath } from '../../../common/book-cover-storage';
|
||||
import { generateThumbnail, imageExt } from '../../metadata/lib/cover';
|
||||
import { MigrationRepository } from '../migration.repository';
|
||||
import { MigrationImportRepository } from './migration-import.repository';
|
||||
import { ScanGateway } from '../../scanner/scan.gateway';
|
||||
@@ -92,16 +93,16 @@ export class CoverImporter {
|
||||
}
|
||||
|
||||
private async importSingleCover(targetBookId: number, booksPath: string, coverBytes: Buffer, sourceThumbnailPath: string): Promise<void> {
|
||||
const targetCoverDir = coverDirPath(booksPath, targetBookId);
|
||||
const targetCoverDir = bookCoverDirPath(booksPath, targetBookId);
|
||||
await mkdir(targetCoverDir, { recursive: true });
|
||||
await this.deleteFilesByPrefix(targetCoverDir, 'cover_custom.');
|
||||
await this.deleteFilesByPrefix(targetCoverDir, COVER_CUSTOM_FILE_PREFIX);
|
||||
|
||||
const coverExt = imageExt(coverBytes);
|
||||
await writeFile(join(targetCoverDir, `cover_custom.${coverExt}`), coverBytes);
|
||||
await writeFile(join(targetCoverDir, `${COVER_CUSTOM_FILE_PREFIX}${coverExt}`), coverBytes);
|
||||
|
||||
const sourceThumbnailBytes = await this.readOptionalFile(sourceThumbnailPath);
|
||||
const thumbnailBytes = sourceThumbnailBytes ?? (await generateThumbnail(coverBytes));
|
||||
await writeFile(join(targetCoverDir, 'thumbnail.jpg'), thumbnailBytes);
|
||||
await writeFile(join(targetCoverDir, COVER_THUMBNAIL_FILE_NAME), thumbnailBytes);
|
||||
|
||||
await this.importRepo.markCoverAsCustom(targetBookId);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
vi.mock('fs', () => ({
|
||||
createReadStream: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('fs/promises', () => ({
|
||||
readdir: vi.fn(),
|
||||
stat: vi.fn(),
|
||||
}));
|
||||
|
||||
import { createReadStream } from 'fs';
|
||||
import { readdir, stat } from 'fs/promises';
|
||||
import type { MockedFunction } from 'vitest';
|
||||
|
||||
import { OpdsController } from './opds.controller';
|
||||
|
||||
const mockCreateReadStream = createReadStream as MockedFunction<typeof createReadStream>;
|
||||
const mockReaddir = readdir as MockedFunction<typeof readdir>;
|
||||
const mockStat = stat as MockedFunction<typeof stat>;
|
||||
|
||||
function makeController() {
|
||||
const opdsService = {} as never;
|
||||
const opdsBookService = {
|
||||
validateBookAccess: vi.fn().mockResolvedValue(undefined),
|
||||
} as never;
|
||||
const config = {
|
||||
get: vi.fn().mockReturnValue('/books'),
|
||||
} as never;
|
||||
|
||||
return {
|
||||
controller: new OpdsController(opdsService, opdsBookService, config),
|
||||
opdsBookService,
|
||||
};
|
||||
}
|
||||
|
||||
function makeReply() {
|
||||
const reply = {
|
||||
header: vi.fn(),
|
||||
type: vi.fn(),
|
||||
status: vi.fn(),
|
||||
send: vi.fn(),
|
||||
};
|
||||
|
||||
reply.header.mockReturnValue(reply);
|
||||
reply.type.mockReturnValue(reply);
|
||||
reply.status.mockReturnValue(reply);
|
||||
|
||||
return reply as never;
|
||||
}
|
||||
|
||||
describe('OpdsController', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
});
|
||||
|
||||
it('serves the preferred stored cover file for OPDS clients', async () => {
|
||||
const { controller, opdsBookService } = makeController();
|
||||
const reply = makeReply();
|
||||
const stream = { kind: 'stream' };
|
||||
|
||||
mockReaddir.mockResolvedValue(['cover_extracted.jpg', 'cover_custom.png'] as never);
|
||||
mockStat.mockResolvedValue({ mtimeMs: 1234 } as never);
|
||||
mockCreateReadStream.mockReturnValue(stream as never);
|
||||
|
||||
await controller.cover(42, { userId: 7, isSuperuser: false } as never, reply);
|
||||
|
||||
expect(opdsBookService.validateBookAccess).toHaveBeenCalledWith(42, 7, false);
|
||||
expect(mockCreateReadStream).toHaveBeenCalledWith('/books/covers/42/cover_custom.png');
|
||||
expect(reply.header).toHaveBeenCalledWith('ETag', '"1234"');
|
||||
expect(reply.type).toHaveBeenCalledWith('image/png');
|
||||
expect(reply.send).toHaveBeenCalledWith(stream);
|
||||
});
|
||||
});
|
||||
@@ -17,6 +17,7 @@ import { join } from 'path';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import type { FastifyReply } from 'fastify';
|
||||
|
||||
import { bookCoverDirPath, bookThumbnailPath, findPreferredBookCoverFileName } from '../../common/book-cover-storage';
|
||||
import { Public } from '../../common/decorators/public.decorator';
|
||||
import { OPDS_MIME_ACQ, OPDS_MIME_NAV, OPDS_MIME_SEARCH, fileMimeType } from './opds-xml.helpers';
|
||||
import { OpdsAuthGuard } from './opds-auth.guard';
|
||||
@@ -193,10 +194,10 @@ export class OpdsController {
|
||||
@Headers('if-none-match') ifNoneMatch?: string,
|
||||
) {
|
||||
await this.opdsBookService.validateBookAccess(bookId, user.userId, user.isSuperuser);
|
||||
const dir = join(this.booksPath, 'covers', String(bookId));
|
||||
const dir = bookCoverDirPath(this.booksPath, bookId);
|
||||
try {
|
||||
const files = await readdir(dir);
|
||||
const cover = files.find((f) => f.startsWith('cover.'));
|
||||
const cover = findPreferredBookCoverFileName(files);
|
||||
if (!cover) throw new NotFoundException('No cover');
|
||||
const coverPath = join(dir, cover);
|
||||
const { mtimeMs } = await stat(coverPath);
|
||||
@@ -223,7 +224,7 @@ export class OpdsController {
|
||||
@Headers('if-none-match') ifNoneMatch?: string,
|
||||
) {
|
||||
await this.opdsBookService.validateBookAccess(bookId, user.userId, user.isSuperuser);
|
||||
const thumbnailPath = join(this.booksPath, 'covers', String(bookId), 'thumbnail.jpg');
|
||||
const thumbnailPath = bookThumbnailPath(this.booksPath, bookId);
|
||||
try {
|
||||
const { mtimeMs } = await stat(thumbnailPath);
|
||||
const etag = `"${Math.floor(mtimeMs)}"`;
|
||||
|
||||
@@ -17,8 +17,8 @@ import { assembleBookCards } from '../book/utils/assemble-book-cards';
|
||||
|
||||
const METADATA_FORMATS = new Set(['epub', 'mobi', 'azw3', 'azw', 'cbz', 'cbr', 'cb7', 'fb2', 'pdf', 'm4b', 'mp3', 'm4a', 'opus', 'ogg', 'flac']);
|
||||
const BATCH_SIZE = 3;
|
||||
const BOOK_EMIT_BUFFER_SIZE = 30;
|
||||
const BOOK_EMIT_FLUSH_INTERVAL_MS = 1500;
|
||||
const BOOK_EMIT_BUFFER_SIZE = 20;
|
||||
const BOOK_EMIT_FLUSH_INTERVAL_MS = 1000;
|
||||
type OrganizationMode = 'book_per_file' | 'book_per_folder';
|
||||
|
||||
interface ScanCounts {
|
||||
|
||||
@@ -340,6 +340,13 @@ describe('UploadService', () => {
|
||||
|
||||
expect(result).toEqual({ bookId: 99, filename: 'Research Paper.pdf', format: 'pdf', sizeBytes: 456 });
|
||||
expect(storage.moveToPath).toHaveBeenCalledWith('/tmp/upload.bin', '/library/Research Paper.pdf');
|
||||
expect(mockParsePdfFile).toHaveBeenCalledWith(
|
||||
'/tmp/upload.bin',
|
||||
expect.objectContaining({
|
||||
extractCover: false,
|
||||
onWarning: expect.any(Function),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('cbz metadata populates tokens', async () => {
|
||||
|
||||
@@ -19,7 +19,7 @@ import type { UploadResult } from '@projectx/types';
|
||||
import { extractEpubMetadata } from '../metadata/lib/epub';
|
||||
import { extractCbzMetadata, extractCbrMetadata, extractCb7Metadata } from '../metadata/lib/cbz-metadata';
|
||||
import { parseMobiFile } from '../metadata/lib/mobi-parser';
|
||||
import { parsePdfFile } from '../metadata/lib/pdf-parser';
|
||||
import { parsePdfFile, type PdfParseWarning } from '../metadata/lib/pdf-parser';
|
||||
|
||||
type Db = NodePgDatabase<typeof schema>;
|
||||
|
||||
@@ -180,7 +180,10 @@ export class UploadService {
|
||||
};
|
||||
}
|
||||
} else if (format === 'pdf') {
|
||||
const pdf = await parsePdfFile(tempPath);
|
||||
const pdf = await parsePdfFile(tempPath, {
|
||||
extractCover: false,
|
||||
onWarning: (warning) => this.logPdfPatternTokenWarning(warning),
|
||||
});
|
||||
if (pdf) {
|
||||
parsed = { title: pdf.title, publisher: pdf.publisher, authors: pdf.authors, seriesName: null, seriesIndex: null };
|
||||
}
|
||||
@@ -220,6 +223,18 @@ export class UploadService {
|
||||
return { errorClass, errorMessage };
|
||||
}
|
||||
|
||||
private logPdfPatternTokenWarning(warning: PdfParseWarning): void {
|
||||
if (warning.code === 'buffered-large-pdf') {
|
||||
this.logger.warn(
|
||||
`[upload.pattern_tokens_pdf] [end] path="${warning.absolutePath}" code=${warning.code} sizeBytes=${warning.sizeBytes ?? 0} thresholdBytes=${warning.thresholdBytes ?? 0} - large pdf buffered in memory`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.logger.warn(
|
||||
`[upload.pattern_tokens_pdf] [fail] path="${warning.absolutePath}" code=${warning.code} errorClass=${warning.errorClass} error="${warning.errorMessage}" - pdf token extraction warning emitted`,
|
||||
);
|
||||
}
|
||||
|
||||
private async destinationExists(absolutePath: string): Promise<boolean> {
|
||||
try {
|
||||
await fsAccess(absolutePath);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { mkdir, writeFile } from 'fs/promises';
|
||||
import { mkdir, readFile, writeFile } from 'fs/promises';
|
||||
import { dirname, join } from 'path';
|
||||
import { afterEach } from 'vitest';
|
||||
|
||||
@@ -7,6 +7,7 @@ import { Permission, type BookBucketFile, type BookBucketFinalizeResult } from '
|
||||
|
||||
import * as schema from '../src/db/schema';
|
||||
import { buildFb2Fixture } from './e2e/book-bucket/book-bucket-fixture-builder';
|
||||
import { createPdfFixture } from './e2e/metadata-write/metadata-write-fixture-builder';
|
||||
import {
|
||||
authHeader,
|
||||
closeBookBucketE2EContext,
|
||||
@@ -141,6 +142,27 @@ describe('Book Bucket ingest + finalize (e2e)', () => {
|
||||
expect(summaryResponse.json()).toMatchObject({ total: 1, ready: 1 });
|
||||
});
|
||||
|
||||
it('ingests pdf uploads and persists extracted cover files', async () => {
|
||||
const fixturePath = await createPdfFixture(context.fixture.rootPath, 'fixtures/book-bucket-cover.pdf', 'Book Bucket PDF Title');
|
||||
const pdfBytes = await readFile(fixturePath);
|
||||
|
||||
const uploadResponse = await uploadBookBucketFile(context, {
|
||||
token: context.adminToken,
|
||||
fileName: 'book-bucket-cover.pdf',
|
||||
content: pdfBytes,
|
||||
contentType: 'application/pdf',
|
||||
});
|
||||
|
||||
expect(uploadResponse.statusCode).toBe(201);
|
||||
const uploaded = uploadResponse.json() as BookBucketFile;
|
||||
const ready = await waitForBookBucketStatus(context, uploaded.id, ['ready']);
|
||||
|
||||
expect(ready.embeddedMetadata?.title).toBe('Book Bucket PDF Title');
|
||||
expect(ready.coverPath).toEqual(expect.stringContaining('.jpg'));
|
||||
expect(ready.coverPath).not.toBeNull();
|
||||
expect(await fileExists(ready.coverPath!)).toBe(true);
|
||||
});
|
||||
|
||||
it('bulk set-target selectAll with status=pending includes pending, extracting, and fetching', async () => {
|
||||
const destination = await createLibraryWithFolder(context);
|
||||
const pendingRow = await createBookBucketRow(context, { fileName: 'pending-target.fb2', status: 'pending' });
|
||||
|
||||
Reference in New Issue
Block a user