mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
refactor(metadata): improve parsing, cover storage, and content type
This commit is contained in:
@@ -22,9 +22,11 @@ describe('book-cover-storage', () => {
|
||||
expect(isExtractedBookCoverFileName('cover_custom.jpg')).toBe(false);
|
||||
});
|
||||
|
||||
it('prefers custom covers over extracted covers', () => {
|
||||
it('prefers custom and extracted covers over legacy cover files', () => {
|
||||
expect(findPreferredBookCoverFileName(['thumbnail.jpg', 'cover_extracted.jpg', 'cover_custom.png'])).toBe('cover_custom.png');
|
||||
expect(findPreferredBookCoverFileName(['thumbnail.jpg', 'cover_extracted.jpg'])).toBe('cover_extracted.jpg');
|
||||
expect(findPreferredBookCoverFileName(['thumbnail.jpg', 'cover.jpg'])).toBe('cover.jpg');
|
||||
expect(findPreferredBookCoverFileName(['thumbnail.jpg', 'cover.png'])).toBe('cover.png');
|
||||
expect(findPreferredBookCoverFileName(['thumbnail.jpg'])).toBeNull();
|
||||
});
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import { join } from 'path';
|
||||
|
||||
export const COVER_CUSTOM_FILE_PREFIX = 'cover_custom.';
|
||||
export const COVER_EXTRACTED_FILE_PREFIX = 'cover_extracted.';
|
||||
export const COVER_LEGACY_FILE_PREFIX = 'cover.';
|
||||
export const COVER_THUMBNAIL_FILE_NAME = 'thumbnail.jpg';
|
||||
|
||||
export function bookCoverDirPath(appDataPath: string, bookId: number): string {
|
||||
@@ -21,9 +22,13 @@ export function isExtractedBookCoverFileName(fileName: string): boolean {
|
||||
}
|
||||
|
||||
export function findPreferredBookCoverFileName(files: readonly string[]): string | null {
|
||||
return files.find(isCustomBookCoverFileName) ?? files.find(isExtractedBookCoverFileName) ?? null;
|
||||
return files.find(isCustomBookCoverFileName) ?? files.find(isExtractedBookCoverFileName) ?? files.find(isLegacyBookCoverFileName) ?? null;
|
||||
}
|
||||
|
||||
export function findExtractedBookCoverFileName(files: readonly string[]): string | null {
|
||||
return files.find(isExtractedBookCoverFileName) ?? null;
|
||||
}
|
||||
|
||||
function isLegacyBookCoverFileName(fileName: string): boolean {
|
||||
return fileName.startsWith(COVER_LEGACY_FILE_PREFIX);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { imageContentTypeFromExtension, imageContentTypeFromPath } from './image-content-type';
|
||||
|
||||
describe('image content type helpers', () => {
|
||||
describe('imageContentTypeFromPath', () => {
|
||||
it('returns image/png for png file paths', () => {
|
||||
expect(imageContentTypeFromPath('/tmp/cover.png')).toBe('image/png');
|
||||
});
|
||||
|
||||
it('returns image/webp for webp file paths', () => {
|
||||
expect(imageContentTypeFromPath('/tmp/cover.webp')).toBe('image/webp');
|
||||
});
|
||||
|
||||
it('falls back to image/jpeg for unknown extensions', () => {
|
||||
expect(imageContentTypeFromPath('/tmp/cover.unknown')).toBe('image/jpeg');
|
||||
});
|
||||
});
|
||||
|
||||
describe('imageContentTypeFromExtension', () => {
|
||||
it('normalizes extension case', () => {
|
||||
expect(imageContentTypeFromExtension('.PNG')).toBe('image/png');
|
||||
});
|
||||
|
||||
it('returns image/jpeg for jpeg aliases', () => {
|
||||
expect(imageContentTypeFromExtension('.jpg')).toBe('image/jpeg');
|
||||
expect(imageContentTypeFromExtension('.jpeg')).toBe('image/jpeg');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
import { extname } from 'path';
|
||||
|
||||
const IMAGE_CONTENT_TYPE_BY_EXT: Record<string, string> = {
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.png': 'image/png',
|
||||
'.gif': 'image/gif',
|
||||
'.webp': 'image/webp',
|
||||
'.bmp': 'image/bmp',
|
||||
'.avif': 'image/avif',
|
||||
};
|
||||
|
||||
export function imageContentTypeFromPath(path: string): string {
|
||||
return imageContentTypeFromExtension(extname(path));
|
||||
}
|
||||
|
||||
export function imageContentTypeFromExtension(extension: string): string {
|
||||
return IMAGE_CONTENT_TYPE_BY_EXT[extension.toLowerCase()] ?? 'image/jpeg';
|
||||
}
|
||||
@@ -19,13 +19,13 @@ import {
|
||||
import { createReadStream } from 'fs';
|
||||
import { stat } from 'fs/promises';
|
||||
import type { FastifyReply } from 'fastify';
|
||||
import { extname } from 'path';
|
||||
import { map, Observable } from 'rxjs';
|
||||
|
||||
import { Permission, AuditAction, AuditResource } from '@projectx/types';
|
||||
import { CurrentUser } from '../../common/decorators/current-user.decorator';
|
||||
import { Auditable } from '../../common/decorators/auditable.decorator';
|
||||
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
|
||||
import { imageContentTypeFromPath } from '../../common/image-content-type';
|
||||
import type { RequestUser } from '../../common/types/request-user';
|
||||
import type { AuthorMetadataCandidate } from '@projectx/types';
|
||||
import { AuthorEnrichmentConfigService } from './author-enrichment-config.service';
|
||||
@@ -228,8 +228,7 @@ export class AuthorsController {
|
||||
reply.header('Cache-Control', 'no-cache');
|
||||
reply.header('ETag', `"${Math.floor(mtimeMs)}"`);
|
||||
|
||||
const ext = extname(imagePath).toLowerCase();
|
||||
reply.type(ext === '.png' ? 'image/png' : 'image/jpeg');
|
||||
reply.type(imageContentTypeFromPath(imagePath));
|
||||
reply.send(createReadStream(imagePath));
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ import { AuditAction, AuditResource } from '@projectx/types';
|
||||
import { CurrentUser } from '../../common/decorators/current-user.decorator';
|
||||
import { Auditable } from '../../common/decorators/auditable.decorator';
|
||||
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
|
||||
import { imageContentTypeFromPath } from '../../common/image-content-type';
|
||||
import type { MultipartRequest } from '../../common/types/multipart-request';
|
||||
import type { RequestUser } from '../../common/types/request-user';
|
||||
import { BookBucketService } from './book-bucket.service';
|
||||
@@ -96,7 +97,7 @@ export class BookBucketController {
|
||||
if (!exists) throw new NotFoundException('Cover file not found on disk');
|
||||
|
||||
const stream = createReadStream(row.coverPath);
|
||||
const contentType = row.coverPath.endsWith('.png') ? 'image/png' : 'image/jpeg';
|
||||
const contentType = imageContentTypeFromPath(row.coverPath);
|
||||
reply.header('Content-Type', contentType);
|
||||
reply.header('Cache-Control', 'private, max-age=3600');
|
||||
return reply.send(stream);
|
||||
|
||||
@@ -23,6 +23,7 @@ import type { FastifyReply } from 'fastify';
|
||||
import { CurrentUser } from '../../common/decorators/current-user.decorator';
|
||||
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
|
||||
import { Auditable } from '../../common/decorators/auditable.decorator';
|
||||
import { imageContentTypeFromPath } from '../../common/image-content-type';
|
||||
import type { RequestUser } from '../../common/types/request-user';
|
||||
import { FileWriteService } from '../file-write/file-write.service';
|
||||
import { BookService } from './book.service';
|
||||
@@ -297,8 +298,7 @@ export class BookController {
|
||||
return;
|
||||
}
|
||||
|
||||
const ext = coverPath.split('.').pop()?.toLowerCase();
|
||||
const contentType = ext === 'png' ? 'image/png' : 'image/jpeg';
|
||||
const contentType = imageContentTypeFromPath(coverPath);
|
||||
reply.header('Cache-Control', 'private, max-age=86400');
|
||||
reply.header('ETag', etag);
|
||||
reply.type(contentType);
|
||||
|
||||
@@ -5,6 +5,7 @@ import { createReadStream } from 'fs';
|
||||
import { join } from 'path';
|
||||
import type { FastifyReply } from 'fastify';
|
||||
import { bookCoverDirPath, bookThumbnailPath, findPreferredBookCoverFileName } from '../../../common/book-cover-storage';
|
||||
import { imageContentTypeFromPath } from '../../../common/image-content-type';
|
||||
import { KoboBookAccessService } from './kobo-book-access.service';
|
||||
|
||||
@Injectable()
|
||||
@@ -51,10 +52,9 @@ export class KoboThumbnailService {
|
||||
reply.status(304).send();
|
||||
return;
|
||||
}
|
||||
const ext = cover.split('.').pop()?.toLowerCase();
|
||||
reply.header('Cache-Control', 'max-age=86400');
|
||||
reply.header('ETag', etag);
|
||||
reply.type(ext === 'png' ? 'image/png' : 'image/jpeg');
|
||||
reply.type(imageContentTypeFromPath(coverPath));
|
||||
reply.send(createReadStream(coverPath));
|
||||
} catch {
|
||||
throw new NotFoundException('No cover image');
|
||||
|
||||
@@ -151,6 +151,22 @@ describe('extractCbzMetadata', () => {
|
||||
expect(r?.description).toBe('Watchmen summary.');
|
||||
});
|
||||
|
||||
it('maps CommunityRating from 0-5 scale into 0-10 scale', async () => {
|
||||
const xml = `<ComicInfo><CommunityRating>4.5</CommunityRating></ComicInfo>`;
|
||||
mockReadFile.mockResolvedValue(buildZipWithComicInfo(xml) as unknown as Buffer);
|
||||
|
||||
const r = await extractCbzMetadata('/book.cbz');
|
||||
expect(r?.rating).toBe(9);
|
||||
});
|
||||
|
||||
it('clamps CommunityRating to a maximum of 10 after scaling', async () => {
|
||||
const xml = `<ComicInfo><CommunityRating>8</CommunityRating></ComicInfo>`;
|
||||
mockReadFile.mockResolvedValue(buildZipWithComicInfo(xml) as unknown as Buffer);
|
||||
|
||||
const r = await extractCbzMetadata('/book.cbz');
|
||||
expect(r?.rating).toBe(10);
|
||||
});
|
||||
|
||||
it('truncates fractional year to integer', async () => {
|
||||
const xml = `<ComicInfo><Year>1986.5</Year></ComicInfo>`;
|
||||
mockReadFile.mockResolvedValue(buildZipWithComicInfo(xml) as unknown as Buffer);
|
||||
|
||||
@@ -180,7 +180,7 @@ function parseCbxRating(value: string | null): number | null {
|
||||
if (!value) return null;
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed)) return null;
|
||||
return Math.min(5, Math.max(0, parsed * 2));
|
||||
return Math.min(10, Math.max(0, parsed * 2));
|
||||
}
|
||||
|
||||
function parseIsbn13(value: string | null): string | null {
|
||||
|
||||
@@ -60,8 +60,20 @@ describe('imageExt', () => {
|
||||
expect(imageExt(Buffer.from([0xff, 0xd8, 0xff, 0xe0]))).toBe('jpg');
|
||||
});
|
||||
|
||||
it('returns "jpg" as default for unknown magic bytes (GIF)', () => {
|
||||
expect(imageExt(Buffer.from([0x47, 0x49, 0x46, 0x38]))).toBe('jpg');
|
||||
it('returns "gif" for GIF magic bytes (GIF8)', () => {
|
||||
expect(imageExt(Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]))).toBe('gif');
|
||||
});
|
||||
|
||||
it('returns "webp" for RIFF/WEBP magic bytes', () => {
|
||||
expect(imageExt(Buffer.from([0x52, 0x49, 0x46, 0x46, 0x24, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50]))).toBe('webp');
|
||||
});
|
||||
|
||||
it('returns "bmp" for BMP magic bytes (BM)', () => {
|
||||
expect(imageExt(Buffer.from([0x42, 0x4d, 0x00, 0x00]))).toBe('bmp');
|
||||
});
|
||||
|
||||
it('returns "jpg" as default for unknown magic bytes', () => {
|
||||
expect(imageExt(Buffer.from([0x00, 0x11, 0x22, 0x33]))).toBe('jpg');
|
||||
});
|
||||
|
||||
it('returns "jpg" when first byte is 0x89 but second byte is not 0x50', () => {
|
||||
|
||||
@@ -14,7 +14,23 @@ const THUMBNAIL_HEIGHT = 600;
|
||||
|
||||
/** Returns extension based on magic bytes, defaulting to 'jpg'. */
|
||||
export function imageExt(bytes: Buffer): string {
|
||||
if (bytes[0] === 0x89 && bytes[1] === 0x50) return 'png';
|
||||
if (bytes.length >= 4 && bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47) return 'png';
|
||||
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'jpg';
|
||||
if (bytes.length >= 4 && bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46 && bytes[3] === 0x38) return 'gif';
|
||||
if (
|
||||
bytes.length >= 12 &&
|
||||
bytes[0] === 0x52 &&
|
||||
bytes[1] === 0x49 &&
|
||||
bytes[2] === 0x46 &&
|
||||
bytes[3] === 0x46 &&
|
||||
bytes[8] === 0x57 &&
|
||||
bytes[9] === 0x45 &&
|
||||
bytes[10] === 0x42 &&
|
||||
bytes[11] === 0x50
|
||||
) {
|
||||
return 'webp';
|
||||
}
|
||||
if (bytes.length >= 2 && bytes[0] === 0x42 && bytes[1] === 0x4d) return 'bmp';
|
||||
return 'jpg';
|
||||
}
|
||||
|
||||
|
||||
@@ -163,6 +163,11 @@ describe('parseFb2File', () => {
|
||||
mockReadFile.mockResolvedValue(makeFb2('<book-title>Book</book-title><date>1984</date>') as unknown as Buffer);
|
||||
expect((await parseFb2File('/book.fb2'))?.publishedYear).toBe(1984);
|
||||
});
|
||||
|
||||
it('parses year from title-info/date value attribute', async () => {
|
||||
mockReadFile.mockResolvedValue(makeFb2('<book-title>Book</book-title><date value="1984-01-01"/>') as unknown as Buffer);
|
||||
expect((await parseFb2File('/book.fb2'))?.publishedYear).toBe(1984);
|
||||
});
|
||||
});
|
||||
|
||||
describe('description (annotation)', () => {
|
||||
|
||||
@@ -11,6 +11,11 @@ const parser = new XMLParser({
|
||||
function text(val: unknown): string | null {
|
||||
if (typeof val === 'string') return val.trim() || null;
|
||||
if (typeof val === 'number') return String(val);
|
||||
if (typeof val === 'object' && val !== null && '#text' in val) {
|
||||
const textNode = (val as Record<string, unknown>)['#text'];
|
||||
if (typeof textNode === 'string') return textNode.trim() || null;
|
||||
if (typeof textNode === 'number') return String(textNode);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -39,6 +44,27 @@ function extractAnnotationText(val: unknown): string {
|
||||
return '';
|
||||
}
|
||||
|
||||
function parseFb2YearNode(val: unknown): number | null {
|
||||
const candidates: string[] = [];
|
||||
const direct = text(val);
|
||||
if (direct) candidates.push(direct);
|
||||
|
||||
if (typeof val === 'object' && val !== null) {
|
||||
const objectNode = val as Record<string, unknown>;
|
||||
const valueAttr = text(objectNode['@_value']);
|
||||
if (valueAttr) candidates.push(valueAttr);
|
||||
}
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const match = candidate.match(/(\d{4})/);
|
||||
if (!match) continue;
|
||||
const year = parseInt(match[1], 10);
|
||||
if (!isNaN(year) && year > 1000 && year < 2200) return year;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export interface Fb2Metadata {
|
||||
title: string | null;
|
||||
description: string | null;
|
||||
@@ -107,10 +133,7 @@ export async function parseFb2File(absolutePath: string): Promise<Fb2Metadata |
|
||||
let publishedYear: number | null = null;
|
||||
const publishInfo = description?.['publish-info'] as Record<string, unknown> | undefined;
|
||||
const yearRaw = publishInfo?.['year'] ?? titleInfo['date'];
|
||||
if (typeof yearRaw === 'string' || typeof yearRaw === 'number') {
|
||||
const y = parseInt(String(yearRaw), 10);
|
||||
if (!isNaN(y) && y > 1000 && y < 2200) publishedYear = y;
|
||||
}
|
||||
publishedYear = parseFb2YearNode(yearRaw);
|
||||
|
||||
// Annotation (description)
|
||||
let annotationDescription: string | null = null;
|
||||
|
||||
@@ -108,6 +108,23 @@ describe('parseOpf', () => {
|
||||
expect(r.authors[0].sortName).toBe('Pratchett, Terry');
|
||||
});
|
||||
|
||||
it('accepts EPUB3 role values in full relator URI form', () => {
|
||||
const xml = epub3Opf(`
|
||||
<dc:creator id="cr1">Ursula Le Guin</dc:creator>
|
||||
<meta refines="#cr1" property="role">http://id.loc.gov/vocabulary/relators/aut</meta>
|
||||
`);
|
||||
const r = parseOpf(xml);
|
||||
expect(r.authors).toHaveLength(1);
|
||||
expect(r.authors[0].name).toBe('Ursula Le Guin');
|
||||
});
|
||||
|
||||
it('treats uppercase EPUB2 role codes as authors', () => {
|
||||
const xml = epub2Opf(`<dc:creator opf:role="AUT">Octavia Butler</dc:creator>`);
|
||||
const r = parseOpf(xml);
|
||||
expect(r.authors).toHaveLength(1);
|
||||
expect(r.authors[0].name).toBe('Octavia Butler');
|
||||
});
|
||||
|
||||
it('returns empty authors array when no creators', () => {
|
||||
const r = parseOpf(epub2Opf('<dc:title>Book</dc:title>'));
|
||||
expect(r.authors).toHaveLength(0);
|
||||
|
||||
@@ -82,6 +82,16 @@ function parseProjectxTags(raw: string | null): string[] {
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeCreatorRole(role: string | null | undefined): string {
|
||||
if (!role) return '';
|
||||
const trimmed = role.trim().toLowerCase();
|
||||
if (!trimmed) return '';
|
||||
const pathParts = trimmed.split('/');
|
||||
const withoutPath = pathParts[pathParts.length - 1] ?? trimmed;
|
||||
const colonParts = withoutPath.split(':');
|
||||
return colonParts[colonParts.length - 1] ?? withoutPath;
|
||||
}
|
||||
|
||||
// Build a map of id → array of refining meta nodes (EPUB 3).
|
||||
function buildRefineMap(metas: unknown[]): Map<string, unknown[]> {
|
||||
const map = new Map<string, unknown[]>();
|
||||
@@ -173,7 +183,7 @@ export function parseOpf(xml: string): ParsedOpf {
|
||||
const role3 = id ? getRefineValue(refineMap, id, 'role') : null;
|
||||
// EPUB 2: opf:role attribute
|
||||
const role2 = (mo['@_opf:role'] ?? mo['@_role']) as string | undefined;
|
||||
const role = role3 ?? role2 ?? 'aut';
|
||||
const role = normalizeCreatorRole(role3 ?? role2 ?? 'aut');
|
||||
if (role !== 'aut' && role !== '') continue; // skip editors, illustrators, etc.
|
||||
|
||||
// EPUB 3: file-as via refines; EPUB 2: opf:file-as attribute
|
||||
|
||||
@@ -187,6 +187,44 @@ describe('parsePdfFile', () => {
|
||||
expect(parsed?.pageCount).toBe(123);
|
||||
});
|
||||
|
||||
it('does not fall back to Info dictionary publisher/pageCount for projectx-authored PDFs when XMP is present', async () => {
|
||||
mockPdfLoad.mockResolvedValue(
|
||||
makePdfDoc({
|
||||
getCreator: () => 'projectx',
|
||||
getProducer: () => 'Legacy Publisher',
|
||||
}) as never,
|
||||
);
|
||||
mockExtractXmpXml.mockReturnValue('<xmp/>');
|
||||
mockParseXmp.mockReturnValue({
|
||||
title: 'XMP Title',
|
||||
subtitle: null,
|
||||
description: null,
|
||||
publisher: null,
|
||||
publishedYear: null,
|
||||
language: null,
|
||||
authors: [],
|
||||
genres: [],
|
||||
tags: [],
|
||||
isbn10: null,
|
||||
isbn13: null,
|
||||
seriesName: null,
|
||||
seriesIndex: null,
|
||||
rating: null,
|
||||
pageCount: null,
|
||||
googleBooksId: null,
|
||||
goodreadsId: null,
|
||||
amazonId: null,
|
||||
hardcoverId: null,
|
||||
openLibraryId: null,
|
||||
itunesId: null,
|
||||
});
|
||||
|
||||
const parsed = await parsePdfFile('/books/book.pdf');
|
||||
|
||||
expect(parsed?.publisher).toBeNull();
|
||||
expect(parsed?.pageCount).toBeNull();
|
||||
});
|
||||
|
||||
it('falls back to legacy ProjectX producer metadata when XMP publisher is absent', async () => {
|
||||
mockPdfLoad.mockResolvedValue(
|
||||
makePdfDoc({
|
||||
|
||||
@@ -83,8 +83,7 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
|
||||
}
|
||||
const doc = await PDFDocument.load(buf, { ignoreEncryption: true });
|
||||
|
||||
// XMP is the authoritative source — richer and semantically correct.
|
||||
// Info Dictionary is used only as fallback for fields XMP doesn't cover.
|
||||
// XMP is the authoritative source; Info Dictionary is fallback-only.
|
||||
const xmpXml = extractXmpXml(doc);
|
||||
const xmp: XmpParsed | null = xmpXml ? parseXmp(xmpXml) : null;
|
||||
const hasXmp = xmp !== null;
|
||||
@@ -95,6 +94,7 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
|
||||
const infoProducer = clean(doc.getProducer());
|
||||
const infoSubject = clean(doc.getSubject());
|
||||
const infoKeywords = splitCommaList(clean(doc.getKeywords()));
|
||||
const isProjectXInfo = infoCreator === PROJECTX_NS_PREFIX;
|
||||
|
||||
const infoAuthors = infoAuthorRaw
|
||||
? infoAuthorRaw
|
||||
@@ -118,18 +118,18 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
|
||||
subtitle: xmp?.subtitle ?? null,
|
||||
authors: hasXmp ? xmp.authors : infoAuthors,
|
||||
description: hasXmp ? xmp.description : infoSubject,
|
||||
publisher: xmp?.publisher ?? (infoCreator === PROJECTX_NS_PREFIX ? infoProducer : null),
|
||||
publisher: hasXmp ? (xmp.publisher ?? null) : isProjectXInfo ? infoProducer : null,
|
||||
publishedYear: xmp?.publishedYear ?? null,
|
||||
language: xmp?.language ?? null,
|
||||
genres: xmp?.genres?.length ? xmp.genres : [],
|
||||
// Info Dict keywords are genres+tags mixed — only use as tags when XMP is absent
|
||||
// Info Dict keywords are genres+tags mixed; only use as tags when XMP is absent.
|
||||
tags: hasXmp ? xmp.tags : infoKeywords,
|
||||
isbn10: xmp?.isbn10 ?? null,
|
||||
isbn13: xmp?.isbn13 ?? null,
|
||||
seriesName: xmp?.seriesName ?? null,
|
||||
seriesIndex: xmp?.seriesIndex ?? null,
|
||||
rating: xmp?.rating ?? null,
|
||||
pageCount: xmp?.pageCount ?? doc.getPageCount(),
|
||||
pageCount: hasXmp ? (xmp.pageCount ?? (isProjectXInfo ? null : doc.getPageCount())) : doc.getPageCount(),
|
||||
googleBooksId: xmp?.googleBooksId ?? null,
|
||||
goodreadsId: xmp?.goodreadsId ?? null,
|
||||
amazonId: xmp?.amazonId ?? null,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { PDFDocument, PDFName } from 'pdf-lib';
|
||||
import { PROJECTX_NS_PREFIX, PROJECTX_NS_URI } from '../../../common/projectx-ns';
|
||||
import { parseXmp } from './pdf-xmp-reader';
|
||||
import { extractXmpXml, parseXmp } from './pdf-xmp-reader';
|
||||
|
||||
// Wraps XMP content in standard RDF envelope
|
||||
function xmpDoc(body: string): string {
|
||||
@@ -250,3 +251,34 @@ describe('parseXmp', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractXmpXml', () => {
|
||||
it('decodes compressed metadata streams', async () => {
|
||||
const xmp = xmpDoc('<dc:title>Compressed title</dc:title>');
|
||||
const doc = await PDFDocument.create();
|
||||
doc.addPage([200, 200]);
|
||||
|
||||
const metadata = doc.context.flateStream(Buffer.from(xmp, 'utf-8'), {
|
||||
Type: PDFName.of('Metadata'),
|
||||
Subtype: PDFName.of('XML'),
|
||||
});
|
||||
const metadataRef = doc.context.register(metadata);
|
||||
doc.catalog.set(PDFName.of('Metadata'), metadataRef);
|
||||
|
||||
const bytes = await doc.save();
|
||||
const loaded = await PDFDocument.load(bytes, { ignoreEncryption: true });
|
||||
const xml = extractXmpXml(loaded);
|
||||
|
||||
expect(xml).toContain('<dc:title>Compressed title</dc:title>');
|
||||
});
|
||||
|
||||
it('returns null when the PDF has no metadata stream', async () => {
|
||||
const doc = await PDFDocument.create();
|
||||
doc.addPage([200, 200]);
|
||||
|
||||
const bytes = await doc.save();
|
||||
const loaded = await PDFDocument.load(bytes, { ignoreEncryption: true });
|
||||
|
||||
expect(extractXmpXml(loaded)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
import { PDFDocument, PDFName, PDFRawStream, PDFRef } from 'pdf-lib';
|
||||
import { decodePDFRawStream, PDFDocument, PDFName, PDFRawStream, PDFRef } from 'pdf-lib';
|
||||
|
||||
import { PROJECTX_NS_PREFIX } from '../../../common/projectx-ns';
|
||||
|
||||
@@ -41,7 +41,12 @@ export function extractXmpXml(doc: PDFDocument): string | null {
|
||||
if (!ref) return null;
|
||||
const stream = ref instanceof PDFRef ? doc.context.lookup(ref) : ref;
|
||||
if (!(stream instanceof PDFRawStream)) return null;
|
||||
return Buffer.from(stream.contents).toString('utf-8');
|
||||
try {
|
||||
const decoded = decodePDFRawStream(stream).decode();
|
||||
return Buffer.from(decoded).toString('utf-8');
|
||||
} catch {
|
||||
return Buffer.from(stream.contents).toString('utf-8');
|
||||
}
|
||||
}
|
||||
|
||||
// Handles plain text, rdf:Alt (Calibre-style i18n), and rdf:Seq/Bag first-item fallback.
|
||||
|
||||
@@ -20,6 +20,7 @@ import type { FastifyReply } from 'fastify';
|
||||
import { bookCoverDirPath, bookThumbnailPath, findPreferredBookCoverFileName } from '../../common/book-cover-storage';
|
||||
import { MAX_OFFSET_ROWS, isOffsetWithinLimit } from '../../common/constants/pagination.constants';
|
||||
import { Public } from '../../common/decorators/public.decorator';
|
||||
import { imageContentTypeFromPath } from '../../common/image-content-type';
|
||||
import { OPDS_MIME_ACQ, OPDS_MIME_NAV, OPDS_MIME_SEARCH, fileMimeType } from './opds-xml.helpers';
|
||||
import { OpdsAuthGuard } from './opds-auth.guard';
|
||||
import type { OpdsRequestUser } from './opds-auth.guard';
|
||||
@@ -215,10 +216,9 @@ export class OpdsController {
|
||||
reply.status(304).send();
|
||||
return;
|
||||
}
|
||||
const ext = cover.split('.').pop()?.toLowerCase();
|
||||
reply.header('Cache-Control', 'no-cache');
|
||||
reply.header('ETag', etag);
|
||||
reply.type(ext === 'png' ? 'image/png' : 'image/jpeg');
|
||||
reply.type(imageContentTypeFromPath(coverPath));
|
||||
reply.send(createReadStream(coverPath));
|
||||
} catch {
|
||||
throw new NotFoundException('No cover');
|
||||
|
||||
@@ -5,6 +5,7 @@ import { readFile } from 'fs/promises';
|
||||
import { createInflateRaw } from 'zlib';
|
||||
import { createExtractorFromData } from 'node-unrar-js';
|
||||
import { getSevenZip } from '../../../common/sevenzip';
|
||||
import { imageContentTypeFromPath } from '../../../common/image-content-type';
|
||||
|
||||
import type { RequestUser } from '../../../common/types/request-user';
|
||||
import { BookService } from '../../book/book.service';
|
||||
@@ -27,10 +28,7 @@ function naturalSort(a: string, b: string): number {
|
||||
}
|
||||
|
||||
function mimeForExt(name: string): string {
|
||||
const ext = name.substring(name.lastIndexOf('.')).toLowerCase();
|
||||
if (ext === '.png') return 'image/png';
|
||||
if (ext === '.webp') return 'image/webp';
|
||||
return 'image/jpeg';
|
||||
return imageContentTypeFromPath(name);
|
||||
}
|
||||
|
||||
// ── CBZ: ZIP byte-offset scan ──────────────────────────────────────────────────
|
||||
|
||||
@@ -5,7 +5,7 @@ import { tmpdir } from 'os';
|
||||
import { fingerprintFile } from './hash';
|
||||
|
||||
const EMPTY_MD5 = 'd41d8cd98f00b204e9800998ecf8427e';
|
||||
// 1 KB = 1024 bytes. The first read position is BASE (1024), so files <= 1024 bytes produce empty hash.
|
||||
// 1 KB = 1024 bytes. The first sampled offset is BASE (1024).
|
||||
const FIRST_READ_POSITION = 1024;
|
||||
|
||||
let tmpDir: string;
|
||||
@@ -27,11 +27,22 @@ describe('files smaller than the first read position', () => {
|
||||
expect(hash).toBe(EMPTY_MD5);
|
||||
});
|
||||
|
||||
it('returns MD5 of empty string for a file with exactly 1024 bytes', async () => {
|
||||
// Position 1024 >= size 1024 → no read
|
||||
it('falls back to hashing from byte 0 for a file with exactly 1024 bytes', async () => {
|
||||
await writeFile(join(tmpDir, 'exact.epub'), Buffer.alloc(FIRST_READ_POSITION, 'A'));
|
||||
const hash = await fingerprintFile(join(tmpDir, 'exact.epub'));
|
||||
expect(hash).toBe(EMPTY_MD5);
|
||||
expect(hash).not.toBe(EMPTY_MD5);
|
||||
});
|
||||
|
||||
it('produces different fingerprints for different tiny files', async () => {
|
||||
const a = Buffer.alloc(900, 'A');
|
||||
const b = Buffer.alloc(900, 'A');
|
||||
b.write('Z', 0, 'ascii');
|
||||
|
||||
await writeFile(join(tmpDir, 'a.epub'), a);
|
||||
await writeFile(join(tmpDir, 'b.epub'), b);
|
||||
|
||||
const [h1, h2] = await Promise.all([fingerprintFile(join(tmpDir, 'a.epub')), fingerprintFile(join(tmpDir, 'b.epub'))]);
|
||||
expect(h1).not.toBe(h2);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -8,6 +8,8 @@ const BLOCK_SIZE = 1024;
|
||||
* Partial MD5 fingerprint — reads 1 KB blocks from exponentially spaced positions
|
||||
* (1 KB, 4 KB, 16 KB, … up to 1 GB) rather than the entire file.
|
||||
* Used only when path and inode lookups both fail (cross-filesystem move detection).
|
||||
* For files too small to reach the first sample offset, fall back to the first
|
||||
* 1 KB so tiny files do not collapse to the same empty hash.
|
||||
*/
|
||||
export async function fingerprintFile(absolutePath: string): Promise<string> {
|
||||
const fh = await open(absolutePath, 'r');
|
||||
@@ -15,12 +17,24 @@ export async function fingerprintFile(absolutePath: string): Promise<string> {
|
||||
const { size } = await fh.stat();
|
||||
const hash = createHash('md5');
|
||||
const buf = Buffer.allocUnsafe(BLOCK_SIZE);
|
||||
let sampled = false;
|
||||
|
||||
for (let i = 0; i <= 10; i++) {
|
||||
const position = BASE << (2 * i);
|
||||
if (position >= size) break;
|
||||
const { bytesRead } = await fh.read(buf, 0, BLOCK_SIZE, position);
|
||||
if (bytesRead > 0) hash.update(buf.subarray(0, bytesRead));
|
||||
if (bytesRead > 0) {
|
||||
hash.update(buf.subarray(0, bytesRead));
|
||||
sampled = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!sampled && size > 0) {
|
||||
const readLength = Math.min(BLOCK_SIZE, size);
|
||||
const { bytesRead } = await fh.read(buf, 0, readLength, 0);
|
||||
if (bytesRead > 0) {
|
||||
hash.update(buf.subarray(0, bytesRead));
|
||||
}
|
||||
}
|
||||
|
||||
return hash.digest('hex');
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { Permission } from '@projectx/types';
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
|
||||
import { UserController } from './user.controller';
|
||||
import { MAX_USER_AVATAR_BYTES } from './user-avatar.service';
|
||||
|
||||
describe('UserController', () => {
|
||||
const userService = {
|
||||
@@ -40,6 +42,39 @@ describe('UserController', () => {
|
||||
expect(userService.updateMe).toHaveBeenCalledWith(7, dto);
|
||||
});
|
||||
|
||||
it('uploads avatar bytes with multipart file limits', async () => {
|
||||
const user = { id: 7 } as any;
|
||||
const buffer = Buffer.from('img');
|
||||
const req = {
|
||||
file: vi.fn().mockResolvedValue({
|
||||
mimetype: 'image/png',
|
||||
toBuffer: vi.fn().mockResolvedValue(buffer),
|
||||
}),
|
||||
} as any;
|
||||
|
||||
await controller.uploadMyAvatar(user, req);
|
||||
|
||||
expect(req.file).toHaveBeenCalledWith({ limits: { fileSize: MAX_USER_AVATAR_BYTES } });
|
||||
expect(userAvatarService.uploadOwnAvatar).toHaveBeenCalledWith(user, buffer, 'image/png');
|
||||
});
|
||||
|
||||
it('maps multipart file-size errors to a bad request response', async () => {
|
||||
const tooLargeError = Object.assign(new Error('request file too large'), {
|
||||
code: 'FST_REQ_FILE_TOO_LARGE',
|
||||
statusCode: 413,
|
||||
});
|
||||
const req = {
|
||||
file: vi.fn().mockResolvedValue({
|
||||
mimetype: 'image/jpeg',
|
||||
toBuffer: vi.fn().mockRejectedValue(tooLargeError),
|
||||
}),
|
||||
} as any;
|
||||
|
||||
await expect(controller.uploadMyAvatar({ id: 7 } as any, req)).rejects.toThrow(BadRequestException);
|
||||
await expect(controller.uploadMyAvatar({ id: 7 } as any, req)).rejects.toThrow('Image exceeds 5 MB limit');
|
||||
expect(userAvatarService.uploadOwnAvatar).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('delegates permission and superuser management and admin reset', async () => {
|
||||
const requester = { id: 1 } as any;
|
||||
const dto = { permissionNames: [Permission.LibraryDownload] };
|
||||
|
||||
@@ -79,9 +79,25 @@ export class UserController {
|
||||
description: () => 'User uploaded a profile picture',
|
||||
})
|
||||
async uploadMyAvatar(@CurrentUser() user: RequestUser, @Req() req: MultipartRequest) {
|
||||
const data = await req.file({ limits: { fileSize: MAX_USER_AVATAR_BYTES } });
|
||||
let data;
|
||||
try {
|
||||
data = await req.file({ limits: { fileSize: MAX_USER_AVATAR_BYTES } });
|
||||
} catch (error) {
|
||||
if (isMultipartFileTooLargeError(error)) {
|
||||
throw new BadRequestException('Image exceeds 5 MB limit');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (!data) throw new BadRequestException('No file provided');
|
||||
const buffer = await data.toBuffer();
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = await data.toBuffer();
|
||||
} catch (error) {
|
||||
if (isMultipartFileTooLargeError(error)) {
|
||||
throw new BadRequestException('Image exceeds 5 MB limit');
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return this.userAvatarService.uploadOwnAvatar(user, buffer, data.mimetype);
|
||||
}
|
||||
|
||||
@@ -230,3 +246,14 @@ export class UserController {
|
||||
return this.userService.adminResetPassword(id, requestingUser);
|
||||
}
|
||||
}
|
||||
|
||||
function isMultipartFileTooLargeError(error: unknown): boolean {
|
||||
if (typeof error !== 'object' || error === null) {
|
||||
return false;
|
||||
}
|
||||
const value = error as { code?: unknown; statusCode?: unknown; message?: unknown };
|
||||
if (value.code === 'FST_REQ_FILE_TOO_LARGE') {
|
||||
return true;
|
||||
}
|
||||
return value.statusCode === 413 && typeof value.message === 'string' && value.message.toLowerCase().includes('file too large');
|
||||
}
|
||||
|
||||
@@ -235,24 +235,42 @@ export async function waitForScanCompletion(db: Db, jobId: number, timeoutMs = 3
|
||||
}
|
||||
|
||||
export async function locateBookFileByRelPath(ctx: MetadataWriteE2EContext, libraryId: number, relPath: string): Promise<LocatedBookFile> {
|
||||
const [row] = await ctx.db
|
||||
.select({
|
||||
bookId: books.id,
|
||||
bookFileId: bookFiles.id,
|
||||
absolutePath: bookFiles.absolutePath,
|
||||
relPath: bookFiles.relPath,
|
||||
format: bookFiles.format,
|
||||
})
|
||||
.from(bookFiles)
|
||||
.innerJoin(books, eq(books.id, bookFiles.bookId))
|
||||
.where(and(eq(books.libraryId, libraryId), eq(bookFiles.relPath, relPath), eq(books.status, 'present')))
|
||||
.limit(1);
|
||||
const query = async (): Promise<LocatedBookFile | null> => {
|
||||
const [row] = await ctx.db
|
||||
.select({
|
||||
bookId: books.id,
|
||||
bookFileId: bookFiles.id,
|
||||
absolutePath: bookFiles.absolutePath,
|
||||
relPath: bookFiles.relPath,
|
||||
format: bookFiles.format,
|
||||
})
|
||||
.from(bookFiles)
|
||||
.innerJoin(books, eq(books.id, bookFiles.bookId))
|
||||
.where(and(eq(books.libraryId, libraryId), eq(bookFiles.relPath, relPath), eq(books.status, 'present')))
|
||||
.limit(1);
|
||||
return row ?? null;
|
||||
};
|
||||
|
||||
if (!row) {
|
||||
throw new Error(`No present book file found for relPath "${relPath}" in library ${libraryId}`);
|
||||
}
|
||||
const waitForRow = async (timeoutMs: number): Promise<LocatedBookFile | null> => {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const row = await query();
|
||||
if (row) return row;
|
||||
await sleep(100);
|
||||
}
|
||||
return query();
|
||||
};
|
||||
|
||||
return row;
|
||||
const initial = await waitForRow(2_000);
|
||||
if (initial) return initial;
|
||||
|
||||
// In CI and shared dev environments, fixture writes can occasionally race the
|
||||
// first scan trigger. Perform one recovery scan before failing hard.
|
||||
await triggerAndWaitForLibraryScan(ctx, libraryId);
|
||||
const recovered = await waitForRow(2_000);
|
||||
if (recovered) return recovered;
|
||||
|
||||
throw new Error(`No present book file found for relPath "${relPath}" in library ${libraryId}`);
|
||||
}
|
||||
|
||||
export async function getLatestWriteLogEntry(db: Db, bookId: number, triggeredBy?: 'auto' | 'sync') {
|
||||
|
||||
Reference in New Issue
Block a user