refactor(metadata): improve parsing, cover storage, and content type

This commit is contained in:
neon
2026-04-10 00:26:45 -06:00
parent 3fb99714d7
commit 698951a245
27 changed files with 387 additions and 56 deletions
+3 -1
View File
@@ -22,9 +22,11 @@ describe('book-cover-storage', () => {
expect(isExtractedBookCoverFileName('cover_custom.jpg')).toBe(false);
});
it('prefers custom covers over extracted covers', () => {
it('prefers custom and extracted covers over legacy cover files', () => {
expect(findPreferredBookCoverFileName(['thumbnail.jpg', 'cover_extracted.jpg', 'cover_custom.png'])).toBe('cover_custom.png');
expect(findPreferredBookCoverFileName(['thumbnail.jpg', 'cover_extracted.jpg'])).toBe('cover_extracted.jpg');
expect(findPreferredBookCoverFileName(['thumbnail.jpg', 'cover.jpg'])).toBe('cover.jpg');
expect(findPreferredBookCoverFileName(['thumbnail.jpg', 'cover.png'])).toBe('cover.png');
expect(findPreferredBookCoverFileName(['thumbnail.jpg'])).toBeNull();
});
+6 -1
View File
@@ -2,6 +2,7 @@ import { join } from 'path';
export const COVER_CUSTOM_FILE_PREFIX = 'cover_custom.';
export const COVER_EXTRACTED_FILE_PREFIX = 'cover_extracted.';
export const COVER_LEGACY_FILE_PREFIX = 'cover.';
export const COVER_THUMBNAIL_FILE_NAME = 'thumbnail.jpg';
export function bookCoverDirPath(appDataPath: string, bookId: number): string {
@@ -21,9 +22,13 @@ export function isExtractedBookCoverFileName(fileName: string): boolean {
}
export function findPreferredBookCoverFileName(files: readonly string[]): string | null {
return files.find(isCustomBookCoverFileName) ?? files.find(isExtractedBookCoverFileName) ?? null;
return files.find(isCustomBookCoverFileName) ?? files.find(isExtractedBookCoverFileName) ?? files.find(isLegacyBookCoverFileName) ?? null;
}
export function findExtractedBookCoverFileName(files: readonly string[]): string | null {
return files.find(isExtractedBookCoverFileName) ?? null;
}
function isLegacyBookCoverFileName(fileName: string): boolean {
return fileName.startsWith(COVER_LEGACY_FILE_PREFIX);
}
@@ -0,0 +1,28 @@
import { imageContentTypeFromExtension, imageContentTypeFromPath } from './image-content-type';
describe('image content type helpers', () => {
describe('imageContentTypeFromPath', () => {
it('returns image/png for png file paths', () => {
expect(imageContentTypeFromPath('/tmp/cover.png')).toBe('image/png');
});
it('returns image/webp for webp file paths', () => {
expect(imageContentTypeFromPath('/tmp/cover.webp')).toBe('image/webp');
});
it('falls back to image/jpeg for unknown extensions', () => {
expect(imageContentTypeFromPath('/tmp/cover.unknown')).toBe('image/jpeg');
});
});
describe('imageContentTypeFromExtension', () => {
it('normalizes extension case', () => {
expect(imageContentTypeFromExtension('.PNG')).toBe('image/png');
});
it('returns image/jpeg for jpeg aliases', () => {
expect(imageContentTypeFromExtension('.jpg')).toBe('image/jpeg');
expect(imageContentTypeFromExtension('.jpeg')).toBe('image/jpeg');
});
});
});
+19
View File
@@ -0,0 +1,19 @@
import { extname } from 'path';
const IMAGE_CONTENT_TYPE_BY_EXT: Record<string, string> = {
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.png': 'image/png',
'.gif': 'image/gif',
'.webp': 'image/webp',
'.bmp': 'image/bmp',
'.avif': 'image/avif',
};
export function imageContentTypeFromPath(path: string): string {
return imageContentTypeFromExtension(extname(path));
}
export function imageContentTypeFromExtension(extension: string): string {
return IMAGE_CONTENT_TYPE_BY_EXT[extension.toLowerCase()] ?? 'image/jpeg';
}
@@ -19,13 +19,13 @@ import {
import { createReadStream } from 'fs';
import { stat } from 'fs/promises';
import type { FastifyReply } from 'fastify';
import { extname } from 'path';
import { map, Observable } from 'rxjs';
import { Permission, AuditAction, AuditResource } from '@projectx/types';
import { CurrentUser } from '../../common/decorators/current-user.decorator';
import { Auditable } from '../../common/decorators/auditable.decorator';
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
import { imageContentTypeFromPath } from '../../common/image-content-type';
import type { RequestUser } from '../../common/types/request-user';
import type { AuthorMetadataCandidate } from '@projectx/types';
import { AuthorEnrichmentConfigService } from './author-enrichment-config.service';
@@ -228,8 +228,7 @@ export class AuthorsController {
reply.header('Cache-Control', 'no-cache');
reply.header('ETag', `"${Math.floor(mtimeMs)}"`);
const ext = extname(imagePath).toLowerCase();
reply.type(ext === '.png' ? 'image/png' : 'image/jpeg');
reply.type(imageContentTypeFromPath(imagePath));
reply.send(createReadStream(imagePath));
}
@@ -26,6 +26,7 @@ import { AuditAction, AuditResource } from '@projectx/types';
import { CurrentUser } from '../../common/decorators/current-user.decorator';
import { Auditable } from '../../common/decorators/auditable.decorator';
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
import { imageContentTypeFromPath } from '../../common/image-content-type';
import type { MultipartRequest } from '../../common/types/multipart-request';
import type { RequestUser } from '../../common/types/request-user';
import { BookBucketService } from './book-bucket.service';
@@ -96,7 +97,7 @@ export class BookBucketController {
if (!exists) throw new NotFoundException('Cover file not found on disk');
const stream = createReadStream(row.coverPath);
const contentType = row.coverPath.endsWith('.png') ? 'image/png' : 'image/jpeg';
const contentType = imageContentTypeFromPath(row.coverPath);
reply.header('Content-Type', contentType);
reply.header('Cache-Control', 'private, max-age=3600');
return reply.send(stream);
+2 -2
View File
@@ -23,6 +23,7 @@ import type { FastifyReply } from 'fastify';
import { CurrentUser } from '../../common/decorators/current-user.decorator';
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
import { Auditable } from '../../common/decorators/auditable.decorator';
import { imageContentTypeFromPath } from '../../common/image-content-type';
import type { RequestUser } from '../../common/types/request-user';
import { FileWriteService } from '../file-write/file-write.service';
import { BookService } from './book.service';
@@ -297,8 +298,7 @@ export class BookController {
return;
}
const ext = coverPath.split('.').pop()?.toLowerCase();
const contentType = ext === 'png' ? 'image/png' : 'image/jpeg';
const contentType = imageContentTypeFromPath(coverPath);
reply.header('Cache-Control', 'private, max-age=86400');
reply.header('ETag', etag);
reply.type(contentType);
@@ -5,6 +5,7 @@ import { createReadStream } from 'fs';
import { join } from 'path';
import type { FastifyReply } from 'fastify';
import { bookCoverDirPath, bookThumbnailPath, findPreferredBookCoverFileName } from '../../../common/book-cover-storage';
import { imageContentTypeFromPath } from '../../../common/image-content-type';
import { KoboBookAccessService } from './kobo-book-access.service';
@Injectable()
@@ -51,10 +52,9 @@ export class KoboThumbnailService {
reply.status(304).send();
return;
}
const ext = cover.split('.').pop()?.toLowerCase();
reply.header('Cache-Control', 'max-age=86400');
reply.header('ETag', etag);
reply.type(ext === 'png' ? 'image/png' : 'image/jpeg');
reply.type(imageContentTypeFromPath(coverPath));
reply.send(createReadStream(coverPath));
} catch {
throw new NotFoundException('No cover image');
@@ -151,6 +151,22 @@ describe('extractCbzMetadata', () => {
expect(r?.description).toBe('Watchmen summary.');
});
it('maps CommunityRating from 0-5 scale into 0-10 scale', async () => {
const xml = `<ComicInfo><CommunityRating>4.5</CommunityRating></ComicInfo>`;
mockReadFile.mockResolvedValue(buildZipWithComicInfo(xml) as unknown as Buffer);
const r = await extractCbzMetadata('/book.cbz');
expect(r?.rating).toBe(9);
});
it('clamps CommunityRating to a maximum of 10 after scaling', async () => {
const xml = `<ComicInfo><CommunityRating>8</CommunityRating></ComicInfo>`;
mockReadFile.mockResolvedValue(buildZipWithComicInfo(xml) as unknown as Buffer);
const r = await extractCbzMetadata('/book.cbz');
expect(r?.rating).toBe(10);
});
it('truncates fractional year to integer', async () => {
const xml = `<ComicInfo><Year>1986.5</Year></ComicInfo>`;
mockReadFile.mockResolvedValue(buildZipWithComicInfo(xml) as unknown as Buffer);
@@ -180,7 +180,7 @@ function parseCbxRating(value: string | null): number | null {
if (!value) return null;
const parsed = Number(value);
if (!Number.isFinite(parsed)) return null;
return Math.min(5, Math.max(0, parsed * 2));
return Math.min(10, Math.max(0, parsed * 2));
}
function parseIsbn13(value: string | null): string | null {
+14 -2
View File
@@ -60,8 +60,20 @@ describe('imageExt', () => {
expect(imageExt(Buffer.from([0xff, 0xd8, 0xff, 0xe0]))).toBe('jpg');
});
it('returns "jpg" as default for unknown magic bytes (GIF)', () => {
expect(imageExt(Buffer.from([0x47, 0x49, 0x46, 0x38]))).toBe('jpg');
it('returns "gif" for GIF magic bytes (GIF8)', () => {
expect(imageExt(Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]))).toBe('gif');
});
it('returns "webp" for RIFF/WEBP magic bytes', () => {
expect(imageExt(Buffer.from([0x52, 0x49, 0x46, 0x46, 0x24, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50]))).toBe('webp');
});
it('returns "bmp" for BMP magic bytes (BM)', () => {
expect(imageExt(Buffer.from([0x42, 0x4d, 0x00, 0x00]))).toBe('bmp');
});
it('returns "jpg" as default for unknown magic bytes', () => {
expect(imageExt(Buffer.from([0x00, 0x11, 0x22, 0x33]))).toBe('jpg');
});
it('returns "jpg" when first byte is 0x89 but second byte is not 0x50', () => {
+17 -1
View File
@@ -14,7 +14,23 @@ const THUMBNAIL_HEIGHT = 600;
/** Returns extension based on magic bytes, defaulting to 'jpg'. */
export function imageExt(bytes: Buffer): string {
if (bytes[0] === 0x89 && bytes[1] === 0x50) return 'png';
if (bytes.length >= 4 && bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47) return 'png';
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'jpg';
if (bytes.length >= 4 && bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46 && bytes[3] === 0x38) return 'gif';
if (
bytes.length >= 12 &&
bytes[0] === 0x52 &&
bytes[1] === 0x49 &&
bytes[2] === 0x46 &&
bytes[3] === 0x46 &&
bytes[8] === 0x57 &&
bytes[9] === 0x45 &&
bytes[10] === 0x42 &&
bytes[11] === 0x50
) {
return 'webp';
}
if (bytes.length >= 2 && bytes[0] === 0x42 && bytes[1] === 0x4d) return 'bmp';
return 'jpg';
}
@@ -163,6 +163,11 @@ describe('parseFb2File', () => {
mockReadFile.mockResolvedValue(makeFb2('<book-title>Book</book-title><date>1984</date>') as unknown as Buffer);
expect((await parseFb2File('/book.fb2'))?.publishedYear).toBe(1984);
});
it('parses year from title-info/date value attribute', async () => {
mockReadFile.mockResolvedValue(makeFb2('<book-title>Book</book-title><date value="1984-01-01"/>') as unknown as Buffer);
expect((await parseFb2File('/book.fb2'))?.publishedYear).toBe(1984);
});
});
describe('description (annotation)', () => {
+27 -4
View File
@@ -11,6 +11,11 @@ const parser = new XMLParser({
function text(val: unknown): string | null {
if (typeof val === 'string') return val.trim() || null;
if (typeof val === 'number') return String(val);
if (typeof val === 'object' && val !== null && '#text' in val) {
const textNode = (val as Record<string, unknown>)['#text'];
if (typeof textNode === 'string') return textNode.trim() || null;
if (typeof textNode === 'number') return String(textNode);
}
return null;
}
@@ -39,6 +44,27 @@ function extractAnnotationText(val: unknown): string {
return '';
}
function parseFb2YearNode(val: unknown): number | null {
const candidates: string[] = [];
const direct = text(val);
if (direct) candidates.push(direct);
if (typeof val === 'object' && val !== null) {
const objectNode = val as Record<string, unknown>;
const valueAttr = text(objectNode['@_value']);
if (valueAttr) candidates.push(valueAttr);
}
for (const candidate of candidates) {
const match = candidate.match(/(\d{4})/);
if (!match) continue;
const year = parseInt(match[1], 10);
if (!isNaN(year) && year > 1000 && year < 2200) return year;
}
return null;
}
export interface Fb2Metadata {
title: string | null;
description: string | null;
@@ -107,10 +133,7 @@ export async function parseFb2File(absolutePath: string): Promise<Fb2Metadata |
let publishedYear: number | null = null;
const publishInfo = description?.['publish-info'] as Record<string, unknown> | undefined;
const yearRaw = publishInfo?.['year'] ?? titleInfo['date'];
if (typeof yearRaw === 'string' || typeof yearRaw === 'number') {
const y = parseInt(String(yearRaw), 10);
if (!isNaN(y) && y > 1000 && y < 2200) publishedYear = y;
}
publishedYear = parseFb2YearNode(yearRaw);
// Annotation (description)
let annotationDescription: string | null = null;
@@ -108,6 +108,23 @@ describe('parseOpf', () => {
expect(r.authors[0].sortName).toBe('Pratchett, Terry');
});
it('accepts EPUB3 role values in full relator URI form', () => {
const xml = epub3Opf(`
<dc:creator id="cr1">Ursula Le Guin</dc:creator>
<meta refines="#cr1" property="role">http://id.loc.gov/vocabulary/relators/aut</meta>
`);
const r = parseOpf(xml);
expect(r.authors).toHaveLength(1);
expect(r.authors[0].name).toBe('Ursula Le Guin');
});
it('treats uppercase EPUB2 role codes as authors', () => {
const xml = epub2Opf(`<dc:creator opf:role="AUT">Octavia Butler</dc:creator>`);
const r = parseOpf(xml);
expect(r.authors).toHaveLength(1);
expect(r.authors[0].name).toBe('Octavia Butler');
});
it('returns empty authors array when no creators', () => {
const r = parseOpf(epub2Opf('<dc:title>Book</dc:title>'));
expect(r.authors).toHaveLength(0);
+11 -1
View File
@@ -82,6 +82,16 @@ function parseProjectxTags(raw: string | null): string[] {
}
}
function normalizeCreatorRole(role: string | null | undefined): string {
if (!role) return '';
const trimmed = role.trim().toLowerCase();
if (!trimmed) return '';
const pathParts = trimmed.split('/');
const withoutPath = pathParts[pathParts.length - 1] ?? trimmed;
const colonParts = withoutPath.split(':');
return colonParts[colonParts.length - 1] ?? withoutPath;
}
// Build a map of id → array of refining meta nodes (EPUB 3).
function buildRefineMap(metas: unknown[]): Map<string, unknown[]> {
const map = new Map<string, unknown[]>();
@@ -173,7 +183,7 @@ export function parseOpf(xml: string): ParsedOpf {
const role3 = id ? getRefineValue(refineMap, id, 'role') : null;
// EPUB 2: opf:role attribute
const role2 = (mo['@_opf:role'] ?? mo['@_role']) as string | undefined;
const role = role3 ?? role2 ?? 'aut';
const role = normalizeCreatorRole(role3 ?? role2 ?? 'aut');
if (role !== 'aut' && role !== '') continue; // skip editors, illustrators, etc.
// EPUB 3: file-as via refines; EPUB 2: opf:file-as attribute
@@ -187,6 +187,44 @@ describe('parsePdfFile', () => {
expect(parsed?.pageCount).toBe(123);
});
it('does not fall back to Info dictionary publisher/pageCount for projectx-authored PDFs when XMP is present', async () => {
mockPdfLoad.mockResolvedValue(
makePdfDoc({
getCreator: () => 'projectx',
getProducer: () => 'Legacy Publisher',
}) as never,
);
mockExtractXmpXml.mockReturnValue('<xmp/>');
mockParseXmp.mockReturnValue({
title: 'XMP Title',
subtitle: null,
description: null,
publisher: null,
publishedYear: null,
language: null,
authors: [],
genres: [],
tags: [],
isbn10: null,
isbn13: null,
seriesName: null,
seriesIndex: null,
rating: null,
pageCount: null,
googleBooksId: null,
goodreadsId: null,
amazonId: null,
hardcoverId: null,
openLibraryId: null,
itunesId: null,
});
const parsed = await parsePdfFile('/books/book.pdf');
expect(parsed?.publisher).toBeNull();
expect(parsed?.pageCount).toBeNull();
});
it('falls back to legacy ProjectX producer metadata when XMP publisher is absent', async () => {
mockPdfLoad.mockResolvedValue(
makePdfDoc({
@@ -83,8 +83,7 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
}
const doc = await PDFDocument.load(buf, { ignoreEncryption: true });
// XMP is the authoritative source — richer and semantically correct.
// Info Dictionary is used only as fallback for fields XMP doesn't cover.
// XMP is the authoritative source; Info Dictionary is fallback-only.
const xmpXml = extractXmpXml(doc);
const xmp: XmpParsed | null = xmpXml ? parseXmp(xmpXml) : null;
const hasXmp = xmp !== null;
@@ -95,6 +94,7 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
const infoProducer = clean(doc.getProducer());
const infoSubject = clean(doc.getSubject());
const infoKeywords = splitCommaList(clean(doc.getKeywords()));
const isProjectXInfo = infoCreator === PROJECTX_NS_PREFIX;
const infoAuthors = infoAuthorRaw
? infoAuthorRaw
@@ -118,18 +118,18 @@ export async function parsePdfFile(absolutePath: string, options: PdfParseOption
subtitle: xmp?.subtitle ?? null,
authors: hasXmp ? xmp.authors : infoAuthors,
description: hasXmp ? xmp.description : infoSubject,
publisher: xmp?.publisher ?? (infoCreator === PROJECTX_NS_PREFIX ? infoProducer : null),
publisher: hasXmp ? (xmp.publisher ?? null) : isProjectXInfo ? infoProducer : null,
publishedYear: xmp?.publishedYear ?? null,
language: xmp?.language ?? null,
genres: xmp?.genres?.length ? xmp.genres : [],
// Info Dict keywords are genres+tags mixed — only use as tags when XMP is absent
// Info Dict keywords are genres+tags mixed; only use as tags when XMP is absent.
tags: hasXmp ? xmp.tags : infoKeywords,
isbn10: xmp?.isbn10 ?? null,
isbn13: xmp?.isbn13 ?? null,
seriesName: xmp?.seriesName ?? null,
seriesIndex: xmp?.seriesIndex ?? null,
rating: xmp?.rating ?? null,
pageCount: xmp?.pageCount ?? doc.getPageCount(),
pageCount: hasXmp ? (xmp.pageCount ?? (isProjectXInfo ? null : doc.getPageCount())) : doc.getPageCount(),
googleBooksId: xmp?.googleBooksId ?? null,
goodreadsId: xmp?.goodreadsId ?? null,
amazonId: xmp?.amazonId ?? null,
@@ -1,5 +1,6 @@
import { PDFDocument, PDFName } from 'pdf-lib';
import { PROJECTX_NS_PREFIX, PROJECTX_NS_URI } from '../../../common/projectx-ns';
import { parseXmp } from './pdf-xmp-reader';
import { extractXmpXml, parseXmp } from './pdf-xmp-reader';
// Wraps XMP content in standard RDF envelope
function xmpDoc(body: string): string {
@@ -250,3 +251,34 @@ describe('parseXmp', () => {
});
});
});
describe('extractXmpXml', () => {
it('decodes compressed metadata streams', async () => {
const xmp = xmpDoc('<dc:title>Compressed title</dc:title>');
const doc = await PDFDocument.create();
doc.addPage([200, 200]);
const metadata = doc.context.flateStream(Buffer.from(xmp, 'utf-8'), {
Type: PDFName.of('Metadata'),
Subtype: PDFName.of('XML'),
});
const metadataRef = doc.context.register(metadata);
doc.catalog.set(PDFName.of('Metadata'), metadataRef);
const bytes = await doc.save();
const loaded = await PDFDocument.load(bytes, { ignoreEncryption: true });
const xml = extractXmpXml(loaded);
expect(xml).toContain('<dc:title>Compressed title</dc:title>');
});
it('returns null when the PDF has no metadata stream', async () => {
const doc = await PDFDocument.create();
doc.addPage([200, 200]);
const bytes = await doc.save();
const loaded = await PDFDocument.load(bytes, { ignoreEncryption: true });
expect(extractXmpXml(loaded)).toBeNull();
});
});
@@ -1,5 +1,5 @@
import { XMLParser } from 'fast-xml-parser';
import { PDFDocument, PDFName, PDFRawStream, PDFRef } from 'pdf-lib';
import { decodePDFRawStream, PDFDocument, PDFName, PDFRawStream, PDFRef } from 'pdf-lib';
import { PROJECTX_NS_PREFIX } from '../../../common/projectx-ns';
@@ -41,7 +41,12 @@ export function extractXmpXml(doc: PDFDocument): string | null {
if (!ref) return null;
const stream = ref instanceof PDFRef ? doc.context.lookup(ref) : ref;
if (!(stream instanceof PDFRawStream)) return null;
return Buffer.from(stream.contents).toString('utf-8');
try {
const decoded = decodePDFRawStream(stream).decode();
return Buffer.from(decoded).toString('utf-8');
} catch {
return Buffer.from(stream.contents).toString('utf-8');
}
}
// Handles plain text, rdf:Alt (Calibre-style i18n), and rdf:Seq/Bag first-item fallback.
+2 -2
View File
@@ -20,6 +20,7 @@ import type { FastifyReply } from 'fastify';
import { bookCoverDirPath, bookThumbnailPath, findPreferredBookCoverFileName } from '../../common/book-cover-storage';
import { MAX_OFFSET_ROWS, isOffsetWithinLimit } from '../../common/constants/pagination.constants';
import { Public } from '../../common/decorators/public.decorator';
import { imageContentTypeFromPath } from '../../common/image-content-type';
import { OPDS_MIME_ACQ, OPDS_MIME_NAV, OPDS_MIME_SEARCH, fileMimeType } from './opds-xml.helpers';
import { OpdsAuthGuard } from './opds-auth.guard';
import type { OpdsRequestUser } from './opds-auth.guard';
@@ -215,10 +216,9 @@ export class OpdsController {
reply.status(304).send();
return;
}
const ext = cover.split('.').pop()?.toLowerCase();
reply.header('Cache-Control', 'no-cache');
reply.header('ETag', etag);
reply.type(ext === 'png' ? 'image/png' : 'image/jpeg');
reply.type(imageContentTypeFromPath(coverPath));
reply.send(createReadStream(coverPath));
} catch {
throw new NotFoundException('No cover');
+2 -4
View File
@@ -5,6 +5,7 @@ import { readFile } from 'fs/promises';
import { createInflateRaw } from 'zlib';
import { createExtractorFromData } from 'node-unrar-js';
import { getSevenZip } from '../../../common/sevenzip';
import { imageContentTypeFromPath } from '../../../common/image-content-type';
import type { RequestUser } from '../../../common/types/request-user';
import { BookService } from '../../book/book.service';
@@ -27,10 +28,7 @@ function naturalSort(a: string, b: string): number {
}
function mimeForExt(name: string): string {
const ext = name.substring(name.lastIndexOf('.')).toLowerCase();
if (ext === '.png') return 'image/png';
if (ext === '.webp') return 'image/webp';
return 'image/jpeg';
return imageContentTypeFromPath(name);
}
// ── CBZ: ZIP byte-offset scan ──────────────────────────────────────────────────
+15 -4
View File
@@ -5,7 +5,7 @@ import { tmpdir } from 'os';
import { fingerprintFile } from './hash';
const EMPTY_MD5 = 'd41d8cd98f00b204e9800998ecf8427e';
// 1 KB = 1024 bytes. The first read position is BASE (1024), so files <= 1024 bytes produce empty hash.
// 1 KB = 1024 bytes. The first sampled offset is BASE (1024).
const FIRST_READ_POSITION = 1024;
let tmpDir: string;
@@ -27,11 +27,22 @@ describe('files smaller than the first read position', () => {
expect(hash).toBe(EMPTY_MD5);
});
it('returns MD5 of empty string for a file with exactly 1024 bytes', async () => {
// Position 1024 >= size 1024 → no read
it('falls back to hashing from byte 0 for a file with exactly 1024 bytes', async () => {
await writeFile(join(tmpDir, 'exact.epub'), Buffer.alloc(FIRST_READ_POSITION, 'A'));
const hash = await fingerprintFile(join(tmpDir, 'exact.epub'));
expect(hash).toBe(EMPTY_MD5);
expect(hash).not.toBe(EMPTY_MD5);
});
it('produces different fingerprints for different tiny files', async () => {
const a = Buffer.alloc(900, 'A');
const b = Buffer.alloc(900, 'A');
b.write('Z', 0, 'ascii');
await writeFile(join(tmpDir, 'a.epub'), a);
await writeFile(join(tmpDir, 'b.epub'), b);
const [h1, h2] = await Promise.all([fingerprintFile(join(tmpDir, 'a.epub')), fingerprintFile(join(tmpDir, 'b.epub'))]);
expect(h1).not.toBe(h2);
});
});
+15 -1
View File
@@ -8,6 +8,8 @@ const BLOCK_SIZE = 1024;
* Partial MD5 fingerprint — reads 1 KB blocks from exponentially spaced positions
* (1 KB, 4 KB, 16 KB, … up to 1 GB) rather than the entire file.
* Used only when path and inode lookups both fail (cross-filesystem move detection).
* For files too small to reach the first sample offset, fall back to the first
* 1 KB so tiny files do not collapse to the same empty hash.
*/
export async function fingerprintFile(absolutePath: string): Promise<string> {
const fh = await open(absolutePath, 'r');
@@ -15,12 +17,24 @@ export async function fingerprintFile(absolutePath: string): Promise<string> {
const { size } = await fh.stat();
const hash = createHash('md5');
const buf = Buffer.allocUnsafe(BLOCK_SIZE);
let sampled = false;
for (let i = 0; i <= 10; i++) {
const position = BASE << (2 * i);
if (position >= size) break;
const { bytesRead } = await fh.read(buf, 0, BLOCK_SIZE, position);
if (bytesRead > 0) hash.update(buf.subarray(0, bytesRead));
if (bytesRead > 0) {
hash.update(buf.subarray(0, bytesRead));
sampled = true;
}
}
if (!sampled && size > 0) {
const readLength = Math.min(BLOCK_SIZE, size);
const { bytesRead } = await fh.read(buf, 0, readLength, 0);
if (bytesRead > 0) {
hash.update(buf.subarray(0, bytesRead));
}
}
return hash.digest('hex');
@@ -1,6 +1,8 @@
import { Permission } from '@projectx/types';
import { BadRequestException } from '@nestjs/common';
import { UserController } from './user.controller';
import { MAX_USER_AVATAR_BYTES } from './user-avatar.service';
describe('UserController', () => {
const userService = {
@@ -40,6 +42,39 @@ describe('UserController', () => {
expect(userService.updateMe).toHaveBeenCalledWith(7, dto);
});
it('uploads avatar bytes with multipart file limits', async () => {
const user = { id: 7 } as any;
const buffer = Buffer.from('img');
const req = {
file: vi.fn().mockResolvedValue({
mimetype: 'image/png',
toBuffer: vi.fn().mockResolvedValue(buffer),
}),
} as any;
await controller.uploadMyAvatar(user, req);
expect(req.file).toHaveBeenCalledWith({ limits: { fileSize: MAX_USER_AVATAR_BYTES } });
expect(userAvatarService.uploadOwnAvatar).toHaveBeenCalledWith(user, buffer, 'image/png');
});
it('maps multipart file-size errors to a bad request response', async () => {
const tooLargeError = Object.assign(new Error('request file too large'), {
code: 'FST_REQ_FILE_TOO_LARGE',
statusCode: 413,
});
const req = {
file: vi.fn().mockResolvedValue({
mimetype: 'image/jpeg',
toBuffer: vi.fn().mockRejectedValue(tooLargeError),
}),
} as any;
await expect(controller.uploadMyAvatar({ id: 7 } as any, req)).rejects.toThrow(BadRequestException);
await expect(controller.uploadMyAvatar({ id: 7 } as any, req)).rejects.toThrow('Image exceeds 5 MB limit');
expect(userAvatarService.uploadOwnAvatar).not.toHaveBeenCalled();
});
it('delegates permission and superuser management and admin reset', async () => {
const requester = { id: 1 } as any;
const dto = { permissionNames: [Permission.LibraryDownload] };
+29 -2
View File
@@ -79,9 +79,25 @@ export class UserController {
description: () => 'User uploaded a profile picture',
})
async uploadMyAvatar(@CurrentUser() user: RequestUser, @Req() req: MultipartRequest) {
const data = await req.file({ limits: { fileSize: MAX_USER_AVATAR_BYTES } });
let data;
try {
data = await req.file({ limits: { fileSize: MAX_USER_AVATAR_BYTES } });
} catch (error) {
if (isMultipartFileTooLargeError(error)) {
throw new BadRequestException('Image exceeds 5 MB limit');
}
throw error;
}
if (!data) throw new BadRequestException('No file provided');
const buffer = await data.toBuffer();
let buffer: Buffer;
try {
buffer = await data.toBuffer();
} catch (error) {
if (isMultipartFileTooLargeError(error)) {
throw new BadRequestException('Image exceeds 5 MB limit');
}
throw error;
}
return this.userAvatarService.uploadOwnAvatar(user, buffer, data.mimetype);
}
@@ -230,3 +246,14 @@ export class UserController {
return this.userService.adminResetPassword(id, requestingUser);
}
}
function isMultipartFileTooLargeError(error: unknown): boolean {
if (typeof error !== 'object' || error === null) {
return false;
}
const value = error as { code?: unknown; statusCode?: unknown; message?: unknown };
if (value.code === 'FST_REQ_FILE_TOO_LARGE') {
return true;
}
return value.statusCode === 413 && typeof value.message === 'string' && value.message.toLowerCase().includes('file too large');
}
@@ -235,24 +235,42 @@ export async function waitForScanCompletion(db: Db, jobId: number, timeoutMs = 3
}
export async function locateBookFileByRelPath(ctx: MetadataWriteE2EContext, libraryId: number, relPath: string): Promise<LocatedBookFile> {
const [row] = await ctx.db
.select({
bookId: books.id,
bookFileId: bookFiles.id,
absolutePath: bookFiles.absolutePath,
relPath: bookFiles.relPath,
format: bookFiles.format,
})
.from(bookFiles)
.innerJoin(books, eq(books.id, bookFiles.bookId))
.where(and(eq(books.libraryId, libraryId), eq(bookFiles.relPath, relPath), eq(books.status, 'present')))
.limit(1);
const query = async (): Promise<LocatedBookFile | null> => {
const [row] = await ctx.db
.select({
bookId: books.id,
bookFileId: bookFiles.id,
absolutePath: bookFiles.absolutePath,
relPath: bookFiles.relPath,
format: bookFiles.format,
})
.from(bookFiles)
.innerJoin(books, eq(books.id, bookFiles.bookId))
.where(and(eq(books.libraryId, libraryId), eq(bookFiles.relPath, relPath), eq(books.status, 'present')))
.limit(1);
return row ?? null;
};
if (!row) {
throw new Error(`No present book file found for relPath "${relPath}" in library ${libraryId}`);
}
const waitForRow = async (timeoutMs: number): Promise<LocatedBookFile | null> => {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
const row = await query();
if (row) return row;
await sleep(100);
}
return query();
};
return row;
const initial = await waitForRow(2_000);
if (initial) return initial;
// In CI and shared dev environments, fixture writes can occasionally race the
// first scan trigger. Perform one recovery scan before failing hard.
await triggerAndWaitForLibraryScan(ctx, libraryId);
const recovered = await waitForRow(2_000);
if (recovered) return recovered;
throw new Error(`No present book file found for relPath "${relPath}" in library ${libraryId}`);
}
export async function getLatestWriteLogEntry(db: Db, bookId: number, triggeredBy?: 'auto' | 'sync') {