test(metadata): expand end-to-end scenarios and assertions

This commit is contained in:
neon
2026-04-01 11:50:59 -06:00
parent a6d84ae20c
commit 157224be9f
22 changed files with 1611 additions and 164 deletions
+7
View File
@@ -3,6 +3,12 @@ run-name: ${{ format('Metadata Write E2E - {0}', github.ref_name) }}
on:
workflow_dispatch:
inputs:
vitest-args:
description: "Optional vitest args (examples: --shard=1/2, --testNamePattern=\"concurrency hardening\")"
required: false
default: ""
type: string
push:
paths:
- "server/src/modules/book/**"
@@ -46,3 +52,4 @@ jobs:
check-name: Server Metadata Write E2E
artifact-name: metadata-write-e2e-report
job-timeout-minutes: 60
vitest-args: ${{ github.event_name == 'workflow_dispatch' && inputs.vitest-args || '' }}
+11 -1
View File
@@ -20,6 +20,11 @@ on:
required: false
type: number
default: 45
vitest-args:
description: "Optional extra args forwarded to vitest (for shard/testNamePattern reruns)"
required: false
type: string
default: ""
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
@@ -65,7 +70,12 @@ jobs:
- name: Run e2e suite
shell: bash
run: pnpm run test:e2e -- ${{ inputs.suite-id }}
run: |
if [ -n "${{ inputs.vitest-args }}" ]; then
pnpm run test:e2e -- ${{ inputs.suite-id }} ${{ inputs.vitest-args }}
else
pnpm run test:e2e -- ${{ inputs.suite-id }}
fi
- name: Publish test report annotations
if: always()
+3
View File
@@ -24,6 +24,9 @@
"test:client": "pnpm --filter client test:unit --run",
"test:e2e": "node ./scripts/test/run-e2e-suite.mjs",
"test:e2e:all": "node ./scripts/test/run-e2e-suite.mjs all",
"test:e2e:metadata-write": "node ./scripts/test/run-e2e-suite.mjs metadata-write",
"test:e2e:metadata-write:atomicity": "node ./scripts/test/run-e2e-suite.mjs metadata-write --testNamePattern=\"atomicity failpoints\"",
"test:e2e:metadata-write:concurrency": "node ./scripts/test/run-e2e-suite.mjs metadata-write --testNamePattern=\"concurrency hardening\"",
"test:e2e:list": "node ./scripts/test/run-e2e-suite.mjs --list",
"coverage": "pnpm run coverage:server && pnpm run coverage:client",
"coverage:server": "pnpm --filter server test:cov",
+3
View File
@@ -33,6 +33,9 @@ function printUsage() {
"Examples:",
" pnpm run test:e2e -- smoke",
" pnpm run test:e2e -- all",
" pnpm run test:e2e -- metadata-write",
" pnpm run test:e2e -- metadata-write --testNamePattern=\"concurrency hardening\"",
" pnpm run test:e2e -- metadata-write --shard=1/2",
" pnpm run test:e2e -- scanner --testNamePattern=book-per-folder-disc-folder-flattening",
" pnpm run test:e2e -- --list",
" pnpm run test:e2e:list",
+12 -2
View File
@@ -29,6 +29,8 @@ import {
} from '../../db/schema';
type Db = NodePgDatabase<typeof schema>;
type DbTransaction = Parameters<Parameters<Db['transaction']>[0]>[0];
type MetadataUpdateExecutor = Pick<Db, 'update'>;
type PatternMetadataRow = {
bookId: number;
title: string | null;
@@ -46,6 +48,10 @@ type PatternMetadataRow = {
export class BookRepository {
constructor(@Inject(DB) private readonly db: Db) {}
async withTransaction<T>(callback: (tx: DbTransaction) => Promise<T>): Promise<T> {
return this.db.transaction((tx) => callback(tx));
}
async findCards(opts: { where: SQL | undefined; orderBy: SQL[]; limit: number; offset: number; userId: number }) {
const { where, orderBy, limit, offset, userId } = opts;
@@ -444,8 +450,12 @@ export class BookRepository {
await this.db.delete(books).where(inArray(books.id, bookIds));
}
async updateMetadataFields(bookId: number, fields: Partial<typeof bookMetadata.$inferInsert>): Promise<void> {
await this.db.update(bookMetadata).set(fields).where(eq(bookMetadata.bookId, bookId));
async updateMetadataFields(
bookId: number,
fields: Partial<typeof bookMetadata.$inferInsert>,
executor: MetadataUpdateExecutor = this.db,
): Promise<void> {
await executor.update(bookMetadata).set(fields).where(eq(bookMetadata.bookId, bookId));
}
async upsertProgress(
+29 -9
View File
@@ -58,6 +58,7 @@ function makeService() {
findAudioProgress: vi.fn(),
upsertAudioProgress: vi.fn(),
updateMetadataFields: vi.fn(),
withTransaction: vi.fn(),
deleteByIds: vi.fn(),
findAllIds: vi.fn(),
};
@@ -70,9 +71,10 @@ function makeService() {
buildOrderBy: vi.fn(),
};
const metadataService = {
replaceAuthors: vi.fn().mockResolvedValue(undefined),
replaceAuthors: vi.fn().mockResolvedValue([]),
replaceGenres: vi.fn().mockResolvedValue(undefined),
replaceTags: vi.fn().mockResolvedValue(undefined),
emitAuthorsReplaced: vi.fn(),
downloadAndSaveCover: vi.fn().mockResolvedValue(undefined),
refreshCoverForBook: vi.fn(),
};
@@ -95,6 +97,13 @@ function makeService() {
const fileWriteService = {
scheduleWrite: vi.fn(),
};
const narratorService = {
replaceForBook: vi.fn().mockResolvedValue(undefined),
};
const comicMetadataService = {
upsert: vi.fn().mockResolvedValue(undefined),
findByBookId: vi.fn().mockResolvedValue(null),
};
const userBookStatusService = {
autoUpdate: vi.fn().mockResolvedValue(undefined),
setManual: vi.fn().mockResolvedValue(undefined),
@@ -102,6 +111,8 @@ function makeService() {
findByBookIds: vi.fn().mockResolvedValue(new Map()),
};
bookRepo.withTransaction.mockImplementation(async (callback: (tx: unknown) => Promise<unknown>) => callback({}));
const service = new BookService(
bookRepo as never,
libraryService as never,
@@ -112,8 +123,8 @@ function makeService() {
config as never,
appSettings as never,
userBookStatusService as never,
{ replaceForBook: vi.fn().mockResolvedValue(undefined) } as never,
{ upsert: vi.fn().mockResolvedValue(undefined), findByBookId: vi.fn().mockResolvedValue(null) } as never,
narratorService as never,
comicMetadataService as never,
embedder as never,
fileWriteService as never,
);
@@ -130,6 +141,8 @@ function makeService() {
userBookStatusService,
embedder,
fileWriteService,
narratorService,
comicMetadataService,
};
}
@@ -638,6 +651,7 @@ describe('BookService', () => {
);
expect(verifySpy).toHaveBeenCalledWith(5, user);
expect(bookRepo.withTransaction).toHaveBeenCalledTimes(1);
expect(bookRepo.updateMetadataFields).toHaveBeenCalledWith(
5,
expect.objectContaining({
@@ -645,13 +659,19 @@ describe('BookService', () => {
rating: 4,
updatedAt: expect.any(Date),
}),
expect.anything(),
);
expect(metadataService.replaceAuthors).toHaveBeenCalledWith(5, [
{ name: 'A1', sortName: null },
{ name: 'A2', sortName: null },
]);
expect(metadataService.replaceGenres).toHaveBeenCalledWith(5, ['Sci-Fi']);
expect(metadataService.replaceTags).toHaveBeenCalledWith(5, ['favorite']);
expect(metadataService.replaceAuthors).toHaveBeenCalledWith(
5,
[
{ name: 'A1', sortName: null },
{ name: 'A2', sortName: null },
],
{ executor: expect.anything(), emitEvent: false },
);
expect(metadataService.replaceGenres).toHaveBeenCalledWith(5, ['Sci-Fi'], { executor: expect.anything() });
expect(metadataService.replaceTags).toHaveBeenCalledWith(5, ['favorite'], { executor: expect.anything() });
expect(metadataService.emitAuthorsReplaced).toHaveBeenCalledWith(5, []);
expect(fileWriteService.scheduleWrite).toHaveBeenCalledWith(5, 'auto', user.id);
expect(embedder.embedBook).toHaveBeenCalledWith(5);
expect(detailSpy).toHaveBeenCalledWith(5, user);
+69 -23
View File
@@ -1,4 +1,4 @@
import { BadRequestException, Injectable, Logger, NotFoundException, Optional } from '@nestjs/common';
import { BadRequestException, Injectable, InternalServerErrorException, Logger, NotFoundException, Optional } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { access, readdir, rm, stat } from 'fs/promises';
import { basename, extname, join } from 'path';
@@ -25,11 +25,24 @@ import { SaveProgressDto } from './dto/save-progress.dto';
import { UpsertAudioProgressDto } from './dto/upsert-audio-progress.dto';
import { UpdateBookMetadataDto } from './dto/update-book-metadata.dto';
const METADATA_UPDATE_FAILPOINTS = [
'afterScalarUpdate',
'afterComicMetadataUpsert',
'afterAuthorsReplace',
'afterNarratorsReplace',
'afterGenresReplace',
'afterTagsReplace',
'beforeTransactionCommit',
] as const;
export type MetadataUpdateFailpoint = (typeof METADATA_UPDATE_FAILPOINTS)[number];
@Injectable()
export class BookService {
private readonly logger = new Logger(BookService.name);
private readonly booksPath: string;
private embeddingRun: Promise<void> | null = null;
private metadataUpdateFailpoint: MetadataUpdateFailpoint | null = null;
constructor(
private readonly bookRepo: BookRepository,
@@ -404,30 +417,46 @@ export class BookService {
}
const scalarFieldCount = Object.keys(scalarFields).length;
if (scalarFieldCount > 0) {
scalarFields.updatedAt = new Date();
await this.bookRepo.updateMetadataFields(id, scalarFields);
}
let replacedAuthorIds: number[] = [];
await this.bookRepo.withTransaction(async (tx) => {
if (scalarFieldCount > 0) {
scalarFields.updatedAt = new Date();
await this.bookRepo.updateMetadataFields(id, scalarFields, tx);
}
this.throwIfMetadataUpdateFailpoint('afterScalarUpdate');
if (dto.comicMetadata) {
await this.comicMetadataService.upsert(id, dto.comicMetadata);
}
if (dto.comicMetadata) {
await this.comicMetadataService.upsert(id, dto.comicMetadata, tx);
}
this.throwIfMetadataUpdateFailpoint('afterComicMetadataUpsert');
if (dto.authors !== undefined) {
await this.metadataService.replaceAuthors(
id,
dto.authors.map((name) => ({ name, sortName: null })),
);
}
if (dto.audioMetadata?.narrators !== undefined) {
await this.narratorService.replaceForBook(id, dto.audioMetadata.narrators);
}
if (dto.genres !== undefined) {
await this.metadataService.replaceGenres(id, dto.genres);
}
if (dto.tags !== undefined) {
await this.metadataService.replaceTags(id, dto.tags);
}
if (dto.authors !== undefined) {
replacedAuthorIds = await this.metadataService.replaceAuthors(
id,
dto.authors.map((name) => ({ name, sortName: null })),
{ executor: tx, emitEvent: false },
);
}
this.throwIfMetadataUpdateFailpoint('afterAuthorsReplace');
if (dto.audioMetadata?.narrators !== undefined) {
await this.narratorService.replaceForBook(id, dto.audioMetadata.narrators, { executor: tx });
}
this.throwIfMetadataUpdateFailpoint('afterNarratorsReplace');
if (dto.genres !== undefined) {
await this.metadataService.replaceGenres(id, dto.genres, { executor: tx });
}
this.throwIfMetadataUpdateFailpoint('afterGenresReplace');
if (dto.tags !== undefined) {
await this.metadataService.replaceTags(id, dto.tags, { executor: tx });
}
this.throwIfMetadataUpdateFailpoint('afterTagsReplace');
this.throwIfMetadataUpdateFailpoint('beforeTransactionCommit');
});
this.metadataService.emitAuthorsReplaced(id, replacedAuthorIds);
this.embedder?.embedBook(id).catch((err: Error) => this.logger.warn(`Embedding failed for book ${id}: ${err.message}`));
this.fileWriteService?.scheduleWrite(id, 'auto', user.id);
@@ -447,6 +476,23 @@ export class BookService {
}
}
setMetadataUpdateFailpointForTests(failpoint: MetadataUpdateFailpoint | null): void {
if (failpoint !== null && !METADATA_UPDATE_FAILPOINTS.includes(failpoint)) {
throw new InternalServerErrorException(`Unknown metadata update failpoint: ${failpoint}`);
}
this.metadataUpdateFailpoint = failpoint;
}
clearMetadataUpdateFailpointForTests(): void {
this.metadataUpdateFailpoint = null;
}
private throwIfMetadataUpdateFailpoint(stage: MetadataUpdateFailpoint): void {
if (this.metadataUpdateFailpoint !== stage) return;
this.metadataUpdateFailpoint = null;
throw new InternalServerErrorException(`Metadata update failpoint triggered: ${stage}`);
}
async embedAll(): Promise<{ queued: number }> {
const event = 'book.embed_all';
const startedAt = Date.now();
@@ -5,7 +5,7 @@ import { randomUUID } from 'crypto';
import type { WriteResult } from '@projectx/types';
import { getSevenZip, type SevenZipFS } from '../../../../common/sevenzip';
import type { BookWritePayload } from '../../interfaces/book-write-payload.interface';
import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
import type { FormatWriter } from '../../interfaces/format-writer.interface';
import type { FormatWriteOptions } from '../../interfaces/format-write-options.interface';
import { replaceFileAtomically } from '../shared/atomic-file-replace';
@@ -19,6 +19,28 @@ import { buildComicInfoXml } from './comic-info-builder';
// is always written before it is added to the archive, with no await in between.
const VFS_XML_PATH = '/ComicInfo.xml';
const ENOENT_ERRNO = 44;
const CBX_WRITABLE_FIELDS = new Set<BookWritePayloadKey>([
'title',
'subtitle',
'description',
'publisher',
'publishedYear',
'language',
'pageCount',
'seriesName',
'seriesIndex',
'isbn10',
'isbn13',
'rating',
'authors',
'genres',
'tags',
'googleBooksId',
'goodreadsId',
'amazonId',
'hardcoverId',
'openLibraryId',
]);
@Injectable()
export class Cb7FormatWriter implements FormatWriter {
@@ -27,7 +49,8 @@ export class Cb7FormatWriter implements FormatWriter {
async write(filePath: string, payload: BookWritePayload, options: FormatWriteOptions): Promise<WriteResult> {
const start = Date.now();
const { fieldMask, dryRun } = options;
const fieldsWritten = resolveFieldsWritten(payload, fieldMask);
const cbxFieldMask = new Set([...fieldMask].filter((key) => CBX_WRITABLE_FIELDS.has(key)));
const fieldsWritten = resolveFieldsWritten(payload, cbxFieldMask);
if (dryRun) {
return { status: 'skipped', reason: 'dry-run', fieldsWritten, durationMs: Date.now() - start };
@@ -61,7 +84,7 @@ export class Cb7FormatWriter implements FormatWriter {
// No ComicInfo.xml in archive - start fresh
}
const xml = buildComicInfoXml(existingXml, payload, fieldMask);
const xml = buildComicInfoXml(existingXml, payload, cbxFieldMask);
const xmlBytes = Buffer.from(xml, 'utf-8');
const fd2 = sz.FS.open(VFS_XML_PATH, 'w+');
@@ -1,13 +1,36 @@
import { Injectable } from '@nestjs/common';
import type { WriteResult } from '@projectx/types';
import type { BookWritePayload } from '../../interfaces/book-write-payload.interface';
import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface';
import type { FormatWriter } from '../../interfaces/format-writer.interface';
import type { FormatWriteOptions } from '../../interfaces/format-write-options.interface';
import { resolveFieldsWritten } from '../shared/resolve-fields-written';
import { buildComicInfoXml } from './comic-info-builder';
import { readComicInfoFromZip, writeComicInfoToZip } from './cbz-zip-patcher';
const CBX_WRITABLE_FIELDS = new Set<BookWritePayloadKey>([
'title',
'subtitle',
'description',
'publisher',
'publishedYear',
'language',
'pageCount',
'seriesName',
'seriesIndex',
'isbn10',
'isbn13',
'rating',
'authors',
'genres',
'tags',
'googleBooksId',
'goodreadsId',
'amazonId',
'hardcoverId',
'openLibraryId',
]);
@Injectable()
export class CbzFormatWriter implements FormatWriter {
readonly format = 'cbz';
@@ -15,14 +38,15 @@ export class CbzFormatWriter implements FormatWriter {
async write(filePath: string, payload: BookWritePayload, options: FormatWriteOptions): Promise<WriteResult> {
const start = Date.now();
const { fieldMask, dryRun } = options;
const fieldsWritten = resolveFieldsWritten(payload, fieldMask);
const cbxFieldMask = new Set([...fieldMask].filter((key) => CBX_WRITABLE_FIELDS.has(key)));
const fieldsWritten = resolveFieldsWritten(payload, cbxFieldMask);
if (dryRun) {
return { status: 'skipped', reason: 'dry-run', fieldsWritten, durationMs: Date.now() - start };
}
const existingXml = await readComicInfoFromZip(filePath);
const xml = buildComicInfoXml(existingXml, payload, fieldMask);
const xml = buildComicInfoXml(existingXml, payload, cbxFieldMask);
await writeComicInfoToZip(filePath, xml);
return { status: 'success', fieldsWritten, durationMs: Date.now() - start };
@@ -35,19 +35,19 @@ function parseComicInfoXml(xml: string): ComicInfoObject {
export function buildComicInfoXml(existingXml: string | null, payload: BookWritePayload, fieldMask: Set<BookWritePayloadKey>): string {
const info: ComicInfoObject = existingXml ? parseComicInfoXml(existingXml) : {};
if (fieldMask.has('title') && payload.title != null) info['Title'] = payload.title;
if (fieldMask.has('description') && payload.description != null) info['Summary'] = stripHtml(payload.description);
if (fieldMask.has('publisher') && payload.publisher != null) info['Publisher'] = payload.publisher;
if (fieldMask.has('seriesName') && payload.seriesName != null) info['Series'] = payload.seriesName;
if (fieldMask.has('seriesIndex') && payload.seriesIndex != null) info['Number'] = formatSeriesIndex(payload.seriesIndex);
if (fieldMask.has('publishedYear') && payload.publishedYear != null) info['Year'] = payload.publishedYear;
if (fieldMask.has('pageCount') && payload.pageCount != null) info['PageCount'] = payload.pageCount;
if (fieldMask.has('language') && payload.language != null) info['LanguageISO'] = payload.language;
if (fieldMask.has('authors') && payload.authors?.length) info['Writer'] = payload.authors.map((a) => a.name).join(', ');
if (fieldMask.has('genres') && payload.genres?.length) info['Genre'] = payload.genres.join(', ');
if (fieldMask.has('tags') && payload.tags?.length) info['Tags'] = payload.tags.join(', ');
if (fieldMask.has('rating') && payload.rating != null) info['CommunityRating'] = formatRating(payload.rating);
if (fieldMask.has('isbn13') && payload.isbn13 != null) info['GTIN'] = payload.isbn13;
setComicInfoField(info, fieldMask, 'title', 'Title', payload.title);
setComicInfoField(info, fieldMask, 'description', 'Summary', payload.description, stripHtml);
setComicInfoField(info, fieldMask, 'publisher', 'Publisher', payload.publisher);
setComicInfoField(info, fieldMask, 'seriesName', 'Series', payload.seriesName);
setComicInfoField(info, fieldMask, 'seriesIndex', 'Number', payload.seriesIndex, formatSeriesIndex);
setComicInfoField(info, fieldMask, 'publishedYear', 'Year', payload.publishedYear);
setComicInfoField(info, fieldMask, 'pageCount', 'PageCount', payload.pageCount);
setComicInfoField(info, fieldMask, 'language', 'LanguageISO', payload.language);
setComicInfoField(info, fieldMask, 'authors', 'Writer', payload.authors?.length ? payload.authors.map((a) => a.name).join(', ') : null);
setComicInfoField(info, fieldMask, 'genres', 'Genre', payload.genres?.length ? payload.genres.join(', ') : null);
setComicInfoField(info, fieldMask, 'tags', 'Tags', payload.tags?.length ? payload.tags.join(', ') : null);
setComicInfoField(info, fieldMask, 'rating', 'CommunityRating', payload.rating, formatRating);
setComicInfoField(info, fieldMask, 'isbn13', 'GTIN', payload.isbn13);
const hasProviderSelection = COMIC_INFO_PROVIDER_ID_KEYS.some((key) => fieldMask.has(key));
if (hasProviderSelection) {
@@ -154,3 +154,20 @@ function resolveManagedTextField(payload: BookWritePayload, key: BookWritePayloa
const value = payload[key];
return typeof value === 'string' ? value : null;
}
function setComicInfoField<TValue>(
info: ComicInfoObject,
fieldMask: Set<BookWritePayloadKey>,
key: BookWritePayloadKey,
comicInfoKey: string,
value: TValue | null | undefined,
formatter?: (value: TValue) => string | number,
): void {
if (!fieldMask.has(key)) return;
if (value == null || (typeof value === 'string' && value === '')) {
delete info[comicInfoKey];
return;
}
info[comicInfoKey] = formatter ? formatter(value) : value;
}
@@ -7,13 +7,14 @@ import * as schema from '../../db/schema';
import { ComicMetadataDto } from '../book/dto/update-book-metadata.dto';
type Db = NodePgDatabase<typeof schema>;
type ComicMetadataExecutor = Pick<Db, 'insert'>;
@Injectable()
export class ComicMetadataRepository {
constructor(@Inject(DB) private readonly db: Db) {}
async upsert(bookId: number, fields: ComicMetadataDto): Promise<void> {
await this.db
async upsert(bookId: number, fields: ComicMetadataDto, executor: ComicMetadataExecutor = this.db): Promise<void> {
await executor
.insert(schema.comicMetadata)
.values({
bookId,
@@ -38,7 +38,7 @@ export class ComicFormatExtractor implements FormatExtractor {
seriesName: comicMetadata?.seriesName ?? null,
seriesIndex: comicMetadata?.seriesIndex ?? null,
authors: comicMetadata?.authors ?? [],
genres: comicMetadata?.tags ?? [],
genres: comicMetadata?.genres?.length ? comicMetadata.genres : (comicMetadata?.tags ?? []),
cover: cover ?? null,
comicMetadata: comicMetadata?.comicMetadata ?? null,
};
@@ -18,7 +18,7 @@ export class EpubFormatExtractor implements FormatExtractor {
seriesName: metadata.seriesName,
seriesIndex: metadata.seriesIndex,
authors: metadata.authors,
genres: metadata.tags,
genres: metadata.genres.length > 0 ? metadata.genres : metadata.tags,
cover: cover ?? null,
};
}
@@ -121,13 +121,15 @@ describe('extractCbzMetadata', () => {
expect(r?.authors.map((a) => a.name)).toEqual(['Writer One', 'Writer Two']);
});
it('merges Genre and Tags into deduplicated tags array', async () => {
it('extracts Genre and Tags into separate deduplicated lists', async () => {
const xml = `<ComicInfo><Genre>Superhero,Fantasy</Genre><Tags>Fantasy,Horror</Tags></ComicInfo>`;
mockReadFile.mockResolvedValue(buildZipWithComicInfo(xml) as unknown as Buffer);
const r = await extractCbzMetadata('/book.cbz');
// "Fantasy" appears in both Genre and Tags — must be deduplicated
expect(r?.tags).toContain('Superhero');
expect(r?.genres).toContain('Superhero');
expect(r?.genres).toContain('Fantasy');
expect(r?.genres?.filter((t) => t === 'Fantasy')).toHaveLength(1);
expect(r?.tags).toContain('Fantasy');
expect(r?.tags).toContain('Horror');
expect(r?.tags?.filter((t) => t === 'Fantasy')).toHaveLength(1);
@@ -25,14 +25,26 @@ export interface ParsedCbzComicMetadata {
export interface ParsedCbzMetadata {
title: string | null;
subtitle: string | null;
seriesName: string | null;
seriesIndex: number | null;
description: string | null;
publisher: string | null;
publishedYear: number | null;
language: string | null;
pageCount: number | null;
rating: number | null;
isbn10: string | null;
isbn13: string | null;
authors: { name: string; sortName: string | null }[];
genres: string[];
tags: string[];
googleBooksId: string | null;
goodreadsId: string | null;
amazonId: string | null;
hardcoverId: string | null;
openLibraryId: string | null;
itunesId: string | null;
comicMetadata: ParsedCbzComicMetadata | null;
}
@@ -124,6 +136,59 @@ function splitDelimited(value: string | null | undefined): string[] {
.filter(Boolean);
}
function parseProjectxManagedNotes(notes: string | null): Map<string, string> {
const fields = new Map<string, string>();
if (!notes) return fields;
for (const line of notes.split('\n')) {
const trimmed = line.trim();
if (!trimmed) continue;
const match = trimmed.match(/^\[projectx:([^\]]+)\]\s*(.*)$/);
if (!match) continue;
const [, key, value] = match;
fields.set(key, value.trim());
}
return fields;
}
function parseProviderIdsFromWebUrl(
webUrl: string | null,
): Partial<Record<'goodreadsId' | 'amazonId' | 'hardcoverId' | 'googleBooksId' | 'openLibraryId', string>> {
if (!webUrl) return {};
if (webUrl.startsWith('https://www.goodreads.com/book/show/')) {
return { goodreadsId: webUrl.slice('https://www.goodreads.com/book/show/'.length) || '' };
}
if (webUrl.startsWith('https://www.amazon.com/dp/')) {
return { amazonId: webUrl.slice('https://www.amazon.com/dp/'.length) || '' };
}
if (webUrl.startsWith('https://hardcover.app/books/')) {
return { hardcoverId: webUrl.slice('https://hardcover.app/books/'.length) || '' };
}
if (webUrl.startsWith('https://books.google.com/books?id=')) {
return { googleBooksId: webUrl.slice('https://books.google.com/books?id='.length) || '' };
}
if (webUrl.startsWith('https://openlibrary.org/works/')) {
return { openLibraryId: webUrl.slice('https://openlibrary.org/works/'.length) || '' };
}
return {};
}
function parseCbxRating(value: string | null): number | null {
if (!value) return null;
const parsed = Number(value);
if (!Number.isFinite(parsed)) return null;
return Math.min(5, Math.max(0, parsed * 2));
}
function parseIsbn13(value: string | null): string | null {
if (!value) return null;
const digits = value.replace(/[^\d]/g, '');
return digits.length === 13 ? digits : null;
}
function parseComicInfoXml(xmlBuf: Buffer): ParsedCbzMetadata | null {
try {
const parsed = xmlParser.parse(xmlBuf.toString('utf-8')) as Record<string, unknown>;
@@ -143,8 +208,12 @@ function parseComicInfoXml(xmlBuf: Buffer): ParsedCbzMetadata | null {
const writers = splitDelimited(str('Writer'));
const year = num('Year');
const managedNotes = parseProjectxManagedNotes(str('Notes'));
const providerIdsFromWeb = parseProviderIdsFromWebUrl(str('Web'));
const tags = [...splitDelimited(str('Genre')), ...splitDelimited(str('Tags'))];
const genres = splitDelimited(str('Genre'));
const tags = splitDelimited(str('Tags'));
const uniqueGenres = [...new Set(genres)];
const uniqueTags = [...new Set(tags)];
const pencillers = splitDelimited(str('Penciller'));
@@ -172,14 +241,26 @@ function parseComicInfoXml(xmlBuf: Buffer): ParsedCbzMetadata | null {
return {
title: str('Title'),
subtitle: managedNotes.get('subtitle') ?? null,
seriesName: str('Series'),
seriesIndex: num('Number'),
description: str('Summary') ?? str('Description'),
publisher: str('Publisher'),
publishedYear: year != null ? Math.floor(year) : null,
language: str('LanguageISO'),
pageCount: num('PageCount'),
rating: parseCbxRating(str('CommunityRating')),
isbn10: managedNotes.get('isbn10') ?? null,
isbn13: parseIsbn13(str('GTIN')),
authors: writers.map((name) => ({ name, sortName: null })),
genres: uniqueGenres,
tags: uniqueTags,
googleBooksId: managedNotes.get('googleBooksId') ?? providerIdsFromWeb.googleBooksId ?? null,
goodreadsId: managedNotes.get('goodreadsId') ?? providerIdsFromWeb.goodreadsId ?? null,
amazonId: managedNotes.get('amazonId') ?? providerIdsFromWeb.amazonId ?? null,
hardcoverId: managedNotes.get('hardcoverId') ?? providerIdsFromWeb.hardcoverId ?? null,
openLibraryId: managedNotes.get('openLibraryId') ?? providerIdsFromWeb.openLibraryId ?? null,
itunesId: null,
comicMetadata: hasComicFields
? {
issueNumber: str('Number'),
@@ -215,14 +296,26 @@ function parseComicBookInfoJson(comment: string): ParsedCbzMetadata | null {
return {
title: (cbi['title'] as string) ?? null,
subtitle: null,
seriesName: (cbi['series'] as string) ?? null,
seriesIndex: cbi['issue'] != null ? (Number.isFinite(Number(cbi['issue'])) ? Number(cbi['issue']) : null) : null,
description: (cbi['comments'] as string) ?? null,
publisher: (cbi['publisher'] as string) ?? null,
publishedYear: (cbi['publicationYear'] as number) ?? null,
language: null,
pageCount: null,
rating: null,
isbn10: null,
isbn13: null,
authors: writers,
genres: [],
tags,
googleBooksId: null,
goodreadsId: null,
amazonId: null,
hardcoverId: null,
openLibraryId: null,
itunesId: null,
comicMetadata: null,
};
} catch {
+71 -2
View File
@@ -9,10 +9,19 @@ export interface ParsedOpf {
publisher: string | null;
publishedYear: number | null;
language: string | null;
pageCount: number | null;
rating: number | null;
seriesName: string | null;
seriesIndex: number | null;
authors: { name: string; sortName: string | null }[];
genres: string[];
tags: string[];
googleBooksId: string | null;
goodreadsId: string | null;
amazonId: string | null;
hardcoverId: string | null;
openLibraryId: string | null;
itunesId: string | null;
}
const parser = new XMLParser({
@@ -53,6 +62,26 @@ function parseYear(raw: string): number | null {
return match ? parseInt(match[0], 10) : null;
}
function parseNumber(raw: string | null): number | null {
if (!raw) return null;
const parsed = Number(raw);
return Number.isFinite(parsed) ? parsed : null;
}
function parseProjectxTags(raw: string | null): string[] {
if (!raw) return [];
try {
const parsed = JSON.parse(raw) as unknown;
if (!Array.isArray(parsed)) return [];
return parsed.map((item) => String(item).trim()).filter(Boolean);
} catch {
return raw
.split(/[,;]/)
.map((item) => item.trim())
.filter(Boolean);
}
}
// Build a map of id → array of refining meta nodes (EPUB 3).
function buildRefineMap(metas: unknown[]): Map<string, unknown[]> {
const map = new Map<string, unknown[]>();
@@ -99,6 +128,17 @@ export function parseOpf(xml: string): ParsedOpf {
return null;
}
function propertyMeta(property: string): string | null {
for (const m of rawMetas) {
if (typeof m !== 'object' || m === null) continue;
const mo = m as Record<string, unknown>;
if (mo['@_property'] === property) {
return getText(m) || null;
}
}
return null;
}
// ── Titles ─────────────────────────────────────────────────────────────────
let title: string | null = null;
let subtitle: string | null = null;
@@ -117,6 +157,7 @@ export function parseOpf(xml: string): ParsedOpf {
}
title ??= getText(rawTitles[0]);
}
subtitle ??= namedMeta('projectx:subtitle');
// ── Authors ────────────────────────────────────────────────────────────────
const authors: { name: string; sortName: string | null }[] = [];
@@ -144,6 +185,12 @@ export function parseOpf(xml: string): ParsedOpf {
// ── Identifiers → ISBN ─────────────────────────────────────────────────────
let isbn10: string | null = null;
let isbn13: string | null = null;
let googleBooksId: string | null = null;
let goodreadsId: string | null = null;
let amazonId: string | null = null;
let hardcoverId: string | null = null;
let openLibraryId: string | null = null;
let itunesId: string | null = null;
for (const ident of toArray(metadata['identifier'])) {
const mo = (typeof ident === 'object' && ident !== null ? ident : {}) as Record<string, unknown>;
@@ -156,10 +203,23 @@ export function parseOpf(xml: string): ParsedOpf {
isbn10 ??= parsed.isbn10;
isbn13 ??= parsed.isbn13;
}
if (value.startsWith('urn:goodreads:')) goodreadsId ??= value.slice('urn:goodreads:'.length) || null;
if (value.startsWith('urn:amazon:')) amazonId ??= value.slice('urn:amazon:'.length) || null;
if (value.startsWith('urn:hardcover:')) hardcoverId ??= value.slice('urn:hardcover:'.length) || null;
if (value.startsWith('urn:google:')) googleBooksId ??= value.slice('urn:google:'.length) || null;
if (value.startsWith('urn:openlibrary:')) openLibraryId ??= value.slice('urn:openlibrary:'.length) || null;
if (value.startsWith('urn:itunes:')) itunesId ??= value.slice('urn:itunes:'.length) || null;
}
// ── Tags ───────────────────────────────────────────────────────────────────
const tags = toArray(metadata['subject']).map(getText).filter(Boolean);
isbn10 ??= propertyMeta('projectx:isbn10') ?? namedMeta('projectx:isbn10');
// ── Genres and tags ────────────────────────────────────────────────────────
const genres = toArray(metadata['subject']).map(getText).filter(Boolean);
const projectxTags = parseProjectxTags(propertyMeta('projectx:tags') ?? namedMeta('projectx:tags'));
const tags = projectxTags.length > 0 ? projectxTags : genres;
const pageCount = parseNumber(propertyMeta('projectx:page_count') ?? namedMeta('projectx:page_count'));
const rating = parseNumber(propertyMeta('projectx:rating') ?? namedMeta('projectx:rating'));
// ── Series (Calibre EPUB2, then EPUB3) ────────────────────────────────────
let seriesName: string | null = namedMeta('calibre:series');
@@ -204,9 +264,18 @@ export function parseOpf(xml: string): ParsedOpf {
publisher,
publishedYear,
language: language || null,
pageCount,
rating,
seriesName: seriesName || null,
seriesIndex,
authors,
genres,
tags,
googleBooksId,
goodreadsId,
amazonId,
hardcoverId,
openLibraryId,
itunesId,
};
}
@@ -94,7 +94,7 @@ describe('parsePdfFile', () => {
title: 'XMP Title',
authors: [{ name: 'XMP Author', sortName: null }],
tags: ['favorite'],
pageCount: 123,
pageCount: 999,
coverBuffer: Buffer.from([0xff, 0xd8, 0xff]),
}),
);
@@ -73,6 +73,7 @@ export async function parsePdfFile(absolutePath: string): Promise<PdfParsed | nu
// Info Dictionary is used only as fallback for fields XMP doesn't cover.
const xmpXml = extractXmpXml(doc);
const xmp: XmpParsed | null = xmpXml ? parseXmp(xmpXml) : null;
const hasXmp = xmp !== null;
const infoTitle = clean(doc.getTitle());
const infoAuthorRaw = clean(doc.getAuthor());
@@ -90,22 +91,22 @@ export async function parsePdfFile(absolutePath: string): Promise<PdfParsed | nu
const coverBuffer = await extractPdfCover(absolutePath);
return {
title: xmp?.title ?? infoTitle,
title: hasXmp ? xmp.title : infoTitle,
subtitle: xmp?.subtitle ?? null,
authors: xmp?.authors?.length ? xmp.authors : infoAuthors,
description: xmp?.description ?? infoSubject,
authors: hasXmp ? xmp.authors : infoAuthors,
description: hasXmp ? xmp.description : infoSubject,
publisher: xmp?.publisher ?? null,
publishedYear: xmp?.publishedYear ?? null,
language: xmp?.language ?? null,
genres: xmp?.genres?.length ? xmp.genres : [],
// Info Dict keywords are genres+tags mixed — only use as tags when no XMP
tags: xmp ? xmp.tags : infoKeywords,
// Info Dict keywords are genres+tags mixed — only use as tags when XMP is absent
tags: hasXmp ? xmp.tags : infoKeywords,
isbn10: xmp?.isbn10 ?? null,
isbn13: xmp?.isbn13 ?? null,
seriesName: xmp?.seriesName ?? null,
seriesIndex: xmp?.seriesIndex ?? null,
rating: xmp?.rating ?? null,
pageCount: doc.getPageCount(),
pageCount: hasXmp ? xmp.pageCount : doc.getPageCount(),
googleBooksId: xmp?.googleBooksId ?? null,
goodreadsId: xmp?.goodreadsId ?? null,
amazonId: xmp?.amazonId ?? null,
+94 -53
View File
@@ -25,6 +25,12 @@ import { generateThumbnail, imageExt } from './lib/cover';
import { MetadataEventsService, METADATA_AUTHORS_REPLACED } from './metadata-events.service';
type Db = NodePgDatabase<typeof schema>;
type RelationMutationExecutor = Pick<Db, 'delete' | 'insert' | 'select'>;
interface RelationMutationOptions {
executor?: RelationMutationExecutor;
emitEvent?: boolean;
}
const AUDIO_FORMATS = ['m4b', 'mp3', 'm4a', 'opus', 'ogg', 'flac'] as const;
const MAX_RELATION_NAME_LENGTH = 200;
@@ -237,7 +243,11 @@ export class MetadataService {
// ── Authors ──────────────────────────────────────────────────────────────────
async replaceAuthors(bookId: number, parsedAuthors: { name: string; sortName: string | null }[]) {
async replaceAuthors(
bookId: number,
parsedAuthors: { name: string; sortName: string | null }[],
options: RelationMutationOptions = {},
): Promise<number[]> {
const normalized = parsedAuthors
.map((author) => ({
name: author.name.trim(),
@@ -253,34 +263,21 @@ export class MetadataService {
return true;
});
const linkedAuthorIds = await this.db.transaction(async (tx) => {
await tx.delete(bookAuthors).where(eq(bookAuthors.bookId, bookId));
const linkedAuthorIds = options.executor
? await this.replaceAuthorsInExecutor(options.executor, bookId, unique)
: await this.db.transaction(async (tx) => this.replaceAuthorsInExecutor(tx, bookId, unique));
if (unique.length === 0) return [] as number[];
const authorIds: number[] = [];
for (let index = 0; index < unique.length; index += 1) {
const { name, sortName } = unique[index];
let [author] = await tx.insert(authors).values({ name, sortName }).onConflictDoNothing().returning();
if (!author) {
[author] = await tx.select().from(authors).where(eq(authors.name, name)).limit(1);
}
if (!author) continue;
await tx.insert(bookAuthors).values({ bookId, authorId: author.id, displayOrder: index }).onConflictDoNothing();
authorIds.push(author.id);
}
return authorIds;
});
if (linkedAuthorIds.length > 0) {
this.metadataEvents?.emit(METADATA_AUTHORS_REPLACED, {
bookId,
authorIds: linkedAuthorIds,
});
const emitEvent = options.emitEvent ?? true;
if (emitEvent && linkedAuthorIds.length > 0) {
this.emitAuthorsReplaced(bookId, linkedAuthorIds);
}
return linkedAuthorIds;
}
emitAuthorsReplaced(bookId: number, authorIds: number[]): void {
if (authorIds.length === 0) return;
this.metadataEvents?.emit(METADATA_AUTHORS_REPLACED, { bookId, authorIds });
}
// ── Genres ───────────────────────────────────────────────────────────────────
@@ -289,46 +286,90 @@ export class MetadataService {
await this.narratorService.replaceForBook(bookId, narratorNames);
}
async replaceGenres(bookId: number, parsedGenres: string[]) {
async replaceGenres(bookId: number, parsedGenres: string[], options: RelationMutationOptions = {}) {
const uniqueGenres = this.normalizeUniqueRelationNames(parsedGenres);
if (options.executor) {
await this.replaceGenresInExecutor(options.executor, bookId, uniqueGenres);
return;
}
await this.db.transaction(async (tx) => {
await tx.delete(bookGenres).where(eq(bookGenres.bookId, bookId));
if (uniqueGenres.length === 0) return;
for (const genreName of uniqueGenres) {
let [genre] = await tx.insert(genres).values({ name: genreName }).onConflictDoNothing().returning();
if (!genre) {
[genre] = await tx.select().from(genres).where(eq(genres.name, genreName)).limit(1);
}
if (!genre) continue;
await tx.insert(bookGenres).values({ bookId, genreId: genre.id }).onConflictDoNothing();
}
await this.replaceGenresInExecutor(tx, bookId, uniqueGenres);
});
}
// ── Tags ─────────────────────────────────────────────────────────────────────
async replaceTags(bookId: number, userTags: string[]) {
async replaceTags(bookId: number, userTags: string[], options: RelationMutationOptions = {}) {
const uniqueTags = this.normalizeUniqueRelationNames(userTags);
if (options.executor) {
await this.replaceTagsInExecutor(options.executor, bookId, uniqueTags);
return;
}
await this.db.transaction(async (tx) => {
await tx.delete(bookTags).where(eq(bookTags.bookId, bookId));
if (uniqueTags.length === 0) return;
for (const tagName of uniqueTags) {
let [tag] = await tx.insert(tags).values({ name: tagName }).onConflictDoNothing().returning();
if (!tag) {
[tag] = await tx.select().from(tags).where(eq(tags.name, tagName)).limit(1);
}
if (!tag) continue;
await tx.insert(bookTags).values({ bookId, tagId: tag.id }).onConflictDoNothing();
}
await this.replaceTagsInExecutor(tx, bookId, uniqueTags);
});
}
private async replaceAuthorsInExecutor(
executor: RelationMutationExecutor,
bookId: number,
uniqueAuthors: { name: string; sortName: string | null }[],
): Promise<number[]> {
await executor.delete(bookAuthors).where(eq(bookAuthors.bookId, bookId));
if (uniqueAuthors.length === 0) return [];
const authorIds: number[] = [];
for (let index = 0; index < uniqueAuthors.length; index += 1) {
const { name, sortName } = uniqueAuthors[index];
let [author] = await executor.insert(authors).values({ name, sortName }).onConflictDoNothing().returning();
if (!author) {
[author] = await executor.select().from(authors).where(eq(authors.name, name)).limit(1);
}
if (!author) continue;
await executor.insert(bookAuthors).values({ bookId, authorId: author.id, displayOrder: index }).onConflictDoNothing();
authorIds.push(author.id);
}
return authorIds;
}
private async replaceGenresInExecutor(executor: RelationMutationExecutor, bookId: number, uniqueGenres: string[]): Promise<void> {
await executor.delete(bookGenres).where(eq(bookGenres.bookId, bookId));
if (uniqueGenres.length === 0) return;
for (const genreName of uniqueGenres) {
let [genre] = await executor.insert(genres).values({ name: genreName }).onConflictDoNothing().returning();
if (!genre) {
[genre] = await executor.select().from(genres).where(eq(genres.name, genreName)).limit(1);
}
if (!genre) continue;
await executor.insert(bookGenres).values({ bookId, genreId: genre.id }).onConflictDoNothing();
}
}
private async replaceTagsInExecutor(executor: RelationMutationExecutor, bookId: number, uniqueTags: string[]): Promise<void> {
await executor.delete(bookTags).where(eq(bookTags.bookId, bookId));
if (uniqueTags.length === 0) return;
for (const tagName of uniqueTags) {
let [tag] = await executor.insert(tags).values({ name: tagName }).onConflictDoNothing().returning();
if (!tag) {
[tag] = await executor.select().from(tags).where(eq(tags.name, tagName)).limit(1);
}
if (!tag) continue;
await executor.insert(bookTags).values({ bookId, tagId: tag.id }).onConflictDoNothing();
}
}
// ── Persistence ──────────────────────────────────────────────────────────────
private async persistMetadata(bookId: number, data: ParsedBookData, format: string): Promise<void> {
@@ -7,29 +7,43 @@ import * as schema from '../../db/schema';
import { bookNarrators, narrators } from '../../db/schema';
type Db = NodePgDatabase<typeof schema>;
type NarratorMutationExecutor = Pick<Db, 'delete' | 'insert' | 'select'>;
@Injectable()
export class NarratorRepository {
constructor(@Inject(DB) private readonly db: Db) {}
async replaceForBook(bookId: number, names: { name: string; sortName: string | null }[]): Promise<void> {
async replaceForBook(bookId: number, names: { name: string; sortName: string | null }[], executor?: NarratorMutationExecutor): Promise<void> {
if (executor) {
await this.replaceForBookWithExecutor(executor, bookId, names);
return;
}
await this.db.transaction(async (tx) => {
await tx.delete(bookNarrators).where(eq(bookNarrators.bookId, bookId));
for (let i = 0; i < names.length; i++) {
const { name, sortName } = names[i];
// onConflictDoUpdate with a no-op SET guarantees a RETURNING row regardless of
// whether the INSERT or the conflict path was taken, avoiding a read-committed
// race where onConflictDoNothing + subsequent SELECT could return nothing.
await tx.insert(narrators).values({ name, sortName }).onConflictDoUpdate({ target: narrators.name, set: { name } });
const [narrator] = await tx.select({ id: narrators.id }).from(narrators).where(eq(narrators.name, name)).limit(1);
if (narrator) {
await tx.insert(bookNarrators).values({ bookId, narratorId: narrator.id, displayOrder: i }).onConflictDoNothing();
}
}
await this.replaceForBookWithExecutor(tx, bookId, names);
});
}
private async replaceForBookWithExecutor(
executor: NarratorMutationExecutor,
bookId: number,
names: { name: string; sortName: string | null }[],
): Promise<void> {
await executor.delete(bookNarrators).where(eq(bookNarrators.bookId, bookId));
for (let i = 0; i < names.length; i++) {
const { name, sortName } = names[i];
// onConflictDoUpdate with a no-op SET guarantees a RETURNING row regardless of
// whether the INSERT or the conflict path was taken, avoiding a read-committed
// race where onConflictDoNothing + subsequent SELECT could return nothing.
await executor.insert(narrators).values({ name, sortName }).onConflictDoUpdate({ target: narrators.name, set: { name } });
const [narrator] = await executor.select({ id: narrators.id }).from(narrators).where(eq(narrators.name, name)).limit(1);
if (narrator) {
await executor.insert(bookNarrators).values({ bookId, narratorId: narrator.id, displayOrder: i }).onConflictDoNothing();
}
}
}
}
@@ -1,17 +1,30 @@
import { Injectable } from '@nestjs/common';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import * as schema from '../../db/schema';
import { NarratorRepository } from './narrator.repository';
type Db = NodePgDatabase<typeof schema>;
type NarratorMutationExecutor = Pick<Db, 'delete' | 'insert' | 'select'>;
@Injectable()
export class NarratorService {
constructor(private readonly narratorRepo: NarratorRepository) {}
async replaceForBook(bookId: number, names: string[] | { name: string; sortName: string | null }[]): Promise<void> {
async replaceForBook(
bookId: number,
names: string[] | { name: string; sortName: string | null }[],
options: { executor?: NarratorMutationExecutor } = {},
): Promise<void> {
const normalized =
names.length > 0 && typeof names[0] !== 'string'
? (names as { name: string; sortName: string | null }[])
: (names as string[]).map((name) => ({ name, sortName: null }));
if (options.executor) {
await this.narratorRepo.replaceForBook(bookId, normalized, options.executor);
return;
}
await this.narratorRepo.replaceForBook(bookId, normalized);
}
}
File diff suppressed because it is too large Load Diff