diff --git a/.github/workflows/e2e-metadata-write.yml b/.github/workflows/e2e-metadata-write.yml index a96855cf0..ceeddcdc6 100644 --- a/.github/workflows/e2e-metadata-write.yml +++ b/.github/workflows/e2e-metadata-write.yml @@ -3,6 +3,12 @@ run-name: ${{ format('Metadata Write E2E - {0}', github.ref_name) }} on: workflow_dispatch: + inputs: + vitest-args: + description: "Optional vitest args (examples: --shard=1/2, --testNamePattern=\"concurrency hardening\")" + required: false + default: "" + type: string push: paths: - "server/src/modules/book/**" @@ -46,3 +52,4 @@ jobs: check-name: Server Metadata Write E2E artifact-name: metadata-write-e2e-report job-timeout-minutes: 60 + vitest-args: ${{ github.event_name == 'workflow_dispatch' && inputs.vitest-args || '' }} diff --git a/.github/workflows/e2e-runner.yml b/.github/workflows/e2e-runner.yml index 04647dfd5..6ef494f94 100644 --- a/.github/workflows/e2e-runner.yml +++ b/.github/workflows/e2e-runner.yml @@ -20,6 +20,11 @@ on: required: false type: number default: 45 + vitest-args: + description: "Optional extra args forwarded to vitest (for shard/testNamePattern reruns)" + required: false + type: string + default: "" env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" @@ -65,7 +70,12 @@ jobs: - name: Run e2e suite shell: bash - run: pnpm run test:e2e -- ${{ inputs.suite-id }} + run: | + if [ -n "${{ inputs.vitest-args }}" ]; then + pnpm run test:e2e -- ${{ inputs.suite-id }} ${{ inputs.vitest-args }} + else + pnpm run test:e2e -- ${{ inputs.suite-id }} + fi - name: Publish test report annotations if: always() diff --git a/package.json b/package.json index e688f60f3..5bbfcd137 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,9 @@ "test:client": "pnpm --filter client test:unit --run", "test:e2e": "node ./scripts/test/run-e2e-suite.mjs", "test:e2e:all": "node ./scripts/test/run-e2e-suite.mjs all", + "test:e2e:metadata-write": "node ./scripts/test/run-e2e-suite.mjs metadata-write", + "test:e2e:metadata-write:atomicity": "node ./scripts/test/run-e2e-suite.mjs metadata-write --testNamePattern=\"atomicity failpoints\"", + "test:e2e:metadata-write:concurrency": "node ./scripts/test/run-e2e-suite.mjs metadata-write --testNamePattern=\"concurrency hardening\"", "test:e2e:list": "node ./scripts/test/run-e2e-suite.mjs --list", "coverage": "pnpm run coverage:server && pnpm run coverage:client", "coverage:server": "pnpm --filter server test:cov", diff --git a/scripts/test/run-e2e-suite.mjs b/scripts/test/run-e2e-suite.mjs index ed4edcdac..62db1ff05 100644 --- a/scripts/test/run-e2e-suite.mjs +++ b/scripts/test/run-e2e-suite.mjs @@ -33,6 +33,9 @@ function printUsage() { "Examples:", " pnpm run test:e2e -- smoke", " pnpm run test:e2e -- all", + " pnpm run test:e2e -- metadata-write", + " pnpm run test:e2e -- metadata-write --testNamePattern=\"concurrency hardening\"", + " pnpm run test:e2e -- metadata-write --shard=1/2", " pnpm run test:e2e -- scanner --testNamePattern=book-per-folder-disc-folder-flattening", " pnpm run test:e2e -- --list", " pnpm run test:e2e:list", diff --git a/server/src/modules/book/book.repository.ts b/server/src/modules/book/book.repository.ts index 5f1397ec7..fcb715580 100644 --- a/server/src/modules/book/book.repository.ts +++ b/server/src/modules/book/book.repository.ts @@ -29,6 +29,8 @@ import { } from '../../db/schema'; type Db = NodePgDatabase; +type DbTransaction = Parameters[0]>[0]; +type MetadataUpdateExecutor = Pick; type PatternMetadataRow = { bookId: number; title: string | null; @@ -46,6 +48,10 @@ type PatternMetadataRow = { export class BookRepository { constructor(@Inject(DB) private readonly db: Db) {} + async withTransaction(callback: (tx: DbTransaction) => Promise): Promise { + return this.db.transaction((tx) => callback(tx)); + } + async findCards(opts: { where: SQL | undefined; orderBy: SQL[]; limit: number; offset: number; userId: number }) { const { where, orderBy, limit, offset, userId } = opts; @@ -444,8 +450,12 @@ export class BookRepository { await this.db.delete(books).where(inArray(books.id, bookIds)); } - async updateMetadataFields(bookId: number, fields: Partial): Promise { - await this.db.update(bookMetadata).set(fields).where(eq(bookMetadata.bookId, bookId)); + async updateMetadataFields( + bookId: number, + fields: Partial, + executor: MetadataUpdateExecutor = this.db, + ): Promise { + await executor.update(bookMetadata).set(fields).where(eq(bookMetadata.bookId, bookId)); } async upsertProgress( diff --git a/server/src/modules/book/book.service.test.ts b/server/src/modules/book/book.service.test.ts index 6dfc1cbe9..087d5e278 100644 --- a/server/src/modules/book/book.service.test.ts +++ b/server/src/modules/book/book.service.test.ts @@ -58,6 +58,7 @@ function makeService() { findAudioProgress: vi.fn(), upsertAudioProgress: vi.fn(), updateMetadataFields: vi.fn(), + withTransaction: vi.fn(), deleteByIds: vi.fn(), findAllIds: vi.fn(), }; @@ -70,9 +71,10 @@ function makeService() { buildOrderBy: vi.fn(), }; const metadataService = { - replaceAuthors: vi.fn().mockResolvedValue(undefined), + replaceAuthors: vi.fn().mockResolvedValue([]), replaceGenres: vi.fn().mockResolvedValue(undefined), replaceTags: vi.fn().mockResolvedValue(undefined), + emitAuthorsReplaced: vi.fn(), downloadAndSaveCover: vi.fn().mockResolvedValue(undefined), refreshCoverForBook: vi.fn(), }; @@ -95,6 +97,13 @@ function makeService() { const fileWriteService = { scheduleWrite: vi.fn(), }; + const narratorService = { + replaceForBook: vi.fn().mockResolvedValue(undefined), + }; + const comicMetadataService = { + upsert: vi.fn().mockResolvedValue(undefined), + findByBookId: vi.fn().mockResolvedValue(null), + }; const userBookStatusService = { autoUpdate: vi.fn().mockResolvedValue(undefined), setManual: vi.fn().mockResolvedValue(undefined), @@ -102,6 +111,8 @@ function makeService() { findByBookIds: vi.fn().mockResolvedValue(new Map()), }; + bookRepo.withTransaction.mockImplementation(async (callback: (tx: unknown) => Promise) => callback({})); + const service = new BookService( bookRepo as never, libraryService as never, @@ -112,8 +123,8 @@ function makeService() { config as never, appSettings as never, userBookStatusService as never, - { replaceForBook: vi.fn().mockResolvedValue(undefined) } as never, - { upsert: vi.fn().mockResolvedValue(undefined), findByBookId: vi.fn().mockResolvedValue(null) } as never, + narratorService as never, + comicMetadataService as never, embedder as never, fileWriteService as never, ); @@ -130,6 +141,8 @@ function makeService() { userBookStatusService, embedder, fileWriteService, + narratorService, + comicMetadataService, }; } @@ -638,6 +651,7 @@ describe('BookService', () => { ); expect(verifySpy).toHaveBeenCalledWith(5, user); + expect(bookRepo.withTransaction).toHaveBeenCalledTimes(1); expect(bookRepo.updateMetadataFields).toHaveBeenCalledWith( 5, expect.objectContaining({ @@ -645,13 +659,19 @@ describe('BookService', () => { rating: 4, updatedAt: expect.any(Date), }), + expect.anything(), ); - expect(metadataService.replaceAuthors).toHaveBeenCalledWith(5, [ - { name: 'A1', sortName: null }, - { name: 'A2', sortName: null }, - ]); - expect(metadataService.replaceGenres).toHaveBeenCalledWith(5, ['Sci-Fi']); - expect(metadataService.replaceTags).toHaveBeenCalledWith(5, ['favorite']); + expect(metadataService.replaceAuthors).toHaveBeenCalledWith( + 5, + [ + { name: 'A1', sortName: null }, + { name: 'A2', sortName: null }, + ], + { executor: expect.anything(), emitEvent: false }, + ); + expect(metadataService.replaceGenres).toHaveBeenCalledWith(5, ['Sci-Fi'], { executor: expect.anything() }); + expect(metadataService.replaceTags).toHaveBeenCalledWith(5, ['favorite'], { executor: expect.anything() }); + expect(metadataService.emitAuthorsReplaced).toHaveBeenCalledWith(5, []); expect(fileWriteService.scheduleWrite).toHaveBeenCalledWith(5, 'auto', user.id); expect(embedder.embedBook).toHaveBeenCalledWith(5); expect(detailSpy).toHaveBeenCalledWith(5, user); diff --git a/server/src/modules/book/book.service.ts b/server/src/modules/book/book.service.ts index 2fc0a502f..7ada90e46 100644 --- a/server/src/modules/book/book.service.ts +++ b/server/src/modules/book/book.service.ts @@ -1,4 +1,4 @@ -import { BadRequestException, Injectable, Logger, NotFoundException, Optional } from '@nestjs/common'; +import { BadRequestException, Injectable, InternalServerErrorException, Logger, NotFoundException, Optional } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { access, readdir, rm, stat } from 'fs/promises'; import { basename, extname, join } from 'path'; @@ -25,11 +25,24 @@ import { SaveProgressDto } from './dto/save-progress.dto'; import { UpsertAudioProgressDto } from './dto/upsert-audio-progress.dto'; import { UpdateBookMetadataDto } from './dto/update-book-metadata.dto'; +const METADATA_UPDATE_FAILPOINTS = [ + 'afterScalarUpdate', + 'afterComicMetadataUpsert', + 'afterAuthorsReplace', + 'afterNarratorsReplace', + 'afterGenresReplace', + 'afterTagsReplace', + 'beforeTransactionCommit', +] as const; + +export type MetadataUpdateFailpoint = (typeof METADATA_UPDATE_FAILPOINTS)[number]; + @Injectable() export class BookService { private readonly logger = new Logger(BookService.name); private readonly booksPath: string; private embeddingRun: Promise | null = null; + private metadataUpdateFailpoint: MetadataUpdateFailpoint | null = null; constructor( private readonly bookRepo: BookRepository, @@ -404,30 +417,46 @@ export class BookService { } const scalarFieldCount = Object.keys(scalarFields).length; - if (scalarFieldCount > 0) { - scalarFields.updatedAt = new Date(); - await this.bookRepo.updateMetadataFields(id, scalarFields); - } + let replacedAuthorIds: number[] = []; + await this.bookRepo.withTransaction(async (tx) => { + if (scalarFieldCount > 0) { + scalarFields.updatedAt = new Date(); + await this.bookRepo.updateMetadataFields(id, scalarFields, tx); + } + this.throwIfMetadataUpdateFailpoint('afterScalarUpdate'); - if (dto.comicMetadata) { - await this.comicMetadataService.upsert(id, dto.comicMetadata); - } + if (dto.comicMetadata) { + await this.comicMetadataService.upsert(id, dto.comicMetadata, tx); + } + this.throwIfMetadataUpdateFailpoint('afterComicMetadataUpsert'); - if (dto.authors !== undefined) { - await this.metadataService.replaceAuthors( - id, - dto.authors.map((name) => ({ name, sortName: null })), - ); - } - if (dto.audioMetadata?.narrators !== undefined) { - await this.narratorService.replaceForBook(id, dto.audioMetadata.narrators); - } - if (dto.genres !== undefined) { - await this.metadataService.replaceGenres(id, dto.genres); - } - if (dto.tags !== undefined) { - await this.metadataService.replaceTags(id, dto.tags); - } + if (dto.authors !== undefined) { + replacedAuthorIds = await this.metadataService.replaceAuthors( + id, + dto.authors.map((name) => ({ name, sortName: null })), + { executor: tx, emitEvent: false }, + ); + } + this.throwIfMetadataUpdateFailpoint('afterAuthorsReplace'); + + if (dto.audioMetadata?.narrators !== undefined) { + await this.narratorService.replaceForBook(id, dto.audioMetadata.narrators, { executor: tx }); + } + this.throwIfMetadataUpdateFailpoint('afterNarratorsReplace'); + + if (dto.genres !== undefined) { + await this.metadataService.replaceGenres(id, dto.genres, { executor: tx }); + } + this.throwIfMetadataUpdateFailpoint('afterGenresReplace'); + + if (dto.tags !== undefined) { + await this.metadataService.replaceTags(id, dto.tags, { executor: tx }); + } + this.throwIfMetadataUpdateFailpoint('afterTagsReplace'); + this.throwIfMetadataUpdateFailpoint('beforeTransactionCommit'); + }); + + this.metadataService.emitAuthorsReplaced(id, replacedAuthorIds); this.embedder?.embedBook(id).catch((err: Error) => this.logger.warn(`Embedding failed for book ${id}: ${err.message}`)); this.fileWriteService?.scheduleWrite(id, 'auto', user.id); @@ -447,6 +476,23 @@ export class BookService { } } + setMetadataUpdateFailpointForTests(failpoint: MetadataUpdateFailpoint | null): void { + if (failpoint !== null && !METADATA_UPDATE_FAILPOINTS.includes(failpoint)) { + throw new InternalServerErrorException(`Unknown metadata update failpoint: ${failpoint}`); + } + this.metadataUpdateFailpoint = failpoint; + } + + clearMetadataUpdateFailpointForTests(): void { + this.metadataUpdateFailpoint = null; + } + + private throwIfMetadataUpdateFailpoint(stage: MetadataUpdateFailpoint): void { + if (this.metadataUpdateFailpoint !== stage) return; + this.metadataUpdateFailpoint = null; + throw new InternalServerErrorException(`Metadata update failpoint triggered: ${stage}`); + } + async embedAll(): Promise<{ queued: number }> { const event = 'book.embed_all'; const startedAt = Date.now(); diff --git a/server/src/modules/file-write/formats/cbx/cb7-format-writer.ts b/server/src/modules/file-write/formats/cbx/cb7-format-writer.ts index 9d9ddbd56..b2c7c5805 100644 --- a/server/src/modules/file-write/formats/cbx/cb7-format-writer.ts +++ b/server/src/modules/file-write/formats/cbx/cb7-format-writer.ts @@ -5,7 +5,7 @@ import { randomUUID } from 'crypto'; import type { WriteResult } from '@projectx/types'; import { getSevenZip, type SevenZipFS } from '../../../../common/sevenzip'; -import type { BookWritePayload } from '../../interfaces/book-write-payload.interface'; +import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface'; import type { FormatWriter } from '../../interfaces/format-writer.interface'; import type { FormatWriteOptions } from '../../interfaces/format-write-options.interface'; import { replaceFileAtomically } from '../shared/atomic-file-replace'; @@ -19,6 +19,28 @@ import { buildComicInfoXml } from './comic-info-builder'; // is always written before it is added to the archive, with no await in between. const VFS_XML_PATH = '/ComicInfo.xml'; const ENOENT_ERRNO = 44; +const CBX_WRITABLE_FIELDS = new Set([ + 'title', + 'subtitle', + 'description', + 'publisher', + 'publishedYear', + 'language', + 'pageCount', + 'seriesName', + 'seriesIndex', + 'isbn10', + 'isbn13', + 'rating', + 'authors', + 'genres', + 'tags', + 'googleBooksId', + 'goodreadsId', + 'amazonId', + 'hardcoverId', + 'openLibraryId', +]); @Injectable() export class Cb7FormatWriter implements FormatWriter { @@ -27,7 +49,8 @@ export class Cb7FormatWriter implements FormatWriter { async write(filePath: string, payload: BookWritePayload, options: FormatWriteOptions): Promise { const start = Date.now(); const { fieldMask, dryRun } = options; - const fieldsWritten = resolveFieldsWritten(payload, fieldMask); + const cbxFieldMask = new Set([...fieldMask].filter((key) => CBX_WRITABLE_FIELDS.has(key))); + const fieldsWritten = resolveFieldsWritten(payload, cbxFieldMask); if (dryRun) { return { status: 'skipped', reason: 'dry-run', fieldsWritten, durationMs: Date.now() - start }; @@ -61,7 +84,7 @@ export class Cb7FormatWriter implements FormatWriter { // No ComicInfo.xml in archive - start fresh } - const xml = buildComicInfoXml(existingXml, payload, fieldMask); + const xml = buildComicInfoXml(existingXml, payload, cbxFieldMask); const xmlBytes = Buffer.from(xml, 'utf-8'); const fd2 = sz.FS.open(VFS_XML_PATH, 'w+'); diff --git a/server/src/modules/file-write/formats/cbx/cbz-format-writer.ts b/server/src/modules/file-write/formats/cbx/cbz-format-writer.ts index 5183f6c5a..9a57e150d 100644 --- a/server/src/modules/file-write/formats/cbx/cbz-format-writer.ts +++ b/server/src/modules/file-write/formats/cbx/cbz-format-writer.ts @@ -1,13 +1,36 @@ import { Injectable } from '@nestjs/common'; import type { WriteResult } from '@projectx/types'; -import type { BookWritePayload } from '../../interfaces/book-write-payload.interface'; +import type { BookWritePayload, BookWritePayloadKey } from '../../interfaces/book-write-payload.interface'; import type { FormatWriter } from '../../interfaces/format-writer.interface'; import type { FormatWriteOptions } from '../../interfaces/format-write-options.interface'; import { resolveFieldsWritten } from '../shared/resolve-fields-written'; import { buildComicInfoXml } from './comic-info-builder'; import { readComicInfoFromZip, writeComicInfoToZip } from './cbz-zip-patcher'; +const CBX_WRITABLE_FIELDS = new Set([ + 'title', + 'subtitle', + 'description', + 'publisher', + 'publishedYear', + 'language', + 'pageCount', + 'seriesName', + 'seriesIndex', + 'isbn10', + 'isbn13', + 'rating', + 'authors', + 'genres', + 'tags', + 'googleBooksId', + 'goodreadsId', + 'amazonId', + 'hardcoverId', + 'openLibraryId', +]); + @Injectable() export class CbzFormatWriter implements FormatWriter { readonly format = 'cbz'; @@ -15,14 +38,15 @@ export class CbzFormatWriter implements FormatWriter { async write(filePath: string, payload: BookWritePayload, options: FormatWriteOptions): Promise { const start = Date.now(); const { fieldMask, dryRun } = options; - const fieldsWritten = resolveFieldsWritten(payload, fieldMask); + const cbxFieldMask = new Set([...fieldMask].filter((key) => CBX_WRITABLE_FIELDS.has(key))); + const fieldsWritten = resolveFieldsWritten(payload, cbxFieldMask); if (dryRun) { return { status: 'skipped', reason: 'dry-run', fieldsWritten, durationMs: Date.now() - start }; } const existingXml = await readComicInfoFromZip(filePath); - const xml = buildComicInfoXml(existingXml, payload, fieldMask); + const xml = buildComicInfoXml(existingXml, payload, cbxFieldMask); await writeComicInfoToZip(filePath, xml); return { status: 'success', fieldsWritten, durationMs: Date.now() - start }; diff --git a/server/src/modules/file-write/formats/cbx/comic-info-builder.ts b/server/src/modules/file-write/formats/cbx/comic-info-builder.ts index a64225e15..dfd14abe9 100644 --- a/server/src/modules/file-write/formats/cbx/comic-info-builder.ts +++ b/server/src/modules/file-write/formats/cbx/comic-info-builder.ts @@ -35,19 +35,19 @@ function parseComicInfoXml(xml: string): ComicInfoObject { export function buildComicInfoXml(existingXml: string | null, payload: BookWritePayload, fieldMask: Set): string { const info: ComicInfoObject = existingXml ? parseComicInfoXml(existingXml) : {}; - if (fieldMask.has('title') && payload.title != null) info['Title'] = payload.title; - if (fieldMask.has('description') && payload.description != null) info['Summary'] = stripHtml(payload.description); - if (fieldMask.has('publisher') && payload.publisher != null) info['Publisher'] = payload.publisher; - if (fieldMask.has('seriesName') && payload.seriesName != null) info['Series'] = payload.seriesName; - if (fieldMask.has('seriesIndex') && payload.seriesIndex != null) info['Number'] = formatSeriesIndex(payload.seriesIndex); - if (fieldMask.has('publishedYear') && payload.publishedYear != null) info['Year'] = payload.publishedYear; - if (fieldMask.has('pageCount') && payload.pageCount != null) info['PageCount'] = payload.pageCount; - if (fieldMask.has('language') && payload.language != null) info['LanguageISO'] = payload.language; - if (fieldMask.has('authors') && payload.authors?.length) info['Writer'] = payload.authors.map((a) => a.name).join(', '); - if (fieldMask.has('genres') && payload.genres?.length) info['Genre'] = payload.genres.join(', '); - if (fieldMask.has('tags') && payload.tags?.length) info['Tags'] = payload.tags.join(', '); - if (fieldMask.has('rating') && payload.rating != null) info['CommunityRating'] = formatRating(payload.rating); - if (fieldMask.has('isbn13') && payload.isbn13 != null) info['GTIN'] = payload.isbn13; + setComicInfoField(info, fieldMask, 'title', 'Title', payload.title); + setComicInfoField(info, fieldMask, 'description', 'Summary', payload.description, stripHtml); + setComicInfoField(info, fieldMask, 'publisher', 'Publisher', payload.publisher); + setComicInfoField(info, fieldMask, 'seriesName', 'Series', payload.seriesName); + setComicInfoField(info, fieldMask, 'seriesIndex', 'Number', payload.seriesIndex, formatSeriesIndex); + setComicInfoField(info, fieldMask, 'publishedYear', 'Year', payload.publishedYear); + setComicInfoField(info, fieldMask, 'pageCount', 'PageCount', payload.pageCount); + setComicInfoField(info, fieldMask, 'language', 'LanguageISO', payload.language); + setComicInfoField(info, fieldMask, 'authors', 'Writer', payload.authors?.length ? payload.authors.map((a) => a.name).join(', ') : null); + setComicInfoField(info, fieldMask, 'genres', 'Genre', payload.genres?.length ? payload.genres.join(', ') : null); + setComicInfoField(info, fieldMask, 'tags', 'Tags', payload.tags?.length ? payload.tags.join(', ') : null); + setComicInfoField(info, fieldMask, 'rating', 'CommunityRating', payload.rating, formatRating); + setComicInfoField(info, fieldMask, 'isbn13', 'GTIN', payload.isbn13); const hasProviderSelection = COMIC_INFO_PROVIDER_ID_KEYS.some((key) => fieldMask.has(key)); if (hasProviderSelection) { @@ -154,3 +154,20 @@ function resolveManagedTextField(payload: BookWritePayload, key: BookWritePayloa const value = payload[key]; return typeof value === 'string' ? value : null; } + +function setComicInfoField( + info: ComicInfoObject, + fieldMask: Set, + key: BookWritePayloadKey, + comicInfoKey: string, + value: TValue | null | undefined, + formatter?: (value: TValue) => string | number, +): void { + if (!fieldMask.has(key)) return; + if (value == null || (typeof value === 'string' && value === '')) { + delete info[comicInfoKey]; + return; + } + + info[comicInfoKey] = formatter ? formatter(value) : value; +} diff --git a/server/src/modules/metadata/comic-metadata.repository.ts b/server/src/modules/metadata/comic-metadata.repository.ts index adb7ef51f..1c89f8047 100644 --- a/server/src/modules/metadata/comic-metadata.repository.ts +++ b/server/src/modules/metadata/comic-metadata.repository.ts @@ -7,13 +7,14 @@ import * as schema from '../../db/schema'; import { ComicMetadataDto } from '../book/dto/update-book-metadata.dto'; type Db = NodePgDatabase; +type ComicMetadataExecutor = Pick; @Injectable() export class ComicMetadataRepository { constructor(@Inject(DB) private readonly db: Db) {} - async upsert(bookId: number, fields: ComicMetadataDto): Promise { - await this.db + async upsert(bookId: number, fields: ComicMetadataDto, executor: ComicMetadataExecutor = this.db): Promise { + await executor .insert(schema.comicMetadata) .values({ bookId, diff --git a/server/src/modules/metadata/extractors/comic-format.extractor.ts b/server/src/modules/metadata/extractors/comic-format.extractor.ts index c2895baeb..2d25e969f 100644 --- a/server/src/modules/metadata/extractors/comic-format.extractor.ts +++ b/server/src/modules/metadata/extractors/comic-format.extractor.ts @@ -38,7 +38,7 @@ export class ComicFormatExtractor implements FormatExtractor { seriesName: comicMetadata?.seriesName ?? null, seriesIndex: comicMetadata?.seriesIndex ?? null, authors: comicMetadata?.authors ?? [], - genres: comicMetadata?.tags ?? [], + genres: comicMetadata?.genres?.length ? comicMetadata.genres : (comicMetadata?.tags ?? []), cover: cover ?? null, comicMetadata: comicMetadata?.comicMetadata ?? null, }; diff --git a/server/src/modules/metadata/extractors/epub-format.extractor.ts b/server/src/modules/metadata/extractors/epub-format.extractor.ts index eb2cecab7..8ba2c4887 100644 --- a/server/src/modules/metadata/extractors/epub-format.extractor.ts +++ b/server/src/modules/metadata/extractors/epub-format.extractor.ts @@ -18,7 +18,7 @@ export class EpubFormatExtractor implements FormatExtractor { seriesName: metadata.seriesName, seriesIndex: metadata.seriesIndex, authors: metadata.authors, - genres: metadata.tags, + genres: metadata.genres.length > 0 ? metadata.genres : metadata.tags, cover: cover ?? null, }; } diff --git a/server/src/modules/metadata/lib/cbz-metadata.test.ts b/server/src/modules/metadata/lib/cbz-metadata.test.ts index 77cd9af1a..f566ba234 100644 --- a/server/src/modules/metadata/lib/cbz-metadata.test.ts +++ b/server/src/modules/metadata/lib/cbz-metadata.test.ts @@ -121,13 +121,15 @@ describe('extractCbzMetadata', () => { expect(r?.authors.map((a) => a.name)).toEqual(['Writer One', 'Writer Two']); }); - it('merges Genre and Tags into deduplicated tags array', async () => { + it('extracts Genre and Tags into separate deduplicated lists', async () => { const xml = `Superhero,FantasyFantasy,Horror`; mockReadFile.mockResolvedValue(buildZipWithComicInfo(xml) as unknown as Buffer); const r = await extractCbzMetadata('/book.cbz'); - // "Fantasy" appears in both Genre and Tags — must be deduplicated - expect(r?.tags).toContain('Superhero'); + expect(r?.genres).toContain('Superhero'); + expect(r?.genres).toContain('Fantasy'); + expect(r?.genres?.filter((t) => t === 'Fantasy')).toHaveLength(1); + expect(r?.tags).toContain('Fantasy'); expect(r?.tags).toContain('Horror'); expect(r?.tags?.filter((t) => t === 'Fantasy')).toHaveLength(1); diff --git a/server/src/modules/metadata/lib/cbz-metadata.ts b/server/src/modules/metadata/lib/cbz-metadata.ts index 43b59dcb1..073772cd5 100644 --- a/server/src/modules/metadata/lib/cbz-metadata.ts +++ b/server/src/modules/metadata/lib/cbz-metadata.ts @@ -25,14 +25,26 @@ export interface ParsedCbzComicMetadata { export interface ParsedCbzMetadata { title: string | null; + subtitle: string | null; seriesName: string | null; seriesIndex: number | null; description: string | null; publisher: string | null; publishedYear: number | null; language: string | null; + pageCount: number | null; + rating: number | null; + isbn10: string | null; + isbn13: string | null; authors: { name: string; sortName: string | null }[]; + genres: string[]; tags: string[]; + googleBooksId: string | null; + goodreadsId: string | null; + amazonId: string | null; + hardcoverId: string | null; + openLibraryId: string | null; + itunesId: string | null; comicMetadata: ParsedCbzComicMetadata | null; } @@ -124,6 +136,59 @@ function splitDelimited(value: string | null | undefined): string[] { .filter(Boolean); } +function parseProjectxManagedNotes(notes: string | null): Map { + const fields = new Map(); + if (!notes) return fields; + + for (const line of notes.split('\n')) { + const trimmed = line.trim(); + if (!trimmed) continue; + const match = trimmed.match(/^\[projectx:([^\]]+)\]\s*(.*)$/); + if (!match) continue; + const [, key, value] = match; + fields.set(key, value.trim()); + } + + return fields; +} + +function parseProviderIdsFromWebUrl( + webUrl: string | null, +): Partial> { + if (!webUrl) return {}; + + if (webUrl.startsWith('https://www.goodreads.com/book/show/')) { + return { goodreadsId: webUrl.slice('https://www.goodreads.com/book/show/'.length) || '' }; + } + if (webUrl.startsWith('https://www.amazon.com/dp/')) { + return { amazonId: webUrl.slice('https://www.amazon.com/dp/'.length) || '' }; + } + if (webUrl.startsWith('https://hardcover.app/books/')) { + return { hardcoverId: webUrl.slice('https://hardcover.app/books/'.length) || '' }; + } + if (webUrl.startsWith('https://books.google.com/books?id=')) { + return { googleBooksId: webUrl.slice('https://books.google.com/books?id='.length) || '' }; + } + if (webUrl.startsWith('https://openlibrary.org/works/')) { + return { openLibraryId: webUrl.slice('https://openlibrary.org/works/'.length) || '' }; + } + + return {}; +} + +function parseCbxRating(value: string | null): number | null { + if (!value) return null; + const parsed = Number(value); + if (!Number.isFinite(parsed)) return null; + return Math.min(5, Math.max(0, parsed * 2)); +} + +function parseIsbn13(value: string | null): string | null { + if (!value) return null; + const digits = value.replace(/[^\d]/g, ''); + return digits.length === 13 ? digits : null; +} + function parseComicInfoXml(xmlBuf: Buffer): ParsedCbzMetadata | null { try { const parsed = xmlParser.parse(xmlBuf.toString('utf-8')) as Record; @@ -143,8 +208,12 @@ function parseComicInfoXml(xmlBuf: Buffer): ParsedCbzMetadata | null { const writers = splitDelimited(str('Writer')); const year = num('Year'); + const managedNotes = parseProjectxManagedNotes(str('Notes')); + const providerIdsFromWeb = parseProviderIdsFromWebUrl(str('Web')); - const tags = [...splitDelimited(str('Genre')), ...splitDelimited(str('Tags'))]; + const genres = splitDelimited(str('Genre')); + const tags = splitDelimited(str('Tags')); + const uniqueGenres = [...new Set(genres)]; const uniqueTags = [...new Set(tags)]; const pencillers = splitDelimited(str('Penciller')); @@ -172,14 +241,26 @@ function parseComicInfoXml(xmlBuf: Buffer): ParsedCbzMetadata | null { return { title: str('Title'), + subtitle: managedNotes.get('subtitle') ?? null, seriesName: str('Series'), seriesIndex: num('Number'), description: str('Summary') ?? str('Description'), publisher: str('Publisher'), publishedYear: year != null ? Math.floor(year) : null, language: str('LanguageISO'), + pageCount: num('PageCount'), + rating: parseCbxRating(str('CommunityRating')), + isbn10: managedNotes.get('isbn10') ?? null, + isbn13: parseIsbn13(str('GTIN')), authors: writers.map((name) => ({ name, sortName: null })), + genres: uniqueGenres, tags: uniqueTags, + googleBooksId: managedNotes.get('googleBooksId') ?? providerIdsFromWeb.googleBooksId ?? null, + goodreadsId: managedNotes.get('goodreadsId') ?? providerIdsFromWeb.goodreadsId ?? null, + amazonId: managedNotes.get('amazonId') ?? providerIdsFromWeb.amazonId ?? null, + hardcoverId: managedNotes.get('hardcoverId') ?? providerIdsFromWeb.hardcoverId ?? null, + openLibraryId: managedNotes.get('openLibraryId') ?? providerIdsFromWeb.openLibraryId ?? null, + itunesId: null, comicMetadata: hasComicFields ? { issueNumber: str('Number'), @@ -215,14 +296,26 @@ function parseComicBookInfoJson(comment: string): ParsedCbzMetadata | null { return { title: (cbi['title'] as string) ?? null, + subtitle: null, seriesName: (cbi['series'] as string) ?? null, seriesIndex: cbi['issue'] != null ? (Number.isFinite(Number(cbi['issue'])) ? Number(cbi['issue']) : null) : null, description: (cbi['comments'] as string) ?? null, publisher: (cbi['publisher'] as string) ?? null, publishedYear: (cbi['publicationYear'] as number) ?? null, language: null, + pageCount: null, + rating: null, + isbn10: null, + isbn13: null, authors: writers, + genres: [], tags, + googleBooksId: null, + goodreadsId: null, + amazonId: null, + hardcoverId: null, + openLibraryId: null, + itunesId: null, comicMetadata: null, }; } catch { diff --git a/server/src/modules/metadata/lib/opf-parser.ts b/server/src/modules/metadata/lib/opf-parser.ts index 21a1b6c26..fb79d1a05 100644 --- a/server/src/modules/metadata/lib/opf-parser.ts +++ b/server/src/modules/metadata/lib/opf-parser.ts @@ -9,10 +9,19 @@ export interface ParsedOpf { publisher: string | null; publishedYear: number | null; language: string | null; + pageCount: number | null; + rating: number | null; seriesName: string | null; seriesIndex: number | null; authors: { name: string; sortName: string | null }[]; + genres: string[]; tags: string[]; + googleBooksId: string | null; + goodreadsId: string | null; + amazonId: string | null; + hardcoverId: string | null; + openLibraryId: string | null; + itunesId: string | null; } const parser = new XMLParser({ @@ -53,6 +62,26 @@ function parseYear(raw: string): number | null { return match ? parseInt(match[0], 10) : null; } +function parseNumber(raw: string | null): number | null { + if (!raw) return null; + const parsed = Number(raw); + return Number.isFinite(parsed) ? parsed : null; +} + +function parseProjectxTags(raw: string | null): string[] { + if (!raw) return []; + try { + const parsed = JSON.parse(raw) as unknown; + if (!Array.isArray(parsed)) return []; + return parsed.map((item) => String(item).trim()).filter(Boolean); + } catch { + return raw + .split(/[,;]/) + .map((item) => item.trim()) + .filter(Boolean); + } +} + // Build a map of id → array of refining meta nodes (EPUB 3). function buildRefineMap(metas: unknown[]): Map { const map = new Map(); @@ -99,6 +128,17 @@ export function parseOpf(xml: string): ParsedOpf { return null; } + function propertyMeta(property: string): string | null { + for (const m of rawMetas) { + if (typeof m !== 'object' || m === null) continue; + const mo = m as Record; + if (mo['@_property'] === property) { + return getText(m) || null; + } + } + return null; + } + // ── Titles ───────────────────────────────────────────────────────────────── let title: string | null = null; let subtitle: string | null = null; @@ -117,6 +157,7 @@ export function parseOpf(xml: string): ParsedOpf { } title ??= getText(rawTitles[0]); } + subtitle ??= namedMeta('projectx:subtitle'); // ── Authors ──────────────────────────────────────────────────────────────── const authors: { name: string; sortName: string | null }[] = []; @@ -144,6 +185,12 @@ export function parseOpf(xml: string): ParsedOpf { // ── Identifiers → ISBN ───────────────────────────────────────────────────── let isbn10: string | null = null; let isbn13: string | null = null; + let googleBooksId: string | null = null; + let goodreadsId: string | null = null; + let amazonId: string | null = null; + let hardcoverId: string | null = null; + let openLibraryId: string | null = null; + let itunesId: string | null = null; for (const ident of toArray(metadata['identifier'])) { const mo = (typeof ident === 'object' && ident !== null ? ident : {}) as Record; @@ -156,10 +203,23 @@ export function parseOpf(xml: string): ParsedOpf { isbn10 ??= parsed.isbn10; isbn13 ??= parsed.isbn13; } + + if (value.startsWith('urn:goodreads:')) goodreadsId ??= value.slice('urn:goodreads:'.length) || null; + if (value.startsWith('urn:amazon:')) amazonId ??= value.slice('urn:amazon:'.length) || null; + if (value.startsWith('urn:hardcover:')) hardcoverId ??= value.slice('urn:hardcover:'.length) || null; + if (value.startsWith('urn:google:')) googleBooksId ??= value.slice('urn:google:'.length) || null; + if (value.startsWith('urn:openlibrary:')) openLibraryId ??= value.slice('urn:openlibrary:'.length) || null; + if (value.startsWith('urn:itunes:')) itunesId ??= value.slice('urn:itunes:'.length) || null; } - // ── Tags ─────────────────────────────────────────────────────────────────── - const tags = toArray(metadata['subject']).map(getText).filter(Boolean); + isbn10 ??= propertyMeta('projectx:isbn10') ?? namedMeta('projectx:isbn10'); + + // ── Genres and tags ──────────────────────────────────────────────────────── + const genres = toArray(metadata['subject']).map(getText).filter(Boolean); + const projectxTags = parseProjectxTags(propertyMeta('projectx:tags') ?? namedMeta('projectx:tags')); + const tags = projectxTags.length > 0 ? projectxTags : genres; + const pageCount = parseNumber(propertyMeta('projectx:page_count') ?? namedMeta('projectx:page_count')); + const rating = parseNumber(propertyMeta('projectx:rating') ?? namedMeta('projectx:rating')); // ── Series (Calibre EPUB2, then EPUB3) ──────────────────────────────────── let seriesName: string | null = namedMeta('calibre:series'); @@ -204,9 +264,18 @@ export function parseOpf(xml: string): ParsedOpf { publisher, publishedYear, language: language || null, + pageCount, + rating, seriesName: seriesName || null, seriesIndex, authors, + genres, tags, + googleBooksId, + goodreadsId, + amazonId, + hardcoverId, + openLibraryId, + itunesId, }; } diff --git a/server/src/modules/metadata/lib/pdf-parser.test.ts b/server/src/modules/metadata/lib/pdf-parser.test.ts index e9c67fa43..67ce17911 100644 --- a/server/src/modules/metadata/lib/pdf-parser.test.ts +++ b/server/src/modules/metadata/lib/pdf-parser.test.ts @@ -94,7 +94,7 @@ describe('parsePdfFile', () => { title: 'XMP Title', authors: [{ name: 'XMP Author', sortName: null }], tags: ['favorite'], - pageCount: 123, + pageCount: 999, coverBuffer: Buffer.from([0xff, 0xd8, 0xff]), }), ); diff --git a/server/src/modules/metadata/lib/pdf-parser.ts b/server/src/modules/metadata/lib/pdf-parser.ts index 3fc19746d..19892f9ed 100644 --- a/server/src/modules/metadata/lib/pdf-parser.ts +++ b/server/src/modules/metadata/lib/pdf-parser.ts @@ -73,6 +73,7 @@ export async function parsePdfFile(absolutePath: string): Promise; +type RelationMutationExecutor = Pick; + +interface RelationMutationOptions { + executor?: RelationMutationExecutor; + emitEvent?: boolean; +} const AUDIO_FORMATS = ['m4b', 'mp3', 'm4a', 'opus', 'ogg', 'flac'] as const; const MAX_RELATION_NAME_LENGTH = 200; @@ -237,7 +243,11 @@ export class MetadataService { // ── Authors ────────────────────────────────────────────────────────────────── - async replaceAuthors(bookId: number, parsedAuthors: { name: string; sortName: string | null }[]) { + async replaceAuthors( + bookId: number, + parsedAuthors: { name: string; sortName: string | null }[], + options: RelationMutationOptions = {}, + ): Promise { const normalized = parsedAuthors .map((author) => ({ name: author.name.trim(), @@ -253,34 +263,21 @@ export class MetadataService { return true; }); - const linkedAuthorIds = await this.db.transaction(async (tx) => { - await tx.delete(bookAuthors).where(eq(bookAuthors.bookId, bookId)); + const linkedAuthorIds = options.executor + ? await this.replaceAuthorsInExecutor(options.executor, bookId, unique) + : await this.db.transaction(async (tx) => this.replaceAuthorsInExecutor(tx, bookId, unique)); - if (unique.length === 0) return [] as number[]; - - const authorIds: number[] = []; - for (let index = 0; index < unique.length; index += 1) { - const { name, sortName } = unique[index]; - - let [author] = await tx.insert(authors).values({ name, sortName }).onConflictDoNothing().returning(); - if (!author) { - [author] = await tx.select().from(authors).where(eq(authors.name, name)).limit(1); - } - if (!author) continue; - - await tx.insert(bookAuthors).values({ bookId, authorId: author.id, displayOrder: index }).onConflictDoNothing(); - authorIds.push(author.id); - } - - return authorIds; - }); - - if (linkedAuthorIds.length > 0) { - this.metadataEvents?.emit(METADATA_AUTHORS_REPLACED, { - bookId, - authorIds: linkedAuthorIds, - }); + const emitEvent = options.emitEvent ?? true; + if (emitEvent && linkedAuthorIds.length > 0) { + this.emitAuthorsReplaced(bookId, linkedAuthorIds); } + + return linkedAuthorIds; + } + + emitAuthorsReplaced(bookId: number, authorIds: number[]): void { + if (authorIds.length === 0) return; + this.metadataEvents?.emit(METADATA_AUTHORS_REPLACED, { bookId, authorIds }); } // ── Genres ─────────────────────────────────────────────────────────────────── @@ -289,46 +286,90 @@ export class MetadataService { await this.narratorService.replaceForBook(bookId, narratorNames); } - async replaceGenres(bookId: number, parsedGenres: string[]) { + async replaceGenres(bookId: number, parsedGenres: string[], options: RelationMutationOptions = {}) { const uniqueGenres = this.normalizeUniqueRelationNames(parsedGenres); + if (options.executor) { + await this.replaceGenresInExecutor(options.executor, bookId, uniqueGenres); + return; + } + await this.db.transaction(async (tx) => { - await tx.delete(bookGenres).where(eq(bookGenres.bookId, bookId)); - if (uniqueGenres.length === 0) return; - - for (const genreName of uniqueGenres) { - let [genre] = await tx.insert(genres).values({ name: genreName }).onConflictDoNothing().returning(); - if (!genre) { - [genre] = await tx.select().from(genres).where(eq(genres.name, genreName)).limit(1); - } - if (!genre) continue; - - await tx.insert(bookGenres).values({ bookId, genreId: genre.id }).onConflictDoNothing(); - } + await this.replaceGenresInExecutor(tx, bookId, uniqueGenres); }); } // ── Tags ───────────────────────────────────────────────────────────────────── - async replaceTags(bookId: number, userTags: string[]) { + async replaceTags(bookId: number, userTags: string[], options: RelationMutationOptions = {}) { const uniqueTags = this.normalizeUniqueRelationNames(userTags); + if (options.executor) { + await this.replaceTagsInExecutor(options.executor, bookId, uniqueTags); + return; + } + await this.db.transaction(async (tx) => { - await tx.delete(bookTags).where(eq(bookTags.bookId, bookId)); - if (uniqueTags.length === 0) return; - - for (const tagName of uniqueTags) { - let [tag] = await tx.insert(tags).values({ name: tagName }).onConflictDoNothing().returning(); - if (!tag) { - [tag] = await tx.select().from(tags).where(eq(tags.name, tagName)).limit(1); - } - if (!tag) continue; - - await tx.insert(bookTags).values({ bookId, tagId: tag.id }).onConflictDoNothing(); - } + await this.replaceTagsInExecutor(tx, bookId, uniqueTags); }); } + private async replaceAuthorsInExecutor( + executor: RelationMutationExecutor, + bookId: number, + uniqueAuthors: { name: string; sortName: string | null }[], + ): Promise { + await executor.delete(bookAuthors).where(eq(bookAuthors.bookId, bookId)); + + if (uniqueAuthors.length === 0) return []; + + const authorIds: number[] = []; + for (let index = 0; index < uniqueAuthors.length; index += 1) { + const { name, sortName } = uniqueAuthors[index]; + + let [author] = await executor.insert(authors).values({ name, sortName }).onConflictDoNothing().returning(); + if (!author) { + [author] = await executor.select().from(authors).where(eq(authors.name, name)).limit(1); + } + if (!author) continue; + + await executor.insert(bookAuthors).values({ bookId, authorId: author.id, displayOrder: index }).onConflictDoNothing(); + authorIds.push(author.id); + } + + return authorIds; + } + + private async replaceGenresInExecutor(executor: RelationMutationExecutor, bookId: number, uniqueGenres: string[]): Promise { + await executor.delete(bookGenres).where(eq(bookGenres.bookId, bookId)); + if (uniqueGenres.length === 0) return; + + for (const genreName of uniqueGenres) { + let [genre] = await executor.insert(genres).values({ name: genreName }).onConflictDoNothing().returning(); + if (!genre) { + [genre] = await executor.select().from(genres).where(eq(genres.name, genreName)).limit(1); + } + if (!genre) continue; + + await executor.insert(bookGenres).values({ bookId, genreId: genre.id }).onConflictDoNothing(); + } + } + + private async replaceTagsInExecutor(executor: RelationMutationExecutor, bookId: number, uniqueTags: string[]): Promise { + await executor.delete(bookTags).where(eq(bookTags.bookId, bookId)); + if (uniqueTags.length === 0) return; + + for (const tagName of uniqueTags) { + let [tag] = await executor.insert(tags).values({ name: tagName }).onConflictDoNothing().returning(); + if (!tag) { + [tag] = await executor.select().from(tags).where(eq(tags.name, tagName)).limit(1); + } + if (!tag) continue; + + await executor.insert(bookTags).values({ bookId, tagId: tag.id }).onConflictDoNothing(); + } + } + // ── Persistence ────────────────────────────────────────────────────────────── private async persistMetadata(bookId: number, data: ParsedBookData, format: string): Promise { diff --git a/server/src/modules/narrator/narrator.repository.ts b/server/src/modules/narrator/narrator.repository.ts index f3fa78dc7..a5ee86e00 100644 --- a/server/src/modules/narrator/narrator.repository.ts +++ b/server/src/modules/narrator/narrator.repository.ts @@ -7,29 +7,43 @@ import * as schema from '../../db/schema'; import { bookNarrators, narrators } from '../../db/schema'; type Db = NodePgDatabase; +type NarratorMutationExecutor = Pick; @Injectable() export class NarratorRepository { constructor(@Inject(DB) private readonly db: Db) {} - async replaceForBook(bookId: number, names: { name: string; sortName: string | null }[]): Promise { + async replaceForBook(bookId: number, names: { name: string; sortName: string | null }[], executor?: NarratorMutationExecutor): Promise { + if (executor) { + await this.replaceForBookWithExecutor(executor, bookId, names); + return; + } + await this.db.transaction(async (tx) => { - await tx.delete(bookNarrators).where(eq(bookNarrators.bookId, bookId)); - - for (let i = 0; i < names.length; i++) { - const { name, sortName } = names[i]; - - // onConflictDoUpdate with a no-op SET guarantees a RETURNING row regardless of - // whether the INSERT or the conflict path was taken, avoiding a read-committed - // race where onConflictDoNothing + subsequent SELECT could return nothing. - await tx.insert(narrators).values({ name, sortName }).onConflictDoUpdate({ target: narrators.name, set: { name } }); - - const [narrator] = await tx.select({ id: narrators.id }).from(narrators).where(eq(narrators.name, name)).limit(1); - - if (narrator) { - await tx.insert(bookNarrators).values({ bookId, narratorId: narrator.id, displayOrder: i }).onConflictDoNothing(); - } - } + await this.replaceForBookWithExecutor(tx, bookId, names); }); } + + private async replaceForBookWithExecutor( + executor: NarratorMutationExecutor, + bookId: number, + names: { name: string; sortName: string | null }[], + ): Promise { + await executor.delete(bookNarrators).where(eq(bookNarrators.bookId, bookId)); + + for (let i = 0; i < names.length; i++) { + const { name, sortName } = names[i]; + + // onConflictDoUpdate with a no-op SET guarantees a RETURNING row regardless of + // whether the INSERT or the conflict path was taken, avoiding a read-committed + // race where onConflictDoNothing + subsequent SELECT could return nothing. + await executor.insert(narrators).values({ name, sortName }).onConflictDoUpdate({ target: narrators.name, set: { name } }); + + const [narrator] = await executor.select({ id: narrators.id }).from(narrators).where(eq(narrators.name, name)).limit(1); + + if (narrator) { + await executor.insert(bookNarrators).values({ bookId, narratorId: narrator.id, displayOrder: i }).onConflictDoNothing(); + } + } + } } diff --git a/server/src/modules/narrator/narrator.service.ts b/server/src/modules/narrator/narrator.service.ts index 9f505812e..ddcec6d59 100644 --- a/server/src/modules/narrator/narrator.service.ts +++ b/server/src/modules/narrator/narrator.service.ts @@ -1,17 +1,30 @@ import { Injectable } from '@nestjs/common'; +import { NodePgDatabase } from 'drizzle-orm/node-postgres'; +import * as schema from '../../db/schema'; import { NarratorRepository } from './narrator.repository'; +type Db = NodePgDatabase; +type NarratorMutationExecutor = Pick; + @Injectable() export class NarratorService { constructor(private readonly narratorRepo: NarratorRepository) {} - async replaceForBook(bookId: number, names: string[] | { name: string; sortName: string | null }[]): Promise { + async replaceForBook( + bookId: number, + names: string[] | { name: string; sortName: string | null }[], + options: { executor?: NarratorMutationExecutor } = {}, + ): Promise { const normalized = names.length > 0 && typeof names[0] !== 'string' ? (names as { name: string; sortName: string | null }[]) : (names as string[]).map((name) => ({ name, sortName: null })); + if (options.executor) { + await this.narratorRepo.replaceForBook(bookId, normalized, options.executor); + return; + } await this.narratorRepo.replaceForBook(bookId, normalized); } } diff --git a/server/test/metadata-write.e2e-spec.ts b/server/test/metadata-write.e2e-spec.ts index ab373b556..5e3a3ebea 100644 --- a/server/test/metadata-write.e2e-spec.ts +++ b/server/test/metadata-write.e2e-spec.ts @@ -5,6 +5,7 @@ import { eq } from 'drizzle-orm'; import { Permission, type LibraryFileSyncProgressEvent } from '@projectx/types'; import * as schema from '../src/db/schema'; +import { BookService, type MetadataUpdateFailpoint } from '../src/modules/book/book.service'; import { extractCb7Metadata, extractCbzMetadata } from '../src/modules/metadata/lib/cbz-metadata'; import { extractEpubMetadata } from '../src/modules/metadata/lib/epub'; import { parsePdfFile } from '../src/modules/metadata/lib/pdf-parser'; @@ -46,6 +47,133 @@ interface PreparedLibraryResult { books: Record; } +interface ExpectedWritePayload { + title: string; + subtitle: string; + description: string; + publisher: string; + publishedYear: number; + language: string; + pageCount: number; + seriesName: string; + seriesIndex: number; + isbn10: string; + isbn13: string; + rating: number; + authors: string[]; + genres: string[]; + tags: string[]; + googleBooksId: string; + goodreadsId: string; + amazonId: string; + hardcoverId: string; + openLibraryId: string; + itunesId: string; +} + +const MATRIX_SET_PAYLOAD = { + title: 'Matrix Title', + subtitle: 'Matrix Subtitle', + description: 'Matrix Description', + publisher: 'Matrix Publisher', + publishedYear: 2007, + language: 'en', + pageCount: 321, + seriesName: 'Matrix Series', + seriesIndex: 3.5, + isbn10: '123456789X', + isbn13: '9781234567890', + rating: 5, + authors: ['Matrix Author A', 'Matrix Author B'], + genres: ['Matrix Genre A', 'Matrix Genre B'], + tags: ['MatrixTagA', 'MatrixTagB'], + googleBooksId: 'gbook-123', + goodreadsId: 'goodreads-123', + amazonId: 'B00TEST123', + hardcoverId: 'hardcover-123', + openLibraryId: 'OL123W', + itunesId: 'itunes-123', +} as const; + +const MATRIX_CLEAR_PAYLOAD = { + title: null, + subtitle: null, + description: null, + publisher: null, + publishedYear: null, + language: null, + pageCount: null, + seriesName: null, + seriesIndex: null, + isbn10: null, + isbn13: null, + rating: null, + authors: [], + genres: [], + tags: [], + googleBooksId: null, + goodreadsId: null, + amazonId: null, + hardcoverId: null, + openLibraryId: null, + itunesId: null, +} as const; + +const METADATA_UPDATE_FAILPOINT_SEQUENCE: MetadataUpdateFailpoint[] = [ + 'afterScalarUpdate', + 'afterComicMetadataUpsert', + 'afterAuthorsReplace', + 'afterNarratorsReplace', + 'afterGenresReplace', + 'afterTagsReplace', + 'beforeTransactionCommit', +]; + +const ATOMICITY_BASELINE_PAYLOAD = { + title: 'Atomic Baseline Title', + subtitle: 'Atomic Baseline Subtitle', + description: 'Atomic Baseline Description', + publisher: 'Atomic Baseline Publisher', + publishedYear: 2001, + language: 'en', + pageCount: 111, + seriesName: 'Atomic Baseline Series', + seriesIndex: 1.5, + isbn10: '1111111111', + isbn13: '9781111111111', + rating: 4, + authors: ['Atomic Baseline Author A', 'Atomic Baseline Author B'], + genres: ['Atomic Baseline Genre'], + tags: ['AtomicBaselineTag'], + googleBooksId: 'atomic-baseline-google', + goodreadsId: 'atomic-baseline-goodreads', + amazonId: 'ATOMICBASE01', + hardcoverId: 'atomic-baseline-hardcover', + openLibraryId: 'ATOMICBASE-OL', + itunesId: 'atomic-baseline-itunes', + audibleId: 'ATOMICBASEAUD', + comicvineId: 'atomic-baseline-comicvine', + audioMetadata: { + narrators: ['Atomic Baseline Narrator'], + durationSeconds: 4200, + abridged: true, + chapters: [{ title: 'Atomic Baseline Chapter', startMs: 0 }], + }, + comicMetadata: { + issueNumber: '1', + volumeName: 'Atomic Baseline Volume', + pencillers: ['Atomic Baseline Penciller'], + inkers: ['Atomic Baseline Inker'], + colorists: ['Atomic Baseline Colorist'], + letterers: ['Atomic Baseline Letterer'], + coverArtists: ['Atomic Baseline CoverArtist'], + characters: ['Atomic Baseline Character'], + teams: ['Atomic Baseline Team'], + locations: ['Atomic Baseline Location'], + storyArcs: ['Atomic Baseline StoryArc'], + }, +} as const; + async function writeScenarioReport(results: ScenarioRunResult[]): Promise { const reportDir = process.env.JUNIT_OUTPUT ? dirname(process.env.JUNIT_OUTPUT) : join(process.cwd(), '..', 'test-results', 'server'); await mkdir(reportDir, { recursive: true }); @@ -76,6 +204,10 @@ describe('Metadata write operations (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () }); afterEach((taskContext) => { + if (context) { + context.app.get(BookService).clearMetadataUpdateFailpointForTests(); + } + const result = taskContext.task.result; if (!result) return; @@ -190,6 +322,121 @@ describe('Metadata write operations (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () expect(genreRows.map((row) => row.name)).toEqual(['DB Genre']); expect(tagRows.map((row) => row.name)).toEqual(['DB Tag']); }); + + it('applies clear semantics for scalar and relational metadata through PATCH /books/:id/metadata', async () => { + const library = await createLibraryWithFolder(context, { mode: 'book_per_file' }); + await createPdfFixture(library.folderPath, 'db-clear/book.pdf'); + await triggerAndWaitForLibraryScan(context, library.libraryId); + const book = await locateBookFileByRelPath(context, library.libraryId, 'db-clear/book.pdf'); + + const setResponse = await context.app.inject({ + method: 'PATCH', + url: `/api/v1/books/${book.bookId}/metadata`, + headers: authHeader(context.adminToken), + payload: MATRIX_SET_PAYLOAD, + }); + expect(setResponse.statusCode).toBe(200); + + const clearResponse = await context.app.inject({ + method: 'PATCH', + url: `/api/v1/books/${book.bookId}/metadata`, + headers: authHeader(context.adminToken), + payload: MATRIX_CLEAR_PAYLOAD, + }); + expect(clearResponse.statusCode).toBe(200); + + const [metadata] = await context.db + .select({ + title: schema.bookMetadata.title, + subtitle: schema.bookMetadata.subtitle, + description: schema.bookMetadata.description, + publisher: schema.bookMetadata.publisher, + publishedYear: schema.bookMetadata.publishedYear, + language: schema.bookMetadata.language, + pageCount: schema.bookMetadata.pageCount, + seriesName: schema.bookMetadata.seriesName, + seriesIndex: schema.bookMetadata.seriesIndex, + isbn10: schema.bookMetadata.isbn10, + isbn13: schema.bookMetadata.isbn13, + rating: schema.bookMetadata.rating, + googleBooksId: schema.bookMetadata.googleBooksId, + goodreadsId: schema.bookMetadata.goodreadsId, + amazonId: schema.bookMetadata.amazonId, + hardcoverId: schema.bookMetadata.hardcoverId, + openLibraryId: schema.bookMetadata.openLibraryId, + itunesId: schema.bookMetadata.itunesId, + }) + .from(schema.bookMetadata) + .where(eq(schema.bookMetadata.bookId, book.bookId)) + .limit(1); + + expect(metadata).toMatchObject({ + title: null, + subtitle: null, + description: null, + publisher: null, + publishedYear: null, + language: null, + pageCount: null, + seriesName: null, + seriesIndex: null, + isbn10: null, + isbn13: null, + rating: null, + googleBooksId: null, + goodreadsId: null, + amazonId: null, + hardcoverId: null, + openLibraryId: null, + itunesId: null, + }); + + const [authorRows, genreRows, tagRows] = await Promise.all([ + context.db.select({ bookId: schema.bookAuthors.bookId }).from(schema.bookAuthors).where(eq(schema.bookAuthors.bookId, book.bookId)), + context.db.select({ bookId: schema.bookGenres.bookId }).from(schema.bookGenres).where(eq(schema.bookGenres.bookId, book.bookId)), + context.db.select({ bookId: schema.bookTags.bookId }).from(schema.bookTags).where(eq(schema.bookTags.bookId, book.bookId)), + ]); + + expect(authorRows).toHaveLength(0); + expect(genreRows).toHaveLength(0); + expect(tagRows).toHaveLength(0); + }); + }); + + describe('atomicity failpoints', () => { + it('rolls back metadata mutations for each failpoint stage', async () => { + const library = await createLibraryWithFolder(context, { mode: 'book_per_file' }); + await createPdfFixture(library.folderPath, 'atomicity/book.pdf', 'Atomicity Seed PDF'); + await triggerAndWaitForLibraryScan(context, library.libraryId); + const book = await locateBookFileByRelPath(context, library.libraryId, 'atomicity/book.pdf'); + + await setFileWriteSettings(context.db, { enabled: false }); + const baselineResponse = await context.app.inject({ + method: 'PATCH', + url: `/api/v1/books/${book.bookId}/metadata`, + headers: authHeader(context.adminToken), + payload: ATOMICITY_BASELINE_PAYLOAD, + }); + expect(baselineResponse.statusCode).toBe(200); + + const baselineState = await readMetadataMutationState(context, book.bookId); + const bookService = context.app.get(BookService); + + for (const failpoint of METADATA_UPDATE_FAILPOINT_SEQUENCE) { + bookService.setMetadataUpdateFailpointForTests(failpoint); + + const failureResponse = await context.app.inject({ + method: 'PATCH', + url: `/api/v1/books/${book.bookId}/metadata`, + headers: authHeader(context.adminToken), + payload: buildAtomicityFailurePayload(failpoint), + }); + expect(failureResponse.statusCode).toBe(500); + + const afterFailureState = await readMetadataMutationState(context, book.bookId); + expect(afterFailureState).toEqual(baselineState); + } + }); }); describe('auto-write on metadata patch', () => { @@ -283,30 +530,19 @@ describe('Metadata write operations (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () cbx: { enabled: true, formats: ['cbz', 'cb7'] }, }); - const updates: Record = { - pdf: { title: 'Sync PDF Title', author: 'Sync PDF Author' }, - epub: { title: 'Sync EPUB Title', author: 'Sync EPUB Author' }, - cbz: { title: 'Sync CBZ Title', author: 'Sync CBZ Author' }, - cb7: { title: 'Sync CB7 Title', author: 'Sync CB7 Author' }, - }; - for (const format of Object.keys(prepared.books) as SupportedFormat[]) { const book = prepared.books[format]; - const payload = updates[format]; const patchResponse = await context.app.inject({ method: 'PATCH', url: `/api/v1/books/${book.bookId}/metadata`, headers: authHeader(context.adminToken), - payload: { - title: payload.title, - authors: [payload.author], - publisher: `Publisher ${format.toUpperCase()}`, - publishedYear: 2020, - }, + payload: MATRIX_SET_PAYLOAD, }); expect(patchResponse.statusCode).toBe(200); } + await sleep(250); + await setFileWriteSettings(context.db, { enabled: true, writeCover: false, @@ -331,23 +567,322 @@ describe('Metadata write operations (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () const book = prepared.books[format]; const logEntry = await getLatestWriteLogEntry(context.db, book.bookId, 'sync'); expect(logEntry?.status).toBe('success'); + if (format === 'cbz' || format === 'cb7') { + expect(logEntry?.fieldsWritten ?? []).not.toContain('itunesId'); + } else { + expect(logEntry?.fieldsWritten ?? []).toContain('itunesId'); + } } const parsedPdf = await parsePdfFile(prepared.books.pdf.absolutePath); - expect(parsedPdf?.title).toBe(updates.pdf.title); - expect(parsedPdf?.authors.map((author) => author.name)).toContain(updates.pdf.author); + expect(parsedPdf).toMatchObject({ + title: MATRIX_SET_PAYLOAD.title, + subtitle: MATRIX_SET_PAYLOAD.subtitle, + description: MATRIX_SET_PAYLOAD.description, + publisher: MATRIX_SET_PAYLOAD.publisher, + publishedYear: MATRIX_SET_PAYLOAD.publishedYear, + language: MATRIX_SET_PAYLOAD.language, + pageCount: MATRIX_SET_PAYLOAD.pageCount, + seriesName: MATRIX_SET_PAYLOAD.seriesName, + seriesIndex: MATRIX_SET_PAYLOAD.seriesIndex, + isbn10: MATRIX_SET_PAYLOAD.isbn10, + isbn13: MATRIX_SET_PAYLOAD.isbn13, + rating: MATRIX_SET_PAYLOAD.rating, + googleBooksId: MATRIX_SET_PAYLOAD.googleBooksId, + goodreadsId: MATRIX_SET_PAYLOAD.goodreadsId, + amazonId: MATRIX_SET_PAYLOAD.amazonId, + hardcoverId: MATRIX_SET_PAYLOAD.hardcoverId, + openLibraryId: MATRIX_SET_PAYLOAD.openLibraryId, + itunesId: MATRIX_SET_PAYLOAD.itunesId, + }); + expect(parsedPdf?.authors.map((author) => author.name)).toEqual(MATRIX_SET_PAYLOAD.authors); + expect(parsedPdf?.genres).toEqual(MATRIX_SET_PAYLOAD.genres); + expect(parsedPdf?.tags).toEqual(MATRIX_SET_PAYLOAD.tags); const parsedEpub = await extractEpubMetadata(prepared.books.epub.absolutePath); - expect(parsedEpub?.title).toBe(updates.epub.title); - expect(parsedEpub?.authors.map((author) => author.name)).toContain(updates.epub.author); + expect(parsedEpub).toMatchObject({ + title: MATRIX_SET_PAYLOAD.title, + subtitle: MATRIX_SET_PAYLOAD.subtitle, + description: MATRIX_SET_PAYLOAD.description, + publisher: MATRIX_SET_PAYLOAD.publisher, + publishedYear: MATRIX_SET_PAYLOAD.publishedYear, + language: MATRIX_SET_PAYLOAD.language, + pageCount: MATRIX_SET_PAYLOAD.pageCount, + seriesName: MATRIX_SET_PAYLOAD.seriesName, + seriesIndex: MATRIX_SET_PAYLOAD.seriesIndex, + isbn10: MATRIX_SET_PAYLOAD.isbn10, + isbn13: MATRIX_SET_PAYLOAD.isbn13, + rating: MATRIX_SET_PAYLOAD.rating, + googleBooksId: MATRIX_SET_PAYLOAD.googleBooksId, + goodreadsId: MATRIX_SET_PAYLOAD.goodreadsId, + amazonId: MATRIX_SET_PAYLOAD.amazonId, + hardcoverId: MATRIX_SET_PAYLOAD.hardcoverId, + openLibraryId: MATRIX_SET_PAYLOAD.openLibraryId, + itunesId: MATRIX_SET_PAYLOAD.itunesId, + }); + expect(parsedEpub?.authors.map((author) => author.name)).toEqual(MATRIX_SET_PAYLOAD.authors); + expect(parsedEpub?.genres).toEqual(MATRIX_SET_PAYLOAD.genres); + expect(parsedEpub?.tags).toEqual(MATRIX_SET_PAYLOAD.tags); const parsedCbz = await extractCbzMetadata(prepared.books.cbz.absolutePath); - expect(parsedCbz?.title).toBe(updates.cbz.title); - expect(parsedCbz?.authors.map((author) => author.name)).toContain(updates.cbz.author); + expect(parsedCbz).toMatchObject({ + title: MATRIX_SET_PAYLOAD.title, + subtitle: MATRIX_SET_PAYLOAD.subtitle, + description: MATRIX_SET_PAYLOAD.description, + publisher: MATRIX_SET_PAYLOAD.publisher, + publishedYear: MATRIX_SET_PAYLOAD.publishedYear, + language: MATRIX_SET_PAYLOAD.language, + pageCount: MATRIX_SET_PAYLOAD.pageCount, + seriesName: MATRIX_SET_PAYLOAD.seriesName, + seriesIndex: MATRIX_SET_PAYLOAD.seriesIndex, + isbn10: MATRIX_SET_PAYLOAD.isbn10, + isbn13: MATRIX_SET_PAYLOAD.isbn13, + rating: MATRIX_SET_PAYLOAD.rating, + googleBooksId: MATRIX_SET_PAYLOAD.googleBooksId, + goodreadsId: MATRIX_SET_PAYLOAD.goodreadsId, + amazonId: MATRIX_SET_PAYLOAD.amazonId, + hardcoverId: MATRIX_SET_PAYLOAD.hardcoverId, + openLibraryId: MATRIX_SET_PAYLOAD.openLibraryId, + itunesId: null, + }); + expect(parsedCbz?.authors.map((author) => author.name)).toEqual(MATRIX_SET_PAYLOAD.authors); + expect(parsedCbz?.genres).toEqual(MATRIX_SET_PAYLOAD.genres); + expect(parsedCbz?.tags).toEqual(MATRIX_SET_PAYLOAD.tags); const parsedCb7 = await extractCb7Metadata(prepared.books.cb7.absolutePath); - expect(parsedCb7?.title).toBe(updates.cb7.title); - expect(parsedCb7?.authors.map((author) => author.name)).toContain(updates.cb7.author); + expect(parsedCb7).toMatchObject({ + title: MATRIX_SET_PAYLOAD.title, + subtitle: MATRIX_SET_PAYLOAD.subtitle, + description: MATRIX_SET_PAYLOAD.description, + publisher: MATRIX_SET_PAYLOAD.publisher, + publishedYear: MATRIX_SET_PAYLOAD.publishedYear, + language: MATRIX_SET_PAYLOAD.language, + pageCount: MATRIX_SET_PAYLOAD.pageCount, + seriesName: MATRIX_SET_PAYLOAD.seriesName, + seriesIndex: MATRIX_SET_PAYLOAD.seriesIndex, + isbn10: MATRIX_SET_PAYLOAD.isbn10, + isbn13: MATRIX_SET_PAYLOAD.isbn13, + rating: MATRIX_SET_PAYLOAD.rating, + googleBooksId: MATRIX_SET_PAYLOAD.googleBooksId, + goodreadsId: MATRIX_SET_PAYLOAD.goodreadsId, + amazonId: MATRIX_SET_PAYLOAD.amazonId, + hardcoverId: MATRIX_SET_PAYLOAD.hardcoverId, + openLibraryId: MATRIX_SET_PAYLOAD.openLibraryId, + itunesId: null, + }); + expect(parsedCb7?.authors.map((author) => author.name)).toEqual(MATRIX_SET_PAYLOAD.authors); + expect(parsedCb7?.genres).toEqual(MATRIX_SET_PAYLOAD.genres); + expect(parsedCb7?.tags).toEqual(MATRIX_SET_PAYLOAD.tags); + }); + + it('clears supported metadata in DB and files across all formats', async () => { + const prepared = await prepareLibraryWithAllFormats(context); + + await setFileWriteSettings(context.db, { + enabled: false, + cbx: { enabled: true, formats: ['cbz', 'cb7'] }, + }); + + for (const format of Object.keys(prepared.books) as SupportedFormat[]) { + const book = prepared.books[format]; + + const setResponse = await context.app.inject({ + method: 'PATCH', + url: `/api/v1/books/${book.bookId}/metadata`, + headers: authHeader(context.adminToken), + payload: MATRIX_SET_PAYLOAD, + }); + expect(setResponse.statusCode).toBe(200); + + const clearResponse = await context.app.inject({ + method: 'PATCH', + url: `/api/v1/books/${book.bookId}/metadata`, + headers: authHeader(context.adminToken), + payload: MATRIX_CLEAR_PAYLOAD, + }); + expect(clearResponse.statusCode).toBe(200); + } + + for (const format of Object.keys(prepared.books) as SupportedFormat[]) { + const book = prepared.books[format]; + + const [metadata] = await context.db + .select({ + title: schema.bookMetadata.title, + subtitle: schema.bookMetadata.subtitle, + description: schema.bookMetadata.description, + publisher: schema.bookMetadata.publisher, + publishedYear: schema.bookMetadata.publishedYear, + language: schema.bookMetadata.language, + pageCount: schema.bookMetadata.pageCount, + seriesName: schema.bookMetadata.seriesName, + seriesIndex: schema.bookMetadata.seriesIndex, + isbn10: schema.bookMetadata.isbn10, + isbn13: schema.bookMetadata.isbn13, + rating: schema.bookMetadata.rating, + googleBooksId: schema.bookMetadata.googleBooksId, + goodreadsId: schema.bookMetadata.goodreadsId, + amazonId: schema.bookMetadata.amazonId, + hardcoverId: schema.bookMetadata.hardcoverId, + openLibraryId: schema.bookMetadata.openLibraryId, + itunesId: schema.bookMetadata.itunesId, + }) + .from(schema.bookMetadata) + .where(eq(schema.bookMetadata.bookId, book.bookId)) + .limit(1); + + expect(metadata).toMatchObject({ + title: null, + subtitle: null, + description: null, + publisher: null, + publishedYear: null, + language: null, + pageCount: null, + seriesName: null, + seriesIndex: null, + isbn10: null, + isbn13: null, + rating: null, + googleBooksId: null, + goodreadsId: null, + amazonId: null, + hardcoverId: null, + openLibraryId: null, + itunesId: null, + }); + + const [authorRows, genreRows, tagRows] = await Promise.all([ + context.db.select({ bookId: schema.bookAuthors.bookId }).from(schema.bookAuthors).where(eq(schema.bookAuthors.bookId, book.bookId)), + context.db.select({ bookId: schema.bookGenres.bookId }).from(schema.bookGenres).where(eq(schema.bookGenres.bookId, book.bookId)), + context.db.select({ bookId: schema.bookTags.bookId }).from(schema.bookTags).where(eq(schema.bookTags.bookId, book.bookId)), + ]); + + expect(authorRows).toHaveLength(0); + expect(genreRows).toHaveLength(0); + expect(tagRows).toHaveLength(0); + } + + await sleep(250); + + await setFileWriteSettings(context.db, { + enabled: true, + writeCover: false, + cbx: { enabled: true, formats: ['cbz', 'cb7'] }, + }); + + const syncResponse = await context.app.inject({ + method: 'POST', + url: `/api/v1/libraries/${prepared.libraryId}/write-metadata-to-files`, + headers: authHeader(context.adminToken), + }); + + expect(syncResponse.statusCode).toBe(200); + const events = parseSseEvents(syncResponse.body); + const doneEvent = events.find( + (event): event is Extract => 'done' in event && event.done === true, + ); + expect(doneEvent).toMatchObject({ processed: 4, succeeded: 4, failed: 0, skipped: 0 }); + + const parsedPdf = await parsePdfFile(prepared.books.pdf.absolutePath); + expect(parsedPdf).toMatchObject({ + title: null, + subtitle: null, + description: null, + publisher: null, + publishedYear: null, + language: null, + pageCount: null, + seriesName: null, + seriesIndex: null, + isbn10: null, + isbn13: null, + rating: null, + googleBooksId: null, + goodreadsId: null, + amazonId: null, + hardcoverId: null, + openLibraryId: null, + itunesId: null, + }); + expect(parsedPdf?.authors).toEqual([]); + expect(parsedPdf?.genres).toEqual([]); + expect(parsedPdf?.tags).toEqual([]); + + const parsedEpub = await extractEpubMetadata(prepared.books.epub.absolutePath); + expect(parsedEpub).toMatchObject({ + title: null, + subtitle: null, + description: null, + publisher: null, + publishedYear: null, + language: null, + pageCount: null, + seriesName: null, + seriesIndex: null, + isbn10: null, + isbn13: null, + rating: null, + googleBooksId: null, + goodreadsId: null, + amazonId: null, + hardcoverId: null, + openLibraryId: null, + itunesId: null, + }); + expect(parsedEpub?.authors).toEqual([]); + expect(parsedEpub?.genres).toEqual([]); + expect(parsedEpub?.tags).toEqual([]); + + const parsedCbz = await extractCbzMetadata(prepared.books.cbz.absolutePath); + expect(parsedCbz).toMatchObject({ + title: null, + subtitle: null, + description: null, + publisher: null, + publishedYear: null, + language: null, + pageCount: null, + seriesName: null, + seriesIndex: null, + isbn10: null, + isbn13: null, + rating: null, + googleBooksId: null, + goodreadsId: null, + amazonId: null, + hardcoverId: null, + openLibraryId: null, + itunesId: null, + }); + expect(parsedCbz?.authors).toEqual([]); + expect(parsedCbz?.genres).toEqual([]); + expect(parsedCbz?.tags).toEqual([]); + + const parsedCb7 = await extractCb7Metadata(prepared.books.cb7.absolutePath); + expect(parsedCb7).toMatchObject({ + title: null, + subtitle: null, + description: null, + publisher: null, + publishedYear: null, + language: null, + pageCount: null, + seriesName: null, + seriesIndex: null, + isbn10: null, + isbn13: null, + rating: null, + googleBooksId: null, + goodreadsId: null, + amazonId: null, + hardcoverId: null, + openLibraryId: null, + itunesId: null, + }); + expect(parsedCb7?.authors).toEqual([]); + expect(parsedCb7?.genres).toEqual([]); + expect(parsedCb7?.tags).toEqual([]); }); it('supports dry-run sync and records skipped write-log entries', async () => { @@ -368,6 +903,8 @@ describe('Metadata write operations (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () }); expect(patchResponse.statusCode).toBe(200); + await sleep(250); + await setFileWriteSettings(context.db, { enabled: true }); const dryRunResponse = await context.app.inject({ @@ -442,6 +979,8 @@ describe('Metadata write operations (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () expect(patchResponse.statusCode).toBe(200); } + await sleep(250); + await setFileWriteSettings(context.db, { enabled: true, cbx: { enabled: true, formats: ['cbz', 'cb7'] } }); const syncResponse = await context.app.inject({ @@ -467,6 +1006,293 @@ describe('Metadata write operations (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, () }); }); + describe('concurrency hardening', () => { + it('resolves parallel PATCH conflicts to one complete payload shape', async () => { + const library = await createLibraryWithFolder(context, { mode: 'book_per_file' }); + await createPdfFixture(library.folderPath, 'concurrency/parallel-patch.pdf', 'Parallel Patch Seed'); + await triggerAndWaitForLibraryScan(context, library.libraryId); + const book = await locateBookFileByRelPath(context, library.libraryId, 'concurrency/parallel-patch.pdf'); + + const payloadA = { + title: 'Parallel Patch Title A', + subtitle: 'Parallel Patch Subtitle A', + description: 'Parallel Patch Description A', + publisher: 'Parallel Patch Publisher A', + publishedYear: 2003, + language: 'en', + pageCount: 203, + seriesName: 'Parallel Patch Series A', + seriesIndex: 1.1, + isbn10: '1234500001', + isbn13: '9781234500001', + rating: 4, + authors: ['Parallel Patch Author A1', 'Parallel Patch Author A2'], + genres: ['Parallel Patch Genre A'], + tags: ['ParallelPatchTagA'], + googleBooksId: 'parallel-a-google', + goodreadsId: 'parallel-a-goodreads', + amazonId: 'PARALLELA01', + hardcoverId: 'parallel-a-hardcover', + openLibraryId: 'PARALLELA-OL', + itunesId: 'parallel-a-itunes', + } as const; + + const payloadB = { + title: 'Parallel Patch Title B', + subtitle: 'Parallel Patch Subtitle B', + description: 'Parallel Patch Description B', + publisher: 'Parallel Patch Publisher B', + publishedYear: 2004, + language: 'fr', + pageCount: 204, + seriesName: 'Parallel Patch Series B', + seriesIndex: 2.2, + isbn10: '1234500002', + isbn13: '9781234500002', + rating: 5, + authors: ['Parallel Patch Author B1', 'Parallel Patch Author B2'], + genres: ['Parallel Patch Genre B'], + tags: ['ParallelPatchTagB'], + googleBooksId: 'parallel-b-google', + goodreadsId: 'parallel-b-goodreads', + amazonId: 'PARALLELB02', + hardcoverId: 'parallel-b-hardcover', + openLibraryId: 'PARALLELB-OL', + itunesId: 'parallel-b-itunes', + } as const; + + const [responseA, responseB] = await Promise.all([ + context.app.inject({ + method: 'PATCH', + url: `/api/v1/books/${book.bookId}/metadata`, + headers: authHeader(context.adminToken), + payload: payloadA, + }), + context.app.inject({ + method: 'PATCH', + url: `/api/v1/books/${book.bookId}/metadata`, + headers: authHeader(context.adminToken), + payload: payloadB, + }), + ]); + + expect(responseA.statusCode).toBe(200); + expect(responseB.statusCode).toBe(200); + + await waitForWriteLogEntry(context.db, book.bookId, { + triggeredBy: 'auto', + status: 'success', + timeoutMs: 20_000, + }); + + const dbState = await readBookWriteState(context, book.bookId); + const winningPayload = matchesBookWriteState(dbState, payloadA) ? payloadA : matchesBookWriteState(dbState, payloadB) ? payloadB : null; + expect(winningPayload).not.toBeNull(); + + const parsedPdf = await parsePdfFile(book.absolutePath); + expect(parsedPdf).toMatchObject({ + title: winningPayload!.title, + subtitle: winningPayload!.subtitle, + description: winningPayload!.description, + publisher: winningPayload!.publisher, + publishedYear: winningPayload!.publishedYear, + language: winningPayload!.language, + pageCount: winningPayload!.pageCount, + seriesName: winningPayload!.seriesName, + seriesIndex: winningPayload!.seriesIndex, + isbn10: winningPayload!.isbn10, + isbn13: winningPayload!.isbn13, + rating: winningPayload!.rating, + googleBooksId: winningPayload!.googleBooksId, + goodreadsId: winningPayload!.goodreadsId, + amazonId: winningPayload!.amazonId, + hardcoverId: winningPayload!.hardcoverId, + openLibraryId: winningPayload!.openLibraryId, + itunesId: winningPayload!.itunesId, + }); + expect(parsedPdf?.authors.map((author) => author.name)).toEqual(winningPayload!.authors); + expect(parsedPdf?.genres).toEqual(winningPayload!.genres); + expect(parsedPdf?.tags).toEqual(winningPayload!.tags); + }); + + it('converges to latest PATCH state when PATCH overlaps with sync run', async () => { + const library = await createLibraryWithFolder(context, { mode: 'book_per_file' }); + const relPaths = ['concurrency/overlap/a.pdf', 'concurrency/overlap/b.pdf', 'concurrency/overlap/c.pdf', 'concurrency/overlap/d.pdf']; + await Promise.all(relPaths.map((relPath) => createPdfFixture(library.folderPath, relPath, `Overlap Seed ${relPath}`))); + await triggerAndWaitForLibraryScan(context, library.libraryId); + + const books = await Promise.all(relPaths.map((relPath) => locateBookFileByRelPath(context, library.libraryId, relPath))); + const targetBook = books[0]!; + + await setFileWriteSettings(context.db, { enabled: false }); + for (const located of books) { + const seedResponse = await context.app.inject({ + method: 'PATCH', + url: `/api/v1/books/${located.bookId}/metadata`, + headers: authHeader(context.adminToken), + payload: { + title: `Overlap Seed ${located.bookId}`, + authors: ['Overlap Seed Author'], + tags: ['OverlapSeedTag'], + }, + }); + expect(seedResponse.statusCode).toBe(200); + } + + await sleep(250); + + await setFileWriteSettings(context.db, { + enabled: true, + writeCover: false, + epub: { enabled: true }, + pdf: { enabled: true }, + cbx: { enabled: true, formats: ['cbz', 'cb7'] }, + }); + + const overlapPayload = { + title: 'Overlap Final Title', + subtitle: 'Overlap Final Subtitle', + description: 'Overlap Final Description', + publisher: 'Overlap Final Publisher', + publishedYear: 2017, + language: 'es', + pageCount: 417, + seriesName: 'Overlap Final Series', + seriesIndex: 7.7, + isbn10: '7654321098', + isbn13: '9787654321098', + rating: 5, + authors: ['Overlap Final Author A', 'Overlap Final Author B'], + genres: ['Overlap Final Genre'], + tags: ['OverlapFinalTag'], + googleBooksId: 'overlap-google', + goodreadsId: 'overlap-goodreads', + amazonId: 'OVERLAP017', + hardcoverId: 'overlap-hardcover', + openLibraryId: 'OVERLAP-OL', + itunesId: 'overlap-itunes', + } as const; + + const syncPromise = context.app.inject({ + method: 'POST', + url: `/api/v1/libraries/${library.libraryId}/write-metadata-to-files`, + headers: authHeader(context.adminToken), + }); + + await sleep(75); + + const patchResponse = await context.app.inject({ + method: 'PATCH', + url: `/api/v1/books/${targetBook.bookId}/metadata`, + headers: authHeader(context.adminToken), + payload: overlapPayload, + }); + expect(patchResponse.statusCode).toBe(200); + + const syncResponse = await syncPromise; + expect(syncResponse.statusCode).toBe(200); + const syncEvents = parseSseEvents(syncResponse.body); + const doneEvent = syncEvents.find( + (event): event is Extract => 'done' in event && event.done === true, + ); + expect(doneEvent).toMatchObject({ processed: books.length, failed: 0 }); + + await waitForWriteLogEntry(context.db, targetBook.bookId, { + triggeredBy: 'auto', + status: 'success', + timeoutMs: 20_000, + }); + + const targetState = await readBookWriteState(context, targetBook.bookId); + expect(matchesBookWriteState(targetState, overlapPayload)).toBe(true); + + const parsedPdf = await parsePdfFile(targetBook.absolutePath); + expect(parsedPdf).toMatchObject({ + title: overlapPayload.title, + subtitle: overlapPayload.subtitle, + description: overlapPayload.description, + publisher: overlapPayload.publisher, + publishedYear: overlapPayload.publishedYear, + language: overlapPayload.language, + pageCount: overlapPayload.pageCount, + seriesName: overlapPayload.seriesName, + seriesIndex: overlapPayload.seriesIndex, + isbn10: overlapPayload.isbn10, + isbn13: overlapPayload.isbn13, + rating: overlapPayload.rating, + googleBooksId: overlapPayload.googleBooksId, + goodreadsId: overlapPayload.goodreadsId, + amazonId: overlapPayload.amazonId, + hardcoverId: overlapPayload.hardcoverId, + openLibraryId: overlapPayload.openLibraryId, + itunesId: overlapPayload.itunesId, + }); + expect(parsedPdf?.authors.map((author) => author.name)).toEqual(overlapPayload.authors); + expect(parsedPdf?.genres).toEqual(overlapPayload.genres); + expect(parsedPdf?.tags).toEqual(overlapPayload.tags); + }); + + it('allows concurrent sync runs on the same library with consistent outcomes', async () => { + const prepared = await prepareLibraryWithAllFormats(context); + + await setFileWriteSettings(context.db, { + enabled: false, + writeCover: false, + cbx: { enabled: true, formats: ['cbz', 'cb7'] }, + }); + + for (const format of Object.keys(prepared.books) as SupportedFormat[]) { + const book = prepared.books[format]; + const patchResponse = await context.app.inject({ + method: 'PATCH', + url: `/api/v1/books/${book.bookId}/metadata`, + headers: authHeader(context.adminToken), + payload: MATRIX_SET_PAYLOAD, + }); + expect(patchResponse.statusCode).toBe(200); + } + + await sleep(250); + + await setFileWriteSettings(context.db, { + enabled: true, + writeCover: false, + cbx: { enabled: true, formats: ['cbz', 'cb7'] }, + }); + + const [syncA, syncB] = await Promise.all([ + context.app.inject({ + method: 'POST', + url: `/api/v1/libraries/${prepared.libraryId}/write-metadata-to-files`, + headers: authHeader(context.adminToken), + }), + context.app.inject({ + method: 'POST', + url: `/api/v1/libraries/${prepared.libraryId}/write-metadata-to-files`, + headers: authHeader(context.adminToken), + }), + ]); + + expect(syncA.statusCode).toBe(200); + expect(syncB.statusCode).toBe(200); + + const doneA = parseSseEvents(syncA.body).find( + (event): event is Extract => 'done' in event && event.done === true, + ); + const doneB = parseSseEvents(syncB.body).find( + (event): event is Extract => 'done' in event && event.done === true, + ); + expect(doneA).toMatchObject({ processed: 4, succeeded: 4, failed: 0, skipped: 0 }); + expect(doneB).toMatchObject({ processed: 4, succeeded: 4, failed: 0, skipped: 0 }); + + for (const format of Object.keys(prepared.books) as SupportedFormat[]) { + const book = prepared.books[format]; + const syncSuccessLogs = await countSyncSuccessLogs(context, book.bookId); + expect(syncSuccessLogs).toBe(2); + } + }); + }); + describe('access control', () => { it('enforces permission and library access for PATCH /books/:id/metadata', async () => { const library = await createLibraryWithFolder(context, { mode: 'book_per_file' }); @@ -600,3 +1426,227 @@ function parseSseEvents(rawBody: string): LibraryFileSyncProgressEvent[] { return events; } + +function buildAtomicityFailurePayload(failpoint: MetadataUpdateFailpoint) { + const token = failpoint + .replace(/[^a-z]/gi, '') + .toLowerCase() + .slice(0, 10); + return { + title: `Atomic Failure Title ${token}`, + subtitle: `Atomic Failure Subtitle ${token}`, + description: `Atomic Failure Description ${token}`, + publisher: `Atomic Failure Publisher ${token}`, + publishedYear: 2020, + language: 'de', + pageCount: 520, + seriesName: `Atomic Failure Series ${token}`, + seriesIndex: 9.9, + isbn10: '9999999999', + isbn13: '9789999999999', + rating: 2, + authors: [`Atomic Failure Author ${token}`], + genres: [`Atomic Failure Genre ${token}`], + tags: [`AtomicFailureTag${token}`], + googleBooksId: `atomic-fail-g-${token}`, + goodreadsId: `atomic-fail-gr-${token}`, + amazonId: `ATFAIL${token.slice(0, 5).toUpperCase()}`, + hardcoverId: `atomic-fail-hc-${token}`, + openLibraryId: `ATFAIL-${token.toUpperCase()}`, + itunesId: `atomic-fail-it-${token}`, + audibleId: `ATF${token.slice(0, 8).toUpperCase()}`, + comicvineId: `atomic-fail-cv-${token}`, + audioMetadata: { + narrators: [`Atomic Failure Narrator ${token}`], + durationSeconds: 777, + abridged: false, + chapters: [{ title: `Atomic Failure Chapter ${token}`, startMs: 1000 }], + }, + comicMetadata: { + issueNumber: '9', + volumeName: `Atomic Failure Volume ${token}`, + pencillers: [`Atomic Failure Penciller ${token}`], + inkers: [`Atomic Failure Inker ${token}`], + colorists: [`Atomic Failure Colorist ${token}`], + letterers: [`Atomic Failure Letterer ${token}`], + coverArtists: [`Atomic Failure CoverArtist ${token}`], + characters: [`Atomic Failure Character ${token}`], + teams: [`Atomic Failure Team ${token}`], + locations: [`Atomic Failure Location ${token}`], + storyArcs: [`Atomic Failure StoryArc ${token}`], + }, + }; +} + +async function readMetadataMutationState(context: MetadataWriteE2EContext, bookId: number) { + const [metadata, authors, genres, tags, narrators, comicRows] = await Promise.all([ + context.db + .select({ + title: schema.bookMetadata.title, + subtitle: schema.bookMetadata.subtitle, + description: schema.bookMetadata.description, + publisher: schema.bookMetadata.publisher, + publishedYear: schema.bookMetadata.publishedYear, + language: schema.bookMetadata.language, + pageCount: schema.bookMetadata.pageCount, + seriesName: schema.bookMetadata.seriesName, + seriesIndex: schema.bookMetadata.seriesIndex, + isbn10: schema.bookMetadata.isbn10, + isbn13: schema.bookMetadata.isbn13, + rating: schema.bookMetadata.rating, + googleBooksId: schema.bookMetadata.googleBooksId, + goodreadsId: schema.bookMetadata.goodreadsId, + amazonId: schema.bookMetadata.amazonId, + hardcoverId: schema.bookMetadata.hardcoverId, + openLibraryId: schema.bookMetadata.openLibraryId, + itunesId: schema.bookMetadata.itunesId, + audibleId: schema.bookMetadata.audibleId, + comicvineId: schema.bookMetadata.comicvineId, + durationSeconds: schema.bookMetadata.durationSeconds, + abridged: schema.bookMetadata.abridged, + chapters: schema.bookMetadata.chapters, + }) + .from(schema.bookMetadata) + .where(eq(schema.bookMetadata.bookId, bookId)) + .limit(1), + context.db + .select({ name: schema.authors.name }) + .from(schema.bookAuthors) + .innerJoin(schema.authors, eq(schema.authors.id, schema.bookAuthors.authorId)) + .where(eq(schema.bookAuthors.bookId, bookId)) + .orderBy(schema.bookAuthors.displayOrder), + context.db + .select({ name: schema.genres.name }) + .from(schema.bookGenres) + .innerJoin(schema.genres, eq(schema.genres.id, schema.bookGenres.genreId)) + .where(eq(schema.bookGenres.bookId, bookId)), + context.db + .select({ name: schema.tags.name }) + .from(schema.bookTags) + .innerJoin(schema.tags, eq(schema.tags.id, schema.bookTags.tagId)) + .where(eq(schema.bookTags.bookId, bookId)), + context.db + .select({ name: schema.narrators.name }) + .from(schema.bookNarrators) + .innerJoin(schema.narrators, eq(schema.narrators.id, schema.bookNarrators.narratorId)) + .where(eq(schema.bookNarrators.bookId, bookId)) + .orderBy(schema.bookNarrators.displayOrder), + context.db.select().from(schema.comicMetadata).where(eq(schema.comicMetadata.bookId, bookId)).limit(1), + ]); + + const comic = comicRows[0] ?? null; + + return { + metadata: metadata[0] ?? null, + authors: authors.map((row) => row.name), + genres: genres.map((row) => row.name).sort(), + tags: tags.map((row) => row.name).sort(), + narrators: narrators.map((row) => row.name), + comicMetadata: comic + ? { + issueNumber: comic.issueNumber ?? null, + volumeName: comic.volumeName ?? null, + pencillers: [...(comic.pencillers ?? [])].sort(), + inkers: [...(comic.inkers ?? [])].sort(), + colorists: [...(comic.colorists ?? [])].sort(), + letterers: [...(comic.letterers ?? [])].sort(), + coverArtists: [...(comic.coverArtists ?? [])].sort(), + characters: [...(comic.characters ?? [])].sort(), + teams: [...(comic.teams ?? [])].sort(), + locations: [...(comic.locations ?? [])].sort(), + storyArcs: [...(comic.storyArcs ?? [])].sort(), + } + : null, + }; +} + +async function readBookWriteState(context: MetadataWriteE2EContext, bookId: number) { + const [metadata, authors, genres, tags] = await Promise.all([ + context.db + .select({ + title: schema.bookMetadata.title, + subtitle: schema.bookMetadata.subtitle, + description: schema.bookMetadata.description, + publisher: schema.bookMetadata.publisher, + publishedYear: schema.bookMetadata.publishedYear, + language: schema.bookMetadata.language, + pageCount: schema.bookMetadata.pageCount, + seriesName: schema.bookMetadata.seriesName, + seriesIndex: schema.bookMetadata.seriesIndex, + isbn10: schema.bookMetadata.isbn10, + isbn13: schema.bookMetadata.isbn13, + rating: schema.bookMetadata.rating, + googleBooksId: schema.bookMetadata.googleBooksId, + goodreadsId: schema.bookMetadata.goodreadsId, + amazonId: schema.bookMetadata.amazonId, + hardcoverId: schema.bookMetadata.hardcoverId, + openLibraryId: schema.bookMetadata.openLibraryId, + itunesId: schema.bookMetadata.itunesId, + }) + .from(schema.bookMetadata) + .where(eq(schema.bookMetadata.bookId, bookId)) + .limit(1), + context.db + .select({ name: schema.authors.name }) + .from(schema.bookAuthors) + .innerJoin(schema.authors, eq(schema.authors.id, schema.bookAuthors.authorId)) + .where(eq(schema.bookAuthors.bookId, bookId)) + .orderBy(schema.bookAuthors.displayOrder), + context.db + .select({ name: schema.genres.name }) + .from(schema.bookGenres) + .innerJoin(schema.genres, eq(schema.genres.id, schema.bookGenres.genreId)) + .where(eq(schema.bookGenres.bookId, bookId)), + context.db + .select({ name: schema.tags.name }) + .from(schema.bookTags) + .innerJoin(schema.tags, eq(schema.tags.id, schema.bookTags.tagId)) + .where(eq(schema.bookTags.bookId, bookId)), + ]); + + return { + ...(metadata[0] ?? {}), + authors: authors.map((row) => row.name), + genres: genres.map((row) => row.name).sort(), + tags: tags.map((row) => row.name).sort(), + }; +} + +function matchesBookWriteState(state: Awaited>, payload: ExpectedWritePayload): boolean { + return ( + state.title === payload.title && + state.subtitle === payload.subtitle && + state.description === payload.description && + state.publisher === payload.publisher && + state.publishedYear === payload.publishedYear && + state.language === payload.language && + state.pageCount === payload.pageCount && + state.seriesName === payload.seriesName && + state.seriesIndex === payload.seriesIndex && + state.isbn10 === payload.isbn10 && + state.isbn13 === payload.isbn13 && + state.rating === payload.rating && + state.googleBooksId === payload.googleBooksId && + state.goodreadsId === payload.goodreadsId && + state.amazonId === payload.amazonId && + state.hardcoverId === payload.hardcoverId && + state.openLibraryId === payload.openLibraryId && + state.itunesId === payload.itunesId && + JSON.stringify(state.authors) === JSON.stringify(payload.authors) && + JSON.stringify(state.genres) === JSON.stringify([...payload.genres].sort()) && + JSON.stringify(state.tags) === JSON.stringify([...payload.tags].sort()) + ); +} + +async function countSyncSuccessLogs(context: MetadataWriteE2EContext, bookId: number): Promise { + const rows = await context.db + .select({ triggeredBy: schema.fileWriteLog.triggeredBy, status: schema.fileWriteLog.status }) + .from(schema.fileWriteLog) + .where(eq(schema.fileWriteLog.bookId, bookId)); + + return rows.filter((row) => row.triggeredBy === 'sync' && row.status === 'success').length; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +}