mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
fix(kobo): render fixed-layout comics full screen on the device
Detect the OPF rendition:layout declaration and announce pre-paginated books as EPUB3FL instead of KEPUB, which is what selects the device's fixed-layout renderer. Kepub conversion is left alone so KoboSpan progress sync still works. The flag is stored per file and backfilled during the sync reconcile pass, so existing libraries converge without a rescan. Closes #1044
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1 @@
|
||||
ALTER TABLE "book_files" ADD COLUMN "is_fixed_layout" boolean;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -526,6 +526,13 @@
|
||||
"when": 1787522267109,
|
||||
"tag": "0074_add_scan_jobs_library_started_at_index",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 75,
|
||||
"version": "7",
|
||||
"when": 1787530767419,
|
||||
"tag": "0075_add_book_file_fixed_layout",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,19 @@
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { bigint, check, foreignKey, index, integer, numeric, pgTable, serial, timestamp, unique, uniqueIndex, varchar } from 'drizzle-orm/pg-core';
|
||||
import {
|
||||
bigint,
|
||||
boolean,
|
||||
check,
|
||||
foreignKey,
|
||||
index,
|
||||
integer,
|
||||
numeric,
|
||||
pgTable,
|
||||
serial,
|
||||
timestamp,
|
||||
unique,
|
||||
uniqueIndex,
|
||||
varchar,
|
||||
} from 'drizzle-orm/pg-core';
|
||||
|
||||
import { libraryFolders, libraries } from './libraries';
|
||||
|
||||
@@ -67,6 +81,8 @@ export const bookFiles = pgTable(
|
||||
role: varchar('role', { length: 20 }).notNull().default('content'),
|
||||
sortOrder: integer('sort_order'),
|
||||
durationSeconds: integer('duration_seconds'),
|
||||
// null means "not determined yet"; rows predating this column are backfilled lazily on Kobo sync.
|
||||
isFixedLayout: boolean('is_fixed_layout'),
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(),
|
||||
updatedAt: timestamp('updated_at', { withTimezone: true })
|
||||
.defaultNow()
|
||||
|
||||
@@ -4,6 +4,7 @@ import { CommonModule } from '../../common/common.module';
|
||||
import { AchievementModule } from '../achievement/achievement.module';
|
||||
import { AnnotationModule } from '../annotation/annotation.module';
|
||||
import { BookModule } from '../book/book.module';
|
||||
import { MetadataModule } from '../metadata/metadata.module';
|
||||
import { PositionConverterModule } from '../position-converter/position-converter.module';
|
||||
import { ReadingSessionModule } from '../reading-session/reading-session.module';
|
||||
import { SmartScopeModule } from '../smart-scope/smart-scope.module';
|
||||
@@ -41,6 +42,7 @@ import { KoboAnalyticsService } from './services/kobo-analytics.service';
|
||||
AchievementModule,
|
||||
AnnotationModule,
|
||||
BookModule,
|
||||
MetadataModule,
|
||||
PositionConverterModule,
|
||||
UserModule,
|
||||
UserBookStatusModule,
|
||||
|
||||
@@ -73,11 +73,13 @@ function makeDb(state?: Partial<QueueState>) {
|
||||
execute: [...(state?.execute ?? [])],
|
||||
};
|
||||
const chains: ReturnType<typeof makeChain>[] = [];
|
||||
const updateChains: ReturnType<typeof makeChain>[] = [];
|
||||
const transactionChains: ReturnType<typeof makeChain>[] = [];
|
||||
const transactionExecutes: ReturnType<typeof vi.fn>[] = [];
|
||||
|
||||
return {
|
||||
__chains: chains,
|
||||
__updateChains: updateChains,
|
||||
__transactionChains: transactionChains,
|
||||
__transactionExecutes: transactionExecutes,
|
||||
query: {
|
||||
@@ -94,7 +96,11 @@ function makeDb(state?: Partial<QueueState>) {
|
||||
return chain;
|
||||
}),
|
||||
insert: vi.fn(() => makeChain(queue.insert.shift() ?? [])),
|
||||
update: vi.fn(() => makeChain(queue.update.shift() ?? [])),
|
||||
update: vi.fn(() => {
|
||||
const chain = makeChain(queue.update.shift() ?? []);
|
||||
updateChains.push(chain);
|
||||
return chain;
|
||||
}),
|
||||
delete: vi.fn(() => makeChain(queue.delete.shift() ?? [])),
|
||||
execute: vi.fn(() => Promise.resolve({ rows: queue.execute.shift() ?? [] })),
|
||||
transaction: vi.fn(
|
||||
@@ -164,6 +170,9 @@ describe('KoboSyncService', () => {
|
||||
const smartScopeService = {
|
||||
findKoboSyncScopes: vi.fn(),
|
||||
};
|
||||
const metadataExtractionService = {
|
||||
detectFixedLayout: vi.fn(),
|
||||
};
|
||||
|
||||
function makeIdentity(bookId: number, needsLegacyNumericRemoval = false) {
|
||||
return {
|
||||
@@ -187,6 +196,7 @@ describe('KoboSyncService', () => {
|
||||
bookIdentityService as never,
|
||||
queryBuilder as never,
|
||||
smartScopeService as never,
|
||||
metadataExtractionService as never,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -196,6 +206,7 @@ describe('KoboSyncService', () => {
|
||||
smartScopeService.findKoboSyncScopes.mockResolvedValue([]);
|
||||
readingStateService.getRawState.mockResolvedValue(null);
|
||||
contentFilterRepository.findByUserId.mockResolvedValue([]);
|
||||
metadataExtractionService.detectFixedLayout.mockResolvedValue(null);
|
||||
bookIdentityService.ensureForBooks.mockImplementation((_userId: number, bookIds: number[], needsLegacyNumericRemoval: boolean) =>
|
||||
makeIdentityMap(bookIds, needsLegacyNumericRemoval),
|
||||
);
|
||||
@@ -236,8 +247,8 @@ describe('KoboSyncService', () => {
|
||||
|
||||
expect(createSpy).toHaveBeenNthCalledWith(1, 5, 101, eligible, new Set());
|
||||
expect(createSpy).toHaveBeenNthCalledWith(2, 5, 202, eligible, new Set());
|
||||
expect((service as any).fetchEligibleSnapshotRows).toHaveBeenNthCalledWith(1, 5, false, expect.any(Map));
|
||||
expect((service as any).fetchEligibleSnapshotRows).toHaveBeenNthCalledWith(2, 5, true, expect.any(Map));
|
||||
expect((service as any).fetchEligibleSnapshotRows).toHaveBeenNthCalledWith(1, 5, false, expect.any(Map), true);
|
||||
expect((service as any).fetchEligibleSnapshotRows).toHaveBeenNthCalledWith(2, 5, true, expect.any(Map), true);
|
||||
expect(pageSpy).toHaveBeenNthCalledWith(1, 5, 11, 'device-a-token', 'https://reader.example.com', expect.any(Map), expect.any(Function));
|
||||
expect(pageSpy).toHaveBeenNthCalledWith(2, 5, 22, 'device-b-token', 'https://reader.example.com', expect.any(Map), expect.any(Function));
|
||||
});
|
||||
@@ -1085,6 +1096,214 @@ describe('KoboSyncService', () => {
|
||||
expect(nativeKepub.hash).toBe(kepub.hash);
|
||||
});
|
||||
|
||||
describe('fixed-layout (comic) delivery', () => {
|
||||
const settings = { convertToKepub: true, forceEnableHyphenation: false, kepubConversionLimitMb: 1 };
|
||||
|
||||
it('announces a fixed-layout epub as EPUB3FL so the device renders it full screen', () => {
|
||||
const service = makeService(makeDb());
|
||||
|
||||
expect((service as any).getDeliveryInfo('epub', 1024, settings, true).format).toBe('EPUB3FL');
|
||||
});
|
||||
|
||||
it('announces a natively fixed-layout kepub as EPUB3FL', () => {
|
||||
const service = makeService(makeDb());
|
||||
|
||||
expect((service as any).getDeliveryInfo('kepub', 1024, settings, true).format).toBe('EPUB3FL');
|
||||
});
|
||||
|
||||
it('still announces EPUB3FL when kepub conversion is disabled or the file is over the limit', () => {
|
||||
const service = makeService(makeDb());
|
||||
|
||||
expect((service as any).getDeliveryInfo('epub', 1024, { ...settings, convertToKepub: false }, true).format).toBe('EPUB3FL');
|
||||
expect((service as any).getDeliveryInfo('epub', 2 * 1024 * 1024, settings, true).format).toBe('EPUB3FL');
|
||||
});
|
||||
|
||||
it('leaves a PDF alone, because the fixed-layout format only applies to EPUB', () => {
|
||||
const service = makeService(makeDb());
|
||||
|
||||
expect((service as any).getDeliveryInfo('pdf', 1024, settings, true).format).toBe('PDF');
|
||||
});
|
||||
|
||||
it('falls back to the reflowable formats when the flag is false or not yet known', () => {
|
||||
const service = makeService(makeDb());
|
||||
|
||||
expect((service as any).getDeliveryInfo('epub', 1024, settings, false).format).toBe('KEPUB');
|
||||
expect((service as any).getDeliveryInfo('epub', 1024, settings, null).format).toBe('KEPUB');
|
||||
expect((service as any).getDeliveryInfo('epub', 1024, { ...settings, convertToKepub: false }, null).format).toBe('EPUB3');
|
||||
});
|
||||
|
||||
it('separates hyphenated from plain kepub bytes even though both announce EPUB3FL', () => {
|
||||
const service = makeService(makeDb());
|
||||
|
||||
const plain = (service as any).getDeliveryInfo('epub', 1024, settings, true);
|
||||
const hyphenated = (service as any).getDeliveryInfo('epub', 1024, { ...settings, forceEnableHyphenation: true }, true);
|
||||
|
||||
expect(plain.format).toBe('EPUB3FL');
|
||||
expect(hyphenated.format).toBe('EPUB3FL');
|
||||
expect(hyphenated.hash).not.toBe(plain.hash);
|
||||
});
|
||||
|
||||
it('ignores hyphenation in the hash when no conversion runs, because the bytes are identical', () => {
|
||||
const service = makeService(makeDb());
|
||||
|
||||
const noConversion = { ...settings, convertToKepub: false };
|
||||
const plain = (service as any).getDeliveryInfo('epub', 1024, noConversion, true);
|
||||
const hyphenated = (service as any).getDeliveryInfo('epub', 1024, { ...noConversion, forceEnableHyphenation: true }, true);
|
||||
|
||||
expect(hyphenated.hash).toBe(plain.hash);
|
||||
});
|
||||
|
||||
it('changes the delivery hash so an already-synced comic is re-announced to the device', () => {
|
||||
const service = makeService(makeDb());
|
||||
|
||||
const before = (service as any).getDeliveryInfo('epub', 1024, settings, null);
|
||||
const after = (service as any).getDeliveryInfo('epub', 1024, settings, true);
|
||||
|
||||
expect(after.hash).not.toBe(before.hash);
|
||||
});
|
||||
});
|
||||
|
||||
describe('fixed-layout backfill', () => {
|
||||
function candidate(fileId: number, overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
fileId,
|
||||
fileAbsolutePath: `/books/${fileId}.epub`,
|
||||
fileFormat: 'epub',
|
||||
isFixedLayout: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
it('detects, returns, and persists the flag for files that have never been checked', async () => {
|
||||
const db = makeDb();
|
||||
const service = makeService(db);
|
||||
metadataExtractionService.detectFixedLayout.mockImplementation((path: string) => Promise.resolve(path === '/books/1.epub'));
|
||||
|
||||
const resolved = await (service as any).backfillFixedLayout([candidate(1), candidate(2)]);
|
||||
|
||||
expect(resolved).toEqual(
|
||||
new Map([
|
||||
[1, true],
|
||||
[2, false],
|
||||
]),
|
||||
);
|
||||
expect(metadataExtractionService.detectFixedLayout).toHaveBeenCalledTimes(2);
|
||||
// One grouped write per distinct value, never one per file.
|
||||
expect(db.update).toHaveBeenCalledTimes(2);
|
||||
expect(db.__updateChains[0]!.set).toHaveBeenCalledWith({ isFixedLayout: true });
|
||||
expect(db.__updateChains[1]!.set).toHaveBeenCalledWith({ isFixedLayout: false });
|
||||
});
|
||||
|
||||
it('never re-reads a file whose flag is already stored', async () => {
|
||||
const db = makeDb();
|
||||
const service = makeService(db);
|
||||
|
||||
const resolved = await (service as any).backfillFixedLayout([candidate(1, { isFixedLayout: true }), candidate(2, { isFixedLayout: false })]);
|
||||
|
||||
expect(resolved.size).toBe(0);
|
||||
expect(metadataExtractionService.detectFixedLayout).not.toHaveBeenCalled();
|
||||
expect(db.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('ignores formats that cannot declare a layout', async () => {
|
||||
const db = makeDb();
|
||||
const service = makeService(db);
|
||||
|
||||
await (service as any).backfillFixedLayout([
|
||||
candidate(1, { fileFormat: 'pdf' }),
|
||||
candidate(2, { fileFormat: 'cbz' }),
|
||||
candidate(3, { fileFormat: null }),
|
||||
]);
|
||||
|
||||
expect(metadataExtractionService.detectFixedLayout).not.toHaveBeenCalled();
|
||||
expect(db.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('checks kepub files too, since a native kepub carries the same OPF', async () => {
|
||||
const service = makeService(makeDb());
|
||||
metadataExtractionService.detectFixedLayout.mockResolvedValue(true);
|
||||
|
||||
const resolved = await (service as any).backfillFixedLayout([candidate(1, { fileFormat: 'kepub' })]);
|
||||
|
||||
expect(resolved.get(1)).toBe(true);
|
||||
});
|
||||
|
||||
it('caps how many files one sync opens, leaving the rest for the next sync', async () => {
|
||||
const db = makeDb();
|
||||
const service = makeService(db);
|
||||
metadataExtractionService.detectFixedLayout.mockResolvedValue(false);
|
||||
|
||||
const resolved = await (service as any).backfillFixedLayout(Array.from({ length: 600 }, (_unused, index) => candidate(index + 1)));
|
||||
|
||||
expect(metadataExtractionService.detectFixedLayout).toHaveBeenCalledTimes(500);
|
||||
expect(resolved.size).toBe(500);
|
||||
});
|
||||
|
||||
it('leaves an unreadable file unresolved instead of recording it as reflowable', async () => {
|
||||
const db = makeDb();
|
||||
const service = makeService(db);
|
||||
metadataExtractionService.detectFixedLayout.mockResolvedValue(null);
|
||||
|
||||
const resolved = await (service as any).backfillFixedLayout([candidate(1)]);
|
||||
|
||||
expect(resolved.size).toBe(0);
|
||||
expect(db.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('treats a thrown detection as unknown and keeps processing the rest', async () => {
|
||||
const db = makeDb();
|
||||
const service = makeService(db);
|
||||
metadataExtractionService.detectFixedLayout.mockImplementation((path: string) =>
|
||||
path === '/books/1.epub' ? Promise.reject(new Error('permission denied')) : Promise.resolve(true),
|
||||
);
|
||||
|
||||
const resolved = await (service as any).backfillFixedLayout([candidate(1), candidate(2)]);
|
||||
|
||||
expect(resolved).toEqual(new Map([[2, true]]));
|
||||
expect(db.__updateChains[0]!.set).toHaveBeenCalledWith({ isFixedLayout: true });
|
||||
});
|
||||
|
||||
it('still announces the right format for this sync when the flag cannot be persisted', async () => {
|
||||
const db = makeDb();
|
||||
db.update.mockImplementationOnce(() => {
|
||||
throw new Error('database is read only');
|
||||
});
|
||||
const service = makeService(db);
|
||||
metadataExtractionService.detectFixedLayout.mockResolvedValue(true);
|
||||
|
||||
const resolved = await (service as any).backfillFixedLayout([candidate(1)]);
|
||||
|
||||
expect(resolved.get(1)).toBe(true);
|
||||
});
|
||||
|
||||
it('does nothing at all once every eligible file has been checked', async () => {
|
||||
const db = makeDb();
|
||||
const service = makeService(db);
|
||||
|
||||
await (service as any).backfillFixedLayout([]);
|
||||
|
||||
expect(metadataExtractionService.detectFixedLayout).not.toHaveBeenCalled();
|
||||
expect(db.update).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// Delivery hashes are stored per device: any drift re-marks every book as changed and makes
|
||||
// every device re-download its whole library. These are the values the pre-EPUB3FL
|
||||
// implementation produced and they must not move.
|
||||
it('keeps delivery hashes for non-comic books byte-identical to the previous implementation', () => {
|
||||
const service = makeService(makeDb());
|
||||
const base = { convertToKepub: true, forceEnableHyphenation: false, kepubConversionLimitMb: 1 };
|
||||
|
||||
expect((service as any).getDeliveryInfo('pdf', 1024, base, null).hash).toBe('6c4931d80e7cbf0d');
|
||||
expect((service as any).getDeliveryInfo('kepub', 1024, base, null).hash).toBe('2f55f4d028ef859c');
|
||||
expect((service as any).getDeliveryInfo('epub', 1024, base, null).hash).toBe('2f55f4d028ef859c');
|
||||
expect((service as any).getDeliveryInfo('epub', 1024, { ...base, forceEnableHyphenation: true }, null).hash).toBe('990477a32a0f0d75');
|
||||
expect((service as any).getDeliveryInfo('epub', 1024, { ...base, convertToKepub: false }, null).hash).toBe('90fb0bb55cca2fd2');
|
||||
expect((service as any).getDeliveryInfo('epub', 2 * 1024 * 1024, base, null).hash).toBe('90fb0bb55cca2fd2');
|
||||
// Hyphenation must not leak into the hash of a book that is never converted.
|
||||
expect((service as any).getDeliveryInfo('epub', 2 * 1024 * 1024, { ...base, forceEnableHyphenation: true }, null).hash).toBe('90fb0bb55cca2fd2');
|
||||
});
|
||||
|
||||
it('fetchEligibleSnapshotRows and fetchEligibleBooksByIds map DB rows into sync payload objects', async () => {
|
||||
const db = makeDb({
|
||||
select: [
|
||||
@@ -1158,4 +1377,80 @@ describe('KoboSyncService', () => {
|
||||
);
|
||||
expect(books.get(5)?.metadataHash).toBe(snapshotRows[0].metadataHash);
|
||||
});
|
||||
|
||||
describe('fixed-layout books end to end', () => {
|
||||
function comicRows(isFixedLayout: boolean | null) {
|
||||
const shared = {
|
||||
bookId: 5,
|
||||
title: 'Manga Vol. 1',
|
||||
language: 'English',
|
||||
isbn10: null,
|
||||
isbn13: null,
|
||||
seriesName: null,
|
||||
seriesIndex: null,
|
||||
metadataUpdatedAt: new Date('2026-01-01T00:00:00.000Z'),
|
||||
fileFormat: 'epub',
|
||||
fileSizeBytes: 1234,
|
||||
fileHash: 'file-hash',
|
||||
isFixedLayout,
|
||||
};
|
||||
return {
|
||||
select: [
|
||||
[{ ...shared, fileId: 77, fileAbsolutePath: '/books/manga.epub', authorNamesCsv: 'Mangaka' }],
|
||||
[{ ...shared, description: null, publisher: null, publishedYear: null, addedAt: new Date(), updatedAt: new Date() }],
|
||||
[{ bookId: 5, name: 'Mangaka' }],
|
||||
[],
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
it('announces a stored fixed-layout book as EPUB3FL in the payload the device receives', async () => {
|
||||
const db = makeDb(comicRows(true));
|
||||
const service = makeService(db);
|
||||
vi.spyOn(service as any, 'buildEligibleBooksWhereClause').mockResolvedValue({ where: true });
|
||||
|
||||
await (service as any).fetchEligibleSnapshotRows(8, false, new Map());
|
||||
const books = await (service as any).fetchEligibleBooksByIds(8, [5], false, new Map());
|
||||
const metadata = (service as any).buildBookMetadata(books.get(5), 'tok', 'https://base') as Record<string, unknown>;
|
||||
|
||||
expect((metadata.DownloadUrls as Array<Record<string, unknown>>)[0]!.Format).toBe('EPUB3FL');
|
||||
});
|
||||
|
||||
it('backfills an unchecked comic during the reconcile pass and hashes it as EPUB3FL immediately', async () => {
|
||||
const db = makeDb(comicRows(null));
|
||||
const service = makeService(db);
|
||||
vi.spyOn(service as any, 'buildEligibleBooksWhereClause').mockResolvedValue({ where: true });
|
||||
metadataExtractionService.detectFixedLayout.mockResolvedValue(true);
|
||||
|
||||
const rows = await (service as any).fetchEligibleSnapshotRows(8, false, new Map(), true);
|
||||
|
||||
expect(metadataExtractionService.detectFixedLayout).toHaveBeenCalledWith('/books/manga.epub', 'epub');
|
||||
expect(db.__updateChains[0]!.set).toHaveBeenCalledWith({ isFixedLayout: true });
|
||||
// The same pass that learns the flag also stores the EPUB3FL hash, so the device is told
|
||||
// about the comic on this sync rather than the next one.
|
||||
expect(rows[0].deliveryHash).toBe('6a8354534ee2c5c8');
|
||||
});
|
||||
|
||||
it('does no filesystem work on the mid-page pass that only needs eligible ids', async () => {
|
||||
const db = makeDb(comicRows(null));
|
||||
const service = makeService(db);
|
||||
vi.spyOn(service as any, 'buildEligibleBooksWhereClause').mockResolvedValue({ where: true });
|
||||
|
||||
await (service as any).fetchEligibleSnapshotRows(8, false, new Map());
|
||||
|
||||
expect(metadataExtractionService.detectFixedLayout).not.toHaveBeenCalled();
|
||||
expect(db.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not open files for a book whose flag is already stored', async () => {
|
||||
const db = makeDb(comicRows(false));
|
||||
const service = makeService(db);
|
||||
vi.spyOn(service as any, 'buildEligibleBooksWhereClause').mockResolvedValue({ where: true });
|
||||
|
||||
const rows = await (service as any).fetchEligibleSnapshotRows(8, false, new Map(), true);
|
||||
|
||||
expect(metadataExtractionService.detectFixedLayout).not.toHaveBeenCalled();
|
||||
expect(rows[0].deliveryHash).toBe('2f55f4d028ef859c');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
import { createHash } from 'crypto';
|
||||
import { Inject, Injectable, InternalServerErrorException } from '@nestjs/common';
|
||||
import { Inject, Injectable, InternalServerErrorException, Logger } from '@nestjs/common';
|
||||
import { SQL, and, asc, eq, inArray, isNull, lte, or, sql } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import { DB } from '../../../db/db.module';
|
||||
import * as schema from '../../../db/schema';
|
||||
import { mapWithConcurrency } from '../../../common/utils/batch.utils';
|
||||
import { buildContentFilterClauses } from '../../../common/utils/content-filter-sql.utils';
|
||||
import { sanitizeLogValue } from '../../../common/utils/log-sanitize.utils';
|
||||
import { resolveTimeZone } from '../../../common/utils/timezone.utils';
|
||||
import { ContentFilterRepository } from '../../user/content-filter.repository';
|
||||
import { BookQueryBuilder } from '../../book/book-query-builder.service';
|
||||
import { MetadataExtractionService } from '../../metadata/metadata-extraction.service';
|
||||
import { SmartScopeService } from '../../smart-scope/smart-scope.service';
|
||||
import { KoboBookAccessService } from './kobo-book-access.service';
|
||||
import { KoboBookIdentityService } from './kobo-book-identity.service';
|
||||
@@ -26,6 +29,12 @@ const SNAPSHOT_RECONCILE_BATCH_SIZE = 5000;
|
||||
const SNAPSHOT_CREATE_BATCH_SIZE = 5000;
|
||||
const METADATA_SERIALIZER_VERSION = 2;
|
||||
|
||||
// Fixed-layout detection opens the EPUB, so a sync never resolves the whole backlog at once.
|
||||
// Each pass persists what it learns, so a library converges over successive syncs and every
|
||||
// later sync reads the stored answer instead of touching the filesystem.
|
||||
const FIXED_LAYOUT_BACKFILL_LIMIT = 500;
|
||||
const FIXED_LAYOUT_BACKFILL_CONCURRENCY = 8;
|
||||
|
||||
type EligibleSnapshotRow = {
|
||||
bookId: number;
|
||||
fileHash: string | null;
|
||||
@@ -56,7 +65,16 @@ type KoboDeliverySettings = {
|
||||
kepubConversionLimitMb: number;
|
||||
};
|
||||
|
||||
type KoboDeliveryFormat = 'EPUB3' | 'KEPUB' | 'PDF';
|
||||
// EPUB3FL tells the device to use its fixed-layout renderer, which is what makes a comic fill
|
||||
// the screen instead of picking up the reflowable reading margins.
|
||||
type KoboDeliveryFormat = 'EPUB3' | 'EPUB3FL' | 'KEPUB' | 'PDF';
|
||||
|
||||
type FixedLayoutCandidate = {
|
||||
fileId: number;
|
||||
fileAbsolutePath: string;
|
||||
fileFormat: string | null;
|
||||
isFixedLayout: boolean | null;
|
||||
};
|
||||
|
||||
type KoboDeliveryInfo = {
|
||||
format: KoboDeliveryFormat;
|
||||
@@ -99,6 +117,8 @@ export interface KoboBookEntry {
|
||||
|
||||
@Injectable()
|
||||
export class KoboSyncService {
|
||||
private readonly logger = new Logger(KoboSyncService.name);
|
||||
|
||||
constructor(
|
||||
@Inject(DB) private readonly db: Db,
|
||||
private readonly bookAccessService: KoboBookAccessService,
|
||||
@@ -107,6 +127,7 @@ export class KoboSyncService {
|
||||
private readonly bookIdentityService: KoboBookIdentityService,
|
||||
private readonly queryBuilder: BookQueryBuilder,
|
||||
private readonly smartScopeService: SmartScopeService,
|
||||
private readonly metadataExtractionService: MetadataExtractionService,
|
||||
) {}
|
||||
|
||||
async getDelta(
|
||||
@@ -129,7 +150,7 @@ export class KoboSyncService {
|
||||
|
||||
const hadDeviceSnapshot = snapshot ? true : await this.hasDeviceSnapshot(userId);
|
||||
const legacyNumericRemovalBookIds = snapshot ? new Set<number>() : await this.getLegacyNumericRemovalBookIds(userId, deviceId);
|
||||
const eligibleSnapshotRows = await this.fetchEligibleSnapshotRows(userId, hadDeviceSnapshot, smartScopeMatchCache);
|
||||
const eligibleSnapshotRows = await this.fetchEligibleSnapshotRows(userId, hadDeviceSnapshot, smartScopeMatchCache, true);
|
||||
|
||||
if (!snapshot) {
|
||||
const created = await this.createSnapshot(userId, deviceId, eligibleSnapshotRows, legacyNumericRemovalBookIds);
|
||||
@@ -891,36 +912,112 @@ export class KoboSyncService {
|
||||
return createHash('sha256').update(metaStr).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
/**
|
||||
* `hyphenate` describes the bytes actually delivered, not the announced format, so a hyphenated
|
||||
* kepub and a plain one never collide even when both are announced as EPUB3FL.
|
||||
*/
|
||||
private buildDeliveryHash(format: KoboDeliveryFormat, hyphenate: boolean): string {
|
||||
return createHash('sha256')
|
||||
.update([format, format === 'KEPUB' && hyphenate ? 'hyphenate' : 'plain'].join('|'))
|
||||
.update([format, hyphenate ? 'hyphenate' : 'plain'].join('|'))
|
||||
.digest('hex')
|
||||
.slice(0, 16);
|
||||
}
|
||||
|
||||
private getDeliveryInfo(fileFormat: string | null, fileSizeBytes: number | null, settings: KoboDeliverySettings): KoboDeliveryInfo {
|
||||
private getDeliveryInfo(
|
||||
fileFormat: string | null,
|
||||
fileSizeBytes: number | null,
|
||||
settings: KoboDeliverySettings,
|
||||
isFixedLayout: boolean | null,
|
||||
): KoboDeliveryInfo {
|
||||
const normalizedFormat = (fileFormat ?? 'epub').toLowerCase();
|
||||
|
||||
if (normalizedFormat === 'pdf') {
|
||||
return { format: 'PDF', hash: this.buildDeliveryHash('PDF', false) };
|
||||
}
|
||||
|
||||
const limitBytes = settings.kepubConversionLimitMb * 1024 * 1024;
|
||||
const convertsToKepub = normalizedFormat === 'epub' && settings.convertToKepub && (!fileSizeBytes || fileSizeBytes <= limitBytes);
|
||||
const hyphenated = convertsToKepub && settings.forceEnableHyphenation;
|
||||
|
||||
// Announcing EPUB3FL is what earns the full-screen renderer; the bytes are deliberately left
|
||||
// alone. A kepub renders full screen just the same once the format says fixed layout, and
|
||||
// keeping the conversion preserves the KoboSpan positions two-way progress sync depends on.
|
||||
if (isFixedLayout && (normalizedFormat === 'epub' || normalizedFormat === 'kepub')) {
|
||||
return { format: 'EPUB3FL', hash: this.buildDeliveryHash('EPUB3FL', hyphenated) };
|
||||
}
|
||||
|
||||
if (normalizedFormat === 'kepub') {
|
||||
return { format: 'KEPUB', hash: this.buildDeliveryHash('KEPUB', false) };
|
||||
}
|
||||
|
||||
if (normalizedFormat === 'epub') {
|
||||
const limitBytes = settings.kepubConversionLimitMb * 1024 * 1024;
|
||||
const withinLimit = !fileSizeBytes || fileSizeBytes <= limitBytes;
|
||||
|
||||
if (settings.convertToKepub && withinLimit) {
|
||||
return { format: 'KEPUB', hash: this.buildDeliveryHash('KEPUB', settings.forceEnableHyphenation) };
|
||||
}
|
||||
if (convertsToKepub) {
|
||||
return { format: 'KEPUB', hash: this.buildDeliveryHash('KEPUB', hyphenated) };
|
||||
}
|
||||
|
||||
return { format: 'EPUB3', hash: this.buildDeliveryHash('EPUB3', false) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the fixed-layout flag for eligible files that have never been checked, and stores what
|
||||
* it finds so later syncs read the column instead of the filesystem. Capped per sync because this
|
||||
* runs on the whole-library pass; anything left over is picked up by the next sync, which only
|
||||
* delays a comic's re-announcement rather than losing it.
|
||||
*
|
||||
* A file that cannot be read stays null so it is retried later: persisting a read failure as
|
||||
* `false` would permanently mark a comic reflowable.
|
||||
*/
|
||||
private async backfillFixedLayout(rows: readonly FixedLayoutCandidate[]): Promise<Map<number, boolean>> {
|
||||
const pending = rows
|
||||
.filter((row) => row.isFixedLayout === null && (row.fileFormat === 'epub' || row.fileFormat === 'kepub'))
|
||||
.slice(0, FIXED_LAYOUT_BACKFILL_LIMIT);
|
||||
const resolved = new Map<number, boolean>();
|
||||
if (pending.length === 0) return resolved;
|
||||
|
||||
const startedAt = Date.now();
|
||||
const detected = await mapWithConcurrency(pending, FIXED_LAYOUT_BACKFILL_CONCURRENCY, async (row) => {
|
||||
try {
|
||||
return await this.metadataExtractionService.detectFixedLayout(row.fileAbsolutePath, row.fileFormat!);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
const fileIdsByValue = new Map<boolean, number[]>();
|
||||
detected.forEach((value, index) => {
|
||||
if (value === null) return;
|
||||
const fileId = pending[index]!.fileId;
|
||||
resolved.set(fileId, value);
|
||||
const bucket = fileIdsByValue.get(value) ?? [];
|
||||
bucket.push(fileId);
|
||||
fileIdsByValue.set(value, bucket);
|
||||
});
|
||||
|
||||
try {
|
||||
for (const [value, fileIds] of fileIdsByValue) {
|
||||
// Still-null guard: keeps the write idempotent, avoids clobbering a value a concurrent
|
||||
// scan just wrote, and keeps book_files.updatedAt (the KOReader library token and OPDS
|
||||
// entry timestamps) from being bumped by a sync that changed nothing.
|
||||
await this.db
|
||||
.update(schema.bookFiles)
|
||||
.set({ isFixedLayout: value })
|
||||
.where(and(inArray(schema.bookFiles.id, fileIds), isNull(schema.bookFiles.isFixedLayout)));
|
||||
}
|
||||
} catch (err) {
|
||||
// The detected values are still returned, so this sync announces the right format either
|
||||
// way; only the persistence that saves the next sync the work is lost.
|
||||
const error = err instanceof Error ? err : new Error(String(err));
|
||||
this.logger.warn(
|
||||
`[kobo.fixed_layout_backfill] [fail] candidates=${pending.length} errorClass=${error.constructor.name} error="${sanitizeLogValue(error.message)}" - could not persist fixed layout flags`,
|
||||
);
|
||||
return resolved;
|
||||
}
|
||||
|
||||
this.logger.debug(
|
||||
`[kobo.fixed_layout_backfill] [end] candidates=${pending.length} resolved=${resolved.size} fixedLayout=${fileIdsByValue.get(true)?.length ?? 0} durationMs=${Date.now() - startedAt} - fixed layout flags backfilled`,
|
||||
);
|
||||
return resolved;
|
||||
}
|
||||
|
||||
private async getDeliverySettings(userId: number): Promise<KoboDeliverySettings> {
|
||||
const settings = await this.db.query.koboSyncSettings.findFirst({
|
||||
where: eq(schema.koboSyncSettings.userId, userId),
|
||||
@@ -990,6 +1087,7 @@ export class KoboSyncService {
|
||||
userId: number,
|
||||
needsLegacyNumericRemovalForNewMappings: boolean,
|
||||
smartScopeMatchCache: SmartScopeMatchCache,
|
||||
backfillFixedLayout = false,
|
||||
): Promise<EligibleSnapshotRow[]> {
|
||||
const whereClause = await this.buildEligibleBooksWhereClause(userId, smartScopeMatchCache);
|
||||
if (!whereClause) return [];
|
||||
@@ -1008,6 +1106,9 @@ export class KoboSyncService {
|
||||
fileFormat: schema.bookFiles.format,
|
||||
fileSizeBytes: schema.bookFiles.sizeBytes,
|
||||
fileHash: schema.bookFiles.fileHash,
|
||||
fileId: schema.bookFiles.id,
|
||||
fileAbsolutePath: schema.bookFiles.absolutePath,
|
||||
isFixedLayout: schema.bookFiles.isFixedLayout,
|
||||
authorNamesCsv: sql<string>`coalesce(string_agg(${schema.authors.name}, ',' ORDER BY ${schema.bookAuthors.displayOrder}, ${schema.bookAuthors.authorId}), '')`,
|
||||
})
|
||||
.from(schema.books)
|
||||
@@ -1028,6 +1129,9 @@ export class KoboSyncService {
|
||||
schema.bookFiles.format,
|
||||
schema.bookFiles.sizeBytes,
|
||||
schema.bookFiles.fileHash,
|
||||
schema.bookFiles.id,
|
||||
schema.bookFiles.absolutePath,
|
||||
schema.bookFiles.isFixedLayout,
|
||||
);
|
||||
|
||||
const identitiesById = await this.bookIdentityService.ensureForBooks(
|
||||
@@ -1036,8 +1140,11 @@ export class KoboSyncService {
|
||||
needsLegacyNumericRemovalForNewMappings,
|
||||
);
|
||||
|
||||
const fixedLayoutByFileId = backfillFixedLayout ? await this.backfillFixedLayout(rows) : new Map<number, boolean>();
|
||||
|
||||
return rows.map((row) => {
|
||||
const delivery = this.getDeliveryInfo(row.fileFormat, row.fileSizeBytes, deliverySettings);
|
||||
const isFixedLayout = row.isFixedLayout ?? fixedLayoutByFileId.get(row.fileId) ?? null;
|
||||
const delivery = this.getDeliveryInfo(row.fileFormat, row.fileSizeBytes, deliverySettings, isFixedLayout);
|
||||
const identity = identitiesById.get(row.bookId);
|
||||
const coverImageId = identity
|
||||
? this.bookIdentityService.buildVersionedCoverImageId(identity.coverImageId, row.metadataUpdatedAt)
|
||||
@@ -1091,6 +1198,7 @@ export class KoboSyncService {
|
||||
fileFormat: schema.bookFiles.format,
|
||||
fileSizeBytes: schema.bookFiles.sizeBytes,
|
||||
fileHash: schema.bookFiles.fileHash,
|
||||
isFixedLayout: schema.bookFiles.isFixedLayout,
|
||||
metadataUpdatedAt: schema.bookMetadata.updatedAt,
|
||||
addedAt: schema.books.addedAt,
|
||||
updatedAt: schema.books.updatedAt,
|
||||
@@ -1142,7 +1250,7 @@ export class KoboSyncService {
|
||||
const byId = new Map<number, KoboBookEntry>();
|
||||
for (const row of rows) {
|
||||
const authors = authorsByBook.get(row.bookId) ?? [];
|
||||
const delivery = this.getDeliveryInfo(row.fileFormat, row.fileSizeBytes, deliverySettings);
|
||||
const delivery = this.getDeliveryInfo(row.fileFormat, row.fileSizeBytes, deliverySettings, row.isFixedLayout);
|
||||
const identity = identitiesById.get(row.bookId);
|
||||
if (!identity) continue;
|
||||
const coverImageId = this.bookIdentityService.buildVersionedCoverImageId(identity.coverImageId, row.metadataUpdatedAt);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { extractEpubCover } from '../lib/cover-epub';
|
||||
import { extractEpubMetadata } from '../lib/epub';
|
||||
import { extractEpubMetadata, isPrePaginated } from '../lib/epub';
|
||||
import type { FormatExtractor, ParsedBookData } from './format-extractor.interface';
|
||||
import { mapOpfMetadata } from './opf-metadata.mapper';
|
||||
|
||||
@@ -7,6 +7,6 @@ export class EpubFormatExtractor implements FormatExtractor {
|
||||
async extract(absolutePath: string): Promise<ParsedBookData | null> {
|
||||
const [metadata, cover] = await Promise.all([extractEpubMetadata(absolutePath), extractEpubCover(absolutePath).catch(() => null)]);
|
||||
if (!metadata) return null;
|
||||
return mapOpfMetadata(metadata, cover ?? null);
|
||||
return { ...mapOpfMetadata(metadata, cover ?? null), isFixedLayout: isPrePaginated(metadata) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +39,11 @@ export interface ParsedBookData {
|
||||
chapters?: { title: string; startMs: number }[];
|
||||
// ebook-specific extras
|
||||
pageCount?: number | null;
|
||||
/**
|
||||
* Fixed-layout (`rendition:layout: pre-paginated`) declaration. Only EPUB extraction sets it,
|
||||
* because it describes the book file itself, not the bibliographic metadata a sidecar can carry.
|
||||
*/
|
||||
isFixedLayout?: boolean | null;
|
||||
comicMetadata?: ParsedCbzComicMetadata | null;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,7 +4,8 @@ vi.mock('fs/promises', () => ({
|
||||
readFile: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../lib/epub', () => ({
|
||||
vi.mock('../lib/epub', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('../lib/epub')>()),
|
||||
extractEpubMetadata: vi.fn(),
|
||||
}));
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ vi.mock('./opf-parser', () => ({ parseOpf: vi.fn() }));
|
||||
import * as unzipper from 'unzipper';
|
||||
|
||||
import { parseOpf } from './opf-parser';
|
||||
import { extractEpubMetadata } from './epub';
|
||||
import { extractEpubFixedLayout, extractEpubMetadata, isPrePaginated } from './epub';
|
||||
|
||||
const mockOpenFile = (unzipper as any).Open.file as vi.Mock;
|
||||
const mockParseOpf = parseOpf as MockedFunction<typeof parseOpf>;
|
||||
@@ -142,3 +142,71 @@ describe('extractEpubMetadata', () => {
|
||||
await expect(extractEpubMetadata('/books/y.epub')).resolves.toEqual(expect.objectContaining({ isbn10: '0306406152', isbn13: null }));
|
||||
});
|
||||
});
|
||||
|
||||
describe('isPrePaginated', () => {
|
||||
it('recognizes the fixed-layout declaration', () => {
|
||||
expect(isPrePaginated({ renditionLayout: 'pre-paginated' })).toBe(true);
|
||||
});
|
||||
|
||||
it('tolerates surrounding whitespace and casing from hand-edited OPFs', () => {
|
||||
expect(isPrePaginated({ renditionLayout: ' Pre-Paginated \n' })).toBe(true);
|
||||
});
|
||||
|
||||
it('treats a reflowable, empty, or absent declaration as not fixed layout', () => {
|
||||
expect(isPrePaginated({ renditionLayout: 'reflowable' })).toBe(false);
|
||||
expect(isPrePaginated({ renditionLayout: '' })).toBe(false);
|
||||
expect(isPrePaginated({ renditionLayout: null })).toBe(false);
|
||||
});
|
||||
|
||||
it('does not match a value that merely contains the keyword', () => {
|
||||
expect(isPrePaginated({ renditionLayout: 'not-pre-paginated' })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractEpubFixedLayout', () => {
|
||||
beforeEach(() => vi.resetAllMocks());
|
||||
|
||||
function mockEpub(renditionLayout: string | null) {
|
||||
const containerXml = `<container><rootfiles><rootfile full-path="content.opf" /></rootfiles></container>`;
|
||||
mockOpenFile.mockResolvedValue({
|
||||
files: [zipFile('META-INF/container.xml', containerXml), zipFile('content.opf', '<package/>')],
|
||||
});
|
||||
mockParseOpf.mockReturnValue({ renditionLayout } as never);
|
||||
}
|
||||
|
||||
it('reports a comic as fixed layout', async () => {
|
||||
mockEpub('pre-paginated');
|
||||
await expect(extractEpubFixedLayout('/books/manga.epub')).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it('reports a novel as not fixed layout', async () => {
|
||||
mockEpub('reflowable');
|
||||
await expect(extractEpubFixedLayout('/books/novel.epub')).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('reports an EPUB that declares nothing as not fixed layout', async () => {
|
||||
mockEpub(null);
|
||||
await expect(extractEpubFixedLayout('/books/plain.epub')).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('returns null rather than false when the file cannot be opened, so the answer is retried', async () => {
|
||||
mockOpenFile.mockRejectedValue(new Error('ENOENT'));
|
||||
await expect(extractEpubFixedLayout('/books/missing.epub')).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('returns null when the archive has no container.xml to locate the OPF', async () => {
|
||||
mockOpenFile.mockResolvedValue({ files: [] });
|
||||
await expect(extractEpubFixedLayout('/books/broken.epub')).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('returns null when the OPF itself cannot be parsed', async () => {
|
||||
const containerXml = `<container><rootfiles><rootfile full-path="content.opf" /></rootfiles></container>`;
|
||||
mockOpenFile.mockResolvedValue({
|
||||
files: [zipFile('META-INF/container.xml', containerXml), zipFile('content.opf', '<package/>')],
|
||||
});
|
||||
mockParseOpf.mockImplementation(() => {
|
||||
throw new Error('malformed xml');
|
||||
});
|
||||
await expect(extractEpubFixedLayout('/books/corrupt.epub')).resolves.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -35,6 +35,29 @@ async function findOpfPath(zip: unzipper.CentralDirectory): Promise<string> {
|
||||
return opfPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when an OPF declares the EPUB fixed-layout. Comics and manga converted by tools
|
||||
* such as Kindle Comic Converter set this; readers must render them without reflow margins.
|
||||
*/
|
||||
export function isPrePaginated(opf: Pick<ParsedOpf, 'renditionLayout'>): boolean {
|
||||
return opf.renditionLayout?.trim().toLowerCase() === 'pre-paginated';
|
||||
}
|
||||
|
||||
/**
|
||||
* Read only the fixed-layout flag from an EPUB. Returns null when the file cannot be opened
|
||||
* or parsed, which callers must keep distinct from a parsed `false` so a transient read
|
||||
* failure is never persisted as "this book reflows".
|
||||
*/
|
||||
export async function extractEpubFixedLayout(absolutePath: string): Promise<boolean | null> {
|
||||
try {
|
||||
const zip = await unzipper.Open.file(absolutePath);
|
||||
const opfPath = await findOpfPath(zip);
|
||||
return isPrePaginated(parseOpf(await readFileFromZip(zip, opfPath)));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Open an EPUB file and extract metadata from its OPF.
|
||||
* Returns null if the file is not a valid EPUB or parsing fails.
|
||||
|
||||
@@ -905,4 +905,47 @@ describe('parseOpf', () => {
|
||||
expect(parseOpf(xml).coverHref).toBe('epub3-cover.jpg');
|
||||
});
|
||||
});
|
||||
|
||||
describe('rendition layout', () => {
|
||||
it('parses the pre-paginated declaration a comic converter writes', () => {
|
||||
const xml = epub3Opf(`
|
||||
<dc:title>Manga Vol. 1</dc:title>
|
||||
<meta property="rendition:spread">landscape</meta>
|
||||
<meta property="rendition:layout">pre-paginated</meta>
|
||||
`);
|
||||
expect(parseOpf(xml).renditionLayout).toBe('pre-paginated');
|
||||
});
|
||||
|
||||
it('parses a reflowable declaration without confusing it for a fixed layout', () => {
|
||||
const xml = epub3Opf(`
|
||||
<dc:title>Novel</dc:title>
|
||||
<meta property="rendition:layout">reflowable</meta>
|
||||
`);
|
||||
expect(parseOpf(xml).renditionLayout).toBe('reflowable');
|
||||
});
|
||||
|
||||
it('is null when the OPF declares no layout at all', () => {
|
||||
expect(parseOpf(epub3Opf('<dc:title>Novel</dc:title>')).renditionLayout).toBeNull();
|
||||
});
|
||||
|
||||
it('reads the declaration through a namespace-prefixed meta element', () => {
|
||||
const xml = `<?xml version="1.0" encoding="utf-8"?>
|
||||
<opf:package xmlns:opf="http://www.idpf.org/2007/opf" version="3.0">
|
||||
<opf:metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:title>Manga Vol. 2</dc:title>
|
||||
<opf:meta property="rendition:layout">pre-paginated</opf:meta>
|
||||
</opf:metadata>
|
||||
</opf:package>`;
|
||||
expect(parseOpf(xml).renditionLayout).toBe('pre-paginated');
|
||||
});
|
||||
|
||||
it('ignores a spine-level fixed-layout override so a mostly reflowable book stays reflowable', () => {
|
||||
const xml = `<?xml version="1.0" encoding="utf-8"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf" version="3.0">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:title>Novel</dc:title></metadata>
|
||||
<spine><itemref idref="map" properties="rendition:layout-pre-paginated"/></spine>
|
||||
</package>`;
|
||||
expect(parseOpf(xml).renditionLayout).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -32,6 +32,8 @@ export interface ParsedOpf {
|
||||
itunesId: string | null;
|
||||
customMetadata: Record<string, string>;
|
||||
coverHref: string | null;
|
||||
/** Raw `rendition:layout` value; `pre-paginated` marks a fixed-layout book such as a comic. */
|
||||
renditionLayout: string | null;
|
||||
}
|
||||
|
||||
const parser = new XMLParser({
|
||||
@@ -551,6 +553,7 @@ export function parseOpf(xml: string): ParsedOpf {
|
||||
itunesId,
|
||||
customMetadata,
|
||||
coverHref,
|
||||
renditionLayout: propertyMeta('rendition:layout'),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,11 @@ const mocks = vi.hoisted(() => ({
|
||||
opf: vi.fn(),
|
||||
pdf: vi.fn(),
|
||||
pdfOptions: undefined as { onWarning?: (warning: Record<string, unknown>) => void; extractCover?: boolean } | undefined,
|
||||
epubFixedLayout: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('./lib/epub', () => ({
|
||||
extractEpubFixedLayout: mocks.epubFixedLayout,
|
||||
}));
|
||||
|
||||
vi.mock('./lib/cover', () => ({
|
||||
@@ -188,3 +193,35 @@ describe('MetadataExtractionService', () => {
|
||||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('sizeBytes=10000000 thresholdBytes=5000000'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('MetadataExtractionService.detectFixedLayout', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.epubFixedLayout.mockResolvedValue(null);
|
||||
});
|
||||
|
||||
it('reads the declaration for an epub', async () => {
|
||||
mocks.epubFixedLayout.mockResolvedValue(true);
|
||||
|
||||
await expect(new MetadataExtractionService().detectFixedLayout('/books/manga.epub', 'epub')).resolves.toBe(true);
|
||||
expect(mocks.epubFixedLayout).toHaveBeenCalledWith('/books/manga.epub');
|
||||
});
|
||||
|
||||
it('reads the declaration for a kepub, which carries the same OPF', async () => {
|
||||
mocks.epubFixedLayout.mockResolvedValue(true);
|
||||
|
||||
await expect(new MetadataExtractionService().detectFixedLayout('/books/manga.kepub', 'kepub')).resolves.toBe(true);
|
||||
});
|
||||
|
||||
it('passes a reflowable answer through unchanged', async () => {
|
||||
mocks.epubFixedLayout.mockResolvedValue(false);
|
||||
|
||||
await expect(new MetadataExtractionService().detectFixedLayout('/books/novel.epub', 'epub')).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('returns null without touching the filesystem for a format that cannot declare a layout', async () => {
|
||||
await expect(new MetadataExtractionService().detectFixedLayout('/books/comic.cbz', 'cbz')).resolves.toBeNull();
|
||||
await expect(new MetadataExtractionService().detectFixedLayout('/books/scan.pdf', 'pdf')).resolves.toBeNull();
|
||||
expect(mocks.epubFixedLayout).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -10,6 +10,7 @@ import { MobiFormatExtractor } from './extractors/mobi-format.extractor';
|
||||
import { OpfFormatExtractor } from './extractors/opf-format.extractor';
|
||||
import { PdfFormatExtractor } from './extractors/pdf-format.extractor';
|
||||
import { extractCover } from './lib/cover';
|
||||
import { extractEpubFixedLayout } from './lib/epub';
|
||||
import type { PdfParseWarning } from './lib/pdf-parser';
|
||||
|
||||
export const METADATA_AUDIO_FORMATS = ['m4b', 'mp3', 'm4a', 'opus', 'ogg', 'flac'] as const;
|
||||
@@ -56,6 +57,16 @@ export class MetadataExtractionService {
|
||||
return (await this.extractors.get(format)?.extract(absolutePath)) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads only the fixed-layout declaration from an EPUB, without the full metadata parse.
|
||||
* Returns null when the format cannot carry the declaration or the file cannot be read, so
|
||||
* callers can retry rather than caching a read failure as a definitive answer.
|
||||
*/
|
||||
async detectFixedLayout(absolutePath: string, format: string): Promise<boolean | null> {
|
||||
if (format !== 'epub' && format !== 'kepub') return null;
|
||||
return extractEpubFixedLayout(absolutePath);
|
||||
}
|
||||
|
||||
async extractWithCoverFallback(absolutePath: string, format: string): Promise<MetadataExtractionResult> {
|
||||
const metadata = await this.extract(absolutePath, format);
|
||||
if (metadata) return { metadata, cover: metadata.cover };
|
||||
|
||||
@@ -17,7 +17,8 @@ vi.mock('./lib/cbz-metadata', () => ({
|
||||
extractCb7Metadata: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('./lib/epub', () => ({
|
||||
vi.mock('./lib/epub', async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import('./lib/epub')>()),
|
||||
extractEpubMetadata: vi.fn(),
|
||||
}));
|
||||
|
||||
@@ -1193,6 +1194,91 @@ describe('MetadataService', () => {
|
||||
await expect(service.extractAudioChaptersAndNarrators(72, '/tmp/book.pdf', 'pdf')).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
describe('fixed-layout persistence', () => {
|
||||
function epubOpf(renditionLayout: string | null) {
|
||||
return {
|
||||
title: 'Manga Vol. 1',
|
||||
subtitle: null,
|
||||
description: null,
|
||||
isbn10: null,
|
||||
isbn13: null,
|
||||
publisher: null,
|
||||
publishedDate: null,
|
||||
publishedYear: null,
|
||||
language: null,
|
||||
pageCount: null,
|
||||
rating: null,
|
||||
seriesName: null,
|
||||
seriesIndex: null,
|
||||
authors: [],
|
||||
genres: [],
|
||||
tags: [],
|
||||
customMetadata: {},
|
||||
coverHref: null,
|
||||
renditionLayout,
|
||||
};
|
||||
}
|
||||
|
||||
it('records a comic as fixed layout so Kobo sync can announce it as EPUB3FL', async () => {
|
||||
const { db, updateSet } = makeDb();
|
||||
const service = makeService(db);
|
||||
mockExtractEpubMetadata.mockResolvedValueOnce(epubOpf('pre-paginated') as never);
|
||||
|
||||
await service.extractAndSave(42, '/books/manga.epub', 'epub');
|
||||
|
||||
expect(updateSet).toHaveBeenCalledWith({ isFixedLayout: true });
|
||||
});
|
||||
|
||||
it('records a reflowable novel explicitly, so it is never re-read on every sync', async () => {
|
||||
const { db, updateSet } = makeDb();
|
||||
const service = makeService(db);
|
||||
mockExtractEpubMetadata.mockResolvedValueOnce(epubOpf('reflowable') as never);
|
||||
|
||||
await service.extractAndSave(42, '/books/novel.epub', 'epub');
|
||||
|
||||
expect(updateSet).toHaveBeenCalledWith({ isFixedLayout: false });
|
||||
});
|
||||
|
||||
it('records an EPUB that declares no layout as reflowable', async () => {
|
||||
const { db, updateSet } = makeDb();
|
||||
const service = makeService(db);
|
||||
mockExtractEpubMetadata.mockResolvedValueOnce(epubOpf(null) as never);
|
||||
|
||||
await service.extractAndSave(42, '/books/plain.epub', 'epub');
|
||||
|
||||
expect(updateSet).toHaveBeenCalledWith({ isFixedLayout: false });
|
||||
});
|
||||
|
||||
it('writes nothing for a format whose extractor cannot report a layout', async () => {
|
||||
const { db, updateSet } = makeDb();
|
||||
const service = makeService(db);
|
||||
vi.spyOn(service, 'replaceAuthors').mockResolvedValue(undefined);
|
||||
vi.spyOn(service, 'replaceGenres').mockResolvedValue(undefined);
|
||||
vi.spyOn(service, 'replaceTags').mockResolvedValue(undefined);
|
||||
mockParsePdfFile.mockResolvedValueOnce({
|
||||
title: 'Scan',
|
||||
subtitle: null,
|
||||
description: null,
|
||||
isbn10: null,
|
||||
isbn13: null,
|
||||
publisher: null,
|
||||
publishedYear: null,
|
||||
language: null,
|
||||
seriesName: null,
|
||||
seriesIndex: null,
|
||||
authors: [],
|
||||
genres: [],
|
||||
tags: [],
|
||||
rating: null,
|
||||
pageCount: 10,
|
||||
} as never);
|
||||
|
||||
await service.extractAndSave(42, '/books/scan.pdf', 'pdf');
|
||||
|
||||
expect(updateSet).not.toHaveBeenCalledWith(expect.objectContaining({ isFixedLayout: expect.anything() }));
|
||||
});
|
||||
});
|
||||
|
||||
it('extractAudioFileDuration writes duration only when parser returns a value', async () => {
|
||||
const { db, updateSet } = makeDb();
|
||||
const service = makeService(db);
|
||||
|
||||
@@ -111,7 +111,11 @@ export class MetadataService {
|
||||
return false;
|
||||
}
|
||||
|
||||
await Promise.all([this.persistMetadata(bookId, data, format), data.cover ? this.persistCover(bookId, data.cover, true) : Promise.resolve()]);
|
||||
await Promise.all([
|
||||
this.persistMetadata(bookId, data, format),
|
||||
data.cover ? this.persistCover(bookId, data.cover, true) : Promise.resolve(),
|
||||
this.persistFixedLayout(bookId, absolutePath, data.isFixedLayout),
|
||||
]);
|
||||
|
||||
await this.scoreService.calculateAndSave(bookId);
|
||||
|
||||
@@ -255,6 +259,27 @@ export class MetadataService {
|
||||
|
||||
// ── Audio helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Records whether the extracted file is a fixed-layout EPUB. Kobo sync reads this to announce
|
||||
* comics as EPUB3FL so the device renders them full screen instead of adding reflow margins.
|
||||
*/
|
||||
private async persistFixedLayout(bookId: number, absolutePath: string, isFixedLayout: boolean | null | undefined): Promise<void> {
|
||||
if (isFixedLayout == null) return;
|
||||
// `is distinct from` keeps a re-extraction of an unchanged file from touching the row at all.
|
||||
// Any write bumps book_files.updatedAt, which feeds the KOReader library token and OPDS entry
|
||||
// timestamps, so a no-op update would invalidate both for nothing.
|
||||
await this.db
|
||||
.update(schema.bookFiles)
|
||||
.set({ isFixedLayout })
|
||||
.where(
|
||||
and(
|
||||
eq(schema.bookFiles.bookId, bookId),
|
||||
eq(schema.bookFiles.absolutePath, absolutePath),
|
||||
sql`${schema.bookFiles.isFixedLayout} is distinct from ${isFixedLayout}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async extractAudioFileDuration(bookId: number, absolutePath: string): Promise<void> {
|
||||
const durationSeconds = await parseAudioDuration(absolutePath);
|
||||
if (durationSeconds === null) return;
|
||||
|
||||
Reference in New Issue
Block a user