diff --git a/server/src/modules/kobo/services/kepub-conversion.service.test.ts b/server/src/modules/kobo/services/kepub-conversion.service.test.ts index be933ce30..b8ecba968 100644 --- a/server/src/modules/kobo/services/kepub-conversion.service.test.ts +++ b/server/src/modules/kobo/services/kepub-conversion.service.test.ts @@ -1,5 +1,6 @@ vi.mock('fs/promises', () => ({ mkdir: vi.fn(), mkdtemp: vi.fn(), rename: vi.fn(), rm: vi.fn(), stat: vi.fn() })); vi.mock('child_process', () => ({ execFile: vi.fn() })); +vi.mock('./kepub-href-decoding', () => ({ findDecodableHrefs: vi.fn(), writeWithDecodedHrefs: vi.fn(), restoreEncodedHrefs: vi.fn() })); import { Test } from '@nestjs/testing'; import { storageConfig } from '../../../config/config'; @@ -7,6 +8,7 @@ import { execFile } from 'child_process'; import { mkdir, mkdtemp, rename, rm, stat } from 'fs/promises'; import { KepubConversionService } from './kepub-conversion.service'; +import { findDecodableHrefs, restoreEncodedHrefs, writeWithDecodedHrefs } from './kepub-href-decoding'; import { KepubifyBinaryService } from './kepubify-binary.service'; const execMock = vi.mocked(execFile); @@ -36,6 +38,9 @@ describe('KepubConversionService', () => { vi.mocked(rename).mockResolvedValue(undefined); vi.mocked(rm).mockResolvedValue(undefined); successfulConversion(); + vi.mocked(findDecodableHrefs).mockResolvedValue(new Map()); + vi.mocked(writeWithDecodedHrefs).mockResolvedValue(undefined); + vi.mocked(restoreEncodedHrefs).mockResolvedValue(1); const module = await Test.createTestingModule({ providers: [ KepubConversionService, @@ -137,4 +142,42 @@ describe('KepubConversionService', () => { expect(rename).not.toHaveBeenCalled(); expect(rm).not.toHaveBeenCalled(); }); + + describe('books listing chapters by percent-encoded hrefs (#1136)', () => { + const decoded = new Map([['text/chapter one.xhtml', 'text/chapter%20one.xhtml']]); + + it('converts them through a decoded copy, restores their hrefs, and caches them under their own key', async () => { + vi.mocked(findDecodableHrefs).mockResolvedValue(decoded); + + await expect(service.getKepubPath(input)).resolves.toBe(`${cacheDir}/abc-hrefs.kepub.epub`); + + expect(writeWithDecodedHrefs).toHaveBeenCalledWith(input.sourcePath, `${tempDir}/source.epub`, decoded); + expect(execMock).toHaveBeenCalledWith( + '/tools/kepubify', + ['--output', `${tempDir}/converted.kepub.epub`, `${tempDir}/source.epub`], + { timeout: 60_000 }, + expect.any(Function), + ); + expect(restoreEncodedHrefs).toHaveBeenCalledWith(`${tempDir}/converted.kepub.epub`, tempPath, decoded); + expect(rename).toHaveBeenCalledExactlyOnceWith(tempPath, `${cacheDir}/abc-hrefs.kepub.epub`); + }); + + it('inspects a source once, not on every lookup', async () => { + statMock.mockResolvedValue({} as never); + + await service.getKepubPath(input); + await service.getKepubPath(input); + + expect(findDecodableHrefs).toHaveBeenCalledTimes(1); + }); + + it('converts a source it cannot inspect as is, under the key it always had', async () => { + vi.mocked(findDecodableHrefs).mockRejectedValue(new Error('not a zip')); + + await expect(service.getKepubPath(input)).resolves.toBe(`${cacheDir}/abc.kepub.epub`); + + expect(writeWithDecodedHrefs).not.toHaveBeenCalled(); + expect(execMock).toHaveBeenCalledWith('/tools/kepubify', ['--output', tempPath, input.sourcePath], { timeout: 60_000 }, expect.any(Function)); + }); + }); }); diff --git a/server/src/modules/kobo/services/kepub-conversion.service.ts b/server/src/modules/kobo/services/kepub-conversion.service.ts index efdb6c51d..e49f542e8 100644 --- a/server/src/modules/kobo/services/kepub-conversion.service.ts +++ b/server/src/modules/kobo/services/kepub-conversion.service.ts @@ -4,16 +4,18 @@ import { mkdir, mkdtemp, rename, rm, stat, opendir, lstat, realpath } from 'fs/p import { join } from 'path'; import { promisify } from 'util'; -import { Inject, Injectable, BadRequestException } from '@nestjs/common'; +import { Inject, Injectable, BadRequestException, Logger } from '@nestjs/common'; import type { ConfigType } from '@nestjs/config'; import { storageConfig } from '../../../config/config'; import { KepubifyBinaryService } from './kepubify-binary.service'; +import { findDecodableHrefs, restoreEncodedHrefs, writeWithDecodedHrefs, type DecodedHrefs } from './kepub-href-decoding'; const execFileAsync = promisify(execFile); const KEPUBIFY_TIMEOUT_MS = 60_000; const AUDIOLESS_EPUB_CACHE_VERSION = 1; +const HREF_DECODING_MEMO_LIMIT = 256; interface KepubConversionInput { sourcePath: string; @@ -30,6 +32,7 @@ interface KepubConversionInput { @Injectable() export class KepubConversionService { + private readonly logger = new Logger(KepubConversionService.name); private readonly kepubCachePath: string; private readonly appDataPath: string; private active = 0; @@ -37,6 +40,7 @@ export class KepubConversionService { private readonly leases = new Map(); private publication: Promise = Promise.resolve(); private readonly conversions = new Map>(); + private readonly hrefDecodings = new Map>(); constructor( @Inject(storageConfig.KEY) storage: ConfigType, @@ -67,10 +71,13 @@ export class KepubConversionService { const cacheDir = join(this.kepubCachePath, String(input.bookId)); const fileHash = input.fileHash ?? 'nohash'; // Keep the original two key shapes byte for byte so existing cache entries still hit. - const legacyKey = `${fileHash}${input.audioless ? `-noaudio-v${AUDIOLESS_EPUB_CACHE_VERSION}` : ''}${input.hyphenate ? '-hyph' : ''}`; + // A book kepubify reads as is keeps its key byte for byte, so only the books it used to leave + // unconverted are converted again. + const decodedHrefs = await this.decodableHrefs(input); + const legacyKey = `${fileHash}${input.audioless ? `-noaudio-v${AUDIOLESS_EPUB_CACHE_VERSION}` : ''}${input.hyphenate ? '-hyph' : ''}${decodedHrefs.size ? '-hrefs' : ''}`; const cacheKey = input.sourceRevision ? createHash('sha256') - .update(`${input.sourceRevision}:${await this.kepubifyBinaryService.getVersion()}:${input.hyphenate}`) + .update(`${input.sourceRevision}:${await this.kepubifyBinaryService.getVersion()}:${input.hyphenate}${decodedHrefs.size ? ':hrefs' : ''}`) .digest('hex') : legacyKey; const cachedPath = join(cacheDir, `${cacheKey}.kepub.epub`); @@ -88,7 +95,7 @@ export class KepubConversionService { const concurrent = this.conversions.get(cachedPath); if (concurrent) return concurrent; - const conversion = this.convertToCache(input, cacheDir, cachedPath); + const conversion = this.convertToCache(input, cacheDir, cachedPath, decodedHrefs); this.conversions.set(cachedPath, conversion); try { return await conversion; @@ -97,7 +104,7 @@ export class KepubConversionService { } } - private async convertToCache(input: KepubConversionInput, cacheDir: string, cachedPath: string): Promise { + private async convertToCache(input: KepubConversionInput, cacheDir: string, cachedPath: string, decodedHrefs: DecodedHrefs): Promise { if (this.active >= 2) { if (this.waiters.length >= 32) throw new BadRequestException({ code: 'DELIVERY_BUSY' }); await new Promise((resolve, reject) => { @@ -124,8 +131,18 @@ export class KepubConversionService { const tempDir = await mkdtemp(join(cacheDir, '.conversion-')); const outputPath = join(tempDir, 'book.kepub.epub'); try { - const args = input.hyphenate ? ['--hyphenate', '--output', outputPath, input.sourcePath] : ['--output', outputPath, input.sourcePath]; - await execFileAsync(binaryPath, args, { timeout: KEPUBIFY_TIMEOUT_MS }); + if (decodedHrefs.size) { + const sourcePath = join(tempDir, 'source.epub'); + const convertedPath = join(tempDir, 'converted.kepub.epub'); + await writeWithDecodedHrefs(input.sourcePath, sourcePath, decodedHrefs); + await this.runKepubify(binaryPath, sourcePath, convertedPath, input.hyphenate); + const restored = await restoreEncodedHrefs(convertedPath, outputPath, decodedHrefs); + this.logger.log( + `[kobo.kepub_href_decoding] [end] bookId=${input.bookId} decoded=${decodedHrefs.size} restored=${restored} - converted chapters kepubify could not find by their encoded href`, + ); + } else { + await this.runKepubify(binaryPath, input.sourcePath, outputPath, input.hyphenate); + } // Readers and the cache recovery pass must never see a partially written conversion. const publication = this.publication.then(async () => { if (input.sourceRevision) await this.evict(cachedPath, (await stat(outputPath)).size); @@ -143,6 +160,22 @@ export class KepubConversionService { else this.active--; } } + private async runKepubify(binaryPath: string, sourcePath: string, outputPath: string, hyphenate: boolean): Promise { + const args = hyphenate ? ['--hyphenate', '--output', outputPath, sourcePath] : ['--output', outputPath, sourcePath]; + await execFileAsync(binaryPath, args, { timeout: KEPUBIFY_TIMEOUT_MS }); + } + + private decodableHrefs(input: KepubConversionInput): Promise { + const key = `${input.sourcePath}\0${input.sourceRevision ?? input.fileHash ?? ''}`; + let pending = this.hrefDecodings.get(key); + if (!pending) { + pending = findDecodableHrefs(input.sourcePath).catch(() => new Map()); + if (this.hrefDecodings.size >= HREF_DECODING_MEMO_LIMIT) this.hrefDecodings.delete(this.hrefDecodings.keys().next().value!); + this.hrefDecodings.set(key, pending); + } + return pending; + } + private async evict(publishingPath: string, requiredBytes: number): Promise { let total = requiredBytes; let count = 1; @@ -157,7 +190,7 @@ export class KepubConversionService { if (Date.now() - info.mtimeMs > 20 * 60_000) await rm(path, { recursive: true, force: true }); continue; } - if (!entry.isFile() || !/^[a-f0-9]{32,64}(?:-noaudio-v\d+)?(?:-hyph)?\.kepub\.epub$/.test(entry.name)) continue; + if (!entry.isFile() || !/^[a-f0-9]{32,64}(?:-noaudio-v\d+)?(?:-hyph)?(?:-hrefs)?\.kepub\.epub$/.test(entry.name)) continue; const info = await stat(path); total += info.size; count++; diff --git a/server/src/modules/kobo/services/kepub-href-decoding.test.ts b/server/src/modules/kobo/services/kepub-href-decoding.test.ts new file mode 100644 index 000000000..4abf162f2 --- /dev/null +++ b/server/src/modules/kobo/services/kepub-href-decoding.test.ts @@ -0,0 +1,150 @@ +import { execFile } from 'child_process'; +import { mkdtemp, rm, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { promisify } from 'util'; +import { ZipArchive } from 'archiver'; +import * as unzipper from 'unzipper'; + +import { findDecodableHrefs, restoreEncodedHrefs, writeWithDecodedHrefs } from './kepub-href-decoding'; +import { KepubifyBinaryService } from './kepubify-binary.service'; + +const execFileAsync = promisify(execFile); + +const CONTAINER_XML = ` + + +`; + +// Hrefs are URLs, so files stored with spaces, `&` or `#` in their names are listed percent-encoded. +const CONTENT_OPF = ` + + Encoded hrefsxen + + + + + + + + + +`; + +function chapter(text: string): string { + return ` +t +

${text} The first sentence of the chapter.

A second paragraph follows it.

`; +} + +const ENTRIES: [string, string][] = [ + ['OEBPS/text/chapter one.xhtml', chapter('Spaces.')], + ['OEBPS/text/notes & more.xhtml', chapter('Ampersand.')], + ['OEBPS/text/part#1.xhtml', chapter('Hash.')], + // Stored with the escapes in its name, so the href already resolves as written. + ['OEBPS/text/stored%20as%20written.xhtml', chapter('Literal.')], + ['OEBPS/text/plain.xhtml', chapter('Plain.')], +]; + +async function buildEpub(path: string): Promise { + const archive = new ZipArchive({ zlib: { level: 6 } }); + const chunks: Buffer[] = []; + archive.on('data', (chunk: Buffer) => chunks.push(chunk)); + const done = new Promise((resolve, reject) => { + archive.on('end', resolve); + archive.on('error', reject); + }); + archive.append('application/epub+zip', { name: 'mimetype', store: true }); + archive.append(CONTAINER_XML, { name: 'META-INF/container.xml' }); + archive.append(CONTENT_OPF, { name: 'OEBPS/content.opf' }); + for (const [name, body] of ENTRIES) archive.append(body, { name }); + await archive.finalize(); + await done; + await writeFile(path, Buffer.concat(chunks)); +} + +async function readEntries(path: string): Promise> { + const zip = await unzipper.Open.file(path); + const entries = new Map(); + for (const file of zip.files) if (file.type === 'File') entries.set(file.path, (await file.buffer()).toString('utf8')); + return entries; +} + +function hrefAttributes(opf: string): string[] { + return [...opf.matchAll(/\bhref="([^"]*)"/g)].map((match) => match[1]!); +} + +function hasSpans(xhtml: string | undefined): boolean { + return (xhtml ?? '').includes('koboSpan'); +} + +describe('kepub href decoding (real kepubify)', () => { + let dir: string; + let epubPath: string; + let kepubify: string; + + beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'bookorbit-kepub-hrefs-')); + epubPath = join(dir, 'book.epub'); + await buildEpub(epubPath); + kepubify = await new KepubifyBinaryService().getBinaryPath(); + }, 60_000); + + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('finds the hrefs that resolve only once decoded, keyed by the stored name', async () => { + const decoded = await findDecodableHrefs(epubPath); + + expect(Object.fromEntries(decoded)).toEqual({ + 'text/chapter one.xhtml': 'text/chapter%20one.xhtml', + 'text/chapter one.xhtml#start': 'text/chapter%20one.xhtml#start', + 'text/notes & more.xhtml': 'text/notes%20%26%20more.xhtml', + 'text/part#1.xhtml': 'text/part%231.xhtml', + }); + }); + + it('documents why: kepubify leaves a chapter listed by its encoded href unconverted', async () => { + const out = join(dir, 'as-is.kepub.epub'); + await execFileAsync(kepubify, ['--output', out, epubPath], { timeout: 60_000 }); + const entries = await readEntries(out); + + expect(hasSpans(entries.get('OEBPS/text/chapter one.xhtml'))).toBe(false); + expect(hasSpans(entries.get('OEBPS/text/plain.xhtml'))).toBe(true); + }, 60_000); + + it('converts every chapter and hands back the package with the book’s own hrefs', async () => { + const decoded = await findDecodableHrefs(epubPath); + const source = join(dir, 'decoded.epub'); + const converted = join(dir, 'converted.kepub.epub'); + const restoredPath = join(dir, 'restored.kepub.epub'); + + await writeWithDecodedHrefs(epubPath, source, decoded); + const decodedOpf = (await readEntries(source)).get('OEBPS/content.opf')!; + expect(decodedOpf).toContain('href="text/notes & more.xhtml"'); + expect(decodedOpf).toContain('href="text/stored%20as%20written.xhtml"'); + + await execFileAsync(kepubify, ['--output', converted, source], { timeout: 60_000 }); + const restored = await restoreEncodedHrefs(converted, restoredPath, decoded); + const entries = await readEntries(restoredPath); + + expect(restored).toBe(4); + for (const name of ['OEBPS/text/chapter one.xhtml', 'OEBPS/text/notes & more.xhtml', 'OEBPS/text/part#1.xhtml', 'OEBPS/text/plain.xhtml']) { + expect(hasSpans(entries.get(name)), name).toBe(true); + } + // kepubify adds a title page of its own once it can read the first chapter; every other href is the book's. + const hrefs = hrefAttributes(entries.get('OEBPS/content.opf')!); + expect(hrefs.filter((href) => !href.startsWith('kepubify-'))).toEqual(hrefAttributes(CONTENT_OPF)); + const first = (await unzipper.Open.file(restoredPath)).files[0]!; + expect(first.path).toBe('mimetype'); + expect(first.compressionMethod).toBe(0); + }, 60_000); + + it('rejects input that is not an archive, so the caller falls back to converting it as is', async () => { + const notAnEpub = join(dir, 'not-an.epub'); + await writeFile(notAnEpub, 'plain text'); + + await expect(findDecodableHrefs(notAnEpub)).rejects.toThrow(); + }); +}); diff --git a/server/src/modules/kobo/services/kepub-href-decoding.ts b/server/src/modules/kobo/services/kepub-href-decoding.ts new file mode 100644 index 000000000..c2ceb377c --- /dev/null +++ b/server/src/modules/kobo/services/kepub-href-decoding.ts @@ -0,0 +1,159 @@ +import { createWriteStream } from 'fs'; +import { posix } from 'path'; +import { ZipArchive } from 'archiver'; +import * as unzipper from 'unzipper'; + +// kepubify looks a manifest href up in the archive exactly as written, so a chapter listed as +// `Text/My%20Chapter.xhtml` but stored as `Text/My Chapter.xhtml` is shipped unconverted. With no +// KoboSpans in it a Kobo can only bookmark the chapter's start, and every synced position snaps +// back there (#1136). The fix converts a copy whose OPF names those files as stored, then puts the +// book's own spelling back so the Kobo still reports the paths it always did. + +const HREF_ATTRIBUTE = /\bhref=(["'])([^"']*)\1/g; +const PERCENT_ESCAPE = /%[0-9A-Fa-f]{2}/; +const MAX_PACKAGE_BYTES = 4 * 1024 * 1024; +const MAX_ENTRIES = 20_000; + +/** The book's own href attribute text, keyed by the href as stored in the archive. */ +export type DecodedHrefs = ReadonlyMap; + +interface PackageDocument { + files: unzipper.File[]; + opfPath: string; + opf: string; +} + +/** The hrefs that resolve only once decoded, or an empty map when kepubify can read the book as is. */ +export async function findDecodableHrefs(epubPath: string): Promise> { + const pkg = await readPackageDocument(epubPath); + const decoded = new Map(); + if (!pkg) return decoded; + const names = new Set(pkg.files.map((file) => file.path)); + const opfDir = posix.dirname(pkg.opfPath); + for (const [, , raw] of pkg.opf.matchAll(HREF_ATTRIBUTE)) { + if (!PERCENT_ESCAPE.test(raw)) continue; + const href = unescapeXml(raw); + const plain = decodeHref(href); + if (!plain || plain === href) continue; + if (names.has(resolveHref(opfDir, hrefPath(href))) || !names.has(resolveHref(opfDir, decodeHref(hrefPath(href)) ?? ''))) continue; + decoded.set(plain, raw); + } + return decoded; +} + +/** Copies the book with the given hrefs decoded in its OPF, for kepubify to convert. */ +export async function writeWithDecodedHrefs(epubPath: string, outputPath: string, decoded: DecodedHrefs): Promise { + await rewriteOpf(epubPath, outputPath, (opf) => + opf.replace(HREF_ATTRIBUTE, (match, quote: string, raw: string) => { + const plain = decodeHref(unescapeXml(raw)); + return plain !== null && decoded.get(plain) === raw ? `href=${quote}${escapeXml(plain)}${quote}` : match; + }), + ); +} + +/** Puts the book's own href spelling back into kepubify's output; returns how many it restored. */ +export async function restoreEncodedHrefs(kepubPath: string, outputPath: string, decoded: DecodedHrefs): Promise { + let restored = 0; + await rewriteOpf(kepubPath, outputPath, (opf) => + // kepubify re-serializes the package, so match on the attribute's value rather than its text. + opf.replace(HREF_ATTRIBUTE, (match, quote: string, raw: string) => { + const original = decoded.get(unescapeXml(raw)); + if (original === undefined) return match; + restored++; + return `href=${quote}${original}${quote}`; + }), + ); + return restored; +} + +async function rewriteOpf(epubPath: string, outputPath: string, rewrite: (opf: string) => string): Promise { + const pkg = await readPackageDocument(epubPath); + if (!pkg) throw new Error('EPUB package document not found'); + const archive = new ZipArchive({ zlib: { level: 6 } }); + const output = createWriteStream(outputPath); + let fail!: (error: Error) => void; + const failure = new Promise((_resolve, reject) => { + fail = reject; + }); + failure.catch(() => {}); + archive.on('error', fail); + output.on('error', fail); + const closed = new Promise((resolve) => output.once('close', resolve)); + archive.pipe(output); + + // One entry at a time: appending them all up front opens a read stream per entry at once. + const append = (source: Buffer | NodeJS.ReadableStream, options: { name: string; store?: boolean }) => + Promise.race([ + new Promise((resolve) => { + archive.once('entry', () => resolve()); + if (!Buffer.isBuffer(source)) source.once('error', fail); + archive.append(source as Buffer, options); + }), + failure, + ]); + + try { + const mimetype = pkg.files.find((file) => file.path === 'mimetype'); + if (mimetype) await append(await readBounded(mimetype), { name: 'mimetype', store: true }); + for (const file of pkg.files) { + if (file.type !== 'File' || file.path === 'mimetype') continue; + await append(file.path === pkg.opfPath ? Buffer.from(rewrite(pkg.opf), 'utf8') : file.stream(), { name: file.path }); + } + void archive.finalize(); + await Promise.race([closed, failure]); + } catch (error) { + output.destroy(); + throw error; + } +} + +async function readPackageDocument(epubPath: string): Promise { + const zip = await unzipper.Open.file(epubPath); + if (zip.files.length > MAX_ENTRIES) return null; + const container = zip.files.find((file) => file.path === 'META-INF/container.xml'); + if (!container) return null; + const opfPath = /full-path=(["'])([^"']+)\1/.exec((await readBounded(container)).toString('utf8'))?.[2]; + const opfEntry = opfPath ? zip.files.find((file) => file.path === opfPath) : undefined; + if (!opfPath || !opfEntry) return null; + return { files: zip.files, opfPath, opf: (await readBounded(opfEntry)).toString('utf8') }; +} + +async function readBounded(file: unzipper.File): Promise { + if (file.uncompressedSize > MAX_PACKAGE_BYTES) throw new Error(`EPUB entry ${file.path} is too large to read`); + return file.buffer(); +} + +/** Decodes the path of an href; a fragment is left as written. */ +function decodeHref(href: string): string | null { + const path = hrefPath(href); + try { + return decodeURIComponent(path) + href.slice(path.length); + } catch { + return null; + } +} + +const XML_ESCAPES: Record = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' }; +const XML_ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }; + +function escapeXml(value: string): string { + return value.replace(/[&<>"']/g, (char) => XML_ESCAPES[char] ?? char); +} + +function unescapeXml(value: string): string { + return value.replace(/&(#x[0-9A-Fa-f]+|#\d+|amp|lt|gt|quot|apos);/g, (match, entity: string) => { + if (!entity.startsWith('#')) return XML_ENTITIES[entity] ?? match; + const codePoint = entity.startsWith('#x') ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10); + return codePoint <= 0x10ffff ? String.fromCodePoint(codePoint) : match; + }); +} + +/** The href without its fragment, cut at the first literal `#` so an encoded `%23` stays in the path. */ +function hrefPath(href: string): string { + const hash = href.indexOf('#'); + return hash === -1 ? href : href.slice(0, hash); +} + +function resolveHref(opfDir: string, path: string): string { + return posix.normalize(posix.join(opfDir, path)); +} diff --git a/server/src/modules/kobo/services/kobo-download-hash-backfill.service.ts b/server/src/modules/kobo/services/kobo-download-hash-backfill.service.ts index 5641829e4..3f0b78c94 100644 --- a/server/src/modules/kobo/services/kobo-download-hash-backfill.service.ts +++ b/server/src/modules/kobo/services/kobo-download-hash-backfill.service.ts @@ -12,7 +12,7 @@ import { KoboDownloadRepository } from '../kobo-download.repository'; const SETTING_KEY = 'kobo_download_hash_backfill'; const VERSION = 1; const EVENT = 'kobo.download_hash_backfill'; -const CACHE_FILE = /^([0-9a-f]{32})(?:-noaudio-v[1-9]\d*)?(?:-hyph)?\.kepub\.epub$/; +const CACHE_FILE = /^([0-9a-f]{32})(?:-noaudio-v[1-9]\d*)?(?:-hyph)?(?:-hrefs)?\.kepub\.epub$/; @Injectable() export class KoboDownloadHashBackfillService implements OnApplicationBootstrap { diff --git a/server/src/modules/position-converter/epub-dom.service.ts b/server/src/modules/position-converter/epub-dom.service.ts index 145fc0f25..f8e76bea6 100644 --- a/server/src/modules/position-converter/epub-dom.service.ts +++ b/server/src/modules/position-converter/epub-dom.service.ts @@ -51,7 +51,18 @@ function resolveHref(href: string, basePath: string): string { } function findInZip(files: unzipper.File[], path: string): unzipper.File | undefined { - const clean = normalizeZipPath(path); + const entry = matchZipEntry(files, normalizeZipPath(path)); + if (entry || !path.includes('%')) return entry; + // Manifest hrefs are URLs, so a chapter stored as `My Chapter.xhtml` is listed as `My%20Chapter.xhtml`. + try { + const decoded = decodeURIComponent(path); + return decoded === path ? undefined : matchZipEntry(files, normalizeZipPath(decoded)); + } catch { + return undefined; + } +} + +function matchZipEntry(files: unzipper.File[], clean: string): unzipper.File | undefined { const cleanLower = clean.toLowerCase(); let ciMatch: unzipper.File | undefined; for (const file of files) { diff --git a/server/src/modules/position-converter/kepub-dom.service.test.ts b/server/src/modules/position-converter/kepub-dom.service.test.ts index e74f0e917..2592e3efd 100644 --- a/server/src/modules/position-converter/kepub-dom.service.test.ts +++ b/server/src/modules/position-converter/kepub-dom.service.test.ts @@ -29,7 +29,7 @@ const CH2 = ` 2

Second chapter text.

`; -async function buildKepub(path: string): Promise { +async function buildKepub(path: string, opf = CONTENT_OPF, chapterNames = ['OEBPS/text/ch1.xhtml', 'OEBPS/text/ch2.xhtml']): Promise { const archive = new ZipArchive({ zlib: { level: 0 } }); const chunks: Buffer[] = []; archive.on('data', (chunk: Buffer) => chunks.push(chunk)); @@ -39,9 +39,9 @@ async function buildKepub(path: string): Promise { }); archive.append('application/epub+zip', { name: 'mimetype', store: true }); archive.append(CONTAINER_XML, { name: 'META-INF/container.xml' }); - archive.append(CONTENT_OPF, { name: 'OEBPS/content.opf' }); - archive.append(CH1, { name: 'OEBPS/text/ch1.xhtml' }); - archive.append(CH2, { name: 'OEBPS/text/ch2.xhtml' }); + archive.append(opf, { name: 'OEBPS/content.opf' }); + archive.append(CH1, { name: chapterNames[0] }); + archive.append(CH2, { name: chapterNames[1] }); await archive.finalize(); await done; await writeFile(path, Buffer.concat(chunks)); @@ -89,3 +89,37 @@ describe('KepubDomService', () => { expect(await service.getSpine(join(dir, 'nope.kepub.epub'))).toBeNull(); }); }); + +// A chapter stored as `chapter one.xhtml` is listed, and reported by the Kobo, as `chapter%20one.xhtml` (#1136). +describe('KepubDomService with percent-encoded manifest hrefs', () => { + let dir: string; + let kepubPath: string; + + beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'bookorbit-kepub-encoded-')); + kepubPath = join(dir, 'encoded.kepub.epub'); + await buildKepub( + kepubPath, + CONTENT_OPF.replace('text/ch1.xhtml', 'text/chapter%20one.xhtml').replace('text/ch2.xhtml', 'text/notes%20%26%20more.xhtml'), + ['OEBPS/text/chapter one.xhtml', 'OEBPS/text/notes & more.xhtml'], + ); + }); + + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('loads a chapter whose stored name only matches its href once decoded', async () => { + const doc = await new KepubDomService().getChapterByIndex(kepubPath, 1); + + expect(doc?.index.collapsed).toBe('Second chapter text.'); + }); + + it('matches the chapter the Kobo reports, encoded or not', async () => { + const service = new KepubDomService(); + + expect(await service.findChapterIndexByFilename(kepubPath, 'OEBPS/text/chapter%20one.xhtml')).toBe(0); + expect(await service.findChapterIndexByFilename(kepubPath, 'OEBPS/text/chapter one.xhtml')).toBe(0); + expect(await service.findChapterIndexByFilename(kepubPath, 'OEBPS/text/notes%20%26%20more.xhtml')).toBe(1); + }); +}); diff --git a/server/src/modules/position-converter/kepub-dom.service.ts b/server/src/modules/position-converter/kepub-dom.service.ts index 7097b8f64..f9e0bca6e 100644 --- a/server/src/modules/position-converter/kepub-dom.service.ts +++ b/server/src/modules/position-converter/kepub-dom.service.ts @@ -59,7 +59,8 @@ export class KepubDomService { const normalized = normalizeChapterPath(chapterFilename); if (!normalized) return null; const lower = normalized.toLowerCase(); - const hrefs = entry.spine.hrefs; + // The Kobo reports a chapter by its manifest href, encoded or not, so compare both sides decoded. + const hrefs = entry.spine.hrefs.map(normalizeChapterPath); let index = hrefs.findIndex((href) => href === normalized); if (index < 0) index = hrefs.findIndex((href) => href.toLowerCase() === lower);