Migrations/bookorbit
1
0
Fork 0
mirror of https://github.com/bookorbit/bookorbit.git synced 2026-11-03 00:11:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(kobo): preserve koreader matching for converted downloads

Browse Source 
Register the identity of delivered bytes so Kobo conversions and audio
removal continue matching their source books in KOReader. Keep download
registration recoverable without allowing database lock waits to stall it.

Preserve manual corrections across cached device identities, make review
semantics explicit, and keep cache invalidation bounded for large libraries.
Add unit, database, lifecycle and scale regressions with one generated migration.

Closes #1697
This commit is contained in:
neon
2026-10-07 00:07:50 +00:00
parent 019c432262
commit 58756565fc
51 changed files with 25561 additions and 735 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/features/settings/KoreaderSettings.vue
+14 -13
View File
@@ -454,9 +454,9 @@ async function handleRefreshUnmatched() {
try {
await fetchUnmatchedBooks()
unmatchedPage.value = 1
toast.success(t('settings.reader.koreader.hashLinks.toast.unmatchedRefreshed'))
toast.success(t('settings.reader.koreader.hashLinks.toast.reviewRefreshed'))
} catch {
toast.error(t('settings.reader.koreader.hashLinks.toast.unmatchedRefreshFailed'))
toast.error(t('settings.reader.koreader.hashLinks.toast.reviewRefreshFailed'))
}
}
@@ -499,10 +499,10 @@ async function handleDismissUnmatchedBook() {
dismissingHash.value = dismissConfirmBook.value.hash
try {
await dismissUnmatchedBook(dismissConfirmBook.value.hash)
toast.success(t('settings.reader.koreader.hashLinks.toast.unmatchedDismissed'))
toast.success(t('settings.reader.koreader.hashLinks.toast.reviewDismissed'))
handleCloseDismiss()
} catch (e) {
toast.error(e instanceof Error ? e.message : t('settings.reader.koreader.hashLinks.toast.dismissFailed'))
toast.error(e instanceof Error ? e.message : t('settings.reader.koreader.hashLinks.toast.reviewDismissFailed'))
} finally {
dismissingHash.value = null
}
@@ -521,14 +521,14 @@ async function handleDismissAllUnmatchedBooks() {
try {
const result = await dismissAllUnmatchedBooks()
toast.success(
t('settings.reader.koreader.hashLinks.toast.allDismissed', {
t('settings.reader.koreader.hashLinks.toast.reviewAllDismissed', {
count: result.count,
}),
)
dismissAllConfirmOpen.value = false
unmatchedPage.value = 1
} catch (e) {
toast.error(e instanceof Error ? e.message : t('settings.reader.koreader.hashLinks.toast.dismissAllFailed'))
toast.error(e instanceof Error ? e.message : t('settings.reader.koreader.hashLinks.toast.reviewDismissAllFailed'))
} finally {
dismissingAll.value = false
}
@@ -1152,7 +1152,7 @@ async function handleDownloadPlugin() {
<div class="mb-8">
<div class="mb-2 flex items-center justify-between gap-3">
<p class="settings-group-label mb-0">
{{ t('settings.reader.koreader.hashLinks.unmatchedTitle') }}
{{ t('settings.reader.koreader.hashLinks.reviewTitle') }}
</p>
<div class="flex items-center gap-2">
<Button
@@ -1171,12 +1171,13 @@ async function handleDownloadPlugin() {
</Button>
</div>
</div>
<p class="settings-hint mb-3">{{ t('settings.reader.koreader.hashLinks.reviewHint') }}</p>
<div class="settings-card">
<div v-if="unmatchedLoading" class="px-4 py-5 bg-card text-sm text-muted-foreground md:px-5">
{{ t('settings.reader.koreader.hashLinks.loadingUnmatched') }}
{{ t('settings.reader.koreader.hashLinks.loadingReview') }}
</div>
<div v-else-if="unmatchedBooks.length === 0" class="px-4 py-5 bg-card text-sm text-muted-foreground md:px-5">
{{ t('settings.reader.koreader.hashLinks.noUnmatched') }}
{{ t('settings.reader.koreader.hashLinks.noReview') }}
</div>
<template v-else>
<div v-for="book in pagedUnmatchedBooks" :key="book.hash" class="px-4 py-4 bg-card md:px-5">
@@ -1469,7 +1470,7 @@ async function handleDownloadPlugin() {
</p>
<p class="mt-1 text-sm text-muted-foreground">
{{
t('settings.reader.koreader.hashLinks.unlinkConfirmBody', {
t('settings.reader.koreader.hashLinks.manualUnlinkConfirmBody', {
title: linkedBookTitle(unlinkConfirmLink),
})
}}
@@ -1544,7 +1545,7 @@ async function handleDownloadPlugin() {
}}
</p>
<p class="mt-1 text-sm text-muted-foreground">
{{ t('settings.reader.koreader.hashLinks.dismissConfirmBody') }}
{{ t('settings.reader.koreader.hashLinks.reviewDismissConfirmBody') }}
</p>
<div class="mt-4 rounded-md border border-border bg-muted/40 px-3 py-2 text-xs text-muted-foreground">
<p class="font-mono text-foreground truncate">
@@ -1575,7 +1576,7 @@ async function handleDownloadPlugin() {
<div class="relative w-full rounded-t-lg border border-border bg-card p-4 shadow-xl md:max-w-md md:rounded-lg md:p-5">
<p class="text-base font-semibold text-foreground">
{{
t('settings.reader.koreader.hashLinks.dismissAllConfirmTitle', {
t('settings.reader.koreader.hashLinks.reviewDismissAllConfirmTitle', {
count: unmatchedBooks.length,
})
}}
@@ -1583,7 +1584,7 @@ async function handleDownloadPlugin() {
<div class="mt-2 flex items-start gap-2 rounded-md border border-destructive/30 bg-destructive/10 px-3 py-2 text-sm text-destructive">
<AlertTriangle :size="14" class="mt-0.5 shrink-0" />
<p>
{{ t('settings.reader.koreader.hashLinks.dismissAllConfirmBody') }}
{{ t('settings.reader.koreader.hashLinks.reviewDismissAllConfirmBody') }}
</p>
</div>
<div class="mt-4 flex items-center justify-end gap-2">
client/src/features/settings/__tests__/KoreaderSettings.spec.ts
+8 -6
View File
@@ -329,7 +329,7 @@ describe('KoreaderSettings', () => {
'Setup',
'Devices',
'Plugin Activity',
'Unmatched KOReader Books',
'KOReader Books to Review',
'Manual KOReader Links',
'Setup Guide',
'Danger Zone',
@@ -346,7 +346,8 @@ describe('KoreaderSettings', () => {
expect(wrapper.text()).toContain('Update available')
expect(wrapper.text()).toContain('latest plugin v1.6.0')
expect(wrapper.text()).toContain('Matched books')
expect(wrapper.text()).toContain('No unmatched KOReader books.')
expect(wrapper.text()).toContain('No KOReader books to review.')
expect(wrapper.text()).toContain('Unlinked books can still sync through an automatic match.')
expect(wrapper.text()).toContain('No manual KOReader links.')
expect(wrapper.text()).toContain('2 deleted highlights awaiting KOReader plugin acknowledgement.')
expect(wrapper.text()).toContain('3 highlight positions need attention.')
@@ -677,8 +678,8 @@ describe('KoreaderSettings', () => {
await buttonByText(wrapper, 'Dismiss all')!.trigger('click')
expect(wrapper.text()).toContain('Dismiss all 2 unmatched books?')
expect(wrapper.text()).toContain('clears your entire unmatched books list')
expect(wrapper.text()).toContain('Dismiss all 2 books to review?')
expect(wrapper.text()).toContain('clears your entire books to review list')
const dismissAllButtons = wrapper.findAll('button').filter((button) => button.text() === 'Dismiss all')
await dismissAllButtons[dismissAllButtons.length - 1]!.trigger('click')
@@ -700,7 +701,7 @@ describe('KoreaderSettings', () => {
await buttonByText(wrapper, 'Cancel')!.trigger('click')
expect(koreaderMock.dismissAllUnmatchedBooks).not.toHaveBeenCalled()
expect(wrapper.text()).not.toContain('clears your entire unmatched books list')
expect(wrapper.text()).not.toContain('clears your entire books to review list')
})
it('keeps the confirm dialog open and reports an error when dismissing all unmatched books fails', async () => {
@@ -719,7 +720,7 @@ describe('KoreaderSettings', () => {
await flushPromises()
expect(koreaderMock.dismissAllUnmatchedBooks).toHaveBeenCalledTimes(1)
expect(wrapper.text()).toContain('clears your entire unmatched books list')
expect(wrapper.text()).toContain('clears your entire books to review list')
})
it('loads more BookOrbit search results from the link dialog', async () => {
@@ -778,6 +779,7 @@ describe('KoreaderSettings', () => {
await buttonByText(wrapper, 'Unlink')!.trigger('click')
expect(wrapper.text()).toContain('Unlink KOReader book?')
expect(wrapper.text()).toContain('Future syncs will use an automatic match if one exists.')
expect(wrapper.text()).toContain('Already synced stats will stay on their current BookOrbit book.')
const unlinkButtons = wrapper.findAll('button').filter((button) => button.text() === 'Unlink')
client/src/locales/en.json
+16 -2
View File
@@ -2886,7 +2886,13 @@
"unmatchedDismissed": "Unmatched book dismissed",
"dismissFailed": "Failed to dismiss unmatched book",
"allDismissed": "{count, plural, =0 {Dismissed no unmatched books} one {Dismissed # unmatched book} other {Dismissed # unmatched books}}",
"dismissAllFailed": "Failed to dismiss all unmatched books"
"dismissAllFailed": "Failed to dismiss all unmatched books",
"reviewRefreshed": "Books to review refreshed",
"reviewRefreshFailed": "Failed to refresh books to review",
"reviewDismissed": "Book dismissed from review",
"reviewDismissFailed": "Failed to dismiss book from review",
"reviewAllDismissed": "{count, plural, =0 {Dismissed no books from review} one {Dismissed # book from review} other {Dismissed # books from review}}",
"reviewDismissAllFailed": "Failed to dismiss all books from review"
},
"modal": {
"linkTitle": "Link KOReader book",
@@ -2939,7 +2945,15 @@
"syncedStatsNote": "Already synced stats will stay on their current BookOrbit book.",
"chooseDifferent": "Choose different",
"saving": "Saving...",
"confirmLink": "Confirm link"
"confirmLink": "Confirm link",
"reviewTitle": "KOReader Books to Review",
"reviewHint": "Books without an automatic match and books you unlinked appear here. Unlinked books can still sync through an automatic match. Link a book to override that match, or dismiss it to remove it from this list.",
"loadingReview": "Loading books to review...",
"noReview": "No KOReader books to review.",
"manualUnlinkConfirmBody": "Remove the manual link to {title} and return this book to the review list. Future syncs will use an automatic match if one exists. Already synced stats will stay on their current BookOrbit book.",
"reviewDismissConfirmBody": "This removes it from your books to review. Automatic matching and existing sync data are kept. If a device reports the hash again without a match, it can reappear.",
"reviewDismissAllConfirmTitle": "{count, plural, =0 {No books to review} one {Dismiss all # book to review?} other {Dismiss all # books to review?}}",
"reviewDismissAllConfirmBody": "This clears your entire books to review list, including entries on other pages. Automatic matching and existing sync data are kept. If a device reports a hash again without a match, it can reappear."
},
"remove": "Remove",
"removing": "Removing...",
docs/KOREADER_HASH_MATCHING.md
+21
View File
@@ -0,0 +1,21 @@
# KOReader download matching
KOReader identifies books with a partial file hash. Kobo conversion and audio removal change the bytes, so the downloaded hash can differ from the library file's hash. BookOrbit registers the hash of the same open file descriptor it streams, preserving the source hash and explicit manual links.
## Registration and recovery
Delivered hashes are stored in file hash history. Registration and the owning library's match-cache revision update commit together. Duplicate deliveries do not increment that revision. Registration uses a transaction-local one-second lock timeout, so a busy library falls back to recovery rather than holding up the download for the database's full statement timeout.
If registration fails, an immutable hash-only entry is saved in the application's private retry directory. Replay runs at startup and every 30 seconds, with bounded batches and a retained directory cursor. Malformed files and symbolic links are removed without following links; directories are skipped. Failed registrations remain for another pass. Recovery does not need the delivered archive, including temporary audioless EPUBs that have already been deleted.
If both the database and retry storage fail, the download still succeeds and failures are logged. The lock timeout bounds database lock waits, not connection acquisition or all query execution.
## Manual links and review
An explicit manual link takes precedence over automatic hash matches. Removing it returns the hash to **KOReader Books to Review**. Automatic matching can still resolve that hash and sync progress; the review entry allows the user to choose another manual target. Dismissing a review entry does not disable automatic matching or delete synced progress. Previously synced statistics stay on their existing book.
The existing API routes and response fields remain compatible. The file-timestamp component of the match-cache token uses a global indexed lookup, so another library's file changes can trigger extra match refreshes. History revisions and manual links remain scoped to the user's access. Manifest pagination continues across token changes.
## Historical recovery limit
The one-time backfill recovers delivered identities from retained KEPUB cache artifacts whose source identity is still verifiable. It cannot recover pre-fix temporary audioless outputs, over-limit conversions, or other delivered variants whose bytes have already been deleted. Those hashes need a manual link or another download. New deliveries use registration and durable retry instead of depending on retained artifacts.
scripts/e2e/suite-registry.mjs
+19
View File
@@ -920,6 +920,25 @@ export const E2E_SUITES = Object.freeze({
...SHARED_DB_AND_HELPER_PATHS,
],
},
"kobo-koreader-download": {
id: "kobo-koreader-download",
name: "Kobo KOReader Download Identity",
timeout: 60,
lane: "full",
description: "Kobo delivered hashes, KOReader matching, cache recovery and access isolation",
vitestTarget: "test/kobo-koreader-download.e2e-spec.ts",
junitOutput: `${TEST_RESULTS_DIR}/kobo-koreader-download-e2e-junit.xml`,
prepareDedicatedDatabase: true,
useDedicatedDatabase: true,
changedPaths: [
"server/src/modules/kobo/**",
"server/src/modules/koreader/**",
"server/src/common/utils/file-hash.utils.ts",
"server/test/kobo-koreader-download.e2e-spec.ts",
"server/test/e2e/reader-state-isolation/**",
...SHARED_DB_AND_HELPER_PATHS,
],
},
"kobo-cover-identity": {
id: "kobo-cover-identity",
name: "Kobo Cover Identity",
server/src/common/utils/file-hash.utils.test.ts
+59
View File
@@ -0,0 +1,59 @@
import { createHash } from 'crypto';
import { mkdtemp, open, rename, rm, writeFile } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { computeFileHash, computeFileHashFromHandle } from './file-hash.utils';
describe('KOReader hash from an open file', () => {
let root: string;
beforeEach(async () => {
root = await mkdtemp(join(tmpdir(), 'open-file-hash-'));
});
afterEach(async () => {
await rm(root, { recursive: true, force: true });
});
it('matches path hashing and leaves ownership and stream offset with the caller', async () => {
const path = join(root, 'book.epub');
const bytes = Buffer.alloc(8192, 'A');
bytes.fill('B', 1024, 2048);
bytes.fill('C', 4096, 5120);
await writeFile(path, bytes);
const handle = await open(path, 'r');
try {
const expected = createHash('md5')
.update(Buffer.concat([Buffer.alloc(1024, 'A'), Buffer.alloc(1024, 'B'), Buffer.alloc(1024, 'C')]))
.digest('hex');
await expect(computeFileHashFromHandle(handle)).resolves.toBe(expected);
await expect(computeFileHash(path)).resolves.toBe(expected);
const buffer = Buffer.alloc(1);
await handle.read(buffer, 0, 1, null);
expect(buffer.toString()).toBe('A');
await expect(handle.stat()).resolves.toEqual(expect.objectContaining({ size: 8192 }));
} finally {
await handle.close();
}
});
it('hashes the opened artifact after its path is atomically replaced', async () => {
const path = join(root, 'book.epub');
const bytes = Buffer.from('original bytes');
await writeFile(path, bytes);
const handle = await open(path, 'r');
try {
await rename(path, join(root, 'previous.epub'));
await writeFile(path, 'replacement bytes');
await expect(computeFileHashFromHandle(handle)).resolves.toBe(createHash('md5').update(bytes).digest('hex'));
expect(await computeFileHash(path)).not.toBe(await computeFileHashFromHandle(handle));
} finally {
await handle.close();
}
});
it('propagates descriptor failures without closing a caller-owned handle', async () => {
const failure = new Error('Read failed');
const handle = { stat: vi.fn().mockResolvedValue({ size: 100 }), read: vi.fn().mockRejectedValue(failure), close: vi.fn() };
await expect(computeFileHashFromHandle(handle as never)).rejects.toBe(failure);
expect(handle.close).not.toHaveBeenCalled();
});
});
server/src/common/utils/file-hash.utils.ts
+38
View File
@@ -0,0 +1,38 @@
import { createHash } from 'crypto';
import { open } from 'fs/promises';
import type { FileHandle } from 'fs/promises';
const BASE = 1024;
const BLOCK_SIZE = 1024;
/**
* Partial MD5 fingerprint matching KOReader's util.partialMD5 algorithm.
* Reads 1 KB blocks from positions 0, 1 KB, 4 KB, 16 KB, … up to 1 GB.
* The loop starts at i=-1 so that `BASE << (2 * -1)` overflows 32-bit to 0,
* producing the same byte sequence KOReader uses for its document hash.
*/
export async function computeFileHash(absolutePath: string): Promise<string> {
const fh = await open(absolutePath, 'r');
try {
return await computeFileHashFromHandle(fh);
} finally {
await fh.close();
}
}
export async function computeFileHashFromHandle(fh: FileHandle): Promise<string> {
const { size } = await fh.stat();
const hash = createHash('md5');
const buf = Buffer.allocUnsafe(BLOCK_SIZE);
for (let i = -1; i <= 10; i++) {
const position = BASE << (2 * i); // BASE << -2 overflows 32-bit to 0 when i = -1
if (position >= size) break;
const { bytesRead } = await fh.read(buf, 0, BLOCK_SIZE, position);
if (bytesRead > 0) {
hash.update(buf.subarray(0, bytesRead));
}
}
return hash.digest('hex');
}
server/src/db/book-file-hash-history.test.ts
+51
View File
@@ -0,0 +1,51 @@
import { eq } from 'drizzle-orm';
import { createCapturingDb } from '../common/test-utils/capture-sql-db';
import * as schema from './schema';
import {
deleteBookFilesWithHashInvalidation,
deleteBooksWithHashInvalidation,
deleteLibraryFoldersWithHashInvalidation,
recordBookFileHashHistory,
} from './book-file-hash-history';
const HASH = 'a'.repeat(32);
describe('hash identity revision writes', () => {
it('commits insertion and scoped invalidation in one statement and bumps only newly inserted identities', async () => {
const { db, queries } = createCapturingDb();
await recordBookFileHashHistory(db, 22, HASH, 'external_change');
expect(queries).toHaveLength(1);
const query = queries[0]!;
expect(query.sql).toContain('with registered as');
expect(query.sql).toContain('on conflict (book_file_id, file_hash) do nothing');
expect(query.sql).toContain('returning book_file_id');
expect(query.sql).toContain('update "libraries"');
expect(query.sql).toContain('koreader_hash_revision = koreader_hash_revision + 1');
expect(query.sql).toContain('from registered');
expect(query.params).toEqual([HASH, 'external_change', 22]);
});
it.each([
{ table: 'book_files', run: deleteBookFilesWithHashInvalidation, filter: eq(schema.bookFiles.id, 22) },
{ table: 'books', run: deleteBooksWithHashInvalidation, filter: eq(schema.books.id, 22) },
{ table: 'library_folders', run: deleteLibraryFoldersWithHashInvalidation, filter: eq(schema.libraryFolders.id, 22) },
])('commits $table deletion and invalidation together, including cascading history deletion', async ({ table, run, filter }) => {
const { db, queries } = createCapturingDb();
await run(db, filter);
expect(queries).toHaveLength(1);
const query = queries[0]!;
expect(query.sql).toContain('with deleted as');
expect(query.sql).toContain(`delete from "${table}" where`);
expect(query.sql).toContain('update "libraries"');
expect(query.sql).toContain('koreader_hash_revision = koreader_hash_revision + 1');
expect(query.sql).toContain('deleted');
expect(query.params).toEqual([22]);
});
it('propagates database failure so callers cannot commit a mutation without its invalidation', async () => {
const failure = new Error('Database unavailable');
const db = { execute: vi.fn().mockRejectedValue(failure) };
await expect(deleteBooksWithHashInvalidation(db as never, eq(schema.books.id, 22))).rejects.toBe(failure);
await expect(recordBookFileHashHistory(db as never, 22, HASH, 'kobo_download')).rejects.toBe(failure);
});
});
server/src/db/book-file-hash-history.ts
+65
View File
@@ -0,0 +1,65 @@
import { sql, type SQL } from 'drizzle-orm';
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
import * as schema from './schema';
export async function recordBookFileHashHistory(
db: Pick<NodePgDatabase<typeof schema>, 'execute'>,
bookFileId: number,
fileHash: string,
reason: string,
): Promise<void> {
// Registration and invalidation commit together. Duplicate deliveries do not invalidate caches.
// Selecting the source also lets recovery discard entries for files that were deleted.
await db.execute(sql`
with registered as (
insert into ${schema.bookFileHashHistory} (book_file_id, file_hash, reason)
select ${schema.bookFiles.id}, ${fileHash}, ${reason}
from ${schema.bookFiles} where ${schema.bookFiles.id} = ${bookFileId}
on conflict (book_file_id, file_hash) do nothing
returning book_file_id
)
update ${schema.libraries}
set koreader_hash_revision = koreader_hash_revision + 1
where id in (
select ${schema.books.libraryId} from registered
join ${schema.bookFiles} on ${schema.bookFiles.id} = registered.book_file_id
join ${schema.books} on ${schema.books.id} = ${schema.bookFiles.bookId}
)
`);
}
export async function deleteBookFilesWithHashInvalidation(db: Pick<NodePgDatabase<typeof schema>, 'execute'>, filter: SQL): Promise<void> {
await db.execute(sql`
with deleted as (
delete from ${schema.bookFiles} where ${filter} returning book_id
)
update ${schema.libraries}
set koreader_hash_revision = koreader_hash_revision + 1
where id in (
select ${schema.books.libraryId} from ${schema.books}
join deleted on deleted.book_id = ${schema.books.id}
)
`);
}
export async function deleteBooksWithHashInvalidation(db: Pick<NodePgDatabase<typeof schema>, 'execute'>, filter: SQL): Promise<void> {
await db.execute(sql`
with deleted as (
delete from ${schema.books} where ${filter} returning library_id
)
update ${schema.libraries}
set koreader_hash_revision = koreader_hash_revision + 1
where id in (select library_id from deleted)
`);
}
export async function deleteLibraryFoldersWithHashInvalidation(db: Pick<NodePgDatabase<typeof schema>, 'execute'>, filter: SQL): Promise<void> {
await db.execute(sql`
with deleted as (
delete from ${schema.libraryFolders} where ${filter} returning library_id
)
update ${schema.libraries}
set koreader_hash_revision = koreader_hash_revision + 1
where id in (select library_id from deleted)
`);
}
server/src/db/migrations/0103_fix_kobo_koreader_download_matching.sql
+5
View File
@@ -0,0 +1,5 @@
ALTER TABLE "book_file_hash_history" DROP CONSTRAINT "book_file_hash_history_reason_chk";--> statement-breakpoint
ALTER TABLE "libraries" ADD COLUMN "koreader_hash_revision" bigint DEFAULT 0 NOT NULL;--> statement-breakpoint
ALTER TABLE "koreader_unmatched_books" ADD COLUMN "manual_link_requested" boolean DEFAULT false NOT NULL;--> statement-breakpoint
CREATE INDEX "book_files_change_timestamp_idx" ON "book_files" USING btree (greatest("created_at", "updated_at") desc);--> statement-breakpoint
ALTER TABLE "book_file_hash_history" ADD CONSTRAINT "book_file_hash_history_reason_chk" CHECK ("book_file_hash_history"."reason" in ('file_write', 'external_change', 'rescan', 'kobo_download'));
server/src/db/migrations/meta/0103_snapshot.json
+22717
View File
File diff suppressed because it is too large Load Diff
server/src/db/migrations/meta/_journal.json
+7
View File
@@ -722,6 +722,13 @@
"when": 1791126075111,
"tag": "0102_add_book_journal_and_annotation_stars",
"breakpoints": true
},
{
"idx": 103,
"version": "7",
"when": 1791330236442,
"tag": "0103_fix_kobo_koreader_download_matching",
"breakpoints": true
}
]
}
server/src/db/schema/books.ts
+1
View File
@@ -102,6 +102,7 @@ export const bookFiles = pgTable(
uniqueIndex('book_files_public_id_uidx').on(t.publicId),
uniqueIndex('book_files_absolute_path_uidx').on(t.absolutePath),
index('book_files_book_id_idx').on(t.bookId),
index('book_files_change_timestamp_idx').on(sql`greatest(${t.createdAt}, ${t.updatedAt}) desc`),
index('book_files_library_folder_id_idx').on(t.libraryFolderId),
index('book_files_file_hash_idx').on(t.fileHash),
index('book_files_ino_idx').on(t.ino),
server/src/db/schema/koreader.ts
+2 -1
View File
@@ -150,7 +150,7 @@ export const bookFileHashHistory = pgTable(
(t) => [
uniqueIndex('book_file_hash_history_book_file_id_file_hash_idx').on(t.bookFileId, t.fileHash),
index('book_file_hash_history_file_hash_idx').on(t.fileHash),
check('book_file_hash_history_reason_chk', sql`${t.reason} in ('file_write', 'external_change', 'rescan')`),
check('book_file_hash_history_reason_chk', sql`${t.reason} in ('file_write', 'external_change', 'rescan', 'kobo_download')`),
],
);
@@ -200,6 +200,7 @@ export const koreaderUnmatchedBooks = pgTable(
lastOpen: bigint('last_open', { mode: 'number' }),
source: varchar('source', { length: 20 }).notNull().default('statistics'),
metadataAmbiguous: boolean('metadata_ambiguous').notNull().default(false),
manualLinkRequested: boolean('manual_link_requested').notNull().default(false),
firstSeenAt: timestamp('first_seen_at', { withTimezone: true }).defaultNow().notNull(),
lastSeenAt: timestamp('last_seen_at', { withTimezone: true }).defaultNow().notNull(),
},
server/src/db/schema/libraries.ts
+1
View File
@@ -85,6 +85,7 @@ export const libraries = pgTable(
scanMode: varchar('scan_mode', { length: 20 }).notNull().default('auto'),
pollInterval: integer('poll_interval_seconds').default(300),
koreaderHashRevision: bigint('koreader_hash_revision', { mode: 'number' }).notNull().default(0),
createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(),
updatedAt: timestamp('updated_at', { withTimezone: true })
.defaultNow()
server/src/modules/architecture/architecture-boundaries.test.ts
-1
View File
@@ -48,7 +48,6 @@ describe('Architecture boundaries', () => {
'src/modules/kobo/services/kobo-book-access.service.ts',
'src/modules/kobo/services/kobo-book-identity.service.ts',
'src/modules/kobo/services/kobo-device.service.ts',
'src/modules/kobo/services/kobo-download.service.ts',
'src/modules/kobo/services/kobo-reading-state.service.ts',
'src/modules/kobo/services/kobo-settings.service.ts',
'src/modules/kobo/services/kobo-sync-history.service.ts',
server/src/modules/book-move/book-move.repository.ts
+2 -1
View File
@@ -1,3 +1,4 @@
import { deleteBooksWithHashInvalidation } from '../../db/book-file-hash-history';
import { Inject, Injectable } from '@nestjs/common';
import { and, asc, eq, inArray, sql } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
@@ -363,7 +364,7 @@ export class BookMoveRepository {
.limit(1);
if (duplicate && duplicate.libraryId === input.targetLibraryId) {
await tx.delete(books).where(eq(books.id, duplicate.id));
await deleteBooksWithHashInvalidation(tx, eq(books.id, duplicate.id));
mergedBookId = duplicate.id;
}
}
server/src/modules/book/book.repository.test.ts
+10 -4
View File
@@ -163,6 +163,7 @@ describe('BookRepository', () => {
const stateDeleteWhere = vi.fn().mockResolvedValue(undefined);
const bookDeleteWhere = vi.fn().mockResolvedValue(undefined);
const tx = {
execute: vi.fn().mockResolvedValue(undefined),
select: vi.fn().mockReturnValueOnce(bookSelect).mockReturnValueOnce(folderSelect),
update: vi.fn().mockReturnValue({ set: updateSet }),
delete: vi.fn().mockReturnValueOnce({ where: stateDeleteWhere }).mockReturnValueOnce({ where: bookDeleteWhere }),
@@ -174,7 +175,8 @@ describe('BookRepository', () => {
expect(db.transaction).toHaveBeenCalledTimes(1);
expect(tx.update).toHaveBeenCalledTimes(1);
expect(tx.delete).toHaveBeenCalledTimes(2);
expect(tx.delete).toHaveBeenCalledTimes(1);
expect(tx.execute).toHaveBeenCalledTimes(1);
const invalidationQuery = new PgDialect().sqlToQuery(stateDeleteWhere.mock.calls[0]![0]);
expect(invalidationQuery.sql).toContain('"library_dir_scan_state"."library_folder_id" = $1');
expect(invalidationQuery.sql).toContain('"library_dir_scan_state"."dir_path" in');
@@ -188,6 +190,7 @@ describe('BookRepository', () => {
const updateWhere = vi.fn().mockResolvedValue(undefined);
const stateDeleteWhere = vi.fn().mockRejectedValue(new Error('invalidation failed'));
const tx = {
execute: vi.fn().mockResolvedValue(undefined),
select: vi.fn().mockReturnValueOnce(bookSelect).mockReturnValueOnce(folderSelect),
update: vi.fn().mockReturnValue({ set: vi.fn().mockReturnValue({ where: updateWhere }) }),
delete: vi.fn().mockReturnValue({ where: stateDeleteWhere }),
@@ -209,6 +212,7 @@ describe('BookRepository', () => {
const bookSelect = makeSelectChain('for', bookRows);
const folderSelect = makeSelectChain('for', [{ id: 7 }]);
const tx = {
execute: vi.fn().mockResolvedValue(undefined),
select: vi.fn().mockReturnValueOnce(bookSelect).mockReturnValueOnce(folderSelect),
update: vi.fn().mockReturnValue({ set: vi.fn().mockReturnValue({ where: vi.fn().mockResolvedValue(undefined) }) }),
delete: vi.fn().mockReturnValue({ where: vi.fn().mockResolvedValue(undefined) }),
@@ -218,7 +222,8 @@ describe('BookRepository', () => {
await repo.deleteByIdsAndInvalidateScanState(bookRows.map((row) => row.id));
expect(tx.delete).toHaveBeenCalledTimes(3);
expect(tx.delete).toHaveBeenCalledTimes(2);
expect(tx.execute).toHaveBeenCalledTimes(1);
});
it('findCards loads card rows and related collections for the current user', async () => {
@@ -943,6 +948,7 @@ describe('BookRepository', () => {
const updateWhere = vi.fn().mockResolvedValue(undefined);
const updateBuilder = { set: vi.fn().mockReturnValue({ where: updateWhere }) };
const db = {
execute: vi.fn().mockResolvedValue(undefined),
delete: vi.fn().mockReturnValue(deleteBuilder),
update: vi.fn().mockReturnValue(updateBuilder),
};
@@ -950,8 +956,8 @@ describe('BookRepository', () => {
await repo.deleteByIds([10, 11]);
await repo.updateMetadataFields(10, { title: 'Updated' });
expect(db.delete).toHaveBeenCalledTimes(1);
expect(deleteWhere).toHaveBeenCalledTimes(1);
expect(db.execute).toHaveBeenCalledTimes(1);
expect(new PgDialect().sqlToQuery(db.execute.mock.calls[0]![0]).sql).toContain('delete from "books"');
expect(db.update).toHaveBeenCalledTimes(2);
expect(updateBuilder.set).toHaveBeenNthCalledWith(1, { title: 'Updated' });
expect(updateBuilder.set).toHaveBeenNthCalledWith(2, expect.objectContaining({ updatedAt: expect.any(Date) }));
server/src/modules/book/book.repository.ts
+5 -3
View File
@@ -1,3 +1,4 @@
import { deleteBookFilesWithHashInvalidation, deleteBooksWithHashInvalidation } from '../../db/book-file-hash-history';
import { BadRequestException, Inject, Injectable, Optional } from '@nestjs/common';
import { SQL, and, asc, count, eq, inArray, isNotNull, isNull, lte, ne, or, sql } from 'drizzle-orm';
import { SUPPORTED_BOOK_FORMATS } from '../upload/upload-validator.service';
@@ -1591,7 +1592,7 @@ export class BookRepository {
}
async deleteBookFile(fileId: number): Promise<void> {
await this.db.delete(bookFiles).where(eq(bookFiles.id, fileId));
await deleteBookFilesWithHashInvalidation(this.db, eq(bookFiles.id, fileId));
}
async updateBookFile(
@@ -2149,7 +2150,7 @@ export class BookRepository {
}
async deleteByIds(bookIds: number[]): Promise<void> {
await this.db.delete(books).where(inArray(books.id, bookIds));
await deleteBooksWithHashInvalidation(this.db, inArray(books.id, bookIds));
}
async deleteByIdsAndInvalidateScanState(bookIds: number[]): Promise<void> {
@@ -2203,7 +2204,8 @@ export class BookRepository {
}
}
await tx.delete(books).where(
await deleteBooksWithHashInvalidation(
tx,
inArray(
books.id,
rows.map((row) => row.id),
server/src/modules/file-write/file-rename.repository.test.ts
+4 -2
View File
@@ -215,6 +215,7 @@ describe('FileRenameRepository', () => {
const setCalls: unknown[] = [];
const deleteWhere = vi.fn().mockResolvedValue(undefined);
const tx = {
execute: vi.fn().mockResolvedValue(undefined),
select: vi.fn().mockReturnValue({
from: vi.fn().mockReturnThis(),
where: vi.fn().mockReturnThis(),
@@ -251,13 +252,14 @@ describe('FileRenameRepository', () => {
expect.objectContaining({ bookId: 99, libraryFolderId: 7, absolutePath: '/library/Author/Book/old.opf', relPath: 'Author/Book/old.opf' }),
expect.objectContaining({ primaryFileId: 10, status: 'present' }),
]);
expect(tx.delete).toHaveBeenCalledTimes(1);
expect(deleteWhere).toHaveBeenCalledTimes(1);
expect(tx.execute).toHaveBeenCalledTimes(1);
expect(tx.delete).not.toHaveBeenCalled();
});
it('applyExistingFolderMerge does not replace an existing target primary file', async () => {
const setCalls: unknown[] = [];
const tx = {
execute: vi.fn().mockResolvedValue(undefined),
select: vi.fn().mockReturnValue({
from: vi.fn().mockReturnThis(),
where: vi.fn().mockReturnThis(),
server/src/modules/file-write/file-rename.repository.ts
+2 -1
View File
@@ -1,3 +1,4 @@
import { deleteBooksWithHashInvalidation } from '../../db/book-file-hash-history';
import { Inject, Injectable } from '@nestjs/common';
import { and, asc, eq, inArray } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
@@ -241,7 +242,7 @@ export class FileRenameRepository {
.where(eq(bookFiles.id, update.id));
}
await tx.delete(books).where(eq(books.id, input.sourceBookId));
await deleteBooksWithHashInvalidation(tx, eq(books.id, input.sourceBookId));
await tx
.update(books)
server/src/modules/file-write/file-write.repository.ts
+2 -1
View File
@@ -1,3 +1,4 @@
import { recordBookFileHashHistory } from '../../db/book-file-hash-history';
import { Inject, Injectable } from '@nestjs/common';
import { and, asc, desc, eq, isNull, ne, sql } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
@@ -155,7 +156,7 @@ export class FileWriteRepository {
}
async recordHashHistory(bookFileId: number, fileHash: string, reason: string): Promise<void> {
await this.db.insert(schema.bookFileHashHistory).values({ bookFileId, fileHash, reason }).onConflictDoNothing();
await recordBookFileHashHistory(this.db, bookFileId, fileHash, reason);
}
async loadPayload(bookId: number) {
server/src/modules/kobo/kobo-download.repository.test.ts
+95
View File
@@ -0,0 +1,95 @@
import { Test } from '@nestjs/testing';
import { DB } from '../../db/db.module';
import { createCapturingDb } from '../../common/test-utils/capture-sql-db';
import { KoboDownloadRepository } from './kobo-download.repository';
const HASH = 'a'.repeat(32);
async function makeRepository(db: unknown) {
const module = await Test.createTestingModule({ providers: [KoboDownloadRepository, { provide: DB, useValue: db }] }).compile();
return module.get(KoboDownloadRepository);
}
describe('KoboDownloadRepository', () => {
it('adds intrinsic download identity without replacing original hashes or manual mappings', async () => {
const { db, queries } = createCapturingDb();
const repository = await makeRepository(db);
await repository.registerDeliveredHash(22, HASH);
expect(queries.map((query) => query.sql)).toEqual(['begin', "set local lock_timeout = '1s'", expect.stringContaining('insert into'), 'commit']);
expect(queries[2]!.sql).toContain('insert into "book_file_hash_history"');
expect(queries[2]!.params).toEqual(expect.arrayContaining([HASH, 'kobo_download', 22]));
expect(queries[2]!.sql).toContain('on conflict (book_file_id, file_hash) do nothing');
expect(queries[2]!.sql).toContain('update "libraries"');
expect(queries[2]!.sql).toContain('koreader_hash_revision = koreader_hash_revision + 1');
expect(queries[2]!.sql).not.toContain('koreader_book_hash_links');
});
it('uses a file-and-hash conflict key so repeated registration is idempotent and collisions remain representable', async () => {
const { db, queries } = createCapturingDb();
const repository = await makeRepository(db);
await repository.registerDeliveredHash(22, HASH);
await repository.registerDeliveredHash(22, HASH);
await repository.registerDeliveredHash(33, HASH);
const registrations = queries.filter((query) => query.sql.includes('insert into'));
expect(registrations).toHaveLength(3);
expect(registrations.map((query) => query.params)).toEqual([
[HASH, 'kobo_download', 22],
[HASH, 'kobo_download', 22],
[HASH, 'kobo_download', 33],
]);
expect(registrations.every((query) => query.sql.includes('do nothing'))).toBe(true);
});
it('propagates persistence failure for the registration service to queue', async () => {
const failure = new Error('Database unavailable');
const tx = { execute: vi.fn().mockResolvedValueOnce(undefined).mockRejectedValueOnce(failure) };
const repository = await makeRepository({ transaction: vi.fn((callback) => callback(tx)) });
await expect(repository.registerDeliveredHash(22, HASH)).rejects.toBe(failure);
});
it('honors the selected primary file inside its book without imposing a new role restriction', async () => {
const { db, queries } = createCapturingDb();
const repository = await makeRepository(db);
await repository.findPrimaryFile(11, 22);
expect(queries[0]!.sql).toContain('"bookFiles"."book_id" =');
expect(queries[0]!.sql).toContain('"bookFiles"."id" =');
expect(queries[0]!.sql).not.toContain('"bookFiles"."role" =');
expect(queries[0]!.params).toEqual(expect.arrayContaining([11, 22]));
});
it('selects only book identity and primary-file identity', async () => {
const { db, queries } = createCapturingDb();
const repository = await makeRepository(db);
await repository.findBook(11);
expect(queries[0]!.sql).toContain('"id", "primary_file_id"');
expect(queries[0]!.params).toContain(11);
});
it('bounds cache-source lookup and verifies book, role and original/historical source identity', async () => {
const { db, queries } = createCapturingDb();
const repository = await makeRepository(db);
await expect(repository.findCachedSourceFileId(11, HASH)).resolves.toBeNull();
const query = queries[0]!;
expect(query.sql).toContain('select distinct');
expect(query.sql).toContain('left join "book_file_hash_history"');
expect(query.sql).toContain('"book_files"."book_id" =');
expect(query.sql).toContain('"book_files"."role" =');
expect(query.sql).toContain('"book_file_hash_history"."reason" <>');
expect(query.sql).toContain('limit');
expect(query.params).toEqual(expect.arrayContaining([11, HASH, 'content', 'kobo_download', 2]));
});
it.each([
{ name: 'unique source', matches: [{ id: 22 }], result: 22 },
{ name: 'deleted or stale source', matches: [], result: null },
{ name: 'ambiguous source', matches: [{ id: 22 }, { id: 33 }], result: null },
])('recovers only a $name', async ({ matches, result }) => {
const chain = { from: vi.fn(), leftJoin: vi.fn(), where: vi.fn(), limit: vi.fn().mockResolvedValue(matches) };
chain.from.mockReturnValue(chain);
chain.leftJoin.mockReturnValue(chain);
chain.where.mockReturnValue(chain);
const repository = await makeRepository({ selectDistinct: vi.fn().mockReturnValue(chain) });
await expect(repository.findCachedSourceFileId(11, HASH)).resolves.toBe(result);
expect(chain.limit).toHaveBeenCalledWith(2);
});
});
server/src/modules/kobo/kobo-download.repository.ts
+56
View File
@@ -0,0 +1,56 @@
import { Inject, Injectable } from '@nestjs/common';
import { and, eq, ne, or, sql } from 'drizzle-orm';
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { DB } from '../../db/db.module';
import * as schema from '../../db/schema';
import { recordBookFileHashHistory } from '../../db/book-file-hash-history';
@Injectable()
export class KoboDownloadRepository {
constructor(@Inject(DB) private readonly db: NodePgDatabase<typeof schema>) {}
findBook(bookId: number) {
return this.db.query.books.findFirst({
where: eq(schema.books.id, bookId),
columns: { id: true, primaryFileId: true },
});
}
findPrimaryFile(bookId: number, fileId: number) {
return this.db.query.bookFiles.findFirst({
where: and(eq(schema.bookFiles.bookId, bookId), eq(schema.bookFiles.id, fileId)),
});
}
async registerDeliveredHash(bookFileId: number, fileHash: string): Promise<void> {
await this.db.transaction(async (tx) => {
// A busy library must fall back to the durable queue before delaying the download.
await tx.execute(sql`set local lock_timeout = '1s'`);
await recordBookFileHashHistory(tx, bookFileId, fileHash, 'kobo_download');
});
}
async findCachedSourceFileId(bookId: number, sourceHash: string): Promise<number | null> {
const matches = await this.db
.selectDistinct({ id: schema.bookFiles.id })
.from(schema.bookFiles)
.leftJoin(
schema.bookFileHashHistory,
and(
eq(schema.bookFileHashHistory.bookFileId, schema.bookFiles.id),
eq(schema.bookFileHashHistory.fileHash, sourceHash),
ne(schema.bookFileHashHistory.reason, 'kobo_download'),
),
)
.where(
and(
eq(schema.bookFiles.bookId, bookId),
eq(schema.bookFiles.role, 'content'),
or(eq(schema.bookFiles.fileHash, sourceHash), eq(schema.bookFileHashHistory.fileHash, sourceHash)),
),
)
.limit(2);
return matches.length === 1 ? matches[0]!.id : null;
}
}
server/src/modules/kobo/kobo.module.test.ts
+8
View File
@@ -16,6 +16,10 @@ import { KoboAnnotationMaterializerService } from './services/kobo-annotation-ma
import { KoboBookAccessService } from './services/kobo-book-access.service';
import { KoboBookIdentityService } from './services/kobo-book-identity.service';
import { KoboDeviceService } from './services/kobo-device.service';
import { KoboDownloadRepository } from './kobo-download.repository';
import { KoboDownloadHashBackfillService } from './services/kobo-download-hash-backfill.service';
import { AppSettingsModule } from '../app-settings/app-settings.module';
import { KoboDownloadHashRegistrationService } from './services/kobo-download-hash-registration.service';
import { KoboDownloadService } from './services/kobo-download.service';
import { KoboKepubContextService } from './services/kobo-kepub-context.service';
import { KoboPositionRebuildService } from './services/kobo-position-rebuild.service';
@@ -40,6 +44,7 @@ describe('KoboModule', () => {
const imports = Reflect.getMetadata(MODULE_METADATA.IMPORTS, KoboModule) as unknown[];
expect(imports).toContain(BookModule);
expect(imports).toContain(AppSettingsModule);
expect(imports).toContain(ReadingSessionModule);
expect(imports).toContain(AnnotationModule);
expect(imports).toContain(PositionConverterModule);
@@ -62,6 +67,9 @@ describe('KoboModule', () => {
KoboSyncHistoryService,
KoboThumbnailService,
KoboDownloadService,
KoboDownloadRepository,
KoboDownloadHashBackfillService,
KoboDownloadHashRegistrationService,
KoboProxyService,
KoboAnalyticsResolverService,
KoboAnalyticsService,
server/src/modules/kobo/kobo.module.ts
+8
View File
@@ -24,6 +24,10 @@ import { KoboAnnotationExchangeService } from './services/kobo-annotation-exchan
import { KoboAnnotationMaterializerService } from './services/kobo-annotation-materializer.service';
import { KoboDeviceService } from './services/kobo-device.service';
import { KoboDownloadService } from './services/kobo-download.service';
import { KoboDownloadRepository } from './kobo-download.repository';
import { KoboDownloadHashRegistrationService } from './services/kobo-download-hash-registration.service';
import { KoboDownloadHashBackfillService } from './services/kobo-download-hash-backfill.service';
import { AppSettingsModule } from '../app-settings/app-settings.module';
import { KoboKepubContextService } from './services/kobo-kepub-context.service';
import { KoboPositionRebuildService } from './services/kobo-position-rebuild.service';
import { KoboProgressBridgeService } from './services/kobo-progress-bridge.service';
@@ -39,6 +43,7 @@ import { KoboAnalyticsService } from './services/kobo-analytics.service';
@Module({
imports: [
AppSettingsModule,
CommonModule,
AchievementModule,
AnnotationModule,
@@ -70,6 +75,9 @@ import { KoboAnalyticsService } from './services/kobo-analytics.service';
KoboSyncHistoryService,
KoboThumbnailService,
KoboDownloadService,
KoboDownloadRepository,
KoboDownloadHashBackfillService,
KoboDownloadHashRegistrationService,
KoboProxyService,
KoboAnalyticsResolverService,
KoboAnalyticsService,
server/src/modules/kobo/services/kepub-conversion.service.test.ts
+110 -66
View File
@@ -1,96 +1,140 @@
vi.mock('fs/promises', () => ({
mkdir: vi.fn(),
stat: vi.fn(),
}));
vi.mock('child_process', () => ({
execFile: vi.fn(),
}));
vi.mock('fs/promises', () => ({ mkdir: vi.fn(), mkdtemp: vi.fn(), rename: vi.fn(), rm: vi.fn(), stat: vi.fn() }));
vi.mock('child_process', () => ({ execFile: vi.fn() }));
import { Test } from '@nestjs/testing';
import { storageConfig } from '../../../config/config';
import { execFile } from 'child_process';
import { mkdir, stat } from 'fs/promises';
import { mkdir, mkdtemp, rename, rm, stat } from 'fs/promises';
import { KepubConversionService } from './kepub-conversion.service';
import { KepubifyBinaryService } from './kepubify-binary.service';
const execMock = vi.mocked(execFile);
const statMock = vi.mocked(stat);
const mkdirMock = vi.mocked(mkdir);
const execFileMock = vi.mocked(execFile);
const input = { sourcePath: '/books/source.epub', fileHash: 'abc', bookId: 44, hyphenate: false };
const cacheDir = '/app-data/.kepub-cache/44';
const tempDir = `${cacheDir}/.conversion-unit-test`;
const tempPath = `${tempDir}/book.kepub.epub`;
function makeService() {
const config = { get: vi.fn().mockReturnValue('/app-data') };
const kepubifyBinaryService = { getBinaryPath: vi.fn().mockResolvedValue('/tools/kepubify') };
return { service: new KepubConversionService(config as never, kepubifyBinaryService as never), kepubifyBinaryService };
function successfulConversion() {
execMock.mockImplementation((_path, _args, _options, callback) => {
callback?.(null, '', '');
return {} as never;
});
}
describe('KepubConversionService', () => {
beforeEach(() => {
vi.clearAllMocks();
let service: KepubConversionService;
let binary: { getBinaryPath: ReturnType<typeof vi.fn> };
beforeEach(async () => {
vi.resetAllMocks();
binary = { getBinaryPath: vi.fn().mockResolvedValue('/tools/kepubify') };
statMock.mockRejectedValue(Object.assign(new Error('No cached file'), { code: 'ENOENT' }));
vi.mocked(mkdir).mockResolvedValue(undefined);
vi.mocked(mkdtemp).mockImplementation((prefix) => Promise.resolve(`${prefix}unit-test`) as never);
vi.mocked(rename).mockResolvedValue(undefined);
vi.mocked(rm).mockResolvedValue(undefined);
successfulConversion();
const module = await Test.createTestingModule({
providers: [
KepubConversionService,
{ provide: storageConfig.KEY, useValue: { appDataPath: '/app-data' } },
{ provide: KepubifyBinaryService, useValue: binary },
],
}).compile();
service = module.get(KepubConversionService);
});
it('returns an existing cached kepub path without running kepubify', async () => {
const { service, kepubifyBinaryService } = makeService();
statMock.mockResolvedValueOnce({} as never);
await expect(service.getKepubPath({ sourcePath: '/books/source.epub', fileHash: 'abc', bookId: 44, hyphenate: false })).resolves.toBe(
'/app-data/.kepub-cache/44/abc.kepub.epub',
);
expect(kepubifyBinaryService.getBinaryPath).not.toHaveBeenCalled();
expect(execFileMock).not.toHaveBeenCalled();
it('returns existing conversions without invoking the binary or altering the cache', async () => {
statMock.mockResolvedValue({} as never);
await expect(service.getKepubPath(input)).resolves.toBe(`${cacheDir}/abc.kepub.epub`);
expect(binary.getBinaryPath).not.toHaveBeenCalled();
expect(execMock).not.toHaveBeenCalled();
expect(mkdir).not.toHaveBeenCalled();
expect(rename).not.toHaveBeenCalled();
});
it('converts and caches a kepub when the cache is missing', async () => {
const { service } = makeService();
statMock.mockRejectedValueOnce(new Error('cache miss'));
execFileMock.mockImplementation((_path, _args, _options, cb) => {
cb?.(null, '', '');
return {} as never;
});
await expect(service.getKepubPath({ sourcePath: '/books/source.epub', fileHash: 'hash', bookId: 44, hyphenate: true })).resolves.toBe(
'/app-data/.kepub-cache/44/hash-hyph.kepub.epub',
);
expect(mkdirMock).toHaveBeenCalledWith('/app-data/.kepub-cache/44', { recursive: true });
expect(execFileMock).toHaveBeenCalledWith(
it.each([
{ hash: 'abc', hyphenate: false, audioless: false, key: 'abc' },
{ hash: 'abc', hyphenate: true, audioless: false, key: 'abc-hyph' },
{ hash: 'abc', hyphenate: false, audioless: true, key: 'abc-noaudio-v1' },
{ hash: 'abc', hyphenate: true, audioless: true, key: 'abc-noaudio-v1-hyph' },
{ hash: null, hyphenate: false, audioless: false, key: 'nohash' },
{ hash: undefined, hyphenate: false, audioless: false, key: 'nohash' },
])('publishes a complete conversion with the compatible $key cache key', async ({ hash, hyphenate, audioless, key }) => {
await expect(service.getKepubPath({ ...input, fileHash: hash, hyphenate, audioless })).resolves.toBe(`${cacheDir}/${key}.kepub.epub`);
expect(mkdir).toHaveBeenCalledWith(cacheDir, { recursive: true });
expect(mkdtemp).toHaveBeenCalledWith(`${cacheDir}/.conversion-`);
expect(execMock).toHaveBeenCalledWith(
'/tools/kepubify',
['--hyphenate', '--output', '/app-data/.kepub-cache/44/hash-hyph.kepub.epub', '/books/source.epub'],
[...(hyphenate ? ['--hyphenate'] : []), '--output', tempPath, input.sourcePath],
{ timeout: 60_000 },
expect.any(Function),
);
expect(rename).toHaveBeenCalledExactlyOnceWith(tempPath, `${cacheDir}/${key}.kepub.epub`);
expect(rm).toHaveBeenCalledExactlyOnceWith(tempDir, { recursive: true, force: true });
expect(execMock.mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(rename).mock.invocationCallOrder[0]!);
expect(vi.mocked(rename).mock.invocationCallOrder[0]).toBeLessThan(vi.mocked(rm).mock.invocationCallOrder[0]!);
});
it('gives a stripped source its own cache entry so it cannot be served as the full archive', async () => {
const { service } = makeService();
statMock.mockRejectedValueOnce(new Error('cache miss'));
execFileMock.mockImplementation((_path, _args, _options, cb) => {
cb?.(null, '', '');
it('coalesces simultaneous misses and publishes only after conversion finishes', async () => {
let complete!: () => void;
execMock.mockImplementation((_path, _args, _options, callback) => {
complete = () => callback?.(null, '', '');
return {} as never;
});
// Same book and same file hash as the full archive; only the audioless flag separates them.
await expect(
service.getKepubPath({ sourcePath: '/tmp/kobo-epub/book.epub', fileHash: 'abc', bookId: 44, hyphenate: true, audioless: true }),
).resolves.toBe('/app-data/.kepub-cache/44/abc-noaudio-v1-hyph.kepub.epub');
const first = service.getKepubPath(input);
const second = service.getKepubPath(input);
await vi.waitFor(() => expect(execMock).toHaveBeenCalledTimes(1));
const third = service.getKepubPath(input);
expect(rename).not.toHaveBeenCalled();
expect(rm).not.toHaveBeenCalled();
complete();
await expect(Promise.all([first, second, third])).resolves.toEqual(Array(3).fill(`${cacheDir}/abc.kepub.epub`));
expect(execMock).toHaveBeenCalledTimes(1);
expect(rename).toHaveBeenCalledTimes(1);
});
it('uses a stable nohash cache key when file hash is unavailable', async () => {
const { service } = makeService();
statMock.mockRejectedValueOnce(new Error('cache miss'));
execFileMock.mockImplementation((_path, _args, _options, cb) => {
cb?.(null, '', '');
it('does not coalesce different source hashes, books, or conversion options', async () => {
await Promise.all([
service.getKepubPath(input),
service.getKepubPath({ ...input, fileHash: 'def' }),
service.getKepubPath({ ...input, bookId: 45 }),
service.getKepubPath({ ...input, hyphenate: true }),
service.getKepubPath({ ...input, audioless: true }),
]);
expect(execMock).toHaveBeenCalledTimes(5);
expect(rename).toHaveBeenCalledTimes(5);
});
it('cleans up a failed conversion without publishing it, and permits a retry', async () => {
execMock.mockImplementationOnce((_path, _args, _options, callback) => {
callback?.(new Error('Conversion failed'), '', '');
return {} as never;
});
await expect(service.getKepubPath(input)).rejects.toThrow('Conversion failed');
expect(rename).not.toHaveBeenCalled();
expect(rm).toHaveBeenCalledWith(tempDir, { recursive: true, force: true });
await expect(service.getKepubPath(input)).resolves.toBe(`${cacheDir}/abc.kepub.epub`);
expect(execMock).toHaveBeenCalledTimes(2);
});
await expect(service.getKepubPath({ sourcePath: '/books/source.epub', fileHash: null, bookId: 44, hyphenate: false })).resolves.toBe(
'/app-data/.kepub-cache/44/nohash.kepub.epub',
);
it('cleans up when atomic publication fails and allows a retry', async () => {
vi.mocked(rename).mockRejectedValueOnce(new Error('Rename failed'));
await expect(service.getKepubPath(input)).rejects.toThrow('Rename failed');
expect(rm).toHaveBeenCalledWith(tempDir, { recursive: true, force: true });
await expect(service.getKepubPath(input)).resolves.toBe(`${cacheDir}/abc.kepub.epub`);
});
expect(execFileMock).toHaveBeenCalledWith(
'/tools/kepubify',
['--output', '/app-data/.kepub-cache/44/nohash.kepub.epub', '/books/source.epub'],
{ timeout: 60_000 },
expect.any(Function),
);
it.each(['binary', 'directory', 'temporary-directory'])('does not publish after %s preparation fails', async (stage) => {
const error = new Error('Preparation failed');
if (stage === 'binary') binary.getBinaryPath.mockRejectedValue(error);
if (stage === 'directory') vi.mocked(mkdir).mockRejectedValue(error);
if (stage === 'temporary-directory') vi.mocked(mkdtemp).mockRejectedValue(error);
await expect(service.getKepubPath(input)).rejects.toThrow('Preparation failed');
expect(execMock).not.toHaveBeenCalled();
expect(rename).not.toHaveBeenCalled();
expect(rm).not.toHaveBeenCalled();
});
});
server/src/modules/kobo/services/kepub-conversion.service.ts
+33 -8
View File
@@ -1,10 +1,12 @@
import { execFile } from 'child_process';
import { mkdir, stat } from 'fs/promises';
import { mkdir, mkdtemp, rename, rm, stat } from 'fs/promises';
import { join } from 'path';
import { promisify } from 'util';
import { Injectable } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Inject, Injectable } from '@nestjs/common';
import type { ConfigType } from '@nestjs/config';
import { storageConfig } from '../../../config/config';
import { KepubifyBinaryService } from './kepubify-binary.service';
@@ -27,12 +29,13 @@ interface KepubConversionInput {
@Injectable()
export class KepubConversionService {
private readonly kepubCachePath: string;
private readonly conversions = new Map<string, Promise<string>>();
constructor(
private readonly config: ConfigService,
@Inject(storageConfig.KEY) storage: ConfigType<typeof storageConfig>,
private readonly kepubifyBinaryService: KepubifyBinaryService,
) {
this.kepubCachePath = join(this.config.get<string>('storage.appDataPath')!, '.kepub-cache');
this.kepubCachePath = join(storage.appDataPath, '.kepub-cache');
}
async getKepubPath(input: KepubConversionInput): Promise<string> {
@@ -41,6 +44,8 @@ export class KepubConversionService {
// Keep the original two key shapes byte for byte so existing cache entries still hit.
const cacheKey = `${fileHash}${input.audioless ? `-noaudio-v${AUDIOLESS_EPUB_CACHE_VERSION}` : ''}${input.hyphenate ? '-hyph' : ''}`;
const cachedPath = join(cacheDir, `${cacheKey}.kepub.epub`);
const pending = this.conversions.get(cachedPath);
if (pending) return pending;
try {
await stat(cachedPath);
@@ -49,10 +54,30 @@ export class KepubConversionService {
// Cache miss.
}
const concurrent = this.conversions.get(cachedPath);
if (concurrent) return concurrent;
const conversion = this.convertToCache(input, cacheDir, cachedPath);
this.conversions.set(cachedPath, conversion);
try {
return await conversion;
} finally {
this.conversions.delete(cachedPath);
}
}
private async convertToCache(input: KepubConversionInput, cacheDir: string, cachedPath: string): Promise<string> {
const binaryPath = await this.kepubifyBinaryService.getBinaryPath();
await mkdir(cacheDir, { recursive: true });
const args = input.hyphenate ? ['--hyphenate', '--output', cachedPath, input.sourcePath] : ['--output', cachedPath, input.sourcePath];
await execFileAsync(binaryPath, args, { timeout: KEPUBIFY_TIMEOUT_MS });
return cachedPath;
const tempDir = await mkdtemp(join(cacheDir, '.conversion-'));
const outputPath = join(tempDir, 'book.kepub.epub');
try {
const args = input.hyphenate ? ['--hyphenate', '--output', outputPath, input.sourcePath] : ['--output', outputPath, input.sourcePath];
await execFileAsync(binaryPath, args, { timeout: KEPUBIFY_TIMEOUT_MS });
// Readers and the cache recovery pass must never see a partially written conversion.
await rename(outputPath, cachedPath);
return cachedPath;
} finally {
await rm(tempDir, { recursive: true, force: true });
}
}
}
server/src/modules/kobo/services/kobo-download-file-handle.test.ts
+75
View File
@@ -0,0 +1,75 @@
import { Test } from '@nestjs/testing';
import { mkdtemp, readFile, rename, rm, writeFile } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import type { ReadStream } from 'fs';
import { computeFileHash } from '../../../common/utils/file-hash.utils';
import { AudiolessEpubService } from '../../book/audioless-epub.service';
import { KoboDownloadRepository } from '../kobo-download.repository';
import { KoboBookAccessService } from './kobo-book-access.service';
import { KepubConversionService } from './kepub-conversion.service';
import { KoboDownloadHashRegistrationService } from './kobo-download-hash-registration.service';
import { KoboDownloadService } from './kobo-download.service';
import { KoboSettingsService } from './kobo-settings.service';
describe('Kobo delivered file descriptor', () => {
it('streams the exact bytes it hashed even if the path is replaced during registration', async () => {
const root = await mkdtemp(join(tmpdir(), 'kobo-delivery-descriptor-'));
try {
const path = join(root, 'download.kepub.epub');
const original = Buffer.from('The originally delivered artifact');
const replacement = Buffer.from('A different concurrently published artifact with a different length');
await writeFile(path, original);
const originalHash = await computeFileHash(path);
const register = vi.fn().mockImplementation(async () => {
await rename(path, join(root, 'previous.kepub.epub'));
await writeFile(path, replacement);
});
const module = await Test.createTestingModule({
providers: [
KoboDownloadService,
{ provide: KoboDownloadHashRegistrationService, useValue: { record: register } },
{
provide: KoboDownloadRepository,
useValue: {
findBook: vi.fn().mockResolvedValue({ id: 11, primaryFileId: 22 }),
findPrimaryFile: vi
.fn()
.mockResolvedValue({ id: 22, format: 'epub', absolutePath: path, sizeBytes: original.length, fileHash: 'b'.repeat(32) }),
registerDeliveredHash: register,
},
},
{ provide: KepubConversionService, useValue: { getKepubPath: vi.fn().mockResolvedValue(path) } },
{
provide: KoboSettingsService,
useValue: { getSettings: vi.fn().mockResolvedValue({ convertToKepub: true, kepubConversionLimitMb: 10 }) },
},
{ provide: KoboBookAccessService, useValue: { assertBookAccessible: vi.fn().mockResolvedValue(undefined) } },
{ provide: AudiolessEpubService, useValue: {} },
],
}).compile();
let consume!: Promise<Buffer>;
const reply = {
header: vi.fn().mockReturnThis(),
type: vi.fn().mockReturnThis(),
send: vi.fn((stream: ReadStream) => {
consume = (async () => {
const chunks: Buffer[] = [];
for await (const chunk of stream) chunks.push(Buffer.from(chunk));
return Buffer.concat(chunks);
})();
}),
};
await module.get(KoboDownloadService).streamBook(7, 11, reply as never);
await expect(consume).resolves.toEqual(original);
expect(register).toHaveBeenCalledExactlyOnceWith(22, originalHash);
expect(reply.header).toHaveBeenCalledWith('Content-Length', original.length);
expect(await readFile(path)).toEqual(replacement);
expect(await computeFileHash(path)).not.toBe(originalHash);
await module.close();
} finally {
await rm(root, { recursive: true, force: true });
}
});
});
server/src/modules/kobo/services/kobo-download-hash-backfill.service.test.ts
+187
View File
@@ -0,0 +1,187 @@
import { Test } from '@nestjs/testing';
import { mkdir, mkdtemp, rm, symlink, writeFile } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import type { Mock } from 'vitest';
import { computeFileHash } from '../../../common/utils/file-hash.utils';
import { storageConfig } from '../../../config/config';
import { AppSettingsService } from '../../app-settings/app-settings.service';
import { KoboDownloadRepository } from '../kobo-download.repository';
import { KoboDownloadHashBackfillService } from './kobo-download-hash-backfill.service';
const SOURCE_HASH = 'a'.repeat(32);
const KEY = 'kobo_download_hash_backfill';
describe('KoboDownloadHashBackfillService', () => {
let root: string;
let cache: string;
let service: KoboDownloadHashBackfillService;
let repository: {
findCachedSourceFileId: Mock<KoboDownloadRepository['findCachedSourceFileId']>;
registerDeliveredHash: Mock<KoboDownloadRepository['registerDeliveredHash']>;
};
let settings: { getValue: ReturnType<typeof vi.fn>; setValue: ReturnType<typeof vi.fn> };
beforeEach(async () => {
root = await mkdtemp(join(tmpdir(), 'kobo-hash-backfill-'));
cache = join(root, '.kepub-cache');
repository = { findCachedSourceFileId: vi.fn().mockResolvedValue(22), registerDeliveredHash: vi.fn().mockResolvedValue(undefined) };
settings = { getValue: vi.fn().mockResolvedValue(null), setValue: vi.fn().mockResolvedValue(undefined) };
const module = await Test.createTestingModule({
providers: [
KoboDownloadHashBackfillService,
{ provide: KoboDownloadRepository, useValue: repository },
{ provide: AppSettingsService, useValue: settings },
{ provide: storageConfig.KEY, useValue: { appDataPath: root } },
],
}).compile();
service = module.get(KoboDownloadHashBackfillService);
});
afterEach(async () => {
await rm(root, { recursive: true, force: true });
});
async function cachedFile(book = '11', name = `${SOURCE_HASH}.kepub.epub`, content = 'Converted KEPUB bytes') {
const directory = join(cache, book);
await mkdir(directory, { recursive: true });
const path = join(directory, name);
await writeFile(path, content);
return path;
}
it('recovers actual cached bytes against a verified source and marks completion', async () => {
const path = await cachedFile();
await expect(service.run()).resolves.toEqual({ registered: 1, skipped: 0, failed: 0 });
expect(repository.findCachedSourceFileId).toHaveBeenCalledExactlyOnceWith(11, SOURCE_HASH);
expect(repository.registerDeliveredHash).toHaveBeenCalledExactlyOnceWith(22, await computeFileHash(path));
expect(repository.registerDeliveredHash).not.toHaveBeenCalledWith(22, SOURCE_HASH);
expect(settings.setValue).toHaveBeenCalledWith(KEY, '1');
});
it.each(['-hyph', '-noaudio-v1', '-noaudio-v1-hyph', '-noaudio-v2-hyph'])(
'recovers the %s variant using its original source hash',
async (suffix) => {
const path = await cachedFile('11', `${SOURCE_HASH}${suffix}.kepub.epub`);
await service.run();
expect(repository.findCachedSourceFileId).toHaveBeenCalledWith(11, SOURCE_HASH);
expect(repository.registerDeliveredHash).toHaveBeenCalledWith(22, await computeFileHash(path));
},
);
it('does no filesystem or repository work after completed recovery', async () => {
settings.getValue.mockResolvedValue('1');
await cachedFile();
await expect(service.run()).resolves.toEqual({ registered: 0, skipped: 0, failed: 0 });
expect(repository.findCachedSourceFileId).not.toHaveBeenCalled();
expect(settings.setValue).not.toHaveBeenCalled();
});
it('runs for an older recovery version', async () => {
settings.getValue.mockResolvedValue('0');
await cachedFile();
await service.run();
expect(repository.registerDeliveredHash).toHaveBeenCalledTimes(1);
});
it('completes safely when there is no conversion cache', async () => {
await expect(service.run()).resolves.toEqual({ registered: 0, skipped: 0, failed: 0 });
expect(settings.setValue).toHaveBeenCalledWith(KEY, '1');
});
it.each([
'nohash.kepub.epub',
'bad.kepub.epub',
`${SOURCE_HASH}-hyph-noaudio-v1.kepub.epub`,
`${SOURCE_HASH}.epub`,
`${SOURCE_HASH}-noaudio-v0.kepub.epub`,
])('skips unverifiable cache keys (%s)', async (name) => {
await cachedFile('11', name);
await expect(service.run()).resolves.toEqual({ registered: 0, skipped: 1, failed: 0 });
expect(repository.findCachedSourceFileId).not.toHaveBeenCalled();
});
it.each(['0', '01', '-1', '2147483648', '9007199254740992', 'other'])('skips invalid book directories (%s)', async (book) => {
await cachedFile(book);
await service.run();
expect(repository.findCachedSourceFileId).not.toHaveBeenCalled();
});
it('ignores unfinished conversion directories and non-file entries', async () => {
await mkdir(join(cache, '11', '.conversion-unfinished'), { recursive: true });
await mkdir(join(cache, '11', `${SOURCE_HASH}.kepub.epub`));
await service.run();
expect(repository.findCachedSourceFileId).not.toHaveBeenCalled();
expect(repository.registerDeliveredHash).not.toHaveBeenCalled();
});
it('skips ambiguous or missing source identities without guessing', async () => {
repository.findCachedSourceFileId.mockResolvedValue(null);
await cachedFile();
await expect(service.run()).resolves.toEqual({ registered: 0, skipped: 1, failed: 0 });
expect(repository.registerDeliveredHash).not.toHaveBeenCalled();
});
it('rejects a cache root symlink outside the configured data directory', async () => {
const outside = join(root, 'outside');
await mkdir(outside);
await symlink(outside, cache);
await expect(service.run()).rejects.toThrow('KEPUB cache must not be a symbolic link');
expect(settings.setValue).not.toHaveBeenCalled();
expect(repository.registerDeliveredHash).not.toHaveBeenCalled();
});
it('does not follow book-directory or cache-file symlinks', async () => {
const outside = join(root, 'outside');
await mkdir(outside);
await mkdir(join(cache, '12'), { recursive: true });
await writeFile(join(outside, `${SOURCE_HASH}.kepub.epub`), 'Outside bytes');
await symlink(outside, join(cache, '11'));
await symlink(join(outside, `${SOURCE_HASH}.kepub.epub`), join(cache, '12', `${SOURCE_HASH}.kepub.epub`));
await service.run();
expect(repository.findCachedSourceFileId).not.toHaveBeenCalled();
expect(repository.registerDeliveredHash).not.toHaveBeenCalled();
});
it('reports individual failures, continues with other artifacts, and leaves recovery retryable', async () => {
await cachedFile('11');
await cachedFile('12');
repository.registerDeliveredHash.mockRejectedValueOnce(new Error('Database unavailable'));
await expect(service.run()).resolves.toEqual({ registered: 1, skipped: 0, failed: 1 });
expect(settings.setValue).not.toHaveBeenCalled();
await expect(service.run()).resolves.toEqual({ registered: 2, skipped: 0, failed: 0 });
expect(settings.setValue).toHaveBeenCalledWith(KEY, '1');
});
it('does not mark recovery complete when source lookup fails', async () => {
await cachedFile();
repository.findCachedSourceFileId.mockRejectedValue(new Error('Lookup failed'));
await expect(service.run()).resolves.toEqual({ registered: 0, skipped: 0, failed: 1 });
expect(settings.setValue).not.toHaveBeenCalled();
});
it('keeps work sequential across many cache files and coalesces concurrent recovery calls', async () => {
for (let book = 1; book <= 30; book++) await cachedFile(String(book));
let active = 0;
let peak = 0;
repository.registerDeliveredHash.mockImplementation(async () => {
active++;
peak = Math.max(peak, active);
await Promise.resolve();
active--;
});
const first = service.run();
expect(service.run()).toBe(first);
await expect(first).resolves.toEqual({ registered: 30, skipped: 0, failed: 0 });
expect(peak).toBe(1);
expect(settings.getValue).toHaveBeenCalledTimes(1);
});
it('propagates failure to save completion so recovery can be retried', async () => {
await cachedFile();
settings.setValue.mockRejectedValueOnce(new Error('Marker write failed'));
await expect(service.run()).rejects.toThrow('Marker write failed');
await expect(service.run()).resolves.toEqual({ registered: 1, skipped: 0, failed: 0 });
});
});
server/src/modules/kobo/services/kobo-download-hash-backfill.service.ts
+110
View File
@@ -0,0 +1,110 @@
import { Inject, Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
import type { ConfigType } from '@nestjs/config';
import { opendir, realpath } from 'fs/promises';
import { join } from 'path';
import { computeFileHash } from '../../../common/utils/file-hash.utils';
import { sanitizeLogValue } from '../../../common/utils/log-sanitize.utils';
import { storageConfig } from '../../../config/config';
import { AppSettingsService } from '../../app-settings/app-settings.service';
import { KoboDownloadRepository } from '../kobo-download.repository';
const SETTING_KEY = 'kobo_download_hash_backfill';
const VERSION = 1;
const EVENT = 'kobo.download_hash_backfill';
const CACHE_FILE = /^([0-9a-f]{32})(?:-noaudio-v[1-9]\d*)?(?:-hyph)?\.kepub\.epub$/;
@Injectable()
export class KoboDownloadHashBackfillService implements OnApplicationBootstrap {
private readonly logger = new Logger(KoboDownloadHashBackfillService.name);
private running: Promise<{ registered: number; skipped: number; failed: number }> | null = null;
constructor(
private readonly repository: KoboDownloadRepository,
private readonly appSettings: AppSettingsService,
@Inject(storageConfig.KEY) private readonly storage: ConfigType<typeof storageConfig>,
) {}
onApplicationBootstrap(): void {
const startedAt = Date.now();
void this.run().catch((error: unknown) => this.logFailure(error, startedAt));
}
run(): Promise<{ registered: number; skipped: number; failed: number }> {
if (this.running) return this.running;
this.running = this.recover().finally(() => {
this.running = null;
});
return this.running;
}
private async recover(): Promise<{ registered: number; skipped: number; failed: number }> {
const totals = { registered: 0, skipped: 0, failed: 0 };
const marker = await this.appSettings.getValue(SETTING_KEY);
if (marker === String(VERSION)) return totals;
const startedAt = Date.now();
this.logger.log(`[${EVENT}] [start] version=${VERSION} - cached download identity recovery started`);
try {
const appDataPath = await realpath(this.storage.appDataPath);
const cachePath = join(appDataPath, '.kepub-cache');
if ((await realpath(cachePath)) !== cachePath) throw new Error('KEPUB cache must not be a symbolic link');
const directories = await opendir(cachePath);
// Streaming directory iterators and sequential work keep memory and database load bounded.
for await (const directory of directories) {
const bookId = Number(directory.name);
if (!directory.isDirectory() || !/^[1-9]\d*$/.test(directory.name) || !Number.isSafeInteger(bookId) || bookId > 2_147_483_647) {
totals.skipped++;
continue;
}
try {
const bookPath = join(cachePath, directory.name);
if ((await realpath(bookPath)) !== bookPath) {
totals.skipped++;
continue;
}
const files = await opendir(bookPath);
for await (const file of files) {
const match = CACHE_FILE.exec(file.name);
if (!file.isFile() || !match) {
totals.skipped++;
continue;
}
try {
const filePath = join(bookPath, file.name);
if ((await realpath(filePath)) !== filePath) {
totals.skipped++;
continue;
}
const fileId = await this.repository.findCachedSourceFileId(bookId, match[1]!);
if (fileId === null) {
totals.skipped++;
continue;
}
await this.repository.registerDeliveredHash(fileId, await computeFileHash(filePath));
totals.registered++;
} catch (error) {
totals.failed++;
this.logFailure(error, startedAt, bookId);
}
}
} catch (error) {
totals.failed++;
this.logFailure(error, startedAt, bookId);
}
}
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
}
if (totals.failed === 0) await this.appSettings.setValue(SETTING_KEY, String(VERSION));
this.logger.log(
`[${EVENT}] [end] version=${VERSION} durationMs=${Date.now() - startedAt} registered=${totals.registered} skipped=${totals.skipped} failed=${totals.failed} - cached download identity recovery completed`,
);
return totals;
}
private logFailure(error: unknown, startedAt: number, bookId?: number): void {
this.logger.warn(
`[${EVENT}] [fail] version=${VERSION} bookId=${bookId ?? 'none'} durationMs=${Date.now() - startedAt} errorClass=${error instanceof Error ? error.name : 'UnknownError'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - cached download identity recovery failed`,
);
}
}
server/src/modules/kobo/services/kobo-download-hash-registration.service.test.ts
+237
View File
@@ -0,0 +1,237 @@
import { Logger } from '@nestjs/common';
import { Test } from '@nestjs/testing';
import { mkdir, mkdtemp, opendir, readFile, readdir, rm, symlink, writeFile } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { storageConfig } from '../../../config/config';
import { KoboDownloadRepository } from '../kobo-download.repository';
import { KoboDownloadHashRegistrationService } from './kobo-download-hash-registration.service';
const HASH = 'a'.repeat(32);
vi.mock('fs/promises', async (importOriginal) => {
const actual = await importOriginal<typeof import('fs/promises')>();
return { ...actual, opendir: vi.fn(actual.opendir) };
});
describe('KoboDownloadHashRegistrationService', () => {
let root: string;
let directory: string;
let repository: { registerDeliveredHash: ReturnType<typeof vi.fn<(fileId: number, hash: string) => Promise<void>>> };
let service: KoboDownloadHashRegistrationService;
let warnings: ReturnType<typeof vi.spyOn>;
let services: KoboDownloadHashRegistrationService[];
async function createService() {
const module = await Test.createTestingModule({
providers: [
KoboDownloadHashRegistrationService,
{ provide: KoboDownloadRepository, useValue: repository },
{ provide: storageConfig.KEY, useValue: { appDataPath: root } },
],
}).compile();
const instance = module.get(KoboDownloadHashRegistrationService);
services.push(instance);
return instance;
}
beforeEach(async () => {
services = [];
vi.mocked(opendir).mockClear();
root = await mkdtemp(join(tmpdir(), 'kobo-hash-registration-'));
directory = join(root, '.kobo-download-hashes');
repository = { registerDeliveredHash: vi.fn().mockResolvedValue(undefined) };
service = await createService();
warnings = vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
});
afterEach(async () => {
await Promise.all(services.map((instance) => instance.onModuleDestroy()));
warnings.mockRestore();
await rm(root, { recursive: true, force: true });
});
it('registers immediately without creating a retry directory when the database works', async () => {
await service.record(22, HASH);
expect(repository.registerDeliveredHash).toHaveBeenCalledExactlyOnceWith(22, HASH);
await expect(readdir(directory)).rejects.toMatchObject({ code: 'ENOENT' });
expect(warnings).not.toHaveBeenCalled();
});
it('persists a failed registration without book bytes, paths or metadata', async () => {
repository.registerDeliveredHash.mockRejectedValue(new Error('Database unavailable'));
await expect(service.record(22, HASH)).resolves.toBeUndefined();
expect(await readdir(directory)).toEqual([`22-${HASH}`]);
expect(warnings).toHaveBeenCalledWith(expect.stringContaining('[fail]'));
});
it('deduplicates concurrent deliveries of the same identity', async () => {
repository.registerDeliveredHash.mockRejectedValue(new Error('Database unavailable'));
await Promise.all(Array.from({ length: 20 }, () => service.record(22, HASH)));
expect(await readdir(directory)).toEqual([`22-${HASH}`]);
});
it('retains different source files and variants independently', async () => {
repository.registerDeliveredHash.mockRejectedValue(new Error('Database unavailable'));
await service.record(22, HASH);
await service.record(33, HASH);
await service.record(22, 'b'.repeat(32));
expect((await readdir(directory)).sort()).toEqual([`22-${HASH}`, `22-${'b'.repeat(32)}`, `33-${HASH}`]);
});
it('recovers after restart even if the downloaded temporary archive is gone', async () => {
repository.registerDeliveredHash.mockRejectedValueOnce(new Error('Database unavailable'));
await service.record(22, HASH);
const restarted = await createService();
await restarted.replay();
expect(repository.registerDeliveredHash).toHaveBeenLastCalledWith(22, HASH);
expect(await readdir(directory)).toEqual([]);
});
it('retains pending identities through repeated failures and removes them after success', async () => {
repository.registerDeliveredHash.mockRejectedValue(new Error('Database unavailable'));
await service.record(22, HASH);
await service.replay();
await service.replay();
expect(await readdir(directory)).toEqual([`22-${HASH}`]);
repository.registerDeliveredHash.mockResolvedValue(undefined);
await service.replay();
expect(await readdir(directory)).toEqual([]);
});
it('coalesces overlapping replay requests', async () => {
await mkdir(directory);
await writeFile(join(directory, `22-${HASH}`), '');
let resolve!: () => void;
repository.registerDeliveredHash.mockImplementation(
() =>
new Promise<void>((done) => {
resolve = done;
}),
);
const first = service.replay();
expect(service.replay()).toBe(first);
await vi.waitFor(() => expect(repository.registerDeliveredHash).toHaveBeenCalledTimes(1));
resolve();
await first;
expect(await readdir(directory)).toEqual([]);
});
it('bounds replay work and continues the remaining entries on the next tick', async () => {
await mkdir(directory);
for (let id = 1; id <= 1005; id++) await writeFile(join(directory, `${id}-${HASH}`), '');
await service.replay();
expect(repository.registerDeliveredHash).toHaveBeenCalledTimes(1000);
expect(await readdir(directory)).toHaveLength(5);
await service.replay();
expect(repository.registerDeliveredHash).toHaveBeenCalledTimes(1005);
expect(await readdir(directory)).toEqual([]);
});
it('removes malformed files and symlinks without following them or deleting directories', async () => {
await mkdir(directory);
for (const name of ['0-' + HASH, '2147483648-' + HASH, '22-not-a-hash', '-22-' + HASH]) await writeFile(join(directory, name), '');
await mkdir(join(directory, `22-${HASH}`));
await symlink(join(root, 'missing'), join(directory, `33-${HASH}`));
await service.replay();
expect(repository.registerDeliveredHash).not.toHaveBeenCalled();
expect(await readdir(directory)).toEqual([`22-${HASH}`]);
expect(warnings).toHaveBeenCalledWith(expect.stringContaining('removedEntries=5 skippedEntries=1'));
});
it.each(['file', 'directory', 'symlink'])('makes bounded forward progress past more than a batch of invalid %s entries', async (kind) => {
await mkdir(directory);
const target = join(root, 'outside-entry');
await writeFile(target, 'preserve outside data');
for (let i = 0; i < 1001; i++) {
const path = join(directory, `junk-${i}`);
if (kind === 'file') await writeFile(path, '');
else if (kind === 'directory') await mkdir(path);
else await symlink(target, path);
}
const validName = `22-${HASH}`;
await writeFile(join(directory, validName), '');
const entries = (await readdir(directory, { withFileTypes: true })).sort((a, b) => Number(a.name === validName) - Number(b.name === validName));
const cursor = { read: vi.fn(() => Promise.resolve(entries.shift() ?? null)), close: vi.fn().mockResolvedValue(undefined) };
vi.mocked(opendir).mockResolvedValueOnce(cursor as never);
await service.replay();
expect(cursor.read).toHaveBeenCalledTimes(1000);
expect(repository.registerDeliveredHash).not.toHaveBeenCalled();
expect(cursor.close).not.toHaveBeenCalled();
await service.replay();
expect(repository.registerDeliveredHash).toHaveBeenCalledExactlyOnceWith(22, HASH);
expect(cursor.read).toHaveBeenCalledTimes(1003);
expect(cursor.close).toHaveBeenCalledTimes(1);
expect(opendir).toHaveBeenCalledTimes(1);
expect(await readFile(target, 'utf8')).toBe('preserve outside data');
expect(await readdir(directory)).toHaveLength(kind === 'directory' ? 1001 : 0);
});
it('continues past a failed identity and retries it on the next complete pass', async () => {
await mkdir(directory);
await writeFile(join(directory, `22-${HASH}`), '');
await writeFile(join(directory, `33-${HASH}`), '');
const entries = (await readdir(directory, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name));
vi.mocked(opendir).mockResolvedValueOnce({ read: vi.fn(() => Promise.resolve(entries.shift() ?? null)), close: vi.fn() } as never);
repository.registerDeliveredHash.mockRejectedValueOnce(new Error('Library locked'));
await service.replay();
expect(await readdir(directory)).toHaveLength(2);
await service.replay();
expect(repository.registerDeliveredHash).toHaveBeenLastCalledWith(33, HASH);
expect(await readdir(directory)).toEqual([`22-${HASH}`]);
await service.replay();
expect(repository.registerDeliveredHash).toHaveBeenLastCalledWith(22, HASH);
expect(await readdir(directory)).toEqual([]);
});
it('closes a partial traversal on shutdown and does not reopen it', async () => {
await mkdir(directory);
const entries = Array.from({ length: 1001 }, () => ({ name: 'directory', isFile: () => false, isSymbolicLink: () => false }));
const cursor = { read: vi.fn(() => Promise.resolve(entries.shift() ?? null)), close: vi.fn().mockResolvedValue(undefined) };
vi.mocked(opendir).mockResolvedValueOnce(cursor as never);
await service.replay();
await service.onModuleDestroy();
await service.replay();
expect(cursor.read).toHaveBeenCalledTimes(1000);
expect(cursor.close).toHaveBeenCalledTimes(1);
expect(opendir).toHaveBeenCalledTimes(1);
});
it('does not write through a symlink retry directory and still allows the download', async () => {
const outside = join(root, 'outside');
await mkdir(outside);
await symlink(outside, directory);
repository.registerDeliveredHash.mockRejectedValue(new Error('Database unavailable'));
await expect(service.record(22, HASH)).resolves.toBeUndefined();
await service.replay();
expect(await readdir(outside)).toEqual([]);
expect(warnings).toHaveBeenCalledWith(expect.stringContaining('symbolic link'));
});
it('allows downloads when both database and retry storage are unavailable and logs both failures', async () => {
await writeFile(directory, 'not a directory');
repository.registerDeliveredHash.mockRejectedValue(new Error('Database unavailable'));
await expect(service.record(22, HASH)).resolves.toBeUndefined();
expect(warnings).toHaveBeenCalledTimes(2);
expect(warnings).toHaveBeenLastCalledWith(expect.stringContaining('identity retry could not be persisted'));
});
it.each([
[0, HASH],
[22, '../unsafe'],
[2147483648, HASH],
])('does not persist invalid identity %s/%s', async (id, hash) => {
repository.registerDeliveredHash.mockRejectedValue(new Error('Database unavailable'));
await service.record(id as number, hash as string);
await expect(readdir(directory)).rejects.toMatchObject({ code: 'ENOENT' });
});
it('tolerates startup with no pending directory', async () => {
service.onApplicationBootstrap();
await service.replay();
expect(warnings).not.toHaveBeenCalled();
expect(repository.registerDeliveredHash).not.toHaveBeenCalled();
});
});
server/src/modules/kobo/services/kobo-download-hash-registration.service.ts
+152
View File
@@ -0,0 +1,152 @@
import { Inject, Injectable, Logger, OnApplicationBootstrap, OnModuleDestroy } from '@nestjs/common';
import type { ConfigType } from '@nestjs/config';
import { Interval } from '@nestjs/schedule';
import { mkdir, open, opendir, realpath, unlink } from 'fs/promises';
import type { Dir } from 'fs';
import { join } from 'path';
import { sanitizeLogValue } from '../../../common/utils/log-sanitize.utils';
import { storageConfig } from '../../../config/config';
import { KoboDownloadRepository } from '../kobo-download.repository';
const EVENT = 'kobo.download_hash_registration';
const ENTRY = /^([1-9]\d*)-([0-9a-f]{32})$/;
const REPLAY_BATCH = 1000;
@Injectable()
export class KoboDownloadHashRegistrationService implements OnApplicationBootstrap, OnModuleDestroy {
private readonly logger = new Logger(KoboDownloadHashRegistrationService.name);
private replaying: Promise<void> | null = null;
private entries: Dir | null = null;
private stopped = false;
constructor(
private readonly repository: KoboDownloadRepository,
@Inject(storageConfig.KEY) private readonly storage: ConfigType<typeof storageConfig>,
) {}
onApplicationBootstrap(): void {
void this.replay();
}
async onModuleDestroy(): Promise<void> {
this.stopped = true;
await this.replaying;
await this.closeEntries();
}
async record(fileId: number, hash: string): Promise<void> {
const startedAt = Date.now();
try {
await this.repository.registerDeliveredHash(fileId, hash);
return;
} catch (error) {
this.logFailure(error, startedAt, fileId, 'database registration failed; queuing retry');
}
try {
if (!Number.isSafeInteger(fileId) || fileId <= 0 || fileId > 2_147_483_647 || !/^[0-9a-f]{32}$/.test(hash)) {
throw new Error('Invalid delivered identity');
}
const directory = await this.pendingDirectory(true);
// The immutable filename carries the identity, so recovery never needs the temporary book.
const entry = await open(join(directory, `${fileId}-${hash}`), 'wx', 0o600).catch((error: NodeJS.ErrnoException) => {
if (error.code === 'EEXIST') return null;
throw error;
});
if (entry) {
try {
await entry.sync();
} finally {
await entry.close();
}
}
const handle = await open(directory, 'r');
try {
await handle.sync();
} finally {
await handle.close();
}
} catch (error) {
this.logFailure(error, startedAt, fileId, 'identity retry could not be persisted');
}
}
@Interval(30_000)
replay(): Promise<void> {
if (this.replaying) return this.replaying;
if (this.stopped) return Promise.resolve();
this.replaying = this.replayPending().finally(() => {
this.replaying = null;
});
return this.replaying;
}
private async pendingDirectory(create: boolean): Promise<string> {
const root = await realpath(this.storage.appDataPath);
const directory = join(root, '.kobo-download-hashes');
if (create)
await mkdir(directory, { mode: 0o700 }).catch((error: NodeJS.ErrnoException) => {
if (error.code !== 'EEXIST') throw error;
});
if ((await realpath(directory)) !== directory) throw new Error('Hash retry directory must not be a symbolic link');
return directory;
}
private async replayPending(): Promise<void> {
const startedAt = Date.now();
let removedEntries = 0;
let skippedEntries = 0;
try {
const directory = await this.pendingDirectory(false);
// Retain the cursor between bounded batches so even directories and failed entries
// cannot permanently hide later identities. Failed identities remain for the next pass.
this.entries ??= await opendir(directory);
for (let scanned = 0; scanned < REPLAY_BATCH && !this.stopped; scanned++) {
const entry = await this.entries.read();
if (!entry) {
await this.closeEntries();
break;
}
const match = ENTRY.exec(entry.name);
const fileId = Number(match?.[1]);
if (!entry.isFile() || !match || !Number.isSafeInteger(fileId) || fileId > 2_147_483_647) {
if (entry.isFile() || entry.isSymbolicLink()) {
await this.removeEntry(directory, entry.name);
removedEntries++;
} else {
skippedEntries++;
}
continue;
}
await this.repository.registerDeliveredHash(fileId, match[2]!);
await this.removeEntry(directory, entry.name);
}
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') this.logFailure(error, startedAt, undefined, 'queued identity replay failed');
} finally {
if (removedEntries || skippedEntries) {
this.logger.warn(
`[${EVENT}] [end] durationMs=${Date.now() - startedAt} removedEntries=${removedEntries} skippedEntries=${skippedEntries} - invalid retry entries encountered`,
);
}
}
}
private async removeEntry(directory: string, name: string): Promise<void> {
await unlink(join(directory, name)).catch((error: NodeJS.ErrnoException) => {
if (error.code !== 'ENOENT') throw error;
});
}
private async closeEntries(): Promise<void> {
const entries = this.entries;
this.entries = null;
await entries?.close();
}
private logFailure(error: unknown, startedAt: number, fileId: number | undefined, message: string): void {
this.logger.warn(
`[${EVENT}] [fail] fileId=${fileId ?? 'none'} durationMs=${Date.now() - startedAt} errorClass=${error instanceof Error ? error.name : 'UnknownError'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - ${message}`,
);
}
}
server/src/modules/kobo/services/kobo-download.service.test.ts
+295 -336
View File
@@ -1,377 +1,336 @@
vi.mock('fs/promises', () => ({
stat: vi.fn(),
mkdtemp: vi.fn(),
rm: vi.fn(),
}));
vi.mock('fs/promises', () => ({ open: vi.fn(), stat: vi.fn(), mkdtemp: vi.fn(), rm: vi.fn() }));
vi.mock('fs', () => ({ createReadStream: vi.fn() }));
vi.mock('../../../common/utils/file-hash.utils', () => ({ computeFileHashFromHandle: vi.fn() }));
vi.mock('fs', () => ({
createReadStream: vi.fn(),
}));
import { NotFoundException } from '@nestjs/common';
import { ForbiddenException, InternalServerErrorException, NotFoundException } from '@nestjs/common';
import { Test } from '@nestjs/testing';
import { EventEmitter } from 'events';
import { createReadStream } from 'fs';
import { mkdtemp, rm, stat } from 'fs/promises';
import { mkdtemp, open, rm, stat } from 'fs/promises';
import { computeFileHashFromHandle } from '../../../common/utils/file-hash.utils';
import { AudiolessEpubService } from '../../book/audioless-epub.service';
import { KoboDownloadRepository } from '../kobo-download.repository';
import { KoboBookAccessService } from './kobo-book-access.service';
import { KepubConversionService } from './kepub-conversion.service';
import { KoboSettingsService } from './kobo-settings.service';
import { KoboDownloadHashRegistrationService } from './kobo-download-hash-registration.service';
import { KoboDownloadService } from './kobo-download.service';
const DELIVERED_HASH = 'a'.repeat(32);
const ORIGINAL_HASH = 'b'.repeat(32);
const TEMP_DIR = '/runtime-temp/kobo-download';
const SOURCE = '/books/source.epub';
const CONVERTED = '/app-data/.kepub-cache/11/converted.kepub.epub';
const STRIPPED = `${TEMP_DIR}/book.epub`;
const SOURCE_FILE = {
id: 22,
format: 'epub',
absolutePath: SOURCE,
fileHash: ORIGINAL_HASH,
sizeBytes: 5 * 1024 * 1024,
mediaOverlayAvailable: false,
};
const statMock = vi.mocked(stat);
const mkdtempMock = vi.mocked(mkdtemp);
const rmMock = vi.mocked(rm);
const createReadStreamMock = vi.mocked(createReadStream);
function makeReply() {
return {
header: vi.fn().mockReturnThis(),
type: vi.fn().mockReturnThis(),
send: vi.fn().mockReturnThis(),
};
}
const openMock = vi.mocked(open);
const hashMock = vi.mocked(computeFileHashFromHandle);
const readStreamMock = vi.mocked(createReadStream);
function makeDeps() {
return {
db: {
query: {
books: { findFirst: vi.fn() },
bookFiles: { findFirst: vi.fn() },
},
repository: {
findBook: vi.fn().mockResolvedValue({ id: 11, primaryFileId: 22 }),
findPrimaryFile: vi.fn().mockResolvedValue(SOURCE_FILE),
registerDeliveredHash: vi.fn().mockResolvedValue(undefined),
},
kepubConversionService: { getKepubPath: vi.fn() },
settingsService: { getSettings: vi.fn() },
bookAccessService: { assertBookAccessible: vi.fn() },
audiolessEpubService: { writeArchive: vi.fn() },
conversion: { getKepubPath: vi.fn().mockResolvedValue(CONVERTED) },
settings: { getSettings: vi.fn().mockResolvedValue({ convertToKepub: true, forceEnableHyphenation: false, kepubConversionLimitMb: 10 }) },
access: { assertBookAccessible: vi.fn().mockResolvedValue(undefined) },
audioless: { writeArchive: vi.fn().mockResolvedValue({ removedEntries: 9, sanitizedEntries: 9 }) },
};
}
function makeService(deps: ReturnType<typeof makeDeps>) {
return new KoboDownloadService(
deps.db as never,
deps.kepubConversionService as never,
deps.settingsService as never,
deps.bookAccessService as never,
deps.audiolessEpubService as never,
);
function makeReply() {
return { header: vi.fn().mockReturnThis(), type: vi.fn().mockReturnThis(), send: vi.fn().mockReturnThis() };
}
/** A narrated EPUB whose audio makes it far larger than the conversion limit. */
function narratedEpubFile() {
return {
id: 22,
format: 'epub',
absolutePath: '/books/narrated.epub',
fileHash: 'h1',
sizeBytes: 1_500 * 1024 * 1024,
mediaOverlayAvailable: true,
};
function deferred<T>() {
let resolve!: (value: T) => void;
const promise = new Promise<T>((done) => {
resolve = done;
});
return { promise, resolve };
}
function kepubSettings(overrides: Record<string, unknown> = {}) {
return {
convertToKepub: true,
forceEnableHyphenation: false,
kepubConversionLimitMb: 10,
twoWayProgressSync: false,
...overrides,
};
function errno(code: string) {
return Object.assign(new Error('Filesystem operation failed'), { code });
}
describe('KoboDownloadService', () => {
beforeEach(() => {
vi.clearAllMocks();
});
let deps: ReturnType<typeof makeDeps>;
let service: KoboDownloadService;
let reply: ReturnType<typeof makeReply>;
let stream: EventEmitter & { destroy: ReturnType<typeof vi.fn> };
let handle: { stat: ReturnType<typeof vi.fn>; close: ReturnType<typeof vi.fn>; createReadStream: ReturnType<typeof vi.fn> };
it('throws when target book does not exist', async () => {
const deps = makeDeps();
deps.db.query.books.findFirst.mockResolvedValue(null);
const service = makeService(deps);
await expect(service.streamBook(7, 11, makeReply() as never)).rejects.toThrow(NotFoundException);
});
it('throws when book file is missing after access check', async () => {
const deps = makeDeps();
deps.db.query.books.findFirst.mockResolvedValue({ id: 11, primaryFileId: 22 });
deps.db.query.bookFiles.findFirst.mockResolvedValue(null);
deps.bookAccessService.assertBookAccessible.mockResolvedValue(undefined);
const service = makeService(deps);
await expect(service.streamBook(7, 11, makeReply() as never)).rejects.toThrow('No file found for this book');
expect(deps.bookAccessService.assertBookAccessible).toHaveBeenCalledWith(7, 11);
});
it('uses direct file streaming for pdf files', async () => {
const deps = makeDeps();
deps.db.query.books.findFirst.mockResolvedValue({ id: 11, primaryFileId: 22 });
deps.db.query.bookFiles.findFirst.mockResolvedValue({
id: 22,
format: 'pdf',
absolutePath: '/books/file.pdf',
fileHash: 'hash',
sizeBytes: 100,
});
deps.bookAccessService.assertBookAccessible.mockResolvedValue(undefined);
const service = makeService(deps);
const streamFileSpy = vi.spyOn(service as any, 'streamFile').mockResolvedValue(undefined);
await service.streamBook(7, 11, makeReply() as never);
expect(streamFileSpy).toHaveBeenCalledWith('/books/file.pdf', 22, 'pdf', expect.anything());
});
it('streams native kepub files without conversion', async () => {
const deps = makeDeps();
deps.db.query.books.findFirst.mockResolvedValue({ id: 11, primaryFileId: 22 });
deps.db.query.bookFiles.findFirst.mockResolvedValue({
id: 22,
format: 'kepub',
absolutePath: '/books/file.kepub.epub',
fileHash: 'h1',
sizeBytes: 5 * 1024 * 1024,
});
deps.bookAccessService.assertBookAccessible.mockResolvedValue(undefined);
const service = makeService(deps);
const streamFileSpy = vi.spyOn(service as any, 'streamFile').mockResolvedValue(undefined);
await service.streamBook(7, 11, makeReply() as never);
expect(streamFileSpy).toHaveBeenCalledWith('/books/file.kepub.epub', 22, 'kepub.epub', expect.anything());
expect(deps.settingsService.getSettings).not.toHaveBeenCalled();
expect(deps.kepubConversionService.getKepubPath).not.toHaveBeenCalled();
});
it('converts epub to kepub when enabled and within conversion size limit', async () => {
const deps = makeDeps();
deps.db.query.books.findFirst.mockResolvedValue({ id: 11, primaryFileId: 22 });
deps.db.query.bookFiles.findFirst.mockResolvedValue({
id: 22,
format: 'epub',
absolutePath: '/books/file.epub',
fileHash: 'h1',
sizeBytes: 5 * 1024 * 1024,
});
deps.bookAccessService.assertBookAccessible.mockResolvedValue(undefined);
deps.settingsService.getSettings.mockResolvedValue({
convertToKepub: true,
forceEnableHyphenation: true,
kepubConversionLimitMb: 10,
twoWayProgressSync: false,
});
const service = makeService(deps);
const streamKepubSpy = vi.spyOn(service as any, 'streamKepub').mockResolvedValue(undefined);
await service.streamBook(7, 11, makeReply() as never);
expect(streamKepubSpy).toHaveBeenCalledWith('/books/file.epub', 'h1', 11, 22, true, false, expect.anything(), undefined);
});
it('strips narration before the size check so a read-along book still converts', async () => {
const deps = makeDeps();
deps.db.query.books.findFirst.mockResolvedValue({ id: 11, primaryFileId: 22 });
deps.db.query.bookFiles.findFirst.mockResolvedValue(narratedEpubFile());
deps.bookAccessService.assertBookAccessible.mockResolvedValue(undefined);
deps.settingsService.getSettings.mockResolvedValue(kepubSettings());
deps.audiolessEpubService.writeArchive.mockResolvedValue({ removedEntries: 9, sanitizedEntries: 9 });
mkdtempMock.mockResolvedValue('/tmp/kobo-epub' as never);
// The rebuilt copy is a couple of megabytes, well under the ten megabyte limit the original
// 1.5 GB archive could never meet.
statMock.mockResolvedValueOnce({ size: 2 * 1024 * 1024 } as never);
const service = makeService(deps);
const streamKepubSpy = vi.spyOn(service as any, 'streamKepub').mockResolvedValue(undefined);
await service.streamBook(7, 11, makeReply() as never);
expect(deps.audiolessEpubService.writeArchive).toHaveBeenCalledWith('/books/narrated.epub', '/tmp/kobo-epub/book.epub');
expect(streamKepubSpy).toHaveBeenCalledWith('/tmp/kobo-epub/book.epub', 'h1', 11, 22, false, true, expect.anything(), expect.any(Function));
});
it('streams the stripped copy when kepub conversion is turned off', async () => {
const deps = makeDeps();
deps.db.query.books.findFirst.mockResolvedValue({ id: 11, primaryFileId: 22 });
deps.db.query.bookFiles.findFirst.mockResolvedValue(narratedEpubFile());
deps.bookAccessService.assertBookAccessible.mockResolvedValue(undefined);
deps.settingsService.getSettings.mockResolvedValue(kepubSettings({ convertToKepub: false }));
deps.audiolessEpubService.writeArchive.mockResolvedValue({ removedEntries: 9, sanitizedEntries: 9 });
mkdtempMock.mockResolvedValue('/tmp/kobo-epub' as never);
statMock.mockResolvedValueOnce({ size: 2 * 1024 * 1024 } as never);
const service = makeService(deps);
const streamFileSpy = vi.spyOn(service as any, 'streamFile').mockResolvedValue(undefined);
await service.streamBook(7, 11, makeReply() as never);
expect(streamFileSpy).toHaveBeenCalledWith('/tmp/kobo-epub/book.epub', 22, 'epub', expect.anything(), expect.any(Function));
});
it('serves the original archive and cleans up when the rebuild fails', async () => {
const deps = makeDeps();
deps.db.query.books.findFirst.mockResolvedValue({ id: 11, primaryFileId: 22 });
deps.db.query.bookFiles.findFirst.mockResolvedValue(narratedEpubFile());
deps.bookAccessService.assertBookAccessible.mockResolvedValue(undefined);
deps.settingsService.getSettings.mockResolvedValue(kepubSettings({ convertToKepub: false }));
deps.audiolessEpubService.writeArchive.mockRejectedValue(new Error('corrupt archive'));
mkdtempMock.mockResolvedValue('/tmp/kobo-epub' as never);
rmMock.mockResolvedValue(undefined as never);
const service = makeService(deps);
const streamFileSpy = vi.spyOn(service as any, 'streamFile').mockResolvedValue(undefined);
await service.streamBook(7, 11, makeReply() as never);
expect(rmMock).toHaveBeenCalledWith('/tmp/kobo-epub', { recursive: true, force: true });
expect(streamFileSpy).toHaveBeenCalledWith('/books/narrated.epub', 22, 'epub', expect.anything());
});
it('leaves books without narration on the original path', async () => {
const deps = makeDeps();
deps.db.query.books.findFirst.mockResolvedValue({ id: 11, primaryFileId: 22 });
deps.db.query.bookFiles.findFirst.mockResolvedValue({
id: 22,
format: 'epub',
absolutePath: '/books/plain.epub',
fileHash: 'h1',
sizeBytes: 5 * 1024 * 1024,
mediaOverlayAvailable: false,
});
deps.bookAccessService.assertBookAccessible.mockResolvedValue(undefined);
deps.settingsService.getSettings.mockResolvedValue(kepubSettings({ convertToKepub: false }));
const service = makeService(deps);
const streamFileSpy = vi.spyOn(service as any, 'streamFile').mockResolvedValue(undefined);
await service.streamBook(7, 11, makeReply() as never);
expect(deps.audiolessEpubService.writeArchive).not.toHaveBeenCalled();
expect(streamFileSpy).toHaveBeenCalledWith('/books/plain.epub', 22, 'epub', expect.anything());
});
it('removes the rebuilt copy only once the response has finished reading it', async () => {
const deps = makeDeps();
const service = makeService(deps);
const reply = makeReply();
const listeners: Record<string, () => void> = {};
const stream = {
once: vi.fn((event: string, handler: () => void) => {
listeners[event] = handler;
}),
beforeEach(async () => {
vi.resetAllMocks();
deps = makeDeps();
reply = makeReply();
stream = Object.assign(new EventEmitter(), { destroy: vi.fn() });
handle = {
stat: vi.fn().mockImplementation(() => statMock('' as never)),
close: vi.fn().mockResolvedValue(undefined),
createReadStream: vi.fn().mockImplementation(() => readStreamMock(openMock.mock.calls.at(-1)![0] as never)),
};
statMock.mockResolvedValueOnce({ size: 2048 } as never);
createReadStreamMock.mockReturnValue(stream as never);
const cleanup = vi.fn().mockResolvedValue(undefined);
openMock.mockResolvedValue(handle as never);
statMock.mockResolvedValue({ size: 4096 } as never);
hashMock.mockResolvedValue(DELIVERED_HASH);
vi.mocked(mkdtemp).mockResolvedValue(TEMP_DIR as never);
vi.mocked(rm).mockResolvedValue(undefined);
readStreamMock.mockReturnValue(stream as never);
const module = await Test.createTestingModule({
providers: [
KoboDownloadService,
{ provide: KoboDownloadRepository, useValue: deps.repository },
{ provide: KoboDownloadHashRegistrationService, useValue: { record: deps.repository.registerDeliveredHash } },
{ provide: KepubConversionService, useValue: deps.conversion },
{ provide: KoboSettingsService, useValue: deps.settings },
{ provide: KoboBookAccessService, useValue: deps.access },
{ provide: AudiolessEpubService, useValue: deps.audioless },
],
}).compile();
service = module.get(KoboDownloadService);
});
await (service as any).streamFile('/tmp/kobo-epub/book.epub', 99, 'epub', reply, cleanup);
function setFile(patch: Record<string, unknown>) {
deps.repository.findPrimaryFile.mockResolvedValue({ ...SOURCE_FILE, ...patch });
}
// Every delivery branch runs the public method with real preparation/streaming logic.
it.each([
{ name: 'PDF', format: 'pdf', path: '/books/source.pdf', conversion: false, mime: 'application/pdf', extension: 'pdf' },
{
name: 'native KEPUB',
format: 'kepub',
path: '/books/source.kepub.epub',
conversion: false,
mime: 'application/epub+zip',
extension: 'kepub.epub',
},
{ name: 'converted EPUB', format: 'epub', path: SOURCE, conversion: true, mime: 'application/epub+zip', extension: 'kepub.epub' },
{ name: 'unknown format', format: 'other', path: '/books/source.other', conversion: false, mime: 'application/octet-stream', extension: 'other' },
{ name: 'uppercase EPUB', format: 'EPUB', path: SOURCE, conversion: true, mime: 'application/epub+zip', extension: 'kepub.epub' },
{ name: 'missing stored format', format: null, path: SOURCE, conversion: true, mime: 'application/epub+zip', extension: 'kepub.epub' },
])('registers the delivered bytes for $name', async ({ format, path, conversion, mime, extension }) => {
setFile({ format, absolutePath: path });
await service.streamBook(7, 11, reply as never);
const deliveredPath = conversion ? CONVERTED : path;
expect(openMock).toHaveBeenCalledExactlyOnceWith(deliveredPath, 'r');
expect(hashMock).toHaveBeenCalledExactlyOnceWith(handle);
expect(handle.createReadStream).toHaveBeenCalledWith({ start: 0 });
expect(deps.repository.registerDeliveredHash).toHaveBeenCalledExactlyOnceWith(22, DELIVERED_HASH);
expect(readStreamMock).toHaveBeenCalledExactlyOnceWith(deliveredPath);
expect(reply.send).toHaveBeenCalledWith(stream);
expect(cleanup).not.toHaveBeenCalled();
listeners.close();
expect(cleanup).toHaveBeenCalledTimes(1);
expect(reply.type).toHaveBeenCalledWith(mime);
expect(reply.header).toHaveBeenCalledWith('Content-Length', 4096);
expect(reply.header).toHaveBeenCalledWith('Content-Disposition', `attachment; filename="book-22.${extension}"`);
expect(deps.conversion.getKepubPath).toHaveBeenCalledTimes(conversion ? 1 : 0);
});
it('falls back to epub stream when conversion is disabled or over limit', async () => {
const deps = makeDeps();
deps.db.query.books.findFirst.mockResolvedValue({ id: 11, primaryFileId: 22 });
deps.db.query.bookFiles.findFirst.mockResolvedValue({
id: 22,
format: 'epub',
absolutePath: '/books/file.epub',
fileHash: null,
sizeBytes: 20 * 1024 * 1024,
});
deps.bookAccessService.assertBookAccessible.mockResolvedValue(undefined);
deps.settingsService.getSettings.mockResolvedValue({
convertToKepub: true,
forceEnableHyphenation: false,
kepubConversionLimitMb: 10,
twoWayProgressSync: false,
});
const service = makeService(deps);
const streamFileSpy = vi.spyOn(service as any, 'streamFile').mockResolvedValue(undefined);
await service.streamBook(7, 11, makeReply() as never);
expect(streamFileSpy).toHaveBeenCalledWith('/books/file.epub', 22, 'epub', expect.anything());
it.each([
{ name: 'conversion disabled', settings: { convertToKepub: false }, size: 1024 },
{ name: 'over size limit', settings: { convertToKepub: true }, size: 11 * 1024 * 1024 },
])('registers an original EPUB when $name', async ({ settings, size }) => {
deps.settings.getSettings.mockResolvedValue({ forceEnableHyphenation: false, kepubConversionLimitMb: 10, ...settings });
setFile({ sizeBytes: size });
await service.streamBook(7, 11, reply as never);
expect(deps.conversion.getKepubPath).not.toHaveBeenCalled();
expect(openMock).toHaveBeenCalledWith(SOURCE, 'r');
expect(deps.repository.registerDeliveredHash).toHaveBeenCalledWith(22, DELIVERED_HASH);
expect(readStreamMock).toHaveBeenCalledWith(SOURCE);
});
it('streamFile writes headers and stream payload and throws when source path is missing', async () => {
const deps = makeDeps();
const service = makeService(deps);
const reply = makeReply();
const stream = {} as never;
statMock.mockResolvedValueOnce({ size: 1234 } as never);
createReadStreamMock.mockReturnValue(stream);
await (service as any).streamFile('/books/book.epub', 99, 'epub', reply);
expect(reply.header).toHaveBeenCalledWith('Content-Length', 1234);
expect(reply.header).toHaveBeenCalledWith('Content-Disposition', 'attachment; filename="book-99.epub"');
expect(reply.type).toHaveBeenCalledWith('application/epub+zip');
expect(reply.send).toHaveBeenCalledWith(stream);
statMock.mockRejectedValueOnce(new Error('missing'));
await expect((service as any).streamFile('/books/missing.epub', 99, 'epub', reply)).rejects.toThrow(NotFoundException);
it.each([10 * 1024 * 1024, 0, null])('converts at the exact limit or without a known size (%s)', async (sizeBytes) => {
setFile({ sizeBytes });
await service.streamBook(7, 11, reply as never);
expect(openMock).toHaveBeenCalledWith(CONVERTED, 'r');
});
it('streamFile uses application/epub+zip for kepub.epub format', async () => {
const deps = makeDeps();
const service = makeService(deps);
const reply = makeReply();
statMock.mockResolvedValueOnce({ size: 4096 } as never);
createReadStreamMock.mockReturnValue({} as never);
await (service as any).streamFile('/cache/44/hash.kepub.epub', 55, 'kepub.epub', reply);
expect(reply.type).toHaveBeenCalledWith('application/epub+zip');
expect(reply.header).toHaveBeenCalledWith('Content-Disposition', 'attachment; filename="book-55.kepub.epub"');
});
it('streamFile falls back to application/octet-stream for unknown formats', async () => {
const deps = makeDeps();
const service = makeService(deps);
const reply = makeReply();
statMock.mockResolvedValueOnce({ size: 100 } as never);
createReadStreamMock.mockReturnValue({} as never);
await (service as any).streamFile('/books/book.xyz', 10, 'xyz', reply);
expect(reply.type).toHaveBeenCalledWith('application/octet-stream');
});
it('streamKepub streams the shared conversion path', async () => {
const deps = makeDeps();
const service = makeService(deps);
deps.kepubConversionService.getKepubPath.mockResolvedValue('/app-data/.kepub-cache/44/abc.kepub.epub');
const streamFileSpy = vi.spyOn(service as any, 'streamFile').mockResolvedValue(undefined);
await (service as any).streamKepub('/books/source.epub', 'abc', 44, 55, false, false, makeReply());
expect(deps.kepubConversionService.getKepubPath).toHaveBeenCalledWith({
sourcePath: '/books/source.epub',
fileHash: 'abc',
bookId: 44,
hyphenate: false,
it('passes original source identity and hyphenation to conversion but registers the resulting identity', async () => {
deps.settings.getSettings.mockResolvedValue({ convertToKepub: true, forceEnableHyphenation: true, kepubConversionLimitMb: 10 });
await service.streamBook(7, 11, reply as never);
expect(deps.conversion.getKepubPath).toHaveBeenCalledWith({
sourcePath: SOURCE,
fileHash: ORIGINAL_HASH,
bookId: 11,
hyphenate: true,
audioless: false,
});
expect(streamFileSpy).toHaveBeenCalledWith('/app-data/.kepub-cache/44/abc.kepub.epub', 55, 'kepub.epub', expect.anything(), undefined);
expect(deps.repository.registerDeliveredHash).not.toHaveBeenCalledWith(22, ORIGINAL_HASH);
});
it('streamKepub hands the stripped copy its own cache entry and cleanup', async () => {
const deps = makeDeps();
const service = makeService(deps);
const cleanup = vi.fn().mockResolvedValue(undefined);
deps.kepubConversionService.getKepubPath.mockResolvedValue('/app-data/.kepub-cache/44/abc-noaudio.kepub.epub');
const streamFileSpy = vi.spyOn(service as any, 'streamFile').mockResolvedValue(undefined);
await (service as any).streamKepub('/tmp/kobo-epub/book.epub', 'abc', 44, 55, false, true, makeReply(), cleanup);
expect(deps.kepubConversionService.getKepubPath).toHaveBeenCalledWith(expect.objectContaining({ audioless: true }));
// The temp rebuild outlives the conversion: whichever file is sent carries the cleanup.
expect(streamFileSpy).toHaveBeenCalledWith('/app-data/.kepub-cache/44/abc-noaudio.kepub.epub', 55, 'kepub.epub', expect.anything(), cleanup);
it('registers each cached delivery, including a different authorized user', async () => {
await service.streamBook(7, 11, reply as never);
await service.streamBook(8, 11, makeReply() as never);
expect(deps.access.assertBookAccessible.mock.calls).toEqual([
[7, 11],
[8, 11],
]);
expect(openMock.mock.calls).toEqual([
[CONVERTED, 'r'],
[CONVERTED, 'r'],
]);
expect(deps.repository.registerDeliveredHash.mock.calls).toEqual([
[22, DELIVERED_HASH],
[22, DELIVERED_HASH],
]);
});
it('streamKepub falls back when conversion fails', async () => {
const deps = makeDeps();
deps.kepubConversionService.getKepubPath.mockRejectedValue(new Error('convert failed'));
const service = makeService(deps);
const streamFileSpy = vi.spyOn(service as any, 'streamFile').mockResolvedValue(undefined);
it('uses a nohash conversion key without confusing it with the delivered hash', async () => {
setFile({ fileHash: null });
await service.streamBook(7, 11, reply as never);
expect(deps.conversion.getKepubPath).toHaveBeenCalledWith(expect.objectContaining({ fileHash: 'nohash' }));
expect(deps.repository.registerDeliveredHash).toHaveBeenCalledWith(22, DELIVERED_HASH);
});
await (service as any).streamKepub('/books/source.epub', 'hash', 44, 55, false, false, makeReply());
it.each([
{ name: 'conversion succeeds', enabled: true, oversized: false, failed: false, path: CONVERTED },
{ name: 'conversion disabled', enabled: false, oversized: false, failed: false, path: STRIPPED },
{ name: 'stripped copy over limit', enabled: true, oversized: true, failed: false, path: STRIPPED },
{ name: 'conversion fails', enabled: true, oversized: false, failed: true, path: STRIPPED },
])('registers the final narration-free file when $name', async ({ enabled, oversized, failed, path }) => {
setFile({ mediaOverlayAvailable: true, sizeBytes: 1500 * 1024 * 1024 });
deps.settings.getSettings.mockResolvedValue({ convertToKepub: enabled, forceEnableHyphenation: false, kepubConversionLimitMb: 10 });
statMock.mockResolvedValueOnce({ size: (oversized ? 11 : 2) * 1024 * 1024 } as never);
if (failed) deps.conversion.getKepubPath.mockRejectedValue(new Error('Conversion failed'));
await service.streamBook(7, 11, reply as never);
expect(deps.audioless.writeArchive).toHaveBeenCalledWith(SOURCE, STRIPPED);
expect(openMock).toHaveBeenCalledExactlyOnceWith(path, 'r');
expect(readStreamMock).toHaveBeenCalledWith(path);
expect(deps.repository.registerDeliveredHash).toHaveBeenCalledWith(22, DELIVERED_HASH);
expect(rm).not.toHaveBeenCalled();
stream.emit('close');
expect(rm).toHaveBeenCalledExactlyOnceWith(TEMP_DIR, { recursive: true, force: true });
if (enabled && !oversized)
expect(deps.conversion.getKepubPath).toHaveBeenCalledWith(expect.objectContaining({ sourcePath: STRIPPED, audioless: true }));
});
expect(streamFileSpy).toHaveBeenLastCalledWith('/books/source.epub', 55, 'epub', expect.anything(), undefined);
it.each([false, true])('falls back safely when narration stripping fails (conversion=%s)', async (convertToKepub) => {
setFile({ mediaOverlayAvailable: true });
deps.settings.getSettings.mockResolvedValue({ convertToKepub, forceEnableHyphenation: false, kepubConversionLimitMb: 10 });
deps.audioless.writeArchive.mockRejectedValue(new Error('Invalid archive'));
await service.streamBook(7, 11, reply as never);
expect(rm).toHaveBeenCalledExactlyOnceWith(TEMP_DIR, { recursive: true, force: true });
expect(openMock).toHaveBeenCalledWith(convertToKepub ? CONVERTED : SOURCE, 'r');
if (convertToKepub) expect(deps.conversion.getKepubPath).toHaveBeenCalledWith(expect.objectContaining({ sourcePath: SOURCE, audioless: false }));
stream.emit('close');
expect(rm).toHaveBeenCalledTimes(1);
});
it('registers the actual original EPUB if both transformations fail', async () => {
setFile({ mediaOverlayAvailable: true });
deps.audioless.writeArchive.mockRejectedValue(new Error('Invalid archive'));
deps.conversion.getKepubPath.mockRejectedValue(new Error('Conversion failed'));
await service.streamBook(7, 11, reply as never);
expect(openMock).toHaveBeenCalledExactlyOnceWith(SOURCE, 'r');
expect(readStreamMock).toHaveBeenCalledWith(SOURCE);
});
it('does not prepare or register a book that is missing', async () => {
deps.repository.findBook.mockResolvedValue(undefined);
await expect(service.streamBook(7, 11, reply as never)).rejects.toThrow(NotFoundException);
expect(deps.access.assertBookAccessible).not.toHaveBeenCalled();
expect(deps.repository.findPrimaryFile).not.toHaveBeenCalled();
expect(hashMock).not.toHaveBeenCalled();
});
it('does not inspect, convert, hash or register an inaccessible book', async () => {
deps.access.assertBookAccessible.mockRejectedValue(new ForbiddenException());
await expect(service.streamBook(7, 11, reply as never)).rejects.toThrow(ForbiddenException);
expect(deps.repository.findPrimaryFile).not.toHaveBeenCalled();
expect(deps.conversion.getKepubPath).not.toHaveBeenCalled();
expect(hashMock).not.toHaveBeenCalled();
expect(deps.repository.registerDeliveredHash).not.toHaveBeenCalled();
expect(reply.send).not.toHaveBeenCalled();
});
it('requires a content file belonging to the requested book', async () => {
deps.repository.findPrimaryFile.mockResolvedValue(undefined);
await expect(service.streamBook(7, 11, reply as never)).rejects.toThrow('No file found for this book');
expect(deps.repository.findPrimaryFile).toHaveBeenCalledWith(11, 22);
expect(hashMock).not.toHaveBeenCalled();
});
it('does not substitute another file when there is no primary file', async () => {
deps.repository.findBook.mockResolvedValue({ id: 11, primaryFileId: null });
deps.repository.findPrimaryFile.mockResolvedValue(undefined);
await expect(service.streamBook(7, 11, reply as never)).rejects.toThrow(NotFoundException);
expect(deps.repository.findPrimaryFile).toHaveBeenCalledWith(11, -1);
});
it('waits for hashing before registering or streaming', async () => {
const hashing = deferred<string>();
hashMock.mockReturnValue(hashing.promise);
const download = service.streamBook(7, 11, reply as never);
await vi.waitFor(() => expect(hashMock).toHaveBeenCalled());
expect(deps.repository.registerDeliveredHash).not.toHaveBeenCalled();
expect(reply.header).not.toHaveBeenCalled();
expect(readStreamMock).not.toHaveBeenCalled();
hashing.resolve(DELIVERED_HASH);
await download;
expect(reply.send).toHaveBeenCalled();
});
it('waits for identity persistence before sending headers or bytes', async () => {
const registration = deferred<void>();
deps.repository.registerDeliveredHash.mockReturnValue(registration.promise);
const download = service.streamBook(7, 11, reply as never);
await vi.waitFor(() => expect(deps.repository.registerDeliveredHash).toHaveBeenCalled());
expect(reply.header).not.toHaveBeenCalled();
expect(readStreamMock).not.toHaveBeenCalled();
registration.resolve();
await download;
expect(reply.send).toHaveBeenCalledWith(stream);
});
it.each(['ENOENT', 'ENOTDIR'])('reports a missing delivered file (%s) without a hash link', async (code) => {
hashMock.mockRejectedValue(errno(code));
await expect(service.streamBook(7, 11, reply as never)).rejects.toThrow(NotFoundException);
expect(deps.repository.registerDeliveredHash).not.toHaveBeenCalled();
expect(reply.send).not.toHaveBeenCalled();
});
it.each(['hash', 'stat', 'stream'])('does not silently deliver an unregistered fallback after a %s failure', async (stage) => {
const failure = new Error('Preparation failed');
if (stage === 'hash') hashMock.mockRejectedValue(failure);
if (stage === 'stat') statMock.mockRejectedValue(failure);
if (stage === 'stream')
readStreamMock.mockImplementation(() => {
throw failure;
});
await expect(service.streamBook(7, 11, reply as never)).rejects.toThrow(InternalServerErrorException);
expect(deps.conversion.getKepubPath).toHaveBeenCalledTimes(1);
expect(hashMock).toHaveBeenCalledExactlyOnceWith(handle);
expect(reply.send).not.toHaveBeenCalled();
expect(handle.close).toHaveBeenCalledTimes(1);
});
it('continues streaming when identity registration unexpectedly rejects', async () => {
deps.repository.registerDeliveredHash.mockRejectedValue(new Error('Database unavailable'));
await expect(service.streamBook(7, 11, reply as never)).resolves.toBeUndefined();
expect(reply.send).toHaveBeenCalledWith(stream);
expect(handle.close).not.toHaveBeenCalled();
});
it('streams narration-free files despite registration failure and cleans up after close', async () => {
setFile({ mediaOverlayAvailable: true });
deps.repository.registerDeliveredHash.mockRejectedValue(new Error('Database unavailable'));
vi.mocked(rm).mockRejectedValue(new Error('Cleanup failed'));
await expect(service.streamBook(7, 11, reply as never)).resolves.toBeUndefined();
expect(reply.send).toHaveBeenCalledWith(stream);
expect(rm).not.toHaveBeenCalled();
stream.emit('close');
expect(rm).toHaveBeenCalledExactlyOnceWith(TEMP_DIR, { recursive: true, force: true });
});
});
server/src/modules/kobo/services/kobo-download.service.ts
+44 -20
View File
@@ -1,23 +1,21 @@
import { createReadStream } from 'fs';
import { mkdtemp, rm, stat } from 'fs/promises';
import type { ReadStream } from 'fs';
import { mkdtemp, open, rm, stat } from 'fs/promises';
import type { FileHandle } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { Inject, Injectable, Logger, NotFoundException } from '@nestjs/common';
import { and, eq } from 'drizzle-orm';
import { HttpException, Injectable, InternalServerErrorException, Logger, NotFoundException } from '@nestjs/common';
import type { FastifyReply } from 'fastify';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { computeFileHashFromHandle } from '../../../common/utils/file-hash.utils';
import { sanitizeLogValue } from '../../../common/utils/log-sanitize.utils';
import { DB } from '../../../db/db.module';
import * as schema from '../../../db/schema';
import { AudiolessEpubService } from '../../book/audioless-epub.service';
import { KoboDownloadHashRegistrationService } from './kobo-download-hash-registration.service';
import { KoboDownloadRepository } from '../kobo-download.repository';
import { KoboBookAccessService } from './kobo-book-access.service';
import { KepubConversionService } from './kepub-conversion.service';
import { KoboSettingsService } from './kobo-settings.service';
type Db = NodePgDatabase<typeof schema>;
/** A narration-free rebuild, living in a temp directory for the length of one download. */
interface AudiolessCopy {
path: string;
@@ -36,7 +34,8 @@ export class KoboDownloadService {
private readonly logger = new Logger(KoboDownloadService.name);
constructor(
@Inject(DB) private readonly db: Db,
private readonly repository: KoboDownloadRepository,
private readonly hashRegistration: KoboDownloadHashRegistrationService,
private readonly kepubConversionService: KepubConversionService,
private readonly settingsService: KoboSettingsService,
private readonly bookAccessService: KoboBookAccessService,
@@ -44,14 +43,12 @@ export class KoboDownloadService {
) {}
async streamBook(userId: number, bookId: number, reply: FastifyReply) {
const book = await this.db.query.books.findFirst({ where: eq(schema.books.id, bookId) });
const book = await this.repository.findBook(bookId);
if (!book) throw new NotFoundException('Book not found');
await this.bookAccessService.assertBookAccessible(userId, bookId);
const file = await this.db.query.bookFiles.findFirst({
where: and(eq(schema.bookFiles.bookId, bookId), eq(schema.bookFiles.id, book.primaryFileId ?? -1)),
});
const file = await this.repository.findPrimaryFile(bookId, book.primaryFileId ?? -1);
if (!file) throw new NotFoundException('No file found for this book');
@@ -120,18 +117,45 @@ export class KoboDownloadService {
}
private async streamFile(absolutePath: string, fileId: number, format: string, reply: FastifyReply, cleanup?: () => Promise<void>) {
const startedAt = Date.now();
let handle: FileHandle | undefined;
let stream: ReadStream | undefined;
let cleanupStarted = false;
const release = async () => {
if (cleanupStarted) return;
cleanupStarted = true;
await cleanup?.().catch(() => undefined);
};
try {
const { size } = await stat(absolutePath);
handle = await open(absolutePath, 'r');
const [{ size }, hash] = await Promise.all([handle.stat(), computeFileHashFromHandle(handle)]);
// Persist the identity of the bytes being sent, including cache hits and fallback EPUBs.
// File history preserves ambiguity and never replaces a user's explicit hash link.
await this.hashRegistration.record(fileId, hash).catch((error: unknown) => {
this.logger.warn(
`[kobo.download_hash_registration] [fail] fileId=${fileId} durationMs=${Date.now() - startedAt} errorClass=${error instanceof Error ? error.name : 'UnknownError'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - identity registration failed`,
);
});
reply.header('Content-Length', size);
reply.header('Content-Disposition', `attachment; filename="book-${fileId}.${format}"`);
reply.type(MIME[format] ?? 'application/octet-stream');
const stream = createReadStream(absolutePath);
// Hash and stream the same open file even if its cache path is replaced concurrently.
stream = handle.createReadStream({ start: 0 });
// A temp rebuild is only safe to delete once the response has finished reading it.
if (cleanup) stream.once('close', () => void cleanup().catch(() => undefined));
if (cleanup) stream.once('close', () => void release());
reply.send(stream);
} catch {
await cleanup?.().catch(() => undefined);
throw new NotFoundException('File not found on disk');
} catch (err) {
stream?.destroy();
await handle?.close().catch(() => undefined);
await release();
if ((err as NodeJS.ErrnoException).code === 'ENOENT' || (err as NodeJS.ErrnoException).code === 'ENOTDIR') {
throw new NotFoundException('File not found on disk');
}
if (err instanceof HttpException) throw err;
this.logger.warn(
`[kobo.download] [fail] fileId=${fileId} durationMs=${Date.now() - startedAt} errorClass=${err instanceof Error ? err.name : 'UnknownError'} error="${sanitizeLogValue(err instanceof Error ? err.message : String(err))}" - download preparation failed`,
);
throw new InternalServerErrorException('Unable to prepare book download');
}
}
server/src/modules/koreader/koreader-hash-link.service.test.ts
+40 -27
View File
@@ -1,3 +1,5 @@
import { Test } from '@nestjs/testing';
import { BookService } from '../book/book.service';
import { BadRequestException, ConflictException, Logger, NotFoundException } from '@nestjs/common';
import { beforeEach, describe, expect, it, vi } from 'vitest';
@@ -19,7 +21,7 @@ describe('KoreaderHashLinkService', () => {
listBookHashLinks: ReturnType<typeof vi.fn>;
upsertBookHashLink: ReturnType<typeof vi.fn>;
getBookHashLink: ReturnType<typeof vi.fn>;
deleteBookHashLink: ReturnType<typeof vi.fn>;
unlinkBookHashLink: ReturnType<typeof vi.fn>;
upsertUnmatchedBooks: ReturnType<typeof vi.fn>;
findBookFileIdByBookId: ReturnType<typeof vi.fn>;
dismissUnmatchedBook: ReturnType<typeof vi.fn>;
@@ -48,7 +50,7 @@ describe('KoreaderHashLinkService', () => {
};
}
beforeEach(() => {
beforeEach(async () => {
vi.restoreAllMocks();
vi.clearAllMocks();
@@ -62,7 +64,7 @@ describe('KoreaderHashLinkService', () => {
listBookHashLinks: vi.fn().mockResolvedValue([]),
upsertBookHashLink: vi.fn().mockResolvedValue(undefined),
getBookHashLink: vi.fn().mockResolvedValue(null),
deleteBookHashLink: vi.fn().mockResolvedValue(null),
unlinkBookHashLink: vi.fn().mockResolvedValue(null),
upsertUnmatchedBooks: vi.fn().mockResolvedValue(undefined),
findBookFileIdByBookId: vi.fn(),
dismissUnmatchedBook: vi.fn().mockResolvedValue(null),
@@ -77,7 +79,10 @@ describe('KoreaderHashLinkService', () => {
vi.spyOn(Logger.prototype, 'debug').mockImplementation(() => undefined);
vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
service = new KoreaderHashLinkService(mockRepo as unknown as KoreaderRepository, mockBookService as never);
const module = await Test.createTestingModule({
providers: [KoreaderHashLinkService, { provide: KoreaderRepository, useValue: mockRepo }, { provide: BookService, useValue: mockBookService }],
}).compile();
service = module.get(KoreaderHashLinkService);
});
describe('listUnmatchedBooks', () => {
@@ -131,6 +136,16 @@ describe('KoreaderHashLinkService', () => {
expect(mockRepo.clearUnmatchedBooks).toHaveBeenCalledWith(7, [HASH_A]);
});
it('keeps a pending manual override visible even after intrinsic resolution succeeds', async () => {
mockRepo.listUnmatchedBooks.mockResolvedValue([makeUnmatchedRow({ manualLinkRequested: true })]);
mockRepo.resolveBookFilesByHashes.mockResolvedValue(new Map([[HASH_A, { bookFileId: 44, bookId: 55, libraryId: 1 }]]));
const rows = await service.listUnmatchedBooks(user);
expect(rows).toHaveLength(1);
expect(rows[0]!.hash).toBe(HASH_A);
expect(rows[0]).not.toHaveProperty('manualLinkRequested');
expect(mockRepo.resolveBookFilesByHashes).toHaveBeenCalledWith([], [1, 2], 7);
});
it('does not call clearUnmatchedBooks when nothing resolves', async () => {
mockRepo.listUnmatchedBooks.mockResolvedValue([makeUnmatchedRow()]);
mockRepo.resolveBookFilesByHashes.mockResolvedValue(new Map());
@@ -158,25 +173,25 @@ describe('KoreaderHashLinkService', () => {
await expect(service.linkUnmatchedBook(user, HASH_A.toUpperCase(), 55)).resolves.toEqual({ hash: HASH_A, bookId: 55, bookFileId: 44 });
expect(mockBookService.verifyBookAccess).toHaveBeenCalledWith(55, user);
expect(mockRepo.resolveBookFileByHash).toHaveBeenCalledWith(HASH_A, [1, 2]);
expect(mockRepo.resolveBookFileByHash).not.toHaveBeenCalled();
expect(mockRepo.getBookHashLink).toHaveBeenCalledWith(7, HASH_A);
expect(mockRepo.upsertBookHashLink).toHaveBeenCalledWith(7, HASH_A, 44, {
title: 'Stats title',
authors: 'Stats author',
lastOpen: 100,
});
expect(mockRepo.clearUnmatchedBooks).toHaveBeenCalledWith(7, [HASH_A]);
expect(mockRepo.clearUnmatchedBooks).toHaveBeenCalledWith(7, [HASH_A], true);
});
it('does not create a manual link when the hash already resolves to the same file intrinsically', async () => {
it('preserves explicit intent even when the hash already resolves to the same file intrinsically', async () => {
mockRepo.getUnmatchedBook.mockResolvedValue(makeUnmatchedRow());
mockRepo.findBookFileIdByBookId.mockResolvedValue(44);
mockRepo.resolveBookFileByHash.mockResolvedValue({ id: 44, bookId: 55, libraryId: 1 });
await service.linkUnmatchedBook(user, HASH_A, 55);
expect(mockRepo.upsertBookHashLink).not.toHaveBeenCalled();
expect(mockRepo.clearUnmatchedBooks).toHaveBeenCalledWith(7, [HASH_A]);
expect(mockRepo.upsertBookHashLink).toHaveBeenCalledWith(7, HASH_A, 44, expect.any(Object));
expect(mockRepo.clearUnmatchedBooks).toHaveBeenCalledWith(7, [HASH_A], true);
});
it('rejects linking historical or ambiguous unmatched rows', async () => {
@@ -205,15 +220,15 @@ describe('KoreaderHashLinkService', () => {
expect(mockRepo.upsertBookHashLink).not.toHaveBeenCalled();
});
it('rejects links when the hash already belongs to a different accessible book file', async () => {
it('allows an explicit override of an intrinsic match to another book', async () => {
mockRepo.getUnmatchedBook.mockResolvedValue(makeUnmatchedRow());
mockRepo.findBookFileIdByBookId.mockResolvedValue(44);
mockRepo.resolveBookFileByHash.mockResolvedValue({ id: 99, bookId: 88, libraryId: 1 });
await expect(service.linkUnmatchedBook(user, HASH_A, 55)).rejects.toThrow(ConflictException);
await expect(service.linkUnmatchedBook(user, HASH_A, 55)).resolves.toEqual({ hash: HASH_A, bookId: 55, bookFileId: 44 });
expect(mockRepo.upsertBookHashLink).not.toHaveBeenCalled();
expect(mockRepo.clearUnmatchedBooks).not.toHaveBeenCalled();
expect(mockRepo.upsertBookHashLink).toHaveBeenCalled();
expect(mockRepo.clearUnmatchedBooks).toHaveBeenCalledWith(7, [HASH_A], true);
});
it('rejects linking when the hash is already manually linked to a different file', async () => {
@@ -276,7 +291,7 @@ describe('KoreaderHashLinkService', () => {
expect(mockBookService.verifyBookAccess).toHaveBeenCalledWith(77, user);
expect(mockRepo.upsertBookHashLink).toHaveBeenCalledWith(7, HASH_A, 66);
expect(mockRepo.clearUnmatchedBooks).toHaveBeenCalledWith(7, [HASH_A]);
expect(mockRepo.clearUnmatchedBooks).toHaveBeenCalledWith(7, [HASH_A], true);
});
it('rejects relinking when no manual link exists', async () => {
@@ -298,21 +313,21 @@ describe('KoreaderHashLinkService', () => {
expect(mockRepo.upsertBookHashLink).not.toHaveBeenCalled();
});
it('rejects relinking when the hash already matches a different book intrinsically', async () => {
it('allows relinking despite an intrinsic match to a different book', async () => {
mockRepo.getBookHashLink.mockResolvedValue({ bookFileId: 44 });
mockRepo.findBookFileIdByBookId.mockResolvedValue(66);
mockRepo.resolveBookFileByHash.mockResolvedValue({ id: 99, bookId: 88, libraryId: 1 });
await expect(service.relinkManualHashLink(user, HASH_A, 77)).rejects.toThrow(ConflictException);
await expect(service.relinkManualHashLink(user, HASH_A, 77)).resolves.toEqual({ hash: HASH_A, bookId: 77, bookFileId: 66 });
expect(mockRepo.upsertBookHashLink).not.toHaveBeenCalled();
expect(mockRepo.clearUnmatchedBooks).not.toHaveBeenCalled();
expect(mockRepo.upsertBookHashLink).toHaveBeenCalledWith(7, HASH_A, 66);
expect(mockRepo.clearUnmatchedBooks).toHaveBeenCalledWith(7, [HASH_A], true);
});
});
describe('unlinkManualHashLink', () => {
it('unlinks a manual hash link and restores it as unmatched when no intrinsic match exists', async () => {
mockRepo.deleteBookHashLink.mockResolvedValue({
mockRepo.unlinkBookHashLink.mockResolvedValue({
hash: HASH_A,
bookFileId: 44,
koreaderTitle: 'KOReader Title',
@@ -323,14 +338,12 @@ describe('KoreaderHashLinkService', () => {
await expect(service.unlinkManualHashLink(user, HASH_A)).resolves.toEqual({ hash: HASH_A });
expect(mockRepo.deleteBookHashLink).toHaveBeenCalledWith(7, HASH_A);
expect(mockRepo.upsertUnmatchedBooks).toHaveBeenCalledWith(7, [
{ hash: HASH_A, title: 'KOReader Title', authors: 'KOReader Author', lastOpen: 100, source: 'file', metadataAmbiguous: false },
]);
expect(mockRepo.unlinkBookHashLink).toHaveBeenCalledWith(7, HASH_A);
expect(mockRepo.unlinkBookHashLink).toHaveBeenCalledWith(7, HASH_A);
});
it('does not restore the hash as unmatched when it still resolves intrinsically', async () => {
mockRepo.deleteBookHashLink.mockResolvedValue({
it('restores a manual-link request even when the hash resolves intrinsically', async () => {
mockRepo.unlinkBookHashLink.mockResolvedValue({
hash: HASH_A,
bookFileId: 44,
koreaderTitle: 'KOReader Title',
@@ -341,11 +354,11 @@ describe('KoreaderHashLinkService', () => {
await expect(service.unlinkManualHashLink(user, HASH_A)).resolves.toEqual({ hash: HASH_A });
expect(mockRepo.upsertUnmatchedBooks).not.toHaveBeenCalled();
expect(mockRepo.unlinkBookHashLink).toHaveBeenCalledWith(7, HASH_A);
});
it('rejects unlinking when no manual link exists', async () => {
mockRepo.deleteBookHashLink.mockResolvedValue(null);
mockRepo.unlinkBookHashLink.mockResolvedValue(null);
await expect(service.unlinkManualHashLink(user, HASH_A)).rejects.toThrow(NotFoundException);
server/src/modules/koreader/koreader-hash-link.service.ts
+16 -45
View File
@@ -23,9 +23,7 @@ const UNMATCHED_BOOK_DISMISS_ALL_EVENT = 'koreader.unmatched_book_dismiss_all';
/**
* Manages the KOReader "unmatched book" queue and the manual hash links used
* to resolve hashes that BookOrbit cannot match intrinsically (via file hash
* or hash history). Split out of KoreaderService to keep that class focused
* on credentials and progress sync.
* to resolve unknown hashes or override an intrinsic match explicitly.
*/
@Injectable()
export class KoreaderHashLinkService {
@@ -41,14 +39,17 @@ export class KoreaderHashLinkService {
if (rows.length === 0) return [];
const accessibleLibraryIds = await this.repo.getAccessibleLibraryIds(user.id);
const hashes = rows.map((row) => row.hash);
const hashes = rows.filter((row) => !row.manualLinkRequested).map((row) => row.hash);
const resolved = await this.repo.resolveBookFilesByHashes(hashes, accessibleLibraryIds, user.id);
if (resolved.size > 0) {
await this.repo.clearUnmatchedBooks(user.id, [...resolved.keys()]);
await this.repo.clearUnmatchedBooks(
user.id,
hashes.filter((hash) => resolved.has(hash)),
);
}
return rows
.filter((row) => !resolved.has(row.hash))
.filter((row) => row.manualLinkRequested || !resolved.has(row.hash))
.map((row) => ({
hash: row.hash,
title: row.title ?? null,
@@ -73,25 +74,17 @@ export class KoreaderHashLinkService {
const bookFileId = await this.repo.findBookFileIdByBookId(bookId);
if (!bookFileId) throw new BadRequestException('Book has no file to link');
const accessibleLibraryIds = await this.repo.getAccessibleLibraryIds(user.id);
const existingIntrinsicMatch = await this.repo.resolveBookFileByHash(normalizedHash, accessibleLibraryIds);
if (existingIntrinsicMatch && existingIntrinsicMatch.id !== bookFileId) {
throw new ConflictException('KOReader hash already matches a different book');
}
const existingLink = await this.repo.getBookHashLink(user.id, normalizedHash);
if (existingLink && existingLink.bookFileId !== bookFileId) {
throw new ConflictException('KOReader hash is already linked to a different book');
}
if (!existingIntrinsicMatch) {
await this.repo.upsertBookHashLink(user.id, normalizedHash, bookFileId, {
title: unmatched?.title ?? null,
authors: unmatched?.authors ?? null,
lastOpen: unmatched?.lastOpen ?? null,
});
}
await this.repo.clearUnmatchedBooks(user.id, [normalizedHash]);
await this.repo.upsertBookHashLink(user.id, normalizedHash, bookFileId, {
title: unmatched.title ?? null,
authors: unmatched.authors ?? null,
lastOpen: unmatched.lastOpen ?? null,
});
await this.repo.clearUnmatchedBooks(user.id, [normalizedHash], true);
this.logger.log(
`[${HASH_LINK_CREATE_EVENT}] [end] userId=${user.id} hash=${normalizedHash.slice(0, 8)} bookId=${bookId} bookFileId=${bookFileId} durationMs=${Date.now() - startedAtMs} - hash link create completed`,
@@ -131,14 +124,8 @@ export class KoreaderHashLinkService {
const bookFileId = await this.repo.findBookFileIdByBookId(bookId);
if (!bookFileId) throw new BadRequestException('Book has no file to link');
const accessibleLibraryIds = await this.repo.getAccessibleLibraryIds(user.id);
const existingIntrinsicMatch = await this.repo.resolveBookFileByHash(normalizedHash, accessibleLibraryIds);
if (existingIntrinsicMatch && existingIntrinsicMatch.id !== bookFileId) {
throw new ConflictException('KOReader hash already matches a different book');
}
await this.repo.upsertBookHashLink(user.id, normalizedHash, bookFileId);
await this.repo.clearUnmatchedBooks(user.id, [normalizedHash]);
await this.repo.clearUnmatchedBooks(user.id, [normalizedHash], true);
this.logger.log(
`[${HASH_LINK_UPDATE_EVENT}] [end] userId=${user.id} hash=${normalizedHash.slice(0, 8)} bookId=${bookId} bookFileId=${bookFileId} durationMs=${Date.now() - startedAtMs} - hash link update completed`,
@@ -151,27 +138,11 @@ export class KoreaderHashLinkService {
const normalizedHash = this.normalizeKoreaderHash(hash);
this.logger.log(`[${HASH_LINK_DELETE_EVENT}] [start] userId=${user.id} hash=${normalizedHash.slice(0, 8)} - hash link delete started`);
const deleted = await this.repo.deleteBookHashLink(user.id, normalizedHash);
const deleted = await this.repo.unlinkBookHashLink(user.id, normalizedHash);
if (!deleted) throw new NotFoundException('KOReader manual link not found');
const accessibleLibraryIds = await this.repo.getAccessibleLibraryIds(user.id);
const existingIntrinsicMatch = await this.repo.resolveBookFileByHash(normalizedHash, accessibleLibraryIds);
const restoredAsUnmatched = !existingIntrinsicMatch;
if (restoredAsUnmatched) {
await this.repo.upsertUnmatchedBooks(user.id, [
{
hash: normalizedHash,
title: deleted.koreaderTitle,
authors: deleted.koreaderAuthors,
lastOpen: deleted.koreaderLastOpen,
source: 'file',
metadataAmbiguous: false,
},
]);
}
this.logger.log(
`[${HASH_LINK_DELETE_EVENT}] [end] userId=${user.id} hash=${normalizedHash.slice(0, 8)} durationMs=${Date.now() - startedAtMs} restoredAsUnmatched=${restoredAsUnmatched} - hash link delete completed`,
`[${HASH_LINK_DELETE_EVENT}] [end] userId=${user.id} hash=${normalizedHash.slice(0, 8)} durationMs=${Date.now() - startedAtMs} restoredAsUnmatched=true - hash link delete completed`,
);
return { hash: normalizedHash };
}
server/src/modules/koreader/koreader-plugin.repository.test.ts
+49 -2
View File
@@ -1,6 +1,9 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { sqlChunkText } from '../../common/test-utils/sql-chunk-text';
import { KoreaderPluginRepository } from './koreader-plugin.repository';
import { Test } from '@nestjs/testing';
import { DB } from '../../db';
import { createCapturingDb } from '../../common/test-utils/capture-sql-db';
function makeQueryChain(result: unknown) {
const chain: Record<string, unknown> = {
@@ -9,7 +12,9 @@ function makeQueryChain(result: unknown) {
},
};
chain.from = vi.fn().mockReturnValue(chain);
chain.innerJoin = vi.fn().mockReturnValue(chain);
chain.where = vi.fn().mockReturnValue(chain);
chain.orderBy = vi.fn().mockReturnValue(chain);
chain.limit = vi.fn().mockReturnValue(chain);
return chain;
}
@@ -38,9 +43,10 @@ describe('KoreaderPluginRepository', () => {
let db: ReturnType<typeof makeDb>;
let repo: KoreaderPluginRepository;
beforeEach(() => {
beforeEach(async () => {
db = makeDb();
repo = new KoreaderPluginRepository(db as never);
const module = await Test.createTestingModule({ providers: [KoreaderPluginRepository, { provide: DB, useValue: db }] }).compile();
repo = module.get(KoreaderPluginRepository);
});
describe('getPluginTotals', () => {
@@ -90,6 +96,47 @@ describe('KoreaderPluginRepository', () => {
});
});
describe('getHashHistoryVersion', () => {
it('avoids querying for a user without library access', async () => {
await expect(repo.getHashHistoryVersion([])).resolves.toBe('');
expect(db.select).not.toHaveBeenCalled();
});
it('preserves full bigint precision in the revision token', async () => {
db.select.mockReturnValue(
makeQueryChain([
{ id: 31, revision: '9007199254740993' },
{ id: 32, revision: '2' },
]),
);
await expect(repo.getHashHistoryVersion([31, 32])).resolves.toBe('31:9007199254740993,32:2');
});
it.each([null, [31, 32]])('reads library revisions without scanning files or history for access %s', async (libraries) => {
const captured = createCapturingDb();
const module = await Test.createTestingModule({ providers: [KoreaderPluginRepository, { provide: DB, useValue: captured.db }] }).compile();
await module.get(KoreaderPluginRepository).getHashHistoryVersion(libraries);
expect(captured.queries).toHaveLength(1);
const query = captured.queries[0]!;
expect(query.sql).toContain('"koreader_hash_revision"::text');
expect(query.sql).toContain('order by "libraries"."id"');
expect(query.sql).not.toMatch(/count|join|book_files|book_file_hash_history/);
expect(query.params).toEqual(libraries ?? []);
});
});
describe('getGlobalMaxFileTimestamp', () => {
it('uses an indexed, bounded expression lookup without a library join', async () => {
const captured = createCapturingDb();
const module = await Test.createTestingModule({ providers: [KoreaderPluginRepository, { provide: DB, useValue: captured.db }] }).compile();
await module.get(KoreaderPluginRepository).getGlobalMaxFileTimestamp();
expect(captured.queries[0]!.sql).toContain('order by greatest(');
expect(captured.queries[0]!.sql).toContain('desc limit');
expect(captured.queries[0]!.sql).not.toMatch(/max\(|join|books/);
expect(captured.queries[0]!.params).toEqual([1]);
});
});
describe('getHashLinkVersion', () => {
it('returns manual link count and newest update timestamp', async () => {
db.select.mockReturnValue(makeQueryChain([{ count: '3', maxTs: '2026-06-02T10:00:00.000Z' }]));
server/src/modules/koreader/koreader-plugin.repository.ts
+16 -7
View File
@@ -520,15 +520,14 @@ export class KoreaderPluginRepository {
};
}
async getLibraryMaxFileTimestamp(accessibleLibraryIds: number[] | null): Promise<Date | null> {
if (accessibleLibraryIds !== null && accessibleLibraryIds.length === 0) return null;
const libraryFilter = accessibleLibraryIds ? inArray(schema.books.libraryId, accessibleLibraryIds) : undefined;
async getGlobalMaxFileTimestamp(): Promise<Date | null> {
// A global file change may invalidate an unrelated user's token, which is harmless.
// The expression index makes this one probe instead of scanning all accessible files.
const [row] = await this.db
.select({ maxTs: sql<Date | string | null>`max(greatest(${schema.bookFiles.createdAt}, ${schema.bookFiles.updatedAt}))` })
.select({ maxTs: sql<Date | string | null>`greatest(${schema.bookFiles.createdAt}, ${schema.bookFiles.updatedAt})` })
.from(schema.bookFiles)
.innerJoin(schema.books, eq(schema.books.id, schema.bookFiles.bookId))
.where(libraryFilter);
.orderBy(sql`greatest(${schema.bookFiles.createdAt}, ${schema.bookFiles.updatedAt}) desc`)
.limit(1);
return row?.maxTs ? new Date(row.maxTs) : null;
}
@@ -544,4 +543,14 @@ export class KoreaderPluginRepository {
return { count: Number(row?.count ?? 0), maxTs: row?.maxTs ? new Date(row.maxTs) : null };
}
async getHashHistoryVersion(accessibleLibraryIds: number[] | null): Promise<string> {
if (accessibleLibraryIds !== null && accessibleLibraryIds.length === 0) return '';
const rows = await this.db
.select({ id: schema.libraries.id, revision: sql<string>`${schema.libraries.koreaderHashRevision}::text` })
.from(schema.libraries)
.where(accessibleLibraryIds ? inArray(schema.libraries.id, accessibleLibraryIds) : undefined)
.orderBy(schema.libraries.id);
return rows.map((row) => `${row.id}:${row.revision}`).join(',');
}
}
server/src/modules/koreader/koreader-plugin.service.test.ts
+99 -27
View File
@@ -1,16 +1,17 @@
import { Logger } from '@nestjs/common';
import { Test } from '@nestjs/testing';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import type { RequestUser } from '../../common/types/request-user';
import { ACHIEVEMENT_EVENT_BOOK_RATING_CHANGED, type AchievementEventsService } from '../achievement/achievement-events.service';
import type { BookService } from '../book/book.service';
import type { UserBookNoteService } from '../user-book-note/user-book-note.service';
import type { UserBookStatusService } from '../user-book-status/user-book-status.service';
import { ACHIEVEMENT_EVENT_BOOK_RATING_CHANGED, AchievementEventsService } from '../achievement/achievement-events.service';
import { BookService } from '../book/book.service';
import { UserBookNoteService } from '../user-book-note/user-book-note.service';
import { UserBookStatusService } from '../user-book-status/user-book-status.service';
import type { BookStatesUploadDto, BulkProgressDto, MatchCheckDto, SweepCompleteDto } from './dto';
import type { KoreaderPluginRepository } from './koreader-plugin.repository';
import { KoreaderPluginRepository } from './koreader-plugin.repository';
import { KoreaderPluginService } from './koreader-plugin.service';
import type { KoreaderRepository } from './koreader.repository';
import type { KoreaderService } from './koreader.service';
import { KoreaderRepository } from './koreader.repository';
import { KoreaderService } from './koreader.service';
const DEVICE_ID = 'abcdef12-3456-7890-abcd-ef1234567890';
const HASH_A = 'a'.repeat(32);
@@ -31,7 +32,7 @@ describe('KoreaderPluginService', () => {
upsertUnmatchedBooks: ReturnType<typeof vi.fn>;
clearUnmatchedBooks: ReturnType<typeof vi.fn>;
restoreDevice: ReturnType<typeof vi.fn>;
upsertBookHashLink: ReturnType<typeof vi.fn>;
createBookHashLinkIfAbsent: ReturnType<typeof vi.fn>;
};
let pluginRepo: {
getRatings: ReturnType<typeof vi.fn>;
@@ -39,8 +40,9 @@ describe('KoreaderPluginService', () => {
upsertSweep: ReturnType<typeof vi.fn>;
listSweeps: ReturnType<typeof vi.fn>;
getPluginTotals: ReturnType<typeof vi.fn>;
getLibraryMaxFileTimestamp: ReturnType<typeof vi.fn>;
getGlobalMaxFileTimestamp: ReturnType<typeof vi.fn>;
getHashLinkVersion: ReturnType<typeof vi.fn>;
getHashHistoryVersion: ReturnType<typeof vi.fn>;
};
let koreaderService: { applyBulkProgress: ReturnType<typeof vi.fn> };
let userBookStatusService: {
@@ -57,7 +59,7 @@ describe('KoreaderPluginService', () => {
let bookService: { verifyFileAccess: ReturnType<typeof vi.fn> };
let service: KoreaderPluginService;
beforeEach(() => {
beforeEach(async () => {
vi.restoreAllMocks();
vi.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
vi.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
@@ -68,7 +70,7 @@ describe('KoreaderPluginService', () => {
upsertUnmatchedBooks: vi.fn().mockResolvedValue(undefined),
clearUnmatchedBooks: vi.fn().mockResolvedValue(undefined),
restoreDevice: vi.fn().mockResolvedValue(undefined),
upsertBookHashLink: vi.fn().mockResolvedValue(undefined),
createBookHashLinkIfAbsent: vi.fn().mockResolvedValue(undefined),
};
pluginRepo = {
getRatings: vi.fn().mockResolvedValue(new Map()),
@@ -76,8 +78,9 @@ describe('KoreaderPluginService', () => {
upsertSweep: vi.fn().mockResolvedValue(new Date('2026-06-09T10:00:00.000Z')),
listSweeps: vi.fn().mockResolvedValue([]),
getPluginTotals: vi.fn().mockResolvedValue({ matchedBooks: 0, pageStatEvents: 0, annotations: 0, unmatchedBooks: 0 }),
getLibraryMaxFileTimestamp: vi.fn().mockResolvedValue(new Date('2026-06-01T00:00:00.000Z')),
getGlobalMaxFileTimestamp: vi.fn().mockResolvedValue(new Date('2026-06-01T00:00:00.000Z')),
getHashLinkVersion: vi.fn().mockResolvedValue({ count: 0, maxTs: null }),
getHashHistoryVersion: vi.fn().mockResolvedValue('1:0'),
};
koreaderService = { applyBulkProgress: vi.fn().mockResolvedValue({ shared: 0, stale: 0 }) };
userBookStatusService = {
@@ -95,15 +98,19 @@ describe('KoreaderPluginService', () => {
};
achievementEvents = { emit: vi.fn() };
bookService = { verifyFileAccess: vi.fn() };
service = new KoreaderPluginService(
koreaderRepo as unknown as KoreaderRepository,
pluginRepo as unknown as KoreaderPluginRepository,
koreaderService as unknown as KoreaderService,
userBookStatusService as unknown as UserBookStatusService,
userBookNoteService as unknown as UserBookNoteService,
achievementEvents as unknown as AchievementEventsService,
bookService as unknown as BookService,
);
const module = await Test.createTestingModule({
providers: [
KoreaderPluginService,
{ provide: KoreaderRepository, useValue: koreaderRepo },
{ provide: KoreaderPluginRepository, useValue: pluginRepo },
{ provide: KoreaderService, useValue: koreaderService },
{ provide: UserBookStatusService, useValue: userBookStatusService },
{ provide: UserBookNoteService, useValue: userBookNoteService },
{ provide: AchievementEventsService, useValue: achievementEvents },
{ provide: BookService, useValue: bookService },
],
}).compile();
service = module.get(KoreaderPluginService);
});
describe('matchCheck', () => {
@@ -120,6 +127,9 @@ describe('KoreaderPluginService', () => {
});
it('links an explicitly downloaded catalog file when its hash resolved to another file', async () => {
koreaderRepo.resolveBookFilesByHashes
.mockResolvedValueOnce(new Map([[HASH_A, { bookFileId: 10, bookId: 20, libraryId: 1 }]]))
.mockResolvedValueOnce(new Map([[HASH_A, { bookFileId: 11, bookId: 20, libraryId: 1 }]]));
bookService.verifyFileAccess.mockResolvedValue({ id: 11, bookId: 20, libraryId: 1, format: 'epub', role: 'content' });
const dto = {
...deviceFields(),
@@ -130,7 +140,7 @@ describe('KoreaderPluginService', () => {
const result = await service.matchCheck(makeUser(), dto);
expect(bookService.verifyFileAccess).toHaveBeenCalledWith(11, makeUser());
expect(koreaderRepo.upsertBookHashLink).toHaveBeenCalledWith(7, HASH_A, 11, {
expect(koreaderRepo.createBookHashLinkIfAbsent).toHaveBeenCalledWith(7, HASH_A, 11, {
title: 'Read-along edition',
authors: null,
lastOpen: null,
@@ -139,7 +149,9 @@ describe('KoreaderPluginService', () => {
});
it('links an opened file using its previously verified file identity', async () => {
koreaderRepo.resolveBookFilesByHashes.mockResolvedValue(new Map());
koreaderRepo.resolveBookFilesByHashes
.mockResolvedValueOnce(new Map())
.mockResolvedValueOnce(new Map([[HASH_B, { bookFileId: 11, bookId: 21, libraryId: 1 }]]));
bookService.verifyFileAccess.mockResolvedValue({ id: 11, bookId: 21, libraryId: 1, format: 'epub', role: 'content' });
const dto = {
...deviceFields(),
@@ -149,7 +161,7 @@ describe('KoreaderPluginService', () => {
const result = await service.matchCheck(makeUser(), dto);
expect(koreaderRepo.upsertBookHashLink).toHaveBeenCalledWith(7, HASH_B, 11, {
expect(koreaderRepo.createBookHashLinkIfAbsent).toHaveBeenCalledWith(7, HASH_B, 11, {
title: 'Recovered book',
authors: null,
lastOpen: null,
@@ -157,6 +169,45 @@ describe('KoreaderPluginService', () => {
expect(result.matches).toEqual([{ hash: HASH_B, bookId: 21, bookFileId: 11 }]);
});
it.each(['file', 'current_file'])(
'keeps the resolved manual target when a %s candidate has a stale file ID from another book',
async (source) => {
bookService.verifyFileAccess.mockResolvedValue({ id: 11, bookId: 21, libraryId: 1, format: 'epub', role: 'content' });
const result = await service.matchCheck(makeUser(), {
...deviceFields(),
hashes: [HASH_A],
books: [{ hash: HASH_A, source, bookFileId: 11 }],
} as MatchCheckDto);
expect(result.matches).toEqual([{ hash: HASH_A, bookId: 20, bookFileId: 10 }]);
expect(koreaderRepo.createBookHashLinkIfAbsent).not.toHaveBeenCalled();
},
);
it('keeps a manual file choice within the same book when conditional link creation declines the stale candidate', async () => {
bookService.verifyFileAccess.mockResolvedValue({ id: 11, bookId: 20, libraryId: 1, format: 'epub', role: 'content' });
const result = await service.matchCheck(makeUser(), {
...deviceFields(),
hashes: [HASH_A],
books: [{ hash: HASH_A, source: 'current_file', bookFileId: 11 }],
} as MatchCheckDto);
expect(result.matches).toEqual([{ hash: HASH_A, bookId: 20, bookFileId: 10 }]);
expect(koreaderRepo.resolveBookFilesByHashes).toHaveBeenCalledTimes(2);
});
it('returns a concurrently created manual target after conditional candidate linking', async () => {
koreaderRepo.resolveBookFilesByHashes
.mockResolvedValueOnce(new Map())
.mockResolvedValueOnce(new Map([[HASH_B, { bookFileId: 30, bookId: 40, libraryId: 1 }]]));
bookService.verifyFileAccess.mockResolvedValue({ id: 11, bookId: 21, libraryId: 1, format: 'epub', role: 'content' });
const result = await service.matchCheck(makeUser(), {
...deviceFields(),
hashes: [HASH_B],
books: [{ hash: HASH_B, source: 'current_file', bookFileId: 11 }],
} as MatchCheckDto);
expect(result.matches).toEqual([{ hash: HASH_B, bookId: 40, bookFileId: 30 }]);
expect(koreaderRepo.resolveBookFilesByHashes).toHaveBeenLastCalledWith([HASH_B], [1], 7);
});
it('does not trust a file id reported only by a statistics row', async () => {
koreaderRepo.resolveBookFilesByHashes.mockResolvedValue(new Map());
const dto = {
@@ -168,7 +219,7 @@ describe('KoreaderPluginService', () => {
const result = await service.matchCheck(makeUser(), dto);
expect(bookService.verifyFileAccess).not.toHaveBeenCalled();
expect(koreaderRepo.upsertBookHashLink).not.toHaveBeenCalled();
expect(koreaderRepo.createBookHashLinkIfAbsent).not.toHaveBeenCalled();
expect(result.matches).toEqual([]);
});
@@ -183,7 +234,7 @@ describe('KoreaderPluginService', () => {
const result = await service.matchCheck(makeUser(), dto);
expect(koreaderRepo.upsertBookHashLink).not.toHaveBeenCalled();
expect(koreaderRepo.createBookHashLinkIfAbsent).not.toHaveBeenCalled();
expect(result.matches).toEqual([]);
});
@@ -197,7 +248,7 @@ describe('KoreaderPluginService', () => {
await service.matchCheck(makeUser(), dto);
expect(bookService.verifyFileAccess).not.toHaveBeenCalled();
expect(koreaderRepo.upsertBookHashLink).not.toHaveBeenCalled();
expect(koreaderRepo.createBookHashLinkIfAbsent).not.toHaveBeenCalled();
});
it('persists unmatched candidate metadata from the device statistics database', async () => {
@@ -269,6 +320,13 @@ describe('KoreaderPluginService', () => {
await expect(service.getLibraryVersion(7)).resolves.toBe(result.libraryVersion);
});
it('skips the global file timestamp when no library is accessible', async () => {
koreaderRepo.getAccessibleLibraryIds.mockResolvedValue([]);
await service.getLibraryVersion(7);
expect(pluginRepo.getGlobalMaxFileTimestamp).not.toHaveBeenCalled();
expect(pluginRepo.getHashHistoryVersion).toHaveBeenCalledWith([]);
});
it('changes the library version token when the accessible library set changes', async () => {
const dto = { ...deviceFields(), hashes: [HASH_A] } as MatchCheckDto;
@@ -302,6 +360,20 @@ describe('KoreaderPluginService', () => {
});
});
describe('hash history freshness', () => {
it.each(['1:1', '1:2', '1:9007199254740993'])('invalidates cached matches for revision %s', async (version) => {
const before = await service.getLibraryVersion(7);
pluginRepo.getHashHistoryVersion.mockResolvedValue(version);
expect(await service.getLibraryVersion(7)).not.toBe(before);
expect(pluginRepo.getHashHistoryVersion).toHaveBeenCalledWith([1]);
});
it('stays stable for unchanged revisions', async () => {
pluginRepo.getHashHistoryVersion.mockResolvedValue('1:1');
const first = await service.getLibraryVersion(7);
expect(await service.getLibraryVersion(7)).toBe(first);
});
});
describe('uploadBookStates', () => {
function statesDto(books: BookStatesUploadDto['books']): BookStatesUploadDto {
return { ...deviceFields(), books } as BookStatesUploadDto;
server/src/modules/koreader/koreader-plugin.service.ts
+15 -8
View File
@@ -114,7 +114,7 @@ export class KoreaderPluginService {
const accessibleLibraryIds = await this.koreaderRepo.getAccessibleLibraryIds(user.id);
const hashes = [...new Set(dto.hashes.map((hash) => hash.toLowerCase()))];
const resolved = await this.koreaderRepo.resolveBookFilesByHashes(hashes, accessibleLibraryIds, user.id);
const linked = await this.linkRequestedFileHashes(user, hashes, resolved, this.buildCandidateMap(dto));
const linked = await this.linkRequestedFileHashes(user, hashes, resolved, this.buildCandidateMap(dto), accessibleLibraryIds);
for (const [hash, match] of linked) resolved.set(hash, match);
const matchedHashes = [...resolved.keys()];
const unmatchedCandidates = this.buildUnmatchedCandidates(hashes, matchedHashes, dto);
@@ -295,15 +295,17 @@ export class KoreaderPluginService {
}
private async computeLibraryVersion(userId: number, accessibleLibraryIds: number[] | null): Promise<string> {
const [fileMaxTs, linkVersion] = await Promise.all([
this.pluginRepo.getLibraryMaxFileTimestamp(accessibleLibraryIds),
const [fileMaxTs, linkVersion, historyVersion] = await Promise.all([
accessibleLibraryIds?.length === 0 ? null : this.pluginRepo.getGlobalMaxFileTimestamp(),
this.pluginRepo.getHashLinkVersion(userId),
this.pluginRepo.getHashHistoryVersion(accessibleLibraryIds),
]);
const maxTs = maxDate(fileMaxTs, linkVersion.maxTs);
const libraryKey = accessibleLibraryIds === null ? 'all' : [...accessibleLibraryIds].sort((a, b) => a - b).join(',');
const linkKey = `${linkVersion.count}:${linkVersion.maxTs ? linkVersion.maxTs.toISOString() : 'none'}`;
const historyKey = historyVersion;
return createHash('md5') // codeql[js/weak-cryptographic-algorithm] - non-security cache token
.update(`${libraryKey}|${maxTs ? maxTs.toISOString() : 'none'}|${linkKey}`)
.update(`${libraryKey}|${maxTs ? maxTs.toISOString() : 'none'}|${linkKey}|${historyKey}`)
.digest('hex')
.slice(0, 16);
}
@@ -321,8 +323,9 @@ export class KoreaderPluginService {
hashes: string[],
resolved: Map<string, ResolvedBookFile>,
candidates: Map<string, MatchCheckBookCandidate>,
accessibleLibraryIds: number[] | null,
): Promise<Map<string, ResolvedBookFile>> {
const linked = new Map<string, ResolvedBookFile>();
const attempted: string[] = [];
for (const hash of hashes) {
const candidate = candidates.get(hash);
if (!candidate?.bookFileId || (candidate.source !== 'file' && candidate.source !== 'current_file')) continue;
@@ -330,15 +333,19 @@ export class KoreaderPluginService {
const file = await this.bookService.verifyFileAccess(candidate.bookFileId, user).catch(() => null);
if (!file || file.role !== 'content') continue;
const existing = resolved.get(hash);
if (existing && existing.bookId !== file.bookId) continue;
await this.koreaderRepo.upsertBookHashLink(user.id, hash, file.id, {
// A device's cached file ID cannot overwrite a manual choice made since its last sync.
await this.koreaderRepo.createBookHashLinkIfAbsent(user.id, hash, file.id, {
title: candidate.title ?? null,
authors: candidate.authors ?? null,
lastOpen: candidate.lastOpen ?? null,
});
linked.set(hash, { bookFileId: file.id, bookId: file.bookId, libraryId: file.libraryId, format: file.format });
attempted.push(hash);
}
return linked;
// Resolve committed links again, including a manual link created concurrently with access checks.
return attempted.length ? this.koreaderRepo.resolveBookFilesByHashes(attempted, accessibleLibraryIds, user.id) : new Map();
}
private buildUnmatchedCandidates(hashes: string[], matchedHashes: string[], dto: MatchCheckDto) {
server/src/modules/koreader/koreader.repository.test.ts
+201 -27
View File
@@ -1,9 +1,16 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { Test } from '@nestjs/testing';
import type { SQL } from 'drizzle-orm';
import { PgDialect } from 'drizzle-orm/pg-core';
import { createCapturingDb } from '../../common/test-utils/capture-sql-db';
import { sqlChunkText } from '../../common/test-utils/sql-chunk-text';
import { KoreaderRepository } from './koreader.repository';
import { DB } from '../../db';
async function makeRepository(db: unknown) {
const module = await Test.createTestingModule({ providers: [KoreaderRepository, { provide: DB, useValue: db }] }).compile();
return module.get(KoreaderRepository);
}
function makeQueryChain(result: unknown) {
const chain: Record<string, unknown> = {
@@ -44,9 +51,9 @@ describe('KoreaderRepository', () => {
let db: ReturnType<typeof makeDb>;
let repo: KoreaderRepository;
beforeEach(() => {
beforeEach(async () => {
db = makeDb();
repo = new KoreaderRepository(db as never);
repo = await makeRepository(db);
});
describe('progress resets', () => {
@@ -93,6 +100,62 @@ describe('KoreaderRepository', () => {
});
describe('resolveBookFileByHash', () => {
beforeEach(() => db.select.mockReturnValue(makeQueryChain([])));
it.each(['current', 'history'])('honors an explicit user link even when a unique %s identity exists', async (source) => {
const intrinsic = { id: 10, bookId: 20, libraryId: 1, format: 'epub' };
const linked = { id: 11, bookId: 21, libraryId: 1, format: 'epub' };
db.select
.mockReturnValueOnce(makeQueryChain(source === 'current' ? [intrinsic] : []))
.mockReturnValueOnce(makeQueryChain(source === 'history' ? [intrinsic] : []))
.mockReturnValueOnce(makeQueryChain([linked]));
await expect(repo.resolveBookFileByHash('explicit', [1], 7)).resolves.toEqual(linked);
});
it.each([undefined, 7])('rejects a current/history collision across books for user %s', async (userId) => {
db.select
.mockReturnValueOnce(makeQueryChain([{ id: 10, bookId: 20, libraryId: 1, format: 'epub' }]))
.mockReturnValueOnce(makeQueryChain([{ id: 11, bookId: 21, libraryId: 1, format: 'epub' }]));
await expect(repo.resolveBookFileByHash('collision', [1], userId)).resolves.toBeNull();
});
it('allows an explicit user link to disambiguate a current/history collision', async () => {
const linked = { id: 11, bookId: 21, libraryId: 1, format: 'epub' };
db.select
.mockReturnValueOnce(makeQueryChain([{ id: 10, bookId: 20, libraryId: 1, format: 'epub' }]))
.mockReturnValueOnce(makeQueryChain([{ ...linked }]))
.mockReturnValueOnce(makeQueryChain([linked]));
await expect(repo.resolveBookFileByHash('collision', [1], 7)).resolves.toEqual(linked);
});
it.each([undefined, 7])('deduplicates a file present in current and delivered history for user %s', async (userId) => {
const file = { id: 10, bookId: 20, libraryId: 1, format: 'pdf' };
db.select.mockReturnValueOnce(makeQueryChain([file])).mockReturnValueOnce(makeQueryChain([{ ...file }]));
await expect(repo.resolveBookFileByHash('unchanged', [1], userId)).resolves.toEqual({ id: 10, bookId: 20, libraryId: 1, format: 'pdf' });
expect(db.select).toHaveBeenCalledTimes(userId === undefined ? 2 : 3);
});
it('preserves manual selection for different files of one book across current and history', async () => {
const linked = { id: 11, bookId: 20, libraryId: 1, format: 'epub' };
db.select
.mockReturnValueOnce(makeQueryChain([{ id: 10, bookId: 20, libraryId: 1, format: 'epub' }]))
.mockReturnValueOnce(makeQueryChain([{ ...linked }]))
.mockReturnValueOnce(makeQueryChain([linked]));
await expect(repo.resolveBookFileByHash('same-book', [1], 7)).resolves.toEqual(linked);
});
it('checks both intrinsic sources with library filters and bounded ambiguity queries', async () => {
const captured = createCapturingDb();
const repository = await makeRepository(captured.db);
await repository.resolveBookFileByHash('collision', [31, 32]);
expect(captured.queries).toHaveLength(2);
for (const query of captured.queries) {
expect(query.sql).toContain('"books"."library_id" in');
expect(query.sql).toContain('limit');
expect(query.params).toEqual(expect.arrayContaining(['collision', 31, 32, 2]));
}
});
it('short-circuits when accessible libraries are empty', async () => {
await expect(repo.resolveBookFileByHash('hash', [])).resolves.toBeNull();
expect(db.select).not.toHaveBeenCalled();
@@ -100,7 +163,7 @@ describe('KoreaderRepository', () => {
it('returns null when accessible libraries is null and no file found', async () => {
const emptyChain = makeQueryChain([]);
db.select.mockReturnValue(emptyChain);
db.select.mockReturnValueOnce(emptyChain);
const result = await repo.resolveBookFileByHash('hash', null);
@@ -110,25 +173,25 @@ describe('KoreaderRepository', () => {
it('returns the book file when found by current hash', async () => {
const file = { id: 10, bookId: 20, libraryId: 1, format: 'epub' };
db.select.mockReturnValue(makeQueryChain([file]));
db.select.mockReturnValueOnce(makeQueryChain([file]));
const result = await repo.resolveBookFileByHash('abc123', null);
expect(result).toEqual(file);
expect(db.select).toHaveBeenCalledTimes(1);
expect(db.select).toHaveBeenCalledTimes(2);
});
it('returns the oldest file when current hash matches stay within one book', async () => {
const firstFile = { id: 10, bookId: 20, libraryId: 1, format: 'epub' };
const secondFile = { id: 11, bookId: 20, libraryId: 1, format: 'epub' };
db.select.mockReturnValue(makeQueryChain([firstFile, secondFile]));
db.select.mockReturnValueOnce(makeQueryChain([firstFile, secondFile]));
await expect(repo.resolveBookFileByHash('abc123', null)).resolves.toEqual(firstFile);
expect(db.select).toHaveBeenCalledTimes(1);
expect(db.select).toHaveBeenCalledTimes(2);
});
it('returns null when a current hash matches different books', async () => {
db.select.mockReturnValue(
db.select.mockReturnValueOnce(
makeQueryChain([
{ id: 10, bookId: 20, libraryId: 1, format: 'epub' },
{ id: 11, bookId: 21, libraryId: 1, format: 'epub' },
@@ -136,7 +199,7 @@ describe('KoreaderRepository', () => {
);
await expect(repo.resolveBookFileByHash('abc123', null)).resolves.toBeNull();
expect(db.select).toHaveBeenCalledTimes(1);
expect(db.select).toHaveBeenCalledTimes(2);
});
it('uses a user-scoped manual link when a current hash matches different books', async () => {
@@ -148,28 +211,33 @@ describe('KoreaderRepository', () => {
{ id: 11, bookId: 21, libraryId: 1, format: 'epub' },
]),
)
.mockReturnValueOnce(makeQueryChain([]))
.mockReturnValueOnce(makeQueryChain([manualFile]));
await expect(repo.resolveBookFileByHash('abc123', null, 7)).resolves.toEqual(manualFile);
expect(db.select).toHaveBeenCalledTimes(2);
expect(db.select).toHaveBeenCalledTimes(3);
});
it('prefers a user-scoped link when a current hash matches multiple files in one book', async () => {
const linkedFile = { id: 11, bookId: 20, libraryId: 1, format: 'epub' };
db.select
.mockReturnValueOnce(makeQueryChain([{ id: 10, bookId: 20, libraryId: 1, format: 'epub', matchingFileCount: 2 }]))
.mockReturnValueOnce(makeQueryChain([{ id: 10, bookId: 20, libraryId: 1, format: 'epub' }]))
.mockReturnValueOnce(makeQueryChain([]))
.mockReturnValueOnce(makeQueryChain([linkedFile]));
await expect(repo.resolveBookFileByHash('abc123', null, 7)).resolves.toEqual(linkedFile);
expect(db.select).toHaveBeenCalledTimes(2);
expect(db.select).toHaveBeenCalledTimes(3);
});
it('keeps the deterministic first file when a same-book duplicate has no user-scoped link', async () => {
const firstFile = { id: 10, bookId: 20, libraryId: 1, format: 'epub', matchingFileCount: 2 };
db.select.mockReturnValueOnce(makeQueryChain([firstFile])).mockReturnValueOnce(makeQueryChain([]));
const firstFile = { id: 10, bookId: 20, libraryId: 1, format: 'epub' };
db.select
.mockReturnValueOnce(makeQueryChain([firstFile]))
.mockReturnValueOnce(makeQueryChain([]))
.mockReturnValueOnce(makeQueryChain([]));
await expect(repo.resolveBookFileByHash('abc123', null, 7)).resolves.toEqual({ id: 10, bookId: 20, libraryId: 1, format: 'epub' });
expect(db.select).toHaveBeenCalledTimes(2);
expect(db.select).toHaveBeenCalledTimes(3);
});
it('falls back to hash history when current hash lookup returns nothing', async () => {
@@ -229,6 +297,72 @@ describe('KoreaderRepository', () => {
});
describe('resolveBookFilesByHashes', () => {
beforeEach(() => db.select.mockReturnValue(makeQueryChain([])));
it.each(['current', 'history'])('honors an explicit user link over a unique %s identity in a batch', async (source) => {
const intrinsic = { hash: 'explicit', bookFileId: 11, bookId: 21, libraryId: 31, format: 'epub' };
const linked = { hash: 'explicit', bookFileId: 12, bookId: 22, libraryId: 31, format: 'epub' };
db.select
.mockReturnValueOnce(makeQueryChain(source === 'current' ? [intrinsic] : []))
.mockReturnValueOnce(makeQueryChain(source === 'history' ? [intrinsic] : []))
.mockReturnValueOnce(makeQueryChain([linked]));
const result = await repo.resolveBookFilesByHashes(['explicit'], [31], 7);
expect(result.get('explicit')).toEqual({ bookFileId: 12, bookId: 22, libraryId: 31, format: 'epub' });
});
it.each([undefined, 7])('omits current/history collisions without losing other batch matches for user %s', async (userId) => {
db.select
.mockReturnValueOnce(
makeQueryChain([
{ hash: 'collision', bookFileId: 11, bookId: 21, libraryId: 31, format: 'epub' },
{ hash: 'safe', bookFileId: 13, bookId: 23, libraryId: 31, format: 'pdf' },
]),
)
.mockReturnValueOnce(makeQueryChain([{ hash: 'collision', bookFileId: 12, bookId: 22, libraryId: 31, format: 'epub' }]));
const result = await repo.resolveBookFilesByHashes(['collision', 'safe'], [31], userId);
expect(result.has('collision')).toBe(false);
expect(result.get('safe')).toEqual({ bookFileId: 13, bookId: 23, libraryId: 31, format: 'pdf' });
});
it('uses a user link to resolve a current/history collision in a batch', async () => {
const linked = { hash: 'collision', bookFileId: 12, bookId: 22, libraryId: 31, format: 'epub' };
db.select
.mockReturnValueOnce(makeQueryChain([{ hash: 'collision', bookFileId: 11, bookId: 21, libraryId: 31, format: 'epub' }]))
.mockReturnValueOnce(makeQueryChain([linked]))
.mockReturnValueOnce(makeQueryChain([linked]));
const result = await repo.resolveBookFilesByHashes(['collision'], [31], 7);
expect(result.get('collision')).toEqual({ bookFileId: 12, bookId: 22, libraryId: 31, format: 'epub' });
});
it('deduplicates current/history rows of an unchanged delivered file while checking manual links', async () => {
const file = { hash: 'unchanged', bookFileId: 11, bookId: 21, libraryId: 31, format: 'pdf' };
db.select.mockReturnValueOnce(makeQueryChain([file])).mockReturnValueOnce(makeQueryChain([{ ...file }]));
const result = await repo.resolveBookFilesByHashes(['unchanged'], [31], 7);
expect(result.get('unchanged')).toEqual({ bookFileId: 11, bookId: 21, libraryId: 31, format: 'pdf' });
expect(db.select).toHaveBeenCalledTimes(3);
});
it('preserves manual selection of same-book files split across current and history in a batch', async () => {
const linked = { hash: 'same-book', bookFileId: 12, bookId: 21, libraryId: 31, format: 'epub' };
db.select
.mockReturnValueOnce(makeQueryChain([{ hash: 'same-book', bookFileId: 11, bookId: 21, libraryId: 31, format: 'epub' }]))
.mockReturnValueOnce(makeQueryChain([linked]))
.mockReturnValueOnce(makeQueryChain([linked]));
const result = await repo.resolveBookFilesByHashes(['same-book'], [31], 7);
expect(result.get('same-book')).toEqual({ bookFileId: 12, bookId: 21, libraryId: 31, format: 'epub' });
});
it('queries every requested hash in both library-scoped intrinsic sources', async () => {
const captured = createCapturingDb();
const repository = await makeRepository(captured.db);
await repository.resolveBookFilesByHashes(['current', 'delivered'], [31, 32]);
expect(captured.queries).toHaveLength(2);
for (const query of captured.queries) {
expect(query.sql).toContain('"books"."library_id" in');
expect(query.params).toEqual(expect.arrayContaining(['current', 'delivered', 31, 32]));
}
});
it('returns an empty map when no hashes are provided', async () => {
const result = await repo.resolveBookFilesByHashes([], null);
@@ -243,7 +377,7 @@ describe('KoreaderRepository', () => {
expect(db.select).not.toHaveBeenCalled();
});
it('resolves direct hashes first and falls back to hash history for missing hashes', async () => {
it('resolves current and historical identities in the same batch', async () => {
db.select
.mockReturnValueOnce(
makeQueryChain([
@@ -261,13 +395,13 @@ describe('KoreaderRepository', () => {
expect(db.select).toHaveBeenCalledTimes(2);
});
it('skips hash history lookup when all hashes resolve directly', async () => {
it('checks history even when all hashes resolve directly', async () => {
db.select.mockReturnValueOnce(makeQueryChain([{ hash: 'current', bookFileId: 11, bookId: 21, libraryId: 31, format: 'pdf' }]));
const result = await repo.resolveBookFilesByHashes(['current'], null);
expect(result.get('current')).toEqual({ bookFileId: 11, bookId: 21, libraryId: 31, format: 'pdf' });
expect(db.select).toHaveBeenCalledTimes(1);
expect(db.select).toHaveBeenCalledTimes(2);
});
it('resolves duplicate current hash rows when they belong to one book', async () => {
@@ -281,7 +415,7 @@ describe('KoreaderRepository', () => {
const result = await repo.resolveBookFilesByHashes(['current'], null);
expect(result.get('current')).toEqual({ bookFileId: 11, bookId: 21, libraryId: 31, format: 'epub' });
expect(db.select).toHaveBeenCalledTimes(1);
expect(db.select).toHaveBeenCalledTimes(2);
});
it('omits a current hash that matches different books', async () => {
@@ -295,7 +429,7 @@ describe('KoreaderRepository', () => {
const result = await repo.resolveBookFilesByHashes(['current'], null);
expect(result.has('current')).toBe(false);
expect(db.select).toHaveBeenCalledTimes(1);
expect(db.select).toHaveBeenCalledTimes(2);
});
it('uses a user-scoped manual link for an ambiguous current hash', async () => {
@@ -306,12 +440,13 @@ describe('KoreaderRepository', () => {
{ hash: 'current', bookFileId: 12, bookId: 22, libraryId: 31, format: 'epub' },
]),
)
.mockReturnValueOnce(makeQueryChain([]))
.mockReturnValueOnce(makeQueryChain([{ hash: 'current', bookFileId: 12, bookId: 22, libraryId: 31, format: 'epub' }]));
const result = await repo.resolveBookFilesByHashes(['current'], null, 7);
expect(result.get('current')).toEqual({ bookFileId: 12, bookId: 22, libraryId: 31, format: 'epub' });
expect(db.select).toHaveBeenCalledTimes(2);
expect(db.select).toHaveBeenCalledTimes(3);
});
it('prefers a user-scoped link for duplicate current hashes within one book', async () => {
@@ -322,12 +457,13 @@ describe('KoreaderRepository', () => {
{ hash: 'current', bookFileId: 12, bookId: 21, libraryId: 31, format: 'epub' },
]),
)
.mockReturnValueOnce(makeQueryChain([]))
.mockReturnValueOnce(makeQueryChain([{ hash: 'current', bookFileId: 12, bookId: 21, libraryId: 31, format: 'epub' }]));
const result = await repo.resolveBookFilesByHashes(['current'], null, 7);
expect(result.get('current')).toEqual({ bookFileId: 12, bookId: 21, libraryId: 31, format: 'epub' });
expect(db.select).toHaveBeenCalledTimes(2);
expect(db.select).toHaveBeenCalledTimes(3);
});
it('keeps the deterministic first file when a same-book duplicate has no user-scoped bulk link', async () => {
@@ -338,12 +474,13 @@ describe('KoreaderRepository', () => {
{ hash: 'current', bookFileId: 12, bookId: 21, libraryId: 31, format: 'epub' },
]),
)
.mockReturnValueOnce(makeQueryChain([]))
.mockReturnValueOnce(makeQueryChain([]));
const result = await repo.resolveBookFilesByHashes(['current'], null, 7);
expect(result.get('current')).toEqual({ bookFileId: 11, bookId: 21, libraryId: 31, format: 'epub' });
expect(db.select).toHaveBeenCalledTimes(2);
expect(db.select).toHaveBeenCalledTimes(3);
});
it('omits a historical hash that matches different books', async () => {
@@ -570,6 +707,19 @@ describe('KoreaderRepository', () => {
);
});
it('creates automatic candidate links without replacing manual links or reviving pending corrections', async () => {
const { db: capturedDb, queries } = createCapturingDb();
const repository = await makeRepository(capturedDb);
await repository.createBookHashLinkIfAbsent(7, 'a'.repeat(32), 44, { title: ' Title ', authors: ' Author ', lastOpen: 100 });
expect(queries).toHaveLength(1);
const query = queries[0]!;
expect(query.sql).toContain('on conflict (user_id, hash) do nothing');
expect(query.sql).toContain('where not exists');
expect(query.sql).toContain('manual_link_requested = true');
expect(query.sql).not.toContain('do update');
expect(query.params).toEqual([7, 'a'.repeat(32), 44, 'Title', 'Author', 100, 7, 'a'.repeat(32)]);
});
it('upserts a manual hash link with null metadata when none is provided', async () => {
const onConflictDoUpdate = vi.fn().mockResolvedValue(undefined);
const values = vi.fn().mockReturnValue({ onConflictDoUpdate });
@@ -600,11 +750,35 @@ describe('KoreaderRepository', () => {
it('deletes and returns a user-scoped manual hash link', async () => {
const row = { hash: 'a'.repeat(32), bookFileId: 44, koreaderTitle: 'Title', koreaderAuthors: 'Author', koreaderLastOpen: 100 };
const chain = makeQueryChain([row]);
db.delete.mockReturnValue(chain);
db.execute.mockResolvedValue({ rows: [row] });
await expect(repo.unlinkBookHashLink(7, 'a'.repeat(32))).resolves.toEqual(row);
const query = sqlChunkText(db.execute.mock.calls[0]![0]);
expect(query).toContain('with deleted as');
expect(query).toContain('manual_link_requested');
});
});
await expect(repo.deleteBookHashLink(7, 'a'.repeat(32))).resolves.toEqual(row);
expect(chain.returning).toHaveBeenCalledTimes(1);
describe('pending manual override protection', () => {
it.each([false, true])('only clears pending manual requests when explicitly requested (%s)', async (includeManual) => {
const captured = createCapturingDb();
const module = await Test.createTestingModule({ providers: [KoreaderRepository, { provide: DB, useValue: captured.db }] }).compile();
await module.get(KoreaderRepository).clearUnmatchedBooks(7, ['a'.repeat(32)], includeManual);
const query = captured.queries[0]!;
expect(query.sql).toContain('"user_id" =');
expect(query.sql.includes('"manual_link_requested" =')).toBe(!includeManual);
});
it('restores a relinkable request atomically with user-scoped link deletion', async () => {
const captured = createCapturingDb();
const module = await Test.createTestingModule({ providers: [KoreaderRepository, { provide: DB, useValue: captured.db }] }).compile();
await expect(module.get(KoreaderRepository).unlinkBookHashLink(7, 'a'.repeat(32))).resolves.toBeNull();
expect(captured.queries).toHaveLength(1);
const query = captured.queries[0]!;
expect(query.sql).toContain('with deleted as');
expect(query.sql).toContain('where user_id = $1 and hash = $2');
expect(query.sql).toContain('insert into "koreader_unmatched_books"');
expect(query.sql).toContain('manual_link_requested = true');
expect(query.params).toEqual([7, 'a'.repeat(32)]);
});
});
server/src/modules/koreader/koreader.repository.ts
+79 -81
View File
@@ -17,6 +17,7 @@ type KoreaderUnmatchedCandidate = {
lastOpen?: number | null;
source?: KoreaderUnmatchedSource;
metadataAmbiguous?: boolean;
manualLinkRequested?: boolean;
};
type KoreaderHashLinkMetadata = {
title?: string | null;
@@ -25,7 +26,6 @@ type KoreaderHashLinkMetadata = {
};
type HashedResolvedBookFile = ResolvedBookFileByHashes & { hash: string | null };
type CountedResolvedBookFile = ResolvedBookFileByHash & { matchingFileCount: number };
function resolveUniqueBook<T extends { bookId: number }>(rows: T[]): T | null {
const first = rows[0];
@@ -33,15 +33,7 @@ function resolveUniqueBook<T extends { bookId: number }>(rows: T[]): T | null {
return rows.every((row) => row.bookId === first.bookId) ? first : null;
}
function hasDuplicateFileMatches(rows: CountedResolvedBookFile[]): boolean {
return rows.length > 1 || rows.some((row) => Number(row.matchingFileCount) > 1);
}
function withoutMatchCount(row: CountedResolvedBookFile): ResolvedBookFileByHash {
return { id: row.id, bookId: row.bookId, libraryId: row.libraryId, format: row.format };
}
function addUnambiguousHashMatches(rows: HashedResolvedBookFile[], result: Map<string, ResolvedBookFileByHashes>): Set<string> {
function addUnambiguousHashMatches(rows: HashedResolvedBookFile[], result: Map<string, ResolvedBookFileByHashes>): void {
const rowsByHash = new Map<string, HashedResolvedBookFile[]>();
for (const row of rows) {
@@ -62,8 +54,6 @@ function addUnambiguousHashMatches(rows: HashedResolvedBookFile[], result: Map<s
});
}
}
return new Set(rowsByHash.keys());
}
export interface ReadingProgressUpsert {
@@ -133,7 +123,6 @@ export class KoreaderRepository {
bookId: schema.bookFiles.bookId,
libraryId: schema.books.libraryId,
format: schema.bookFiles.format,
matchingFileCount: sql<number>`count(*) over (partition by ${schema.bookFiles.fileHash})`,
})
.from(schema.bookFiles)
.innerJoin(schema.books, eq(schema.books.id, schema.bookFiles.bookId))
@@ -141,20 +130,12 @@ export class KoreaderRepository {
.orderBy(asc(schema.bookFiles.bookId), asc(schema.bookFiles.id))
.limit(2);
if (byFileHash.length > 0) {
const resolved = resolveUniqueBook(byFileHash);
if (resolved && (!hasDuplicateFileMatches(byFileHash) || userId === undefined)) return withoutMatchCount(resolved);
if (userId === undefined) return null;
return (await this.resolveManualBookFileByHash(hash, accessibleLibraryIds, userId)) ?? (resolved ? withoutMatchCount(resolved) : null);
}
const byFileHashHistory = await this.db
.selectDistinctOn([schema.bookFiles.bookId], {
id: schema.bookFiles.id,
bookId: schema.bookFiles.bookId,
libraryId: schema.books.libraryId,
format: schema.bookFiles.format,
matchingFileCount: sql<number>`count(*) over (partition by ${schema.bookFileHashHistory.fileHash})`,
})
.from(schema.bookFileHashHistory)
.innerJoin(schema.bookFiles, eq(schema.bookFiles.id, schema.bookFileHashHistory.bookFileId))
@@ -163,14 +144,18 @@ export class KoreaderRepository {
.orderBy(asc(schema.bookFiles.bookId), asc(schema.bookFiles.id))
.limit(2);
if (byFileHashHistory.length > 0) {
const resolved = resolveUniqueBook(byFileHashHistory);
if (resolved && (!hasDuplicateFileMatches(byFileHashHistory) || userId === undefined)) return withoutMatchCount(resolved);
if (userId === undefined) return null;
return (await this.resolveManualBookFileByHash(hash, accessibleLibraryIds, userId)) ?? (resolved ? withoutMatchCount(resolved) : null);
// A delivered variant can collide with another book's current hash. Both sources must
// agree before a device's progress can be assigned, even when a current hash exists.
const candidates = [...byFileHash];
for (const row of byFileHashHistory) {
if (!candidates.some((candidate) => candidate.id === row.id)) candidates.push(row);
}
return userId === undefined ? null : this.resolveManualBookFileByHash(hash, accessibleLibraryIds, userId);
if (userId !== undefined) {
const manual = await this.resolveManualBookFileByHash(hash, accessibleLibraryIds, userId);
if (manual) return manual;
}
const resolved = resolveUniqueBook(candidates);
return resolved;
}
async resolveBookFilesByHashes(
@@ -197,46 +182,28 @@ export class KoreaderRepository {
.where(and(inArray(schema.bookFiles.fileHash, hashes), libraryFilter))
.orderBy(asc(schema.bookFiles.id));
const directHashes = addUnambiguousHashMatches(direct, result);
const duplicateHashes = new Set<string>();
const directCounts = new Map<string, number>();
for (const row of direct) {
if (!row.hash) continue;
directCounts.set(row.hash, (directCounts.get(row.hash) ?? 0) + 1);
}
for (const [hash, count] of directCounts) {
if (count > 1) duplicateHashes.add(hash);
}
const history = await this.db
.select({
hash: schema.bookFileHashHistory.fileHash,
bookFileId: schema.bookFiles.id,
bookId: schema.bookFiles.bookId,
libraryId: schema.books.libraryId,
format: schema.bookFiles.format,
})
.from(schema.bookFileHashHistory)
.innerJoin(schema.bookFiles, eq(schema.bookFiles.id, schema.bookFileHashHistory.bookFileId))
.innerJoin(schema.books, eq(schema.books.id, schema.bookFiles.bookId))
.where(and(inArray(schema.bookFileHashHistory.fileHash, hashes), libraryFilter))
.orderBy(asc(schema.bookFiles.id));
const missing = hashes.filter((hash) => !directHashes.has(hash));
if (missing.length > 0) {
const history = await this.db
.select({
hash: schema.bookFileHashHistory.fileHash,
bookFileId: schema.bookFiles.id,
bookId: schema.bookFiles.bookId,
libraryId: schema.books.libraryId,
format: schema.bookFiles.format,
})
.from(schema.bookFileHashHistory)
.innerJoin(schema.bookFiles, eq(schema.bookFiles.id, schema.bookFileHashHistory.bookFileId))
.innerJoin(schema.books, eq(schema.books.id, schema.bookFiles.bookId))
.where(and(inArray(schema.bookFileHashHistory.fileHash, missing), libraryFilter))
.orderBy(asc(schema.bookFiles.id));
addUnambiguousHashMatches(history, result);
const historyCounts = new Map<string, number>();
for (const row of history) {
if (!row.hash) continue;
historyCounts.set(row.hash, (historyCounts.get(row.hash) ?? 0) + 1);
}
for (const [hash, count] of historyCounts) {
if (count > 1) duplicateHashes.add(hash);
}
const uniqueFiles = new Map<string, HashedResolvedBookFile>();
for (const row of [...direct, ...history]) {
const key = `${row.hash}:${row.bookFileId}`;
if (!uniqueFiles.has(key)) uniqueFiles.set(key, row);
}
const linkCandidates = hashes.filter((hash) => !result.has(hash) || duplicateHashes.has(hash));
if (linkCandidates.length === 0 || userId === undefined) return result;
const candidates = [...uniqueFiles.values()];
addUnambiguousHashMatches(candidates, result);
if (userId === undefined) return result;
const manualLinks = await this.db
.select({
@@ -249,7 +216,7 @@ export class KoreaderRepository {
.from(schema.koreaderBookHashLinks)
.innerJoin(schema.bookFiles, eq(schema.bookFiles.id, schema.koreaderBookHashLinks.bookFileId))
.innerJoin(schema.books, eq(schema.books.id, schema.bookFiles.bookId))
.where(and(eq(schema.koreaderBookHashLinks.userId, userId), inArray(schema.koreaderBookHashLinks.hash, linkCandidates), libraryFilter));
.where(and(eq(schema.koreaderBookHashLinks.userId, userId), inArray(schema.koreaderBookHashLinks.hash, hashes), libraryFilter));
for (const row of manualLinks) {
result.set(row.hash, { bookFileId: row.bookFileId, bookId: row.bookId, libraryId: row.libraryId, format: row.format });
@@ -294,6 +261,7 @@ export class KoreaderRepository {
lastOpen: candidate.lastOpen ?? null,
source: candidate.source ?? 'statistics',
metadataAmbiguous: candidate.metadataAmbiguous ?? false,
manualLinkRequested: candidate.manualLinkRequested ?? false,
lastSeenAt: now,
})),
)
@@ -321,16 +289,19 @@ export class KoreaderRepository {
`,
source: sql`
case
when ${schema.koreaderUnmatchedBooks.manualLinkRequested} then 'file'
when ${incomingSourceRank} >= ${storedSourceRank} then excluded.source
else ${schema.koreaderUnmatchedBooks.source}
end
`,
metadataAmbiguous: sql`
case
when ${schema.koreaderUnmatchedBooks.manualLinkRequested} then false
when ${incomingSourceRank} >= ${storedSourceRank} then excluded.metadata_ambiguous
else ${schema.koreaderUnmatchedBooks.metadataAmbiguous}
end
`,
manualLinkRequested: sql`${schema.koreaderUnmatchedBooks.manualLinkRequested} or excluded.manual_link_requested`,
lastSeenAt: now,
},
});
@@ -351,11 +322,17 @@ export class KoreaderRepository {
});
}
async clearUnmatchedBooks(userId: number, hashes: string[]): Promise<void> {
async clearUnmatchedBooks(userId: number, hashes: string[], includeManualRequests = false): Promise<void> {
if (hashes.length === 0) return;
await this.db
.delete(schema.koreaderUnmatchedBooks)
.where(and(eq(schema.koreaderUnmatchedBooks.userId, userId), inArray(schema.koreaderUnmatchedBooks.hash, hashes)));
.where(
and(
eq(schema.koreaderUnmatchedBooks.userId, userId),
inArray(schema.koreaderUnmatchedBooks.hash, hashes),
includeManualRequests ? undefined : eq(schema.koreaderUnmatchedBooks.manualLinkRequested, false),
),
);
}
async dismissUnmatchedBook(userId: number, hash: string) {
@@ -485,6 +462,20 @@ export class KoreaderRepository {
});
}
async createBookHashLinkIfAbsent(userId: number, hash: string, bookFileId: number, metadata: KoreaderHashLinkMetadata = {}): Promise<void> {
await this.db.execute(sql`
insert into ${schema.koreaderBookHashLinks}
(user_id, hash, book_file_id, koreader_title, koreader_authors, koreader_last_open)
select ${userId}, ${hash}, ${bookFileId}, ${metadata.title?.trim() || null},
${metadata.authors?.trim() || null}, ${metadata.lastOpen ?? null}
where not exists (
select 1 from ${schema.koreaderUnmatchedBooks}
where user_id = ${userId} and hash = ${hash} and manual_link_requested = true
)
on conflict (user_id, hash) do nothing
`);
}
async getBookHashLink(userId: number, hash: string): Promise<{ bookFileId: number } | null> {
const [row] = await this.db
.select({ bookFileId: schema.koreaderBookHashLinks.bookFileId })
@@ -494,18 +485,25 @@ export class KoreaderRepository {
return row ?? null;
}
async deleteBookHashLink(userId: number, hash: string) {
const [row] = await this.db
.delete(schema.koreaderBookHashLinks)
.where(and(eq(schema.koreaderBookHashLinks.userId, userId), eq(schema.koreaderBookHashLinks.hash, hash)))
.returning({
hash: schema.koreaderBookHashLinks.hash,
bookFileId: schema.koreaderBookHashLinks.bookFileId,
koreaderTitle: schema.koreaderBookHashLinks.koreaderTitle,
koreaderAuthors: schema.koreaderBookHashLinks.koreaderAuthors,
koreaderLastOpen: schema.koreaderBookHashLinks.koreaderLastOpen,
});
return row ?? null;
async unlinkBookHashLink(userId: number, hash: string) {
const result = await this.db.execute<Pick<typeof schema.koreaderUnmatchedBooks.$inferSelect, 'hash'>>(sql`
with deleted as (
delete from ${schema.koreaderBookHashLinks}
where user_id = ${userId} and hash = ${hash}
returning *
)
insert into ${schema.koreaderUnmatchedBooks}
(user_id, hash, title, authors, last_open, source, metadata_ambiguous, manual_link_requested)
select user_id, hash, koreader_title, koreader_authors, koreader_last_open, 'file', false, true
from deleted
on conflict (user_id, hash) do update set
title = coalesce(excluded.title, koreader_unmatched_books.title),
authors = coalesce(excluded.authors, koreader_unmatched_books.authors),
last_open = coalesce(excluded.last_open, koreader_unmatched_books.last_open),
source = 'file', metadata_ambiguous = false, manual_link_requested = true, last_seen_at = now()
returning hash
`);
return result.rows[0] ?? null;
}
async getAccessibleLibraryIds(userId: number): Promise<number[] | null> {
server/src/modules/library/library.repository.ts
+3 -2
View File
@@ -1,3 +1,4 @@
import { deleteLibraryFoldersWithHashInvalidation } from '../../db/book-file-hash-history';
import { Inject, Injectable } from '@nestjs/common';
import { and, asc, eq, getTableColumns, gt, inArray, isNotNull, lte, or, sql } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
@@ -283,11 +284,11 @@ export class LibraryRepository {
}
deleteFolder(id: number) {
return this.db.delete(libraryFolders).where(eq(libraryFolders.id, id));
return deleteLibraryFoldersWithHashInvalidation(this.db, eq(libraryFolders.id, id));
}
deleteFoldersByLibrary(libraryId: number) {
return this.db.delete(libraryFolders).where(eq(libraryFolders.libraryId, libraryId));
return deleteLibraryFoldersWithHashInvalidation(this.db, eq(libraryFolders.libraryId, libraryId));
}
async updateDisplayOrders(order: { id: number; displayOrder: number }[]) {
server/src/modules/scanner/lib/hash.ts
+1 -33
View File
@@ -1,33 +1 @@
import { createHash } from 'crypto';
import { open } from 'fs/promises';
const BASE = 1024;
const BLOCK_SIZE = 1024;
/**
* Partial MD5 fingerprint matching KOReader's util.partialMD5 algorithm.
* Reads 1 KB blocks from positions 0, 1 KB, 4 KB, 16 KB, … up to 1 GB.
* The loop starts at i=-1 so that `BASE << (2 * -1)` overflows 32-bit to 0,
* producing the same byte sequence KOReader uses for its document hash.
*/
export async function computeFileHash(absolutePath: string): Promise<string> {
const fh = await open(absolutePath, 'r');
try {
const { size } = await fh.stat();
const hash = createHash('md5');
const buf = Buffer.allocUnsafe(BLOCK_SIZE);
for (let i = -1; i <= 10; i++) {
const position = BASE << (2 * i); // BASE << -2 overflows 32-bit to 0 when i = -1
if (position >= size) break;
const { bytesRead } = await fh.read(buf, 0, BLOCK_SIZE, position);
if (bytesRead > 0) {
hash.update(buf.subarray(0, bytesRead));
}
}
return hash.digest('hex');
} finally {
await fh.close();
}
}
export { computeFileHash } from '../../../common/utils/file-hash.utils';
server/src/modules/scanner/scanner.repository.test.ts
+2 -1
View File
@@ -47,6 +47,7 @@ function makeDb() {
const db: any = {};
Object.assign(db, {
execute: vi.fn().mockResolvedValue(undefined),
select: vi.fn(() => next('select')),
insert: vi.fn(() => next('insert')),
update: vi.fn(() => next('update')),
@@ -321,7 +322,7 @@ describe('ScannerRepository', () => {
await repo.deleteBookFile(21);
await repo.updateBookFolderPath(4, '/books/D');
expect(db.delete).toHaveBeenCalledTimes(1);
expect(db.execute).toHaveBeenCalledTimes(1);
expect(db.update).toHaveBeenCalledTimes(1);
});
server/src/modules/scanner/scanner.repository.ts
+3 -2
View File
@@ -1,3 +1,4 @@
import { deleteBookFilesWithHashInvalidation, deleteBooksWithHashInvalidation } from '../../db/book-file-hash-history';
import { Inject, Injectable } from '@nestjs/common';
import { and, asc, desc, eq, gte, inArray, lt, ne, or, sql } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
@@ -401,7 +402,7 @@ export class ScannerRepository {
.set({ orphaned: true, orphanedHash: file.fileHash })
.where(and(eq(koreaderDeviceProgress.bookFileId, id), eq(koreaderDeviceProgress.orphaned, false)));
}
await this.db.delete(bookFiles).where(eq(bookFiles.id, id));
await deleteBookFilesWithHashInvalidation(this.db, eq(bookFiles.id, id));
}
async findBookFileWithContextByIno(ino: bigint, libraryId?: number) {
@@ -503,7 +504,7 @@ export class ScannerRepository {
.limit(1);
if (!duplicate || duplicate.bookId === input.sourceBookId) return null;
await tx.delete(books).where(eq(books.id, input.duplicateBookId));
await deleteBooksWithHashInvalidation(tx, eq(books.id, input.duplicateBookId));
await tx
.update(books)
.set({
server/src/modules/upload/upload-processor.service.test.ts
+6 -5
View File
@@ -46,6 +46,7 @@ describe('UploadProcessorService', () => {
const deleteWhere = vi.fn();
const tx = {
execute: vi.fn().mockResolvedValue(undefined),
select: vi.fn(() => ({ from: selectFrom })),
insert: vi.fn((table: unknown) => {
if (table === books) {
@@ -633,9 +634,9 @@ describe('UploadProcessorService', () => {
it('clears the file rows before the books that own them', async () => {
await service.deleteUnitBookRecords({ bookIds: [42], createdBookIds: [42], attachedFileIds: [777], replacedPrimaries: [] });
expect(tx.delete).toHaveBeenNthCalledWith(1, bookFiles);
expect(tx.delete).toHaveBeenNthCalledWith(2, bookFiles);
expect(tx.delete).toHaveBeenNthCalledWith(3, books);
expect(dialect.sqlToQuery(tx.execute.mock.calls[0]![0]).sql).toContain('delete from "book_files"');
expect(dialect.sqlToQuery(tx.execute.mock.calls[1]![0]).sql).toContain('delete from "book_files"');
expect(dialect.sqlToQuery(tx.execute.mock.calls[2]![0]).sql).toContain('delete from "books"');
});
const dialect = new PgDialect();
@@ -649,9 +650,9 @@ describe('UploadProcessorService', () => {
replacedPrimaries: [{ bookId: 99, previousPrimaryFileId: 500, primaryFileId: 777 }],
});
expect(tx.delete).toHaveBeenCalledWith(bookFiles);
expect(tx.execute).toHaveBeenCalledTimes(1);
expect(updateBooksSet).toHaveBeenCalledWith({ primaryFileId: 500, updatedAt: expect.any(Date) });
expect(deleteWhere.mock.invocationCallOrder[0]).toBeLessThan(updateBooksSet.mock.invocationCallOrder[0]!);
expect(tx.execute.mock.invocationCallOrder[0]).toBeLessThan(updateBooksSet.mock.invocationCallOrder[0]!);
// Only a primary that was cleared, or is still the one the unit chose, is undone, and only
// onto a file that is still this book's.
server/src/modules/upload/upload-processor.service.ts
+4 -3
View File
@@ -1,3 +1,4 @@
import { deleteBookFilesWithHashInvalidation, deleteBooksWithHashInvalidation } from '../../db/book-file-hash-history';
import { Inject, Injectable, InternalServerErrorException, Logger, Optional } from '@nestjs/common';
import { stat } from 'fs/promises';
import { basename } from 'path';
@@ -213,14 +214,14 @@ export class UploadProcessorService {
await this.db.transaction(async (tx) => {
if (records.attachedFileIds.length > 0) {
await tx.delete(bookFiles).where(inArray(bookFiles.id, records.attachedFileIds));
await deleteBookFilesWithHashInvalidation(tx, inArray(bookFiles.id, records.attachedFileIds));
}
for (const replaced of records.replacedPrimaries) await this.restorePrimaryFile(tx, replaced);
if (records.createdBookIds.length > 0) {
// `books.primary_file_id` is `on delete set null`, so clearing the files first leaves
// nothing pointing at a row that is about to disappear.
await tx.delete(bookFiles).where(inArray(bookFiles.bookId, records.createdBookIds));
await tx.delete(books).where(inArray(books.id, records.createdBookIds));
await deleteBookFilesWithHashInvalidation(tx, inArray(bookFiles.bookId, records.createdBookIds));
await deleteBooksWithHashInvalidation(tx, inArray(books.id, records.createdBookIds));
}
});
// Metadata may already have written covers for the books this unit created.
server/test/kobo-koreader-download.e2e-spec.ts
+539
View File
@@ -0,0 +1,539 @@
import { randomUUID } from 'crypto';
import { readFile, readdir, stat, writeFile } from 'fs/promises';
import { join } from 'path';
import { and, eq, sql } from 'drizzle-orm';
import { Permission } from '@bookorbit/types';
import * as schema from '../src/db/schema';
import { deleteBookFilesWithHashInvalidation, deleteBooksWithHashInvalidation } from '../src/db/book-file-hash-history';
import { AudiolessEpubService } from '../src/modules/book/audioless-epub.service';
import { KoboSettingsService } from '../src/modules/kobo/services/kobo-settings.service';
import { KoboDownloadHashRegistrationService } from '../src/modules/kobo/services/kobo-download-hash-registration.service';
import { computeFileHash } from '../src/common/utils/file-hash.utils';
import { KoreaderRepository } from '../src/modules/koreader/koreader.repository';
import { AppSettingsService } from '../src/modules/app-settings/app-settings.service';
import { KoboDownloadRepository } from '../src/modules/kobo/kobo-download.repository';
import { KoboDownloadHashBackfillService } from '../src/modules/kobo/services/kobo-download-hash-backfill.service';
import { KepubConversionService } from '../src/modules/kobo/services/kepub-conversion.service';
import { createEpubFixture } from './e2e/reader-state-isolation/reader-state-isolation-fixture-builder';
import {
authHeader,
closeReaderStateIsolationE2EContext,
createLibraryWithFolder,
createReaderStateIsolationE2EContext,
createUserAndLogin,
grantLibraryAccess,
locateBookByAbsolutePath,
triggerAndWaitForLibraryScan,
type ReaderStateIsolationE2EContext,
type LocatedBookFile,
type TestUserSession,
} from './e2e/reader-state-isolation/reader-state-isolation-harness';
describe('Kobo download identity in KOReader (e2e)', { timeout: 180_000 }, () => {
let ctx: ReaderStateIsolationE2EContext;
let source: LocatedBookFile;
let other: LocatedBookFile;
let user: TestUserSession;
let outsider: TestUserSession;
let deviceToken: string;
let downloadPath: string;
let deliveredHash: string;
let originalHash: string;
function koHeaders(session = user) {
return { 'x-auth-user': session.username, 'x-auth-key': session.password };
}
async function match(hash: string, session = user, cachedFileId?: number) {
const response = await ctx.app.inject({
method: 'POST',
url: '/api/v1/koreader/plugin/match-check',
headers: koHeaders(session),
payload: {
deviceId: 'kobo-download-e2e',
deviceModel: 'Kobo',
pluginVersion: '1.5.5',
hashes: [hash],
...(cachedFileId ? { books: [{ hash, source: 'current_file', bookFileId: cachedFileId, title: 'Cached KOReader title' }] } : {}),
},
});
expect(response.statusCode).toBe(201);
return response.json() as { matches: { hash: string; bookId: number; bookFileId: number }[]; libraryVersion: string };
}
async function download() {
return ctx.app.inject({ method: 'GET', url: `/api/v1/kobo/${deviceToken}/v1/books/${source.bookId}/download` });
}
async function withCurrentHashCollision(run: () => Promise<void>) {
const [stored] = await ctx.db.select({ hash: schema.bookFiles.fileHash }).from(schema.bookFiles).where(eq(schema.bookFiles.id, other.bookFileId));
await ctx.db.update(schema.bookFiles).set({ fileHash: deliveredHash }).where(eq(schema.bookFiles.id, other.bookFileId));
try {
await run();
} finally {
await ctx.db.update(schema.bookFiles).set({ fileHash: stored!.hash }).where(eq(schema.bookFiles.id, other.bookFileId));
await ctx.db
.delete(schema.koreaderBookHashLinks)
.where(and(eq(schema.koreaderBookHashLinks.userId, user.userId), eq(schema.koreaderBookHashLinks.hash, deliveredHash)));
}
}
beforeAll(async () => {
ctx = await createReaderStateIsolationE2EContext();
const library = await createLibraryWithFolder(ctx);
const sourcePath = await createEpubFixture(library.folderPath, 'source.epub', {
title: 'Original download source',
uid: `urn:uuid:${randomUUID()}`,
});
const otherPath = await createEpubFixture(library.folderPath, 'other.epub', { title: 'Independent book', uid: `urn:uuid:${randomUUID()}` });
await triggerAndWaitForLibraryScan(ctx, library.libraryId);
source = await locateBookByAbsolutePath(ctx, sourcePath);
other = await locateBookByAbsolutePath(ctx, otherPath);
originalHash = await computeFileHash(sourcePath);
user = await createUserAndLogin(ctx, { permissions: [Permission.KoboSync, Permission.KoreaderSync] });
outsider = await createUserAndLogin(ctx, { permissions: [Permission.KoreaderSync] });
await grantLibraryAccess(ctx, user.userId, library.libraryId);
for (const session of [user, outsider]) {
const credentials = await ctx.app.inject({
method: 'POST',
url: '/api/v1/koreader/credentials',
headers: authHeader(session.accessToken),
payload: { username: session.username, password: session.password },
});
expect(credentials.statusCode).toBe(201);
}
const device = await ctx.app.inject({
method: 'POST',
url: '/api/v1/kobo/devices',
headers: authHeader(user.accessToken),
payload: { name: 'Kobo download regression' },
});
expect(device.statusCode).toBe(201);
deviceToken = (device.json() as { token: string }).token;
const collection = await ctx.app.inject({
method: 'POST',
url: '/api/v1/collections',
headers: authHeader(user.accessToken),
payload: { name: 'Kobo download regression', icon: 'book', syncToKobo: true },
});
expect(collection.statusCode).toBe(201);
const add = await ctx.app.inject({
method: 'POST',
url: `/api/v1/collections/${(collection.json() as { id: number }).id}/books`,
headers: authHeader(user.accessToken),
payload: { bookIds: [source.bookId] },
});
expect(add.statusCode).toBe(201);
const settings = await ctx.app.inject({
method: 'PATCH',
url: '/api/v1/kobo/settings',
headers: authHeader(user.accessToken),
payload: { convertToKepub: true, storeSync: false },
});
expect(settings.statusCode).toBe(200);
const sync = await ctx.app.inject({ method: 'GET', url: `/api/v1/kobo/${deviceToken}/v1/library/sync` });
expect(sync.statusCode).toBe(200);
// The external converter is substituted with a different valid EPUB. The download, hashing,
// history persistence, matching, access checks and progress endpoints are the real app.
downloadPath = await createEpubFixture(join(ctx.fixture.booksPath, '.kepub-cache', String(source.bookId)), `${originalHash}.kepub.epub`, {
title: 'Delivered Kobo artifact',
uid: `urn:uuid:${randomUUID()}`,
});
deliveredHash = await computeFileHash(downloadPath);
expect(deliveredHash).not.toBe(originalHash);
vi.spyOn(ctx.app.get(KepubConversionService), 'getKepubPath').mockResolvedValue(downloadPath);
});
afterAll(async () => {
vi.restoreAllMocks();
if (ctx) await closeReaderStateIsolationE2EContext(ctx);
});
it('matches a converted download with the unchanged KOReader request and leaves the source hash intact', async () => {
const before = await match(deliveredHash);
const outsiderBefore = await match(deliveredHash, outsider);
expect(before.matches).toEqual([]);
const response = await download();
expect(response.statusCode).toBe(200);
expect(response.rawPayload).toEqual(await readFile(downloadPath));
const after = await match(deliveredHash);
expect(after.matches).toEqual([{ hash: deliveredHash, bookId: source.bookId, bookFileId: source.bookFileId }]);
expect(after.libraryVersion).not.toBe(before.libraryVersion);
expect((await match(deliveredHash, outsider)).libraryVersion).toBe(outsiderBefore.libraryVersion);
const [stored] = await ctx.db
.select({ hash: schema.bookFiles.fileHash })
.from(schema.bookFiles)
.where(eq(schema.bookFiles.id, source.bookFileId));
expect(stored!.hash).toBe(originalHash);
expect((await match(originalHash)).matches[0]!.bookFileId).toBe(source.bookFileId);
});
it('registers cached/repeated downloads idempotently', async () => {
const version = (await match(deliveredHash)).libraryVersion;
expect((await download()).statusCode).toBe(200);
expect((await download()).statusCode).toBe(200);
const history = await ctx.db
.select()
.from(schema.bookFileHashHistory)
.where(and(eq(schema.bookFileHashHistory.bookFileId, source.bookFileId), eq(schema.bookFileHashHistory.fileHash, deliveredHash)));
expect(history).toHaveLength(1);
expect(history[0]!.reason).toBe('kobo_download');
expect((await match(deliveredHash)).libraryVersion).toBe(version);
});
it('ships a download while its library is locked, rolls back registration, then replays its queued identity', async () => {
const path = await createEpubFixture(ctx.fixture.rootPath, 'lock-wait.epub', { title: 'Lock wait regression', uid: `urn:uuid:${randomUUID()}` });
const hash = await computeFileHash(path);
const [book] = await ctx.db.select({ libraryId: schema.books.libraryId }).from(schema.books).where(eq(schema.books.id, source.bookId));
const [before] = await ctx.db
.select({ revision: schema.libraries.koreaderHashRevision })
.from(schema.libraries)
.where(eq(schema.libraries.id, book!.libraryId));
let release!: () => void;
let ready!: () => void;
const locked = new Promise<void>((resolve) => {
ready = resolve;
});
const hold = new Promise<void>((resolve) => {
release = resolve;
});
const blocker = ctx.db.transaction(async (tx) => {
await tx.execute(sql`select id from libraries where id = ${book!.libraryId} for update`);
ready();
await hold;
});
// Surface a setup failure without waiting indefinitely for the ready signal.
await Promise.race([locked, blocker]);
try {
const duplicateStart = Date.now();
expect((await download()).statusCode).toBe(200);
expect(Date.now() - duplicateStart).toBeLessThan(1000);
vi.spyOn(ctx.app.get(KepubConversionService), 'getKepubPath').mockResolvedValue(path);
const startedAt = Date.now();
const response = await download();
const durationMs = Date.now() - startedAt;
expect(response.statusCode).toBe(200);
expect(response.rawPayload).toEqual(await readFile(path));
expect(durationMs).toBeGreaterThanOrEqual(850);
expect(durationMs).toBeLessThan(5000);
expect((await match(hash)).matches).toEqual([]);
expect(await readdir(join(ctx.fixture.booksPath, '.kobo-download-hashes'))).toContain(`${source.bookFileId}-${hash}`);
const history = await ctx.db.select().from(schema.bookFileHashHistory).where(eq(schema.bookFileHashHistory.fileHash, hash));
expect(history).toEqual([]);
const [during] = await ctx.db
.select({ revision: schema.libraries.koreaderHashRevision })
.from(schema.libraries)
.where(eq(schema.libraries.id, book!.libraryId));
expect(during!.revision).toBe(before!.revision);
} finally {
release();
await blocker;
vi.spyOn(ctx.app.get(KepubConversionService), 'getKepubPath').mockResolvedValue(downloadPath);
}
await ctx.app.get(KoboDownloadHashRegistrationService).replay();
expect((await match(hash)).matches).toEqual([{ hash, bookId: source.bookId, bookFileId: source.bookFileId }]);
const [after] = await ctx.db
.select({ revision: schema.libraries.koreaderHashRevision })
.from(schema.libraries)
.where(eq(schema.libraries.id, book!.libraryId));
expect(after!.revision).toBe(before!.revision + 1);
});
it('does not expose the generated identity to a user without access to the source library', async () => {
expect((await match(deliveredHash, outsider)).matches).toEqual([]);
});
it('records KOReader progress on the source book file after a converted download', async () => {
const response = await ctx.app.inject({
method: 'PUT',
url: '/api/v1/koreader/syncs/progress',
headers: koHeaders(),
payload: { document: deliveredHash, percentage: 0.25, device: 'Kobo regression', device_id: 'kobo-download-e2e' },
});
expect(response.statusCode).toBe(200);
const progress = await ctx.db
.select()
.from(schema.koreaderDeviceProgress)
.where(and(eq(schema.koreaderDeviceProgress.userId, user.userId), eq(schema.koreaderDeviceProgress.bookFileId, source.bookFileId)));
expect(progress).toHaveLength(1);
expect(progress[0]!.percentage).toBeCloseTo(0.25);
expect(progress[0]!.orphaned).toBe(false);
});
it('allows relink, unlink, automatic sync, list, and link again after intrinsic registration', async () => {
await ctx.db.insert(schema.koreaderBookHashLinks).values({ userId: user.userId, hash: deliveredHash, bookFileId: source.bookFileId });
try {
const relinked = await ctx.app.inject({
method: 'PATCH',
url: `/api/v1/koreader/hash-links/${deliveredHash}`,
headers: authHeader(user.accessToken),
payload: { bookId: other.bookId },
});
expect(relinked.statusCode).toBe(200);
expect((await match(deliveredHash)).matches[0]!.bookFileId).toBe(other.bookFileId);
expect((await match(deliveredHash, user, source.bookFileId)).matches[0]!.bookFileId).toBe(other.bookFileId);
const [preserved] = await ctx.db
.select()
.from(schema.koreaderBookHashLinks)
.where(and(eq(schema.koreaderBookHashLinks.userId, user.userId), eq(schema.koreaderBookHashLinks.hash, deliveredHash)));
expect(preserved!.bookFileId).toBe(other.bookFileId);
const unlinked = await ctx.app.inject({
method: 'DELETE',
url: `/api/v1/koreader/hash-links/${deliveredHash}`,
headers: authHeader(user.accessToken),
});
expect(unlinked.statusCode).toBe(200);
// Background matching resumes intrinsic resolution without removing the user's correction queue.
expect((await match(deliveredHash)).matches[0]!.bookFileId).toBe(source.bookFileId);
await ctx.app
.get(KoreaderRepository)
.upsertUnmatchedBooks(user.userId, [
{ hash: deliveredHash, source: 'current_file', metadataAmbiguous: true, title: 'Ambiguous metadata from background sync' },
]);
const pending = await ctx.app.inject({ method: 'GET', url: '/api/v1/koreader/unmatched-books', headers: authHeader(user.accessToken) });
expect(pending.statusCode).toBe(200);
expect(pending.json()).toEqual(expect.arrayContaining([expect.objectContaining({ hash: deliveredHash })]));
const linked = await ctx.app.inject({
method: 'POST',
url: `/api/v1/koreader/unmatched-books/${deliveredHash}/link`,
headers: authHeader(user.accessToken),
payload: { bookId: other.bookId },
});
expect(linked.statusCode).toBe(201);
expect((await match(deliveredHash)).matches[0]!.bookFileId).toBe(other.bookFileId);
const after = await ctx.app.inject({ method: 'GET', url: '/api/v1/koreader/unmatched-books', headers: authHeader(user.accessToken) });
expect(after.json()).not.toEqual(expect.arrayContaining([expect.objectContaining({ hash: deliveredHash })]));
} finally {
await ctx.db
.delete(schema.koreaderBookHashLinks)
.where(and(eq(schema.koreaderBookHashLinks.userId, user.userId), eq(schema.koreaderBookHashLinks.hash, deliveredHash)));
await ctx.db
.delete(schema.koreaderUnmatchedBooks)
.where(and(eq(schema.koreaderUnmatchedBooks.userId, user.userId), eq(schema.koreaderUnmatchedBooks.hash, deliveredHash)));
}
});
it('does not allow another user to relink or unlink a manual override', async () => {
await ctx.db.insert(schema.koreaderBookHashLinks).values({ userId: user.userId, hash: deliveredHash, bookFileId: source.bookFileId });
try {
for (const method of ['PATCH', 'DELETE'] as const) {
const response = await ctx.app.inject({
method,
url: `/api/v1/koreader/hash-links/${deliveredHash}`,
headers: authHeader(outsider.accessToken),
...(method === 'PATCH' ? { payload: { bookId: other.bookId } } : {}),
});
expect(response.statusCode).toBe(404);
}
expect((await match(deliveredHash)).matches[0]!.bookFileId).toBe(source.bookFileId);
} finally {
await ctx.db
.delete(schema.koreaderBookHashLinks)
.where(and(eq(schema.koreaderBookHashLinks.userId, user.userId), eq(schema.koreaderBookHashLinks.hash, deliveredHash)));
}
});
it('delivers an uncached audioless EPUB during a database outage and recovers identity after temporary cleanup', async () => {
const repository = ctx.app.get(KoboDownloadRepository);
const settings = ctx.app.get(KoboSettingsService);
const storedSettings = await settings.getSettings(user.userId);
const settingsSpy = vi.spyOn(settings, 'getSettings').mockResolvedValue({ ...storedSettings, convertToKepub: false });
const temporaryPaths: string[] = [];
const stripSpy = vi.spyOn(ctx.app.get(AudiolessEpubService), 'writeArchive').mockImplementation(async (input, output) => {
temporaryPaths.push(output);
await writeFile(output, Buffer.concat([await readFile(input), Buffer.from('narration-free delivery regression')]));
return { removedEntries: 1, sanitizedEntries: 1 };
});
await ctx.db.update(schema.bookFiles).set({ mediaOverlayAvailable: true }).where(eq(schema.bookFiles.id, source.bookFileId));
const registrationSpy = vi.spyOn(repository, 'registerDeliveredHash').mockRejectedValue(new Error('Transient database outage'));
let queuedHash: string;
try {
const response = await download();
expect(response.statusCode).toBe(200);
expect(response.rawPayload.length).toBeGreaterThan(0);
queuedHash = registrationSpy.mock.calls[0]![1];
expect(queuedHash).not.toBe(originalHash);
expect((await match(queuedHash)).matches).toEqual([]);
} finally {
registrationSpy.mockRestore();
settingsSpy.mockRestore();
stripSpy.mockRestore();
await ctx.db.update(schema.bookFiles).set({ mediaOverlayAvailable: false }).where(eq(schema.bookFiles.id, source.bookFileId));
}
expect(temporaryPaths).toHaveLength(1);
await expect(stat(temporaryPaths[0]!)).rejects.toMatchObject({ code: 'ENOENT' });
await ctx.app.get(KoboDownloadHashRegistrationService).replay();
expect((await match(queuedHash)).matches).toEqual([{ hash: queuedHash, bookId: source.bookId, bookFileId: source.bookFileId }]);
});
it('increments the scoped revision once for concurrent duplicate registrations and supports rollback', async () => {
const repository = ctx.app.get(KoboDownloadRepository);
const before = (await match(deliveredHash)).libraryVersion;
const outsiderBefore = (await match(deliveredHash, outsider)).libraryVersion;
const [sourceBook] = await ctx.db.select({ libraryId: schema.books.libraryId }).from(schema.books).where(eq(schema.books.id, source.bookId));
const [beforeRevision] = await ctx.db
.select({ revision: schema.libraries.koreaderHashRevision })
.from(schema.libraries)
.where(eq(schema.libraries.id, sourceBook!.libraryId));
const hash = 'd'.repeat(32);
await Promise.all(Array.from({ length: 10 }, () => repository.registerDeliveredHash(source.bookFileId, hash)));
const rows = await ctx.db
.select()
.from(schema.bookFileHashHistory)
.where(and(eq(schema.bookFileHashHistory.bookFileId, source.bookFileId), eq(schema.bookFileHashHistory.fileHash, hash)));
expect(rows).toHaveLength(1);
const [afterRevision] = await ctx.db
.select({ revision: schema.libraries.koreaderHashRevision })
.from(schema.libraries)
.where(eq(schema.libraries.id, sourceBook!.libraryId));
expect(afterRevision!.revision).toBe(beforeRevision!.revision + 1);
const after = (await match(deliveredHash)).libraryVersion;
expect(after).not.toBe(before);
expect((await match(deliveredHash, outsider)).libraryVersion).toBe(outsiderBefore);
await repository.registerDeliveredHash(source.bookFileId, hash);
expect((await match(deliveredHash)).libraryVersion).toBe(after);
await expect(
ctx.db.transaction(async (tx) => {
await deleteBookFilesWithHashInvalidation(tx, eq(schema.bookFiles.id, source.bookFileId));
throw new Error('Roll back deletion');
}),
).rejects.toThrow('Roll back deletion');
expect((await match(deliveredHash)).libraryVersion).toBe(after);
expect((await match(deliveredHash)).matches[0]!.bookFileId).toBe(source.bookFileId);
});
it('uses the timestamp expression index for the bounded version lookup', async () => {
await ctx.db.transaction(async (tx) => {
await tx.execute(sql`set local enable_seqscan = off`);
const plan = await tx.execute(
sql`explain (format json) select greatest(created_at, updated_at) from book_files order by greatest(created_at, updated_at) desc limit 1`,
);
const serialized = JSON.stringify(plan.rows);
expect(serialized).toContain('book_files_change_timestamp_idx');
expect(serialized).not.toContain('Seq Scan');
});
});
it('preserves an existing explicit link even when a subsequent download registers a different intrinsic identity', async () => {
await ctx.db.insert(schema.koreaderBookHashLinks).values({ userId: user.userId, hash: deliveredHash, bookFileId: other.bookFileId });
try {
expect((await download()).statusCode).toBe(200);
expect((await match(deliveredHash)).matches).toEqual([{ hash: deliveredHash, bookId: other.bookId, bookFileId: other.bookFileId }]);
const response = await ctx.app.inject({
method: 'PUT',
url: '/api/v1/koreader/syncs/progress',
headers: koHeaders(),
payload: { document: deliveredHash, percentage: 0.6, device: 'Kobo regression', device_id: 'explicit-prior-link' },
});
expect(response.statusCode).toBe(200);
const progress = await ctx.db
.select()
.from(schema.koreaderDeviceProgress)
.where(and(eq(schema.koreaderDeviceProgress.userId, user.userId), eq(schema.koreaderDeviceProgress.deviceId, 'explicit-prior-link')));
expect(progress).toHaveLength(1);
expect(progress[0]!.bookFileId).toBe(other.bookFileId);
} finally {
await ctx.db
.delete(schema.koreaderBookHashLinks)
.where(and(eq(schema.koreaderBookHashLinks.userId, user.userId), eq(schema.koreaderBookHashLinks.hash, deliveredHash)));
}
});
it('rejects a delivered/current hash collision instead of assigning progress to the other book', async () => {
await withCurrentHashCollision(async () => {
expect((await match(deliveredHash)).matches).toEqual([]);
const before = await ctx.db.select().from(schema.koreaderDeviceProgress).where(eq(schema.koreaderDeviceProgress.userId, user.userId));
const response = await ctx.app.inject({
method: 'PUT',
url: '/api/v1/koreader/syncs/progress',
headers: koHeaders(),
payload: { document: deliveredHash, percentage: 0.99, device: 'Kobo regression', device_id: 'cross-source-collision' },
});
expect(response.statusCode).toBe(404);
const after = await ctx.db.select().from(schema.koreaderDeviceProgress).where(eq(schema.koreaderDeviceProgress.userId, user.userId));
expect(after).toEqual(before);
});
});
it('honors an explicit source-file link for a delivered/current collision on match and progress', async () => {
await withCurrentHashCollision(async () => {
await ctx.db.insert(schema.koreaderBookHashLinks).values({ userId: user.userId, hash: deliveredHash, bookFileId: source.bookFileId });
expect((await download()).statusCode).toBe(200);
expect((await match(deliveredHash)).matches).toEqual([{ hash: deliveredHash, bookId: source.bookId, bookFileId: source.bookFileId }]);
const response = await ctx.app.inject({
method: 'PUT',
url: '/api/v1/koreader/syncs/progress',
headers: koHeaders(),
payload: { document: deliveredHash, percentage: 0.5, device: 'Kobo regression', device_id: 'cross-source-linked' },
});
expect(response.statusCode).toBe(200);
const progress = await ctx.db
.select()
.from(schema.koreaderDeviceProgress)
.where(and(eq(schema.koreaderDeviceProgress.userId, user.userId), eq(schema.koreaderDeviceProgress.deviceId, 'cross-source-linked')));
expect(progress).toHaveLength(1);
expect(progress[0]!.bookFileId).toBe(source.bookFileId);
expect(progress[0]!.percentage).toBeCloseTo(0.5);
});
});
it('preserves ambiguity when two source books have the same delivered hash', async () => {
const repository = ctx.app.get(KoboDownloadRepository);
await repository.registerDeliveredHash(other.bookFileId, deliveredHash);
expect((await match(deliveredHash)).matches).toEqual([]);
});
it('leaves explicit user disambiguation intact across subsequent downloads', async () => {
await ctx.db.insert(schema.koreaderBookHashLinks).values({ userId: user.userId, hash: deliveredHash, bookFileId: source.bookFileId });
expect((await download()).statusCode).toBe(200);
expect((await match(deliveredHash)).matches[0]!.bookFileId).toBe(source.bookFileId);
const [link] = await ctx.db
.select()
.from(schema.koreaderBookHashLinks)
.where(and(eq(schema.koreaderBookHashLinks.userId, user.userId), eq(schema.koreaderBookHashLinks.hash, deliveredHash)));
expect(link!.bookFileId).toBe(source.bookFileId);
});
it('recovers an already downloaded artifact from the retained cache without another download', async () => {
await ctx.db.delete(schema.bookFileHashHistory).where(eq(schema.bookFileHashHistory.fileHash, deliveredHash));
await ctx.db.delete(schema.koreaderBookHashLinks).where(eq(schema.koreaderBookHashLinks.hash, deliveredHash));
expect((await match(deliveredHash)).matches).toEqual([]);
await ctx.app.get(AppSettingsService).setValue('kobo_download_hash_backfill', '0');
const recovered = await ctx.app.get(KoboDownloadHashBackfillService).run();
expect(recovered.registered).toBeGreaterThanOrEqual(1);
expect(recovered.failed).toBe(0);
expect((await match(deliveredHash)).matches).toEqual([{ hash: deliveredHash, bookId: source.bookId, bookFileId: source.bookFileId }]);
});
it('does not retarget a retained conversion after its original source identity is lost', async () => {
const repository = ctx.app.get(KoboDownloadRepository);
await expect(repository.findCachedSourceFileId(source.bookId, originalHash)).resolves.toBe(source.bookFileId);
await ctx.db
.update(schema.bookFiles)
.set({ fileHash: 'c'.repeat(32) })
.where(eq(schema.bookFiles.id, source.bookFileId));
await expect(repository.findCachedSourceFileId(source.bookId, originalHash)).resolves.toBeNull();
await ctx.db.insert(schema.bookFileHashHistory).values({ bookFileId: source.bookFileId, fileHash: originalHash, reason: 'external_change' });
await expect(repository.findCachedSourceFileId(source.bookId, originalHash)).resolves.toBe(source.bookFileId);
});
it('invalidates a cached token when a non-newest book and its historical identities are deleted', async () => {
const library = await createLibraryWithFolder(ctx);
const path = await createEpubFixture(library.folderPath, 'deleted.epub', { title: 'Deletion regression', uid: `urn:uuid:${randomUUID()}` });
await triggerAndWaitForLibraryScan(ctx, library.libraryId);
const target = await locateBookByAbsolutePath(ctx, path);
await grantLibraryAccess(ctx, user.userId, library.libraryId);
const repository = ctx.app.get(KoboDownloadRepository);
const hash = 'e'.repeat(32);
await repository.registerDeliveredHash(target.bookFileId, hash);
await ctx.db
.update(schema.bookFiles)
.set({ createdAt: new Date('2000-01-01'), updatedAt: new Date('2000-01-01') })
.where(eq(schema.bookFiles.id, target.bookFileId));
const before = (await match(hash)).libraryVersion;
await deleteBooksWithHashInvalidation(ctx.db, eq(schema.books.id, target.bookId));
const after = await match(hash);
expect(after.libraryVersion).not.toBe(before);
expect(after.matches).toEqual([]);
});
});
server/test/koreader-bulk-manifest.e2e-spec.ts
+29
View File
@@ -179,4 +179,33 @@ describe('KOReader bulk manifest at scale (e2e)', { timeout: 600_000 }, () => {
// have no usable index, which is the regression this fixture exists to catch.
expect(booksScan!['Node Type']).toMatch(/^Index (Only )?Scan$/);
});
it('keeps version queries bounded with 30,000 files and historical identities', async () => {
await ctx.db.execute(sql`
insert into book_file_hash_history (book_file_id, file_hash, reason)
select f.id, f.file_hash, 'rescan' from book_files f
join books b on b.id = f.book_id
where b.library_id in (${library.libraryId}, ${decoyLibrary.libraryId})
on conflict (book_file_id, file_hash) do nothing
`);
await ctx.db.execute(sql`analyze book_file_hash_history`);
const timestampPlan = await ctx.db.execute(sql`
explain (analyze, buffers, format json)
select greatest(created_at, updated_at) from book_files
order by greatest(created_at, updated_at) desc limit 1
`);
const scans = collectScanNodes(timestampPlan.rows[0]!['QUERY PLAN']);
expect(scans).toHaveLength(1);
expect(scans[0]!['Node Type']).toContain('Index');
expect(JSON.stringify(timestampPlan.rows)).toContain('book_files_change_timestamp_idx');
expect((scans[0] as { 'Actual Rows': number })['Actual Rows']).toBe(1);
const revisionPlan = await ctx.db.execute(sql`
explain (analyze, buffers, format json)
select id, koreader_hash_revision::text from libraries
where id = ${library.libraryId} order by id
`);
const revisionScans = collectScanNodes(revisionPlan.rows[0]!['QUERY PLAN']);
expect(revisionScans).toHaveLength(1);
expect(revisionScans[0]!['Relation Name']).toBe('libraries');
expect((revisionScans[0] as { 'Actual Rows': number })['Actual Rows']).toBe(1);
});
});