fix(koreader): keep manual links to a different copy matched and suggest links

Report a linked copy whose bytes differ from the library file as positionIncompatible instead of
unmatched, so the plugin keeps the match, skips highlights without a failure and still syncs progress.
Treat a position-less place within a page as the same place and jump to the nearest page.
Suggest library books with the same title and authors for books to review, with bulk linking.
See docs/KOREADER_HASH_MATCHING.md.

Closes #1872
Closes #1919
This commit is contained in:
neon
2026-10-26 15:06:04 +00:00
parent 834da8c954
commit 1c27e5dd05
36 changed files with 1634 additions and 63 deletions
@@ -127,8 +127,9 @@ describe('KoreaderAnnotationExchangeService', () => {
books: [{ hash: HASH_A, applied: [], deleted: [] }],
});
expect(result).toEqual({ results: [], unmatched: [HASH_A] });
expect(ack).toEqual({ results: [], unmatched: [HASH_A] });
// Reported apart from `unmatched`: plugins drop the match of a book listed there.
expect(result).toEqual({ results: [], unmatched: [], positionIncompatible: [HASH_A] });
expect(ack).toEqual({ results: [], unmatched: [], positionIncompatible: [HASH_A] });
expect(annotationSync.ingestDeviceAnnotations).not.toHaveBeenCalled();
expect(annotationSync.applyExchangeAck).not.toHaveBeenCalled();
});
@@ -88,11 +88,17 @@ export interface ExchangeBookResult {
export interface ExchangeResponse {
results: ExchangeBookResult[];
unmatched: string[];
/**
* Matched books whose device copy differs from the library file, so its positions cannot be placed.
* Reported apart from `unmatched`, which plugins read as "not in your library" and act on.
*/
positionIncompatible: string[];
}
export interface ExchangeAckResponse {
results: { hash: string; acked: number }[];
unmatched: string[];
positionIncompatible: string[];
}
@Injectable()
@@ -125,15 +131,20 @@ export class KoreaderAnnotationExchangeService {
const results: ExchangeBookResult[] = [];
const unmatched: string[] = [];
const positionIncompatible: string[] = [];
let pushedTotal = 0;
for (const book of dto.books) {
const hash = book.hash.toLowerCase();
const match = matches.get(hash);
if (!match || match.progressCompatible === false) {
if (!match) {
unmatched.push(hash);
continue;
}
if (match.progressCompatible === false) {
positionIncompatible.push(hash);
continue;
}
const result = await this.exchangeBook(
user.id,
exchangeDeviceId(dto.deviceId, hash),
@@ -149,9 +160,9 @@ export class KoreaderAnnotationExchangeService {
}
this.logger.log(
`[${EXCHANGE_EVENT}] [end] userId=${user.id} deviceId=${dto.deviceId.slice(0, 8)} durationMs=${Date.now() - startedAtMs} books=${results.length} pushed=${pushedTotal} unmatched=${unmatched.length} - annotation exchange completed`,
`[${EXCHANGE_EVENT}] [end] userId=${user.id} deviceId=${dto.deviceId.slice(0, 8)} durationMs=${Date.now() - startedAtMs} books=${results.length} pushed=${pushedTotal} unmatched=${unmatched.length} positionIncompatible=${positionIncompatible.length} - annotation exchange completed`,
);
return { results, unmatched };
return { results, unmatched, positionIncompatible };
} catch (error) {
const errorClass = error instanceof Error ? error.constructor.name : 'UnknownError';
this.logger.warn(
@@ -198,13 +209,18 @@ export class KoreaderAnnotationExchangeService {
const results: { hash: string; acked: number }[] = [];
const unmatched: string[] = [];
const positionIncompatible: string[] = [];
for (const book of dto.books) {
const hash = book.hash.toLowerCase();
const match = matches.get(hash);
if (!match || match.progressCompatible === false) {
if (!match) {
unmatched.push(hash);
continue;
}
if (match.progressCompatible === false) {
positionIncompatible.push(hash);
continue;
}
const { acked } = await this.annotationSync.applyExchangeAck({
userId: user.id,
source: 'koreader',
@@ -229,9 +245,9 @@ export class KoreaderAnnotationExchangeService {
}
this.logger.log(
`[${ACK_EVENT}] [end] userId=${user.id} deviceId=${dto.deviceId.slice(0, 8)} durationMs=${Date.now() - startedAtMs} books=${results.length} unmatched=${unmatched.length} - annotation exchange ack applied`,
`[${ACK_EVENT}] [end] userId=${user.id} deviceId=${dto.deviceId.slice(0, 8)} durationMs=${Date.now() - startedAtMs} books=${results.length} unmatched=${unmatched.length} positionIncompatible=${positionIncompatible.length} - annotation exchange ack applied`,
);
return { results, unmatched };
return { results, unmatched, positionIncompatible };
} catch (error) {
const errorClass = error instanceof Error ? error.constructor.name : 'UnknownError';
this.logger.warn(
@@ -100,10 +100,26 @@ describe('KoreaderBookmarkExchangeService', () => {
const result = await service.exchange(makeUser(), makeExchangeDto([makeBook({ hash: HASH_B })]));
expect(result.unmatched).toEqual([HASH_B]);
expect(result.positionIncompatible).toEqual([]);
expect(result.results).toEqual([]);
expect(bookmarkSync.listForSync).not.toHaveBeenCalled();
});
it('reports a matched copy whose bytes differ from the library file apart from unmatched books', async () => {
koreaderRepo.resolveBookFilesByHashes.mockResolvedValue(
new Map([[HASH_A, { bookFileId: 10, bookId: 20, libraryId: 1, progressCompatible: false }]]),
);
const result = await service.exchange(
makeUser(),
makeExchangeDto([makeBook({ changes: [{ datetime: DATETIME, pos: POS, pageno: 42, chapter: 'Chapter 3' }] }), makeBook({ hash: HASH_B })]),
);
expect(result).toEqual({ results: [], unmatched: [HASH_B], positionIncompatible: [HASH_A] });
expect(bookmarkSync.upsertFromDevice).not.toHaveBeenCalled();
expect(bookmarkSync.listForSync).not.toHaveBeenCalled();
});
it('creates a bookmark from an unknown device key and links it', async () => {
const created = makeBookmarkRow({ id: 101, origin: 'koreader', devicePos: POS, pageno: 42 });
bookmarkSync.upsertFromDevice.mockResolvedValue({ row: created, outcome: 'created' });
@@ -373,6 +389,20 @@ describe('KoreaderBookmarkExchangeService', () => {
return { deviceId: DEVICE_ID, deviceModel: 'Kobo', pluginVersion: '0.5.0', books } as BookmarkExchangeAckDto;
}
it('acknowledges nothing for a copy whose bytes differ from the library file', async () => {
koreaderRepo.resolveBookFilesByHashes.mockResolvedValue(
new Map([[HASH_A, { bookFileId: 10, bookId: 20, libraryId: 1, progressCompatible: false }]]),
);
const result = await service.exchangeAck(
makeUser(),
makeAckDto([{ hash: HASH_A, applied: [{ serverId: 100, status: 'applied', key: 'f'.repeat(32), datetime: DATETIME }], deleted: [] }]),
);
expect(result).toEqual({ results: [], unmatched: [], positionIncompatible: [HASH_A] });
expect(bookmarkRepo.upsertLink).not.toHaveBeenCalled();
});
it('links an applied bookmark under the identity the device reported', async () => {
bookmarkSync.listForSync.mockResolvedValue([makeBookmarkRow()]);
@@ -55,11 +55,14 @@ export interface BookmarkExchangeBookResult {
export interface BookmarkExchangeResponse {
results: BookmarkExchangeBookResult[];
unmatched: string[];
/** Matched books whose device copy differs from the library file; see the annotation exchange. */
positionIncompatible: string[];
}
export interface BookmarkExchangeAckResponse {
results: { hash: string; acked: number }[];
unmatched: string[];
positionIncompatible: string[];
}
/** The device identity of a bookmark: md5 of `${datetime}|${pos}`, matching the plugin. */
@@ -89,15 +92,20 @@ export class KoreaderBookmarkExchangeService {
const matches = await this.resolveBooks(user.id, dto.books);
const results: BookmarkExchangeBookResult[] = [];
const unmatched: string[] = [];
const positionIncompatible: string[] = [];
let pushedTotal = 0;
for (const book of dto.books) {
const hash = book.hash.toLowerCase();
const match = matches.get(hash);
if (!match || match.progressCompatible === false) {
if (!match) {
unmatched.push(hash);
continue;
}
if (match.progressCompatible === false) {
positionIncompatible.push(hash);
continue;
}
const result = await this.exchangeBook(user.id, exchangeDeviceId(dto.deviceId, hash), hash, match.bookId, match.bookFileId, book);
pushedTotal += result.toApply.add.length + result.toApply.delete.length;
results.push(result);
@@ -106,9 +114,9 @@ export class KoreaderBookmarkExchangeService {
await this.purgeExpiredTombstones(user.id);
this.logger.log(
`[${EXCHANGE_EVENT}] [end] userId=${user.id} deviceId=${dto.deviceId.slice(0, 8)} durationMs=${Date.now() - startedAtMs} books=${results.length} pushed=${pushedTotal} unmatched=${unmatched.length} - bookmark exchange completed`,
`[${EXCHANGE_EVENT}] [end] userId=${user.id} deviceId=${dto.deviceId.slice(0, 8)} durationMs=${Date.now() - startedAtMs} books=${results.length} pushed=${pushedTotal} unmatched=${unmatched.length} positionIncompatible=${positionIncompatible.length} - bookmark exchange completed`,
);
return { results, unmatched };
return { results, unmatched, positionIncompatible };
} catch (error) {
this.logFailure(EXCHANGE_EVENT, user.id, dto.deviceId, startedAtMs, error);
throw error;
@@ -121,22 +129,27 @@ export class KoreaderBookmarkExchangeService {
const matches = await this.resolveBooks(user.id, dto.books);
const results: { hash: string; acked: number }[] = [];
const unmatched: string[] = [];
const positionIncompatible: string[] = [];
for (const book of dto.books) {
const hash = book.hash.toLowerCase();
const match = matches.get(hash);
if (!match || match.progressCompatible === false) {
if (!match) {
unmatched.push(hash);
continue;
}
if (match.progressCompatible === false) {
positionIncompatible.push(hash);
continue;
}
const acked = await this.ackBook(user.id, exchangeDeviceId(dto.deviceId, hash), match.bookId, book);
results.push({ hash, acked });
}
this.logger.log(
`[${ACK_EVENT}] [end] userId=${user.id} deviceId=${dto.deviceId.slice(0, 8)} durationMs=${Date.now() - startedAtMs} books=${results.length} unmatched=${unmatched.length} - bookmark exchange ack applied`,
`[${ACK_EVENT}] [end] userId=${user.id} deviceId=${dto.deviceId.slice(0, 8)} durationMs=${Date.now() - startedAtMs} books=${results.length} unmatched=${unmatched.length} positionIncompatible=${positionIncompatible.length} - bookmark exchange ack applied`,
);
return { results, unmatched };
return { results, unmatched, positionIncompatible };
} catch (error) {
this.logFailure(ACK_EVENT, user.id, dto.deviceId, startedAtMs, error);
throw error;
@@ -26,6 +26,7 @@ describe('KoreaderHashLinkService', () => {
findBookFileIdByBookId: ReturnType<typeof vi.fn>;
dismissUnmatchedBook: ReturnType<typeof vi.fn>;
dismissAllUnmatchedBooks: ReturnType<typeof vi.fn>;
findBooksByTitleKeys: ReturnType<typeof vi.fn>;
};
let mockBookService: {
verifyBookAccess: ReturnType<typeof vi.fn>;
@@ -69,6 +70,7 @@ describe('KoreaderHashLinkService', () => {
findBookFileIdByBookId: vi.fn(),
dismissUnmatchedBook: vi.fn().mockResolvedValue(null),
dismissAllUnmatchedBooks: vi.fn().mockResolvedValue(0),
findBooksByTitleKeys: vi.fn().mockResolvedValue([]),
};
mockBookService = {
@@ -164,6 +166,149 @@ describe('KoreaderHashLinkService', () => {
});
});
describe('listUnmatchedBookSuggestions', () => {
function makeLibraryBook(overrides: Record<string, unknown> = {}) {
return {
bookId: 501,
titleKey: 'lab alpha',
title: 'Lab Alpha',
subtitle: null,
language: 'en',
publishedYear: 2026,
format: 'epub',
role: 'content',
libraryName: 'Novels',
authors: ['Ada Tester'],
...overrides,
};
}
it('suggests the one library book with the same title and the same set of authors', async () => {
mockRepo.listUnmatchedBooks.mockResolvedValue([makeUnmatchedRow({ title: ' Lab ALPHA ', authors: 'Bo Sample\nAda Tester' })]);
mockRepo.findBooksByTitleKeys.mockResolvedValue([makeLibraryBook({ authors: ['Ada Tester', 'Bo Sample'] })]);
await expect(service.listUnmatchedBookSuggestions(user)).resolves.toEqual([
{
hash: HASH_A,
candidates: [
{
bookId: 501,
title: 'Lab Alpha',
subtitle: null,
authors: ['Ada Tester', 'Bo Sample'],
format: 'epub',
language: 'en',
publishedYear: 2026,
libraryName: 'Novels',
},
],
moreCandidates: false,
},
]);
expect(mockRepo.findBooksByTitleKeys).toHaveBeenCalledWith(['lab alpha'], [1, 2], null, 26);
});
it('lists every edition that shares the title and authors, so none is presented as the only match', async () => {
mockRepo.listUnmatchedBooks.mockResolvedValue([makeUnmatchedRow({ title: 'Lab Alpha', authors: 'Ada Tester' })]);
mockRepo.findBooksByTitleKeys.mockResolvedValue([
makeLibraryBook({ bookId: 501 }),
makeLibraryBook({ bookId: 502, subtitle: 'Second edition' }),
]);
const [suggestion] = await service.listUnmatchedBookSuggestions(user);
expect(suggestion!.candidates.map((candidate) => candidate.bookId)).toEqual([501, 502]);
expect(suggestion!.moreCandidates).toBe(false);
});
it('suggests nothing when the author sets differ in any way', async () => {
mockRepo.listUnmatchedBooks.mockResolvedValue([
makeUnmatchedRow({ hash: HASH_A, title: 'Lab Alpha', authors: 'Ada Tester' }),
makeUnmatchedRow({ hash: HASH_B, title: 'Lab Alpha', authors: 'Ada Tester\nBo Sample\nCy Third' }),
]);
mockRepo.findBooksByTitleKeys.mockResolvedValue([makeLibraryBook({ authors: ['Ada Tester', 'Bo Sample'] })]);
await expect(service.listUnmatchedBookSuggestions(user)).resolves.toEqual([]);
});
it('suggests nothing for a different title, even with the same author', async () => {
mockRepo.listUnmatchedBooks.mockResolvedValue([makeUnmatchedRow({ title: 'Lab Alpha!', authors: 'Ada Tester' })]);
mockRepo.findBooksByTitleKeys.mockResolvedValue([makeLibraryBook({ titleKey: 'lab alpha!', title: 'Lab Alpha' })]);
await expect(service.listUnmatchedBookSuggestions(user)).resolves.toEqual([]);
});
it('only considers rows a link would accept, with a title and authors', async () => {
mockRepo.listUnmatchedBooks.mockResolvedValue([
makeUnmatchedRow({ hash: 'c'.repeat(32), source: 'statistics' }),
makeUnmatchedRow({ hash: 'd'.repeat(32), metadataAmbiguous: true }),
makeUnmatchedRow({ hash: 'e'.repeat(32), manualLinkRequested: true }),
makeUnmatchedRow({ hash: 'f'.repeat(32), title: null }),
makeUnmatchedRow({ hash: '1'.repeat(32), authors: ' ' }),
]);
await expect(service.listUnmatchedBookSuggestions(user)).resolves.toEqual([]);
expect(mockRepo.findBooksByTitleKeys).not.toHaveBeenCalled();
expect(mockRepo.getAccessibleLibraryIds).not.toHaveBeenCalled();
});
it('never offers a book whose primary file is audio or not content, because the link targets that file', async () => {
mockRepo.listUnmatchedBooks.mockResolvedValue([makeUnmatchedRow({ title: 'Lab Alpha', authors: 'Ada Tester' })]);
mockRepo.findBooksByTitleKeys.mockResolvedValue([
makeLibraryBook({ bookId: 601, format: 'm4b' }),
makeLibraryBook({ bookId: 602, role: 'supplement' }),
makeLibraryBook({ bookId: 603, format: 'pdf' }),
]);
const [suggestion] = await service.listUnmatchedBookSuggestions(user);
expect(suggestion!.candidates.map((candidate) => candidate.bookId)).toEqual([603]);
});
it('flags more candidates when the title was shared by more books than were checked', async () => {
mockRepo.listUnmatchedBooks.mockResolvedValue([makeUnmatchedRow({ title: 'Lab Alpha', authors: 'Ada Tester' })]);
const others = Array.from({ length: 25 }, (_, index) => makeLibraryBook({ bookId: 700 + index, authors: ['Someone Else'] }));
mockRepo.findBooksByTitleKeys.mockResolvedValue([makeLibraryBook({ bookId: 699 }), ...others]);
const [suggestion] = await service.listUnmatchedBookSuggestions(user);
expect(suggestion!.candidates.map((candidate) => candidate.bookId)).toEqual([699]);
expect(suggestion!.moreCandidates).toBe(true);
});
it('caps the candidates and reports that more exist', async () => {
mockRepo.listUnmatchedBooks.mockResolvedValue([makeUnmatchedRow({ title: 'Lab Alpha', authors: 'Ada Tester' })]);
mockRepo.findBooksByTitleKeys.mockResolvedValue(Array.from({ length: 7 }, (_, index) => makeLibraryBook({ bookId: 800 + index })));
const [suggestion] = await service.listUnmatchedBookSuggestions(user);
expect(suggestion!.candidates).toHaveLength(5);
expect(suggestion!.moreCandidates).toBe(true);
});
it('applies content filters for regular users and skips them for superusers', async () => {
const filters = { includeTagIds: [3], includeGenreIds: [], excludeTagIds: [], excludeGenreIds: [] };
mockRepo.listUnmatchedBooks.mockResolvedValue([makeUnmatchedRow({ title: 'Lab Alpha', authors: 'Ada Tester' })]);
await service.listUnmatchedBookSuggestions({ id: 7, settings: {}, contentFilters: filters } as never);
await service.listUnmatchedBookSuggestions({ id: 7, settings: {}, contentFilters: filters, isSuperuser: true } as never);
expect(mockRepo.findBooksByTitleKeys).toHaveBeenNthCalledWith(1, ['lab alpha'], [1, 2], filters, 26);
expect(mockRepo.findBooksByTitleKeys).toHaveBeenNthCalledWith(2, ['lab alpha'], [1, 2], null, 26);
});
it('never writes', async () => {
mockRepo.listUnmatchedBooks.mockResolvedValue([makeUnmatchedRow({ title: 'Lab Alpha', authors: 'Ada Tester' })]);
mockRepo.findBooksByTitleKeys.mockResolvedValue([makeLibraryBook()]);
await service.listUnmatchedBookSuggestions(user);
expect(mockRepo.upsertBookHashLink).not.toHaveBeenCalled();
expect(mockRepo.clearUnmatchedBooks).not.toHaveBeenCalled();
expect(mockRepo.resolveBookFilesByHashes).not.toHaveBeenCalled();
});
});
describe('linkUnmatchedBook', () => {
it('links an unmatched hash to an accessible book file', async () => {
mockRepo.getUnmatchedBook.mockResolvedValue(makeUnmatchedRow());
@@ -258,6 +403,7 @@ describe('KoreaderHashLinkService', () => {
koreaderTitle: 'KOReader Title',
koreaderAuthors: 'KOReader Author',
koreaderLastOpen: 100,
positionCompatible: false,
createdAt,
updatedAt,
},
@@ -273,6 +419,7 @@ describe('KoreaderHashLinkService', () => {
koreaderTitle: 'KOReader Title',
koreaderAuthors: 'KOReader Author',
koreaderLastOpen: 100,
positionCompatible: false,
createdAt: '2026-06-01T10:00:00.000Z',
updatedAt: '2026-06-02T11:00:00.000Z',
},
@@ -1,16 +1,20 @@
import { BadRequestException, ConflictException, Injectable, Logger, NotFoundException } from '@nestjs/common';
import { isAudioFormat } from '@bookorbit/types';
import type {
DismissAllKoreaderUnmatchedBooksResult,
DismissKoreaderUnmatchedBookResult,
KoreaderLinkSuggestionCandidate,
KoreaderManualHashLink,
KoreaderUnmatchedBook,
KoreaderUnmatchedBookSuggestion,
LinkKoreaderUnmatchedBookResult,
UnlinkKoreaderManualHashLinkResult,
} from '@bookorbit/types';
import type { RequestUser } from '../../common/types/request-user';
import { BookService } from '../book/book.service';
import { KoreaderRepository } from './koreader.repository';
import { authorKeys, koreaderAuthorKeys, linkSuggestionTextKey, sameAuthorKeys } from './koreader-link-suggestion.utils';
const MD5_HEX = /^[0-9a-f]{32}$/;
const UNMATCHED_BOOK_LIMIT = 100;
@@ -20,6 +24,10 @@ const HASH_LINK_UPDATE_EVENT = 'koreader.hash_link_update';
const HASH_LINK_DELETE_EVENT = 'koreader.hash_link_delete';
const UNMATCHED_BOOK_DISMISS_EVENT = 'koreader.unmatched_book_dismiss';
const UNMATCHED_BOOK_DISMISS_ALL_EVENT = 'koreader.unmatched_book_dismiss_all';
const HASH_LINK_SUGGEST_EVENT = 'koreader.hash_link_suggest';
const SUGGESTION_CANDIDATE_LIMIT = 5;
/** Books checked per title. A title shared by more books than this never yields a single suggestion. */
const SUGGESTION_TITLE_SCAN_LIMIT = 25;
/**
* Manages the KOReader "unmatched book" queue and the manual hash links used
@@ -60,6 +68,72 @@ export class KoreaderHashLinkService {
}));
}
/**
* Library books with the same title and the same set of authors as a linkable book to review. Read
* only: nothing is linked until the user confirms, through the same route as a manual link.
*/
async listUnmatchedBookSuggestions(user: RequestUser): Promise<KoreaderUnmatchedBookSuggestion[]> {
const startedAtMs = Date.now();
const rows = await this.repo.listUnmatchedBooks(user.id, UNMATCHED_BOOK_LIMIT);
const wanted = rows.flatMap((row) => {
if (row.manualLinkRequested || !isLinkableUnmatchedBook(row)) return [];
const titleKey = linkSuggestionTextKey(row.title);
const authors = koreaderAuthorKeys(row.authors);
return titleKey && authors.length > 0 ? [{ hash: row.hash, titleKey, authors }] : [];
});
if (wanted.length === 0) return [];
const accessibleLibraryIds = await this.repo.getAccessibleLibraryIds(user.id);
const books = await this.repo.findBooksByTitleKeys(
[...new Set(wanted.map((entry) => entry.titleKey))],
accessibleLibraryIds,
user.isSuperuser ? null : user.contentFilters,
SUGGESTION_TITLE_SCAN_LIMIT + 1,
);
const booksByTitle = new Map<string, typeof books>();
for (const book of books) {
const list = booksByTitle.get(book.titleKey) ?? [];
list.push(book);
booksByTitle.set(book.titleKey, list);
}
const suggestions: KoreaderUnmatchedBookSuggestion[] = [];
for (const entry of wanted) {
const titleBooks = booksByTitle.get(entry.titleKey) ?? [];
const truncated = titleBooks.length > SUGGESTION_TITLE_SCAN_LIMIT;
const candidates: KoreaderLinkSuggestionCandidate[] = titleBooks
.slice(0, SUGGESTION_TITLE_SCAN_LIMIT)
.filter(
(book) =>
book.role === 'content' &&
!(book.format && isAudioFormat(book.format)) &&
linkSuggestionTextKey(book.title) === entry.titleKey &&
sameAuthorKeys(authorKeys(book.authors), entry.authors),
)
.map((book) => ({
bookId: book.bookId,
title: book.title ?? null,
subtitle: book.subtitle ?? null,
authors: book.authors,
format: book.format ?? null,
language: book.language ?? null,
publishedYear: book.publishedYear ?? null,
libraryName: book.libraryName ?? null,
}));
if (candidates.length === 0) continue;
suggestions.push({
hash: entry.hash,
candidates: candidates.slice(0, SUGGESTION_CANDIDATE_LIMIT),
moreCandidates: truncated || candidates.length > SUGGESTION_CANDIDATE_LIMIT,
});
}
this.logger.log(
`[${HASH_LINK_SUGGEST_EVENT}] [end] userId=${user.id} reviewed=${wanted.length} suggested=${suggestions.length} single=${suggestions.filter((entry) => entry.candidates.length === 1 && !entry.moreCandidates).length} durationMs=${Date.now() - startedAtMs} - link suggestions listed`,
);
return suggestions;
}
async linkUnmatchedBook(user: RequestUser, hash: string, bookId: number): Promise<LinkKoreaderUnmatchedBookResult> {
const startedAtMs = Date.now();
const normalizedHash = this.normalizeKoreaderHash(hash);
@@ -105,6 +179,7 @@ export class KoreaderHashLinkService {
koreaderTitle: row.koreaderTitle ?? null,
koreaderAuthors: row.koreaderAuthors ?? null,
koreaderLastOpen: row.koreaderLastOpen ?? null,
positionCompatible: row.positionCompatible !== false,
createdAt: row.createdAt.toISOString(),
updatedAt: row.updatedAt.toISOString(),
}));
@@ -0,0 +1,41 @@
import { describe, expect, it } from 'vitest';
import { authorKeys, koreaderAuthorKeys, linkSuggestionTextKey, sameAuthorKeys } from './koreader-link-suggestion.utils';
describe('linkSuggestionTextKey', () => {
it('folds case, compatibility forms and whitespace only', () => {
expect(linkSuggestionTextKey(' The Hobbit\t')).toBe('the hobbit');
expect(linkSuggestionTextKey('THE Hobbit')).toBe('the hobbit');
expect(linkSuggestionTextKey('Office')).toBe('office');
});
it('keeps punctuation, so near-identical titles stay distinct', () => {
expect(linkSuggestionTextKey("Ender's Game")).not.toBe(linkSuggestionTextKey('Ender’s Game'));
expect(linkSuggestionTextKey('It')).not.toBe(linkSuggestionTextKey('It!'));
expect(linkSuggestionTextKey('Dune: Messiah')).not.toBe(linkSuggestionTextKey('Dune Messiah'));
});
it('returns null for missing or blank values', () => {
expect(linkSuggestionTextKey(null)).toBeNull();
expect(linkSuggestionTextKey(undefined)).toBeNull();
expect(linkSuggestionTextKey(' ')).toBeNull();
});
});
describe('author keys', () => {
it('splits KOReader authors on new lines and ignores order, case and duplicates', () => {
expect(koreaderAuthorKeys('Bo Sample\nAda Tester\n\nada tester')).toEqual(['ada tester', 'bo sample']);
expect(authorKeys(['Ada Tester', 'Bo Sample'])).toEqual(['ada tester', 'bo sample']);
});
it('does not split on commas, which belong to "Last, First" names', () => {
expect(koreaderAuthorKeys('Tester, Ada')).toEqual(['tester, ada']);
});
it('compares author sets exactly', () => {
expect(sameAuthorKeys(['a', 'b'], ['a', 'b'])).toBe(true);
expect(sameAuthorKeys(['a'], ['a', 'b'])).toBe(false);
expect(sameAuthorKeys(['a', 'b'], ['a'])).toBe(false);
expect(sameAuthorKeys([], [])).toBe(false);
});
});
@@ -0,0 +1,28 @@
/**
* Strict comparison keys for suggesting a manual link from an unmatched KOReader book's title and
* authors. Only case, Unicode compatibility forms and whitespace are folded: a missed suggestion
* costs one manual search, while a wrong one sends a device copy's reading history to another book.
*/
export function linkSuggestionTextKey(value: string | null | undefined): string | null {
if (!value) return null;
const key = value.normalize('NFKC').toLowerCase().replace(/\s+/g, ' ').trim();
return key.length > 0 ? key : null;
}
/** KOReader reports several authors on separate lines. */
export function koreaderAuthorKeys(authors: string | null | undefined): string[] {
return authorKeys((authors ?? '').split('\n'));
}
export function authorKeys(names: readonly (string | null | undefined)[]): string[] {
const keys = new Set<string>();
for (const name of names) {
const key = linkSuggestionTextKey(name);
if (key) keys.add(key);
}
return [...keys].sort();
}
export function sameAuthorKeys(a: readonly string[], b: readonly string[]): boolean {
return a.length > 0 && a.length === b.length && a.every((key, index) => key === b[index]);
}
@@ -349,7 +349,7 @@ describe('KOReader plugin update source wiring', () => {
expect(bookSync).toContain('if not ctx.snap.stats_row_ambiguous then return book and book.fileId or nil end');
expect(bookSync).toContain('metadata_ambiguous = ctx.snap.metadata_ambiguous');
expect(bookSync).toContain('if ctx.snap.stats_metadata_ambiguous then');
expect(bookSync).toContain('ctx.state:setMatched(match.hash, match.bookFileId, match.bookId, ctx.snap.file)');
expect(bookSync).toContain('ctx.state:setMatched(match.hash, match.bookFileId, match.bookId, ctx.snap.file, nil, match.positionCompatible)');
expect(bookSync).toContain('if ctx.snap.stats_row_ambiguous then upload.bookFileId = stats_target.bookFileId end');
expect(bookSync).not.toContain(
'if book then\n if ctx.snap.file and not book.file then\n book.file = ctx.snap.file\n end\n return step(ctx, stepStats)\n end',
@@ -126,6 +126,23 @@ describe('KoreaderPluginService', () => {
expect(result.libraryVersion).toMatch(/^[0-9a-f]{16}$/);
});
it('marks a match whose device copy cannot place positions, and only that one', async () => {
koreaderRepo.resolveBookFilesByHashes.mockResolvedValue(
new Map([
[HASH_A, { bookFileId: 10, bookId: 20, libraryId: 1, progressCompatible: false }],
[HASH_B, { bookFileId: 11, bookId: 21, libraryId: 1 }],
]),
);
const dto = { ...deviceFields(), hashes: [HASH_A, HASH_B] } as MatchCheckDto;
const result = await service.matchCheck(makeUser(), dto);
expect(result.matches).toEqual([
{ hash: HASH_A, bookId: 20, bookFileId: 10, positionCompatible: false },
{ hash: HASH_B, bookId: 21, bookFileId: 11 },
]);
});
it('links an explicitly downloaded catalog file when its hash resolved to another file', async () => {
koreaderRepo.resolveBookFilesByHashes
.mockResolvedValueOnce(new Map([[HASH_A, { bookFileId: 10, bookId: 20, libraryId: 1 }]]))
@@ -43,7 +43,8 @@ const READ_STATUS_TO_DEVICE_STATUS: Record<ReadStatus, string> = {
};
export interface MatchCheckResult {
matches: { hash: string; bookId: number; bookFileId: number }[];
/** `positionCompatible: false` marks a copy whose bytes differ from the library file: progress syncs as a percentage and highlights do not sync. */
matches: { hash: string; bookId: number; bookFileId: number; positionCompatible?: false }[];
libraryVersion: string;
}
@@ -140,6 +141,7 @@ export class KoreaderPluginService {
hash,
bookId: match.bookId,
bookFileId: match.bookFileId,
...(match.progressCompatible === false ? { positionCompatible: false as const } : {}),
}));
const libraryVersion = await this.computeLibraryVersion(user.id, accessibleLibraryIds);
@@ -174,6 +174,12 @@ export class KoreaderController {
return this.hashLinkService.listUnmatchedBooks(user);
}
@RequirePermission(Permission.KoreaderSync)
@Get('unmatched-books/suggestions')
listUnmatchedBookSuggestions(@CurrentUser() user: RequestUser) {
return this.hashLinkService.listUnmatchedBookSuggestions(user);
}
@RequirePermission(Permission.KoreaderSync)
@Post('unmatched-books/:hash/link')
linkUnmatchedBook(@CurrentUser() user: RequestUser, @Param('hash') hash: string, @Body() dto: LinkKoreaderUnmatchedBookDto) {
@@ -1438,3 +1438,58 @@ describe('KoreaderRepository', () => {
});
});
});
describe('KoreaderRepository.findBooksByTitleKeys', () => {
it('short-circuits without a title or an accessible library', async () => {
const captured = createCapturingDb();
const repository = await makeRepository(captured.db);
await expect(repository.findBooksByTitleKeys([], null, null, 26)).resolves.toEqual([]);
await expect(repository.findBooksByTitleKeys(['lab alpha'], [], null, 26)).resolves.toEqual([]);
expect(captured.queries).toHaveLength(0);
});
it('looks titles up by the indexed lowercase title, capped per title and scoped to present books in the libraries', async () => {
const captured = createCapturingDb();
const repository = await makeRepository(captured.db);
await repository.findBooksByTitleKeys(['lab alpha', 'lab beta'], [3, 4], null, 26);
expect(captured.queries).toHaveLength(1);
const query = captured.queries[0]!;
expect(query.sql).toContain('lower("book_metadata"."title") in');
expect(query.sql).toContain('row_number() over (partition by lower("book_metadata"."title")');
expect(query.sql).toContain('"books"."status" =');
expect(query.sql).toContain('"books"."library_id" in');
expect(query.sql).toContain('"books"."primary_file_id"');
expect(query.sql).not.toContain('"book_tags"');
expect(query.params).toEqual(expect.arrayContaining(['lab alpha', 'lab beta', 'present', 3, 4, 26]));
});
it('applies the content filters it is given', async () => {
const captured = createCapturingDb();
const repository = await makeRepository(captured.db);
await repository.findBooksByTitleKeys(
['lab alpha'],
null,
{ includeTagIds: [9], includeGenreIds: [], excludeTagIds: [], excludeGenreIds: [] },
26,
);
expect(captured.queries[0]!.sql).toContain('"book_tags"');
expect(captured.queries[0]!.sql).not.toContain('"books"."library_id" in');
});
});
describe('KoreaderRepository.listBookHashLinks', () => {
it('reports whether each linked copy can place positions in the linked file', async () => {
const captured = createCapturingDb();
const repository = await makeRepository(captured.db);
await repository.listBookHashLinks(7, 100, [1]);
expect(captured.queries[0]!.sql).toContain('"book_files"."file_hash" = "koreader_book_hash_links"."hash"');
expect(captured.queries[0]!.sql).toContain("'kobo_download'");
});
});
@@ -1,9 +1,10 @@
import { Inject, Injectable, NotFoundException } from '@nestjs/common';
import { and, asc, desc, eq, inArray, notExists, or, sql, type SQL, type SQLWrapper } from 'drizzle-orm';
import { and, asc, desc, eq, inArray, lte, notExists, or, sql, type SQL, type SQLWrapper } from 'drizzle-orm';
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
import type { ReadAloudDeliveryPreference } from '@bookorbit/types';
import type { ContentFilterRules, ReadAloudDeliveryPreference } from '@bookorbit/types';
import { chunk } from '../../common/utils/batch.utils';
import { buildContentFilterClauses } from '../../common/utils/content-filter-sql.utils';
import { DB } from '../../db';
import * as schema from '../../db/schema';
@@ -500,6 +501,7 @@ export class KoreaderRepository {
koreaderTitle: schema.koreaderBookHashLinks.koreaderTitle,
koreaderAuthors: schema.koreaderBookHashLinks.koreaderAuthors,
koreaderLastOpen: schema.koreaderBookHashLinks.koreaderLastOpen,
positionCompatible: compatibleProgressHash(schema.koreaderBookHashLinks.hash),
createdAt: schema.koreaderBookHashLinks.createdAt,
updatedAt: schema.koreaderBookHashLinks.updatedAt,
})
@@ -532,6 +534,80 @@ export class KoreaderRepository {
return rows.map((row) => ({ ...row, bookAuthors: authorsByBook.get(row.bookId) ?? [] }));
}
/**
* Present books in the given libraries whose lowercased title is one of `titleKeys`, at most
* `perTitleLimit` per title (lowest ids first), with their primary file and authors. Pass
* `contentFilters` for a non-superuser so hidden books are never offered.
*/
async findBooksByTitleKeys(
titleKeys: string[],
accessibleLibraryIds: number[] | null,
contentFilters: ContentFilterRules | null,
perTitleLimit: number,
) {
if (titleKeys.length === 0) return [];
if (accessibleLibraryIds !== null && accessibleLibraryIds.length === 0) return [];
const titleKey = sql<string>`lower(${schema.bookMetadata.title})`;
const ranked = this.db
.select({
bookId: sql<number>`${schema.books.id}`.as('ranked_book_id'),
titleKey: sql<string>`lower(${schema.bookMetadata.title})`.as('ranked_title_key'),
rank: sql<number>`row_number() over (partition by lower(${schema.bookMetadata.title}) order by ${schema.books.id})`.as('ranked_rank'),
})
.from(schema.bookMetadata)
.innerJoin(schema.books, eq(schema.books.id, schema.bookMetadata.bookId))
.where(
and(
inArray(titleKey, titleKeys),
eq(schema.books.status, 'present'),
accessibleLibraryIds ? inArray(schema.books.libraryId, accessibleLibraryIds) : undefined,
...(contentFilters ? buildContentFilterClauses(contentFilters, this.db) : []),
),
)
.as('ranked');
const rows = await this.db
.select({
bookId: ranked.bookId,
titleKey: ranked.titleKey,
title: schema.bookMetadata.title,
subtitle: schema.bookMetadata.subtitle,
language: schema.bookMetadata.language,
publishedYear: schema.bookMetadata.publishedYear,
format: schema.bookFiles.format,
role: schema.bookFiles.role,
libraryName: schema.libraries.name,
})
.from(ranked)
.innerJoin(schema.books, eq(schema.books.id, ranked.bookId))
.innerJoin(schema.bookMetadata, eq(schema.bookMetadata.bookId, schema.books.id))
.innerJoin(schema.bookFiles, eq(schema.bookFiles.id, schema.books.primaryFileId))
.innerJoin(schema.libraries, eq(schema.libraries.id, schema.books.libraryId))
.where(lte(ranked.rank, perTitleLimit))
.orderBy(asc(ranked.bookId));
const authorsByBook = new Map<number, string[]>();
for (const ids of chunk(
rows.map((row) => row.bookId),
BATCH_QUERY_SIZE,
)) {
const authorRows = await this.db
.select({ bookId: schema.bookAuthors.bookId, name: schema.authors.name })
.from(schema.bookAuthors)
.innerJoin(schema.authors, eq(schema.authors.id, schema.bookAuthors.authorId))
.where(inArray(schema.bookAuthors.bookId, ids))
.orderBy(schema.bookAuthors.bookId, schema.bookAuthors.displayOrder);
for (const row of authorRows) {
const list = authorsByBook.get(row.bookId) ?? [];
list.push(row.name);
authorsByBook.set(row.bookId, list);
}
}
return rows.map((row) => ({ ...row, authors: authorsByBook.get(row.bookId) ?? [] }));
}
async upsertBookHashLink(userId: number, hash: string, bookFileId: number, metadata: KoreaderHashLinkMetadata = {}) {
const now = new Date();
await this.db
@@ -109,6 +109,7 @@
"GET /koreader/plugin/version",
"GET /koreader/syncs/progress/:document",
"GET /koreader/unmatched-books",
"GET /koreader/unmatched-books/suggestions",
"GET /koreader/users/auth",
"GET /libraries/:id|:libraryId",
"GET /libraries/:id|:libraryId/access",
@@ -419,7 +419,10 @@ describe('Kobo download identity in KOReader (e2e)', { timeout: 180_000 }, () =>
await ctx.db.insert(schema.koreaderBookHashLinks).values({ userId: user.userId, hash: deliveredHash, bookFileId: other.bookFileId });
try {
expect((await download()).statusCode).toBe(200);
expect((await match(deliveredHash)).matches).toEqual([{ hash: deliveredHash, bookId: other.bookId, bookFileId: other.bookFileId }]);
// The delivered bytes belong to the source book, so the linked file cannot place their positions.
expect((await match(deliveredHash)).matches).toEqual([
{ hash: deliveredHash, bookId: other.bookId, bookFileId: other.bookFileId, positionCompatible: false },
]);
const response = await ctx.app.inject({
method: 'PUT',
url: '/api/v1/koreader/syncs/progress',