mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
perf(metadata): index hardcover title author fuzzy matching for large libraries
Extract fuzzy title and author matching into HardcoverImportFuzzyIndex with token indexing and yield pauses to prevent event loop lockups during large library imports. Closes #881
This commit is contained in:
neon
@@ -0,0 +1,401 @@
|
||||
import { distance } from 'fastest-levenshtein';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { HardcoverImportFuzzyIndex } from './hardcover-import-fuzzy-index';
|
||||
import type { HardcoverImportLocalBook } from './hardcover.repository';
|
||||
|
||||
const ACCEPT_SCORE = 86;
|
||||
const TIE_MARGIN = 5;
|
||||
const MAX_MATCH_TEXT_LENGTH = 512;
|
||||
|
||||
interface ReferenceMatch {
|
||||
bookId: number;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
function localBook(bookId: number, title: string | null, authors: string[]): HardcoverImportLocalBook {
|
||||
return {
|
||||
bookId,
|
||||
primaryFileId: null,
|
||||
primaryFileFormat: null,
|
||||
title,
|
||||
isbn13: null,
|
||||
isbn10: null,
|
||||
hardcoverMetadataId: null,
|
||||
authors,
|
||||
status: null,
|
||||
startedAt: null,
|
||||
finishedAt: null,
|
||||
progress: null,
|
||||
};
|
||||
}
|
||||
|
||||
async function findMatch(index: HardcoverImportFuzzyIndex, title: string | null, authors: string[]): Promise<ReferenceMatch | null> {
|
||||
const match = await index.findMatch(title, authors);
|
||||
return match ? { bookId: match.book.bookId, confidence: match.confidence } : null;
|
||||
}
|
||||
|
||||
describe('HardcoverImportFuzzyIndex', () => {
|
||||
it('preserves the previous fuzzy scoring decisions across representative string shapes', async () => {
|
||||
const values = ['Dune', 'Dune Messiah', 'Messiah Dune', 'Dune Mesiah', 'A', 'A B', 'The Left Hand of Darkness', 'Unrelated'];
|
||||
|
||||
for (const localTitle of values) {
|
||||
for (const localAuthor of values) {
|
||||
const book = localBook(1, localTitle, [localAuthor]);
|
||||
const index = new HardcoverImportFuzzyIndex([book]);
|
||||
|
||||
for (const queryTitle of values) {
|
||||
for (const queryAuthor of values) {
|
||||
const expected = referenceMatch(queryTitle, [queryAuthor], [book]);
|
||||
const actual = await findMatch(index, queryTitle, [queryAuthor]);
|
||||
expect(actual, `${queryTitle} / ${queryAuthor} -> ${localTitle} / ${localAuthor}`).toEqual(expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('matches a brute-force scan over a library where candidates compete', async () => {
|
||||
const books = buildContentiousLibrary(140);
|
||||
const index = new HardcoverImportFuzzyIndex(books);
|
||||
|
||||
let resolved = 0;
|
||||
for (const query of buildQueries(books)) {
|
||||
const expected = referenceMatch(query.title, query.authors, books);
|
||||
const actual = await findMatch(index, query.title, query.authors);
|
||||
expect(actual, `${String(query.title)} / ${query.authors.join(', ')}`).toEqual(expected);
|
||||
if (actual) resolved++;
|
||||
}
|
||||
|
||||
expect(resolved, 'the corpus should exercise resolved matches, not only rejections').toBeGreaterThan(100);
|
||||
});
|
||||
|
||||
it('finds a title that overlaps only by shared words', async () => {
|
||||
// The Hardcover title ends in a long run of characters the library has never seen, so every rare bigram and the
|
||||
// single rarest token all belong to that run. Only probing the shared words as well can reach the local book.
|
||||
const shared = 'alpha beta gamma delta epsilon zeta eta theta';
|
||||
const books = [localBook(1, `${shared} aaaa`, ['Ann Leckie'])];
|
||||
const index = new HardcoverImportFuzzyIndex(books);
|
||||
|
||||
await expect(findMatch(index, `${shared} ${'qxwj'.repeat(20)}`, ['Ann Leckie'])).resolves.toEqual({ bookId: 1, confidence: 92 });
|
||||
});
|
||||
|
||||
it('keeps ambiguous matches unresolved', async () => {
|
||||
const index = new HardcoverImportFuzzyIndex([localBook(1, 'Dune', ['Frank Herbert']), localBook(2, 'Dune', ['Frank Herbert'])]);
|
||||
|
||||
await expect(index.findMatch('Dune', ['Frank Herbert'])).resolves.toBeNull();
|
||||
});
|
||||
|
||||
// An exact title and author hit is decided by lookup, without comparing it against the rest of the library.
|
||||
// That deliberately resolves near ties the pre-index scan refused to call, which surfaces the book for review
|
||||
// instead of silently dropping the row as unmatched. Nothing is imported without the user selecting the row.
|
||||
describe('exact title and author hits skip the tie margin', () => {
|
||||
it('resolves a duplicate edition whose other copy spells the author differently', async () => {
|
||||
const books = [localBook(1, 'The Left Hand of Darkness', ['Ursula K. Le Guin']), localBook(2, 'The Left Hand of Darkness', ['Ursula Le Guin'])];
|
||||
const index = new HardcoverImportFuzzyIndex(books);
|
||||
|
||||
expect(bruteForceMatch('The Left Hand of Darkness', ['Ursula K. Le Guin'], books)).toBeNull();
|
||||
await expect(findMatch(index, 'The Left Hand of Darkness', ['Ursula K. Le Guin'])).resolves.toEqual({ bookId: 1, confidence: 100 });
|
||||
});
|
||||
|
||||
it('resolves the first book of a series against a later volume by the same author', async () => {
|
||||
const books = [localBook(1, 'Dune', ['Frank Herbert']), localBook(2, 'Dune Messiah', ['Frank Herbert'])];
|
||||
const index = new HardcoverImportFuzzyIndex(books);
|
||||
|
||||
expect(bruteForceMatch('Dune', ['Frank Herbert'], books)).toBeNull();
|
||||
await expect(findMatch(index, 'Dune', ['Frank Herbert'])).resolves.toEqual({ bookId: 1, confidence: 100 });
|
||||
});
|
||||
|
||||
it('still refuses when the exact hit is itself ambiguous', async () => {
|
||||
const books = [
|
||||
localBook(1, 'Ancillary Justice', ['Ann Leckie']),
|
||||
localBook(2, 'Ancillary Justice', ['Ann Leckie']),
|
||||
localBook(3, 'Ancillary Justice', ['Someone Else']),
|
||||
];
|
||||
const index = new HardcoverImportFuzzyIndex(books);
|
||||
|
||||
await expect(index.findMatch('Ancillary Justice', ['Ann Leckie'])).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('falls back to the scored scan when only the author token order differs', async () => {
|
||||
const books = [localBook(1, 'Ubik', ['Philip K. Dick'])];
|
||||
const index = new HardcoverImportFuzzyIndex(books);
|
||||
|
||||
await expect(findMatch(index, 'Ubik', ['Dick Philip'])).resolves.toEqual({ bookId: 1, confidence: 99 });
|
||||
});
|
||||
});
|
||||
|
||||
it('matches a 42,000-book library without holding the event loop', { timeout: 60_000 }, async () => {
|
||||
const books = buildLargeLibrary(42_000);
|
||||
const index = new HardcoverImportFuzzyIndex(books);
|
||||
const queries = Array.from({ length: 40 }, (_, i) => {
|
||||
const book = books[(i * 349) % books.length]!;
|
||||
return i % 2 === 0
|
||||
? { title: `${book.title!.slice(0, -1)}x`, authors: [...book.authors], expected: book.bookId }
|
||||
: { title: largeLibraryTitle(), authors: [largeLibraryAuthor()], expected: null };
|
||||
});
|
||||
|
||||
let eventLoopTurns = 0;
|
||||
let running = true;
|
||||
const countTurn = (): void => {
|
||||
eventLoopTurns++;
|
||||
if (running) setImmediate(countTurn);
|
||||
};
|
||||
setImmediate(countTurn);
|
||||
|
||||
const startedAt = performance.now();
|
||||
for (const query of queries) {
|
||||
const match = await index.findMatch(query.title, query.authors);
|
||||
expect(match?.book.bookId ?? null, `${query.title} / ${query.authors.join(', ')}`).toBe(query.expected);
|
||||
}
|
||||
const elapsedMs = performance.now() - startedAt;
|
||||
running = false;
|
||||
|
||||
// A blocking scan cannot reach the event loop at all, so the count matters far more than the wall clock.
|
||||
expect(eventLoopTurns).toBeGreaterThan(50);
|
||||
expect(elapsedMs).toBeLessThan(30_000);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Reference implementation: the pre-index scan, plus the exact lookups the index resolves ahead of it.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function referenceMatch(title: string | null, authors: string[], books: HardcoverImportLocalBook[]): ReferenceMatch | null {
|
||||
const query = prepareQuery(title, authors);
|
||||
if (!query) return null;
|
||||
|
||||
const candidates = books
|
||||
.filter((book) => book.title?.trim() && book.authors.length > 0)
|
||||
.map((book) => ({ book, title: normalizeTitle(book.title!), authors: book.authors.map(normalizeName).filter((author) => author.length > 0) }))
|
||||
.filter((candidate) => candidate.title.length > 0 && candidate.authors.length > 0);
|
||||
|
||||
const exact = candidates.filter(
|
||||
(candidate) => candidate.title === query.title && candidate.authors.some((author) => query.authors.includes(author)),
|
||||
);
|
||||
if (exact.length > 0) return exact.length === 1 ? { bookId: exact[0]!.book.bookId, confidence: 100 } : null;
|
||||
|
||||
const queryTokenKeys = query.authors.map(sortedTokenKey);
|
||||
const tokenSet = candidates.filter(
|
||||
(candidate) => candidate.title === query.title && candidate.authors.some((author) => queryTokenKeys.includes(sortedTokenKey(author))),
|
||||
);
|
||||
if (tokenSet.length > 0) return tokenSet.length === 1 ? { bookId: tokenSet[0]!.book.bookId, confidence: 99 } : null;
|
||||
|
||||
return bruteForceMatch(title, authors, books);
|
||||
}
|
||||
|
||||
function bruteForceMatch(title: string | null, authors: string[], books: HardcoverImportLocalBook[]): ReferenceMatch | null {
|
||||
if (!title?.trim() || authors.length === 0) return null;
|
||||
|
||||
const scored = books
|
||||
.filter((book) => book.title?.trim() && book.authors.length > 0)
|
||||
.map((book) => {
|
||||
const titleScore = scoreTitle(title, book.title!);
|
||||
const authorScore = scoreAuthors(authors, book.authors);
|
||||
return { book, titleScore, authorScore, confidence: Math.round((titleScore * 0.7 + authorScore * 0.3) * 100) };
|
||||
})
|
||||
.filter((row) => row.titleScore >= 0.8 && row.authorScore >= 0.75 && row.confidence >= ACCEPT_SCORE)
|
||||
.sort((left, right) => right.confidence - left.confidence);
|
||||
|
||||
const best = scored[0];
|
||||
if (!best) return null;
|
||||
const second = scored[1];
|
||||
if (second && best.confidence - second.confidence <= TIE_MARGIN) return null;
|
||||
return { bookId: best.book.bookId, confidence: best.confidence };
|
||||
}
|
||||
|
||||
function prepareQuery(title: string | null, authors: string[]): { title: string; authors: string[] } | null {
|
||||
if (!title?.trim() || authors.length === 0) return null;
|
||||
const normalizedTitle = normalizeTitle(title);
|
||||
const normalizedAuthors = authors.map(normalizeName).filter((author) => author.length > 0);
|
||||
if (!normalizedTitle || normalizedAuthors.length === 0) return null;
|
||||
return { title: normalizedTitle, authors: normalizedAuthors };
|
||||
}
|
||||
|
||||
function scoreTitle(leftValue: string, rightValue: string): number {
|
||||
const left = normalizeTitle(leftValue);
|
||||
const right = normalizeTitle(rightValue);
|
||||
if (!left || !right) return 0;
|
||||
if (left === right) return 1;
|
||||
if (left.startsWith(right) || right.startsWith(left)) return 0.94;
|
||||
if (left.includes(right) || right.includes(left)) return 0.9;
|
||||
const tokenScore = tokenOverlap(left, right);
|
||||
const editScore = normalizedLevenshtein(left, right);
|
||||
return Math.max(tokenScore, editScore >= 0.7 ? editScore : 0);
|
||||
}
|
||||
|
||||
function scoreAuthors(hardcoverAuthors: string[], localAuthors: string[]): number {
|
||||
let best = 0;
|
||||
for (const hardcoverAuthor of hardcoverAuthors) {
|
||||
for (const localAuthor of localAuthors) best = Math.max(best, scoreAuthor(hardcoverAuthor, localAuthor));
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
function scoreAuthor(leftValue: string, rightValue: string): number {
|
||||
const left = normalizeName(leftValue);
|
||||
const right = normalizeName(rightValue);
|
||||
if (!left || !right) return 0;
|
||||
if (left === right) return 1;
|
||||
if (sortedTokenKey(left) === sortedTokenKey(right)) return 0.98;
|
||||
const overlap = tokenOverlap(left, right);
|
||||
const edit = normalizedLevenshtein(left, right);
|
||||
return Math.max(overlap, edit >= 0.76 ? edit : 0);
|
||||
}
|
||||
|
||||
function normalizeTitle(value: string): string {
|
||||
const stripped = value.split(/:\s+| - /)[0] ?? value;
|
||||
return normalizeName(stripped);
|
||||
}
|
||||
|
||||
function normalizeName(value: string): string {
|
||||
return value
|
||||
.slice(0, MAX_MATCH_TEXT_LENGTH)
|
||||
.normalize('NFKD')
|
||||
.replace(/[̀-ͯ]/g, '')
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function tokenize(value: string): string[] {
|
||||
return value.split(' ').filter((token) => token.length > 1);
|
||||
}
|
||||
|
||||
function sortedTokenKey(value: string): string {
|
||||
return tokenize(value).sort().join('\0');
|
||||
}
|
||||
|
||||
function tokenOverlap(leftValue: string, rightValue: string): number {
|
||||
const left = new Set(tokenize(leftValue));
|
||||
const right = new Set(tokenize(rightValue));
|
||||
if (left.size === 0 || right.size === 0) return 0;
|
||||
let overlap = 0;
|
||||
for (const token of left) {
|
||||
if (right.has(token)) overlap++;
|
||||
}
|
||||
return overlap / Math.max(left.size, right.size);
|
||||
}
|
||||
|
||||
function normalizedLevenshtein(left: string, right: string): number {
|
||||
const maxLength = Math.max(left.length, right.length);
|
||||
return maxLength === 0 ? 1 : 1 - distance(left, right) / maxLength;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Corpora
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function createRandom(seed: number): () => number {
|
||||
let state = seed;
|
||||
return () => {
|
||||
state = (state + 0x6d2b79f5) | 0;
|
||||
let value = Math.imul(state ^ (state >>> 15), 1 | state);
|
||||
value = (value + Math.imul(value ^ (value >>> 7), 61 | value)) ^ value;
|
||||
return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
const TITLE_WORDS = ['Dune', 'Messiah', 'Shadow', 'Empire', 'of', 'the', 'Left', 'Hand', 'Darkness', 'Ancillary', 'Justice', 'Sea', 'Émile', 'II'];
|
||||
const GIVEN_NAMES = ['Frank', 'Ursula', 'Ann', 'Anne', 'Jo', 'J', 'Ted', 'Émile', 'Mary', 'Marie'];
|
||||
const FAMILY_NAMES = ['Herbert', 'Le Guin', 'Leckie', 'Chiang', 'Zola', 'García Márquez', 'Adams', 'Adam', 'K'];
|
||||
|
||||
// Titles and authors are drawn from a deliberately small pool so that near-duplicate editions, shared authors and
|
||||
// tie-margin rejections all occur, which is what separates an index lookup from a full scan.
|
||||
function buildContentiousLibrary(count: number): HardcoverImportLocalBook[] {
|
||||
const random = createRandom(0x5eed);
|
||||
const pick = <T>(items: T[]): T => items[Math.floor(random() * items.length)]!;
|
||||
const makeTitle = (): string => {
|
||||
const wordCount = random() < 0.35 ? 4 + Math.floor(random() * 3) : 1 + Math.floor(random() * 3);
|
||||
const words = Array.from({ length: wordCount }, () => pick(TITLE_WORDS)).join(' ');
|
||||
if (random() < 0.15) return `${words}: ${pick(TITLE_WORDS)}`;
|
||||
if (random() < 0.2) return words.toUpperCase();
|
||||
return words;
|
||||
};
|
||||
const makeAuthor = (): string => {
|
||||
const roll = random();
|
||||
if (roll < 0.1) return `${pick(GIVEN_NAMES)}. ${pick(FAMILY_NAMES)}`;
|
||||
if (roll < 0.15) return pick(FAMILY_NAMES);
|
||||
return `${pick(GIVEN_NAMES)} ${pick(FAMILY_NAMES)}`;
|
||||
};
|
||||
|
||||
const books: HardcoverImportLocalBook[] = [];
|
||||
for (let bookId = 0; bookId < count; bookId++) {
|
||||
const roll = random();
|
||||
if (roll < 0.04) books.push(localBook(bookId, makeTitle(), []));
|
||||
else if (roll < 0.07) books.push(localBook(bookId, null, [makeAuthor()]));
|
||||
else if (roll < 0.1) books.push(localBook(bookId, ' ', [makeAuthor()]));
|
||||
else if (roll < 0.14) books.push(localBook(bookId, makeTitle(), ['!!!', makeAuthor()]));
|
||||
else if (roll < 0.22 && books.length > 0) {
|
||||
const source = books[Math.floor(random() * books.length)]!;
|
||||
books.push(localBook(bookId, source.title, [...source.authors]));
|
||||
} else if (roll < 0.3) books.push(localBook(bookId, makeTitle(), [makeAuthor(), makeAuthor()]));
|
||||
else books.push(localBook(bookId, makeTitle(), [makeAuthor()]));
|
||||
}
|
||||
return books;
|
||||
}
|
||||
|
||||
function buildQueries(books: HardcoverImportLocalBook[]): Array<{ title: string; authors: string[] }> {
|
||||
const random = createRandom(0xc0ffee);
|
||||
const perturb = (value: string): string => {
|
||||
if (value.length === 0) return value;
|
||||
const at = Math.floor(random() * value.length);
|
||||
const roll = random();
|
||||
if (roll < 0.3) return value.slice(0, at) + value.slice(at + 1);
|
||||
if (roll < 0.6) return `${value.slice(0, at)}e${value.slice(at)}`;
|
||||
if (roll < 0.85) return `${value.slice(0, at)}x${value.slice(at + 1)}`;
|
||||
return value.split(' ').reverse().join(' ');
|
||||
};
|
||||
// Edits spread across the string are what the bigram bound has to survive: a single typo leaves almost every
|
||||
// bigram intact, so only multi-edit queries prove the candidate prefix is wide enough.
|
||||
const scatterEdits = (value: string, count: number): string => {
|
||||
let result = value;
|
||||
for (let edit = 0; edit < count; edit++) {
|
||||
const at = Math.floor(((edit + 1) * result.length) / (count + 1));
|
||||
result = `${result.slice(0, at)}z${result.slice(at + 1)}`;
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
const queries: Array<{ title: string; authors: string[] }> = [];
|
||||
for (const book of books) {
|
||||
const title = book.title?.trim() ? book.title : 'Dune';
|
||||
const authors = book.authors.length > 0 ? book.authors : ['Frank Herbert'];
|
||||
queries.push({ title, authors: [authors[0]!] });
|
||||
queries.push({ title, authors });
|
||||
queries.push({ title: perturb(title), authors: [authors[0]!] });
|
||||
queries.push({ title, authors: [perturb(authors[0]!)] });
|
||||
queries.push({ title: `${title}: Special Edition`, authors: [authors[0]!] });
|
||||
queries.push({ title: title.split(' ')[0]!, authors: [authors[0]!] });
|
||||
queries.push({ title: title.toUpperCase(), authors: [authors[0]!.split(' ').reverse().join(' ')] });
|
||||
queries.push({ title: scatterEdits(title, 2), authors: [authors[0]!] });
|
||||
queries.push({ title: scatterEdits(title, 3), authors: [scatterEdits(authors[0]!, 2)] });
|
||||
}
|
||||
return queries;
|
||||
}
|
||||
|
||||
const LARGE_SYLLABLES =
|
||||
'ka ro mi tel vin sar dou lem pra zin hol fer nue quas bir wen ta shi glo urn eld myr pod cav rin thu neb wor lax jem ovi dast fen gri hup jolt kres lun mab nyx opa'.split(
|
||||
' ',
|
||||
);
|
||||
const LARGE_COMMON = ['the', 'of', 'and', 'in', 'last', 'dark', 'lost', 'night', 'song', 'house'];
|
||||
const largeRandom = createRandom(0xb00c);
|
||||
|
||||
function largeWord(): string {
|
||||
const count = 2 + Math.floor(largeRandom() * 2);
|
||||
return Array.from({ length: count }, () => LARGE_SYLLABLES[Math.floor(largeRandom() * LARGE_SYLLABLES.length)]!).join('');
|
||||
}
|
||||
|
||||
function largeLibraryTitle(): string {
|
||||
const common = (): string => LARGE_COMMON[Math.floor(largeRandom() * LARGE_COMMON.length)]!;
|
||||
return [common(), largeWord(), common(), largeWord()].join(' ');
|
||||
}
|
||||
|
||||
function largeLibraryAuthor(): string {
|
||||
return `${largeWord()} ${largeWord()}`;
|
||||
}
|
||||
|
||||
function buildLargeLibrary(count: number): HardcoverImportLocalBook[] {
|
||||
return Array.from({ length: count }, (_, bookId) => localBook(bookId, largeLibraryTitle(), [largeLibraryAuthor()]));
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
import { distance } from 'fastest-levenshtein';
|
||||
|
||||
import type { HardcoverImportLocalBook } from './hardcover.repository';
|
||||
|
||||
const TITLE_AUTHOR_ACCEPT_SCORE = 86;
|
||||
const TITLE_AUTHOR_TIE_MARGIN = 5;
|
||||
const MATCH_YIELD_INTERVAL_MS = 8;
|
||||
const MATCH_YIELD_CHECK_SIZE = 128;
|
||||
const MAX_MATCH_TEXT_LENGTH = 512;
|
||||
|
||||
interface PreparedText {
|
||||
normalized: string;
|
||||
tokens: string[];
|
||||
sortedTokenKey: string;
|
||||
}
|
||||
|
||||
// Every comparison puts the Hardcover book on the left and a library book on the right, so the left side is
|
||||
// prepared once per query while the right side stays lean enough to hold for the whole library.
|
||||
interface PreparedQueryText extends PreparedText {
|
||||
tokenSet: Set<string>;
|
||||
bigramCounts: Map<string, number>;
|
||||
}
|
||||
|
||||
interface IndexedBook {
|
||||
book: HardcoverImportLocalBook;
|
||||
title: PreparedText;
|
||||
authors: PreparedText[];
|
||||
}
|
||||
|
||||
export interface HardcoverImportFuzzyMatch {
|
||||
book: HardcoverImportLocalBook;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
interface ExactLookupResult {
|
||||
resolved: boolean;
|
||||
match: HardcoverImportFuzzyMatch | null;
|
||||
}
|
||||
|
||||
// One budget for the whole index, not one per matching step: a preview scores every Hardcover book against the
|
||||
// same index, and a per-step budget lets short steps and cheap rows accumulate into a long unbroken block.
|
||||
class YieldBudget {
|
||||
private deadline = performance.now() + MATCH_YIELD_INTERVAL_MS;
|
||||
private processed = 0;
|
||||
|
||||
isExhausted(): boolean {
|
||||
if (++this.processed < MATCH_YIELD_CHECK_SIZE) return false;
|
||||
this.processed = 0;
|
||||
return performance.now() >= this.deadline;
|
||||
}
|
||||
|
||||
async yield(): Promise<void> {
|
||||
await yieldToEventLoop();
|
||||
this.deadline = performance.now() + MATCH_YIELD_INTERVAL_MS;
|
||||
}
|
||||
}
|
||||
|
||||
export class HardcoverImportFuzzyIndex {
|
||||
private readonly yieldBudget = new YieldBudget();
|
||||
private readonly books: IndexedBook[] = [];
|
||||
private readonly titleBigrams = new Map<string, number[]>();
|
||||
private readonly titleCharacters = new Map<string, number[]>();
|
||||
private readonly titlesByNormalizedName = new Map<string, number[]>();
|
||||
private readonly titleTokens = new Map<string, number[]>();
|
||||
private readonly authorBigrams = new Map<string, number[]>();
|
||||
private readonly authorsByNormalizedName = new Map<string, number[]>();
|
||||
private readonly authorsByTokenKey = new Map<string, number[]>();
|
||||
private readonly authorTokens = new Map<string, number[]>();
|
||||
private readonly tokenlessAuthors = new Set<number>();
|
||||
private readonly exactTitleAuthors = new Map<string, number[]>();
|
||||
private readonly tokenSetTitleAuthors = new Map<string, number[]>();
|
||||
|
||||
constructor(books: HardcoverImportLocalBook[]) {
|
||||
for (const book of books) {
|
||||
if (!book.title?.trim() || book.authors.length === 0) continue;
|
||||
|
||||
const title = prepareText(normalizeTitle(book.title));
|
||||
const authors = book.authors.map((author) => prepareText(normalizeName(author))).filter((author) => author.normalized.length > 0);
|
||||
if (!title.normalized || authors.length === 0) continue;
|
||||
|
||||
const bookIndex = this.books.length;
|
||||
this.books.push({ book, title, authors });
|
||||
|
||||
addPostings(this.titleBigrams, bigramOccurrenceTokens(title.normalized), bookIndex);
|
||||
addPostings(this.titleCharacters, new Set(title.normalized), bookIndex);
|
||||
addPosting(this.titlesByNormalizedName, title.normalized, bookIndex);
|
||||
addPostings(this.titleTokens, new Set(title.tokens), bookIndex);
|
||||
|
||||
const authorBigrams = new Set<string>();
|
||||
const normalizedAuthorNames = new Set<string>();
|
||||
const authorTokenKeys = new Set<string>();
|
||||
const authorTokens = new Set<string>();
|
||||
for (const author of authors) {
|
||||
for (const bigram of bigramOccurrenceTokens(author.normalized)) authorBigrams.add(bigram);
|
||||
normalizedAuthorNames.add(author.normalized);
|
||||
authorTokenKeys.add(author.sortedTokenKey);
|
||||
for (const token of author.tokens) authorTokens.add(token);
|
||||
if (author.tokens.length === 0) this.tokenlessAuthors.add(bookIndex);
|
||||
addPosting(this.exactTitleAuthors, pairKey(title.normalized, author.normalized), bookIndex);
|
||||
addPosting(this.tokenSetTitleAuthors, pairKey(title.normalized, author.sortedTokenKey), bookIndex);
|
||||
}
|
||||
addPostings(this.authorBigrams, authorBigrams, bookIndex);
|
||||
addPostings(this.authorsByNormalizedName, normalizedAuthorNames, bookIndex);
|
||||
addPostings(this.authorsByTokenKey, authorTokenKeys, bookIndex);
|
||||
addPostings(this.authorTokens, authorTokens, bookIndex);
|
||||
}
|
||||
}
|
||||
|
||||
async findMatch(title: string | null, authors: string[]): Promise<HardcoverImportFuzzyMatch | null> {
|
||||
if (!title?.trim() || authors.length === 0) return null;
|
||||
|
||||
const preparedTitle = prepareQueryText(normalizeTitle(title));
|
||||
const preparedAuthors = authors.map((author) => prepareQueryText(normalizeName(author))).filter((author) => author.normalized.length > 0);
|
||||
if (!preparedTitle.normalized || preparedAuthors.length === 0) return null;
|
||||
|
||||
const exact = this.findExactMatch(preparedTitle, preparedAuthors);
|
||||
if (exact.resolved) return exact.match;
|
||||
|
||||
const titleCandidates = await this.findTitleCandidates(preparedTitle);
|
||||
if (titleCandidates.size === 0) return null;
|
||||
const authorCandidates = await this.findAuthorCandidates(preparedAuthors);
|
||||
if (authorCandidates.size === 0) return null;
|
||||
|
||||
const candidateIndexes = await this.narrowCandidates(titleCandidates, authorCandidates, preparedTitle, preparedAuthors);
|
||||
|
||||
let best: HardcoverImportFuzzyMatch | null = null;
|
||||
let secondConfidence: number | null = null;
|
||||
|
||||
for (let index = 0; index < candidateIndexes.length; index++) {
|
||||
if (this.yieldBudget.isExhausted()) await this.yieldBudget.yield();
|
||||
|
||||
const indexedBook = this.books[candidateIndexes[index]!]!;
|
||||
const titleScore = scoreTitle(preparedTitle, indexedBook.title);
|
||||
const authorScore = scoreAuthors(preparedAuthors, indexedBook.authors);
|
||||
const confidence = Math.round((titleScore * 0.7 + authorScore * 0.3) * 100);
|
||||
if (titleScore < 0.8 || authorScore < 0.75 || confidence < TITLE_AUTHOR_ACCEPT_SCORE) continue;
|
||||
|
||||
if (!best || confidence > best.confidence) {
|
||||
secondConfidence = best?.confidence ?? secondConfidence;
|
||||
best = { book: indexedBook.book, confidence };
|
||||
} else if (secondConfidence == null || confidence > secondConfidence) {
|
||||
secondConfidence = confidence;
|
||||
}
|
||||
}
|
||||
|
||||
if (!best) return null;
|
||||
if (secondConfidence != null && best.confidence - secondConfidence <= TITLE_AUTHOR_TIE_MARGIN) return null;
|
||||
return best;
|
||||
}
|
||||
|
||||
private findExactMatch(title: PreparedText, authors: PreparedText[]): ExactLookupResult {
|
||||
const exactCandidate = findUniqueCandidate(
|
||||
authors.map((author) => this.exactTitleAuthors.get(pairKey(title.normalized, author.normalized)) ?? []),
|
||||
);
|
||||
if (exactCandidate.resolved) {
|
||||
return {
|
||||
resolved: true,
|
||||
match: exactCandidate.bookIndex == null ? null : { book: this.books[exactCandidate.bookIndex]!.book, confidence: 100 },
|
||||
};
|
||||
}
|
||||
|
||||
const tokenSetCandidate = findUniqueCandidate(
|
||||
authors.map((author) => this.tokenSetTitleAuthors.get(pairKey(title.normalized, author.sortedTokenKey)) ?? []),
|
||||
);
|
||||
if (tokenSetCandidate.resolved) {
|
||||
return {
|
||||
resolved: true,
|
||||
match: tokenSetCandidate.bookIndex == null ? null : { book: this.books[tokenSetCandidate.bookIndex]!.book, confidence: 99 },
|
||||
};
|
||||
}
|
||||
return { resolved: false, match: null };
|
||||
}
|
||||
|
||||
private async findTitleCandidates(title: PreparedQueryText): Promise<Set<number>> {
|
||||
const candidates = new Set<number>();
|
||||
if (title.normalized.length === 1) {
|
||||
await addPostingsToSetYielding(candidates, this.titleCharacters, [title.normalized], this.yieldBudget);
|
||||
return candidates;
|
||||
}
|
||||
|
||||
const bigramKeys = bigramOccurrenceTokens(title.normalized);
|
||||
const editPrefixSize = Math.min(bigramKeys.length, maxEditPrefixSize(title.normalized.length, 0.8));
|
||||
await addPostingsToSetYielding(candidates, this.titleBigrams, selectRarestKeys(this.titleBigrams, bigramKeys, editPrefixSize), this.yieldBudget);
|
||||
|
||||
const tokenPrefixSize = maxTokenPrefixSize(title.tokenSet.size, 0.8);
|
||||
const tokenKeys = selectRarestKeys(this.titleTokens, title.tokenSet, tokenPrefixSize);
|
||||
await addPostingsToSetYielding(candidates, this.titleTokens, tokenKeys, this.yieldBudget);
|
||||
await addContainedTitles(candidates, this.titlesByNormalizedName, title.normalized, this.yieldBudget);
|
||||
return candidates;
|
||||
}
|
||||
|
||||
private async findAuthorCandidates(authors: PreparedQueryText[]): Promise<Set<number>> {
|
||||
const candidates = new Set<number>();
|
||||
for (const author of authors) {
|
||||
await addPostingsToSetYielding(candidates, this.authorsByNormalizedName, [author.normalized], this.yieldBudget);
|
||||
await addPostingsToSetYielding(candidates, this.authorsByTokenKey, [author.sortedTokenKey], this.yieldBudget);
|
||||
|
||||
const bigramKeys = bigramOccurrenceTokens(author.normalized);
|
||||
const editPrefixSize = Math.min(bigramKeys.length, maxEditPrefixSize(author.normalized.length, 0.76));
|
||||
const rarestBigrams = selectRarestKeys(this.authorBigrams, bigramKeys, editPrefixSize);
|
||||
await addPostingsToSetYielding(candidates, this.authorBigrams, rarestBigrams, this.yieldBudget);
|
||||
|
||||
const tokenPrefixSize = maxTokenPrefixSize(author.tokenSet.size, 0.75);
|
||||
const tokenKeys = selectRarestKeys(this.authorTokens, author.tokenSet, tokenPrefixSize);
|
||||
await addPostingsToSetYielding(candidates, this.authorTokens, tokenKeys, this.yieldBudget);
|
||||
if (author.tokens.length === 0) {
|
||||
await addIndexesToSetYielding(candidates, this.tokenlessAuthors, this.yieldBudget);
|
||||
}
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
// Only the intersection can ever be scored, so both postings lists are narrowed together. Testing each side
|
||||
// separately would run the verification work over every title candidate plus every author candidate instead.
|
||||
private async narrowCandidates(
|
||||
titleCandidates: Set<number>,
|
||||
authorCandidates: Set<number>,
|
||||
title: PreparedQueryText,
|
||||
authors: PreparedQueryText[],
|
||||
): Promise<number[]> {
|
||||
const smaller = titleCandidates.size <= authorCandidates.size ? titleCandidates : authorCandidates;
|
||||
const larger = smaller === titleCandidates ? authorCandidates : titleCandidates;
|
||||
|
||||
const candidateIndexes: number[] = [];
|
||||
for (const bookIndex of smaller) {
|
||||
if (this.yieldBudget.isExhausted()) await this.yieldBudget.yield();
|
||||
if (!larger.has(bookIndex)) continue;
|
||||
|
||||
const indexedBook = this.books[bookIndex]!;
|
||||
if (!isPossibleTitleMatch(title, indexedBook.title)) continue;
|
||||
if (!authors.some((author) => indexedBook.authors.some((localAuthor) => isPossibleAuthorMatch(author, localAuthor)))) continue;
|
||||
candidateIndexes.push(bookIndex);
|
||||
}
|
||||
return candidateIndexes.sort((left, right) => left - right);
|
||||
}
|
||||
}
|
||||
|
||||
function addPostings(map: Map<string, number[]>, keys: Iterable<string>, bookIndex: number): void {
|
||||
for (const key of keys) addPosting(map, key, bookIndex);
|
||||
}
|
||||
|
||||
function addPosting(map: Map<string, number[]>, key: string, bookIndex: number): void {
|
||||
const postings = map.get(key);
|
||||
if (postings) postings.push(bookIndex);
|
||||
else map.set(key, [bookIndex]);
|
||||
}
|
||||
|
||||
function pairKey(left: string, right: string): string {
|
||||
return `${left}\0${right}`;
|
||||
}
|
||||
|
||||
function findUniqueCandidate(postingLists: number[][]): { resolved: boolean; bookIndex: number | null } {
|
||||
let candidate: number | null = null;
|
||||
for (const postings of postingLists) {
|
||||
for (const bookIndex of postings) {
|
||||
if (candidate == null) candidate = bookIndex;
|
||||
else if (candidate !== bookIndex) return { resolved: true, bookIndex: null };
|
||||
}
|
||||
}
|
||||
return candidate == null ? { resolved: false, bookIndex: null } : { resolved: true, bookIndex: candidate };
|
||||
}
|
||||
|
||||
async function addPostingsToSetYielding(result: Set<number>, map: Map<string, number[]>, keys: Iterable<string>, budget: YieldBudget): Promise<void> {
|
||||
for (const key of keys) {
|
||||
for (const bookIndex of map.get(key) ?? []) {
|
||||
result.add(bookIndex);
|
||||
if (budget.isExhausted()) await budget.yield();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function addIndexesToSetYielding(result: Set<number>, indexes: Iterable<number>, budget: YieldBudget): Promise<void> {
|
||||
for (const bookIndex of indexes) {
|
||||
result.add(bookIndex);
|
||||
if (budget.isExhausted()) await budget.yield();
|
||||
}
|
||||
}
|
||||
|
||||
function selectRarestKeys(map: Map<string, number[]>, keys: Iterable<string>, limit: number): string[] {
|
||||
if (limit <= 0) return [];
|
||||
return [...keys].sort((left, right) => (map.get(left)?.length ?? 0) - (map.get(right)?.length ?? 0)).slice(0, limit);
|
||||
}
|
||||
|
||||
function bigramOccurrenceTokens(value: string): string[] {
|
||||
const result: string[] = [];
|
||||
const occurrences = new Map<string, number>();
|
||||
for (let index = 0; index < value.length - 1; index++) {
|
||||
const bigram = value.slice(index, index + 2);
|
||||
const occurrence = occurrences.get(bigram) ?? 0;
|
||||
result.push(`${bigram}\0${occurrence}`);
|
||||
occurrences.set(bigram, occurrence + 1);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function maxEditPrefixSize(queryLength: number, threshold: number): number {
|
||||
if (queryLength < 2) return 0;
|
||||
// One edit changes at most two bigrams. Searching this many rare query bigrams guarantees that every candidate within the threshold shares one.
|
||||
const maxDistance = Math.ceil(((1 - threshold) * queryLength) / threshold);
|
||||
return maxDistance * 2 + 1;
|
||||
}
|
||||
|
||||
function maxTokenPrefixSize(queryTokenCount: number, threshold: number): number {
|
||||
if (queryTokenCount === 0) return 0;
|
||||
// A candidate meeting the overlap threshold cannot avoid every token in this rare-token prefix.
|
||||
return queryTokenCount - Math.ceil(threshold * queryTokenCount) + 1;
|
||||
}
|
||||
|
||||
async function addContainedTitles(
|
||||
result: Set<number>,
|
||||
titlesByNormalizedName: Map<string, number[]>,
|
||||
query: string,
|
||||
budget: YieldBudget,
|
||||
): Promise<void> {
|
||||
for (let start = 0; start < query.length; start++) {
|
||||
for (let end = start + 1; end <= query.length; end++) {
|
||||
if (budget.isExhausted()) await budget.yield();
|
||||
const postings = titlesByNormalizedName.get(query.slice(start, end));
|
||||
if (postings) await addIndexesToSetYielding(result, postings, budget);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function prepareText(normalized: string): PreparedText {
|
||||
const tokens = tokenize(normalized);
|
||||
return {
|
||||
normalized,
|
||||
tokens,
|
||||
sortedTokenKey: [...tokens].sort().join('\0'),
|
||||
};
|
||||
}
|
||||
|
||||
function prepareQueryText(normalized: string): PreparedQueryText {
|
||||
const prepared = prepareText(normalized);
|
||||
return {
|
||||
...prepared,
|
||||
tokenSet: new Set(prepared.tokens),
|
||||
bigramCounts: countBigrams(normalized),
|
||||
};
|
||||
}
|
||||
|
||||
function countBigrams(value: string): Map<string, number> {
|
||||
const counts = new Map<string, number>();
|
||||
for (let index = 0; index < value.length - 1; index++) {
|
||||
const bigram = value.slice(index, index + 2);
|
||||
counts.set(bigram, (counts.get(bigram) ?? 0) + 1);
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
function isPossibleTitleMatch(left: PreparedQueryText, right: PreparedText): boolean {
|
||||
if (left.normalized.startsWith(right.normalized) || right.normalized.startsWith(left.normalized)) return true;
|
||||
if (left.normalized.includes(right.normalized) || right.normalized.includes(left.normalized)) return true;
|
||||
if (tokenOverlap(left, right) >= 0.8) return true;
|
||||
return canReachEditSimilarity(left, right, 0.8);
|
||||
}
|
||||
|
||||
function isPossibleAuthorMatch(left: PreparedQueryText, right: PreparedText): boolean {
|
||||
if (left.normalized === right.normalized) return true;
|
||||
if (left.tokens.length === right.tokens.length && left.sortedTokenKey === right.sortedTokenKey) return true;
|
||||
if (tokenOverlap(left, right) >= 0.75) return true;
|
||||
return canReachEditSimilarity(left, right, 0.76);
|
||||
}
|
||||
|
||||
function canReachEditSimilarity(left: PreparedQueryText, right: PreparedText, threshold: number): boolean {
|
||||
const maxLength = Math.max(left.normalized.length, right.normalized.length);
|
||||
const maxDistance = Math.floor((1 - threshold) * maxLength + Number.EPSILON * maxLength);
|
||||
if (Math.abs(left.normalized.length - right.normalized.length) > maxDistance) return false;
|
||||
|
||||
const requiredSharedBigrams = Math.max(0, maxLength - 1 - maxDistance * 2);
|
||||
return countSharedBigrams(left.bigramCounts, right.normalized) >= requiredSharedBigrams;
|
||||
}
|
||||
|
||||
function countSharedBigrams(left: Map<string, number>, rightValue: string): number {
|
||||
const right = countBigrams(rightValue);
|
||||
const smaller = left.size <= right.size ? left : right;
|
||||
const larger = smaller === left ? right : left;
|
||||
let shared = 0;
|
||||
for (const [bigram, count] of smaller) shared += Math.min(count, larger.get(bigram) ?? 0);
|
||||
return shared;
|
||||
}
|
||||
|
||||
function scoreTitle(left: PreparedQueryText, right: PreparedText): number {
|
||||
if (!left.normalized || !right.normalized) return 0;
|
||||
if (left.normalized === right.normalized) return 1;
|
||||
if (left.normalized.startsWith(right.normalized) || right.normalized.startsWith(left.normalized)) return 0.94;
|
||||
if (left.normalized.includes(right.normalized) || right.normalized.includes(left.normalized)) return 0.9;
|
||||
|
||||
const tokenScore = tokenOverlap(left, right);
|
||||
const editScore = normalizedLevenshtein(left.normalized, right.normalized);
|
||||
return Math.max(tokenScore, editScore >= 0.7 ? editScore : 0);
|
||||
}
|
||||
|
||||
function scoreAuthors(hardcoverAuthors: PreparedQueryText[], localAuthors: PreparedText[]): number {
|
||||
let best = 0;
|
||||
for (const hardcoverAuthor of hardcoverAuthors) {
|
||||
for (const localAuthor of localAuthors) best = Math.max(best, scoreAuthor(hardcoverAuthor, localAuthor));
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
function scoreAuthor(left: PreparedQueryText, right: PreparedText): number {
|
||||
if (!left.normalized || !right.normalized) return 0;
|
||||
if (left.normalized === right.normalized) return 1;
|
||||
if (left.tokens.length === right.tokens.length && left.sortedTokenKey === right.sortedTokenKey) return 0.98;
|
||||
const overlap = tokenOverlap(left, right);
|
||||
const edit = normalizedLevenshtein(left.normalized, right.normalized);
|
||||
return Math.max(overlap, edit >= 0.76 ? edit : 0);
|
||||
}
|
||||
|
||||
function normalizeTitle(value: string): string {
|
||||
const stripped = value.split(/:\s+| - /)[0] ?? value;
|
||||
return normalizeName(stripped);
|
||||
}
|
||||
|
||||
function normalizeName(value: string): string {
|
||||
return value
|
||||
.slice(0, MAX_MATCH_TEXT_LENGTH)
|
||||
.normalize('NFKD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function tokenize(value: string): string[] {
|
||||
return value.split(' ').filter((token) => token.length > 1);
|
||||
}
|
||||
|
||||
function tokenOverlap(left: PreparedQueryText, right: PreparedText): number {
|
||||
const rightTokenSet = new Set(right.tokens);
|
||||
if (left.tokenSet.size === 0 || rightTokenSet.size === 0) return 0;
|
||||
let overlap = 0;
|
||||
for (const token of left.tokenSet) {
|
||||
if (rightTokenSet.has(token)) overlap++;
|
||||
}
|
||||
return overlap / Math.max(left.tokenSet.size, rightTokenSet.size);
|
||||
}
|
||||
|
||||
function normalizedLevenshtein(left: string, right: string): number {
|
||||
const maxLength = Math.max(left.length, right.length);
|
||||
return maxLength === 0 ? 1 : 1 - distance(left, right) / maxLength;
|
||||
}
|
||||
|
||||
function yieldToEventLoop(): Promise<void> {
|
||||
return new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
@@ -10,7 +10,6 @@ import {
|
||||
type HardcoverImportSummary,
|
||||
} from '@bookorbit/types';
|
||||
import { BadRequestException, Injectable, Logger, Optional } from '@nestjs/common';
|
||||
import { distance } from 'fastest-levenshtein';
|
||||
|
||||
import type { RequestUser } from '../../common/types/request-user';
|
||||
import { sanitizeLogValue } from '../../common/utils/log-sanitize.utils';
|
||||
@@ -19,13 +18,11 @@ import { UserBookStatusService } from '../user-book-status/user-book-status.serv
|
||||
import { ReadingAttemptService } from '../user-book-status/reading-attempt.service';
|
||||
import { HARDCOVER_STATUS } from './hardcover.constants';
|
||||
import { HardcoverClientService } from './hardcover-client.service';
|
||||
import { HardcoverImportFuzzyIndex } from './hardcover-import-fuzzy-index';
|
||||
import { type HardcoverImportLocalBook, HardcoverRepository } from './hardcover.repository';
|
||||
import { HardcoverSettingsService } from './hardcover-settings.service';
|
||||
|
||||
const IMPORT_PAGE_SIZE = 100;
|
||||
const TITLE_AUTHOR_ACCEPT_SCORE = 86;
|
||||
const TITLE_AUTHOR_TIE_MARGIN = 5;
|
||||
|
||||
const IMPORTABLE_STATUS_MAP: Record<number, { status: HardcoverImportedReadStatus; label: string }> = {
|
||||
[HARDCOVER_STATUS.WANT_TO_READ]: { status: 'want_to_read', label: 'Want to Read' },
|
||||
[HARDCOVER_STATUS.CURRENTLY_READING]: { status: 'reading', label: 'Currently Reading' },
|
||||
@@ -117,10 +114,10 @@ interface HardcoverUserBook {
|
||||
}
|
||||
|
||||
interface LocalIndexes {
|
||||
books: HardcoverImportLocalBook[];
|
||||
byHardcoverId: Map<number, HardcoverImportLocalBook[]>;
|
||||
byHardcoverSlug: Map<string, HardcoverImportLocalBook[]>;
|
||||
byIsbn: Map<string, HardcoverImportLocalBook[]>;
|
||||
fuzzy: HardcoverImportFuzzyIndex;
|
||||
}
|
||||
|
||||
interface MatchResult {
|
||||
@@ -264,7 +261,8 @@ export class HardcoverImportService {
|
||||
);
|
||||
const indexes = this.buildIndexes(localBooks, states);
|
||||
|
||||
const rows = hardcoverBooks.map((book) => this.buildPreviewRow(book, indexes));
|
||||
const rows: HardcoverImportPreviewRow[] = [];
|
||||
for (const book of hardcoverBooks) rows.push(await this.buildPreviewRow(book, indexes));
|
||||
return {
|
||||
summary: summarize(rows),
|
||||
rows,
|
||||
@@ -301,10 +299,10 @@ export class HardcoverImportService {
|
||||
|
||||
private buildIndexes(books: HardcoverImportLocalBook[], states: Array<Awaited<ReturnType<HardcoverRepository['findBookState']>>>): LocalIndexes {
|
||||
const indexes: LocalIndexes = {
|
||||
books,
|
||||
byHardcoverId: new Map(),
|
||||
byHardcoverSlug: new Map(),
|
||||
byIsbn: new Map(),
|
||||
fuzzy: new HardcoverImportFuzzyIndex(books),
|
||||
};
|
||||
const booksById = new Map(books.map((book) => [book.bookId, book]));
|
||||
|
||||
@@ -333,7 +331,7 @@ export class HardcoverImportService {
|
||||
return indexes;
|
||||
}
|
||||
|
||||
private buildPreviewRow(hardcoverBook: HardcoverUserBook, indexes: LocalIndexes): HardcoverImportPreviewRow {
|
||||
private async buildPreviewRow(hardcoverBook: HardcoverUserBook, indexes: LocalIndexes): Promise<HardcoverImportPreviewRow> {
|
||||
const statusMapping = IMPORTABLE_STATUS_MAP[hardcoverBook.status_id] ?? null;
|
||||
const hardcoverAuthors = extractHardcoverAuthors(hardcoverBook.book?.cached_contributors);
|
||||
const latestRead = hardcoverBook.user_book_reads?.find(hasReadSignal) ?? null;
|
||||
@@ -363,7 +361,7 @@ export class HardcoverImportService {
|
||||
return this.toPreviewRow(base, null, null, null, 'skipped', 'Hardcover status is not imported');
|
||||
}
|
||||
|
||||
const match = this.findLocalMatch(hardcoverBook, hardcoverAuthors, indexes);
|
||||
const match = await this.findLocalMatch(hardcoverBook, hardcoverAuthors, indexes);
|
||||
if ('outcome' in match) {
|
||||
return this.toPreviewRow(base, null, null, null, match.outcome, match.reason);
|
||||
}
|
||||
@@ -420,7 +418,11 @@ export class HardcoverImportService {
|
||||
};
|
||||
}
|
||||
|
||||
private findLocalMatch(hardcoverBook: HardcoverUserBook, hardcoverAuthors: string[], indexes: LocalIndexes): MatchResult | MatchFailure {
|
||||
private async findLocalMatch(
|
||||
hardcoverBook: HardcoverUserBook,
|
||||
hardcoverAuthors: string[],
|
||||
indexes: LocalIndexes,
|
||||
): Promise<MatchResult | MatchFailure> {
|
||||
const byHardcoverId = uniqueCandidate(indexes.byHardcoverId.get(hardcoverBook.book_id));
|
||||
if (byHardcoverId === 'ambiguous') return { outcome: 'skipped', reason: 'Multiple BookOrbit books match this Hardcover ID' };
|
||||
if (byHardcoverId) return { book: byHardcoverId, method: 'hardcover_id', confidence: 100 };
|
||||
@@ -441,33 +443,12 @@ export class HardcoverImportService {
|
||||
if (byIsbn) return { book: byIsbn, method: 'isbn', confidence: 100 };
|
||||
}
|
||||
|
||||
const titleMatch = this.findTitleAuthorMatch(hardcoverBook.book?.title ?? null, hardcoverAuthors, indexes.books);
|
||||
if (titleMatch) return titleMatch;
|
||||
const titleMatch = await indexes.fuzzy.findMatch(hardcoverBook.book?.title ?? null, hardcoverAuthors);
|
||||
if (titleMatch) return { ...titleMatch, method: 'title_author' };
|
||||
|
||||
return { outcome: 'unmatched', reason: 'No matching BookOrbit book found' };
|
||||
}
|
||||
|
||||
private findTitleAuthorMatch(title: string | null, authors: string[], localBooks: HardcoverImportLocalBook[]): MatchResult | null {
|
||||
if (!title?.trim() || authors.length === 0) return null;
|
||||
|
||||
const scored = localBooks
|
||||
.filter((book) => book.title?.trim() && book.authors.length > 0)
|
||||
.map((book) => {
|
||||
const titleScore = scoreTitle(title, book.title!);
|
||||
const authorScore = scoreAuthors(authors, book.authors);
|
||||
const confidence = Math.round((titleScore * 0.7 + authorScore * 0.3) * 100);
|
||||
return { book, titleScore, authorScore, confidence };
|
||||
})
|
||||
.filter((row) => row.titleScore >= 0.8 && row.authorScore >= 0.75 && row.confidence >= TITLE_AUTHOR_ACCEPT_SCORE)
|
||||
.sort((a, b) => b.confidence - a.confidence);
|
||||
|
||||
const best = scored[0];
|
||||
if (!best) return null;
|
||||
const second = scored[1];
|
||||
if (second && best.confidence - second.confidence <= TITLE_AUTHOR_TIE_MARGIN) return null;
|
||||
return { book: best.book, method: 'title_author', confidence: best.confidence };
|
||||
}
|
||||
|
||||
private async applyProgressIfRequested(
|
||||
userId: number,
|
||||
row: HardcoverImportPreviewRow,
|
||||
@@ -661,84 +642,6 @@ function extractHardcoverAuthors(value: unknown): string[] {
|
||||
return authors.filter((author) => author.length > 0);
|
||||
}
|
||||
|
||||
function scoreTitle(a: string, b: string): number {
|
||||
const left = normalizeTitle(a);
|
||||
const right = normalizeTitle(b);
|
||||
if (!left || !right) return 0;
|
||||
if (left === right) return 1;
|
||||
if (left.startsWith(right) || right.startsWith(left)) return 0.94;
|
||||
if (left.includes(right) || right.includes(left)) return 0.9;
|
||||
|
||||
const tokenScore = tokenOverlap(left, right);
|
||||
const editScore = normalizedLevenshtein(left, right);
|
||||
return Math.max(tokenScore, editScore >= 0.7 ? editScore : 0);
|
||||
}
|
||||
|
||||
function scoreAuthors(hardcoverAuthors: string[], localAuthors: string[]): number {
|
||||
let best = 0;
|
||||
for (const hardcoverAuthor of hardcoverAuthors) {
|
||||
for (const localAuthor of localAuthors) {
|
||||
best = Math.max(best, scoreAuthor(hardcoverAuthor, localAuthor));
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
function scoreAuthor(a: string, b: string): number {
|
||||
const left = normalizeName(a);
|
||||
const right = normalizeName(b);
|
||||
if (!left || !right) return 0;
|
||||
if (left === right) return 1;
|
||||
const leftTokens = tokenize(left);
|
||||
const rightTokens = tokenize(right);
|
||||
if (haveEqualTokenSet(leftTokens, rightTokens)) return 0.98;
|
||||
const overlap = tokenOverlap(left, right);
|
||||
const edit = normalizedLevenshtein(left, right);
|
||||
return Math.max(overlap, edit >= 0.76 ? edit : 0);
|
||||
}
|
||||
|
||||
function normalizeTitle(value: string): string {
|
||||
const stripped = value.split(/:\s+| - /)[0] ?? value;
|
||||
return normalizeName(stripped);
|
||||
}
|
||||
|
||||
function normalizeName(value: string): string {
|
||||
return value
|
||||
.normalize('NFKD')
|
||||
.replace(/[\u0300-\u036f]/g, '')
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9\s]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function tokenize(value: string): string[] {
|
||||
return value.split(' ').filter((token) => token.length > 1);
|
||||
}
|
||||
|
||||
function tokenOverlap(a: string, b: string): number {
|
||||
const left = new Set(tokenize(a));
|
||||
const right = new Set(tokenize(b));
|
||||
if (left.size === 0 || right.size === 0) return 0;
|
||||
let overlap = 0;
|
||||
for (const token of left) {
|
||||
if (right.has(token)) overlap++;
|
||||
}
|
||||
return overlap / Math.max(left.size, right.size);
|
||||
}
|
||||
|
||||
function normalizedLevenshtein(a: string, b: string): number {
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
return maxLen === 0 ? 1 : 1 - distance(a, b) / maxLen;
|
||||
}
|
||||
|
||||
function haveEqualTokenSet(a: string[], b: string[]): boolean {
|
||||
if (a.length !== b.length) return false;
|
||||
const left = [...a].sort();
|
||||
const right = [...b].sort();
|
||||
return left.every((token, index) => token === right[index]);
|
||||
}
|
||||
|
||||
function roundProgressPercent(value: number | null | undefined): number | null {
|
||||
const finite = toFiniteNumber(value);
|
||||
if (finite == null) return null;
|
||||
|
||||
Reference in New Issue
Block a user