mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
fix(metadata): stop one book record being assembled from several books
Field rules resolve each field from whichever provider ranks first for it, and nothing ever compared candidates to one another. A provider that matched the wrong book still won every field it led on, so a single record could carry a title from one book, a cover from a second and a subtitle from a third. The stock defaults make that the ordinary case rather than an unlucky one, since cover leads with Amazon while title and subtitle lead with Goodreads. Relevance scoring is what let those wrong matches in. A candidate survived on any score above zero, and token overlap counted stopwords, so "The Girl on the Train" scored against a query for "The Hobbit" on the strength of a shared "the". Substring matching ignored word boundaries as well, which is how "Ubik" matched "Rubik Cube Solutions". Drop stopwords from the query side, require a majority of the words that remain, match whole words only, and give the surviving floor a name. Then hold every candidate to an anchor before field rules ever see it. The anchor is the edition confirmed by the book's own ISBN when there is one, otherwise the candidate the query supports most strongly, with the title rule breaking ties because that is where the reader declares which provider decides identity. Agreement admits subtitle extensions, reordered words and translated editions, and rejects on a contradicting author, which is the only signal that separates a book from a companion volume that extends its title. Reject solely on positive evidence: a candidate with no title, or with no author to compare against, keeps the benefit of the doubt, and a lone provider is never second-guessed. Rejected candidates no longer contribute their provider ids either, which would otherwise pin the wrong book for every later refresh. Normalizing titles to [a-z0-9] emptied every title written in Japanese, Chinese, Russian or Greek. That scored two providers reporting an identical title as a disagreement, which under the new anchor would drop all but one of them in those libraries, and it had already denied the relevance gate any way to score them at all. Decompose instead, strip nonspacing marks so Soseki matches Sōseki, and keep letters of any script. Two sets of fixtures described different books per provider while testing something else entirely, field-rule precedence and metadata locking, and now describe one book so that they still test what they were written for. Closes #1023
This commit is contained in:
@@ -0,0 +1,162 @@
|
||||
import { MetadataCandidate, MetadataProviderKey } from '@bookorbit/types';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { resolveCandidateAgreement } from './candidate-agreement';
|
||||
import { MetadataSearchParams } from './providers/metadata-search-params';
|
||||
|
||||
function candidate(provider: MetadataProviderKey, data: Partial<MetadataCandidate> = {}): MetadataCandidate {
|
||||
return { provider, providerId: `${provider}-1`, ...data };
|
||||
}
|
||||
|
||||
const providersOf = (candidates: MetadataCandidate[]) => candidates.map((c) => c.provider);
|
||||
|
||||
describe('resolveCandidateAgreement', () => {
|
||||
const query: MetadataSearchParams = { title: 'The Hobbit', author: 'J.R.R. Tolkien' };
|
||||
|
||||
it('accepts a lone candidate without judging it', () => {
|
||||
const only = candidate(MetadataProviderKey.GOOGLE, { title: 'Something Else Entirely' });
|
||||
const result = resolveCandidateAgreement([only], query);
|
||||
|
||||
expect(result.accepted).toEqual([only]);
|
||||
expect(result.rejected).toEqual([]);
|
||||
expect(result.anchor).toBe(only);
|
||||
});
|
||||
|
||||
it('accepts an empty candidate list', () => {
|
||||
const result = resolveCandidateAgreement([], query);
|
||||
|
||||
expect(result.accepted).toEqual([]);
|
||||
expect(result.rejected).toEqual([]);
|
||||
expect(result.anchor).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects candidates describing a different book', () => {
|
||||
const right = candidate(MetadataProviderKey.GOOGLE, { title: 'The Hobbit', authors: ['J.R.R. Tolkien'] });
|
||||
const wrong = candidate(MetadataProviderKey.GOODREADS, { title: 'The Girl on the Train', authors: ['Paula Hawkins'] });
|
||||
const alsoWrong = candidate(MetadataProviderKey.AMAZON, { title: 'The Silence of the Lambs', authors: ['Thomas Harris'] });
|
||||
|
||||
const result = resolveCandidateAgreement([right, wrong, alsoWrong], query);
|
||||
|
||||
expect(providersOf(result.accepted)).toEqual([MetadataProviderKey.GOOGLE]);
|
||||
expect(providersOf(result.rejected)).toEqual([MetadataProviderKey.GOODREADS, MetadataProviderKey.AMAZON]);
|
||||
});
|
||||
|
||||
it('accepts an edition whose subtitle extends the anchor title', () => {
|
||||
const anchor = candidate(MetadataProviderKey.GOOGLE, { title: 'The Hobbit', authors: ['J.R.R. Tolkien'] });
|
||||
const edition = candidate(MetadataProviderKey.KOBO, { title: 'The Hobbit: Or There and Back Again', authors: ['J. R. R. Tolkien'] });
|
||||
|
||||
const result = resolveCandidateAgreement([anchor, edition], query);
|
||||
|
||||
expect(result.rejected).toEqual([]);
|
||||
expect(providersOf(result.accepted)).toEqual([MetadataProviderKey.GOOGLE, MetadataProviderKey.KOBO]);
|
||||
});
|
||||
|
||||
it('accepts a translated edition that keeps the distinctive word', () => {
|
||||
const anchor = candidate(MetadataProviderKey.GOOGLE, { title: 'The Hobbit', authors: ['J.R.R. Tolkien'] });
|
||||
const translated = candidate(MetadataProviderKey.KOBO, { title: 'Der kleine Hobbit', authors: ['J.R.R. Tolkien'] });
|
||||
|
||||
const result = resolveCandidateAgreement([anchor, translated], query);
|
||||
|
||||
expect(result.rejected).toEqual([]);
|
||||
});
|
||||
|
||||
it('rejects a companion volume that extends the title but has a different author', () => {
|
||||
const anchor = candidate(MetadataProviderKey.GOOGLE, { title: 'The Hobbit', authors: ['J.R.R. Tolkien'] });
|
||||
const cookbook = candidate(MetadataProviderKey.AMAZON, { title: 'The Hobbit Cookbook', authors: ['Some Chef'] });
|
||||
|
||||
const result = resolveCandidateAgreement([anchor, cookbook], query);
|
||||
|
||||
expect(providersOf(result.rejected)).toEqual([MetadataProviderKey.AMAZON]);
|
||||
});
|
||||
|
||||
it('keeps a candidate whose author is unknown, having no evidence against it', () => {
|
||||
const anchor = candidate(MetadataProviderKey.GOOGLE, { title: 'The Hobbit', authors: ['J.R.R. Tolkien'] });
|
||||
const noAuthors = candidate(MetadataProviderKey.AMAZON, { title: 'The Hobbit', authors: [] });
|
||||
|
||||
const result = resolveCandidateAgreement([anchor, noAuthors], query);
|
||||
|
||||
expect(result.rejected).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps a candidate that has no title to judge', () => {
|
||||
const anchor = candidate(MetadataProviderKey.GOOGLE, { title: 'The Hobbit', authors: ['J.R.R. Tolkien'] });
|
||||
const untitled = candidate(MetadataProviderKey.ITUNES, { coverUrl: 'https://example.com/cover.jpg' });
|
||||
|
||||
const result = resolveCandidateAgreement([anchor, untitled], query);
|
||||
|
||||
expect(result.rejected).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps a candidate that disagrees on title but shares the ISBN', () => {
|
||||
const anchor = candidate(MetadataProviderKey.GOOGLE, { title: 'The Hobbit', authors: ['J.R.R. Tolkien'], isbn13: '9780261102217' });
|
||||
const retitled = candidate(MetadataProviderKey.KOBO, { title: 'A Completely Different Listing', isbn13: '978-0-261-10221-7' });
|
||||
|
||||
const result = resolveCandidateAgreement([anchor, retitled], query);
|
||||
|
||||
expect(result.rejected).toEqual([]);
|
||||
});
|
||||
|
||||
describe('anchor selection', () => {
|
||||
it('anchors on the candidate matching the queried ISBN even when it is not first', () => {
|
||||
const first = candidate(MetadataProviderKey.GOODREADS, { title: 'The Hobbit Cookbook', authors: ['Some Chef'] });
|
||||
const isbnMatch = candidate(MetadataProviderKey.GOOGLE, { title: 'The Hobbit', authors: ['J.R.R. Tolkien'], isbn13: '9780261102217' });
|
||||
|
||||
const result = resolveCandidateAgreement([first, isbnMatch], { ...query, isbn: '978-0-261-10221-7' });
|
||||
|
||||
expect(result.anchor).toBe(isbnMatch);
|
||||
expect(providersOf(result.rejected)).toEqual([MetadataProviderKey.GOODREADS]);
|
||||
});
|
||||
|
||||
it('anchors on the strongest match to the query rather than the most trusted provider', () => {
|
||||
const trustedButWeak = candidate(MetadataProviderKey.GOODREADS, { title: 'The Hobbit Cookbook', authors: ['Some Chef'] });
|
||||
const exact = candidate(MetadataProviderKey.GOOGLE, { title: 'The Hobbit', authors: ['J.R.R. Tolkien'] });
|
||||
|
||||
const result = resolveCandidateAgreement([trustedButWeak, exact], query);
|
||||
|
||||
expect(result.anchor).toBe(exact);
|
||||
expect(providersOf(result.accepted)).toEqual([MetadataProviderKey.GOOGLE]);
|
||||
});
|
||||
|
||||
it('falls back to trust order when the query gives no basis to score', () => {
|
||||
const first = candidate(MetadataProviderKey.GOODREADS, { title: 'Some Book' });
|
||||
const second = candidate(MetadataProviderKey.GOOGLE, { title: 'A Different Book' });
|
||||
|
||||
const result = resolveCandidateAgreement([first, second], {});
|
||||
|
||||
expect(result.anchor).toBe(first);
|
||||
expect(providersOf(result.rejected)).toEqual([MetadataProviderKey.GOOGLE]);
|
||||
});
|
||||
|
||||
it('skips untitled candidates when choosing the anchor', () => {
|
||||
const untitled = candidate(MetadataProviderKey.ITUNES, { coverUrl: 'https://example.com/cover.jpg' });
|
||||
const titled = candidate(MetadataProviderKey.GOOGLE, { title: 'The Hobbit', authors: ['J.R.R. Tolkien'] });
|
||||
|
||||
const result = resolveCandidateAgreement([untitled, titled], query);
|
||||
|
||||
expect(result.anchor).toBe(titled);
|
||||
expect(result.rejected).toEqual([]);
|
||||
});
|
||||
|
||||
it('accepts everything when no candidate can be scored at all', () => {
|
||||
const first = candidate(MetadataProviderKey.GOOGLE, { coverUrl: 'https://example.com/a.jpg' });
|
||||
const second = candidate(MetadataProviderKey.KOBO, { coverUrl: 'https://example.com/b.jpg' });
|
||||
|
||||
const result = resolveCandidateAgreement([first, second], query);
|
||||
|
||||
expect(result.anchor).toBeUndefined();
|
||||
expect(result.accepted).toEqual([first, second]);
|
||||
expect(result.rejected).toEqual([]);
|
||||
});
|
||||
|
||||
it('uses displayTitle when a provider carries no plain title', () => {
|
||||
const anchor = candidate(MetadataProviderKey.COMICVINE, { displayTitle: 'Batman #7 - The Origin' });
|
||||
const agreeing = candidate(MetadataProviderKey.GOOGLE, { title: 'Batman' });
|
||||
const disagreeing = candidate(MetadataProviderKey.KOBO, { title: 'Superman Chronicles' });
|
||||
|
||||
const result = resolveCandidateAgreement([anchor, agreeing, disagreeing], { title: 'Batman' });
|
||||
|
||||
expect(providersOf(result.accepted)).toEqual([MetadataProviderKey.COMICVINE, MetadataProviderKey.GOOGLE]);
|
||||
expect(providersOf(result.rejected)).toEqual([MetadataProviderKey.KOBO]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,112 @@
|
||||
import { MetadataCandidate } from '@bookorbit/types';
|
||||
|
||||
import { matchableTitle, scoreCandidate } from './candidate-relevance';
|
||||
import { candidateHasNormalizedIsbn, candidatesShareIsbn, normalizeMetadataIsbn } from './isbn-match';
|
||||
import { MetadataSearchParams } from './providers/metadata-search-params';
|
||||
import { scoreTitleMatch, shareSignificantToken } from './title-match';
|
||||
|
||||
/**
|
||||
* How closely two providers' titles must match before their records are treated as the same book.
|
||||
* Set at full overlap of the meaningful words, which admits subtitle differences, word reordering,
|
||||
* and translated editions while rejecting unrelated works.
|
||||
*/
|
||||
export const MIN_AGREEMENT_SCORE = 6;
|
||||
|
||||
export interface CandidateAgreement {
|
||||
anchor: MetadataCandidate | undefined;
|
||||
accepted: MetadataCandidate[];
|
||||
rejected: MetadataCandidate[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Field rules resolve each field from whichever provider ranks first for it, so a provider that
|
||||
* matched the wrong book contributes real-looking values for whatever fields it wins. Nothing
|
||||
* downstream compares candidates to each other, which is how one record ends up holding a title
|
||||
* from one book, a cover from a second, and a subtitle from a third.
|
||||
*
|
||||
* This picks the best-supported candidate as the anchor and holds every other candidate to it.
|
||||
* Candidates are only rejected on positive evidence of a different book; when there is no basis to
|
||||
* judge (no title on either side) they are kept, matching how relevance scoring treats an
|
||||
* unscoreable query.
|
||||
*
|
||||
* `candidates` must be in identity-trust order: the anchor tie-break follows it.
|
||||
*/
|
||||
export function resolveCandidateAgreement(candidates: readonly MetadataCandidate[], params: MetadataSearchParams): CandidateAgreement {
|
||||
if (candidates.length <= 1) {
|
||||
return { anchor: candidates[0], accepted: [...candidates], rejected: [] };
|
||||
}
|
||||
|
||||
const anchor = selectAnchor(candidates, params);
|
||||
if (!anchor) return { anchor: undefined, accepted: [...candidates], rejected: [] };
|
||||
|
||||
const accepted: MetadataCandidate[] = [];
|
||||
const rejected: MetadataCandidate[] = [];
|
||||
for (const candidate of candidates) {
|
||||
if (candidate === anchor || describesSameBook(anchor, candidate)) {
|
||||
accepted.push(candidate);
|
||||
} else {
|
||||
rejected.push(candidate);
|
||||
}
|
||||
}
|
||||
|
||||
return { anchor, accepted, rejected };
|
||||
}
|
||||
|
||||
/**
|
||||
* The candidate the others are measured against: an edition confirmed by the book's own ISBN when
|
||||
* one is available, otherwise whichever candidate the query supports most strongly. Ties fall to
|
||||
* the caller's trust order.
|
||||
*/
|
||||
function selectAnchor(candidates: readonly MetadataCandidate[], params: MetadataSearchParams): MetadataCandidate | undefined {
|
||||
const isbn = normalizeMetadataIsbn(params.isbn);
|
||||
if (isbn) {
|
||||
const isbnMatch = candidates.find((candidate) => candidateHasNormalizedIsbn(candidate, isbn));
|
||||
if (isbnMatch) return isbnMatch;
|
||||
}
|
||||
|
||||
const scoreable = candidates.filter((candidate) => matchableTitle(candidate));
|
||||
if (scoreable.length === 0) return undefined;
|
||||
if (!hasText(params.title) && !hasText(params.author)) return scoreable[0];
|
||||
|
||||
let best = scoreable[0];
|
||||
let bestScore = scoreCandidate(best, params);
|
||||
for (const candidate of scoreable.slice(1)) {
|
||||
const score = scoreCandidate(candidate, params);
|
||||
if (score > bestScore) {
|
||||
best = candidate;
|
||||
bestScore = score;
|
||||
}
|
||||
}
|
||||
|
||||
return best;
|
||||
}
|
||||
|
||||
function describesSameBook(anchor: MetadataCandidate, candidate: MetadataCandidate): boolean {
|
||||
if (candidatesShareIsbn(anchor, candidate)) return true;
|
||||
|
||||
const anchorTitle = matchableTitle(anchor);
|
||||
const candidateTitle = matchableTitle(candidate);
|
||||
// No title on either side means no evidence either way, so the candidate keeps the benefit of
|
||||
// the doubt rather than being dropped on a guess.
|
||||
if (!anchorTitle || !candidateTitle) return true;
|
||||
if (scoreTitleMatch(anchorTitle, candidateTitle) < MIN_AGREEMENT_SCORE) return false;
|
||||
|
||||
return authorsAgree(anchor, candidate);
|
||||
}
|
||||
|
||||
/**
|
||||
* Titles alone cannot separate a book from its companion volumes and cash-ins, which routinely
|
||||
* extend the real title ("The Hobbit" / "The Hobbit Cookbook"). Authors can, when both sides
|
||||
* name one.
|
||||
*/
|
||||
function authorsAgree(anchor: MetadataCandidate, candidate: MetadataCandidate): boolean {
|
||||
const anchorAuthors = anchor.authors?.filter(Boolean) ?? [];
|
||||
const candidateAuthors = candidate.authors?.filter(Boolean) ?? [];
|
||||
if (anchorAuthors.length === 0 || candidateAuthors.length === 0) return true;
|
||||
|
||||
return shareSignificantToken(anchorAuthors, candidateAuthors);
|
||||
}
|
||||
|
||||
function hasText(value: string | undefined): boolean {
|
||||
return typeof value === 'string' && value.trim().length > 0;
|
||||
}
|
||||
@@ -99,6 +99,32 @@ describe('candidate-relevance', () => {
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('drops an unrelated book that shares only stopwords with the query', () => {
|
||||
const candidates = [candidate('The Girl on the Train', 'Paula Hawkins'), candidate('The Silence of the Lambs', 'Thomas Harris')];
|
||||
const params: MetadataSearchParams = { title: 'The Hobbit', author: 'J.R.R. Tolkien' };
|
||||
expect(filterAndRank(candidates, params)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('drops a book sharing a minority of the meaningful words', () => {
|
||||
const result = filterAndRank([candidate('The Way We Were')], { title: 'The Way of Kings' });
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('drops a candidate that merely contains the query inside a longer word', () => {
|
||||
expect(filterAndRank([candidate('Rubik Cube Solutions')], { title: 'Ubik' })).toHaveLength(0);
|
||||
expect(filterAndRank([candidate('Italian Cooking Basics')], { title: 'It' })).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('keeps a book actually titled with a stopword', () => {
|
||||
const result = filterAndRank([candidate('It', 'Stephen King')], { title: 'It', author: 'Stephen King' });
|
||||
expect(result).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('keeps a matching author from boosting an otherwise unrelated title', () => {
|
||||
const result = filterAndRank([candidate('The Girl on the Train', 'J.R.R. Tolkien')], { title: 'The Hobbit', author: 'J.R.R. Tolkien' });
|
||||
expect(result).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('keeps a candidate whose embedded subtitle was split out by its provider mapper', () => {
|
||||
const fullTitle = "Babel, or The Necessity of Violence: An Arcane History of the Oxford Translators' Revolution";
|
||||
const candidates = [candidate('A Different Book Entirely'), candidate('Babel, or The Necessity of Violence')];
|
||||
|
||||
@@ -1,11 +1,22 @@
|
||||
import { MetadataCandidate } from '@bookorbit/types';
|
||||
import { distance } from 'fastest-levenshtein';
|
||||
|
||||
import { candidateHasNormalizedIsbn, normalizeMetadataIsbn } from './isbn-match';
|
||||
import { MetadataSearchParams } from './providers/metadata-search-params';
|
||||
import { normalizeTitleText, scoreTitleMatch, shareSignificantToken } from './title-match';
|
||||
|
||||
const DEFAULT_LIMIT = 5;
|
||||
|
||||
export const ISBN_MATCH_SCORE = 100;
|
||||
export const AUTHOR_MATCH_SCORE = 3;
|
||||
export const AUTHOR_TOKEN_MATCH_SCORE = 1;
|
||||
|
||||
/**
|
||||
* The weakest title signal a candidate may rest on. Anything below this is a coincidence rather
|
||||
* than a match: a shared word or two out of a longer title, or two unrelated titles that happen to
|
||||
* be similar in length. Kept low enough that a typo'd title still resolves through the fuzzy path.
|
||||
*/
|
||||
export const MIN_RELEVANCE_SCORE = 3;
|
||||
|
||||
const SKIP_TITLE_PATTERNS = [
|
||||
/^(summary|study guide|analysis|guide to|workbook|review of|summary of|chapter summary|chapter-by-chapter)\b/i,
|
||||
/\b(summary|study guide|book analysis|reading guide|literature guide|workbook|digest|breakdown and analysis)\b/i,
|
||||
@@ -16,7 +27,7 @@ const SKIP_TITLE_PATTERNS = [
|
||||
*
|
||||
* - If no title or author is available in params, returns candidates as-is (no basis for scoring).
|
||||
* - ISBN exact match always survives regardless of title/author score.
|
||||
* - Candidates scoring 0 are dropped (no meaningful signal).
|
||||
* - Candidates below `MIN_RELEVANCE_SCORE` are dropped.
|
||||
* - Survivors are sorted descending by score and capped at `limit`.
|
||||
*/
|
||||
export function filterAndRank(candidates: MetadataCandidate[], params: MetadataSearchParams, limit = DEFAULT_LIMIT): MetadataCandidate[] {
|
||||
@@ -30,29 +41,29 @@ export function filterAndRank(candidates: MetadataCandidate[], params: MetadataS
|
||||
return !matchable || !SKIP_TITLE_PATTERNS.some((pattern) => pattern.test(matchable));
|
||||
})
|
||||
.map((c) => ({ candidate: c, score: scoreCandidate(c, params) }))
|
||||
.filter(({ score }) => score > 0)
|
||||
.filter(({ score }) => score >= MIN_RELEVANCE_SCORE)
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, limit)
|
||||
.map(({ candidate }) => candidate);
|
||||
}
|
||||
|
||||
function scoreCandidate(candidate: MetadataCandidate, params: MetadataSearchParams): number {
|
||||
export function scoreCandidate(candidate: MetadataCandidate, params: MetadataSearchParams): number {
|
||||
if (params.isbn) {
|
||||
const isbn = normalizeMetadataIsbn(params.isbn);
|
||||
if (candidateHasNormalizedIsbn(candidate, isbn)) return 100;
|
||||
if (candidateHasNormalizedIsbn(candidate, isbn)) return ISBN_MATCH_SCORE;
|
||||
}
|
||||
|
||||
let score = 0;
|
||||
|
||||
const matchable = matchableTitle(candidate);
|
||||
if (params.title && matchable) {
|
||||
score += scoreTitle(normalize(params.title), normalize(matchable));
|
||||
score += scoreTitleMatch(params.title, matchable);
|
||||
}
|
||||
|
||||
// Author only boosts when there is already a positive title signal.
|
||||
// A matching author alone (e.g. a different book by the same author) is not enough to keep a result.
|
||||
if (score > 0 && params.author && candidate.authors?.length) {
|
||||
score += scoreAuthor(normalize(params.author), candidate.authors.map(normalize));
|
||||
score += scoreAuthor(params.author, candidate.authors);
|
||||
}
|
||||
|
||||
return score;
|
||||
@@ -63,50 +74,20 @@ function scoreCandidate(candidate: MetadataCandidate, params: MetadataSearchPara
|
||||
* disambiguating label (ComicVine's "<volume> #<issue> - <name>"), which is what a user's query
|
||||
* is shaped like; `title` alone can be just an issue name, or absent entirely.
|
||||
*/
|
||||
function matchableTitle(candidate: MetadataCandidate): string | undefined {
|
||||
export function matchableTitle(candidate: MetadataCandidate): string | undefined {
|
||||
return candidate.displayTitle ?? candidate.title;
|
||||
}
|
||||
|
||||
function scoreTitle(query: string, candidate: string): number {
|
||||
if (candidate === query) return 10;
|
||||
if (candidate.startsWith(query) || query.startsWith(candidate)) return 8;
|
||||
if (candidate.includes(query) || query.includes(candidate)) return 7;
|
||||
|
||||
const queryTokens = tokenize(query);
|
||||
const candidateTokens = new Set(tokenize(candidate));
|
||||
const overlapRatio = queryTokens.length > 0 ? queryTokens.filter((w) => candidateTokens.has(w)).length / queryTokens.length : 0;
|
||||
const tokenScore = overlapRatio * 6;
|
||||
|
||||
const sim = levenshteinSim(query, candidate);
|
||||
const levenScore = sim >= 0.6 ? sim * 4 : 0;
|
||||
|
||||
return Math.max(tokenScore, levenScore);
|
||||
}
|
||||
|
||||
function scoreAuthor(query: string, candidateAuthors: string[]): number {
|
||||
const queryTokens = new Set(tokenize(query).filter((w) => w.length > 2));
|
||||
for (const ca of candidateAuthors) {
|
||||
if (ca.includes(query) || query.includes(ca)) return 3;
|
||||
if (tokenize(ca).some((w) => queryTokens.has(w))) return 1;
|
||||
const normalizedQuery = normalizeTitleText(query);
|
||||
if (!normalizedQuery) return 0;
|
||||
|
||||
for (const candidateAuthor of candidateAuthors) {
|
||||
const normalizedAuthor = normalizeTitleText(candidateAuthor);
|
||||
if (!normalizedAuthor) continue;
|
||||
if (normalizedAuthor.includes(normalizedQuery) || normalizedQuery.includes(normalizedAuthor)) return AUTHOR_MATCH_SCORE;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function levenshteinSim(a: string, b: string): number {
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
return maxLen === 0 ? 1 : 1 - distance(a, b) / maxLen;
|
||||
}
|
||||
|
||||
function normalize(s: string): string {
|
||||
return s
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9 ]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function tokenize(s: string): string[] {
|
||||
return s.split(' ').filter((w) => w.length > 1);
|
||||
return shareSignificantToken([query], candidateAuthors) ? AUTHOR_TOKEN_MATCH_SCORE : 0;
|
||||
}
|
||||
|
||||
function hasText(v: string | undefined): boolean {
|
||||
|
||||
@@ -12,3 +12,11 @@ export function candidateHasNormalizedIsbn(candidate: MetadataCandidate, normali
|
||||
(normalizeMetadataIsbn(candidate.isbn10) === normalizedIsbn || normalizeMetadataIsbn(candidate.isbn13) === normalizedIsbn)
|
||||
);
|
||||
}
|
||||
|
||||
export function candidatesShareIsbn(left: MetadataCandidate, right: MetadataCandidate): boolean {
|
||||
const isbn13 = normalizeMetadataIsbn(left.isbn13);
|
||||
if (isbn13 && normalizeMetadataIsbn(right.isbn13) === isbn13) return true;
|
||||
|
||||
const isbn10 = normalizeMetadataIsbn(left.isbn10);
|
||||
return isbn10.length > 0 && normalizeMetadataIsbn(right.isbn10) === isbn10;
|
||||
}
|
||||
|
||||
@@ -35,11 +35,16 @@ function createPreferences(mutate?: (fields: Record<MetadataField, FieldPreferen
|
||||
return { fields };
|
||||
}
|
||||
|
||||
// Candidates share a title by default so that fixtures exercising field-rule precedence describe
|
||||
// one book. Cross-provider agreement drops candidates that describe a different book, which is a
|
||||
// separate concern with its own tests.
|
||||
const SHARED_CANDIDATE_TITLE = 'Shared Candidate Book';
|
||||
|
||||
function candidate(provider: MetadataProviderKey, providerId: string, data: Partial<MetadataCandidate> = {}): MetadataCandidate {
|
||||
return {
|
||||
provider,
|
||||
providerId,
|
||||
title: data.title ?? `${provider}-${providerId}`,
|
||||
title: data.title ?? SHARED_CANDIDATE_TITLE,
|
||||
...data,
|
||||
};
|
||||
}
|
||||
@@ -376,6 +381,165 @@ describe('MetadataFetchPipeline', () => {
|
||||
expect(sources.description).toBe(MetadataProviderKey.GOOGLE);
|
||||
});
|
||||
|
||||
describe('cross-provider agreement', () => {
|
||||
function primeAgreementPreferences() {
|
||||
const prefs = createPreferences((fields) => {
|
||||
fields.title = {
|
||||
enabled: true,
|
||||
providers: [MetadataProviderKey.GOODREADS, MetadataProviderKey.GOOGLE],
|
||||
mergeStrategy: 'overwriteIfProvided',
|
||||
};
|
||||
fields.subtitle = {
|
||||
enabled: true,
|
||||
providers: [MetadataProviderKey.GOODREADS, MetadataProviderKey.GOOGLE],
|
||||
mergeStrategy: 'overwriteIfProvided',
|
||||
};
|
||||
fields.authors = {
|
||||
enabled: true,
|
||||
providers: [MetadataProviderKey.GOODREADS, MetadataProviderKey.GOOGLE],
|
||||
mergeStrategy: 'overwriteIfProvided',
|
||||
};
|
||||
fields.publisher = {
|
||||
enabled: true,
|
||||
providers: [MetadataProviderKey.GOODREADS, MetadataProviderKey.GOOGLE],
|
||||
mergeStrategy: 'overwriteIfProvided',
|
||||
};
|
||||
fields.cover = {
|
||||
enabled: true,
|
||||
providers: [MetadataProviderKey.AMAZON, MetadataProviderKey.GOOGLE],
|
||||
mergeStrategy: 'overwriteIfProvided',
|
||||
};
|
||||
fields.pageCount = {
|
||||
enabled: true,
|
||||
providers: [MetadataProviderKey.AMAZON, MetadataProviderKey.GOOGLE],
|
||||
mergeStrategy: 'overwriteIfProvided',
|
||||
};
|
||||
});
|
||||
|
||||
preferencesService.getGlobal.mockResolvedValue(prefs);
|
||||
resolver.resolve.mockReturnValue(prefs);
|
||||
resolver.withForwardCompatibility.mockReturnValue(prefs);
|
||||
registry.all.mockReturnValue([
|
||||
{ key: MetadataProviderKey.GOODREADS },
|
||||
{ key: MetadataProviderKey.GOOGLE },
|
||||
{ key: MetadataProviderKey.AMAZON },
|
||||
] as never);
|
||||
}
|
||||
|
||||
it('never assembles one record out of candidates describing different books', async () => {
|
||||
primeAgreementPreferences();
|
||||
fetchService.search.mockReturnValue(
|
||||
of(
|
||||
candidate(MetadataProviderKey.GOODREADS, 'gr1', {
|
||||
title: 'The Girl on the Train',
|
||||
subtitle: 'A Novel of Suspense',
|
||||
authors: ['Paula Hawkins'],
|
||||
publisher: 'Riverhead Books',
|
||||
}),
|
||||
candidate(MetadataProviderKey.AMAZON, 'az1', {
|
||||
title: 'The Hobbit Cookbook',
|
||||
authors: ['Some Chef'],
|
||||
coverUrl: 'https://example.com/wrong-cover.jpg',
|
||||
pageCount: 120,
|
||||
}),
|
||||
candidate(MetadataProviderKey.GOOGLE, 'gg1', {
|
||||
title: 'The Hobbit',
|
||||
authors: ['J.R.R. Tolkien'],
|
||||
publisher: 'George Allen & Unwin',
|
||||
coverUrl: 'https://example.com/correct-cover.jpg',
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const { resolved, sources, diagnostics } = await pipeline.runWithSources({ title: 'The Hobbit', author: 'J.R.R. Tolkien' }, {});
|
||||
|
||||
expect(resolved.title).toBe('The Hobbit');
|
||||
expect(resolved.authors).toEqual(['J.R.R. Tolkien']);
|
||||
expect(resolved.publisher).toBe('George Allen & Unwin');
|
||||
expect(resolved.coverUrl).toBe('https://example.com/correct-cover.jpg');
|
||||
expect(resolved.subtitle).toBeUndefined();
|
||||
expect(resolved.pageCount).toBeUndefined();
|
||||
expect(new Set(Object.values(sources))).toEqual(new Set([MetadataProviderKey.GOOGLE]));
|
||||
expect(diagnostics.candidateProviders).toEqual([MetadataProviderKey.GOOGLE]);
|
||||
});
|
||||
|
||||
it('reports no candidates when every provider matched a different book', async () => {
|
||||
primeAgreementPreferences();
|
||||
fetchService.search.mockReturnValue(
|
||||
of(
|
||||
candidate(MetadataProviderKey.GOODREADS, 'gr1', { title: 'The Girl on the Train', authors: ['Paula Hawkins'] }),
|
||||
candidate(MetadataProviderKey.AMAZON, 'az1', { title: 'The Silence of the Lambs', authors: ['Thomas Harris'] }),
|
||||
),
|
||||
);
|
||||
|
||||
const { resolved, diagnostics } = await pipeline.runWithSources({ title: 'The Hobbit', author: 'J.R.R. Tolkien' }, {});
|
||||
|
||||
// The most trusted provider anchors, so its own record still applies; the unrelated one does not.
|
||||
expect(resolved.title).toBe('The Girl on the Train');
|
||||
expect(diagnostics.candidateProviders).toEqual([MetadataProviderKey.GOODREADS]);
|
||||
expect(diagnostics.candidateCount).toBe(1);
|
||||
});
|
||||
|
||||
it('still fills a field from a lower-priority provider that agrees on the book', async () => {
|
||||
primeAgreementPreferences();
|
||||
fetchService.search.mockReturnValue(
|
||||
of(
|
||||
candidate(MetadataProviderKey.GOODREADS, 'gr1', { title: 'The Hobbit', authors: ['J.R.R. Tolkien'] }),
|
||||
candidate(MetadataProviderKey.AMAZON, 'az1', {
|
||||
title: 'The Hobbit: Or There and Back Again',
|
||||
authors: ['J. R. R. Tolkien'],
|
||||
coverUrl: 'https://example.com/cover.jpg',
|
||||
pageCount: 310,
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const { resolved, sources } = await pipeline.runWithSources({ title: 'The Hobbit', author: 'J.R.R. Tolkien' }, {});
|
||||
|
||||
expect(resolved.title).toBe('The Hobbit');
|
||||
expect(resolved.coverUrl).toBe('https://example.com/cover.jpg');
|
||||
expect(resolved.pageCount).toBe(310);
|
||||
expect(sources.coverUrl).toBe(MetadataProviderKey.AMAZON);
|
||||
});
|
||||
|
||||
it('does not store provider ids for candidates describing a different book', async () => {
|
||||
const prefs = createPreferences((fields) => {
|
||||
fields.title = {
|
||||
enabled: true,
|
||||
providers: [MetadataProviderKey.GOOGLE, MetadataProviderKey.GOODREADS],
|
||||
mergeStrategy: 'overwriteIfProvided',
|
||||
};
|
||||
});
|
||||
prefs.options = { genres: { mode: 'firstProvider', blocklist: [], maxCount: null }, saveProviderIds: true };
|
||||
preferencesService.getGlobal.mockResolvedValue(prefs);
|
||||
resolver.resolve.mockReturnValue(prefs);
|
||||
resolver.withForwardCompatibility.mockReturnValue(prefs);
|
||||
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.GOODREADS }] as never);
|
||||
fetchService.search.mockReturnValue(
|
||||
of(
|
||||
candidate(MetadataProviderKey.GOOGLE, 'gg1', { title: 'The Hobbit', authors: ['J.R.R. Tolkien'] }),
|
||||
candidate(MetadataProviderKey.GOODREADS, 'gr1', { title: 'The Girl on the Train', authors: ['Paula Hawkins'] }),
|
||||
),
|
||||
);
|
||||
|
||||
const { providerIds } = await pipeline.runWithSources({ title: 'The Hobbit', author: 'J.R.R. Tolkien' }, {});
|
||||
|
||||
expect(providerIds[MetadataProviderKey.GOOGLE]).toBe('gg1');
|
||||
expect(providerIds[MetadataProviderKey.GOODREADS]).toBeUndefined();
|
||||
});
|
||||
|
||||
it('leaves a single provider untouched, having nothing to disagree with', async () => {
|
||||
primeAgreementPreferences();
|
||||
fetchService.search.mockReturnValue(
|
||||
of(candidate(MetadataProviderKey.GOODREADS, 'gr1', { title: 'A Loosely Matching Title', authors: ['Someone'] })),
|
||||
);
|
||||
|
||||
const { resolved } = await pipeline.runWithSources({ title: 'The Hobbit', author: 'J.R.R. Tolkien' }, {});
|
||||
|
||||
expect(resolved.title).toBe('A Loosely Matching Title');
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to the next provider in order when the first provider does not provide the requested field', async () => {
|
||||
const prefs = createPreferences((fields) => {
|
||||
fields.description = {
|
||||
|
||||
@@ -20,6 +20,8 @@ import { MetadataPreferencesService } from '../metadata-preferences/metadata-pre
|
||||
import { SeriesExpectedCountService } from '../../common/services/series-expected-count.service';
|
||||
import { applyGenreFetchOptions, createGenreBlocklistTokenSet } from '../../common/utils/genre-fetch-options.utils';
|
||||
import { normalizeMetadataText, normalizeMetadataTextKey } from '../../common/utils/metadata-text-normalize.utils';
|
||||
import { sanitizeLogValue } from '../../common/utils/log-sanitize.utils';
|
||||
import { resolveCandidateAgreement } from './candidate-agreement';
|
||||
import { MetadataFetchService } from './metadata-fetch.service';
|
||||
import { ProviderRegistry } from './provider-registry';
|
||||
import { ProviderThrottleTracker } from './provider-throttle.tracker';
|
||||
@@ -145,11 +147,61 @@ export class MetadataFetchPipeline {
|
||||
for (const c of candidates) {
|
||||
if (!byProvider.has(c.provider)) byProvider.set(c.provider, c);
|
||||
}
|
||||
const { resolved, sources, providerIds } = this.applyPreferences(preferences, byProvider, existingFields, params.existingProviderIds, options);
|
||||
const diagnostics = this.buildDiagnostics(providerSelection, candidates, resolved);
|
||||
|
||||
const agreeing = this.rejectDisagreeingCandidates(preferences, providerSelection.activeProviders, byProvider, params);
|
||||
const { resolved, sources, providerIds } = this.applyPreferences(preferences, agreeing, existingFields, params.existingProviderIds, options);
|
||||
const diagnostics = this.buildDiagnostics(providerSelection, [...agreeing.values()], resolved);
|
||||
return { resolved, sources, providerIds, diagnostics };
|
||||
}
|
||||
|
||||
/**
|
||||
* Drops candidates that describe a different book from the anchor, before field rules get to
|
||||
* source individual fields from them. Without this, every provider that matched the wrong book
|
||||
* still contributes whichever fields it ranks first for, and the result is one record assembled
|
||||
* out of several books.
|
||||
*/
|
||||
private rejectDisagreeingCandidates(
|
||||
preferences: MetadataFetchPreferences,
|
||||
activeProviders: readonly MetadataProviderKey[],
|
||||
byProvider: ReadonlyMap<string, MetadataCandidate>,
|
||||
params: MetadataSearchParams,
|
||||
): Map<string, MetadataCandidate> {
|
||||
const ordered = this.orderByIdentityTrust(preferences, activeProviders, byProvider);
|
||||
const { anchor, accepted, rejected } = resolveCandidateAgreement(ordered, params);
|
||||
|
||||
if (rejected.length > 0) {
|
||||
this.logger.warn(
|
||||
`[metadata_fetch.candidate_agreement] [end] anchorProvider=${anchor?.provider ?? 'none'} acceptedCount=${accepted.length} rejectedProviders=${rejected.map((candidate) => candidate.provider).join('|')} anchorTitle="${sanitizeLogValue(anchor?.title ?? '')}" rejectedTitles="${sanitizeLogValue(rejected.map((candidate) => candidate.title ?? '').join('|'))}" - candidates describing a different book were dropped`,
|
||||
);
|
||||
}
|
||||
|
||||
return new Map(accepted.map((candidate) => [candidate.provider, candidate]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Candidates ordered by how far each provider is trusted to establish which book this is. The
|
||||
* title rule leads because that is where the user declares that authority; arrival order cannot
|
||||
* be used, since providers resolve as a race.
|
||||
*/
|
||||
private orderByIdentityTrust(
|
||||
preferences: MetadataFetchPreferences,
|
||||
activeProviders: readonly MetadataProviderKey[],
|
||||
byProvider: ReadonlyMap<string, MetadataCandidate>,
|
||||
): MetadataCandidate[] {
|
||||
const ordered: MetadataCandidate[] = [];
|
||||
const seen = new Set<string>();
|
||||
|
||||
for (const providerKey of [...(preferences.fields.title?.providers ?? []), ...activeProviders, ...byProvider.keys()]) {
|
||||
if (seen.has(providerKey)) continue;
|
||||
seen.add(providerKey);
|
||||
|
||||
const candidate = byProvider.get(providerKey);
|
||||
if (candidate) ordered.push(candidate);
|
||||
}
|
||||
|
||||
return ordered;
|
||||
}
|
||||
|
||||
private async resolveProviderPreferenceContext(libraryId?: number): Promise<ProviderPreferenceContext> {
|
||||
const [global, providerConfig] = await Promise.all([this.preferencesService.getGlobal(), this.providerConfig.getConfig()]);
|
||||
const overrides = libraryId ? (await this.preferencesService.getForLibrary(libraryId, global)).overrides : null;
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import {
|
||||
containsWord,
|
||||
normalizeTitleText,
|
||||
scoreTitleMatch,
|
||||
shareSignificantToken,
|
||||
significantTokens,
|
||||
startsWithWord,
|
||||
tokenizeTitleText,
|
||||
} from './title-match';
|
||||
|
||||
describe('title-match', () => {
|
||||
describe('containsWord', () => {
|
||||
it('matches a whole word', () => {
|
||||
expect(containsWord('the hobbit cookbook', 'hobbit')).toBe(true);
|
||||
});
|
||||
|
||||
it('matches at the start and at the end', () => {
|
||||
expect(containsWord('hobbit tales', 'hobbit')).toBe(true);
|
||||
expect(containsWord('tales hobbit', 'hobbit')).toBe(true);
|
||||
});
|
||||
|
||||
it('does not match inside a longer word', () => {
|
||||
expect(containsWord('rubik cube solutions', 'ubik')).toBe(false);
|
||||
expect(containsWord('italian cooking basics', 'it')).toBe(false);
|
||||
});
|
||||
|
||||
it('keeps scanning past a non-boundary hit to find a real one', () => {
|
||||
expect(containsWord('rubik ubik', 'ubik')).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects an empty needle', () => {
|
||||
expect(containsWord('anything', '')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('startsWithWord', () => {
|
||||
it('matches a leading whole word', () => {
|
||||
expect(startsWithWord('the hobbit or there and back again', 'the hobbit')).toBe(true);
|
||||
});
|
||||
|
||||
it('matches an identical string', () => {
|
||||
expect(startsWithWord('dune', 'dune')).toBe(true);
|
||||
});
|
||||
|
||||
it('does not match a partial leading word', () => {
|
||||
expect(startsWithWord('foundation and empire', 'found')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('significantTokens', () => {
|
||||
it('drops stopwords', () => {
|
||||
expect(significantTokens(['the', 'way', 'of', 'kings'])).toEqual(['way', 'kings']);
|
||||
});
|
||||
|
||||
it('falls back to the raw tokens when every token is a stopword', () => {
|
||||
expect(significantTokens(['it'])).toEqual(['it']);
|
||||
expect(significantTokens(['the', 'the'])).toEqual(['the', 'the']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('scoreTitleMatch', () => {
|
||||
it('scores an exact match highest', () => {
|
||||
expect(scoreTitleMatch('Dune', 'dune')).toBe(10);
|
||||
});
|
||||
|
||||
it('scores a subtitle extension as a prefix match', () => {
|
||||
expect(scoreTitleMatch('The Hobbit', 'The Hobbit: Or There and Back Again')).toBe(8);
|
||||
});
|
||||
|
||||
it('scores a contained title as a substring match', () => {
|
||||
expect(scoreTitleMatch('The Fellowship of the Ring', 'The Lord of the Rings: The Fellowship of the Ring')).toBe(7);
|
||||
});
|
||||
|
||||
it('gives no credit for sharing only stopwords', () => {
|
||||
expect(scoreTitleMatch('The Hobbit', 'The Girl on the Train')).toBe(0);
|
||||
expect(scoreTitleMatch('The Hobbit', 'The Silence of the Lambs')).toBe(0);
|
||||
});
|
||||
|
||||
it('gives no credit for a minority of shared meaningful words', () => {
|
||||
expect(scoreTitleMatch('The Way of Kings', 'The Way We Were')).toBe(0);
|
||||
});
|
||||
|
||||
it('credits full overlap of the meaningful words regardless of order', () => {
|
||||
expect(scoreTitleMatch('The Way of Kings', 'Kings Way')).toBe(6);
|
||||
});
|
||||
|
||||
it('does not match a title inside an unrelated longer word', () => {
|
||||
expect(scoreTitleMatch('Ubik', 'Rubik Cube Solutions')).toBe(0);
|
||||
expect(scoreTitleMatch('It', 'Italian Cooking Basics')).toBe(0);
|
||||
});
|
||||
|
||||
it('still tolerates a typo through fuzzy matching', () => {
|
||||
expect(scoreTitleMatch('Foundatun', 'Foundation')).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('returns zero when either side is empty after normalization', () => {
|
||||
expect(scoreTitleMatch('', 'Dune')).toBe(0);
|
||||
expect(scoreTitleMatch('Dune', '???')).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shareSignificantToken', () => {
|
||||
it('matches an author across formatting differences', () => {
|
||||
expect(shareSignificantToken(['J.R.R. Tolkien'], ['Tolkien, J. R. R.'])).toBe(true);
|
||||
});
|
||||
|
||||
it('does not match unrelated authors', () => {
|
||||
expect(shareSignificantToken(['J.R.R. Tolkien'], ['Paula Hawkins'])).toBe(false);
|
||||
});
|
||||
|
||||
it('ignores short tokens such as initials', () => {
|
||||
expect(shareSignificantToken(['J. R. Smith'], ['J. R. Jones'])).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false when there is nothing significant to compare', () => {
|
||||
expect(shareSignificantToken([], ['Tolkien'])).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('non-Latin scripts and diacritics', () => {
|
||||
it('matches identical titles written in a non-Latin script', () => {
|
||||
expect(scoreTitleMatch('こころ', 'こころ')).toBe(10);
|
||||
expect(scoreTitleMatch('战争与和平', '战争与和平')).toBe(10);
|
||||
expect(scoreTitleMatch('Война и мир', 'Война и мир')).toBe(10);
|
||||
expect(scoreTitleMatch('Ο Άρχοντας των Δαχτυλιδιών', 'Ο Άρχοντας των Δαχτυλιδιών')).toBe(10);
|
||||
});
|
||||
|
||||
it('keeps a non-Latin title scoreable rather than emptying it', () => {
|
||||
expect(normalizeTitleText('こころ')).toBe('こころ');
|
||||
expect(normalizeTitleText('夏目漱石')).toBe('夏目漱石');
|
||||
});
|
||||
|
||||
it('folds diacritics so transliteration differences still match', () => {
|
||||
expect(scoreTitleMatch('Sōseki', 'Soseki')).toBe(10);
|
||||
expect(scoreTitleMatch('Les Misérables', 'Les Miserables')).toBe(10);
|
||||
expect(scoreTitleMatch('Die Vermessung der Welt', 'Die Vermessung der Welt')).toBe(10);
|
||||
});
|
||||
|
||||
it('still separates different non-Latin titles', () => {
|
||||
expect(scoreTitleMatch('こころ', '吾輩は猫である')).toBe(0);
|
||||
});
|
||||
|
||||
it('matches an author name across a diacritic difference', () => {
|
||||
expect(shareSignificantToken(['Natsume Sōseki'], ['Natsume Soseki'])).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeTitleText and tokenizeTitleText', () => {
|
||||
it('lowercases, strips punctuation, and collapses whitespace', () => {
|
||||
expect(normalizeTitleText(' The Hobbit: Or, There & Back Again! ')).toBe('the hobbit or there back again');
|
||||
});
|
||||
|
||||
it('drops single-character tokens', () => {
|
||||
expect(tokenizeTitleText('a tale of 2 cities')).toEqual(['tale', 'of', 'cities']);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,159 @@
|
||||
import { distance } from 'fastest-levenshtein';
|
||||
|
||||
/**
|
||||
* Shared title-comparison primitives.
|
||||
*
|
||||
* Two callers with different thresholds use these: candidate relevance (is this candidate worth
|
||||
* keeping for the query?) and candidate agreement (do two providers describe the same book?).
|
||||
*/
|
||||
|
||||
export const TITLE_MATCH_SCORES = {
|
||||
exact: 10,
|
||||
prefix: 8,
|
||||
substring: 7,
|
||||
tokenOverlap: 6,
|
||||
levenshtein: 4,
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Function words carry no identifying signal, so counting them in token overlap makes any two
|
||||
* titles sharing "the" look related. They are dropped from the query side only: a stopword left in
|
||||
* a candidate can no longer match anything once the query side is stripped.
|
||||
*/
|
||||
const STOPWORDS = new Set([
|
||||
'the',
|
||||
'an',
|
||||
'and',
|
||||
'or',
|
||||
'of',
|
||||
'in',
|
||||
'on',
|
||||
'at',
|
||||
'to',
|
||||
'for',
|
||||
'with',
|
||||
'from',
|
||||
'by',
|
||||
'is',
|
||||
'it',
|
||||
'as',
|
||||
'be',
|
||||
'are',
|
||||
'was',
|
||||
'were',
|
||||
'this',
|
||||
'that',
|
||||
'these',
|
||||
'those',
|
||||
'into',
|
||||
'over',
|
||||
'under',
|
||||
'its',
|
||||
'his',
|
||||
'her',
|
||||
'their',
|
||||
'our',
|
||||
'your',
|
||||
'my',
|
||||
]);
|
||||
|
||||
const MIN_LEVENSHTEIN_SIMILARITY = 0.6;
|
||||
const MIN_TOKEN_OVERLAP_RATIO = 0.5;
|
||||
|
||||
/**
|
||||
* Folds a title to its comparable form: decomposed, stripped of nonspacing marks so that "Sōseki"
|
||||
* and "Soseki" agree, lowercased, and reduced to letters and digits of any script.
|
||||
*
|
||||
* Restricting this to `[a-z0-9]` would empty every title written in a non-Latin script, which makes
|
||||
* two providers reporting the identical Japanese or Russian title look like different books.
|
||||
* Spacing combining marks are kept, since scripts such as Devanagari carry vowels in them.
|
||||
*/
|
||||
export function normalizeTitleText(value: string): string {
|
||||
return value
|
||||
.normalize('NFKD')
|
||||
.replace(/\p{Mn}+/gu, '')
|
||||
.toLowerCase()
|
||||
.replace(/[^\p{L}\p{N}]+/gu, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
export function tokenizeTitleText(value: string): string[] {
|
||||
return value.split(' ').filter((token) => token.length > 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tokens worth matching on. Falls back to the raw tokens when a title is nothing but stopwords,
|
||||
* so a book genuinely titled "It" or "Them" still has something to match.
|
||||
*/
|
||||
export function significantTokens(tokens: string[]): string[] {
|
||||
const significant = tokens.filter((token) => !STOPWORDS.has(token));
|
||||
return significant.length > 0 ? significant : tokens;
|
||||
}
|
||||
|
||||
/**
|
||||
* Substring matching has to respect word boundaries. Without it "Ubik" matches "Rubik Cube
|
||||
* Solutions" and "It" matches "Italian Cooking Basics", both scoring as strongly as a real hit.
|
||||
*/
|
||||
export function containsWord(haystack: string, needle: string): boolean {
|
||||
if (needle.length === 0) return false;
|
||||
|
||||
for (let from = 0; from <= haystack.length - needle.length;) {
|
||||
const index = haystack.indexOf(needle, from);
|
||||
if (index === -1) return false;
|
||||
|
||||
const startsAtBoundary = index === 0 || haystack[index - 1] === ' ';
|
||||
const endIndex = index + needle.length;
|
||||
const endsAtBoundary = endIndex === haystack.length || haystack[endIndex] === ' ';
|
||||
if (startsAtBoundary && endsAtBoundary) return true;
|
||||
|
||||
from = index + 1;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export function startsWithWord(haystack: string, needle: string): boolean {
|
||||
if (needle.length === 0 || !haystack.startsWith(needle)) return false;
|
||||
return haystack.length === needle.length || haystack[needle.length] === ' ';
|
||||
}
|
||||
|
||||
/**
|
||||
* Scores how strongly two titles refer to the same work. Both arguments are raw titles; ordering
|
||||
* matters only in that `query` supplies the tokens the overlap ratio is measured against.
|
||||
*/
|
||||
export function scoreTitleMatch(query: string, candidate: string): number {
|
||||
const normalizedQuery = normalizeTitleText(query);
|
||||
const normalizedCandidate = normalizeTitleText(candidate);
|
||||
if (!normalizedQuery || !normalizedCandidate) return 0;
|
||||
|
||||
if (normalizedCandidate === normalizedQuery) return TITLE_MATCH_SCORES.exact;
|
||||
if (startsWithWord(normalizedCandidate, normalizedQuery) || startsWithWord(normalizedQuery, normalizedCandidate)) {
|
||||
return TITLE_MATCH_SCORES.prefix;
|
||||
}
|
||||
if (containsWord(normalizedCandidate, normalizedQuery) || containsWord(normalizedQuery, normalizedCandidate)) {
|
||||
return TITLE_MATCH_SCORES.substring;
|
||||
}
|
||||
|
||||
const queryTokens = significantTokens(tokenizeTitleText(normalizedQuery));
|
||||
const candidateTokens = new Set(tokenizeTitleText(normalizedCandidate));
|
||||
const overlapRatio = queryTokens.length > 0 ? queryTokens.filter((token) => candidateTokens.has(token)).length / queryTokens.length : 0;
|
||||
// A minority of the query's meaningful words in common is not evidence of the same book;
|
||||
// "The Way of Kings" and "The Way We Were" share exactly one.
|
||||
const tokenScore = overlapRatio > MIN_TOKEN_OVERLAP_RATIO ? overlapRatio * TITLE_MATCH_SCORES.tokenOverlap : 0;
|
||||
|
||||
const similarity = levenshteinSimilarity(normalizedQuery, normalizedCandidate);
|
||||
const levenshteinScore = similarity >= MIN_LEVENSHTEIN_SIMILARITY ? similarity * TITLE_MATCH_SCORES.levenshtein : 0;
|
||||
|
||||
return Math.max(tokenScore, levenshteinScore);
|
||||
}
|
||||
|
||||
export function shareSignificantToken(left: readonly string[], right: readonly string[]): boolean {
|
||||
const leftTokens = new Set(left.flatMap((value) => tokenizeTitleText(normalizeTitleText(value))).filter((token) => token.length > 2));
|
||||
if (leftTokens.size === 0) return false;
|
||||
return right.some((value) => tokenizeTitleText(normalizeTitleText(value)).some((token) => leftTokens.has(token)));
|
||||
}
|
||||
|
||||
function levenshteinSimilarity(a: string, b: string): number {
|
||||
const maxLength = Math.max(a.length, b.length);
|
||||
return maxLength === 0 ? 1 : 1 - distance(a, b) / maxLength;
|
||||
}
|
||||
@@ -68,9 +68,11 @@ export function buildMetadataLockCandidates(
|
||||
openLibrary: {
|
||||
provider: MetadataProviderKey.OPEN_LIBRARY,
|
||||
providerId: `ol-${slug}`,
|
||||
title: `${baseTitle} - open-library`,
|
||||
// Same book as the Goodreads candidate above, as a second provider would report it:
|
||||
// cross-provider agreement drops candidates that describe a different book.
|
||||
title: `${baseTitle} - refreshed (Open Library edition)`,
|
||||
subtitle: 'Open Library subtitle',
|
||||
authors: ['Open Library Author'],
|
||||
authors: ['Refreshed Author'],
|
||||
description: 'Open Library fallback description',
|
||||
publisher: 'Open Library Publisher',
|
||||
publishedYear: 2023,
|
||||
|
||||
Reference in New Issue
Block a user