Migrations/bookorbit
1
0
Fork 0
mirror of https://github.com/bookorbit/bookorbit.git synced 2026-11-03 00:11:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(metadata): stop comicvine searches timing out into no results

Browse Source 
Every comicvine search probed up to eight candidate volumes one request at a time, each holding a
one second rate limiter slot against an endpoint that answers in three to four seconds, so the
fifteen second provider budget ran out before the right volume was reached and the search reported
an empty result. Rank volumes on the name and issue count already carried by the volume search
response, then probe them in concurrent waves, so the runs that can hold the issue are tried first
and within budget.

Bind the work to a soft deadline that aborts requests still in flight. Reaching it now yields the
candidates already found, rather than letting a late detail lookup cross the hard timeout and take
everything with it. A probe or detail lookup that fails or is throttled costs its own result only,
never its siblings or the candidate it was enriching.

Take the series name from a query the user actually typed instead of the stored one, or searching a
comic for a different series silently returned the series it was already filed under. Strip the
zero padding comic filenames add to issue numbers in the provider rather than only in the search
controller, so the background refresh path benefits too.

Report a provider that timed out, was throttled, or failed as its own stream event, since all three
were indistinguishable from a genuinely empty result. Candidates keep the default event and an
unchanged payload, so a client has to read the event name to tell the two apart.

Fixes #1039
This commit is contained in:
neon
2026-08-18 23:04:25 +00:00
parent b25fc71b7f
commit 93e7c7b6b7
22 changed files with 1303 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/modules/metadata-fetch/metadata-fetch-pipeline.test.ts
+51 -47
View File
@@ -61,7 +61,7 @@ describe('MetadataFetchPipeline', () => {
beforeEach(() => {
fetchService = {
search: vi.fn(),
searchCandidates: vi.fn(),
} as unknown as Mocked<MetadataFetchService>;
preferencesService = {
@@ -118,11 +118,11 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(global);
resolver.withForwardCompatibility.mockReturnValue(global);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { title: 'Title' })));
fetchService.searchCandidates.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { title: 'Title' })));
await pipeline.run({ title: 'Query' }, {});
expect(fetchService.search).toHaveBeenCalledWith({ title: 'Query' }, [MetadataProviderKey.GOOGLE, MetadataProviderKey.OPEN_LIBRARY]);
expect(fetchService.searchCandidates).toHaveBeenCalledWith({ title: 'Query' }, [MetadataProviderKey.GOOGLE, MetadataProviderKey.OPEN_LIBRARY]);
});
describe('series expected counts', () => {
@@ -136,7 +136,7 @@ describe('MetadataFetchPipeline', () => {
it('forwards every candidate, not just the one that wins field resolution', async () => {
primePreferences();
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', { seriesName: 'Dune', seriesTotalBooks: 6 }),
candidate(MetadataProviderKey.OPEN_LIBRARY, 'o1', { seriesName: 'Dune', seriesTotalBooks: 8 }),
@@ -155,7 +155,7 @@ describe('MetadataFetchPipeline', () => {
it('still resolves fields when recording the totals fails', async () => {
primePreferences();
seriesExpectedCount.recordFromCandidates.mockRejectedValueOnce(new Error('unreachable'));
fetchService.search.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { title: 'Fetched Title' })));
fetchService.searchCandidates.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { title: 'Fetched Title' })));
const resolved = await pipeline.run({ title: 'Query' }, {});
@@ -188,11 +188,11 @@ describe('MetadataFetchPipeline', () => {
{ key: MetadataProviderKey.OPEN_LIBRARY },
{ key: MetadataProviderKey.KOBO },
] as never);
fetchService.search.mockReturnValue(of(candidate(MetadataProviderKey.KOBO, 'k1', { title: 'Kobo Title' })));
fetchService.searchCandidates.mockReturnValue(of(candidate(MetadataProviderKey.KOBO, 'k1', { title: 'Kobo Title' })));
await pipeline.run({ title: 'Query' }, {});
expect(fetchService.search).toHaveBeenCalledWith({ title: 'Query' }, [MetadataProviderKey.KOBO]);
expect(fetchService.searchCandidates).toHaveBeenCalledWith({ title: 'Query' }, [MetadataProviderKey.KOBO]);
});
it('lets Field Rules run an audiobook provider without switching the search to audiobook editions', async () => {
@@ -213,7 +213,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(global);
resolver.withForwardCompatibility.mockReturnValue(global);
registry.all.mockReturnValue([{ key: MetadataProviderKey.LIBROFM }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.LIBROFM, '9798217174331', {
title: 'Yesteryear: A GMA Book Club Pick',
@@ -232,7 +232,7 @@ describe('MetadataFetchPipeline', () => {
{},
);
expect(fetchService.search).toHaveBeenCalledWith(
expect(fetchService.searchCandidates).toHaveBeenCalledWith(
{
title: 'Yesteryear',
author: 'Caro Claire Burke',
@@ -265,7 +265,7 @@ describe('MetadataFetchPipeline', () => {
const { resolved, diagnostics } = await pipeline.runWithSources({ title: 'Query' }, {});
expect(resolved).toEqual({});
expect(fetchService.search).not.toHaveBeenCalled();
expect(fetchService.searchCandidates).not.toHaveBeenCalled();
expect(diagnostics).toMatchObject({
reason: 'no_active_providers',
activeProviders: [],
@@ -296,7 +296,7 @@ describe('MetadataFetchPipeline', () => {
const { diagnostics } = await pipeline.runWithSources({ title: 'Query' }, {});
expect(fetchService.search).not.toHaveBeenCalled();
expect(fetchService.searchCandidates).not.toHaveBeenCalled();
expect(diagnostics.reason).toBe('providers_throttled');
expect(diagnostics.throttledProviders).toEqual([MetadataProviderKey.GOOGLE]);
});
@@ -314,7 +314,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(global);
resolver.withForwardCompatibility.mockReturnValue(global);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }] as never);
fetchService.search.mockReturnValue(of() as never);
fetchService.searchCandidates.mockReturnValue(of() as never);
const { diagnostics } = await pipeline.runWithSources({ title: 'Query' }, {});
@@ -336,7 +336,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(global);
resolver.withForwardCompatibility.mockReturnValue(global);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }] as never);
fetchService.search.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { title: 'Fetched Title' })));
fetchService.searchCandidates.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { title: 'Fetched Title' })));
const { resolved, diagnostics } = await pipeline.runWithSources({ title: 'Query' }, { title: 'Existing Title' });
@@ -364,7 +364,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', {
title: 'Fetched Title',
@@ -428,7 +428,7 @@ describe('MetadataFetchPipeline', () => {
it('never assembles one record out of candidates describing different books', async () => {
primeAgreementPreferences();
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOODREADS, 'gr1', {
title: 'The Girl on the Train',
@@ -465,7 +465,7 @@ describe('MetadataFetchPipeline', () => {
it('reports no candidates when every provider matched a different book', async () => {
primeAgreementPreferences();
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOODREADS, 'gr1', { title: 'The Girl on the Train', authors: ['Paula Hawkins'] }),
candidate(MetadataProviderKey.AMAZON, 'az1', { title: 'The Silence of the Lambs', authors: ['Thomas Harris'] }),
@@ -482,7 +482,7 @@ describe('MetadataFetchPipeline', () => {
it('still fills a field from a lower-priority provider that agrees on the book', async () => {
primeAgreementPreferences();
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOODREADS, 'gr1', { title: 'The Hobbit', authors: ['J.R.R. Tolkien'] }),
candidate(MetadataProviderKey.AMAZON, 'az1', {
@@ -515,7 +515,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.GOODREADS }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'gg1', { title: 'The Hobbit', authors: ['J.R.R. Tolkien'] }),
candidate(MetadataProviderKey.GOODREADS, 'gr1', { title: 'The Girl on the Train', authors: ['Paula Hawkins'] }),
@@ -530,7 +530,7 @@ describe('MetadataFetchPipeline', () => {
it('leaves a single provider untouched, having nothing to disagree with', async () => {
primeAgreementPreferences();
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(candidate(MetadataProviderKey.GOODREADS, 'gr1', { title: 'A Loosely Matching Title', authors: ['Someone'] })),
);
@@ -553,7 +553,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', { description: undefined }),
candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { description: 'OpenLibrary Description' }),
@@ -579,7 +579,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', { authors: [] }),
candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { authors: ['Fallback Author'] }),
@@ -605,7 +605,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(candidate(MetadataProviderKey.GOOGLE, 'g1', { authors: [] }), candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { authors: [] })),
);
@@ -628,7 +628,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', { authors: [] }),
candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { authors: ['Fallback Author'] }),
@@ -654,7 +654,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', { description: ' ' }),
candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { description: 'Fallback Description' }),
@@ -680,7 +680,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }] as never);
fetchService.search.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { authors: [] })));
fetchService.searchCandidates.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { authors: [] })));
const { resolved, sources } = await pipeline.runWithSources({ title: 'Query' }, { authors: ['Existing Author'] });
@@ -706,7 +706,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }] as never);
fetchService.search.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { durationSeconds: 0, abridged: false })));
fetchService.searchCandidates.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { durationSeconds: 0, abridged: false })));
const { resolved } = await pipeline.runWithSources({ title: 'Query' }, { duration: 60, abridged: true });
@@ -734,7 +734,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', { communityRating: 4.1 }),
candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { communityRating: 4.2, communityRatingCount: 999 }),
@@ -765,7 +765,9 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(of(candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { coverUrl: 'https://img.example/cover.jpg' })));
fetchService.searchCandidates.mockReturnValue(
of(candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { coverUrl: 'https://img.example/cover.jpg' })),
);
const { resolved, sources } = await pipeline.runWithSources({ title: 'Query' }, {});
@@ -786,7 +788,9 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(of(candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { coverUrl: 'https://img.example/cover.jpg' })));
fetchService.searchCandidates.mockReturnValue(
of(candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { coverUrl: 'https://img.example/cover.jpg' })),
);
const { resolved, sources } = await pipeline.runWithSources({ title: 'Query' }, { cover: 'extracted' });
@@ -827,7 +831,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', {
title: 'Provider Title',
@@ -877,7 +881,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.COMICVINE }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.COMICVINE, 'cv1', {
comicMetadata: {
@@ -923,7 +927,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.COMICVINE }, { key: MetadataProviderKey.AMAZON }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.COMICVINE, 'cv1', {
comicMetadata: {
@@ -961,7 +965,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', { genres: ['Sci-Fi', 'Space Opera'] }),
candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { genres: ['Sci-Fi', 'Classic'] }),
@@ -991,7 +995,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', { genres: ['Sci-Fi', 'Audiobook', 'Adult'] }),
candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { genres: ['audiobook', 'Fantasy'] }),
@@ -1021,7 +1025,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', { genres: ['Fiction', 'Fantasy', 'fantasy', 'Adult'] }),
candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { genres: ['fiction', 'Mystery', 'Classic'] }),
@@ -1051,7 +1055,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', { genres: ['audiobook'] }),
candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { genres: ['Fantasy', 'Adventure'] }),
@@ -1081,7 +1085,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }, { key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g1', { genres: ['Fantasy', 'Adventure', 'Epic'] }),
candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { genres: ['Classic'] }),
@@ -1111,7 +1115,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }] as never);
fetchService.search.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { title: 'Fetched Title' })));
fetchService.searchCandidates.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { title: 'Fetched Title' })));
const { resolved, providerIds } = await pipeline.runWithSources({ title: 'Query' }, { title: 'Existing Title' });
@@ -1136,7 +1140,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.AUDNEXUS }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.AUDNEXUS, 'B0TEST12345', {
audibleId: 'B0TEST12345',
@@ -1167,7 +1171,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.HARDCOVER }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(candidate(MetadataProviderKey.HARDCOVER, 'the-name-of-the-wind', { title: 'Fetched Title', hardcoverEditionId: '1001' })),
);
@@ -1194,7 +1198,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }] as never);
fetchService.search.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { title: 'Fetched Title' })));
fetchService.searchCandidates.mockReturnValue(of(candidate(MetadataProviderKey.GOOGLE, 'g1', { title: 'Fetched Title' })));
const { providerIds } = await pipeline.runWithSources({ title: 'Query' }, {});
@@ -1218,7 +1222,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.HARDCOVER }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(candidate(MetadataProviderKey.HARDCOVER, 'the-name-of-the-wind', { title: 'Fetched Title', hardcoverEditionId: '1001' })),
);
@@ -1241,7 +1245,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.GOOGLE }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.GOOGLE, 'g-first', { description: 'First description' }),
candidate(MetadataProviderKey.GOOGLE, 'g-second', { description: 'Second description' }),
@@ -1271,7 +1275,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.AUDIBLE }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.AUDIBLE, 'B002V1NSN2', {
seriesName: 'Sword of Truth',
@@ -1313,7 +1317,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.AUDIBLE }, { key: MetadataProviderKey.GOOGLE }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.AUDIBLE, 'B002V1NSN2', {
seriesName: 'Sword of Truth',
@@ -1352,7 +1356,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(prefs);
resolver.withForwardCompatibility.mockReturnValue(prefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.AUDIBLE }] as never);
fetchService.search.mockReturnValue(
fetchService.searchCandidates.mockReturnValue(
of(
candidate(MetadataProviderKey.AUDIBLE, 'B002V1NSN2', {
seriesName: 'Sword of Truth',
@@ -1391,7 +1395,7 @@ describe('MetadataFetchPipeline', () => {
resolver.resolve.mockReturnValue(resolvedPrefs);
resolver.withForwardCompatibility.mockReturnValue(resolvedPrefs);
registry.all.mockReturnValue([{ key: MetadataProviderKey.OPEN_LIBRARY }] as never);
fetchService.search.mockReturnValue(of(candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { title: 'Library Title' })));
fetchService.searchCandidates.mockReturnValue(of(candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', { title: 'Library Title' })));
const result = await pipeline.run({ title: 'Query' }, {}, 10);
server/src/modules/metadata-fetch/metadata-fetch-pipeline.ts
+1 -1
View File
@@ -132,7 +132,7 @@ export class MetadataFetchPipeline {
? { ...params, includeAudiobookProviders: true }
: params;
const candidates = providerSelection.activeProviders.length
? await firstValueFrom(this.fetchService.search(searchParams, providerSelection.activeProviders).pipe(toArray()), {
? await firstValueFrom(this.fetchService.searchCandidates(searchParams, providerSelection.activeProviders).pipe(toArray()), {
defaultValue: [] as MetadataCandidate[],
})
: [];
server/src/modules/metadata-fetch/metadata-fetch.controller.test.ts
+77 -12
View File
@@ -1,4 +1,10 @@
import { MetadataProviderKey, ProviderConfigurations, ProviderThrottleRuntimeSnapshot } from '@bookorbit/types';
import {
METADATA_PROVIDER_STATUS_EVENT,
MetadataCandidate,
MetadataProviderKey,
ProviderConfigurations,
ProviderThrottleRuntimeSnapshot,
} from '@bookorbit/types';
import type { Mocked } from 'vitest';
import { firstValueFrom, of, toArray } from 'rxjs';
@@ -7,13 +13,17 @@ import { LookupMetadataDto } from './dto/lookup-metadata.dto';
import { MetadataSearchDto } from './dto/metadata-search.dto';
import { MetadataFetchController } from './metadata-fetch.controller';
import { MetadataFetchPipeline } from './metadata-fetch-pipeline';
import { MetadataFetchService } from './metadata-fetch.service';
import { MetadataFetchService, MetadataSearchEvent } from './metadata-fetch.service';
import { ProviderRegistry } from './provider-registry';
import { ProviderConfigService } from '../metadata-preferences/provider-config.service';
import { MetadataPreferencesService } from '../metadata-preferences/metadata-preferences.service';
import { MetadataPreferenceResolver } from '../metadata-preferences/metadata-preference-resolver';
import { ProviderThrottleTracker } from './provider-throttle.tracker';
function candidateEvent(candidate: MetadataCandidate): MetadataSearchEvent {
return { kind: 'candidate', candidate };
}
describe('MetadataFetchController', () => {
let service: Mocked<MetadataFetchService>;
let pipeline: Mocked<MetadataFetchPipeline>;
@@ -119,8 +129,8 @@ describe('MetadataFetchController', () => {
pipeline.getEffectiveProviderKeys.mockResolvedValue([MetadataProviderKey.GOOGLE, MetadataProviderKey.OPEN_LIBRARY]);
service.search.mockReturnValue(
of(
{ provider: MetadataProviderKey.GOOGLE, providerId: 'vol-1', title: 'First' },
{ provider: MetadataProviderKey.OPEN_LIBRARY, providerId: 'ol-1', title: 'Second' },
candidateEvent({ provider: MetadataProviderKey.GOOGLE, providerId: 'vol-1', title: 'First' }),
candidateEvent({ provider: MetadataProviderKey.OPEN_LIBRARY, providerId: 'ol-1', title: 'Second' }),
),
);
@@ -143,6 +153,7 @@ describe('MetadataFetchController', () => {
author: 'Frank Herbert',
isbn: '9780441172719',
existingProviderIds: { [MetadataProviderKey.GOOGLE]: 'vol-1' },
titleIsExplicitQuery: true,
isAudiobook: false,
includeAudiobookProviders: false,
},
@@ -157,7 +168,7 @@ describe('MetadataFetchController', () => {
it('allows explicit book searches to use enabled providers outside field rules', async () => {
service.getStoredProviderContext.mockResolvedValue({ libraryId: 5, providerIds: {} });
pipeline.getEffectiveProviderKeys.mockResolvedValue([MetadataProviderKey.GOOGLE]);
service.search.mockReturnValue(of({ provider: MetadataProviderKey.KOBO, providerId: 'kobo-1', title: 'Kobo Result' }));
service.search.mockReturnValue(of(candidateEvent({ provider: MetadataProviderKey.KOBO, providerId: 'kobo-1', title: 'Kobo Result' })));
const stream = await controller.stream({ bookId: 12, title: 'Dune', providers: [MetadataProviderKey.KOBO] }, user);
await firstValueFrom(stream.pipe(toArray()));
@@ -173,12 +184,14 @@ describe('MetadataFetchController', () => {
preferences.options!.genres.maxCount = 2;
metadataPreferences.getGlobal.mockResolvedValue(preferences);
service.search.mockReturnValue(
of({
provider: MetadataProviderKey.GOOGLE,
providerId: 'vol-1',
title: 'First',
genres: ['Science Fiction', 'science fiction', 'audiobook', 'Space Opera', 'Fantasy'],
}),
of(
candidateEvent({
provider: MetadataProviderKey.GOOGLE,
providerId: 'vol-1',
title: 'First',
genres: ['Science Fiction', 'science fiction', 'audiobook', 'Space Opera', 'Fantasy'],
}),
),
);
const stream = await controller.stream({ title: 'Dune' }, user);
@@ -190,7 +203,7 @@ describe('MetadataFetchController', () => {
});
it('skips stored provider lookup when bookId is not provided', async () => {
service.search.mockReturnValue(of({ provider: MetadataProviderKey.GOOGLE, providerId: 'vol-2', title: 'Only' }));
service.search.mockReturnValue(of(candidateEvent({ provider: MetadataProviderKey.GOOGLE, providerId: 'vol-2', title: 'Only' })));
const dto: MetadataSearchDto = { title: 'Dune' };
const stream = await controller.stream(dto, user);
@@ -203,6 +216,7 @@ describe('MetadataFetchController', () => {
author: undefined,
isbn: undefined,
existingProviderIds: {},
titleIsExplicitQuery: true,
isAudiobook: false,
includeAudiobookProviders: true,
},
@@ -216,6 +230,57 @@ describe('MetadataFetchController', () => {
);
});
it('sends a provider status as its own SSE event so a timeout is not read as an empty result', async () => {
service.getStoredProviderContext.mockResolvedValue({ libraryId: 5, title: 'Dune', seriesName: null, seriesIndex: null, providerIds: {} });
pipeline.getEffectiveProviderKeys.mockResolvedValue([MetadataProviderKey.COMICVINE]);
service.search.mockReturnValue(
of<MetadataSearchEvent>(candidateEvent({ provider: MetadataProviderKey.COMICVINE, providerId: 'cv-1', title: 'Found' }), {
kind: 'status',
status: { provider: MetadataProviderKey.COMICVINE, outcome: 'timeout' },
}),
);
const stream = await controller.stream({ bookId: 12, title: 'Dune' }, user);
const events = await firstValueFrom(stream.pipe(toArray()));
expect(events).toEqual([
{ data: { provider: MetadataProviderKey.COMICVINE, providerId: 'cv-1', title: 'Found' } },
{ type: METADATA_PROVIDER_STATUS_EVENT, data: { provider: MetadataProviderKey.COMICVINE, outcome: 'timeout' } },
]);
});
it('does not treat the prefilled book title as a query the user typed', async () => {
service.getStoredProviderContext.mockResolvedValue({
libraryId: 5,
title: 'The Amazing Spider-Man (2022) Volume 06 Issue 067',
seriesName: 'Amazing Spider-Man',
seriesIndex: 67,
providerIds: {},
});
pipeline.getEffectiveProviderKeys.mockResolvedValue([MetadataProviderKey.COMICVINE]);
service.search.mockReturnValue(of());
await firstValueFrom((await controller.stream({ bookId: 12, title: 'The Amazing Spider-Man (2022) Volume 06 Issue 067' }, user)).pipe(toArray()));
expect(service.search).toHaveBeenCalledWith(expect.objectContaining({ titleIsExplicitQuery: false }), expect.anything());
});
it('marks an edited search title as a query the user typed', async () => {
service.getStoredProviderContext.mockResolvedValue({
libraryId: 5,
title: 'The Amazing Spider-Man (2022) Volume 06 Issue 067',
seriesName: 'Amazing Spider-Man',
seriesIndex: 67,
providerIds: {},
});
pipeline.getEffectiveProviderKeys.mockResolvedValue([MetadataProviderKey.COMICVINE]);
service.search.mockReturnValue(of());
await firstValueFrom((await controller.stream({ bookId: 12, title: 'Daredevil' }, user)).pipe(toArray()));
expect(service.search).toHaveBeenCalledWith(expect.objectContaining({ titleIsExplicitQuery: true }), expect.anything());
});
it('uses enabled provider config when stream providers are omitted', async () => {
providerConfig.getConfig.mockResolvedValue(
makeProviderConfig({
server/src/modules/metadata-fetch/metadata-fetch.controller.ts
+27 -4
View File
@@ -1,5 +1,12 @@
import { Controller, Get, MessageEvent, Query, Sse } from '@nestjs/common';
import { MetadataCandidate, MetadataProviderInfo, MetadataProviderKey, Permission, ProviderThrottleRuntimeSnapshot } from '@bookorbit/types';
import {
METADATA_PROVIDER_STATUS_EVENT,
MetadataCandidate,
MetadataProviderInfo,
MetadataProviderKey,
Permission,
ProviderThrottleRuntimeSnapshot,
} from '@bookorbit/types';
import { map, Observable } from 'rxjs';
import { CurrentUser } from '../../common/decorators/current-user.decorator';
@@ -7,7 +14,7 @@ import type { RequestUser } from '../../common/types/request-user';
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
import { LookupMetadataDto } from './dto/lookup-metadata.dto';
import { MetadataSearchDto } from './dto/metadata-search.dto';
import { MetadataFetchService } from './metadata-fetch.service';
import { MetadataFetchService, MetadataSearchEvent } from './metadata-fetch.service';
import { MetadataFetchPipeline } from './metadata-fetch-pipeline';
import { ProviderRegistry } from './provider-registry';
import { MetadataSearchParams } from './providers/metadata-search-params';
@@ -23,6 +30,16 @@ function normalizeSearchTitle(title: string | undefined): string | undefined {
return title.trim().replace(/#0*(\d+)/g, '#$1');
}
/**
* The search box is prefilled with the book's own title, so an unchanged one carries no intent and
* stored series context should still lead. Anything else is the user asking for something specific.
*/
function isExplicitQuery(searchTitle: string | undefined, storedTitle: string | null | undefined): boolean {
if (!searchTitle?.trim()) return false;
const stored = normalizeSearchTitle(storedTitle ?? undefined);
return searchTitle.trim().toLowerCase() !== (stored ?? '').trim().toLowerCase();
}
function isAudiobookProvider(providerKey: MetadataProviderKey): boolean {
return providerKey === MetadataProviderKey.AUDIBLE || providerKey === MetadataProviderKey.AUDNEXUS || providerKey === MetadataProviderKey.LIBROFM;
}
@@ -79,12 +96,14 @@ export class MetadataFetchController {
Boolean(existingProviderIds[MetadataProviderKey.AUDIBLE] || existingProviderIds[MetadataProviderKey.LIBROFM]);
const isAudiobook = dto.isAudiobook ?? inferredIsAudiobook;
const searchTitle = normalizeSearchTitle(dto.title);
const params: MetadataSearchParams = {
title: normalizeSearchTitle(dto.title),
title: searchTitle,
author: dto.author,
isbn: dto.isbn,
seriesName: storedContext?.seriesName ?? undefined,
seriesIndex: storedContext?.seriesIndex ?? undefined,
titleIsExplicitQuery: isExplicitQuery(searchTitle, storedContext?.title),
existingProviderIds,
isAudiobook,
includeAudiobookProviders: isAudiobook || providerKeys.some(isAudiobookProvider),
@@ -96,7 +115,11 @@ export class MetadataFetchController {
return this.metadataFetchService
.search(params, providerKeys)
.pipe(
map((candidate: MetadataCandidate) => ({ data: applyGenreFetchOptionsToCandidate(candidate, blockedGenreTokens, genreOptions?.maxCount) })),
map((event: MetadataSearchEvent) =>
event.kind === 'candidate'
? { data: applyGenreFetchOptionsToCandidate(event.candidate, blockedGenreTokens, genreOptions?.maxCount) }
: { type: METADATA_PROVIDER_STATUS_EVENT, data: event.status },
),
);
}
server/src/modules/metadata-fetch/metadata-fetch.repository.ts
+2
View File
@@ -10,6 +10,7 @@ type Db = NodePgDatabase<typeof schema>;
export interface StoredProviderIdsRow {
libraryId: number;
title: string | null;
seriesName: string | null;
seriesIndex: number | null;
googleBooksId: string | null;
@@ -35,6 +36,7 @@ export class MetadataFetchRepository {
const [row] = await this.db
.select({
libraryId: books.libraryId,
title: bookMetadata.title,
seriesName: bookMetadata.seriesName,
seriesIndex: bookMetadata.seriesIndex,
googleBooksId: bookMetadata.googleBooksId,
server/src/modules/metadata-fetch/metadata-fetch.service.test.ts
+121 -15
View File
@@ -1,12 +1,13 @@
import { ForbiddenException, Logger, NotFoundException } from '@nestjs/common';
import { MetadataCandidate, MetadataProviderKey } from '@bookorbit/types';
import type { Mocked } from 'vitest';
import { firstValueFrom, toArray } from 'rxjs';
import { filter, firstValueFrom, map, pipe, toArray } from 'rxjs';
import type { RequestUser } from '../../common/types/request-user';
import { MetadataFetchRepository } from './metadata-fetch.repository';
import { MetadataFetchService } from './metadata-fetch.service';
import { MetadataFetchService, MetadataSearchEvent } from './metadata-fetch.service';
import { ProviderRegistry } from './provider-registry';
import { ProviderThrottleError } from './provider-throttle.error';
import { ProviderThrottleTracker } from './provider-throttle.tracker';
import { IdentifiableProvider, MetadataProvider } from './providers/metadata-provider';
import { EMPTY_CONTENT_FILTER_RULES } from '@bookorbit/types';
@@ -43,6 +44,23 @@ function deferred(): { promise: Promise<void>; resolve: () => void } {
return { promise, resolve };
}
type CandidateEvent = Extract<MetadataSearchEvent, { kind: 'candidate' }>;
function isCandidateEvent(event: MetadataSearchEvent): event is CandidateEvent {
return event.kind === 'candidate';
}
function candidatesOnly() {
return pipe(
filter(isCandidateEvent),
map((event: CandidateEvent) => event.candidate),
);
}
function statusesOf(events: MetadataSearchEvent[]) {
return events.filter((event) => event.kind === 'status').map((event) => event.status);
}
describe('MetadataFetchService', () => {
let registry: Mocked<ProviderRegistry>;
let throttleTracker: Mocked<ProviderThrottleTracker>;
@@ -95,7 +113,7 @@ describe('MetadataFetchService', () => {
};
registry.select.mockReturnValue([google, openLibrary]);
const results = await firstValueFrom(service.search({ title: 'Dune' }).pipe(toArray()));
const results = await firstValueFrom(service.search({ title: 'Dune' }).pipe(candidatesOnly(), toArray()));
expect(results).toHaveLength(3);
expect(results).toEqual(
@@ -140,7 +158,7 @@ describe('MetadataFetchService', () => {
const openLibrary = makeBlockedProvider(MetadataProviderKey.OPEN_LIBRARY);
registry.select.mockReturnValue([google, openLibrary]);
const search = firstValueFrom(service.search({ title: 'Dune' }).pipe(toArray()));
const search = firstValueFrom(service.search({ title: 'Dune' }).pipe(candidatesOnly(), toArray()));
await Promise.all([google.started, openLibrary.started]);
@@ -167,7 +185,9 @@ describe('MetadataFetchService', () => {
};
registry.select.mockReturnValue([google]);
const results = await firstValueFrom(service.search({ title: 'Dune', author: 'Frank Herbert', isbn: '9780441013593' }).pipe(toArray()));
const results = await firstValueFrom(
service.search({ title: 'Dune', author: 'Frank Herbert', isbn: '9780441013593' }).pipe(candidatesOnly(), toArray()),
);
expect(results).toEqual([candidate(MetadataProviderKey.GOOGLE, 'g-fallback', 'Dune')]);
expect(google.search).toHaveBeenCalledTimes(2);
@@ -198,7 +218,7 @@ describe('MetadataFetchService', () => {
};
registry.select.mockReturnValue([google]);
const results = await firstValueFrom(service.search({ title: 'Dune', isbn: '9780441013593' }).pipe(toArray()));
const results = await firstValueFrom(service.search({ title: 'Dune', isbn: '9780441013593' }).pipe(candidatesOnly(), toArray()));
expect(results).toEqual([candidate(MetadataProviderKey.GOOGLE, 'g-isbn', 'Dune')]);
expect(google.search).toHaveBeenCalledTimes(1);
@@ -214,7 +234,7 @@ describe('MetadataFetchService', () => {
};
registry.select.mockReturnValue([google]);
const results = await firstValueFrom(service.search({ isbn: '9780441013593' }).pipe(toArray()));
const results = await firstValueFrom(service.search({ isbn: '9780441013593' }).pipe(candidatesOnly(), toArray()));
expect(results).toEqual([]);
expect(google.search).toHaveBeenCalledTimes(1);
@@ -235,7 +255,7 @@ describe('MetadataFetchService', () => {
registry.select.mockReturnValue([audible]);
const results = await firstValueFrom(
service.search({ title: 'Confessor', author: 'Terry Goodkin', isbn: '9781662539374', isAudiobook: true }).pipe(toArray()),
service.search({ title: 'Confessor', author: 'Terry Goodkin', isbn: '9781662539374', isAudiobook: true }).pipe(candidatesOnly(), toArray()),
);
expect(results).toEqual([candidate(MetadataProviderKey.AUDIBLE, 'B002V1NSN2', 'Confessor')]);
@@ -266,7 +286,7 @@ describe('MetadataFetchService', () => {
isAudiobook: true,
maxCandidatesPerProvider: 1,
})
.pipe(toArray()),
.pipe(candidatesOnly(), toArray()),
);
expect(results).toEqual([]);
@@ -284,7 +304,7 @@ describe('MetadataFetchService', () => {
registry.select.mockReturnValue([google]);
const results = await firstValueFrom(
service.search({ title: 'Dune', existingProviderIds: { [MetadataProviderKey.GOOGLE]: 'stored-id' } }).pipe(toArray()),
service.search({ title: 'Dune', existingProviderIds: { [MetadataProviderKey.GOOGLE]: 'stored-id' } }).pipe(candidatesOnly(), toArray()),
);
expect(results).toEqual([candidate(MetadataProviderKey.GOOGLE, 'stored-id', 'Dune')]);
@@ -310,7 +330,7 @@ describe('MetadataFetchService', () => {
registry.select.mockReturnValue([google]);
const results = await firstValueFrom(
service.search({ title: 'Dune', existingProviderIds: { [MetadataProviderKey.GOOGLE]: 'missing' } }).pipe(toArray()),
service.search({ title: 'Dune', existingProviderIds: { [MetadataProviderKey.GOOGLE]: 'missing' } }).pipe(candidatesOnly(), toArray()),
);
expect(results).toEqual([candidate(MetadataProviderKey.GOOGLE, 'search-id', 'Dune')]);
@@ -344,7 +364,7 @@ describe('MetadataFetchService', () => {
isbn: '9789523331587',
existingProviderIds: { [MetadataProviderKey.HARDCOVER]: 'comet-in-moominland' },
})
.pipe(toArray()),
.pipe(candidatesOnly(), toArray()),
);
expect(results).toEqual([candidate(MetadataProviderKey.HARDCOVER, 'comet-in-moominland', 'Kometen kommer')]);
@@ -384,7 +404,7 @@ describe('MetadataFetchService', () => {
author: 'Frank Herbert',
existingProviderIds: { [MetadataProviderKey.GOOGLE]: 'stored-id' },
})
.pipe(toArray()),
.pipe(candidatesOnly(), toArray()),
);
expect(results).toEqual([candidate(MetadataProviderKey.GOOGLE, 'search-id', 'Dune')]);
@@ -416,7 +436,7 @@ describe('MetadataFetchService', () => {
};
registry.select.mockReturnValue([failing, healthy]);
const results = await firstValueFrom(service.search({ title: 'Dune' }).pipe(toArray()));
const results = await firstValueFrom(service.search({ title: 'Dune' }).pipe(candidatesOnly(), toArray()));
expect(results).toEqual([candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', 'Dune')]);
});
@@ -432,7 +452,7 @@ describe('MetadataFetchService', () => {
};
registry.select.mockReturnValue([stalled]);
const searchPromise = firstValueFrom(service.search({ title: 'Dune' }).pipe(toArray()));
const searchPromise = firstValueFrom(service.search({ title: 'Dune' }).pipe(candidatesOnly(), toArray()));
let settled = false;
void searchPromise.then(() => {
settled = true;
@@ -445,6 +465,92 @@ describe('MetadataFetchService', () => {
await expect(searchPromise).resolves.toEqual([]);
});
it('reports a stalled provider as a timeout rather than letting it read as empty', async () => {
vi.useFakeTimers();
const stalled: MetadataProvider = {
key: MetadataProviderKey.COMICVINE,
label: 'ComicVine',
identifiable: false,
search: vi.fn().mockImplementation(() => new Promise<MetadataCandidate[]>(() => undefined)),
};
registry.select.mockReturnValue([stalled]);
const events = firstValueFrom(service.search({ title: 'Dune' }).pipe(toArray()));
await vi.advanceTimersByTimeAsync(15_000);
expect(statusesOf(await events)).toEqual([{ provider: MetadataProviderKey.COMICVINE, outcome: 'timeout' }]);
});
it('reports a throttled provider as throttled', async () => {
const throttled: MetadataProvider = {
key: MetadataProviderKey.COMICVINE,
label: 'ComicVine',
identifiable: false,
search: vi.fn().mockRejectedValue(new ProviderThrottleError(30)),
};
registry.select.mockReturnValue([throttled]);
const events = await firstValueFrom(service.search({ title: 'Dune' }).pipe(toArray()));
expect(statusesOf(events)).toEqual([{ provider: MetadataProviderKey.COMICVINE, outcome: 'throttled' }]);
});
it('reports a provider that errored, alongside the candidates the others found', async () => {
const failing: MetadataProvider = {
key: MetadataProviderKey.GOODREADS,
label: 'Goodreads',
identifiable: false,
search: vi.fn().mockRejectedValue(new Error('bad upstream response')),
};
const healthy: MetadataProvider = {
key: MetadataProviderKey.OPEN_LIBRARY,
label: 'OpenLibrary',
identifiable: false,
search: vi.fn().mockResolvedValue([candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', 'Dune')]),
};
registry.select.mockReturnValue([failing, healthy]);
const events = await firstValueFrom(service.search({ title: 'Dune' }).pipe(toArray()));
expect(statusesOf(events)).toEqual([{ provider: MetadataProviderKey.GOODREADS, outcome: 'failed' }]);
expect(events.filter((event) => event.kind === 'candidate')).toHaveLength(1);
});
it('stays silent about providers that finish, including those that simply found nothing', async () => {
const empty: MetadataProvider = {
key: MetadataProviderKey.OPEN_LIBRARY,
label: 'OpenLibrary',
identifiable: false,
search: vi.fn().mockResolvedValue([]),
};
registry.select.mockReturnValue([empty]);
const events = await firstValueFrom(service.search({ title: 'Dune' }).pipe(toArray()));
expect(statusesOf(events)).toEqual([]);
});
it('keeps provider status out of the candidate-only view the automatic pipeline consumes', async () => {
const failing: MetadataProvider = {
key: MetadataProviderKey.COMICVINE,
label: 'ComicVine',
identifiable: false,
search: vi.fn().mockRejectedValue(new ProviderThrottleError(30)),
};
const healthy: MetadataProvider = {
key: MetadataProviderKey.OPEN_LIBRARY,
label: 'OpenLibrary',
identifiable: false,
search: vi.fn().mockResolvedValue([candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', 'Dune')]),
};
registry.select.mockReturnValue([failing, healthy]);
const candidates = await firstValueFrom(service.searchCandidates({ title: 'Dune' }).pipe(toArray()));
expect(candidates).toEqual([candidate(MetadataProviderKey.OPEN_LIBRARY, 'ol1', 'Dune')]);
});
it('looks up by provider id only for identifiable providers', async () => {
const nonIdentifiable: MetadataProvider = {
key: MetadataProviderKey.AMAZON,
server/src/modules/metadata-fetch/metadata-fetch.service.ts
+38 -9
View File
@@ -1,6 +1,6 @@
import { ForbiddenException, Injectable, Logger, NotFoundException } from '@nestjs/common';
import { MetadataCandidate, MetadataProviderKey } from '@bookorbit/types';
import { from, merge, Observable, switchMap } from 'rxjs';
import { MetadataCandidate, MetadataProviderKey, MetadataProviderSearchOutcome, MetadataProviderSearchStatus } from '@bookorbit/types';
import { filter, from, map, merge, Observable, switchMap } from 'rxjs';
import type { RequestUser } from '../../common/types/request-user';
import { filterAndRank } from './candidate-relevance';
@@ -18,8 +18,22 @@ interface TimedProviderResult {
timedOut: boolean;
}
interface ProviderSearchResult {
candidates: MetadataCandidate[];
/** Null when the provider ran to completion, whether or not it found anything. */
outcome: MetadataProviderSearchOutcome | null;
}
/**
* A search reports two different things: the records it found, and whether a provider was cut off
* before it could finish. Without the second, an interrupted search is indistinguishable from a
* genuinely empty one.
*/
export type MetadataSearchEvent = { kind: 'candidate'; candidate: MetadataCandidate } | { kind: 'status'; status: MetadataProviderSearchStatus };
export interface StoredProviderContext {
libraryId: number;
title: string | null;
seriesName: string | null;
seriesIndex: number | null;
providerIds: Partial<Record<MetadataProviderKey, string>>;
@@ -36,15 +50,23 @@ export class MetadataFetchService {
private readonly metadataFetchRepository: MetadataFetchRepository,
) {}
search(params: MetadataSearchParams, keys?: MetadataProviderKey[]): Observable<MetadataCandidate> {
search(params: MetadataSearchParams, keys?: MetadataProviderKey[]): Observable<MetadataSearchEvent> {
const providers = this.registry.select(keys);
return merge(
...providers.map((provider) =>
from(this.fetchFromProviderWithThrottleHandling(provider, params)).pipe(switchMap((providerResults) => from(providerResults))),
from(this.fetchFromProviderWithThrottleHandling(provider, params)).pipe(switchMap((result) => from(toSearchEvents(provider.key, result)))),
),
);
}
/** Candidate-only view of `search`, for callers that have no use for provider status. */
searchCandidates(params: MetadataSearchParams, keys?: MetadataProviderKey[]): Observable<MetadataCandidate> {
return this.search(params, keys).pipe(
filter((event): event is Extract<MetadataSearchEvent, { kind: 'candidate' }> => event.kind === 'candidate'),
map((event) => event.candidate),
);
}
async lookupById(key: MetadataProviderKey, providerId: string): Promise<MetadataCandidate | null> {
const provider = this.registry.find(key);
if (!provider || !isIdentifiable(provider)) return null;
@@ -60,6 +82,7 @@ export class MetadataFetchService {
const row = await this.getAccessibleStoredProviderIdsRow(bookId, user);
return {
libraryId: row.libraryId,
title: row.title,
seriesName: row.seriesName,
seriesIndex: row.seriesIndex,
providerIds: this.mapStoredProviderIds(row),
@@ -105,7 +128,7 @@ export class MetadataFetchService {
};
}
private async fetchFromProviderWithThrottleHandling(provider: MetadataProvider, params: MetadataSearchParams): Promise<MetadataCandidate[]> {
private async fetchFromProviderWithThrottleHandling(provider: MetadataProvider, params: MetadataSearchParams): Promise<ProviderSearchResult> {
const startedAt = Date.now();
this.logger.log(`[metadata_fetch.provider_search] [start] provider=${provider.key} - provider fetch started`);
@@ -116,28 +139,28 @@ export class MetadataFetchService {
this.logger.warn(
`[metadata_fetch.provider_search] [fail] provider=${provider.key} durationMs=${Date.now() - startedAt} errorClass=TimeoutError error="provider search timed out" - provider fetch failed`,
);
return [];
return { candidates: [], outcome: 'timeout' };
}
this.throttleTracker.clearOnSuccess(provider.key);
this.logger.log(
`[metadata_fetch.provider_search] [end] provider=${provider.key} durationMs=${Date.now() - startedAt} resultCount=${results.length} - provider fetch completed`,
);
return results;
return { candidates: results, outcome: null };
} catch (error) {
if (error instanceof ProviderThrottleError) {
this.throttleTracker.record(provider.key, error.retryAfterSeconds);
this.logger.warn(
`[metadata_fetch.provider_search] [fail] provider=${provider.key} durationMs=${Date.now() - startedAt} errorClass=ProviderThrottleError error="provider throttled" - provider fetch failed`,
);
return [];
return { candidates: [], outcome: 'throttled' };
}
const errorClass = error instanceof Error ? error.name : 'UnknownError';
this.logger.warn(
`[metadata_fetch.provider_search] [fail] provider=${provider.key} durationMs=${Date.now() - startedAt} errorClass=${errorClass} error="${sanitizeLogError(error)}" - provider fetch failed`,
);
return [];
return { candidates: [], outcome: 'failed' };
}
}
@@ -206,6 +229,12 @@ export class MetadataFetchService {
}
}
function toSearchEvents(provider: MetadataProviderKey, result: ProviderSearchResult): MetadataSearchEvent[] {
const events: MetadataSearchEvent[] = result.candidates.map((candidate) => ({ kind: 'candidate', candidate }));
if (result.outcome) events.push({ kind: 'status', status: { provider, outcome: result.outcome } });
return events;
}
function hasText(v: string | undefined): boolean {
return typeof v === 'string' && v.trim().length > 0;
}
server/src/modules/metadata-fetch/providers/comicvine/comicvine.matching.test.ts
+123
View File
@@ -0,0 +1,123 @@
import { describe, expect, it } from 'vitest';
import { normalizeIssueNumber, rankVolumes } from './comicvine.matching';
import { ComicVineVolume } from './comicvine.types';
function volume(id: number, name: string, startYear: string | null, countOfIssues: number): ComicVineVolume {
return {
id,
name,
start_year: startYear,
count_of_issues: countOfIssues,
description: null,
deck: null,
image: null,
publisher: null,
site_detail_url: null,
};
}
function names(volumes: ComicVineVolume[]): string[] {
return volumes.map((v) => `${v.name} (${v.start_year})`);
}
describe('normalizeIssueNumber', () => {
it.each([
['067', '67'],
['0067', '67'],
['67', '67'],
[' 067 ', '67'],
['007.1', '7.1'],
['0.5', '0.5'],
])('strips the padding a filename adds: %s -> %s', (input, expected) => {
expect(normalizeIssueNumber(input)).toBe(expected);
});
it.each([
['0', '0'],
['000', '0'],
])('keeps issue zero addressable: %s -> %s', (input, expected) => {
expect(normalizeIssueNumber(input)).toBe(expected);
});
it('leaves numbers that carry no padding alone', () => {
expect(normalizeIssueNumber('-1')).toBe('-1');
expect(normalizeIssueNumber('1/2')).toBe('1/2');
expect(normalizeIssueNumber('')).toBe('');
});
});
describe('rankVolumes', () => {
const AMAZING_SPIDER_MAN = [
volume(90160, 'The Amazing Spider-Man Annual', '2018', 5),
volume(1218, 'The Amazing Spider-Man', '1963', 441),
volume(88985, 'Amazing Spider-Man: Renew Your Vows', '2016', 23),
volume(142577, 'The Amazing Spider-Man', '2022', 93),
volume(38536, 'Amazing Spider-Man Family', '2008', 8),
volume(84213, 'The Amazing Spider-Man', '2018', 74),
];
it('puts the runs that can hold the issue ahead of spin-offs and annuals', () => {
const ranked = rankVolumes(AMAZING_SPIDER_MAN, 'Amazing Spider-Man', '67');
expect(ranked.slice(0, 3).map((v) => v.id)).toEqual([142577, 84213, 1218]);
});
it('matches names across a leading article and punctuation', () => {
const ranked = rankVolumes([volume(2, 'Batman: The Dark Knight', '2011', 29), volume(1, 'The Batman', '2016', 80)], 'Batman', '12');
expect(ranked[0].id).toBe(1);
});
it('prefers the most recent run among equally good matches', () => {
const ranked = rankVolumes(AMAZING_SPIDER_MAN, 'Amazing Spider-Man', '5');
expect(names(ranked).slice(0, 3)).toEqual(['The Amazing Spider-Man (2022)', 'The Amazing Spider-Man (2018)', 'The Amazing Spider-Man (1963)']);
});
it('keeps an exact name ahead of a longer run with a looser name, so legacy numbering stays reachable', () => {
const ranked = rankVolumes(
[volume(1, 'Amazing Spider-Man Omnibus', '2020', 950), volume(2, 'Amazing Spider-Man', '2022', 93)],
'Amazing Spider-Man',
'900',
);
expect(ranked[0].id).toBe(2);
});
it('demotes rather than drops a run too short to hold the issue', () => {
const ranked = rankVolumes(AMAZING_SPIDER_MAN, 'Amazing Spider-Man', '400');
expect(ranked[0].id).toBe(1218);
expect(ranked.map((v) => v.id)).toHaveLength(AMAZING_SPIDER_MAN.length);
expect(ranked.map((v) => v.id)).toContain(90160);
});
it('treats an unknown issue count as no evidence either way', () => {
const ranked = rankVolumes([volume(1, 'Saga', '2012', 0), volume(2, 'Saga Compendium', '2019', 100)], 'Saga', '60');
expect(ranked[0].id).toBe(1);
});
it('treats a non-numeric issue number as no evidence either way', () => {
const ranked = rankVolumes([volume(1, 'Hellboy Annual', '1997', 1), volume(2, 'Hellboy', '1994', 60)], 'Hellboy', 'Special');
expect(ranked[0].id).toBe(2);
});
it('sorts a volume with no start year last among equals rather than dropping it', () => {
const ranked = rankVolumes([volume(1, 'Akira', null, 38), volume(2, 'Akira', '1988', 38)], 'Akira', '12');
expect(ranked.map((v) => v.id)).toEqual([2, 1]);
});
it('preserves the provider order for volumes it cannot tell apart', () => {
const ranked = rankVolumes([volume(7, 'Nova', '1976', 25), volume(8, 'Nova', '1976', 25)], 'Nova', '12');
expect(ranked.map((v) => v.id)).toEqual([7, 8]);
});
it('returns an empty list unchanged', () => {
expect(rankVolumes([], 'Anything', '1')).toEqual([]);
});
});
server/src/modules/metadata-fetch/providers/comicvine/comicvine.matching.ts
+64
View File
@@ -0,0 +1,64 @@
import { ComicVineVolume } from './comicvine.types';
const LEADING_ARTICLE = /^(?:the|a|an)\s+/;
const NON_ALPHANUMERIC = /[^a-z0-9]+/g;
const LEADING_ZEROS = /^0+(?=\d)/;
/**
* ComicVine stores issue numbers unpadded, so a padded number lifted from a filename
* ("... Issue 067.cbr") matches nothing. The lookahead keeps the final digit so issue #0,
* which real series do publish, survives.
*/
export function normalizeIssueNumber(raw: string): string {
return raw.trim().replace(LEADING_ZEROS, '');
}
function normalizeName(name: string): string {
return name.toLowerCase().replace(LEADING_ARTICLE, '').replace(NON_ALPHANUMERIC, ' ').trim();
}
/**
* `filter=name:` on /volumes/ is a substring match, so a common series name returns spin-offs,
* annuals and collections alongside the runs themselves. Scoring them before spending a request
* each is what keeps the search inside its budget.
*/
function nameScore(volumeName: string, seriesName: string): number {
const volume = normalizeName(volumeName);
const series = normalizeName(seriesName);
if (!volume || !series) return 0;
if (volume === series) return 2;
if (volume.startsWith(`${series} `) || volume.includes(series)) return 1;
return 0;
}
/**
* A volume that never ran this long cannot hold the issue. Treated as a ranking signal rather than
* a filter because publishers restart numbering and then reissue legacy numbers far above the
* volume's own count.
*/
function holdsIssuePlausibly(volume: ComicVineVolume, issueNumber: string): boolean {
const target = parseFloat(issueNumber);
if (!Number.isFinite(target)) return true;
if (!Number.isFinite(volume.count_of_issues) || volume.count_of_issues <= 0) return true;
return volume.count_of_issues >= target;
}
function startYear(volume: ComicVineVolume): number {
const year = volume.start_year ? parseInt(volume.start_year, 10) : NaN;
return Number.isFinite(year) ? year : 0;
}
function score(volume: ComicVineVolume, seriesName: string, issueNumber: string): number {
return nameScore(volume.name, seriesName) * 2 + (holdsIssuePlausibly(volume, issueNumber) ? 1 : 0);
}
/**
* Orders candidate volumes by how likely each is to hold the wanted issue: name match first, then
* whether the volume ran long enough, then the most recent run. Stable within equal scores.
*/
export function rankVolumes(volumes: ComicVineVolume[], seriesName: string, issueNumber: string): ComicVineVolume[] {
return volumes
.map((volume, index) => ({ volume, index, score: score(volume, seriesName, issueNumber) }))
.sort((a, b) => b.score - a.score || startYear(b.volume) - startYear(a.volume) || a.index - b.index)
.map((entry) => entry.volume);
}
server/src/modules/metadata-fetch/providers/comicvine/comicvine.provider.test.ts
+43 -10
View File
@@ -134,8 +134,8 @@ describe('ComicVineProvider', () => {
const results = await provider.search({ title: 'Batman #1' });
expect(client.searchVolumes).toHaveBeenCalledWith('Batman', 'test-key');
expect(client.searchIssuesInVolume).toHaveBeenCalledWith(mockVolume.id, '1', 'test-key');
expect(client.searchVolumes).toHaveBeenCalledWith('Batman', 'test-key', expect.any(AbortSignal));
expect(client.searchIssuesInVolume).toHaveBeenCalledWith(mockVolume.id, '1', 'test-key', expect.any(AbortSignal));
expect(results).toHaveLength(1);
});
@@ -145,8 +145,8 @@ describe('ComicVineProvider', () => {
const results = await provider.search({ title: 'The Origin', seriesName: 'Batman', seriesIndex: 12.5 });
expect(client.searchVolumes).toHaveBeenCalledWith('Batman', 'test-key');
expect(client.searchIssuesInVolume).toHaveBeenCalledWith(mockVolume.id, '12.5', 'test-key');
expect(client.searchVolumes).toHaveBeenCalledWith('Batman', 'test-key', expect.any(AbortSignal));
expect(client.searchIssuesInVolume).toHaveBeenCalledWith(mockVolume.id, '12.5', 'test-key', expect.any(AbortSignal));
expect(client.searchIssues).not.toHaveBeenCalled();
expect(results).toHaveLength(1);
});
@@ -157,8 +157,41 @@ describe('ComicVineProvider', () => {
await provider.search({ title: 'Superman #3', seriesName: 'Batman', seriesIndex: 12.5 });
expect(client.searchVolumes).toHaveBeenCalledWith('Superman', 'test-key');
expect(client.searchIssuesInVolume).toHaveBeenCalledWith(mockVolume.id, '3', 'test-key');
expect(client.searchVolumes).toHaveBeenCalledWith('Superman', 'test-key', expect.any(AbortSignal));
expect(client.searchIssuesInVolume).toHaveBeenCalledWith(mockVolume.id, '3', 'test-key', expect.any(AbortSignal));
});
it('lets a typed series override the stored one while keeping the stored issue number', async () => {
vi.mocked(client.searchVolumes).mockResolvedValue([mockVolume]);
vi.mocked(client.searchIssuesInVolume).mockResolvedValue([mockIssue]);
await provider.search({ title: 'Daredevil', seriesName: 'Amazing Spider-Man', seriesIndex: 67, titleIsExplicitQuery: true });
expect(client.searchVolumes).toHaveBeenCalledWith('Daredevil', 'test-key', expect.any(AbortSignal));
expect(client.searchIssuesInVolume).toHaveBeenCalledWith(mockVolume.id, '67', 'test-key', expect.any(AbortSignal));
});
it("keeps the stored series when the title is the book's own, not something typed", async () => {
vi.mocked(client.searchVolumes).mockResolvedValue([mockVolume]);
vi.mocked(client.searchIssuesInVolume).mockResolvedValue([mockIssue]);
await provider.search({
title: 'The Amazing Spider-Man (2022) Volume 06 Issue 067',
seriesName: 'Amazing Spider-Man',
seriesIndex: 67,
titleIsExplicitQuery: false,
});
expect(client.searchVolumes).toHaveBeenCalledWith('Amazing Spider-Man', 'test-key', expect.any(AbortSignal));
});
it('falls through to the general search for a typed query when no issue number is stored', async () => {
vi.mocked(client.searchIssues).mockResolvedValue([mockIssue]);
await provider.search({ title: 'Daredevil', seriesName: 'Amazing Spider-Man', titleIsExplicitQuery: true });
expect(client.searchVolumes).not.toHaveBeenCalled();
expect(client.searchIssues).toHaveBeenCalledWith('Daredevil', 'test-key', expect.any(AbortSignal));
});
it('uses the general search when the stored series has no issue number', async () => {
@@ -167,7 +200,7 @@ describe('ComicVineProvider', () => {
await provider.search({ title: 'The Origin', seriesName: 'Batman' });
expect(client.searchVolumes).not.toHaveBeenCalled();
expect(client.searchIssues).toHaveBeenCalledWith('The Origin', 'test-key');
expect(client.searchIssues).toHaveBeenCalledWith('The Origin', 'test-key', expect.any(AbortSignal));
});
it('tries volumes sorted by start_year descending', async () => {
@@ -178,7 +211,7 @@ describe('ComicVineProvider', () => {
await provider.search({ title: 'Batman #1' });
expect(client.searchIssuesInVolume).toHaveBeenNthCalledWith(1, newer.id, '1', 'test-key');
expect(client.searchIssuesInVolume).toHaveBeenNthCalledWith(1, newer.id, '1', 'test-key', expect.any(AbortSignal));
});
it('returns empty array when no volumes are found', async () => {
@@ -223,7 +256,7 @@ describe('ComicVineProvider', () => {
await provider.search({ title: 'Batman #1' });
expect(client.getIssueById).toHaveBeenCalledWith(String(issueNoCredits.id), 'test-key');
expect(client.getIssueById).toHaveBeenCalledWith(String(issueNoCredits.id), 'test-key', expect.any(AbortSignal));
});
it('skips detail fetch when issue already has credits', async () => {
@@ -264,7 +297,7 @@ describe('ComicVineProvider', () => {
await provider.search({ title: 'Batman' });
expect(client.searchIssues).toHaveBeenCalledWith('Batman', 'test-key');
expect(client.searchIssues).toHaveBeenCalledWith('Batman', 'test-key', expect.any(AbortSignal));
expect(client.searchVolumes).not.toHaveBeenCalled();
});
server/src/modules/metadata-fetch/providers/comicvine/comicvine.provider.ts
+123 -40
View File
@@ -6,41 +6,70 @@ import { ProviderThrottleTracker } from '../../provider-throttle.tracker';
import { ProviderThrottleError } from '../../provider-throttle.error';
import { IdentifiableProvider } from '../metadata-provider';
import { MetadataSearchParams } from '../metadata-search-params';
import { PROVIDER_LIMITS } from '../provider-constants';
import { normalizeMaxCandidates } from '../provider-utils';
import { PROVIDER_BUDGETS_MS, PROVIDER_LIMITS } from '../provider-constants';
import { createSearchDeadline, normalizeMaxCandidates, SearchDeadline } from '../provider-utils';
import { ComicVineClient } from './comicvine.client';
import { mapIssueToCandidate } from './comicvine.mapper';
import { ComicVineIssue } from './comicvine.types';
import { normalizeIssueNumber, rankVolumes } from './comicvine.matching';
import { ComicVineIssue, ComicVineVolume } from './comicvine.types';
const ISSUE_PATTERN = /^(.*?)\s*#(\d[\d.]*)(.*)$/;
// Every ComicVine call costs a rate-limiter slot, and the volume issue list is slow enough that
// probing candidates one at a time exhausts the provider budget before reaching the right run.
// Probing a wave at a time keeps recall across all VOLUME_PROBE_LIMIT candidates affordable.
const VOLUME_PROBE_LIMIT = 8;
const VOLUME_PROBE_BATCH = 3;
interface ParsedIssueTitle {
seriesName: string;
issueNumber: string;
}
interface VolumeIssueMatch {
issue: ComicVineIssue;
volume?: ComicVineVolume;
}
function parseIssueTitle(title: string): ParsedIssueTitle | null {
const match = ISSUE_PATTERN.exec(title.trim());
if (!match) return null;
const seriesName = match[1].trim();
const issueNumber = match[2].trim();
const issueNumber = normalizeIssueNumber(match[2]);
if (!seriesName || !issueNumber) return null;
return { seriesName, issueNumber };
}
function storedIssueNumber(params: MetadataSearchParams): string | null {
const issueNumber = params.seriesIndex;
if (issueNumber === undefined || !Number.isFinite(issueNumber) || issueNumber < 0) return null;
return String(issueNumber);
}
/**
* Prefer a "<series> #<issue>" title because that is the user typing an explicit target, and fall
* back to the book's stored series. Comic titles hold the issue name alone, so without the fallback
* Prefer a "<series> #<issue>" title because that is the user naming an explicit target. Otherwise
* the stored series leads, since a comic title holds only the issue name and without that fallback
* an already-matched comic would re-search through the far looser general issue search.
*
* A typed query that is not the book's own title overrides the stored series name even so, or
* searching a comic for a different series would silently return the series it is already filed
* under. The stored issue number still applies: it is the one thing a bare series query omits.
*/
function resolveIssueQuery(params: MetadataSearchParams): ParsedIssueTitle | null {
const fromTitle = params.title ? parseIssueTitle(params.title) : null;
if (fromTitle) return fromTitle;
const issueNumber = storedIssueNumber(params);
const typedSeriesName = params.titleIsExplicitQuery ? params.title?.trim() : undefined;
if (typedSeriesName) return issueNumber ? { seriesName: typedSeriesName, issueNumber } : null;
const seriesName = params.seriesName?.trim();
const issueNumber = params.seriesIndex;
if (!seriesName || issueNumber === undefined || !Number.isFinite(issueNumber) || issueNumber < 0) return null;
return { seriesName, issueNumber: String(issueNumber) };
if (!seriesName || !issueNumber) return null;
return { seriesName, issueNumber };
}
function isThrottleRejection(result: PromiseSettledResult<unknown>): boolean {
return result.status === 'rejected' && result.reason instanceof ProviderThrottleError;
}
function hasAnyCredits(issue: ComicVineIssue): boolean {
@@ -78,9 +107,18 @@ export class ComicVineProvider implements IdentifiableProvider {
try {
const maxCandidates = normalizeMaxCandidates(params.maxCandidatesPerProvider, PROVIDER_LIMITS.COMICVINE_MAX_RESULTS);
const parsed = resolveIssueQuery(params);
return parsed
? await this.structuredSearch(parsed.seriesName, parsed.issueNumber, apiKey, maxCandidates, params.signal)
: await this.generalSearch(params.title, apiKey, maxCandidates, params.signal);
const deadline = createSearchDeadline(PROVIDER_BUDGETS_MS.COMICVINE_SEARCH, params.signal);
try {
const matches = parsed
? await this.structuredSearch(parsed.seriesName, parsed.issueNumber, apiKey, maxCandidates, deadline)
: await this.generalSearch(params.title, apiKey, maxCandidates, deadline);
const enriched = await this.enrichWithinBudget(matches, apiKey, deadline);
return enriched.map(({ issue, volume }) => mapIssueToCandidate(issue, { volumeIssueCount: volume?.count_of_issues }));
} finally {
deadline.dispose();
}
} catch (err) {
if (err instanceof ProviderThrottleError) {
this.recordThrottle();
@@ -117,46 +155,91 @@ export class ComicVineProvider implements IdentifiableProvider {
issueNumber: string,
apiKey: string,
maxCandidates: number,
signal?: AbortSignal,
): Promise<MetadataCandidate[]> {
const volumes = signal ? await this.client.searchVolumes(seriesName, apiKey, signal) : await this.client.searchVolumes(seriesName, apiKey);
deadline: SearchDeadline,
): Promise<VolumeIssueMatch[]> {
const volumes = await this.client.searchVolumes(seriesName, apiKey, deadline.signal);
if (volumes.length === 0) {
this.logger.debug(`ComicVine: no volumes found for "${seriesName}"`);
return [];
}
const sorted = [...volumes].sort((a, b) => {
const yearA = a.start_year ? parseInt(a.start_year, 10) : 0;
const yearB = b.start_year ? parseInt(b.start_year, 10) : 0;
return yearB - yearA;
});
const ranked = rankVolumes(volumes, seriesName, issueNumber).slice(0, VOLUME_PROBE_LIMIT);
const matches: VolumeIssueMatch[] = [];
const seenIssueIds = new Set<number>();
for (const volume of sorted.slice(0, 8)) {
const issues = signal
? await this.client.searchIssuesInVolume(volume.id, issueNumber, apiKey, signal)
: await this.client.searchIssuesInVolume(volume.id, issueNumber, apiKey);
if (issues.length > 0) {
const enriched = await Promise.all(issues.slice(0, maxCandidates).map((issue) => this.enrichWithDetails(issue, apiKey, signal)));
return enriched.map((issue) => mapIssueToCandidate(issue, { volumeIssueCount: volume.count_of_issues }));
for (let offset = 0; offset < ranked.length; offset += VOLUME_PROBE_BATCH) {
const wave = ranked.slice(offset, offset + VOLUME_PROBE_BATCH);
// Settled rather than all: one probe being throttled or cut off by the deadline must not
// discard the issues its siblings found.
const probed = await Promise.allSettled(wave.map((volume) => this.probeVolume(volume, issueNumber, apiKey, deadline.signal)));
for (const result of probed) {
if (result.status !== 'fulfilled') continue;
for (const issue of result.value.issues) {
if (seenIssueIds.has(issue.id)) continue;
seenIssueIds.add(issue.id);
matches.push({ issue, volume: result.value.volume });
}
}
if (probed.some(isThrottleRejection)) {
this.recordThrottle();
break;
}
if (matches.length > 0) break;
if (deadline.expired()) {
this.logger.debug(`ComicVine: budget spent after ${offset + wave.length} volume probes for "${seriesName}" #${issueNumber}`);
break;
}
}
this.logger.debug(`ComicVine: no issues found for "${seriesName}" #${issueNumber}`);
return [];
if (matches.length === 0) {
this.logger.debug(`ComicVine: no issues found for "${seriesName}" #${issueNumber}`);
}
return matches.slice(0, maxCandidates);
}
private async generalSearch(query: string, apiKey: string, maxCandidates: number, signal?: AbortSignal): Promise<MetadataCandidate[]> {
const issues = signal ? await this.client.searchIssues(query, apiKey, signal) : await this.client.searchIssues(query, apiKey);
const enriched = await Promise.all(issues.slice(0, maxCandidates).map((issue) => this.enrichWithDetails(issue, apiKey, signal)));
return enriched.map((issue) => mapIssueToCandidate(issue));
private async probeVolume(
volume: ComicVineVolume,
issueNumber: string,
apiKey: string,
signal: AbortSignal,
): Promise<{ volume: ComicVineVolume; issues: ComicVineIssue[] }> {
return { volume, issues: await this.client.searchIssuesInVolume(volume.id, issueNumber, apiKey, signal) };
}
private async enrichWithDetails(issue: ComicVineIssue, apiKey: string, signal?: AbortSignal): Promise<ComicVineIssue> {
if (hasAnyCredits(issue)) return issue;
this.logger.debug(`ComicVine: issue ${issue.id} has no credits from list endpoint, fetching detail`);
const detailed = signal
? await this.client.getIssueById(String(issue.id), apiKey, signal)
: await this.client.getIssueById(String(issue.id), apiKey);
return detailed ?? issue;
private async generalSearch(query: string, apiKey: string, maxCandidates: number, deadline: SearchDeadline): Promise<VolumeIssueMatch[]> {
const issues = await this.client.searchIssues(query, apiKey, deadline.signal);
return issues.slice(0, maxCandidates).map((issue) => ({ issue }));
}
/**
* Detail lookups only add credits, so they are worth a request each only while the budget holds.
* Sequential because the client serializes calls anyway, which lets the deadline actually stop
* the work rather than every lookup being queued up front. A lookup that fails or is cut off
* costs its candidate only the credits, never the candidate itself.
*/
private async enrichWithinBudget(matches: VolumeIssueMatch[], apiKey: string, deadline: SearchDeadline): Promise<VolumeIssueMatch[]> {
const enriched: VolumeIssueMatch[] = [];
let stopped = false;
for (const match of matches) {
if (stopped || hasAnyCredits(match.issue) || deadline.expired()) {
enriched.push(match);
continue;
}
this.logger.debug(`ComicVine: issue ${match.issue.id} has no credits from list endpoint, fetching detail`);
try {
const detailed = await this.client.getIssueById(String(match.issue.id), apiKey, deadline.signal);
enriched.push(detailed ? { ...match, issue: detailed } : match);
} catch (err) {
enriched.push(match);
stopped = true;
if (err instanceof ProviderThrottleError) this.recordThrottle();
}
}
return enriched;
}
}
server/src/modules/metadata-fetch/providers/comicvine/comicvine.search-budget.test.ts
+410
View File
@@ -0,0 +1,410 @@
import { MetadataProviderKey, ProviderConfigurations } from '@bookorbit/types';
import { firstValueFrom, toArray } from 'rxjs';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import * as fetchWithThrottleModule from '../../fetch-with-throttle';
import { MetadataFetchRepository } from '../../metadata-fetch.repository';
import { MetadataFetchService } from '../../metadata-fetch.service';
import { ProviderRegistry } from '../../provider-registry';
import { ProviderThrottleTracker } from '../../provider-throttle.tracker';
import { ProviderConfigService } from '../../../metadata-preferences/provider-config.service';
import { PROVIDER_BUDGETS_MS, PROVIDER_TIMEOUT_MS } from '../provider-constants';
import { ComicVineClient } from './comicvine.client';
import { ComicVineProvider } from './comicvine.provider';
import { ComicVineIssue, ComicVineVolume } from './comicvine.types';
vi.mock('../../fetch-with-throttle', () => ({ fetchWithThrottle: vi.fn() }));
/**
* Regression cover for the ComicVine search collapsing into "No results found" (issue #1039).
* Latencies are the ones measured against the live API in that report: the volume search is fast,
* every issue lookup is slow, and the provider has 15s for the whole search.
*/
const VOLUME_SEARCH_MS = 570;
const ISSUE_PROBE_MS = 3_500;
const ISSUE_DETAIL_MS = 2_000;
const ASM_2022 = 142577;
const ASM_2018 = 84213;
const ASM_1963 = 1218;
const providerConfig: ProviderConfigurations = {
google: { enabled: false, apiKey: '' },
amazon: { enabled: false, domain: 'amazon.com', cookie: '' },
goodreads: { enabled: false },
hardcover: { enabled: false, apiKey: '' },
openLibrary: { enabled: false },
itunes: { enabled: false, coverResolution: 'high' },
audible: { enabled: false, domain: 'com' },
audnexus: { enabled: false },
comicvine: { enabled: true, apiKey: 'test-key' },
ranobedb: { enabled: false },
kobo: { enabled: false, country: 'us', language: 'en' },
lubimyczytac: { enabled: false },
};
function volume(id: number, name: string, startYear: string, countOfIssues: number): ComicVineVolume {
return {
id,
name,
start_year: startYear,
count_of_issues: countOfIssues,
description: null,
deck: null,
image: null,
publisher: null,
site_detail_url: null,
};
}
function issue(id: number, volumeId: number, volumeName: string, issueNumber: string, options: { credits?: boolean } = {}): ComicVineIssue {
const credited = options.credits ?? true;
return {
id,
name: `Amazing Spider-Man #${issueNumber}`,
issue_number: issueNumber,
cover_date: '2022-11-01',
store_date: null,
description: null,
deck: null,
image: null,
volume: { id: volumeId, name: volumeName },
site_detail_url: null,
person_credits: credited ? [{ id: 1, name: 'Zeb Wells', role: 'writer' }] : [],
character_credits: [],
team_credits: [],
story_arc_credits: [],
location_credits: [],
};
}
// What `filter=name:Amazing Spider-Man` actually returns: three runs plus the spin-offs and
// annuals the substring match drags in.
const AMAZING_SPIDER_MAN_VOLUMES: ComicVineVolume[] = [
volume(90160, 'The Amazing Spider-Man Annual', '2018', 5),
volume(ASM_1963, 'The Amazing Spider-Man', '1963', 441),
volume(88985, 'Amazing Spider-Man: Renew Your Vows', '2016', 23),
volume(ASM_2022, 'The Amazing Spider-Man', '2022', 93),
volume(38536, 'Amazing Spider-Man Family', '2008', 8),
volume(ASM_2018, 'The Amazing Spider-Man', '2018', 74),
volume(49036, 'Amazing Spider-Man: Extra!', '2008', 3),
volume(76473, 'The Amazing Spider-Man: Epic Collection', '2013', 24),
];
function abortableDelay(ms: number, signal?: AbortSignal | null): Promise<void> {
return new Promise((resolve, reject) => {
const onAbort = () => {
clearTimeout(timer);
const error = new Error('This operation was aborted');
error.name = 'AbortError';
reject(error);
};
const timer = setTimeout(() => {
signal?.removeEventListener('abort', onAbort);
resolve();
}, ms);
if (signal?.aborted) {
onAbort();
return;
}
signal?.addEventListener('abort', onAbort, { once: true });
});
}
function okResponse(results: unknown): Response {
return { ok: true, status: 200, json: () => Promise.resolve({ status_code: 1, error: 'OK', results }) } as Response;
}
function throttledResponse(): Response {
return { ok: false, status: 420 } as Response;
}
interface HarnessOptions {
volumes?: ComicVineVolume[];
// Issues the /issues/ endpoint holds, keyed "<volumeId>#<issueNumber>".
issues?: Record<string, ComicVineIssue[]>;
generalSearchResults?: ComicVineIssue[];
issueDetails?: Record<string, ComicVineIssue>;
issueDetailMs?: number;
// Volumes and issue details ComicVine answers with HTTP 420, its throttle response.
throttledVolumeIds?: number[];
throttledDetailIds?: string[];
}
interface Harness {
provider: ComicVineProvider;
service: MetadataFetchService;
throttleTracker: ProviderThrottleTracker;
requestUrls: URL[];
probedVolumeIds: number[];
detailIssueIds: string[];
abortedPaths: string[];
}
function buildHarness(options: HarnessOptions = {}): Harness {
const volumes = options.volumes ?? AMAZING_SPIDER_MAN_VOLUMES;
const issues = options.issues ?? {};
const requestUrls: URL[] = [];
const probedVolumeIds: number[] = [];
const detailIssueIds: string[] = [];
const abortedPaths: string[] = [];
const detailMs = options.issueDetailMs ?? ISSUE_DETAIL_MS;
vi.mocked(fetchWithThrottleModule.fetchWithThrottle).mockImplementation(async (url, init) => {
const target = new URL(String(url));
const signal = init?.signal ?? undefined;
requestUrls.push(target);
const respondAfter = async (ms: number, response: () => Response): Promise<Response> => {
try {
await abortableDelay(ms, signal);
} catch (err) {
abortedPaths.push(target.pathname);
throw err;
}
return response();
};
if (target.pathname.endsWith('/volumes/')) {
return respondAfter(VOLUME_SEARCH_MS, () => okResponse(volumes));
}
if (target.pathname.endsWith('/search/')) {
return respondAfter(VOLUME_SEARCH_MS, () => okResponse(options.generalSearchResults ?? []));
}
if (target.pathname.endsWith('/issues/')) {
const filter = target.searchParams.get('filter') ?? '';
const volumeId = Number(/volume:(\d+)/.exec(filter)?.[1]);
const issueNumber = /issue_number:([^,]+)/.exec(filter)?.[1] ?? '';
probedVolumeIds.push(volumeId);
return respondAfter(ISSUE_PROBE_MS, () =>
options.throttledVolumeIds?.includes(volumeId) ? throttledResponse() : okResponse(issues[`${volumeId}#${issueNumber}`] ?? []),
);
}
const issueId = /\/issue\/4000-(\d+)\//.exec(target.pathname)?.[1] ?? '';
detailIssueIds.push(issueId);
return respondAfter(detailMs, () =>
options.throttledDetailIds?.includes(issueId) ? throttledResponse() : okResponse(options.issueDetails?.[issueId] ?? null),
);
});
const throttleTracker = new ProviderThrottleTracker();
vi.spyOn(throttleTracker, 'record');
const provider = new ComicVineProvider(
new ComicVineClient(),
{ getConfig: () => Promise.resolve(providerConfig) } as unknown as ProviderConfigService,
throttleTracker,
);
const service = new MetadataFetchService(new ProviderRegistry([provider]), new ProviderThrottleTracker(), {} as MetadataFetchRepository);
return { provider, service, throttleTracker, requestUrls, probedVolumeIds, detailIssueIds, abortedPaths };
}
async function runToCompletion<T>(work: Promise<T>): Promise<{ value: T; elapsedMs: number }> {
const startedAt = Date.now();
let settledAt = startedAt;
const tracked = work.then((value) => {
settledAt = Date.now();
return value;
});
await vi.advanceTimersByTimeAsync(120_000);
return { value: await tracked, elapsedMs: settledAt - startedAt };
}
describe('ComicVine search budget', () => {
beforeEach(() => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-08-18T12:51:23.000Z'));
});
afterEach(() => {
vi.useRealTimers();
vi.restoreAllMocks();
});
it('keeps its own budget under the timeout that aborts it', () => {
expect(PROVIDER_BUDGETS_MS.COMICVINE_SEARCH).toBeLessThan(PROVIDER_TIMEOUT_MS.SCRAPE);
});
it('finds the reported issue instead of timing out', async () => {
const harness = buildHarness({
issues: {
[`${ASM_2022}#67`]: [issue(900_001, ASM_2022, 'The Amazing Spider-Man', '67')],
[`${ASM_1963}#67`]: [issue(900_002, ASM_1963, 'The Amazing Spider-Man', '67')],
},
});
const { value: candidates, elapsedMs } = await runToCompletion(
firstValueFrom(
harness.service
.search(
// What the controller builds for the reported book: the typed series name plus the
// series and issue already parsed into book_metadata.
{ title: 'Amazing Spider-Man', seriesName: 'Amazing Spider-Man', seriesIndex: 67 },
[MetadataProviderKey.COMICVINE],
)
.pipe(toArray()),
),
);
expect(candidates.length).toBeGreaterThan(0);
expect(elapsedMs).toBeLessThan(PROVIDER_TIMEOUT_MS.SCRAPE);
});
it('spends its requests on the runs that can hold the issue, not on annuals and collections', async () => {
const harness = buildHarness({
issues: { [`${ASM_2022}#67`]: [issue(900_001, ASM_2022, 'The Amazing Spider-Man', '67')] },
});
await runToCompletion(harness.provider.search({ seriesName: 'Amazing Spider-Man', seriesIndex: 67, title: 'Amazing Spider-Man' }));
expect(harness.probedVolumeIds).toEqual([ASM_2022, ASM_2018, ASM_1963]);
});
it('offers every run that has the issue, most recent first, each carrying its own issue count', async () => {
const harness = buildHarness({
issues: {
[`${ASM_2022}#67`]: [issue(900_001, ASM_2022, 'The Amazing Spider-Man', '67')],
[`${ASM_1963}#67`]: [issue(900_002, ASM_1963, 'The Amazing Spider-Man', '67')],
},
});
const { value: candidates } = await runToCompletion(
harness.provider.search({ seriesName: 'Amazing Spider-Man', seriesIndex: 67, title: 'Amazing Spider-Man' }),
);
expect(candidates.map((c) => c.providerId)).toEqual(['900001', '900002']);
expect(candidates.map((c) => c.seriesTotalBooks)).toEqual([93, 441]);
});
it('reaches a run that ranking cannot promote, where probing one at a time ran out of time', async () => {
// Twenty volumes the ranking cannot tell apart, with the match sitting sixth: the case the
// report hit, where the old serial probe never got past the fourth candidate.
const indistinguishable = Array.from({ length: 20 }, (_, i) => volume(i === 5 ? ASM_2022 : 1000 + i, 'The Amazing Spider-Man', '2022', 93));
const harness = buildHarness({
volumes: indistinguishable,
issues: { [`${ASM_2022}#67`]: [issue(900_001, ASM_2022, 'The Amazing Spider-Man', '67')] },
});
const { value: candidates, elapsedMs } = await runToCompletion(
harness.provider.search({ seriesName: 'Amazing Spider-Man', seriesIndex: 67, title: 'Amazing Spider-Man' }),
);
expect(candidates).toHaveLength(1);
expect(harness.probedVolumeIds).toContain(ASM_2022);
expect(elapsedMs).toBeLessThan(PROVIDER_TIMEOUT_MS.SCRAPE);
});
it('gives up on a series ComicVine does not carry without spending the whole budget', async () => {
const harness = buildHarness({ issues: {} });
const { value: candidates, elapsedMs } = await runToCompletion(
harness.provider.search({ seriesName: 'Amazing Spider-Man', seriesIndex: 67, title: 'Amazing Spider-Man' }),
);
expect(candidates).toEqual([]);
expect(harness.probedVolumeIds.length).toBeLessThanOrEqual(8);
expect(elapsedMs).toBeLessThan(PROVIDER_TIMEOUT_MS.SCRAPE);
});
it('strips the zero padding a comic filename adds to the issue number', async () => {
const harness = buildHarness({
issues: { [`${ASM_2022}#67`]: [issue(900_001, ASM_2022, 'The Amazing Spider-Man', '67')] },
});
// The SSE controller normalizes "#067" itself; the background orchestrator does not, so the
// provider has to.
const { value: candidates } = await runToCompletion(harness.provider.search({ title: 'Amazing Spider-Man #067' }));
const probe = harness.requestUrls.find((url) => url.pathname.endsWith('/issues/'));
expect(probe?.searchParams.get('filter')).toBe(`volume:${ASM_2022},issue_number:67`);
expect(candidates).toHaveLength(1);
});
it('stops enriching when the budget runs out but keeps the candidates it found', async () => {
const uncredited = Array.from({ length: 5 }, (_, i) => issue(900_100 + i, ASM_2022, 'The Amazing Spider-Man', '67', { credits: false }));
const harness = buildHarness({
issues: { [`${ASM_2022}#67`]: uncredited },
issueDetails: Object.fromEntries(
uncredited.map((i) => [String(i.id), { ...i, person_credits: [{ id: 1, name: 'Zeb Wells', role: 'writer' }] }]),
),
});
const { value: candidates, elapsedMs } = await runToCompletion(
harness.provider.search({ seriesName: 'Amazing Spider-Man', seriesIndex: 67, title: 'Amazing Spider-Man' }),
);
expect(candidates).toHaveLength(uncredited.length);
expect(harness.detailIssueIds.length).toBeGreaterThan(0);
expect(harness.detailIssueIds.length).toBeLessThan(uncredited.length);
expect(candidates.at(-1)?.authors).toEqual([]);
expect(elapsedMs).toBeLessThan(PROVIDER_TIMEOUT_MS.SCRAPE);
});
it('cuts off a detail lookup that would outlive the search, keeping the candidate', async () => {
const uncredited = [issue(900_100, ASM_2022, 'The Amazing Spider-Man', '67', { credits: false })];
const harness = buildHarness({
issues: { [`${ASM_2022}#67`]: uncredited },
// Long enough that, left to run, it would cross the hard timeout and take everything with it.
issueDetailMs: 12_000,
issueDetails: { '900100': { ...uncredited[0], person_credits: [{ id: 1, name: 'Zeb Wells', role: 'writer' }] } },
});
const { value: candidates, elapsedMs } = await runToCompletion(
harness.provider.search({ seriesName: 'Amazing Spider-Man', seriesIndex: 67, title: 'Amazing Spider-Man' }),
);
expect(harness.abortedPaths).toEqual(['/api/issue/4000-900100/']);
expect(candidates).toHaveLength(1);
expect(candidates[0].authors).toEqual([]);
expect(elapsedMs).toBeLessThan(PROVIDER_TIMEOUT_MS.SCRAPE);
});
it('keeps what a wave found when one of its probes is throttled', async () => {
const harness = buildHarness({
issues: { [`${ASM_2022}#67`]: [issue(900_001, ASM_2022, 'The Amazing Spider-Man', '67')] },
throttledVolumeIds: [ASM_2018],
});
const { value: candidates } = await runToCompletion(
harness.provider.search({ seriesName: 'Amazing Spider-Man', seriesIndex: 67, title: 'Amazing Spider-Man' }),
);
expect(candidates.map((c) => c.providerId)).toEqual(['900001']);
expect(harness.throttleTracker.record).toHaveBeenCalledWith(MetadataProviderKey.COMICVINE, expect.anything());
});
it('keeps the candidate when the detail lookup behind it is throttled', async () => {
const uncredited = [
issue(900_100, ASM_2022, 'The Amazing Spider-Man', '67', { credits: false }),
issue(900_101, ASM_2022, 'The Amazing Spider-Man', '67', { credits: false }),
];
const harness = buildHarness({
issues: { [`${ASM_2022}#67`]: uncredited },
throttledDetailIds: ['900100'],
});
const { value: candidates } = await runToCompletion(
harness.provider.search({ seriesName: 'Amazing Spider-Man', seriesIndex: 67, title: 'Amazing Spider-Man' }),
);
expect(candidates.map((c) => c.providerId)).toEqual(['900100', '900101']);
expect(harness.throttleTracker.record).toHaveBeenCalledWith(MetadataProviderKey.COMICVINE, expect.anything());
// Throttled once, the run stops asking rather than spending the rest of the window on refusals.
expect(harness.detailIssueIds).toEqual(['900100']);
});
it('falls back to the general issue search for a comic with no parsed series', async () => {
const harness = buildHarness({
generalSearchResults: [issue(900_001, ASM_2022, 'The Amazing Spider-Man', '67')],
});
const { value: candidates } = await runToCompletion(harness.provider.search({ title: 'The Amazing Spider-Man (2022) Volume 06 Issue 067' }));
expect(harness.requestUrls[0].pathname).toBe('/api/search/');
expect(harness.probedVolumeIds).toEqual([]);
expect(candidates).toHaveLength(1);
});
});
server/src/modules/metadata-fetch/providers/metadata-search-params.ts
+3
View File
@@ -9,6 +9,9 @@ export interface MetadataSearchParams {
// pairing cannot be recovered from it.
seriesName?: string;
seriesIndex?: number;
// True when `title` is a query the caller typed rather than the book's own stored title. Stored
// series context may then fill in what the query omits, but must not override what it states.
titleIsExplicitQuery?: boolean;
existingProviderIds?: Partial<Record<MetadataProviderKey, string>>;
// Pins a Hardcover refresh to a previously chosen edition instead of re-deriving one by ISBN.
hardcoverEditionId?: string;
server/src/modules/metadata-fetch/providers/provider-constants.ts
+9
View File
@@ -15,6 +15,15 @@ export const PROVIDER_LIMITS = {
DEFAULT_SEARCH_RESULTS: 10,
} as const;
/**
* Soft budget for a provider search that spans many calls. Reaching it aborts the work still in
* flight, so the margin under the hard provider timeout only has to cover assembling the results
* that already arrived.
*/
export const PROVIDER_BUDGETS_MS = {
COMICVINE_SEARCH: PROVIDER_TIMEOUT_MS.SCRAPE - 1_500,
} as const;
export const PROVIDER_DELAYS_MS = {
AMAZON_BETWEEN_REQUESTS: 800,
GOODREADS_BETWEEN_REQUESTS: 600,
server/src/modules/metadata-fetch/providers/provider-utils.ts
+28
View File
@@ -19,6 +19,34 @@ export function buildRequestSignal(timeoutMs: number, parentSignal?: AbortSignal
return AbortSignal.any([timeoutSignal, parentSignal]);
}
export interface SearchDeadline {
/** Epoch ms the budget runs out. */
readonly at: number;
/** Aborts in-flight work when the budget runs out, so partial results survive. */
readonly signal: AbortSignal;
expired(): boolean;
dispose(): void;
}
/**
* Budget for a search that spans many calls. The hard provider timeout throws the whole search
* away; this one cancels what is still running and leaves the caller holding what it already has,
* so it has to sit far enough below the hard timeout for the caller to finish assembling results.
*/
export function createSearchDeadline(budgetMs: number, parentSignal?: AbortSignal): SearchDeadline {
const at = Date.now() + budgetMs;
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), budgetMs);
const signal = parentSignal ? AbortSignal.any([parentSignal, controller.signal]) : controller.signal;
return {
at,
signal,
expired: () => Date.now() >= at,
dispose: () => clearTimeout(timer),
};
}
export function sleep(ms: number, signal?: AbortSignal): Promise<void> {
if (signal?.aborted) {
return Promise.reject(createAbortError());