mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
fix(metadata): accept goodreads pages that preload waf script
Treat challenge.js preload references as valid page content while retaining detection for active AWS WAF runtime markers. Fixes #1240
This commit is contained in:
neon
@@ -88,7 +88,7 @@ describe('GoodreadsProvider', () => {
|
||||
expect(result[0].title).toBe('Some Book');
|
||||
});
|
||||
|
||||
it('should find by ISBN and fetch book details', async () => {
|
||||
it('should fall back to the ISBN page and fetch book details', async () => {
|
||||
const isbnHtml = `
|
||||
<meta property="og:url" content="https://www.goodreads.com/book/show/456.Test_ISBN">
|
||||
`;
|
||||
@@ -103,19 +103,55 @@ describe('GoodreadsProvider', () => {
|
||||
|
||||
global.fetch = vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({ ok: true, json: () => Promise.resolve([]) })
|
||||
.mockResolvedValueOnce({ ok: true, text: () => Promise.resolve(isbnHtml) })
|
||||
.mockResolvedValueOnce({ ok: true, text: () => Promise.resolve(bookHtml) });
|
||||
|
||||
const result = await provider.search({ isbn: '1234567890' });
|
||||
|
||||
expect(global.fetch).toHaveBeenCalledWith('https://www.goodreads.com/book/auto_complete?format=json&q=1234567890', expect.any(Object));
|
||||
expect(global.fetch).toHaveBeenCalledWith('https://www.goodreads.com/book/isbn/1234567890', expect.any(Object));
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe('Test ISBN Book');
|
||||
});
|
||||
|
||||
it('should find an ISBN through autocomplete when book pages are challenged', async () => {
|
||||
const autocomplete = [
|
||||
{
|
||||
bookId: '53180949',
|
||||
bookUrl: '/book/show/53180949-dune',
|
||||
title: 'Dune (Dune, #1)',
|
||||
bookTitleBare: 'Dune',
|
||||
author: { name: 'Frank Herbert' },
|
||||
numPages: 884,
|
||||
},
|
||||
];
|
||||
|
||||
global.fetch = vi.fn((input: Parameters<typeof fetch>[0]) => {
|
||||
const url = fetchUrl(input);
|
||||
if (url.includes('/auto_complete')) return Promise.resolve({ ok: true, status: 200, json: () => Promise.resolve(autocomplete) });
|
||||
return Promise.resolve({ ok: true, status: 202, text: () => Promise.resolve('<div id="challenge-container"></div>') });
|
||||
}) as never;
|
||||
|
||||
const result = await provider.search({ isbn: '9780441172719' });
|
||||
|
||||
expect(result).toEqual([
|
||||
expect.objectContaining({
|
||||
providerId: '53180949',
|
||||
title: 'Dune',
|
||||
authors: ['Frank Herbert'],
|
||||
pageCount: 884,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('should return empty array if ISBN lookup does not find a book ID', async () => {
|
||||
const emptyHtml = `<html><body>No ISBN found</body></html>`;
|
||||
global.fetch = vi.fn().mockResolvedValue({ ok: true, text: () => Promise.resolve(emptyHtml) });
|
||||
global.fetch = vi.fn((input: Parameters<typeof fetch>[0]) => {
|
||||
const url = fetchUrl(input);
|
||||
if (url.includes('/auto_complete')) return Promise.resolve({ ok: true, json: () => Promise.resolve([]) });
|
||||
return Promise.resolve({ ok: true, text: () => Promise.resolve(emptyHtml) });
|
||||
}) as never;
|
||||
|
||||
const result = await provider.search({ isbn: '0000000000' });
|
||||
expect(result).toEqual([]);
|
||||
@@ -330,7 +366,7 @@ describe('GoodreadsProvider', () => {
|
||||
expect(result.find((c) => c.providerId === '1')?.title).toBe('B1');
|
||||
});
|
||||
|
||||
it('reports a bot challenge as a throttle instead of quietly degrading the whole batch', async () => {
|
||||
it('uses autocomplete candidates when detail pages are challenged', async () => {
|
||||
vi.useFakeTimers();
|
||||
const autocomplete = [
|
||||
{ bookId: '1', bookUrl: '/book/show/1.B1', title: 'B1', bookTitleBare: 'B1' },
|
||||
@@ -343,12 +379,11 @@ describe('GoodreadsProvider', () => {
|
||||
return Promise.resolve({ ok: true, status: 202, text: () => Promise.resolve('<div id="challenge-container"></div>') });
|
||||
}) as never;
|
||||
|
||||
// The rejection lands while the timers run, so the assertion has to be attached before then:
|
||||
// a rejected promise nobody is holding is an unhandled rejection, which fails the whole run.
|
||||
const rejects = expect(provider.search({ title: 'B' })).rejects.toBeInstanceOf(ProviderThrottleError);
|
||||
const searchPromise = provider.search({ title: 'B' });
|
||||
await vi.runAllTimersAsync();
|
||||
const result = await searchPromise;
|
||||
|
||||
await rejects;
|
||||
expect(result.map((candidate) => candidate.providerId)).toEqual(['1', '2']);
|
||||
const detailUrls = vi
|
||||
.mocked(global.fetch)
|
||||
.mock.calls.map(([url]) => fetchUrl(url))
|
||||
@@ -356,7 +391,7 @@ describe('GoodreadsProvider', () => {
|
||||
expect(detailUrls).toEqual(['https://www.goodreads.com/book/show/1']);
|
||||
});
|
||||
|
||||
it('salvages the books already scraped when a bot challenge stops the batch part-way', async () => {
|
||||
it('keeps scraped and autocomplete candidates when a challenge stops detail enrichment', async () => {
|
||||
vi.useFakeTimers();
|
||||
const autocomplete = [
|
||||
{
|
||||
@@ -386,18 +421,13 @@ describe('GoodreadsProvider', () => {
|
||||
return Promise.resolve({ ok: true, status: 202, text: () => Promise.resolve('') });
|
||||
}) as never;
|
||||
|
||||
// Attach the handler before the timers run: a rejected promise nobody is holding is an
|
||||
// unhandled rejection, which fails the whole run.
|
||||
const settled = provider.search({ title: 'The First Witch of Boston' }).catch((err: unknown) => err);
|
||||
const searchPromise = provider.search({ title: 'The First Witch of Boston' });
|
||||
await vi.runAllTimersAsync();
|
||||
const error = await settled;
|
||||
const result = await searchPromise;
|
||||
|
||||
expect(error).toBeInstanceOf(ProviderThrottleError);
|
||||
const throttle = error as ProviderThrottleError;
|
||||
expect(throttle.partialCandidates.map((c) => c.providerId)).toEqual(['222794853', '247090873']);
|
||||
// The first book was scraped in full; the challenged one falls back to its autocomplete entry.
|
||||
expect(throttle.partialCandidates[0].title).toBe('The First Witch of Boston');
|
||||
expect(throttle.partialCandidates[1].title).toBe('The First Witch of Boston Book Two');
|
||||
expect(result.map((candidate) => candidate.providerId)).toEqual(['222794853', '247090873']);
|
||||
expect(result[0].title).toBe('The First Witch of Boston');
|
||||
expect(result[1].title).toBe('The First Witch of Boston Book Two');
|
||||
|
||||
const detailUrls = vi
|
||||
.mocked(global.fetch)
|
||||
@@ -418,6 +448,62 @@ describe('GoodreadsProvider', () => {
|
||||
await expect(provider.lookupById('123')).rejects.toBeInstanceOf(ProviderThrottleError);
|
||||
});
|
||||
|
||||
it('uses the exact autocomplete match when a stored-ID detail page is challenged', async () => {
|
||||
const autocomplete = [
|
||||
{
|
||||
bookId: '53413840',
|
||||
bookUrl: '/book/show/53413840-his-wife-s-sister',
|
||||
title: "His Wife's Sister",
|
||||
bookTitleBare: "His Wife's Sister",
|
||||
author: { name: 'A.J. Wills' },
|
||||
numPages: 331,
|
||||
},
|
||||
];
|
||||
|
||||
global.fetch = vi.fn((input: Parameters<typeof fetch>[0]) => {
|
||||
const url = fetchUrl(input);
|
||||
if (url.includes('/auto_complete')) return Promise.resolve({ ok: true, status: 200, json: () => Promise.resolve(autocomplete) });
|
||||
return Promise.resolve({ ok: true, status: 202, text: () => Promise.resolve('<div id="challenge-container"></div>') });
|
||||
}) as never;
|
||||
|
||||
const result = await provider.lookupById('53413840', undefined, {
|
||||
title: "His Wife's Sister",
|
||||
author: 'A.J. Wills',
|
||||
});
|
||||
|
||||
expect(result).toEqual(
|
||||
expect.objectContaining({
|
||||
providerId: '53413840',
|
||||
title: "His Wife's Sister",
|
||||
authors: ['A.J. Wills'],
|
||||
pageCount: 331,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('reports challenge runtime markers on a 200 response as a throttle', async () => {
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
text: () => Promise.resolve('<script>window.awsWafCookieDomainList = ["goodreads.com"];</script>'),
|
||||
}) as never;
|
||||
|
||||
await expect(provider.lookupById('123')).rejects.toBeInstanceOf(ProviderThrottleError);
|
||||
});
|
||||
|
||||
it('parses a valid book page that preloads the AWS WAF challenge script', async () => {
|
||||
const html = `${goodreadsBookHtml('249', 'Tropic of Cancer')}
|
||||
<script src="https://example.sdk.awswaf.com/example/challenge.js" data-nscript="afterInteractive"></script>`;
|
||||
global.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, text: () => Promise.resolve(html) }) as never;
|
||||
|
||||
await expect(provider.lookupById('249')).resolves.toEqual(
|
||||
expect.objectContaining({
|
||||
providerId: '249',
|
||||
title: 'Tropic of Cancer',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves the description empty when an unavailable detail page forces the truncated autocomplete blurb', async () => {
|
||||
vi.useFakeTimers();
|
||||
const autocomplete = [
|
||||
|
||||
@@ -58,16 +58,17 @@ export class GoodreadsProvider implements IdentifiableProvider {
|
||||
// run of retries push the whole search past the hard provider timeout and lose everything.
|
||||
const deadline = createSearchDeadline(PROVIDER_BUDGETS_MS.GOODREADS_SEARCH, params.signal);
|
||||
try {
|
||||
const targets = params.isbn
|
||||
? await this.findIdByIsbn(params.isbn, deadline).then((id): GoodreadsSearchTarget[] => (id ? [{ id }] : []))
|
||||
: await this.searchTargets(params, deadline);
|
||||
const targets = await this.searchTargets(params, deadline);
|
||||
|
||||
// Try the detail scrape first, since it is the only source of a full description. A page that
|
||||
// loads but fails to parse, or one that is briefly unavailable, only costs this one book.
|
||||
const results: MetadataCandidate[] = [];
|
||||
try {
|
||||
for (const target of targets.slice(0, PROVIDER_LIMITS.GOODREADS_MAX_RESULTS)) {
|
||||
const limitedTargets = targets.slice(0, PROVIDER_LIMITS.GOODREADS_MAX_RESULTS);
|
||||
for (let index = 0; index < limitedTargets.length; index++) {
|
||||
const target = limitedTargets[index];
|
||||
let candidate: MetadataCandidate | null = null;
|
||||
let autocompleteCandidate: MetadataCandidate | null = null;
|
||||
let blocked = false;
|
||||
if (!deadline.expired()) {
|
||||
if (results.length > 0) await sleep(PROVIDER_DELAYS_MS.GOODREADS_BETWEEN_REQUESTS, deadline.signal).catch(() => undefined);
|
||||
@@ -78,11 +79,22 @@ export class GoodreadsProvider implements IdentifiableProvider {
|
||||
blocked = detail.outcome === 'blocked';
|
||||
}
|
||||
}
|
||||
if (!candidate && target.item) candidate = mapGoodreadsAutocompleteItem(target.item, target.id);
|
||||
if (!candidate && target.item) {
|
||||
autocompleteCandidate = mapGoodreadsAutocompleteItem(target.item, target.id);
|
||||
candidate = autocompleteCandidate;
|
||||
}
|
||||
if (candidate) results.push(candidate);
|
||||
// The challenge gates every page from here on, but the book it hit still has its
|
||||
// autocomplete candidate, and the books before it are already scraped in full.
|
||||
if (blocked) throw blockedError();
|
||||
if (blocked) {
|
||||
// Goodreads keeps autocomplete available while WAF gates HTML pages. Keep the source
|
||||
// useful with summary candidates instead of putting the whole provider in cooldown.
|
||||
if (!autocompleteCandidate) throw blockedError();
|
||||
for (const remaining of limitedTargets.slice(index + 1)) {
|
||||
if (!remaining.item) continue;
|
||||
const fallback = mapGoodreadsAutocompleteItem(remaining.item, remaining.id);
|
||||
if (fallback) results.push(fallback);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
rethrowWithPartialCandidates(err, results);
|
||||
@@ -94,14 +106,19 @@ export class GoodreadsProvider implements IdentifiableProvider {
|
||||
}
|
||||
}
|
||||
|
||||
async lookupById(providerId: string, signal?: AbortSignal): Promise<MetadataCandidate | null> {
|
||||
async lookupById(providerId: string, signal?: AbortSignal, params?: MetadataSearchParams): Promise<MetadataCandidate | null> {
|
||||
const { enabled } = await this.providerConfig.getConfig().then((c) => c.goodreads);
|
||||
if (!enabled) return null;
|
||||
|
||||
const deadline = createSearchDeadline(PROVIDER_BUDGETS_MS.GOODREADS_SEARCH, signal);
|
||||
try {
|
||||
const { candidate, outcome } = await this.fetchBook(providerId, deadline);
|
||||
if (outcome === 'blocked') throw blockedError();
|
||||
if (outcome === 'blocked') {
|
||||
if (!params) throw blockedError();
|
||||
const targets = await this.searchAutocompleteTargets(params, deadline);
|
||||
const matchingItem = targets.find((target) => target.id === providerId)?.item;
|
||||
return matchingItem ? mapGoodreadsAutocompleteItem(matchingItem, providerId) : null;
|
||||
}
|
||||
return candidate;
|
||||
} finally {
|
||||
deadline.dispose();
|
||||
@@ -109,11 +126,27 @@ export class GoodreadsProvider implements IdentifiableProvider {
|
||||
}
|
||||
|
||||
private async searchTargets(params: MetadataSearchParams, deadline: SearchDeadline): Promise<GoodreadsSearchTarget[]> {
|
||||
const autocompleteTargets = await this.searchAutocompleteTargets(params, deadline);
|
||||
if (autocompleteTargets.length > 0) return autocompleteTargets;
|
||||
if (deadline.expired()) return [];
|
||||
|
||||
if (params.isbn) {
|
||||
const id = await this.findIdByIsbn(params.isbn, deadline);
|
||||
return id ? [{ id }] : [];
|
||||
}
|
||||
|
||||
const query = [params.title, params.author].filter(Boolean).join(' ');
|
||||
if (!query.trim()) return [];
|
||||
|
||||
// Goodreads search pages are frequently gated behind WAF challenges.
|
||||
// Prefer the JSON autocomplete endpoint, then fall back to HTML scraping.
|
||||
const searchUrl = `https://www.goodreads.com/search?q=${encodeURIComponent(query)}&search_type=books`;
|
||||
const { html } = await this.fetchHtml(searchUrl, 'search', { query, deadline });
|
||||
const ids = html ? extractBookIds(html, params.title, PROVIDER_LIMITS.GOODREADS_MAX_RESULTS) : [];
|
||||
return ids.map((id) => ({ id }));
|
||||
}
|
||||
|
||||
private async searchAutocompleteTargets(params: MetadataSearchParams, deadline: SearchDeadline): Promise<GoodreadsSearchTarget[]> {
|
||||
const autocompleteItems: GoodreadsAutocompleteItem[] = [];
|
||||
for (const autocompleteQuery of buildAutocompleteQueries(params)) {
|
||||
if (deadline.expired()) break;
|
||||
@@ -127,14 +160,7 @@ export class GoodreadsProvider implements IdentifiableProvider {
|
||||
);
|
||||
if (Array.isArray(autocomplete)) autocompleteItems.push(...autocomplete);
|
||||
}
|
||||
const autocompleteTargets = rankAutocompleteItems(autocompleteItems, params, PROVIDER_LIMITS.GOODREADS_MAX_RESULTS);
|
||||
if (autocompleteTargets.length > 0) return autocompleteTargets;
|
||||
if (deadline.expired()) return [];
|
||||
|
||||
const searchUrl = `https://www.goodreads.com/search?q=${encodeURIComponent(query)}&search_type=books`;
|
||||
const { html } = await this.fetchHtml(searchUrl, 'search', { query, deadline });
|
||||
const ids = html ? extractBookIds(html, params.title, PROVIDER_LIMITS.GOODREADS_MAX_RESULTS) : [];
|
||||
return ids.map((id) => ({ id }));
|
||||
return rankAutocompleteItems(autocompleteItems, params, PROVIDER_LIMITS.GOODREADS_MAX_RESULTS);
|
||||
}
|
||||
|
||||
private async findIdByIsbn(isbn: string, deadline: SearchDeadline): Promise<string | null> {
|
||||
@@ -274,7 +300,7 @@ function extractNextData(html: string): GoodreadsNextData | null {
|
||||
// AWS WAF serves a JavaScript token challenge with a 202 status instead of the
|
||||
// real page. A plain fetch cannot solve it, so treat it as a failed fetch.
|
||||
function isWafChallenge(status: number, html: string): boolean {
|
||||
return status === 202 || /awsWafCookieDomainList|AwsWafIntegration|id="challenge-container"|challenge\.js/.test(html);
|
||||
return status === 202 || /awsWafCookieDomainList|AwsWafIntegration|id=["']challenge-container["']/.test(html);
|
||||
}
|
||||
|
||||
// Goodreads sheds load with a bare 503 and no Retry-After. 429 never reaches here: fetchWithThrottle
|
||||
@@ -297,6 +323,8 @@ function blockedError(): ProviderThrottleError {
|
||||
}
|
||||
|
||||
function buildAutocompleteQueries(params: MetadataSearchParams): string[] {
|
||||
const isbn = params.isbn?.trim();
|
||||
if (isbn) return [isbn];
|
||||
const title = params.title?.trim();
|
||||
const author = params.author?.trim();
|
||||
const queries = title ? [title, [title, author].filter(Boolean).join(' ')] : [author ?? ''];
|
||||
|
||||
Reference in New Issue
Block a user