Migrations/bookorbit
1
0
Fork 0
mirror of https://github.com/bookorbit/bookorbit.git synced 2026-11-03 00:11:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(metadata): restore Amazon metadata search

Browse Source 
Use Amazon's readable mobile search route because the desktop route now returns an Akamai interstitial. Detect bot challenges and ignore sponsored cards before detail lookups.
This commit is contained in:
neon
2026-09-02 02:30:59 +00:00
parent 5346414a70
commit 6edde6c64d
7 changed files with 102 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/common/utils/amazon-http.utils.ts
+35
View File
@@ -0,0 +1,35 @@
const AMAZON_BOT_CHALLENGE_PATTERNS = [
/validateCaptcha/i,
/captcha/i,
/not a robot/i,
/triggerInterstitialChallenge/i,
/["']bm-verify["']/i,
/\/_sec\/verify\?provider=interstitial/i,
];
const AMAZON_BROWSER_HEADERS: HeadersInit = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.36',
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'accept-language': 'en-US,en;q=0.9',
dnt: '1',
'upgrade-insecure-requests': '1',
'sec-ch-ua': '"Not=A?Brand";v="99", "Google Chrome";v="151", "Chromium";v="151"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'same-origin',
};
export function amazonRequestHeaders(url: string | URL, cookie = ''): HeadersInit {
const origin = new URL(url).origin;
return {
...AMAZON_BROWSER_HEADERS,
referer: `${origin}/`,
...(cookie ? { cookie } : {}),
};
}
export function isAmazonBotChallenge(body: string): boolean {
return AMAZON_BOT_CHALLENGE_PATTERNS.some((pattern) => pattern.test(body));
}
server/src/modules/metadata-fetch/providers/amazon/amazon.provider.test.ts
+19 -2
View File
@@ -76,7 +76,7 @@ describe('AmazonProvider', () => {
const result = await provider.search({ title: 'Test Book' });
expect(global.fetch).toHaveBeenCalledWith(
expect.stringContaining('https://www.amazon.com/s?k=Test%20Book'),
expect.stringContaining('https://www.amazon.com/gp/aw/s?k=Test%20Book'),
expect.objectContaining({
headers: expect.objectContaining({ cookie: 'test-cookie' }),
}),
@@ -106,7 +106,7 @@ describe('AmazonProvider', () => {
expect(global.fetch).toHaveBeenNthCalledWith(
1,
expect.stringContaining('https://www.amazon.com/s?k=9781250165343'),
expect.stringContaining('https://www.amazon.com/gp/aw/s?k=9781250165343'),
expect.objectContaining({
headers: expect.objectContaining({ cookie: 'test-cookie' }),
}),
@@ -184,6 +184,23 @@ describe('AmazonProvider', () => {
await expect(provider.search({ title: 'Test' })).rejects.toBeInstanceOf(ProviderThrottleError);
});
it('should treat Amazon interstitial responses as throttling', async () => {
global.fetch = vi.fn().mockResolvedValue({
ok: true,
status: 200,
text: () =>
Promise.resolve(
'<script>function triggerInterstitialChallenge() { xhr.open("POST", "/_sec/verify?provider=interstitial"); xhr.send(JSON.stringify({"bm-verify":"token"})); }</script>',
),
});
await expect(provider.search({ title: 'Test' })).rejects.toMatchObject({
name: 'ProviderThrottleError',
reason: 'bot challenge',
});
expect(global.fetch).toHaveBeenCalledTimes(1);
});
it('should return empty if no title in parsed page', async () => {
const searchHtml = `<div data-component-type="s-search-result" data-asin="ASIN000001"></div>`;
const bookHtml = `<div>No title</div>`;
server/src/modules/metadata-fetch/providers/amazon/amazon.provider.ts
+6 -14
View File
@@ -2,6 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
import { MetadataCandidate, MetadataProviderKey, parseSeriesIndex } from '@bookorbit/types';
import { ProviderConfigService } from '../../../metadata-preferences/provider-config.service';
import { amazonRequestHeaders, isAmazonBotChallenge } from '../../../../common/utils/amazon-http.utils';
import { sanitizeLogValue } from '../../../../common/utils/log-sanitize.utils';
import { amazonOrigin } from '../../../../common/utils/metadata-provider-hosts.utils';
import { fetchWithThrottle } from '../../fetch-with-throttle';
@@ -12,18 +13,6 @@ import { MetadataSearchParams } from '../metadata-search-params';
import { buildRequestSignal, normalizeMaxCandidates, rethrowWithPartialCandidates, sleep } from '../provider-utils';
import { extractAsins, parseBookPage } from './amazon.scraper';
const HEADERS: HeadersInit = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua': '"Google Chrome";v="137", "Chromium";v="137", "Not_A Brand";v="24"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
};
@Injectable()
export class AmazonProvider implements IdentifiableProvider {
readonly key = MetadataProviderKey.AMAZON;
@@ -65,7 +54,7 @@ export class AmazonProvider implements IdentifiableProvider {
private async searchAsins(params: MetadataSearchParams, domain: string, cookie: string, limit: number, signal?: AbortSignal): Promise<string[]> {
const query = params.isbn?.trim() || [params.title, params.author].filter(Boolean).join(' ');
if (!query) return [];
const url = `${amazonOrigin(domain)}/s?k=${encodeURIComponent(query)}&i=stripbooks`;
const url = `${amazonOrigin(domain)}/gp/aw/s?k=${encodeURIComponent(query)}&i=stripbooks`;
const html = await this.fetchHtml(url, cookie, 'search', query, undefined, signal);
return html ? extractAsins(html, limit) : [];
}
@@ -109,7 +98,7 @@ export class AmazonProvider implements IdentifiableProvider {
providerId?: string,
signal?: AbortSignal,
): Promise<string | null> {
const headers: HeadersInit = cookie ? { ...HEADERS, cookie } : HEADERS;
const headers = amazonRequestHeaders(url, cookie);
const startedAt = Date.now();
const safeQuery = query ? sanitizeLogValue(query) : undefined;
const safeProviderId = providerId ? sanitizeLogValue(providerId) : undefined;
@@ -123,6 +112,9 @@ export class AmazonProvider implements IdentifiableProvider {
return null;
}
const html = await res.text();
if (isAmazonBotChallenge(html)) {
throw new ProviderThrottleError(undefined, 'bot challenge');
}
this.logger.log(
`[amazon] [end] op=${op}${safeQuery ? ` query="${safeQuery}"` : ''}${safeProviderId ? ` providerId="${safeProviderId}"` : ''} status=${res.status} durationMs=${Date.now() - startedAt}`,
);
server/src/modules/metadata-fetch/providers/amazon/amazon.scraper.test.ts
+15
View File
@@ -41,6 +41,21 @@ describe('AmazonScraper', () => {
const result = extractAsins(html, 5);
expect(result).toEqual(['GOOD123456']);
});
it('should skip sponsored cards before following unrelated format links', () => {
const html = `
<div data-component-type="s-search-result" data-asin="SPONSOR001">
<div data-cy="title-recipe">Sponsored Sponsored result</div>
<a href="/dp/SERIES0001">Book 1 of 3: A Series</a>
</div>
<div data-component-type="s-search-result" data-asin="B123456789">
<div data-cy="title-recipe">Wanted Book</div>
<a href="/dp/B123456789">Kindle</a>
</div>
`;
expect(extractAsins(html, 5)).toEqual(['B123456789']);
});
});
describe('parseBookPage', () => {
server/src/modules/metadata-fetch/providers/amazon/amazon.scraper.ts
+2 -1
View File
@@ -64,7 +64,8 @@ export function extractAsins(html: string, limit: number): string[] {
$('div[data-component-type="s-search-result"]').each((_, el) => {
if (results.length >= limit) return false;
const titleText = $(el).find('[data-cy=title-recipe]').text().toLowerCase();
const titleText = $(el).find('[data-cy=title-recipe]').text().trim().toLowerCase();
if (titleText.startsWith('sponsored')) return;
if (SKIP_TITLE_PATTERNS.test(titleText)) return;
// Collect all /dp/ASIN links with their visible label within this result block.
server/src/modules/metadata-preferences/provider-config.service.test.ts
+21
View File
@@ -488,6 +488,27 @@ describe('ProviderConfigService', () => {
expect(result.message).toContain('bot-check');
});
it('returns warning for Amazon interstitial challenge responses', async () => {
db.query.appSettings.findFirst.mockResolvedValue(undefined);
fetchMock.mockResolvedValue(
new Response(
'<script>function triggerInterstitialChallenge() { xhr.open("POST", "/_sec/verify?provider=interstitial"); xhr.send(JSON.stringify({"bm-verify":"token"})); }</script>',
{ status: 200 },
),
);
const result = await service.testProvider(MetadataProviderKey.AMAZON, {
amazon: { cookie: 'session-cookie' },
});
expect(result).toEqual({
key: MetadataProviderKey.AMAZON,
ok: false,
status: 'warning',
message: 'Amazon responded with a bot-check page. Use a fresh browser session cookie.',
});
});
it('falls back to the allowlisted Amazon origin for an untrusted domain', async () => {
db.query.appSettings.findFirst.mockResolvedValue(undefined);
fetchMock.mockResolvedValue(new Response('<html>Amazon results</html>', { status: 200 }));
server/src/modules/metadata-preferences/provider-config.service.ts
+4 -17
View File
@@ -4,6 +4,7 @@ import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { MetadataProviderKey, ProviderConfigurations, ProviderConnectionTestResult, ProviderStatus } from '@bookorbit/types';
import { stripBearerPrefix, toBearerAuthorization } from '../../common/utils/bearer-token.utils';
import { amazonRequestHeaders, isAmazonBotChallenge } from '../../common/utils/amazon-http.utils';
import { sanitizeLogValue } from '../../common/utils/log-sanitize.utils';
import { amazonOrigin, normalizeAmazonDomain, normalizeAudibleDomain } from '../../common/utils/metadata-provider-hosts.utils';
import { DB } from '../../db';
@@ -20,19 +21,6 @@ const HARDCOVER_GRAPHQL_URL = 'https://api.hardcover.app/v1/graphql';
const HARDCOVER_TEST_QUERY = 'query { me { username } }';
const AMAZON_TEST_QUERY = 'books';
const PROVIDER_TEST_TIMEOUT_MS = 10_000;
const AMAZON_CAPTCHA_PATTERNS = [/validateCaptcha/i, /captcha/i, /not a robot/i];
const AMAZON_TEST_HEADERS: HeadersInit = {
'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'accept-language': 'en-US,en;q=0.9',
'sec-ch-ua': '"Google Chrome";v="137", "Chromium";v="137", "Not_A Brand";v="24"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"macOS"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
};
const DEFAULT_CONFIG: ProviderConfigurations = {
google: { enabled: false, apiKey: '' },
amazon: { enabled: true, domain: 'amazon.com', cookie: '' },
@@ -522,10 +510,10 @@ export class ProviderConfigService {
private async testAmazonProvider(config: ProviderConfigurations['amazon']): Promise<ProviderConnectionTestResult> {
const domain = normalizeAmazonDomain(config.domain);
const cookie = this.normalizeAmazonCookie(config.cookie);
const url = new URL('/s', amazonOrigin(domain));
const url = new URL('/gp/aw/s', amazonOrigin(domain));
url.searchParams.set('k', AMAZON_TEST_QUERY);
url.searchParams.set('i', 'stripbooks');
const headers: HeadersInit = cookie ? { ...AMAZON_TEST_HEADERS, cookie } : AMAZON_TEST_HEADERS;
const headers = amazonRequestHeaders(url, cookie);
const response = await fetch(url, { method: 'GET', headers, signal: AbortSignal.timeout(PROVIDER_TEST_TIMEOUT_MS) });
if (!response.ok) {
@@ -538,8 +526,7 @@ export class ProviderConfigService {
}
const body = await response.text();
const captchaDetected = AMAZON_CAPTCHA_PATTERNS.some((pattern) => pattern.test(body));
if (captchaDetected) {
if (isAmazonBotChallenge(body)) {
return {
key: MetadataProviderKey.AMAZON,
ok: false,