From ed3c36e35d21413a6d13719edb62e743b4a722ff Mon Sep 17 00:00:00 2001 From: neon <18559658+neonsolstice@users.noreply.github.com> Date: Thu, 19 Feb 2026 02:17:50 -0600 Subject: [PATCH] fix(metadata): fix Amazon and Goodreads scraper bugs --- server/package.json | 7 +- server/src/db/migrations/meta/_journal.json | 2 +- .../metadata-fetch/metadata-fetch.module.ts | 3 +- .../providers/amazon/amazon.provider.ts | 100 +++++++++ .../providers/amazon/amazon.scraper.ts | 202 ++++++++++++++++++ .../providers/goodreads/goodreads.mapper.ts | 142 ++++++------ .../providers/goodreads/goodreads.types.ts | 48 +---- 7 files changed, 380 insertions(+), 124 deletions(-) create mode 100644 server/src/modules/metadata-fetch/providers/amazon/amazon.provider.ts create mode 100644 server/src/modules/metadata-fetch/providers/amazon/amazon.scraper.ts diff --git a/server/package.json b/server/package.json index cd5dc09e2..884fde932 100644 --- a/server/package.json +++ b/server/package.json @@ -36,13 +36,14 @@ "@nestjs/core": "^11.0.1", "@nestjs/jwt": "^11.0.2", "@nestjs/passport": "^11.0.5", - "@nestjs/platform-fastify": "^11.1.13", - "@nestjs/platform-socket.io": "^11.1.13", + "@nestjs/platform-fastify": "^11.1.14", + "@nestjs/platform-socket.io": "^11.1.14", "@nestjs/schedule": "^6.1.1", - "@nestjs/websockets": "^11.1.13", + "@nestjs/websockets": "^11.1.14", "@parcel/watcher": "^2.5.6", "@projectx/types": "workspace:*", "bcryptjs": "^3.0.3", + "cheerio": "^1.2.0", "class-transformer": "^0.5.1", "class-validator": "^0.14.3", "drizzle-orm": "^0.45.1", diff --git a/server/src/db/migrations/meta/_journal.json b/server/src/db/migrations/meta/_journal.json index 4cf060a29..2a8314fc5 100644 --- a/server/src/db/migrations/meta/_journal.json +++ b/server/src/db/migrations/meta/_journal.json @@ -5,7 +5,7 @@ { "idx": 0, "version": "7", - "when": 1771534544614, + "when": 1771535401494, "tag": "0000_add_provider_ids", "breakpoints": true } diff --git a/server/src/modules/metadata-fetch/metadata-fetch.module.ts b/server/src/modules/metadata-fetch/metadata-fetch.module.ts index 09c7ae06c..be5b84bf9 100644 --- a/server/src/modules/metadata-fetch/metadata-fetch.module.ts +++ b/server/src/modules/metadata-fetch/metadata-fetch.module.ts @@ -5,11 +5,12 @@ import { MetadataFetchController } from './metadata-fetch.controller'; import { MetadataProvider } from './providers/metadata-provider'; import { MetadataFetchService } from './metadata-fetch.service'; import { ProviderRegistry } from './provider-registry'; +import { AmazonProvider } from './providers/amazon/amazon.provider'; import { GoogleProvider } from './providers/google/google.provider'; import { GoodreadsProvider } from './providers/goodreads/goodreads.provider'; import { OpenLibraryProvider } from './providers/open-library/open-library.provider'; -const PROVIDER_CLASSES = [GoogleProvider, GoodreadsProvider, OpenLibraryProvider]; +const PROVIDER_CLASSES = [GoogleProvider, GoodreadsProvider, AmazonProvider, OpenLibraryProvider]; @Module({ providers: [ diff --git a/server/src/modules/metadata-fetch/providers/amazon/amazon.provider.ts b/server/src/modules/metadata-fetch/providers/amazon/amazon.provider.ts new file mode 100644 index 000000000..fc187149e --- /dev/null +++ b/server/src/modules/metadata-fetch/providers/amazon/amazon.provider.ts @@ -0,0 +1,100 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { MetadataCandidate, MetadataProviderKey } from '@projectx/types'; + +import { IdentifiableProvider } from '../metadata-provider'; +import { MetadataSearchParams } from '../metadata-search-params'; +import { extractAsins, parseBookPage } from './amazon.scraper'; + +const HEADERS: HeadersInit = { + 'user-agent': + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36', + accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'accept-language': 'en-US,en;q=0.9', + 'sec-ch-ua': '"Google Chrome";v="137", "Chromium";v="137", "Not_A Brand";v="24"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"macOS"', + 'sec-fetch-dest': 'document', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', +}; + +const MAX_RESULTS = 3; +const BETWEEN_REQUESTS_MS = 800; + +@Injectable() +export class AmazonProvider implements IdentifiableProvider { + readonly key = MetadataProviderKey.AMAZON; + readonly label = 'Amazon'; + readonly identifiable = true as const; + + private readonly logger = new Logger(AmazonProvider.name); + + async search(params: MetadataSearchParams): Promise { + const asins = params.isbn + ? [params.isbn] // try ASIN directly if ISBN provided + : await this.searchAsins(params); + + const results: MetadataCandidate[] = []; + for (const asin of asins.slice(0, MAX_RESULTS)) { + if (results.length > 0) await sleep(BETWEEN_REQUESTS_MS); + const candidate = await this.fetchByAsin(asin); + if (candidate) results.push(candidate); + } + return results; + } + + async lookupById(providerId: string): Promise { + return this.fetchByAsin(providerId); + } + + private async searchAsins(params: MetadataSearchParams): Promise { + const query = [params.title, params.author].filter(Boolean).join(' '); + const url = `https://www.amazon.com/s?k=${encodeURIComponent(query)}&i=stripbooks`; + const html = await this.fetchHtml(url); + return html ? extractAsins(html, MAX_RESULTS) : []; + } + + private async fetchByAsin(asin: string): Promise { + const html = await this.fetchHtml(`https://www.amazon.com/dp/${asin}`); + if (!html) return null; + const data = parseBookPage(html); + if (!data.title) return null; + return { + provider: MetadataProviderKey.AMAZON, + providerId: asin, + title: data.title, + subtitle: data.subtitle, + authors: data.authors?.length ? data.authors : undefined, + description: data.description, + isbn13: data.isbn13, + isbn10: data.isbn10, + publisher: data.publisher, + publishedYear: data.publishedYear, + language: data.language, + pageCount: data.pageCount, + seriesName: data.seriesName, + seriesIndex: data.seriesIndex, + coverUrl: data.coverUrl, + tags: data.tags?.length ? data.tags : undefined, + sourceUrl: `https://www.amazon.com/dp/${asin}`, + }; + } + + private async fetchHtml(url: string): Promise { + try { + const res = await fetch(url, { headers: HEADERS }); + if (!res.ok) { + this.logger.warn(`Amazon returned ${res.status} for ${url}`); + return null; + } + return res.text(); + } catch (err) { + this.logger.warn(`Amazon fetch failed for ${url}: ${err}`); + return null; + } + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/server/src/modules/metadata-fetch/providers/amazon/amazon.scraper.ts b/server/src/modules/metadata-fetch/providers/amazon/amazon.scraper.ts new file mode 100644 index 000000000..001b9fae0 --- /dev/null +++ b/server/src/modules/metadata-fetch/providers/amazon/amazon.scraper.ts @@ -0,0 +1,202 @@ +import * as cheerio from 'cheerio'; +import type { CheerioAPI } from 'cheerio'; + +export interface AmazonBookData { + title?: string; + subtitle?: string; + authors?: string[]; + description?: string; + isbn13?: string; + isbn10?: string; + publisher?: string; + publishedYear?: number; + language?: string; + pageCount?: number; + seriesName?: string; + seriesIndex?: number; + coverUrl?: string; + tags?: string[]; +} + +const SKIP_TITLE_PATTERNS = /box\s*set|collection\s*set|books\s*set|omnibus|summary\s*&\s*study/i; + +export function parseBookPage(html: string): AmazonBookData { + const $ = cheerio.load(html); + const { title, subtitle } = extractTitle($); + return { + title, + subtitle, + authors: extractAuthors($), + description: extractDescription($), + isbn13: extractIsbn($, 'isbn13'), + isbn10: extractIsbn($, 'isbn10'), + publisher: extractPublisher($), + publishedYear: extractPublishedYear($), + language: extractLanguage($), + pageCount: extractPageCount($), + seriesName: extractSeriesName($), + seriesIndex: extractSeriesIndex($), + coverUrl: extractCoverUrl($), + tags: extractCategories($), + }; +} + +export function extractAsins(html: string, limit: number): string[] { + const $ = cheerio.load(html); + const seen = new Set(); + const asins: string[] = []; + + $('div[data-asin]').each((_, el) => { + const asin = $(el).attr('data-asin'); + if (!asin || asin.length !== 10 || seen.has(asin)) return; + const titleText = $(el).find('[data-cy=title-recipe]').text().toLowerCase(); + if (SKIP_TITLE_PATTERNS.test(titleText)) return; + seen.add(asin); + asins.push(asin); + if (asins.length >= limit) return false; + }); + + return asins; +} + +// --- Field extractors --- + +function extractTitle($: CheerioAPI): { title?: string; subtitle?: string } { + const raw = $('#productTitle, #ebooksProductTitle').first().text().trim(); + if (!raw) return {}; + const colon = raw.indexOf(':'); + if (colon > 0) { + return { title: raw.substring(0, colon).trim(), subtitle: raw.substring(colon + 1).trim() }; + } + return { title: raw }; +} + +function extractAuthors($: CheerioAPI): string[] { + const authors: string[] = []; + const seen = new Set(); + $('#bylineInfo .author a, #bylineInfo_feature_div .author a').each((_, el) => { + const name = $(el).text().trim(); + if (name && !seen.has(name)) { + seen.add(name); + authors.push(name); + } + }); + return authors; +} + +function extractDescription($: CheerioAPI): string | undefined { + const expander = $('#bookDescription_feature_div .a-expander-content').first(); + if (expander.length) return expander.text().trim() || undefined; + + const noscript = $('#bookDescription_feature_div noscript').first(); + if (noscript.length) { + const inner = cheerio.load(noscript.html() ?? ''); + return inner.root().text().trim() || undefined; + } + return undefined; +} + +function extractIsbn($: CheerioAPI, field: 'isbn13' | 'isbn10'): string | undefined { + // Newer Amazon layout: rpi-attribute panels + const rpi = $(`#rpi-attribute-book_details-${field} .rpi-attribute-value span`).first().text().trim(); + if (rpi) return rpi.replace(/[^0-9X]/gi, '') || undefined; + + // Older layout: detail bullets + const label = field === 'isbn13' ? /isbn-?13/i : /isbn-?10/i; + const value = detailBulletValue($, label); + return value ? value.replace(/[^0-9X]/gi, '') || undefined : undefined; +} + +function extractPublisher($: CheerioAPI): string | undefined { + // Newer layout + const rpi = $('#rpi-attribute-book_details-publisher .rpi-attribute-value span').first().text().trim(); + if (rpi) return rpi || undefined; + + // Older layout: "Publisher : Penguin (Jan 1, 2020)" + const raw = detailBulletValue($, /publisher/i); + if (!raw) return undefined; + // Strip date in parentheses at the end + return raw.replace(/\s*\([^)]*\)\s*$/, '').trim() || undefined; +} + +function extractPublishedYear($: CheerioAPI): number | undefined { + // Newer layout + const rpiText = $('#rpi-attribute-book_details-publication_date .rpi-attribute-value span').first().text().trim(); + if (rpiText) { + const y = parseYearFromText(rpiText); + if (y) return y; + } + + // Older layout: extract year from parenthesised date in publisher bullet + const publisherBullet = detailBulletValue($, /publisher/i); + const match = publisherBullet.match(/\(([^)]+)\)/); + if (match) { + const y = parseYearFromText(match[1]); + if (y) return y; + } + return undefined; +} + +function extractLanguage($: CheerioAPI): string | undefined { + const rpi = $('#rpi-attribute-language .rpi-attribute-value span').first().text().trim(); + if (rpi) return rpi || undefined; + return detailBulletValue($, /language/i) || undefined; +} + +function extractPageCount($: CheerioAPI): number | undefined { + const rpi = $('#rpi-attribute-book_details-fiona_pages .rpi-attribute-value span').first().text().trim(); + const text = rpi || detailBulletValue($, /print length|pages/i); + const digits = text.match(/\d+/)?.[0]; + return digits ? parseInt(digits, 10) : undefined; +} + +function extractSeriesName($: CheerioAPI): string | undefined { + return ( + $('#rpi-attribute-book_details-series .rpi-attribute-value a span').first().text().trim() || + undefined + ); +} + +function extractSeriesIndex($: CheerioAPI): number | undefined { + const label = $('#rpi-attribute-book_details-series .rpi-attribute-label span').first().text(); + const match = label.match(/book\s+(\d+(?:\.\d+)?)\s+of/i); + if (!match) return undefined; + const n = parseFloat(match[1]); + return Number.isNaN(n) ? undefined : n; +} + +function extractCoverUrl($: CheerioAPI): string | undefined { + const img = $('#landingImage, #imgBlkFront').first(); + const hires = img.attr('data-old-hires'); + if (hires) return hires; + const src = img.attr('src'); + // Strip Amazon's dimension modifier (e.g. ._SX260_) to get the original image + return src ? src.replace(/\._[A-Z0-9_,]+_\./i, '.') : undefined; +} + +function extractCategories($: CheerioAPI): string[] { + const cats: string[] = []; + $('#detailBullets_feature_div .zg_hrsr .a-list-item a').each((_, el) => { + const name = $(el).text().trim().replace(/\s*\(Books\)\s*$/i, ''); + if (name) cats.push(name); + }); + return cats; +} + +// --- Helpers --- + +function detailBulletValue($: CheerioAPI, labelPattern: RegExp): string { + let value = ''; + $('#detailBullets_feature_div .a-text-bold').each((_, el) => { + if (labelPattern.test($(el).text())) { + value = $(el).next('span').text().trim(); + return false; // break + } + }); + return value; +} + +function parseYearFromText(text: string): number | undefined { + const match = text.match(/\b(1[0-9]{3}|2[0-9]{3})\b/); + return match ? parseInt(match[1], 10) : undefined; +} diff --git a/server/src/modules/metadata-fetch/providers/goodreads/goodreads.mapper.ts b/server/src/modules/metadata-fetch/providers/goodreads/goodreads.mapper.ts index 0ad6a22aa..2ff88f5be 100644 --- a/server/src/modules/metadata-fetch/providers/goodreads/goodreads.mapper.ts +++ b/server/src/modules/metadata-fetch/providers/goodreads/goodreads.mapper.ts @@ -2,110 +2,90 @@ import { MetadataCandidate, MetadataProviderKey } from '@projectx/types'; import { GoodreadsApolloBook, - GoodreadsApolloBookDetails, - GoodreadsApolloBookSeries, GoodreadsApolloContributor, - GoodreadsApolloContributorEdge, - GoodreadsApolloGenre, GoodreadsApolloSeries, - GoodreadsApolloTruncatedHtml, - Ref, } from './goodreads.types'; -function deref(state: Record, ref: Ref | unknown): T | undefined { - if (!ref || typeof ref !== 'object') return undefined; - const r = ref as Ref; - if (!r.__ref) return undefined; - return state[r.__ref] as T | undefined; -} - -function stripHtml(html: string): string { - return html - .replace(//gi, '\n') - .replace(/<[^>]+>/g, '') - .replace(/&/g, '&') - .replace(/</g, '<') - .replace(/>/g, '>') - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/ /g, ' ') - .trim(); -} - export function mapGoodreadsApolloState( state: Record, bookId: string, ): MetadataCandidate | null { - const bookKey = Object.keys(state).find((k) => k.startsWith('Book:')); - if (!bookKey) return null; - const book = state[bookKey] as GoodreadsApolloBook; + const book = findByKeyPrefix(state, 'Book:kca:'); + if (!book?.title) return null; - const details = deref(state, book.details); + const contributor = findByKeyPrefix(state, 'Contributor:kca'); + const series = findByKeyPrefix(state, 'Series:kca'); - const authors: string[] = []; - const primaryEdge = deref(state, book.primaryContributorEdge); - const primaryAuthor = deref(state, primaryEdge?.node); - if (primaryAuthor?.name) authors.push(primaryAuthor.name); - for (const edgeRef of book.secondaryContributorEdges ?? []) { - const edge = deref(state, edgeRef); - if (edge?.role?.toLowerCase() === 'author') { - const author = deref(state, edge.node); - if (author?.name) authors.push(author.name); - } - } + const details = book.details; + const firstSeries = book.bookSeries?.[0]; - const descNode = deref(state, book.description); - const descHtml = descNode?.fullContent ?? descNode?.truncatedContent ?? ''; - const description = descHtml ? stripHtml(descHtml) : undefined; + const tags = (book.bookGenres ?? []) + .map((g) => g.genre?.name) + .filter((n): n is string => !!n); - const tags: string[] = []; - for (const entry of book.bookGenres ?? []) { - const genre = deref(state, entry.genre); - if (genre?.name) tags.push(genre.name); - } + const { title, subtitle } = splitTitle(book.title); - let seriesName: string | undefined; - let seriesIndex: number | undefined; - for (const bsRef of book.bookSeries ?? []) { - const bs = deref(state, bsRef); - const series = deref(state, bs?.series); - if (series?.title) { - seriesName = series.title; - if (bs?.userPosition) { - const idx = parseFloat(bs.userPosition); - if (!Number.isNaN(idx)) seriesIndex = idx; - } - break; - } - } - - let publishedYear: number | undefined; - if (details?.publicationTime) { - publishedYear = new Date(details.publicationTime).getFullYear(); - } - - const fullTitle = book.title ?? ''; - const colonIdx = fullTitle.indexOf(':'); - const title = colonIdx > 0 ? fullTitle.substring(0, colonIdx).trim() : fullTitle; - const subtitle = colonIdx > 0 ? fullTitle.substring(colonIdx + 1).trim() : undefined; + const publishedYear = parseEpochYear(details?.publicationTime); + const pageCount = parsePositiveInt(details?.numPages); + const seriesIndex = parseSeriesIndex(firstSeries?.userPosition); return { provider: MetadataProviderKey.GOODREADS, providerId: bookId, title, subtitle, - authors: authors.length ? authors : undefined, - description, - publisher: details?.publisher, + authors: contributor?.name ? [contributor.name] : undefined, + description: normalize(book.description), + publisher: normalize(details?.publisher), publishedYear, - language: details?.language?.name, - pageCount: details?.numPages, - isbn10: details?.isbn, - isbn13: details?.isbn13, + language: normalize(details?.language?.name), + pageCount, + isbn10: normalize(details?.isbn), + isbn13: normalize(details?.isbn13), tags: tags.length ? tags : undefined, coverUrl: book.imageUrl, sourceUrl: `https://www.goodreads.com/book/show/${bookId}`, - seriesName, + seriesName: normalize(series?.title), seriesIndex, }; } + +function findByKeyPrefix(state: Record, prefix: string): T | undefined { + const key = Object.keys(state).find((k) => k.includes(prefix)); + return key ? (state[key] as T) : undefined; +} + +function splitTitle(fullTitle: string): { title: string; subtitle?: string } { + const colon = fullTitle.indexOf(':'); + if (colon > 0) { + return { + title: fullTitle.substring(0, colon).trim(), + subtitle: fullTitle.substring(colon + 1).trim(), + }; + } + return { title: fullTitle }; +} + +function normalize(value: string | undefined | null): string | undefined { + if (!value || value === 'null') return undefined; + return value.trim() || undefined; +} + +function parseEpochYear(value: string | number | undefined): number | undefined { + if (value == null) return undefined; + const ms = typeof value === 'string' ? parseFloat(value) : value; + if (!ms || Number.isNaN(ms)) return undefined; + return new Date(ms).getFullYear(); +} + +function parsePositiveInt(value: string | number | undefined): number | undefined { + if (value == null) return undefined; + const n = typeof value === 'string' ? parseInt(value, 10) : Math.round(value); + return n > 0 && !Number.isNaN(n) ? n : undefined; +} + +function parseSeriesIndex(value: string | undefined): number | undefined { + if (!value) return undefined; + const n = parseFloat(value); + return Number.isNaN(n) ? undefined : n; +} diff --git a/server/src/modules/metadata-fetch/providers/goodreads/goodreads.types.ts b/server/src/modules/metadata-fetch/providers/goodreads/goodreads.types.ts index 06fb4556b..953fd92f4 100644 --- a/server/src/modules/metadata-fetch/providers/goodreads/goodreads.types.ts +++ b/server/src/modules/metadata-fetch/providers/goodreads/goodreads.types.ts @@ -6,58 +6,30 @@ export interface GoodreadsNextData { }; } -export interface Ref { - __ref: string; -} - +// Inline nested objects — NOT __ref pointers export interface GoodreadsApolloBook { title?: string; + description?: string; imageUrl?: string; - description?: Ref; - bookGenres?: Array<{ genre?: Ref }>; - details?: Ref; - primaryContributorEdge?: Ref; - secondaryContributorEdges?: Ref[]; - bookSeries?: Ref[]; - work?: Ref; + details?: GoodreadsApolloDetails; + bookGenres?: Array<{ genre?: { name?: string } }>; + bookSeries?: Array<{ userPosition?: string }>; } -export interface GoodreadsApolloBookDetails { +export interface GoodreadsApolloDetails { + numPages?: string | number; + publicationTime?: string | number; publisher?: string; - publicationTime?: number; - language?: { name?: string }; - numPages?: number; isbn?: string; isbn13?: string; + language?: { name?: string }; } -export interface GoodreadsApolloContributorEdge { - node?: Ref; - role?: string; -} - +// Found by scanning root apolloState for keys starting with these prefixes export interface GoodreadsApolloContributor { name?: string; } -export interface GoodreadsApolloGenre { - name?: string; -} - -export interface GoodreadsApolloBookSeries { - series?: Ref; - userPosition?: string; -} - export interface GoodreadsApolloSeries { title?: string; } - -export interface GoodreadsApolloWork { - stats?: Ref; -} - -export interface GoodreadsApolloTruncatedHtml { - fullContent?: string; - truncatedContent?: string; -}