fix(metadata): fix Amazon and Goodreads scraper bugs

2026-11-03 00:11:37 -06:00 · 2026-02-19 02:17:50 -06:00
parent d4850dc6d7
commit ed3c36e35d
7 changed files with 380 additions and 124 deletions
@@ -36,13 +36,14 @@
    "@nestjs/core": "^11.0.1",
    "@nestjs/jwt": "^11.0.2",
    "@nestjs/passport": "^11.0.5",
-    "@nestjs/platform-fastify": "^11.1.13",
-    "@nestjs/platform-socket.io": "^11.1.13",
+    "@nestjs/platform-fastify": "^11.1.14",
+    "@nestjs/platform-socket.io": "^11.1.14",
    "@nestjs/schedule": "^6.1.1",
-    "@nestjs/websockets": "^11.1.13",
+    "@nestjs/websockets": "^11.1.14",
    "@parcel/watcher": "^2.5.6",
    "@projectx/types": "workspace:*",
    "bcryptjs": "^3.0.3",
+    "cheerio": "^1.2.0",
    "class-transformer": "^0.5.1",
    "class-validator": "^0.14.3",
    "drizzle-orm": "^0.45.1",
@@ -5,7 +5,7 @@
    {
      "idx": 0,
      "version": "7",
-      "when": 1771534544614,
+      "when": 1771535401494,
      "tag": "0000_add_provider_ids",
      "breakpoints": true
    }
@@ -5,11 +5,12 @@ import { MetadataFetchController } from './metadata-fetch.controller';
 import { MetadataProvider } from './providers/metadata-provider';
 import { MetadataFetchService } from './metadata-fetch.service';
 import { ProviderRegistry } from './provider-registry';
+import { AmazonProvider } from './providers/amazon/amazon.provider';
 import { GoogleProvider } from './providers/google/google.provider';
 import { GoodreadsProvider } from './providers/goodreads/goodreads.provider';
 import { OpenLibraryProvider } from './providers/open-library/open-library.provider';

-const PROVIDER_CLASSES = [GoogleProvider, GoodreadsProvider, OpenLibraryProvider];
+const PROVIDER_CLASSES = [GoogleProvider, GoodreadsProvider, AmazonProvider, OpenLibraryProvider];

@Module({
  providers: [
@@ -0,0 +1,100 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { MetadataCandidate, MetadataProviderKey } from '@projectx/types';
+
+import { IdentifiableProvider } from '../metadata-provider';
+import { MetadataSearchParams } from '../metadata-search-params';
+import { extractAsins, parseBookPage } from './amazon.scraper';
+
+const HEADERS: HeadersInit = {
+  'user-agent':
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36',
+  accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+  'accept-language': 'en-US,en;q=0.9',
+  'sec-ch-ua': '"Google Chrome";v="137", "Chromium";v="137", "Not_A Brand";v="24"',
+  'sec-ch-ua-mobile': '?0',
+  'sec-ch-ua-platform': '"macOS"',
+  'sec-fetch-dest': 'document',
+  'sec-fetch-mode': 'navigate',
+  'sec-fetch-site': 'none',
+};
+
+const MAX_RESULTS = 3;
+const BETWEEN_REQUESTS_MS = 800;
+
+@Injectable()
+export class AmazonProvider implements IdentifiableProvider {
+  readonly key = MetadataProviderKey.AMAZON;
+  readonly label = 'Amazon';
+  readonly identifiable = true as const;
+
+  private readonly logger = new Logger(AmazonProvider.name);
+
+  async search(params: MetadataSearchParams): Promise<MetadataCandidate[]> {
+    const asins = params.isbn
+      ? [params.isbn] // try ASIN directly if ISBN provided
+      : await this.searchAsins(params);
+
+    const results: MetadataCandidate[] = [];
+    for (const asin of asins.slice(0, MAX_RESULTS)) {
+      if (results.length > 0) await sleep(BETWEEN_REQUESTS_MS);
+      const candidate = await this.fetchByAsin(asin);
+      if (candidate) results.push(candidate);
+    }
+    return results;
+  }
+
+  async lookupById(providerId: string): Promise<MetadataCandidate | null> {
+    return this.fetchByAsin(providerId);
+  }
+
+  private async searchAsins(params: MetadataSearchParams): Promise<string[]> {
+    const query = [params.title, params.author].filter(Boolean).join(' ');
+    const url = `https://www.amazon.com/s?k=${encodeURIComponent(query)}&i=stripbooks`;
+    const html = await this.fetchHtml(url);
+    return html ? extractAsins(html, MAX_RESULTS) : [];
+  }
+
+  private async fetchByAsin(asin: string): Promise<MetadataCandidate | null> {
+    const html = await this.fetchHtml(`https://www.amazon.com/dp/${asin}`);
+    if (!html) return null;
+    const data = parseBookPage(html);
+    if (!data.title) return null;
+    return {
+      provider: MetadataProviderKey.AMAZON,
+      providerId: asin,
+      title: data.title,
+      subtitle: data.subtitle,
+      authors: data.authors?.length ? data.authors : undefined,
+      description: data.description,
+      isbn13: data.isbn13,
+      isbn10: data.isbn10,
+      publisher: data.publisher,
+      publishedYear: data.publishedYear,
+      language: data.language,
+      pageCount: data.pageCount,
+      seriesName: data.seriesName,
+      seriesIndex: data.seriesIndex,
+      coverUrl: data.coverUrl,
+      tags: data.tags?.length ? data.tags : undefined,
+      sourceUrl: `https://www.amazon.com/dp/${asin}`,
+    };
+  }
+
+  private async fetchHtml(url: string): Promise<string | null> {
+    try {
+      const res = await fetch(url, { headers: HEADERS });
+      if (!res.ok) {
+        this.logger.warn(`Amazon returned ${res.status} for ${url}`);
+        return null;
+      }
+      return res.text();
+    } catch (err) {
+      this.logger.warn(`Amazon fetch failed for ${url}: ${err}`);
+      return null;
+    }
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
@@ -0,0 +1,202 @@
+import * as cheerio from 'cheerio';
+import type { CheerioAPI } from 'cheerio';
+
+export interface AmazonBookData {
+  title?: string;
+  subtitle?: string;
+  authors?: string[];
+  description?: string;
+  isbn13?: string;
+  isbn10?: string;
+  publisher?: string;
+  publishedYear?: number;
+  language?: string;
+  pageCount?: number;
+  seriesName?: string;
+  seriesIndex?: number;
+  coverUrl?: string;
+  tags?: string[];
+}
+
+const SKIP_TITLE_PATTERNS = /box\s*set|collection\s*set|books\s*set|omnibus|summary\s*&\s*study/i;
+
+export function parseBookPage(html: string): AmazonBookData {
+  const $ = cheerio.load(html);
+  const { title, subtitle } = extractTitle($);
+  return {
+    title,
+    subtitle,
+    authors: extractAuthors($),
+    description: extractDescription($),
+    isbn13: extractIsbn($, 'isbn13'),
+    isbn10: extractIsbn($, 'isbn10'),
+    publisher: extractPublisher($),
+    publishedYear: extractPublishedYear($),
+    language: extractLanguage($),
+    pageCount: extractPageCount($),
+    seriesName: extractSeriesName($),
+    seriesIndex: extractSeriesIndex($),
+    coverUrl: extractCoverUrl($),
+    tags: extractCategories($),
+  };
+}
+
+export function extractAsins(html: string, limit: number): string[] {
+  const $ = cheerio.load(html);
+  const seen = new Set<string>();
+  const asins: string[] = [];
+
+  $('div[data-asin]').each((_, el) => {
+    const asin = $(el).attr('data-asin');
+    if (!asin || asin.length !== 10 || seen.has(asin)) return;
+    const titleText = $(el).find('[data-cy=title-recipe]').text().toLowerCase();
+    if (SKIP_TITLE_PATTERNS.test(titleText)) return;
+    seen.add(asin);
+    asins.push(asin);
+    if (asins.length >= limit) return false;
+  });
+
+  return asins;
+}
+
+// --- Field extractors ---
+
+function extractTitle($: CheerioAPI): { title?: string; subtitle?: string } {
+  const raw = $('#productTitle, #ebooksProductTitle').first().text().trim();
+  if (!raw) return {};
+  const colon = raw.indexOf(':');
+  if (colon > 0) {
+    return { title: raw.substring(0, colon).trim(), subtitle: raw.substring(colon + 1).trim() };
+  }
+  return { title: raw };
+}
+
+function extractAuthors($: CheerioAPI): string[] {
+  const authors: string[] = [];
+  const seen = new Set<string>();
+  $('#bylineInfo .author a, #bylineInfo_feature_div .author a').each((_, el) => {
+    const name = $(el).text().trim();
+    if (name && !seen.has(name)) {
+      seen.add(name);
+      authors.push(name);
+    }
+  });
+  return authors;
+}
+
+function extractDescription($: CheerioAPI): string | undefined {
+  const expander = $('#bookDescription_feature_div .a-expander-content').first();
+  if (expander.length) return expander.text().trim() || undefined;
+
+  const noscript = $('#bookDescription_feature_div noscript').first();
+  if (noscript.length) {
+    const inner = cheerio.load(noscript.html() ?? '');
+    return inner.root().text().trim() || undefined;
+  }
+  return undefined;
+}
+
+function extractIsbn($: CheerioAPI, field: 'isbn13' | 'isbn10'): string | undefined {
+  // Newer Amazon layout: rpi-attribute panels
+  const rpi = $(`#rpi-attribute-book_details-${field} .rpi-attribute-value span`).first().text().trim();
+  if (rpi) return rpi.replace(/[^0-9X]/gi, '') || undefined;
+
+  // Older layout: detail bullets
+  const label = field === 'isbn13' ? /isbn-?13/i : /isbn-?10/i;
+  const value = detailBulletValue($, label);
+  return value ? value.replace(/[^0-9X]/gi, '') || undefined : undefined;
+}
+
+function extractPublisher($: CheerioAPI): string | undefined {
+  // Newer layout
+  const rpi = $('#rpi-attribute-book_details-publisher .rpi-attribute-value span').first().text().trim();
+  if (rpi) return rpi || undefined;
+
+  // Older layout: "Publisher : Penguin (Jan 1, 2020)"
+  const raw = detailBulletValue($, /publisher/i);
+  if (!raw) return undefined;
+  // Strip date in parentheses at the end
+  return raw.replace(/\s*\([^)]*\)\s*$/, '').trim() || undefined;
+}
+
+function extractPublishedYear($: CheerioAPI): number | undefined {
+  // Newer layout
+  const rpiText = $('#rpi-attribute-book_details-publication_date .rpi-attribute-value span').first().text().trim();
+  if (rpiText) {
+    const y = parseYearFromText(rpiText);
+    if (y) return y;
+  }
+
+  // Older layout: extract year from parenthesised date in publisher bullet
+  const publisherBullet = detailBulletValue($, /publisher/i);
+  const match = publisherBullet.match(/\(([^)]+)\)/);
+  if (match) {
+    const y = parseYearFromText(match[1]);
+    if (y) return y;
+  }
+  return undefined;
+}
+
+function extractLanguage($: CheerioAPI): string | undefined {
+  const rpi = $('#rpi-attribute-language .rpi-attribute-value span').first().text().trim();
+  if (rpi) return rpi || undefined;
+  return detailBulletValue($, /language/i) || undefined;
+}
+
+function extractPageCount($: CheerioAPI): number | undefined {
+  const rpi = $('#rpi-attribute-book_details-fiona_pages .rpi-attribute-value span').first().text().trim();
+  const text = rpi || detailBulletValue($, /print length|pages/i);
+  const digits = text.match(/\d+/)?.[0];
+  return digits ? parseInt(digits, 10) : undefined;
+}
+
+function extractSeriesName($: CheerioAPI): string | undefined {
+  return (
+    $('#rpi-attribute-book_details-series .rpi-attribute-value a span').first().text().trim() ||
+    undefined
+  );
+}
+
+function extractSeriesIndex($: CheerioAPI): number | undefined {
+  const label = $('#rpi-attribute-book_details-series .rpi-attribute-label span').first().text();
+  const match = label.match(/book\s+(\d+(?:\.\d+)?)\s+of/i);
+  if (!match) return undefined;
+  const n = parseFloat(match[1]);
+  return Number.isNaN(n) ? undefined : n;
+}
+
+function extractCoverUrl($: CheerioAPI): string | undefined {
+  const img = $('#landingImage, #imgBlkFront').first();
+  const hires = img.attr('data-old-hires');
+  if (hires) return hires;
+  const src = img.attr('src');
+  // Strip Amazon's dimension modifier (e.g. ._SX260_) to get the original image
+  return src ? src.replace(/\._[A-Z0-9_,]+_\./i, '.') : undefined;
+}
+
+function extractCategories($: CheerioAPI): string[] {
+  const cats: string[] = [];
+  $('#detailBullets_feature_div .zg_hrsr .a-list-item a').each((_, el) => {
+    const name = $(el).text().trim().replace(/\s*\(Books\)\s*$/i, '');
+    if (name) cats.push(name);
+  });
+  return cats;
+}
+
+// --- Helpers ---
+
+function detailBulletValue($: CheerioAPI, labelPattern: RegExp): string {
+  let value = '';
+  $('#detailBullets_feature_div .a-text-bold').each((_, el) => {
+    if (labelPattern.test($(el).text())) {
+      value = $(el).next('span').text().trim();
+      return false; // break
+    }
+  });
+  return value;
+}
+
+function parseYearFromText(text: string): number | undefined {
+  const match = text.match(/\b(1[0-9]{3}|2[0-9]{3})\b/);
+  return match ? parseInt(match[1], 10) : undefined;
+}
@@ -2,110 +2,90 @@ import { MetadataCandidate, MetadataProviderKey } from '@projectx/types';

 import {
  GoodreadsApolloBook,
-  GoodreadsApolloBookDetails,
-  GoodreadsApolloBookSeries,
  GoodreadsApolloContributor,
-  GoodreadsApolloContributorEdge,
-  GoodreadsApolloGenre,
  GoodreadsApolloSeries,
-  GoodreadsApolloTruncatedHtml,
-  Ref,
 } from './goodreads.types';

-function deref<T>(state: Record<string, unknown>, ref: Ref | unknown): T | undefined {
-  if (!ref || typeof ref !== 'object') return undefined;
-  const r = ref as Ref;
-  if (!r.__ref) return undefined;
-  return state[r.__ref] as T | undefined;
-}
-
-function stripHtml(html: string): string {
-  return html
-    .replace(/<br\s*\/?>/gi, '\n')
-    .replace(/<[^>]+>/g, '')
-    .replace(/&amp;/g, '&')
-    .replace(/&lt;/g, '<')
-    .replace(/&gt;/g, '>')
-    .replace(/&quot;/g, '"')
-    .replace(/&#39;/g, "'")
-    .replace(/&nbsp;/g, ' ')
-    .trim();
-}
-
 export function mapGoodreadsApolloState(
  state: Record<string, unknown>,
  bookId: string,
 ): MetadataCandidate | null {
-  const bookKey = Object.keys(state).find((k) => k.startsWith('Book:'));
-  if (!bookKey) return null;
-  const book = state[bookKey] as GoodreadsApolloBook;
+  const book = findByKeyPrefix<GoodreadsApolloBook>(state, 'Book:kca:');
+  if (!book?.title) return null;

-  const details = deref<GoodreadsApolloBookDetails>(state, book.details);
+  const contributor = findByKeyPrefix<GoodreadsApolloContributor>(state, 'Contributor:kca');
+  const series = findByKeyPrefix<GoodreadsApolloSeries>(state, 'Series:kca');

-  const authors: string[] = [];
-  const primaryEdge = deref<GoodreadsApolloContributorEdge>(state, book.primaryContributorEdge);
-  const primaryAuthor = deref<GoodreadsApolloContributor>(state, primaryEdge?.node);
-  if (primaryAuthor?.name) authors.push(primaryAuthor.name);
-  for (const edgeRef of book.secondaryContributorEdges ?? []) {
-    const edge = deref<GoodreadsApolloContributorEdge>(state, edgeRef);
-    if (edge?.role?.toLowerCase() === 'author') {
-      const author = deref<GoodreadsApolloContributor>(state, edge.node);
-      if (author?.name) authors.push(author.name);
-    }
-  }
+  const details = book.details;
+  const firstSeries = book.bookSeries?.[0];

-  const descNode = deref<GoodreadsApolloTruncatedHtml>(state, book.description);
-  const descHtml = descNode?.fullContent ?? descNode?.truncatedContent ?? '';
-  const description = descHtml ? stripHtml(descHtml) : undefined;
+  const tags = (book.bookGenres ?? [])
+    .map((g) => g.genre?.name)
+    .filter((n): n is string => !!n);

-  const tags: string[] = [];
-  for (const entry of book.bookGenres ?? []) {
-    const genre = deref<GoodreadsApolloGenre>(state, entry.genre);
-    if (genre?.name) tags.push(genre.name);
-  }
+  const { title, subtitle } = splitTitle(book.title);

-  let seriesName: string | undefined;
-  let seriesIndex: number | undefined;
-  for (const bsRef of book.bookSeries ?? []) {
-    const bs = deref<GoodreadsApolloBookSeries>(state, bsRef);
-    const series = deref<GoodreadsApolloSeries>(state, bs?.series);
-    if (series?.title) {
-      seriesName = series.title;
-      if (bs?.userPosition) {
-        const idx = parseFloat(bs.userPosition);
-        if (!Number.isNaN(idx)) seriesIndex = idx;
-      }
-      break;
-    }
-  }
-
-  let publishedYear: number | undefined;
-  if (details?.publicationTime) {
-    publishedYear = new Date(details.publicationTime).getFullYear();
-  }
-
-  const fullTitle = book.title ?? '';
-  const colonIdx = fullTitle.indexOf(':');
-  const title = colonIdx > 0 ? fullTitle.substring(0, colonIdx).trim() : fullTitle;
-  const subtitle = colonIdx > 0 ? fullTitle.substring(colonIdx + 1).trim() : undefined;
+  const publishedYear = parseEpochYear(details?.publicationTime);
+  const pageCount = parsePositiveInt(details?.numPages);
+  const seriesIndex = parseSeriesIndex(firstSeries?.userPosition);

  return {
    provider: MetadataProviderKey.GOODREADS,
    providerId: bookId,
    title,
    subtitle,
-    authors: authors.length ? authors : undefined,
-    description,
-    publisher: details?.publisher,
+    authors: contributor?.name ? [contributor.name] : undefined,
+    description: normalize(book.description),
+    publisher: normalize(details?.publisher),
    publishedYear,
-    language: details?.language?.name,
-    pageCount: details?.numPages,
-    isbn10: details?.isbn,
-    isbn13: details?.isbn13,
+    language: normalize(details?.language?.name),
+    pageCount,
+    isbn10: normalize(details?.isbn),
+    isbn13: normalize(details?.isbn13),
    tags: tags.length ? tags : undefined,
    coverUrl: book.imageUrl,
    sourceUrl: `https://www.goodreads.com/book/show/${bookId}`,
-    seriesName,
+    seriesName: normalize(series?.title),
    seriesIndex,
  };
 }
+
+function findByKeyPrefix<T>(state: Record<string, unknown>, prefix: string): T | undefined {
+  const key = Object.keys(state).find((k) => k.includes(prefix));
+  return key ? (state[key] as T) : undefined;
+}
+
+function splitTitle(fullTitle: string): { title: string; subtitle?: string } {
+  const colon = fullTitle.indexOf(':');
+  if (colon > 0) {
+    return {
+      title: fullTitle.substring(0, colon).trim(),
+      subtitle: fullTitle.substring(colon + 1).trim(),
+    };
+  }
+  return { title: fullTitle };
+}
+
+function normalize(value: string | undefined | null): string | undefined {
+  if (!value || value === 'null') return undefined;
+  return value.trim() || undefined;
+}
+
+function parseEpochYear(value: string | number | undefined): number | undefined {
+  if (value == null) return undefined;
+  const ms = typeof value === 'string' ? parseFloat(value) : value;
+  if (!ms || Number.isNaN(ms)) return undefined;
+  return new Date(ms).getFullYear();
+}
+
+function parsePositiveInt(value: string | number | undefined): number | undefined {
+  if (value == null) return undefined;
+  const n = typeof value === 'string' ? parseInt(value, 10) : Math.round(value);
+  return n > 0 && !Number.isNaN(n) ? n : undefined;
+}
+
+function parseSeriesIndex(value: string | undefined): number | undefined {
+  if (!value) return undefined;
+  const n = parseFloat(value);
+  return Number.isNaN(n) ? undefined : n;
+}
@@ -6,58 +6,30 @@ export interface GoodreadsNextData {
  };
 }

-export interface Ref {
-  __ref: string;
-}
-
+// Inline nested objects — NOT __ref pointers
 export interface GoodreadsApolloBook {
  title?: string;
+  description?: string;
  imageUrl?: string;
-  description?: Ref;
-  bookGenres?: Array<{ genre?: Ref }>;
-  details?: Ref;
-  primaryContributorEdge?: Ref;
-  secondaryContributorEdges?: Ref[];
-  bookSeries?: Ref[];
-  work?: Ref;
+  details?: GoodreadsApolloDetails;
+  bookGenres?: Array<{ genre?: { name?: string } }>;
+  bookSeries?: Array<{ userPosition?: string }>;
 }

-export interface GoodreadsApolloBookDetails {
+export interface GoodreadsApolloDetails {
+  numPages?: string | number;
+  publicationTime?: string | number;
  publisher?: string;
-  publicationTime?: number;
-  language?: { name?: string };
-  numPages?: number;
  isbn?: string;
  isbn13?: string;
+  language?: { name?: string };
 }

-export interface GoodreadsApolloContributorEdge {
-  node?: Ref;
-  role?: string;
-}
-
+// Found by scanning root apolloState for keys starting with these prefixes
 export interface GoodreadsApolloContributor {
  name?: string;
 }

-export interface GoodreadsApolloGenre {
-  name?: string;
-}
-
-export interface GoodreadsApolloBookSeries {
-  series?: Ref;
-  userPosition?: string;
-}
-
 export interface GoodreadsApolloSeries {
  title?: string;
 }
-
-export interface GoodreadsApolloWork {
-  stats?: Ref;
-}
-
-export interface GoodreadsApolloTruncatedHtml {
-  fullContent?: string;
-  truncatedContent?: string;
-}