fix(reader): decode character references in read-aloud text

Block text was unescaped from a six-entry table, so only &, <, >, ", ' and   ever became characters. Every numeric reference survived into the text handed to the TTS provider, and an EPUB that writes its apostrophes as ' had them read out as "hash thirty-nine" while the page showed a normal apostrophe. Which apostrophes broke depended only on how the book was serialized: ' was in the table, ' was not, and both appear in the same sentence often enough. Parse each block instead of stripping tags and unescaping by hand. The parser decodes whatever the reader's own DOM decodes, which is the property that matters here because the narration is read against the page, and it decodes in one pass, so a book that quotes &#39; is still narrated as '. Only the text conversion changed. Block boundaries still come from the same split on block-element opening tags, so block indices stay aligned with Foliate's highlight blocks and audio cannot drift a paragraph away from the highlight. Do not route this through htmlToPlainText. That helper strips tag-shaped text a second time after decoding, which is right for the scraped metadata it was written for but silences book content: a code sample written <div>hi</div> would narrate as "hi", and "5 < 6" would lose its operator. Tests cover both. Fixes #1486
2026-11-03 00:11:37 -06:00 · 2026-09-22 09:51:10 +00:00
parent 970b152529
commit a4809512d8
2 changed files with 92 additions and 13 deletions
@@ -42,6 +42,73 @@ describe('htmlToBlocks', () => {
  it('decodes entities and collapses whitespace within a block', () => {
    expect(htmlToBlocks('<body><p>Tom &amp; Jerry   say\n"hi".</p></body>')).toEqual(['Tom & Jerry say "hi".']);
  });
+
+  // Regression: block text used to be unescaped from a six-entry table, so anything
+  // outside it reached the TTS provider as raw markup. A Kokoro user heard
+  // "dell hash thirty-nine Arbatskaja" for text the reader displayed as "dell'Arbatskaja".
+  describe('character references', () => {
+    it('decodes a decimal numeric reference', () => {
+      expect(htmlToBlocks('<body><p>Lui aveva fatto l&#39;abitudine.</p></body>')).toEqual(["Lui aveva fatto l'abitudine."]);
+    });
+
+    it('decodes a hexadecimal numeric reference in either case', () => {
+      expect(htmlToBlocks('<body><p>dell&#x27;Arbatskaja &#X2014; sì.</p></body>')).toEqual(["dell'Arbatskaja — sì."]);
+    });
+
+    it('decodes named references beyond the five XML ones', () => {
+      expect(htmlToBlocks('<body><p>Caf&eacute; &agrave; Paris&hellip; &ldquo;c&rsquo;est bon&rdquo;</p></body>')).toEqual([
+        'Café à Paris… “c’est bon”',
+      ]);
+    });
+
+    it('still decodes the references the old table covered', () => {
+      expect(htmlToBlocks('<body><p>Tom &amp; Jerry &quot;said&quot; it&apos;s fine.</p></body>')).toEqual(['Tom & Jerry "said" it\'s fine.']);
+    });
+
+    it('collapses a decoded non-breaking space like any other whitespace', () => {
+      expect(htmlToBlocks('<body><p>Chapter&#160;One&nbsp;&nbsp;begins.</p></body>')).toEqual(['Chapter One begins.']);
+    });
+
+    // A second decoding pass would turn this into an apostrophe and silently rewrite a
+    // book that is quoting the markup itself.
+    it('decodes once, leaving an escaped reference escaped', () => {
+      expect(htmlToBlocks('<body><p>Write &amp;#39; for an apostrophe.</p></body>')).toEqual(['Write &#39; for an apostrophe.']);
+    });
+
+    it('leaves a bare ampersand and an unknown reference alone', () => {
+      expect(htmlToBlocks('<body><p>Fish & chips, AT&T, &zzz; too.</p></body>')).toEqual(['Fish & chips, AT&T, &zzz; too.']);
+    });
+
+    // Decoding before the split would let an escaped block tag open a new block and
+    // shift every later index away from Foliate's highlight blocks.
+    it('does not let a decoded reference create a block boundary', () => {
+      expect(htmlToBlocks('<body><p>One &lt;p&gt; two.</p><p>Next.</p></body>')).toEqual(['One <p> two.', 'Next.']);
+    });
+
+    // Escaped markup is content the page displays, so it is content the narration reads.
+    // Discarding it again after decoding would mute a programming book's code samples.
+    it('keeps escaped markup that the page displays', () => {
+      expect(htmlToBlocks('<body><pre>&lt;div class="x"&gt;hi&lt;/div&gt;</pre></body>')).toEqual(['<div class="x">hi</div>']);
+    });
+
+    it('keeps an escaped comparison operator', () => {
+      expect(htmlToBlocks('<body><p>Se 5 &lt; 6, allora va bene.</p></body>')).toEqual(['Se 5 < 6, allora va bene.']);
+    });
+
+    // Block indices are positional, so a paragraph that is nothing but a reference has
+    // to keep its slot rather than being dropped as empty.
+    it('keeps a block whose only content is a reference', () => {
+      expect(htmlToBlocks('<body><p>Before.</p><p>&#8212;</p><p>After.</p></body>')).toEqual(['Before.', '—', 'After.']);
+    });
+
+    it('decodes references inside inline markup', () => {
+      expect(htmlToBlocks('<body><p>Il <em>tram&#8217;s</em> <b>l&#39;ultimo</b>.</p></body>')).toEqual(["Il tram’s l'ultimo."]);
+    });
+
+    it('decodes references in a table row block', () => {
+      expect(htmlToBlocks('<body><table><tr><td>L&#39;autore</td></tr><tr><td>L&#39;anno</td></tr></table></body>')).toEqual(["L'autore", "L'anno"]);
+    });
+  });
 });

 describe('normalizePath', () => {
@@ -1,4 +1,5 @@
 import { Injectable, Logger, NotFoundException } from '@nestjs/common';
+import { load } from 'cheerio';
 import * as unzipper from 'unzipper';
 import { XMLParser } from 'fast-xml-parser';

@@ -35,9 +36,6 @@ export function htmlToBlocks(html: string): string[] {
  // server-side audio blocks and client-side highlight blocks stay in sync.
  const BLOCK_SPLIT_RE =
    /<(?:article|aside|audio|blockquote|caption|details|dialog|div|dl|dt|dd|figure|footer|form|figcaption|h[1-6]|header|hgroup|hr|li|main|math|nav|ol|p|pre|section|tr)[^>]*>/gi;
-  const TAG_RE = /<[^>]+>/g;
-  const ENTITY_RE = /&(?:amp|lt|gt|quot|apos|nbsp);/g;
-  const ENTITY_MAP: Record<string, string> = { '&amp;': '&', '&lt;': '<', '&gt;': '>', '&quot;': '"', '&apos;': "'", '&nbsp;': ' ' };

  // Foliate's getBlocks() starts the first block at the first block-level
  // element and discards any content before it. Anchor extraction at the same
@@ -52,16 +50,30 @@ export function htmlToBlocks(html: string): string[] {
  // inside a paragraph (common in pretty-printed EPUB HTML) never splits one
  // paragraph into several blocks and desyncs the indices.
  const chunks = fromFirstBlock.split(BLOCK_SPLIT_RE);
-  return chunks
-    .map((chunk) => {
-      const text = chunk
-        .replace(TAG_RE, '')
-        .replace(ENTITY_RE, (m) => ENTITY_MAP[m] ?? m)
-        .replace(/\s+/g, ' ')
-        .trim();
-      return text;
-    })
-    .filter((t) => t.length > 0);
+
+  // Only the text conversion below changed: the split above still decides block
+  // boundaries, so block indices stay aligned with Foliate's highlight blocks.
+  return chunks.map((chunk) => blockText(chunk)).filter((t) => t.length > 0);
+}
+
+/**
+ * A block's spoken text, which has to be the text the reader displays for that block,
+ * because the narration is read against the page.
+ *
+ * Each chunk is still an HTML fragment, so it is parsed rather than stripped and unescaped
+ * by hand. The hand-written table this replaced knew six named entities and no numeric
+ * ones, so a reference like `&#39;` survived into the block text and providers read it out
+ * verbatim ("hash thirty-nine") where the page showed a normal apostrophe. Parsing decodes
+ * whatever the reader's own DOM decodes, and decodes once, so a book that quotes `&amp;#39;`
+ * is still narrated as `&#39;`.
+ *
+ * `htmlToPlainText` is deliberately not used here. It strips tag-shaped text a second time
+ * after decoding, which is right for scraped metadata that gets displayed but would silence
+ * content in a book: a code sample written `&lt;div&gt;hi&lt;/div&gt;` would narrate as
+ * "hi", and "5 &lt; 6" would lose its operator.
+ */
+function blockText(chunk: string): string {
+  return load(chunk, undefined, false).root().text().replace(/\s+/g, ' ').trim();
 }

@Injectable()