From da5d0bb4053d346bd01bc3325d5c20cd3285b762 Mon Sep 17 00:00:00 2001 From: neon <18559658+neonsolstice@users.noreply.github.com> Date: Thu, 24 Sep 2026 13:15:24 +0000 Subject: [PATCH] fix(reader): start read along at the selected narrated sentence "Read from here" and the page start resolved the range to the nearest element with any id and matched SMIL entries against that id. Storyteller keeps the book's inline elements inside its sentence spans with their ids intact (Kobo's kobo.N.N, calibre's calibre_link-N), so in those books the nearest id named no entry and narration restarted at the top of the chapter, while playback kept highlighting the same sentences correctly. Start at the first SMIL entry whose element overlaps the range instead, looking each entry up the way foliate resolves the highlight. Foliate already walks entries in reading order through the start filter, so the vendored engine is unchanged. Fixes #1528 --- client/src/features/reader/ReaderView.vue | 33 +-- .../lib/media-overlay-range.test.ts | 217 ++++++++++++++++++ .../media-overlay/lib/media-overlay-range.ts | 30 +++ 3 files changed, 256 insertions(+), 24 deletions(-) create mode 100644 client/src/features/reader/media-overlay/lib/media-overlay-range.test.ts create mode 100644 client/src/features/reader/media-overlay/lib/media-overlay-range.ts diff --git a/client/src/features/reader/ReaderView.vue b/client/src/features/reader/ReaderView.vue index 2ef44c1d3..04ae8a286 100644 --- a/client/src/features/reader/ReaderView.vue +++ b/client/src/features/reader/ReaderView.vue @@ -33,6 +33,7 @@ import { useMediaOverlay } from './media-overlay/composables/useMediaOverlay' import { resolveMediaOverlayResume, type MediaOverlayResumePosition } from './media-overlay/lib/media-overlay-resume' import { injectMediaOverlayHighlightCss, MEDIA_OVERLAY_DEFAULT_ACTIVE_CLASS } from './media-overlay/lib/media-overlay-highlight' import { startMediaOverlayWithFallback } from './media-overlay/lib/media-overlay-start' +import { mediaOverlayEntriesOverlapping } from './media-overlay/lib/media-overlay-range' import TtsResumePrompt from '@/features/tts/components/TtsResumePrompt.vue' import ReaderHeader from './epub/components/ReaderHeader.vue' import ReaderFooter from './epub/components/ReaderFooter.vue' @@ -577,14 +578,6 @@ async function resolveSavedNarrationPos(): Promise that -// SMIL media-overlay fragments point at. -function nearestSentenceId(node: Node | null): string | null { - let el: Element | null = node instanceof Element ? node : (node?.parentElement ?? null) - while (el && !el.id) el = el.parentElement - return el?.id || null -} - // Persist the exact narrated sentence so playback can resume there after reload. watch( () => mediaOverlay.currentFragment.value, @@ -604,11 +597,10 @@ watch( }, ) -// Starts media-overlay narration in `sectionIdx`. When `target` is given it -// begins at that sentence (matched by full SMIL fragment when byId=false, or by -// element id when byId=true); if no matching plays, it falls back to the -// start of the section instead of staying silent. -async function beginNarration(sectionIdx: number, target: string | null, byId: boolean) { +// Starts media-overlay narration in `sectionIdx` at the first that +// `matches` accepts; if none plays, it falls back to the start of the section +// instead of staying silent. +async function beginNarration(sectionIdx: number, matches: ((item: { text: string }) => boolean) | null) { const startId = ++mediaOverlayStartId const mo = getMediaOverlay() if (!mo) { @@ -620,12 +612,6 @@ async function beginNarration(sectionIdx: number, target: string | null, byId: b clearResumeNarrationHighlight() - const matches = target - ? byId - ? (item: { text: string }) => item.text.split('#')[1] === target - : (item: { text: string }) => item.text === target - : null - await mediaOverlay.start( mo, () => startMediaOverlayWithFallback(mo, sectionIdx, matches, () => startId === mediaOverlayStartId && mediaOverlay.isActive.value), @@ -645,13 +631,13 @@ async function handleStartMediaOverlay() { const saved = await resolveSavedNarrationPos() if (saved) { - await beginNarration(saved.section, saved.fragment, false) + await beginNarration(saved.section, (item) => item.text === saved.fragment) return } // No saved sentence: start from the first narrated sentence on the current page. - const visibleId = nearestSentenceId(getVisibleRange()?.startContainer ?? null) - await beginNarration(sectionIndex.value, visibleId, true) + const visible = getVisibleRange() + await beginNarration(sectionIndex.value, visible ? mediaOverlayEntriesOverlapping(visible) : null) } async function handleReadFromHere() { @@ -665,8 +651,7 @@ async function handleReadFromHere() { showSettings.value = false showTapZones.value = false hideOverlays(true) - const id = nearestSentenceId(range?.startContainer ?? null) - await beginNarration(sectionIndex.value, id, true) + await beginNarration(sectionIndex.value, range ? mediaOverlayEntriesOverlapping(range) : null) return } diff --git a/client/src/features/reader/media-overlay/lib/media-overlay-range.test.ts b/client/src/features/reader/media-overlay/lib/media-overlay-range.test.ts new file mode 100644 index 000000000..56170c510 --- /dev/null +++ b/client/src/features/reader/media-overlay/lib/media-overlay-range.test.ts @@ -0,0 +1,217 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +// @ts-expect-error public Foliate asset has no TypeScript declarations. +import { EPUB } from '../../../../../public/assets/foliate/epub.js' +import { mediaOverlayEntriesOverlapping } from './media-overlay-range' + +// Section bodies are Storyteller's own output for these source shapes (its +// `markupChapter`, `epub:type` attributes dropped). Storyteller wraps each +// sentence in a span but keeps the book's inline elements inside it with their +// ids, which is what bookorbit/bookorbit#1528 tripped over. +const sentenceBody = ` +

Chapter One

+

The keeper climbed the stairs. Salt had crusted on every rail.

+

The lamp turned above the harbor. Fishing boats waited for the tide.

+

Gulls circled the breakwater. Rain moved in from the west.

+` +const sentenceIds = ['c-s1', 'c-s2', 'c-s3', 'c-s4', 'c-s5', 'c-s6'] + +const wordBody = ` +

Waves broke. Nets dried.

+` +const wordIds = ['c-s0-w0', 'c-s0-w1', 'c-s1-w0', 'c-s1-w1'] + +function chapter(body: string): Document { + return new DOMParser().parseFromString( + `c${body}`, + 'application/xhtml+xml', + ) +} + +function textPosition(doc: Document, id: string, needle: string): { node: Text; offset: number } { + const root = doc.getElementById(id) + if (!root) throw new Error(`no element #${id}`) + const walker = doc.createTreeWalker(root, NodeFilter.SHOW_TEXT) + for (let node = walker.nextNode(); node; node = walker.nextNode()) { + const offset = (node as Text).data.indexOf(needle) + if (offset >= 0) return { node: node as Text, offset } + } + throw new Error(`no text "${needle}" in #${id}`) +} + +// The range a reader gets by selecting from `from` through `to`, each a word +// looked up inside the element with the given id. +function selection(doc: Document, from: [string, string], to: [string, string] = from): Range { + const start = textPosition(doc, from[1], from[0]) + const end = textPosition(doc, to[1], to[0]) + const range = doc.createRange() + range.setStart(start.node, start.offset) + range.setEnd(end.node, end.offset + to[0].length) + return range +} + +function entry(id: string): { text: string } { + return { text: `OPS/chapter.xhtml#${id}` } +} + +function acceptedIds(range: Range, ids: string[]): string[] { + const matches = mediaOverlayEntriesOverlapping(range) + return ids.filter((id) => matches(entry(id))) +} + +describe('mediaOverlayEntriesOverlapping', () => { + it('accepts the sentence around a Kobo span rather than needing the Kobo id', () => { + const doc = chapter(sentenceBody) + + expect(acceptedIds(selection(doc, ['lamp', 'c-s3']), sentenceIds)).toEqual(['c-s3']) + expect(acceptedIds(selection(doc, ['tide', 'c-s4']), sentenceIds)).toEqual(['c-s4']) + }) + + it('accepts the sentence around a calibre span, with or without the id Storyteller kept', () => { + const doc = chapter(sentenceBody) + + expect(acceptedIds(selection(doc, ['breakwater', 'c-s5']), sentenceIds)).toEqual(['c-s5']) + expect(acceptedIds(selection(doc, ['west', 'c-s6']), sentenceIds)).toEqual(['c-s6']) + }) + + it('accepts the sentence a selection sits in when the span holds only text', () => { + const doc = chapter(sentenceBody) + + expect(acceptedIds(selection(doc, ['keeper', 'c-s1']), sentenceIds)).toEqual(['c-s1']) + expect(acceptedIds(selection(doc, ['rail', 'c-s2']), sentenceIds)).toEqual(['c-s2']) + }) + + it('accepts every sentence a selection touches and none it does not', () => { + const doc = chapter(sentenceBody) + + expect(acceptedIds(selection(doc, ['lamp', 'c-s3'], ['Gulls', 'c-s5']), sentenceIds)).toEqual(['c-s3', 'c-s4', 'c-s5']) + }) + + it('accepts the word span of a word-level overlay past the Kobo id inside it', () => { + const doc = chapter(wordBody) + + expect(acceptedIds(selection(doc, ['Waves', 'c-s0-w0']), wordIds)).toEqual(['c-s0-w0']) + expect(acceptedIds(selection(doc, ['dried', 'c-s1-w1']), wordIds)).toEqual(['c-s1-w1']) + }) + + it('accepts nothing for text the overlay does not narrate', () => { + const doc = chapter(sentenceBody) + + expect(acceptedIds(selection(doc, ['Chapter', 'heading']), sentenceIds)).toEqual([]) + }) + + it('finds an entry that names an anchor instead of an id, as foliate does', () => { + const doc = chapter('

An older book marks its sentences by name.

') + + expect(mediaOverlayEntriesOverlapping(selection(doc, ['older', 'para']))(entry('old-anchor'))).toBe(true) + }) + + it('rejects entries that name no element or no fragment at all', () => { + const doc = chapter(sentenceBody) + const matches = mediaOverlayEntriesOverlapping(selection(doc, ['lamp', 'c-s3'])) + + expect(matches(entry('missing'))).toBe(false) + expect(matches({ text: 'OPS/chapter.xhtml' })).toBe(false) + }) +}) + +class FakeAudio extends EventTarget { + static instances: FakeAudio[] = [] + + paused = true + currentTime = 0 + volume = 1 + playbackRate = 1 + + constructor() { + super() + FakeAudio.instances.push(this) + } + + play(): Promise { + this.paused = false + this.dispatchEvent(new Event('playing')) + return Promise.resolve() + } + + pause(): void { + this.paused = true + } +} + +function smil(ids: string[]): string { + const pars = ids + .map((id, index) => ``) + .join('') + return `${pars}` +} + +// Starts the real vendored engine with the filter and reports the entry it +// highlights, which is the sentence the reader hears first. +async function startedEntry(ids: string[], range: Range): Promise { + const book = new EPUB({ + loadText: async (src: string) => (src === 'OPS/chapter.smil' ? smil(ids) : null), + loadBlob: async () => new Blob(['audio']), + getSize: () => 0, + }) + book.sections = [{ id: 'OPS/chapter.xhtml', mediaOverlay: { href: 'OPS/chapter.smil' } }] + const overlay = book.getMediaOverlay() + const highlights: string[] = [] + overlay.addEventListener('highlight', (event: Event) => highlights.push((event as CustomEvent<{ text: string }>).detail.text)) + + const started = await overlay.start(0, mediaOverlayEntriesOverlapping(range)) + FakeAudio.instances.at(-1)?.dispatchEvent(new Event('canplaythrough')) + overlay.stop() + if (!started) return null + return highlights[0]?.split('#')[1] ?? null +} + +describe('starting Foliate narration from a range', () => { + const originalAudio = globalThis.Audio + const originalCreateObjectURL = URL.createObjectURL + const originalRevokeObjectURL = URL.revokeObjectURL + + beforeEach(() => { + FakeAudio.instances = [] + globalThis.Audio = FakeAudio as unknown as typeof Audio + URL.createObjectURL = vi.fn<(object: Blob | MediaSource) => string>(() => 'blob:media-overlay') + URL.revokeObjectURL = vi.fn<(url: string) => void>() + }) + + afterEach(() => { + globalThis.Audio = originalAudio + URL.createObjectURL = originalCreateObjectURL + URL.revokeObjectURL = originalRevokeObjectURL + vi.restoreAllMocks() + }) + + it('starts at the selected sentence inside a Kobo span', async () => { + const doc = chapter(sentenceBody) + + await expect(startedEntry(sentenceIds, selection(doc, ['lamp', 'c-s3']))).resolves.toBe('c-s3') + }) + + it('starts a selection across sentences at the first one in reading order', async () => { + const doc = chapter(sentenceBody) + + await expect(startedEntry(sentenceIds, selection(doc, ['lamp', 'c-s3'], ['Gulls', 'c-s5']))).resolves.toBe('c-s3') + }) + + it('starts a selection beginning between sentences at the next one', async () => { + const doc = chapter('

First line here. Second line here.

') + + await expect(startedEntry(['t-1', 't-2'], selection(doc, [' ', 'para-9'], ['Second', 't-2']))).resolves.toBe('t-2') + }) + + it('starts a word-level overlay at the selected word', async () => { + const doc = chapter(wordBody) + + await expect(startedEntry(wordIds, selection(doc, ['Waves', 'c-s0-w0']))).resolves.toBe('c-s0-w0') + }) + + it('starts nothing for text the overlay does not narrate', async () => { + const doc = chapter(sentenceBody) + + await expect(startedEntry(sentenceIds, selection(doc, ['Chapter', 'heading']))).resolves.toBeNull() + }) +}) diff --git a/client/src/features/reader/media-overlay/lib/media-overlay-range.ts b/client/src/features/reader/media-overlay/lib/media-overlay-range.ts new file mode 100644 index 000000000..c89870f21 --- /dev/null +++ b/client/src/features/reader/media-overlay/lib/media-overlay-range.ts @@ -0,0 +1,30 @@ +type MediaOverlayItem = { text: string } + +// The element a SMIL reference narrates, looked up the way foliate's +// resolveHref does, so starting and highlighting agree on the element. +function narratedElement(doc: Document, text: string): Element | null { + const id = text.split('#')[1] + if (!id) return null + return doc.getElementById(id) ?? doc.getElementsByName(id)[0] ?? null +} + +// Accepts the SMIL entries whose narrated element overlaps `range`, so +// foliate, which walks entries in reading order, starts at the first narrated +// sentence the range touches. +// +// Only the overlay knows which ids are narrated. Storyteller wraps each +// sentence in its own span but keeps the book's inline elements inside it with +// their ids (Kobo's `kobo.N.N`, calibre's `calibre_link-N`), so the id nearest +// the selected text is often one no SMIL entry names. +export function mediaOverlayEntriesOverlapping(range: Range): (item: MediaOverlayItem) => boolean { + const doc = range.startContainer.ownerDocument ?? (range.startContainer as Document) + return (item) => { + const element = narratedElement(doc, item.text) + if (!element) return false + try { + return range.intersectsNode(element) + } catch { + return false + } + } +}