From 97cd03325fd42e21ac449c392a15299dedee5499 Mon Sep 17 00:00:00 2001 From: neon <18559658+neonsolstice@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:37:41 +0000 Subject: [PATCH] feat(reader): open DjVu documents in the web reader Scanned books are commonly distributed as DjVu, which the scanner only stored as a supplement. Decode pages on the server with DjVuLibre so DjVu imports with a cover and page count, opens in the image reader with its embedded OCR text, and can be the preferred format of an ordinary email recipient. Migration 0113 widens the recipient format check to allow it. Decoding runs on validated private snapshots behind a bounded, reader-first queue, and revalidation and cached pages are answered without a decoder. Released iOS builds cannot open DjVu, so series continuation offers it only to callers sending includeDjvu=true, and the dashboard and notebook read file never name it. The first full scan after upgrading promotes stored .djvu supplements to content. --- .github/workflows/e2e-runner.yml | 4 + Dockerfile | 2 +- .../__tests__/useBookDockUpload.spec.ts | 8 + .../composables/useBookDockUpload.ts | 4 +- .../book/components/BookQuickView.vue | 4 +- .../components/detail/tabs/DetailsTab.vue | 4 +- .../detail/tabs/EditMetadataTab.vue | 11 +- .../table/BookTableCellDispatcher.vue | 4 +- .../table/BookTableCollapsedSeriesCell.vue | 4 +- .../__tests__/BookTableCellDispatcher.spec.ts | 4 +- .../__tests__/useAddBookFile.spec.ts | 7 + .../book/composables/useAddBookFile.ts | 4 +- .../book/composables/useBookFileTree.ts | 2 +- .../features/book/lib/book-formats.test.ts | 2 +- .../widgets/CurrentlyReadingWidget.vue | 4 +- .../components/widgets/LongWaitWidget.vue | 4 +- .../email/lib/__tests__/email-files.spec.ts | 2 + client/src/features/email/lib/email-files.ts | 2 +- .../components/LibraryCreatorScanner.vue | 2 +- .../__tests__/LibraryCreatorScanner.spec.ts | 10 + .../__tests__/useBookUpload.spec.ts | 22 + .../library/composables/useBookUpload.ts | 4 +- client/src/features/reader/ReaderView.vue | 6 +- .../features/reader/cbz/CbzReaderView.spec.ts | 39 + .../src/features/reader/cbz/CbzReaderView.vue | 308 +- .../reader/cbz/composables/useCbz.spec.ts | 84 + .../features/reader/cbz/composables/useCbz.ts | 48 +- .../features/reader/djvu/DjvuTextSheet.vue | 55 + .../djvu/composables/useDjvuPageText.spec.ts | 62 + .../djvu/composables/useDjvuPageText.ts | 36 + .../__tests__/useSeriesNextBook.spec.ts | 2 +- .../shared/composables/useSeriesNextBook.ts | 2 +- .../__tests__/library-ledger-helpers.spec.ts | 3 +- .../settings/libraries/lib/library-formats.ts | 1 + client/src/locales/en.json | 17 + docs/DJVU_READER.md | 86 + packages/types/src/djvu.ts | 3 + packages/types/src/index.ts | 1 + packages/types/src/library.ts | 1 + packages/types/src/reader-settings.ts | 12 + packages/types/src/upload.ts | 1 + scripts/e2e/suite-registry.mjs | 20 + server/.env.example | 6 + server/src/app.module.ts | 4 + server/src/config/config.ts | 7 + server/src/config/env.validation.ts | 14 + .../migrations/0113_add_djvu_email_format.sql | 2 + .../src/db/migrations/meta/0113_snapshot.json | 23806 ++++++++++++++++ server/src/db/migrations/meta/_journal.json | 7 + server/src/db/schema/email-recipients.ts | 2 +- .../book-duplicate-normalize.test.ts | 1 + .../book-duplicate-normalize.ts | 2 +- .../book-duplicates.repository.test.ts | 2 + .../book-duplicates.repository.ts | 4 +- server/src/modules/book/book.controller.ts | 1 + server/src/modules/book/book.service.ts | 7 + .../dashboard-widget.repository.test.ts | 6 + .../src/modules/djvu/djvu-container.test.ts | 82 + server/src/modules/djvu/djvu-container.ts | 106 + .../src/modules/djvu/djvu-document.module.ts | 11 + .../src/modules/djvu/djvu-document.service.ts | 409 + server/src/modules/djvu/djvu-process.test.ts | 113 + server/src/modules/djvu/djvu-process.ts | 74 + .../email/dto/email-recipient.constants.ts | 2 +- .../koreader-progress-position.util.test.ts | 4 +- .../modules/library/library.service.test.ts | 20 +- .../cover-slot-reconciler.service.test.ts | 2 +- .../metadata-extraction.service.test.ts | 83 +- .../metadata/metadata-extraction.service.ts | 25 +- .../src/modules/metadata/metadata.module.ts | 3 +- .../modules/metadata/metadata.service.test.ts | 2 +- .../modules/notebook/notebook.repository.ts | 5 +- .../opds/__tests__/opds-xml.helpers.test.ts | 2 + server/src/modules/opds/opds-xml.helpers.ts | 2 + .../modules/reader/djvu/djvu.controller.ts | 45 + server/src/modules/reader/djvu/djvu.module.ts | 12 + .../modules/reader/djvu/djvu.service.test.ts | 201 + .../src/modules/reader/djvu/djvu.service.ts | 34 + .../reader/djvu/dto/djvu-page-query.dto.ts | 10 + .../src/modules/scanner/lib/classify.test.ts | 2 +- .../src/modules/scanner/lib/release-plan.ts | 20 +- .../src/modules/scanner/scanner.repository.ts | 1 + .../modules/scanner/scanner.service.test.ts | 40 + server/src/modules/scanner/scanner.service.ts | 16 +- .../dto/find-next-series-book.dto.test.ts | 22 + .../series/dto/find-next-series-book.dto.ts | 9 +- .../src/modules/series/series.service.test.ts | 13 +- server/src/modules/series/series.service.ts | 6 +- .../upload/upload-processor.service.ts | 1 + .../upload/upload-validator.service.test.ts | 3 + .../upload/upload-validator.service.ts | 2 + server/test/djvu-reader.e2e-spec.ts | 272 + server/test/e2e/djvu-fixture.ts | 14 + .../reader-state-isolation-harness.ts | 15 +- server/test/email-lifecycle.e2e-spec.ts | 15 + server/test/series-next-book.e2e-spec.ts | 27 +- 96 files changed, 26351 insertions(+), 146 deletions(-) create mode 100644 client/src/features/library/composables/__tests__/useBookUpload.spec.ts create mode 100644 client/src/features/reader/cbz/composables/useCbz.spec.ts create mode 100644 client/src/features/reader/djvu/DjvuTextSheet.vue create mode 100644 client/src/features/reader/djvu/composables/useDjvuPageText.spec.ts create mode 100644 client/src/features/reader/djvu/composables/useDjvuPageText.ts create mode 100644 docs/DJVU_READER.md create mode 100644 packages/types/src/djvu.ts create mode 100644 server/src/db/migrations/0113_add_djvu_email_format.sql create mode 100644 server/src/db/migrations/meta/0113_snapshot.json create mode 100644 server/src/modules/djvu/djvu-container.test.ts create mode 100644 server/src/modules/djvu/djvu-container.ts create mode 100644 server/src/modules/djvu/djvu-document.module.ts create mode 100644 server/src/modules/djvu/djvu-document.service.ts create mode 100644 server/src/modules/djvu/djvu-process.test.ts create mode 100644 server/src/modules/djvu/djvu-process.ts create mode 100644 server/src/modules/reader/djvu/djvu.controller.ts create mode 100644 server/src/modules/reader/djvu/djvu.module.ts create mode 100644 server/src/modules/reader/djvu/djvu.service.test.ts create mode 100644 server/src/modules/reader/djvu/djvu.service.ts create mode 100644 server/src/modules/reader/djvu/dto/djvu-page-query.dto.ts create mode 100644 server/src/modules/series/dto/find-next-series-book.dto.test.ts create mode 100644 server/test/djvu-reader.e2e-spec.ts create mode 100644 server/test/e2e/djvu-fixture.ts diff --git a/.github/workflows/e2e-runner.yml b/.github/workflows/e2e-runner.yml index 3bb6423f4..cbbbb866a 100644 --- a/.github/workflows/e2e-runner.yml +++ b/.github/workflows/e2e-runner.yml @@ -72,6 +72,10 @@ jobs: pnpm-version: ${{ env.PNPM_VERSION }} prefetch: "true" + - name: Install DjVuLibre tools + if: inputs.suite-id == 'djvu-reader' || inputs.suite-id == 'all' + run: sudo apt-get update && sudo apt-get install -y djvulibre-bin util-linux + - name: Run e2e suite shell: bash run: | diff --git a/Dockerfile b/Dockerfile index 76bb3455e..c8a860fb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,7 @@ COPY server/requirements/kobo-cloudscraper.txt /tmp/kobo-cloudscraper-requiremen # pip is build-only here. Leaving it installed also leaves pip/_vendor/vendor.txt, # which Trivy reads as installed msgpack and setuptools and fails the image scan on. RUN apk upgrade --no-cache && \ - apk add --no-cache poppler-utils su-exec ffmpeg python3 py3-pip tini tzdata && \ + apk add --no-cache poppler-utils djvulibre util-linux su-exec ffmpeg python3 py3-pip tini tzdata && \ python3 -m venv /opt/bookorbit-python && \ /opt/bookorbit-python/bin/python -m pip install --no-cache-dir -r /tmp/kobo-cloudscraper-requirements.txt && \ /opt/bookorbit-python/bin/python -m pip uninstall -y pip && \ diff --git a/client/src/features/book-dock/composables/__tests__/useBookDockUpload.spec.ts b/client/src/features/book-dock/composables/__tests__/useBookDockUpload.spec.ts index a6a146e8d..54f52bc65 100644 --- a/client/src/features/book-dock/composables/__tests__/useBookDockUpload.spec.ts +++ b/client/src/features/book-dock/composables/__tests__/useBookDockUpload.spec.ts @@ -115,6 +115,14 @@ describe('useBookDockUpload client-side validation', () => { expect(upload.failures.value).toHaveLength(0) }) + it('sends DjVu to the Book Dock instead of rejecting it as unsupported', async () => { + const { upload } = await loadComposable() + upload.addFiles([book('DjVuLibre-book.DJVU')]) + await flushPromises() + expect(FakeXhr.sent).toHaveLength(1) + expect(upload.failures.value).toHaveLength(0) + }) + it('rejects a file over the configured upload limit before sending it', async () => { const { upload, appInfo } = await loadComposable() appInfo.maxUploadSizeMb.value = 1 diff --git a/client/src/features/book-dock/composables/useBookDockUpload.ts b/client/src/features/book-dock/composables/useBookDockUpload.ts index 94ddb62f7..8ea9c5465 100644 --- a/client/src/features/book-dock/composables/useBookDockUpload.ts +++ b/client/src/features/book-dock/composables/useBookDockUpload.ts @@ -1,11 +1,11 @@ import { computed, ref } from 'vue' import { getValidToken } from '@/lib/api' -import type { BookDockFile } from '@bookorbit/types' +import { UPLOAD_SUPPORTED_FORMATS, type BookDockFile } from '@bookorbit/types' import { useAppInfo } from '@/features/settings/composables/useAppInfo' import { isRetryableUploadFailure, type BookDockUploadFailure } from '../lib/upload-failure' -export const SUPPORTED_FORMATS = ['epub', 'kepub', 'pdf', 'mobi', 'azw3', 'cbz', 'cbr', 'cb7', 'fb2', 'm4b', 'm4a', 'mp3', 'opus', 'ogg', 'flac'] +export const SUPPORTED_FORMATS: readonly string[] = UPLOAD_SUPPORTED_FORMATS export const SUPPORTED_FORMATS_ACCEPT = SUPPORTED_FORMATS.map((f) => `.${f}`).join(',') export type FileUploadStatus = 'pending' | 'uploading' | 'done' | 'error' diff --git a/client/src/features/book/components/BookQuickView.vue b/client/src/features/book/components/BookQuickView.vue index eef9b933d..c598d0cf7 100644 --- a/client/src/features/book/components/BookQuickView.vue +++ b/client/src/features/book/components/BookQuickView.vue @@ -15,7 +15,7 @@ import { useCoverVersions } from '../composables/useCoverVersions' import BookFormatChip from './BookFormatChip.vue' import { bookFormatEntries, fileFormatKey } from '../lib/book-formats' import { displayPublishedDate } from '../lib/published-date' -import { FORMAT_TO_GROUP, getPrimaryBookFile } from '@bookorbit/types' +import { FORMAT_TO_GROUP, getPrimaryBookFile, isComicFormat } from '@bookorbit/types' import { useBookCoverAspectRatio } from '../composables/useBookCoverAspectRatio' import { useDisplaySettings } from '@/composables/useDisplaySettings' import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip' @@ -98,7 +98,7 @@ const quickViewCoverAspectRatio = computed(() => { const primaryFile = computed(() => (detail.value ? getPrimaryBookFile(detail.value.files) : null)) const isPrimaryAudio = computed(() => primaryFile.value?.format != null && FORMAT_TO_GROUP[primaryFile.value.format] === 'audio') -const isPrimaryComic = computed(() => primaryFile.value?.format != null && FORMAT_TO_GROUP[primaryFile.value.format] === 'cbx') +const isPrimaryComic = computed(() => primaryFile.value?.format != null && isComicFormat(primaryFile.value.format)) const formatEntries = computed(() => (detail.value ? bookFormatEntries(detail.value.files, detail.value.formatPriority) : [])) const publishedDisplay = computed(() => (detail.value ? displayPublishedDate(detail.value.publishedDate, detail.value.publishedYear) : null)) diff --git a/client/src/features/book/components/detail/tabs/DetailsTab.vue b/client/src/features/book/components/detail/tabs/DetailsTab.vue index e54bee988..efe5292ae 100644 --- a/client/src/features/book/components/detail/tabs/DetailsTab.vue +++ b/client/src/features/book/components/detail/tabs/DetailsTab.vue @@ -32,7 +32,7 @@ import { readingDateToDateKey } from '@/features/book/lib/reading-date' import { getProviderColor, PROVIDER_SHORT_LABELS } from '@/lib/provider-colors' import { useCoverVersions } from '@/features/book/composables/useCoverVersions' import { useBookCoverAspectRatio } from '@/features/book/composables/useBookCoverAspectRatio' -import { FORMAT_TO_GROUP, getPrimaryBookFile, READER_OPENABLE_FORMATS } from '@bookorbit/types' +import { FORMAT_TO_GROUP, getPrimaryBookFile, READER_OPENABLE_FORMATS, isComicFormat } from '@bookorbit/types' import type { BookDetail, BookKoboState, CustomMetadataBookValue, ReadAloudProgressSync, ReadStatus, UserBookStatus } from '@bookorbit/types' import { STATUS_OPTIONS, STATUS_ICONS, STATUS_COLORS, useBookStatus } from '@/features/book/composables/useBookStatus' import BookDownloadButton from '@/features/book/components/BookDownloadButton.vue' @@ -364,7 +364,7 @@ const isFaceAudio = computed(() => { if (!props.book.covers.ebook && !props.book.covers.audio) return isPrimaryAudio.value return faceMedium(props.book, coverAspectRatio.value) === 'audio' }) -const isPrimaryComic = computed(() => primaryFile.value?.format != null && FORMAT_TO_GROUP[primaryFile.value.format] === 'cbx') +const isPrimaryComic = computed(() => primaryFile.value?.format != null && isComicFormat(primaryFile.value.format)) const formatEntries = computed(() => bookFormatEntries(props.book.files, props.book.formatPriority)) const isMultiTrackAudio = computed(() => formatEntries.value.some((entry) => entry.audio && entry.files.length > 1)) // A multi-file audiobook opens from its first track, so it is one entry in the menu. diff --git a/client/src/features/book/components/detail/tabs/EditMetadataTab.vue b/client/src/features/book/components/detail/tabs/EditMetadataTab.vue index f986c377c..1129e27ab 100644 --- a/client/src/features/book/components/detail/tabs/EditMetadataTab.vue +++ b/client/src/features/book/components/detail/tabs/EditMetadataTab.vue @@ -25,7 +25,14 @@ import type { MetadataProviderInfo, WriteResult, } from '@bookorbit/types' -import { BOOK_FILE_WRITE_FIELD_LABELS, FORMAT_TO_GROUP, getPrimaryBookFile, isValidSeriesIndex, parseSeriesIndex } from '@bookorbit/types' +import { + BOOK_FILE_WRITE_FIELD_LABELS, + FORMAT_TO_GROUP, + getPrimaryBookFile, + isValidSeriesIndex, + parseSeriesIndex, + isComicFormat, +} from '@bookorbit/types' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover' import { api } from '@/lib/api' @@ -119,7 +126,7 @@ const COMIC_FIELD_MAP = { const primaryFile = computed(() => getPrimaryBookFile(props.book.files)) const isPrimaryAudio = computed(() => primaryFile.value?.format != null && FORMAT_TO_GROUP[primaryFile.value.format] === 'audio') -const isPrimaryComic = computed(() => primaryFile.value?.format != null && FORMAT_TO_GROUP[primaryFile.value.format] === 'cbx') +const isPrimaryComic = computed(() => primaryFile.value?.format != null && isComicFormat(primaryFile.value.format)) const fileWriteStatus = computed(() => props.book.fileWriteStatus ?? null) const fileWriteWritableFormats = computed(() => fileWriteStatus.value?.writableFormats ?? []) const fileWriteFieldLabels = computed(() => (fileWriteStatus.value?.writableFields ?? []).map((field) => BOOK_FILE_WRITE_FIELD_LABELS[field])) diff --git a/client/src/features/book/components/table/BookTableCellDispatcher.vue b/client/src/features/book/components/table/BookTableCellDispatcher.vue index b57cd6f51..b9a3bdef6 100644 --- a/client/src/features/book/components/table/BookTableCellDispatcher.vue +++ b/client/src/features/book/components/table/BookTableCellDispatcher.vue @@ -2,7 +2,7 @@ import { computed } from 'vue' import { useI18n } from 'vue-i18n' import { Lock, LockOpen } from '@lucide/vue' -import { FORMAT_TO_GROUP, type BookCard } from '@bookorbit/types' +import { FORMAT_TO_GROUP, isComicFormat, type BookCard } from '@bookorbit/types' import BookTableCoverCell from './BookTableCoverCell.vue' import BookTableTextCell from './BookTableTextCell.vue' import BookTableNumberCell from './BookTableNumberCell.vue' @@ -97,7 +97,7 @@ function handleToggleAllLocks() { const primaryFile = computed(() => props.book.files.find((file) => file.role === 'primary') ?? props.book.files[0] ?? null) const isAudiobook = computed(() => primaryFile.value?.format != null && FORMAT_TO_GROUP[primaryFile.value.format] === 'audio') -const isComic = computed(() => primaryFile.value?.format != null && FORMAT_TO_GROUP[primaryFile.value.format] === 'cbx') +const isComic = computed(() => primaryFile.value?.format != null && isComicFormat(primaryFile.value.format))