feat(reader): open DjVu documents in the web reader

Scanned books are commonly distributed as DjVu, which the scanner only stored as a supplement.
Decode pages on the server with DjVuLibre so DjVu imports with a cover and page count, opens in
the image reader with its embedded OCR text, and can be the preferred format of an ordinary email
recipient. Migration 0113 widens the recipient format check to allow it.

Decoding runs on validated private snapshots behind a bounded, reader-first queue, and
revalidation and cached pages are answered without a decoder. Released iOS builds cannot open
DjVu, so series continuation offers it only to callers sending includeDjvu=true, and the
dashboard and notebook read file never name it. The first full scan after upgrading promotes
stored .djvu supplements to content.

server/test/djvu-reader.e2e-spec.ts

@@ -0,0 +1,272 @@
+import { execFile } from 'child_process';
+import { writeFile } from 'fs/promises';
+import { join } from 'path';
+import { promisify } from 'util';
+import sharp from 'sharp';
+import { djvuBundleFixture, djvuPageFixture } from './e2e/djvu-fixture';
+import { randomUUID } from 'crypto';
+import { waitForCondition } from './e2e/app-harness';
+import {
+  authHeader,
+  closeReaderStateIsolationE2EContext,
+  createLibraryWithFolder,
+  createReaderStateIsolationE2EContext,
+  createUserAndLogin,
+  grantLibraryAccess,
+  locateBookByAbsolutePath,
+  triggerAndWaitForLibraryScan,
+  type LocatedBookFile,
+  type ReaderStateIsolationE2EContext,
+  type TestUserSession,
+} from './e2e/reader-state-isolation/reader-state-isolation-harness';
+
+describe('DjVuLibre reader (e2e)', { timeout: 60_000 }, () => {
+  let ctx: ReaderStateIsolationE2EContext;
+  let book: LocatedBookFile;
+  let reader: TestUserSession;
+  let secondReader: TestUserSession;
+  let outsider: TestUserSession;
+
+  beforeAll(async () => {
+    ctx = await createReaderStateIsolationE2EContext({ realMetadata: true, multipart: true });
+    const library = await createLibraryWithFolder(ctx);
+    const path = join(library.folderPath, 'scanned-book.djvu');
+    await writeFile(path, djvuBundleFixture());
+    await writeFile(join(library.folderPath, 'ocr.txt'), '(page 0 0 80 120 (line 0 0 80 20 (word 0 0 80 20 "Embedded sample text")))');
+    await promisify(execFile)('djvused', [path, '-s', '-e', 'select 1; set-txt ocr.txt'], { cwd: library.folderPath });
+    await triggerAndWaitForLibraryScan(ctx, library.libraryId);
+    book = await locateBookByAbsolutePath(ctx, path);
+    reader = await createUserAndLogin(ctx);
+    secondReader = await createUserAndLogin(ctx);
+    outsider = await createUserAndLogin(ctx);
+    await grantLibraryAccess(ctx, reader.userId, library.libraryId);
+    await grantLibraryAccess(ctx, secondReader.userId, library.libraryId);
+  }, 120_000);
+  afterAll(async () => {
+    if (ctx) await closeReaderStateIsolationE2EContext(ctx);
+  });
+
+  function get(suffix: string, token = reader.accessToken, extraHeaders: Record<string, string> = {}) {
+    return ctx.app.inject({
+      method: 'GET',
+      url: `/api/v1/djvu/files/${book.bookFileId}/pages${suffix}`,
+      headers: { ...authHeader(token), ...extraHeaders },
+    });
+  }
+
+  function upload(url: string, filename: string, bytes: Buffer, token = ctx.adminToken, extraHeaders: Record<string, string> = {}) {
+    const boundary = `djvu-${randomUUID()}`;
+    return ctx.app.inject({
+      method: 'POST',
+      url,
+      headers: { ...authHeader(token), 'content-type': `multipart/form-data; boundary=${boundary}`, ...extraHeaders },
+      payload: Buffer.concat([
+        Buffer.from(`--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${filename}"\r\nContent-Type: image/vnd.djvu\r\n\r\n`),
+        bytes,
+        Buffer.from(`\r\n--${boundary}--\r\n`),
+      ]),
+    });
+  }
+
+  it('extracts page count and first-page covers during scanning and exposes file metadata', async () => {
+    const detail = await ctx.app.inject({ method: 'GET', url: `/api/v1/books/${book.bookId}`, headers: authHeader(reader.accessToken) });
+    expect(detail.json()).toMatchObject({ pageCount: 2, title: 'scanned-book' });
+    const cover = await ctx.app.inject({ method: 'GET', url: `/api/v1/books/${book.bookId}/cover`, headers: authHeader(reader.accessToken) });
+    expect(cover.statusCode).toBe(200);
+    expect((await sharp(cover.rawPayload).metadata()).width).toBeGreaterThan(0);
+    const metadata = await ctx.app.inject({
+      method: 'GET',
+      url: `/api/v1/books/${book.bookId}/metadata-from-file`,
+      headers: authHeader(ctx.adminToken),
+    });
+    expect(metadata.json()).toMatchObject({ title: 'scanned-book', pageCount: 2 });
+  });
+
+  it('imports DjVu through library upload and keeps library format restrictions', async () => {
+    const library = await createLibraryWithFolder(ctx, { allowedFormats: ['djvu'] });
+    const result = await upload(`/api/v1/libraries/${library.libraryId}/upload`, 'uploaded.DJVU', djvuBundleFixture());
+    expect(result.statusCode).toBe(201);
+    expect(result.json()).toMatchObject({ format: 'djvu' });
+    const uploadedId = result.json<{ bookId: number }>().bookId;
+    await waitForCondition(
+      async () => {
+        const detail = await ctx.app.inject({ method: 'GET', url: `/api/v1/books/${uploadedId}`, headers: authHeader(ctx.adminToken) });
+        expect(detail.json<{ pageCount: number | null }>().pageCount).toBe(2);
+        const cover = await ctx.app.inject({ method: 'GET', url: `/api/v1/books/${uploadedId}/cover`, headers: authHeader(ctx.adminToken) });
+        expect(cover.statusCode).toBe(200);
+      },
+      15_000,
+      100,
+    );
+    expect((await ctx.app.inject({ method: 'GET', url: `/api/v1/books/${uploadedId}/cover`, headers: authHeader(ctx.adminToken) })).statusCode).toBe(
+      200,
+    );
+    const restricted = await createLibraryWithFolder(ctx, { allowedFormats: ['epub'] });
+    expect((await upload(`/api/v1/libraries/${restricted.libraryId}/upload`, 'rejected.djvu', djvuPageFixture())).statusCode).toBe(400);
+    expect((await upload(`/api/v1/libraries/${library.libraryId}/upload`, 'corrupt.djvu', Buffer.from('not a DjVu document'))).statusCode).toBe(400);
+  });
+
+  it('advertises DjVu and imports a resumed upload without losing document bytes', async () => {
+    const capabilities = await ctx.app.inject({ method: 'GET', url: '/api/v1/uploads/capabilities', headers: authHeader(ctx.adminToken) });
+    expect(capabilities.statusCode).toBe(200);
+    expect(capabilities.json<{ supportedFormats: string[] }>().supportedFormats).toContain('djvu');
+    const library = await createLibraryWithFolder(ctx, { allowedFormats: ['djvu'] });
+    const bytes = djvuBundleFixture();
+    const created = await ctx.app.inject({
+      method: 'POST',
+      url: '/api/v1/uploads',
+      headers: authHeader(ctx.adminToken),
+      payload: {
+        filename: 'resumed.djvu',
+        sizeBytes: bytes.length,
+        idempotencyKey: randomUUID(),
+        target: { kind: 'library', libraryId: library.libraryId },
+        contentType: 'image/vnd.djvu',
+      },
+    });
+    expect(created.statusCode).toBe(201);
+    const id = created.json<{ id: string }>().id;
+    const split = Math.floor(bytes.length / 2);
+    const first = await upload(`/api/v1/uploads/${id}/chunks`, 'chunk', bytes.subarray(0, split), ctx.adminToken, { 'upload-offset': '0' });
+    expect(first.statusCode).toBe(201);
+    const resumed = await ctx.app.inject({ method: 'GET', url: `/api/v1/uploads/${id}`, headers: authHeader(ctx.adminToken) });
+    expect(resumed.json()).toMatchObject({ receivedBytes: split, status: 'receiving' });
+    const second = await upload(`/api/v1/uploads/${id}/chunks`, 'chunk', bytes.subarray(split), ctx.adminToken, { 'upload-offset': String(split) });
+    expect(second.statusCode).toBe(201);
+    const completed = await ctx.app.inject({ method: 'POST', url: `/api/v1/uploads/${id}/complete`, headers: authHeader(ctx.adminToken) });
+    expect(completed.statusCode).toBe(202);
+    let importedId = 0;
+    await waitForCondition(
+      async () => {
+        const status = await ctx.app.inject({ method: 'GET', url: `/api/v1/uploads/${id}`, headers: authHeader(ctx.adminToken) });
+        const result = status.json<{ status: string; bookId: number }>();
+        expect(result.status).toBe('completed');
+        expect(result.bookId).toBeGreaterThan(0);
+        importedId = result.bookId;
+      },
+      15_000,
+      100,
+    );
+    const files = await ctx.app.inject({ method: 'GET', url: `/api/v1/books/${importedId}`, headers: authHeader(ctx.adminToken) });
+    const fileId = files.json<{ files: { id: number }[] }>().files[0]!.id;
+    const download = await ctx.app.inject({ method: 'GET', url: `/api/v1/books/files/${fileId}/download`, headers: authHeader(ctx.adminToken) });
+    expect(download.statusCode).toBe(200);
+    expect(download.rawPayload).toEqual(bytes);
+    const pages = await ctx.app.inject({ method: 'GET', url: `/api/v1/djvu/files/${fileId}/pages`, headers: authHeader(ctx.adminToken) });
+    expect(pages.json()).toEqual({ pageCount: 2 });
+  });
+
+  it('accepts DjVu in the Book Dock and extracts its cover and page count', async () => {
+    const result = await upload('/api/v1/book-dock/upload', 'dock.djvu', djvuPageFixture());
+    expect(result.statusCode).toBe(201);
+    const id = result.json<{ id: number }>().id;
+    await waitForCondition(
+      async () => {
+        const detail = await ctx.app.inject({ method: 'GET', url: `/api/v1/book-dock/files/${id}`, headers: authHeader(ctx.adminToken) });
+        const data = detail.json<{ status: string; embeddedMetadata: { pageCount?: number } }>();
+        expect(data.status).toBe('ready');
+        expect(data.embeddedMetadata?.pageCount).toBe(1);
+        const cover = await ctx.app.inject({ method: 'GET', url: `/api/v1/book-dock/files/${id}/cover`, headers: authHeader(ctx.adminToken) });
+        expect(cover.statusCode).toBe(200);
+      },
+      15_000,
+      100,
+    );
+  });
+
+  it('adds a DjVu edition to an existing folder book and delivers its original MIME type', async () => {
+    const library = await createLibraryWithFolder(ctx, { mode: 'book_per_folder' });
+    const first = await upload(`/api/v1/libraries/${library.libraryId}/upload`, 'base.djvu', djvuPageFixture());
+    expect(first.statusCode).toBe(201);
+    const bookId = first.json<{ bookId: number }>().bookId;
+    const added = await upload(`/api/v1/books/${bookId}/files`, 'bundle.djvu', djvuBundleFixture());
+    expect(added.statusCode).toBe(201);
+    expect(added.json()).toMatchObject({ format: 'djvu', role: 'content' });
+    const fileId = added.json<{ id: number }>().id;
+    const download = await ctx.app.inject({ method: 'GET', url: `/api/v1/books/files/${fileId}/download`, headers: authHeader(ctx.adminToken) });
+    expect(download.statusCode).toBe(200);
+    expect(download.headers['content-type']).toBe('image/vnd.djvu');
+    expect(download.rawPayload).toEqual(djvuBundleFixture());
+    const pages = await ctx.app.inject({ method: 'GET', url: `/api/v1/djvu/files/${fileId}/pages`, headers: authHeader(ctx.adminToken) });
+    expect(pages.json()).toEqual({ pageCount: 2 });
+  });
+
+  it('renders real bundled pages and returns embedded OCR', async () => {
+    const count = await get('');
+    expect(count.statusCode).toBe(200);
+    expect(count.json()).toEqual({ pageCount: 2 });
+    for (const index of [0, 1]) {
+      const page = await get(`/${index}?size=1280`);
+      expect(page.statusCode).toBe(200);
+      expect(page.headers['content-type']).toBe('image/webp');
+      expect(page.headers['cache-control']).toBe('private, no-cache');
+      const metadata = await sharp(page.rawPayload).metadata();
+      expect(metadata.format).toBe('webp');
+      expect(Math.max(metadata.width!, metadata.height!)).toBeLessThanOrEqual(1280);
+    }
+    const text = await get('/0/text');
+    expect(text.statusCode).toBe(200);
+    expect(text.json()).toEqual({ text: 'Embedded sample text' });
+    expect((await get('/1/text')).json()).toEqual({ text: '' });
+  });
+
+  it('enforces authentication and library access even after cache warming', async () => {
+    const warm = await get('/0?size=1280');
+    const etag = String(warm.headers.etag);
+    expect((await get('/0?size=1280', reader.accessToken, { 'if-none-match': etag })).statusCode).toBe(304);
+    for (const suffix of ['', '/0?size=1280', '/0/text']) {
+      expect((await get(suffix, outsider.accessToken, { 'if-none-match': etag })).statusCode).toBe(403);
+    }
+    const noAuth = await ctx.app.inject({ method: 'GET', url: `/api/v1/djvu/files/${book.bookFileId}/pages/0` });
+    expect(noAuth.statusCode).toBe(401);
+  });
+
+  it('validates pages and render sizes at the HTTP boundary', async () => {
+    expect((await get('/2')).statusCode).toBe(404);
+    expect((await get('/-1')).statusCode).toBe(404);
+    expect((await get('/0?size=99999')).statusCode).toBe(400);
+    expect((await get('/0?size=not-a-number')).statusCode).toBe(400);
+    expect((await get('/0?unknown=1')).statusCode).toBe(400);
+  });
+
+  it('saves paged progress and image-reader preferences separately for each account', async () => {
+    const saved = await ctx.app.inject({
+      method: 'POST',
+      url: `/api/v1/books/files/${book.bookFileId}/progress`,
+      headers: authHeader(reader.accessToken),
+      payload: { pageNumber: 2, percentage: 60 },
+    });
+    expect(saved.statusCode).toBe(201);
+    const progress = await ctx.app.inject({
+      method: 'GET',
+      url: `/api/v1/books/files/${book.bookFileId}/progress`,
+      headers: authHeader(reader.accessToken),
+    });
+    expect(progress.json()).toMatchObject({ pageNumber: 2, percentage: 60 });
+    const second = await ctx.app.inject({
+      method: 'GET',
+      url: `/api/v1/books/files/${book.bookFileId}/progress`,
+      headers: authHeader(secondReader.accessToken),
+    });
+    expect(second.json()?.pageNumber ?? null).toBeNull();
+    const settings = await ctx.app.inject({
+      method: 'PATCH',
+      url: `/api/v1/reader/preferences/${book.bookFileId}`,
+      headers: authHeader(reader.accessToken),
+      payload: { set: { fitMode: 'fit-width' } },
+    });
+    expect(settings.statusCode).toBe(204);
+    const preference = await ctx.app.inject({
+      method: 'GET',
+      url: `/api/v1/reader/preferences/${book.bookFileId}`,
+      headers: authHeader(reader.accessToken),
+    });
+    expect(preference.json()).toMatchObject({ settings: { fitMode: 'fit-width' }, isCustomized: true });
+    const other = await ctx.app.inject({
+      method: 'GET',
+      url: `/api/v1/reader/preferences/${book.bookFileId}`,
+      headers: authHeader(secondReader.accessToken),
+    });
+    expect(other.json()).toEqual({ settings: null, isCustomized: false });
+  });
+});

server/test/e2e/djvu-fixture.ts

@@ -0,0 +1,14 @@
+// Synthetic 80x120 color page generated with c44; no downloaded document or metadata.
+export function djvuPageFixture(): Buffer {
+  return Buffer.from(
+    'QVQmVEZPUk0AAAB5REpWVUlORk8AAAAKAFAAeBgAZAAWAUJHNDQAAAA5AEoBAgBQAHiK/4/WFVL9KZ/P+U/5tbAAX5D9Z9/+vMGwH////04eo4Tb/zvtSoSRsZHVE+e9Q+trAEJHNDQAAAAJAQ///w5zz75/AEJHNDQAAAAHAgr/8OMWzg==',
+    'base64',
+  );
+}
+
+export function djvuBundleFixture(): Buffer {
+  return Buffer.from(
+    'QVQmVEZPUk0AAAE3REpWTURJUk0AAAAngQACAAAAQAAAAML//+C/ih/u82uWgGzIJJ4n4vgfzH1u5mMh7gAeAEZPUk0AAAB5REpWVUlORk8AAAAKAFAAeBgAZAAWAUJHNDQAAAA5AEoBAgBQAHiK/4/WFVL9KZ/P+U/5tbAAX5D9Z9/+vMGwH////04eo4Tb/zvtSoSRsZHVE+e9Q+trAEJHNDQAAAAJAQ///w5zz75/AEJHNDQAAAAHAgr/8OMWzgBGT1JNAAAAeURKVlVJTkZPAAAACgBQAHgYAGQAFgFCRzQ0AAAAOQBKAQIAUAB4iv+P1hVS/Smfz/lP+bWwAF+Q/Wff/rzBsB////9OHqOE2/877UqEkbGR1RPnvUPrawBCRzQ0AAAACQEP//8Oc8++fwBCRzQ0AAAABwIK//DjFs4=',
+    'base64',
+  );
+}

server/test/e2e/reader-state-isolation/reader-state-isolation-harness.ts

@@ -1,6 +1,7 @@
 import { randomUUID } from 'crypto';
 import { hash } from 'bcryptjs';
 import fastifyCookie from '@fastify/cookie';
+import fastifyMultipart from '@fastify/multipart';
 import { ValidationPipe } from '@nestjs/common';
 import { Test } from '@nestjs/testing';
 import { FastifyAdapter, type NestFastifyApplication } from '@nestjs/platform-fastify';
@@ -65,7 +66,9 @@ export function authHeader(token: string): Record<string, string> {
   return { authorization: `Bearer ${token}` };
 }
 
-export async function createReaderStateIsolationE2EContext(): Promise<ReaderStateIsolationE2EContext> {
+export async function createReaderStateIsolationE2EContext(
+  options: { realMetadata?: boolean; multipart?: boolean } = {},
+): Promise<ReaderStateIsolationE2EContext> {
   const fixture = await createReaderStateIsolationFixtureRoot();
   const envSnapshot: EnvSnapshot = {
     appDataPath: process.env.APP_DATA_PATH,
@@ -73,12 +76,11 @@ export async function createReaderStateIsolationE2EContext(): Promise<ReaderStat
 
   process.env.APP_DATA_PATH = fixture.booksPath;
 
-  const moduleFixture = await Test.createTestingModule({
+  const builder = Test.createTestingModule({
     imports: [AppModule],
-  })
-    .overrideProvider(MetadataService)
-    .useValue(makeMetadataNoopMock())
-    .compile();
+  });
+  if (!options.realMetadata) builder.overrideProvider(MetadataService).useValue(makeMetadataNoopMock());
+  const moduleFixture = await builder.compile();
 
   const app = moduleFixture.createNestApplication<NestFastifyApplication>(new FastifyAdapter());
   app.setGlobalPrefix('api/v1');
@@ -91,6 +93,7 @@ export async function createReaderStateIsolationE2EContext(): Promise<ReaderStat
   );
   app.useGlobalFilters(new GlobalExceptionFilter());
   await app.register(fastifyCookie as never);
+  if (options.multipart) await app.register(fastifyMultipart as never);
   await app.init();
   await app.getHttpAdapter().getInstance().ready();
 

server/test/email-lifecycle.e2e-spec.ts

@@ -365,6 +365,21 @@ describe('Email lifecycle (e2e)', { timeout: 180_000 }, () => {
   });
 
   describe('recipients/groups/preferences ownership and conflicts', () => {
+    it('persists DjVu format preferences for ordinary recipients', async () => {
+      const created = await apiAs(senderOnly, 'POST', '/api/v1/email/recipients', {
+        name: 'DjVu documents',
+        email: 'djvu-reader@example.com',
+        deviceType: 'other',
+        preferredFormat: 'djvu',
+      });
+      expect(created.statusCode).toBe(201);
+      const recipient = created.json() as { id: number; preferredFormat: string };
+      expect(recipient.preferredFormat).toBe('djvu');
+      const loaded = await apiAs(senderOnly, 'GET', `/api/v1/email/recipients/${recipient.id}`);
+      expect(loaded.statusCode).toBe(200);
+      expect(loaded.json()).toMatchObject({ preferredFormat: 'djvu', deviceType: 'other' });
+    });
+
     it('enforces recipient ownership and duplicate-email conflicts', async () => {
       const recipientResponse = await apiAs(senderOnly, 'POST', '/api/v1/email/recipients', {
         name: 'Sender Kindle',

server/test/series-next-book.e2e-spec.ts

@@ -94,8 +94,11 @@ describe('Series next readable book (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, ()
   let issue10!: SeededBook;
   let unnumbered!: SeededBook;
 
-  async function requestNext(bookId: number, options: { formatGroup?: string; series?: number; token?: string } = {}) {
-    const query = options.formatGroup ? `?formatGroup=${options.formatGroup}` : '';
+  async function requestNext(bookId: number, options: { formatGroup?: string; series?: number; token?: string; includeDjvu?: boolean } = {}) {
+    const params = new URLSearchParams();
+    if (options.formatGroup) params.set('formatGroup', options.formatGroup);
+    if (options.includeDjvu) params.set('includeDjvu', 'true');
+    const query = params.size > 0 ? `?${params}` : '';
     return ctx.app.inject({
       method: 'GET',
       url: `/api/v1/series/${options.series ?? seriesId}/books/${bookId}/next${query}`,
@@ -190,6 +193,26 @@ describe('Series next readable book (e2e)', { timeout: SCENARIO_TIMEOUT_MS }, ()
     });
   });
 
+  it('offers a DjVu continuation only to a caller that opts in, so released iOS builds never receive one', async () => {
+    const library = await createLibraryWithFolder(ctx, { name: `series-next-djvu-${randomUUID()}` });
+    await grantLibraryAccess(ctx, reader.userId, library.libraryId);
+    const [scanned] = await ctx.db
+      .insert(schema.bookSeries)
+      .values({ name: 'Scanned Run', normalizedName: `scanned run ${randomUUID()}` })
+      .returning({ id: schema.bookSeries.id });
+    const first = await seedBook(ctx, library, scanned!.id, { title: 'Scan 1', seriesIndex: '1', formats: ['cbz'] });
+    const djvu = await seedBook(ctx, library, scanned!.id, { title: 'Scan 2', seriesIndex: '2', formats: ['djvu'] });
+    const third = await seedBook(ctx, library, scanned!.id, { title: 'Scan 3', seriesIndex: '3', formats: ['cbz'] });
+
+    const released = await requestNext(first.bookId, { formatGroup: 'cbx', series: scanned!.id });
+    const anyFormat = await requestNext(first.bookId, { series: scanned!.id });
+    const web = await requestNext(first.bookId, { formatGroup: 'cbx', series: scanned!.id, includeDjvu: true });
+
+    expect(released.json()).toEqual({ next: expect.objectContaining({ bookId: third.bookId, format: 'cbz' }) });
+    expect(anyFormat.json()).toEqual({ next: expect.objectContaining({ bookId: third.bookId, format: 'cbz' }) });
+    expect(web.json()).toEqual({ next: expect.objectContaining({ bookId: djvu.bookId, fileId: djvu.fileIdsByFormat.get('djvu'), format: 'djvu' }) });
+  });
+
   it('returns no next book when the book is not in the requested series', async () => {
     const response = await requestNext(issue1.bookId, { formatGroup: 'cbx', series: otherSeriesId });