fix(books): hash duplicate scan keys to prevent btree index overflow

Raw file hash and exact metadata signatures can exceed PostgreSQL's btree tuple size limit (2704 bytes) for books with large file counts or long author lists, causing duplicate scans to fail. Hashing signature values with md5 ensures indexed duplicate scan keys maintain a fixed length within PostgreSQL index limits.

Closes #762

Closes #768

server/src/modules/book-duplicates/book-duplicates.repository.test.ts

@@ -69,6 +69,37 @@ describe('BookDuplicatesRepository', () => {
     expect(query).not.toContain('SELECT DISTINCT');
   });
 
+  it('hashes the file-hash signature with md5 so the stored value stays within the btree index limit (regression: #762)', async () => {
+    const execute = vi.fn().mockResolvedValue({ rows: [] });
+    const repo = new BookDuplicatesRepository({ execute } as never);
+
+    await repo.insertFileHashKeys(9, [1, 2], { id: 7, isSuperuser: true, contentFilters: EMPTY_CONTENT_FILTER_RULES } as never);
+
+    const query = renderSql(execute.mock.calls[0]![0]).sql.replace(/\s+/g, ' ');
+    // A book with an arbitrarily large number of content files (e.g. a chapterized
+    // audiobook) must never write its full, unbounded concatenated signature as the
+    // indexed `value` - only its fixed-size md5 digest.
+    expect(query).toMatch(/'file_hash',\s*md5\(\s*array_to_string\(\s*array_agg\(/);
+    expect(query.indexOf('md5(')).toBeLessThan(query.indexOf('array_to_string('));
+    expect(query.match(/\bmd5\(/g)).toHaveLength(1);
+  });
+
+  it('hashes the exact-metadata signature with md5 so an unbounded author list cannot overflow the btree index (regression: #762)', async () => {
+    const execute = vi.fn().mockResolvedValue({ rows: [] });
+    const repo = new BookDuplicatesRepository({ execute } as never);
+
+    await repo.insertExactMetadataKeys(9, [1, 2], { id: 7, isSuperuser: true, contentFilters: EMPTY_CONTENT_FILTER_RULES } as never);
+
+    const query = renderSql(execute.mock.calls[0]![0]).sql.replace(/\s+/g, ' ');
+    // A book with an arbitrarily large author list (e.g. an anthology) must never write
+    // the raw title|authors|family concatenation as the indexed `value` - only its
+    // fixed-size md5 digest.
+    expect(query).toContain(
+      "'exact_metadata', md5(eligible.title_key || chr(31) || array_to_string(eligible.author_keys, chr(30)) || chr(31) || family)",
+    );
+    expect(query.match(/\bmd5\(/g)).toHaveLength(1);
+  });
+
   it('limits identical-file signatures to content files in accessible, non-processing books', async () => {
     const execute = vi.fn().mockResolvedValue({ rows: [] });
     const repo = new BookDuplicatesRepository({ execute } as never);

server/src/modules/book-duplicates/book-duplicates.repository.ts

@@ -133,18 +133,22 @@ export class BookDuplicatesRepository {
 
   async insertFileHashKeys(scanId: number, libraryIds: number[], user: RequestUser): Promise<void> {
     const scope = this.scopeWhere(libraryIds, user);
+    // A chapterized audiobook can have enough content files for the raw signature to
+    // exceed PostgreSQL's btree tuple limit, so only its fixed-size digest is indexed.
     await this.db.execute(sql`
       INSERT INTO ${bookDuplicateScanKeys} (scan_id, book_id, kind, value)
       SELECT
         CAST(${scanId} AS integer),
         ${books.id},
         'file_hash',
-        array_to_string(
-          array_agg(
-            lower(btrim(${bookFiles.fileHash})) || ':' || ${bookFiles.sizeBytes}::text
-            ORDER BY lower(btrim(${bookFiles.fileHash})), ${bookFiles.sizeBytes}, ${bookFiles.id}
-          ),
-          chr(30)
+        md5(
+          array_to_string(
+            array_agg(
+              lower(btrim(${bookFiles.fileHash})) || ':' || ${bookFiles.sizeBytes}::text
+              ORDER BY lower(btrim(${bookFiles.fileHash})), ${bookFiles.sizeBytes}, ${bookFiles.id}
+            ),
+            chr(30)
+          )
         )
       FROM ${books}
       JOIN ${bookFiles} ON ${bookFiles.bookId} = ${books.id}
@@ -197,6 +201,8 @@ export class BookDuplicatesRepository {
 
   async insertExactMetadataKeys(scanId: number, libraryIds: number[], user: RequestUser): Promise<void> {
     const scope = this.scopeWhere(libraryIds, user);
+    // Long titles and author lists can exceed PostgreSQL's btree tuple limit, so only
+    // the fixed-size digest of the normalized metadata signature is indexed.
     await this.db.execute(sql`
       WITH eligible AS (
         SELECT
@@ -234,7 +240,7 @@ export class BookDuplicatesRepository {
         CAST(${scanId} AS integer),
         eligible.book_id,
         'exact_metadata',
-        eligible.title_key || chr(31) || array_to_string(eligible.author_keys, chr(30)) || chr(31) || family
+        md5(eligible.title_key || chr(31) || array_to_string(eligible.author_keys, chr(30)) || chr(31) || family)
       FROM eligible
       CROSS JOIN LATERAL unnest(eligible.families) AS family
       WHERE eligible.title_key <> '' AND cardinality(eligible.author_keys) > 0