Files
bookorbit/server/src/modules/scanner/lib/walk-ino-clamp.test.ts
T
neon 8786a18f04 fix(scanner): clamp oversized MergerFS inodes before storage (#30)
* fix(scanner): clamp oversized MergerFS inodes before storage

MergerFS generates synthetic unsigned 64-bit inodes by hashing file paths.
Values above 2^63-1 overflow PostgreSQL's signed bigint column, crashing
the scanner with 'value out of range for type bigint'.

Switch all file stat() calls to stat(path, { bigint: true }) to read
inodes as precise BigInt values instead of lossy JS numbers. Introduce
clampIno(ino: bigint): number which returns 0 for any inode exceeding
PostgreSQL's bigint max - 0 is the existing sentinel meaning 'rename
detection disabled', so no other code needs to change.

The fix covers all four code paths that write ino to a FileStat:
statFilesIntoAcc and buildSingleBookCandidate in walk.ts, plus the two
targeted-scan paths in scanner.service.ts.

Fixes #19

* ci: add maintainer-gated PR GHCR image publish

---------

Signed-off-by: neon <18559658+neonsolstice@users.noreply.github.com>
2026-05-18 17:11:28 -06:00

258 lines
8.4 KiB
TypeScript

/**
* Integration tests verifying that oversized MergerFS inodes are clamped before storage.
*
* These tests mock `fs/promises.stat` to inject synthetic BigInt inode values that would
* otherwise overflow PostgreSQL's bigint column and crash the scanner.
*/
vi.mock('fs/promises', async () => {
const actual = await vi.importActual<typeof import('fs/promises')>('fs/promises');
return { ...actual, stat: vi.fn() };
});
import { stat } from 'fs/promises';
import { mkdir, mkdtemp, rm, writeFile } from 'fs/promises';
import { join } from 'path';
import { tmpdir } from 'os';
import type { MockedFunction } from 'vitest';
import { findBookCandidates, findLooseFileCandidates, buildSingleBookCandidate } from './walk';
const mockStat = stat as MockedFunction<typeof stat>;
// Real stat from fs/promises (before the mock replaces it) is not directly accessible here,
// so we keep FS setup minimal and return fully-shaped mocks.
function makeBigIntStatsMock(ino: bigint, size = 100n): import('fs').BigIntStats {
return {
ino,
size,
mtime: new Date('2024-01-01T00:00:00Z'),
atime: new Date(),
ctime: new Date(),
birthtime: new Date(),
atimeMs: 0n,
mtimeMs: BigInt(new Date('2024-01-01T00:00:00Z').getTime()),
ctimeMs: 0n,
birthtimeMs: 0n,
blksize: 4096n,
blocks: 8n,
dev: 0n,
gid: 0n,
mode: 0o100644n,
nlink: 1n,
rdev: 0n,
uid: 0n,
isFile: () => true,
isDirectory: () => false,
isBlockDevice: () => false,
isCharacterDevice: () => false,
isFIFO: () => false,
isSocket: () => false,
isSymbolicLink: () => false,
} as unknown as import('fs').BigIntStats;
}
function makeDirStatsMock(): import('fs').Stats {
return {
mtimeMs: Date.now(),
isFile: () => false,
isDirectory: () => true,
isBlockDevice: () => false,
isCharacterDevice: () => false,
isFIFO: () => false,
isSocket: () => false,
isSymbolicLink: () => false,
} as unknown as import('fs').Stats;
}
const OVERSIZED_INO = 17237992710316634000n; // exact value from the bug report
const NORMAL_INO = 12345n;
const PG_BIGINT_MAX_INO = 9223372036854775807n;
let suiteRoot: string;
let root: string;
let caseId = 0;
beforeAll(async () => {
suiteRoot = await mkdtemp(join(tmpdir(), 'walk-ino-clamp-'));
});
beforeEach(async () => {
root = join(suiteRoot, `case-${caseId++}`);
await mkdir(root, { recursive: true });
vi.clearAllMocks();
});
afterEach(async () => {
await rm(root, { recursive: true, force: true });
vi.restoreAllMocks();
});
afterAll(async () => {
await rm(suiteRoot, { recursive: true, force: true });
});
async function touchFile(relPath: string): Promise<string> {
const p = join(root, relPath);
await mkdir(join(p, '..'), { recursive: true });
await writeFile(p, 'x');
return p;
}
// Configure the stat mock: return BigIntStats (with given ino) for file calls,
// and a Stats-like object (for dir mtime tracking) for directory calls.
function setupStatMock(fileIno: bigint): void {
mockStat.mockImplementation((filePath: unknown, opts?: unknown) => {
if ((opts as { bigint?: boolean } | undefined)?.bigint === true) {
return Promise.resolve(makeBigIntStatsMock(fileIno));
}
return Promise.resolve(makeDirStatsMock());
});
}
// ── findBookCandidates ────────────────────────────────────────────────────────
describe('findBookCandidates — oversized MergerFS inode', () => {
it('sets ino=0 when stat returns an inode exceeding PostgreSQL bigint range', async () => {
await touchFile('Book/book.epub');
setupStatMock(OVERSIZED_INO);
const { candidates } = await findBookCandidates(root);
expect(candidates).toHaveLength(1);
expect(candidates[0].files[0].ino).toBe(0);
});
it('emits a logger warning that mentions the oversized inode value', async () => {
await touchFile('Book/book.epub');
setupStatMock(OVERSIZED_INO);
const warnings: string[] = [];
await findBookCandidates(root, [], (msg) => warnings.push(msg));
expect(warnings.some((w) => w.includes(String(OVERSIZED_INO)))).toBe(true);
expect(warnings.some((w) => w.includes('rename detection disabled'))).toBe(true);
});
it('preserves normal inodes that are within PostgreSQL bigint range', async () => {
await touchFile('Book/book.epub');
setupStatMock(NORMAL_INO);
const { candidates } = await findBookCandidates(root);
expect(candidates[0].files[0].ino).toBe(Number(NORMAL_INO));
});
it('preserves the PostgreSQL bigint maximum inode without clamping', async () => {
await touchFile('Book/book.epub');
setupStatMock(PG_BIGINT_MAX_INO);
const { candidates } = await findBookCandidates(root);
expect(candidates[0].files[0].ino).not.toBe(0);
expect(candidates[0].files[0].ino).toBeGreaterThan(0);
});
it('does not emit a warning for normal inodes', async () => {
await touchFile('Book/book.epub');
setupStatMock(NORMAL_INO);
const warnings: string[] = [];
await findBookCandidates(root, [], (msg) => warnings.push(msg));
expect(warnings).toHaveLength(0);
});
it('still discovers books correctly even when all inodes are oversized', async () => {
await touchFile('AuthorA/Book1/book.epub');
await touchFile('AuthorB/Book2/book.epub');
setupStatMock(OVERSIZED_INO);
const { candidates } = await findBookCandidates(root);
expect(candidates).toHaveLength(2);
for (const c of candidates) {
expect(c.files[0].ino).toBe(0);
}
});
it('sets ino=0 for the maximum possible unsigned 64-bit inode', async () => {
await touchFile('Book/book.epub');
setupStatMock(18446744073709551615n);
const { candidates } = await findBookCandidates(root);
expect(candidates[0].files[0].ino).toBe(0);
});
});
// ── findLooseFileCandidates ───────────────────────────────────────────────────
describe('findLooseFileCandidates — oversized MergerFS inode', () => {
it('sets ino=0 for oversized inodes in book_per_file mode', async () => {
await touchFile('book.epub');
setupStatMock(OVERSIZED_INO);
const { candidates } = await findLooseFileCandidates(root);
expect(candidates).toHaveLength(1);
expect(candidates[0].files[0].ino).toBe(0);
});
it('passes normal inodes through in book_per_file mode', async () => {
await touchFile('book.epub');
setupStatMock(NORMAL_INO);
const { candidates } = await findLooseFileCandidates(root);
expect(candidates[0].files[0].ino).toBe(Number(NORMAL_INO));
});
});
// ── buildSingleBookCandidate ──────────────────────────────────────────────────
describe('buildSingleBookCandidate — oversized MergerFS inode', () => {
it('sets ino=0 when stat returns an inode exceeding PostgreSQL bigint range', async () => {
await touchFile('Book/book.epub');
setupStatMock(OVERSIZED_INO);
const result = await buildSingleBookCandidate(join(root, 'Book'), root);
expect(result).not.toBeNull();
expect(result!.files[0].ino).toBe(0);
});
it('emits a logger warning for oversized inodes', async () => {
await touchFile('Book/book.epub');
setupStatMock(OVERSIZED_INO);
const warnings: string[] = [];
await buildSingleBookCandidate(join(root, 'Book'), root, [], (msg) => warnings.push(msg));
expect(warnings.some((w) => w.includes(String(OVERSIZED_INO)))).toBe(true);
expect(warnings.some((w) => w.includes('rename detection disabled'))).toBe(true);
});
it('passes normal inodes through unchanged', async () => {
await touchFile('Book/book.epub');
setupStatMock(NORMAL_INO);
const result = await buildSingleBookCandidate(join(root, 'Book'), root);
expect(result!.files[0].ino).toBe(Number(NORMAL_INO));
});
it('still returns correct folderPath and file list with oversized inodes', async () => {
await touchFile('Book/book.epub');
await touchFile('Book/cover.jpg');
setupStatMock(OVERSIZED_INO);
const result = await buildSingleBookCandidate(join(root, 'Book'), root);
expect(result).not.toBeNull();
expect(result!.folderPath).toBe(join(root, 'Book'));
expect(result!.files).toHaveLength(2);
expect(result!.files.every((f) => f.ino === 0)).toBe(true);
});
});