mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
fix(scanner): clamp oversized MergerFS inodes before storage (#30)
* fix(scanner): clamp oversized MergerFS inodes before storage
MergerFS generates synthetic unsigned 64-bit inodes by hashing file paths.
Values above 2^63-1 overflow PostgreSQL's signed bigint column, crashing
the scanner with 'value out of range for type bigint'.
Switch all file stat() calls to stat(path, { bigint: true }) to read
inodes as precise BigInt values instead of lossy JS numbers. Introduce
clampIno(ino: bigint): number which returns 0 for any inode exceeding
PostgreSQL's bigint max - 0 is the existing sentinel meaning 'rename
detection disabled', so no other code needs to change.
The fix covers all four code paths that write ino to a FileStat:
statFilesIntoAcc and buildSingleBookCandidate in walk.ts, plus the two
targeted-scan paths in scanner.service.ts.
Fixes #19
* ci: add maintainer-gated PR GHCR image publish
---------
Signed-off-by: neon <18559658+neonsolstice@users.noreply.github.com>
This commit is contained in:
+109
-3
@@ -23,6 +23,8 @@ env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NODE_VERSION: "24"
|
||||
PNPM_VERSION: "10"
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
dependency-review:
|
||||
@@ -246,7 +248,7 @@ jobs:
|
||||
(needs.dependency-review.result == 'success' || needs.dependency-review.result == 'skipped') &&
|
||||
needs.detect-changes.result == 'success'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
fail-fast: ${{ github.event_name == 'pull_request' }}
|
||||
matrix:
|
||||
package: [server, client]
|
||||
include:
|
||||
@@ -318,7 +320,7 @@ jobs:
|
||||
(needs.dependency-review.result == 'success' || needs.dependency-review.result == 'skipped') &&
|
||||
needs.detect-changes.result == 'success'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
fail-fast: ${{ github.event_name == 'pull_request' }}
|
||||
matrix:
|
||||
package: [server, client]
|
||||
include:
|
||||
@@ -379,7 +381,7 @@ jobs:
|
||||
(needs.dependency-review.result == 'success' || needs.dependency-review.result == 'skipped') &&
|
||||
needs.detect-changes.result == 'success'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
fail-fast: ${{ github.event_name == 'pull_request' }}
|
||||
matrix:
|
||||
package: [server, client]
|
||||
include:
|
||||
@@ -556,6 +558,110 @@ jobs:
|
||||
echo "e2e-setup=${{ needs.e2e-setup.result }}"
|
||||
echo "e2e=${{ needs.e2e.result }}"
|
||||
|
||||
pr-image-scan:
|
||||
name: PR Image Scan
|
||||
runs-on: ubuntu-latest
|
||||
needs: [ci-gate]
|
||||
timeout-minutes: 45
|
||||
if: >
|
||||
always() &&
|
||||
github.event_name == 'pull_request' &&
|
||||
needs.ci-gate.result == 'success'
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
|
||||
- name: Prepare PR image tag
|
||||
id: pr-meta
|
||||
shell: bash
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
short_sha="${HEAD_SHA:0:12}"
|
||||
scan_tag="pr-scan-${PR_NUMBER}-${short_sha}"
|
||||
echo "scan_tag=${scan_tag}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build image
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
push: false
|
||||
load: true
|
||||
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.pr-meta.outputs.scan_tag }}
|
||||
platforms: linux/amd64
|
||||
cache-from: type=gha,scope=ci-yml-pr-image
|
||||
cache-to: type=gha,mode=max,scope=ci-yml-pr-image
|
||||
|
||||
- name: Scan image with Trivy
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.pr-meta.outputs.scan_tag }}
|
||||
format: table
|
||||
vuln-type: os,library
|
||||
severity: CRITICAL,HIGH
|
||||
ignore-unfixed: true
|
||||
exit-code: "1"
|
||||
|
||||
pr-image-publish:
|
||||
name: PR Image Publish
|
||||
runs-on: ubuntu-latest
|
||||
needs: [ci-gate, pr-image-scan]
|
||||
timeout-minutes: 45
|
||||
if: >
|
||||
always() &&
|
||||
github.event_name == 'pull_request' &&
|
||||
needs.ci-gate.result == 'success' &&
|
||||
needs.pr-image-scan.result == 'success' &&
|
||||
github.event.pull_request.head.repo.full_name == github.repository &&
|
||||
contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association)
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
|
||||
- name: Prepare PR publish tag
|
||||
id: pr-meta
|
||||
shell: bash
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
short_sha="${HEAD_SHA:0:12}"
|
||||
publish_tag="pr-${PR_NUMBER}-${short_sha}"
|
||||
echo "publish_tag=${publish_tag}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GHCR_TOKEN != '' && secrets.GHCR_TOKEN || github.token }}
|
||||
|
||||
- name: Build and push PR image
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
push: true
|
||||
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.pr-meta.outputs.publish_tag }}
|
||||
platforms: linux/amd64
|
||||
cache-from: type=gha,scope=ci-yml-pr-image
|
||||
cache-to: type=gha,mode=max,scope=ci-yml-pr-image
|
||||
|
||||
ci-performance:
|
||||
name: CI Performance
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
name: Container Image
|
||||
run-name: >-
|
||||
${{ github.event_name == 'workflow_run' && format('Image - {0} ({1})', github.event.workflow_run.head_commit.message, github.event.workflow_run.head_branch) || github.event_name == 'pull_request' && format('Image check - PR #{0}', github.event.pull_request.number) || format('Image - {0}', github.ref_name) }}
|
||||
${{ github.event_name == 'workflow_run' && format('Image - {0} ({1})', github.event.workflow_run.head_commit.message, github.event.workflow_run.head_branch) || format('Image - {0}', github.ref_name) }}
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
# Break-glass: bypasses CI/E2E gate. Use only when CI is broken and a hotfix image is urgently needed.
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
@@ -18,7 +16,7 @@ on:
|
||||
branches: [main]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.sha }}
|
||||
group: ${{ github.workflow }}-${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
@@ -30,43 +28,6 @@ env:
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
# PR path: amd64-only build and scan for fast feedback. No push.
|
||||
scan:
|
||||
name: Build and Scan Image
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build image
|
||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
push: false
|
||||
load: true
|
||||
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:pr-scan
|
||||
platforms: linux/amd64
|
||||
cache-from: type=gha,scope=container-image-yml-pr
|
||||
cache-to: type=gha,mode=max,scope=container-image-yml-pr
|
||||
|
||||
- name: Scan image with Trivy
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:pr-scan
|
||||
format: table
|
||||
vuln-type: os,library
|
||||
severity: CRITICAL,HIGH
|
||||
ignore-unfixed: true
|
||||
exit-code: "1"
|
||||
|
||||
# Main push path: resolve context once, then build multi-arch natively in parallel.
|
||||
prepare:
|
||||
name: Resolve Build Context
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
/**
|
||||
* Integration tests verifying that oversized MergerFS inodes are clamped before storage.
|
||||
*
|
||||
* These tests mock `fs/promises.stat` to inject synthetic BigInt inode values that would
|
||||
* otherwise overflow PostgreSQL's bigint column and crash the scanner.
|
||||
*/
|
||||
|
||||
vi.mock('fs/promises', async () => {
|
||||
const actual = await vi.importActual<typeof import('fs/promises')>('fs/promises');
|
||||
return { ...actual, stat: vi.fn() };
|
||||
});
|
||||
|
||||
import { stat } from 'fs/promises';
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
import type { MockedFunction } from 'vitest';
|
||||
|
||||
import { findBookCandidates, findLooseFileCandidates, buildSingleBookCandidate } from './walk';
|
||||
|
||||
const mockStat = stat as MockedFunction<typeof stat>;
|
||||
|
||||
// Real stat from fs/promises (before the mock replaces it) is not directly accessible here,
|
||||
// so we keep FS setup minimal and return fully-shaped mocks.
|
||||
function makeBigIntStatsMock(ino: bigint, size = 100n): import('fs').BigIntStats {
|
||||
return {
|
||||
ino,
|
||||
size,
|
||||
mtime: new Date('2024-01-01T00:00:00Z'),
|
||||
atime: new Date(),
|
||||
ctime: new Date(),
|
||||
birthtime: new Date(),
|
||||
atimeMs: 0n,
|
||||
mtimeMs: BigInt(new Date('2024-01-01T00:00:00Z').getTime()),
|
||||
ctimeMs: 0n,
|
||||
birthtimeMs: 0n,
|
||||
blksize: 4096n,
|
||||
blocks: 8n,
|
||||
dev: 0n,
|
||||
gid: 0n,
|
||||
mode: 0o100644n,
|
||||
nlink: 1n,
|
||||
rdev: 0n,
|
||||
uid: 0n,
|
||||
isFile: () => true,
|
||||
isDirectory: () => false,
|
||||
isBlockDevice: () => false,
|
||||
isCharacterDevice: () => false,
|
||||
isFIFO: () => false,
|
||||
isSocket: () => false,
|
||||
isSymbolicLink: () => false,
|
||||
} as unknown as import('fs').BigIntStats;
|
||||
}
|
||||
|
||||
function makeDirStatsMock(): import('fs').Stats {
|
||||
return {
|
||||
mtimeMs: Date.now(),
|
||||
isFile: () => false,
|
||||
isDirectory: () => true,
|
||||
isBlockDevice: () => false,
|
||||
isCharacterDevice: () => false,
|
||||
isFIFO: () => false,
|
||||
isSocket: () => false,
|
||||
isSymbolicLink: () => false,
|
||||
} as unknown as import('fs').Stats;
|
||||
}
|
||||
|
||||
const OVERSIZED_INO = 17237992710316634000n; // exact value from the bug report
|
||||
const NORMAL_INO = 12345n;
|
||||
const PG_BIGINT_MAX_INO = 9223372036854775807n;
|
||||
|
||||
let suiteRoot: string;
|
||||
let root: string;
|
||||
let caseId = 0;
|
||||
|
||||
beforeAll(async () => {
|
||||
suiteRoot = await mkdtemp(join(tmpdir(), 'walk-ino-clamp-'));
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
root = join(suiteRoot, `case-${caseId++}`);
|
||||
await mkdir(root, { recursive: true });
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await rm(suiteRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function touchFile(relPath: string): Promise<string> {
|
||||
const p = join(root, relPath);
|
||||
await mkdir(join(p, '..'), { recursive: true });
|
||||
await writeFile(p, 'x');
|
||||
return p;
|
||||
}
|
||||
|
||||
// Configure the stat mock: return BigIntStats (with given ino) for file calls,
|
||||
// and a Stats-like object (for dir mtime tracking) for directory calls.
|
||||
function setupStatMock(fileIno: bigint): void {
|
||||
mockStat.mockImplementation((filePath: unknown, opts?: unknown) => {
|
||||
if ((opts as { bigint?: boolean } | undefined)?.bigint === true) {
|
||||
return Promise.resolve(makeBigIntStatsMock(fileIno));
|
||||
}
|
||||
return Promise.resolve(makeDirStatsMock());
|
||||
});
|
||||
}
|
||||
|
||||
// ── findBookCandidates ────────────────────────────────────────────────────────
|
||||
|
||||
describe('findBookCandidates — oversized MergerFS inode', () => {
|
||||
it('sets ino=0 when stat returns an inode exceeding PostgreSQL bigint range', async () => {
|
||||
await touchFile('Book/book.epub');
|
||||
setupStatMock(OVERSIZED_INO);
|
||||
|
||||
const { candidates } = await findBookCandidates(root);
|
||||
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].files[0].ino).toBe(0);
|
||||
});
|
||||
|
||||
it('emits a logger warning that mentions the oversized inode value', async () => {
|
||||
await touchFile('Book/book.epub');
|
||||
setupStatMock(OVERSIZED_INO);
|
||||
|
||||
const warnings: string[] = [];
|
||||
await findBookCandidates(root, [], (msg) => warnings.push(msg));
|
||||
|
||||
expect(warnings.some((w) => w.includes(String(OVERSIZED_INO)))).toBe(true);
|
||||
expect(warnings.some((w) => w.includes('rename detection disabled'))).toBe(true);
|
||||
});
|
||||
|
||||
it('preserves normal inodes that are within PostgreSQL bigint range', async () => {
|
||||
await touchFile('Book/book.epub');
|
||||
setupStatMock(NORMAL_INO);
|
||||
|
||||
const { candidates } = await findBookCandidates(root);
|
||||
|
||||
expect(candidates[0].files[0].ino).toBe(Number(NORMAL_INO));
|
||||
});
|
||||
|
||||
it('preserves the PostgreSQL bigint maximum inode without clamping', async () => {
|
||||
await touchFile('Book/book.epub');
|
||||
setupStatMock(PG_BIGINT_MAX_INO);
|
||||
|
||||
const { candidates } = await findBookCandidates(root);
|
||||
|
||||
expect(candidates[0].files[0].ino).not.toBe(0);
|
||||
expect(candidates[0].files[0].ino).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('does not emit a warning for normal inodes', async () => {
|
||||
await touchFile('Book/book.epub');
|
||||
setupStatMock(NORMAL_INO);
|
||||
|
||||
const warnings: string[] = [];
|
||||
await findBookCandidates(root, [], (msg) => warnings.push(msg));
|
||||
|
||||
expect(warnings).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('still discovers books correctly even when all inodes are oversized', async () => {
|
||||
await touchFile('AuthorA/Book1/book.epub');
|
||||
await touchFile('AuthorB/Book2/book.epub');
|
||||
setupStatMock(OVERSIZED_INO);
|
||||
|
||||
const { candidates } = await findBookCandidates(root);
|
||||
|
||||
expect(candidates).toHaveLength(2);
|
||||
for (const c of candidates) {
|
||||
expect(c.files[0].ino).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('sets ino=0 for the maximum possible unsigned 64-bit inode', async () => {
|
||||
await touchFile('Book/book.epub');
|
||||
setupStatMock(18446744073709551615n);
|
||||
|
||||
const { candidates } = await findBookCandidates(root);
|
||||
|
||||
expect(candidates[0].files[0].ino).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── findLooseFileCandidates ───────────────────────────────────────────────────
|
||||
|
||||
describe('findLooseFileCandidates — oversized MergerFS inode', () => {
|
||||
it('sets ino=0 for oversized inodes in book_per_file mode', async () => {
|
||||
await touchFile('book.epub');
|
||||
setupStatMock(OVERSIZED_INO);
|
||||
|
||||
const { candidates } = await findLooseFileCandidates(root);
|
||||
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].files[0].ino).toBe(0);
|
||||
});
|
||||
|
||||
it('passes normal inodes through in book_per_file mode', async () => {
|
||||
await touchFile('book.epub');
|
||||
setupStatMock(NORMAL_INO);
|
||||
|
||||
const { candidates } = await findLooseFileCandidates(root);
|
||||
|
||||
expect(candidates[0].files[0].ino).toBe(Number(NORMAL_INO));
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildSingleBookCandidate ──────────────────────────────────────────────────
|
||||
|
||||
describe('buildSingleBookCandidate — oversized MergerFS inode', () => {
|
||||
it('sets ino=0 when stat returns an inode exceeding PostgreSQL bigint range', async () => {
|
||||
await touchFile('Book/book.epub');
|
||||
setupStatMock(OVERSIZED_INO);
|
||||
|
||||
const result = await buildSingleBookCandidate(join(root, 'Book'), root);
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.files[0].ino).toBe(0);
|
||||
});
|
||||
|
||||
it('emits a logger warning for oversized inodes', async () => {
|
||||
await touchFile('Book/book.epub');
|
||||
setupStatMock(OVERSIZED_INO);
|
||||
|
||||
const warnings: string[] = [];
|
||||
await buildSingleBookCandidate(join(root, 'Book'), root, [], (msg) => warnings.push(msg));
|
||||
|
||||
expect(warnings.some((w) => w.includes(String(OVERSIZED_INO)))).toBe(true);
|
||||
expect(warnings.some((w) => w.includes('rename detection disabled'))).toBe(true);
|
||||
});
|
||||
|
||||
it('passes normal inodes through unchanged', async () => {
|
||||
await touchFile('Book/book.epub');
|
||||
setupStatMock(NORMAL_INO);
|
||||
|
||||
const result = await buildSingleBookCandidate(join(root, 'Book'), root);
|
||||
|
||||
expect(result!.files[0].ino).toBe(Number(NORMAL_INO));
|
||||
});
|
||||
|
||||
it('still returns correct folderPath and file list with oversized inodes', async () => {
|
||||
await touchFile('Book/book.epub');
|
||||
await touchFile('Book/cover.jpg');
|
||||
setupStatMock(OVERSIZED_INO);
|
||||
|
||||
const result = await buildSingleBookCandidate(join(root, 'Book'), root);
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(result!.folderPath).toBe(join(root, 'Book'));
|
||||
expect(result!.files).toHaveLength(2);
|
||||
expect(result!.files.every((f) => f.ino === 0)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile, chmod } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
import { findBookCandidates, buildSingleBookCandidate, findLooseFileCandidates, BookCandidate } from './walk';
|
||||
import { findBookCandidates, buildSingleBookCandidate, findLooseFileCandidates, clampIno, BookCandidate } from './walk';
|
||||
|
||||
let suiteRoot: string;
|
||||
let root: string;
|
||||
@@ -925,3 +925,46 @@ describe('incremental scan — findLooseFileCandidates', () => {
|
||||
expect(second.unchangedDirs.size).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── clampIno ─────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('clampIno', () => {
|
||||
it('passes through zero unchanged', () => {
|
||||
expect(clampIno(0n)).toBe(0);
|
||||
});
|
||||
|
||||
it('passes through typical small filesystem inodes', () => {
|
||||
expect(clampIno(1n)).toBe(1);
|
||||
expect(clampIno(12345n)).toBe(12345);
|
||||
expect(clampIno(1000000n)).toBe(1000000);
|
||||
});
|
||||
|
||||
it('passes through the maximum safe JS integer', () => {
|
||||
expect(clampIno(BigInt(Number.MAX_SAFE_INTEGER))).toBe(Number.MAX_SAFE_INTEGER);
|
||||
});
|
||||
|
||||
it('does not clamp values at the PostgreSQL bigint maximum', () => {
|
||||
// 2^63 - 1: largest value PostgreSQL bigint can store
|
||||
const pgMax = 9223372036854775807n;
|
||||
expect(clampIno(pgMax)).not.toBe(0);
|
||||
expect(clampIno(pgMax)).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('clamps values one above the PostgreSQL bigint maximum to 0', () => {
|
||||
expect(clampIno(9223372036854775808n)).toBe(0);
|
||||
});
|
||||
|
||||
it('clamps the exact MergerFS inode from the bug report to 0', () => {
|
||||
// 17237992710316634000 is the unsigned 64-bit inode reported in the MergerFS issue
|
||||
expect(clampIno(17237992710316634000n)).toBe(0);
|
||||
});
|
||||
|
||||
it('clamps the maximum unsigned 64-bit inode (2^64 - 1) to 0', () => {
|
||||
expect(clampIno(18446744073709551615n)).toBe(0);
|
||||
});
|
||||
|
||||
it('returns a number type', () => {
|
||||
expect(typeof clampIno(1n)).toBe('number');
|
||||
expect(typeof clampIno(17237992710316634000n)).toBe('number');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,6 +3,21 @@ import { basename, dirname, join, relative } from 'path';
|
||||
|
||||
import { classifyFile, isPrimaryFormat, isAudioFormat, type FileRole } from './classify';
|
||||
|
||||
const PG_BIGINT_MAX = 9223372036854775807n; // 2^63 - 1: upper bound of PostgreSQL bigint
|
||||
|
||||
/**
|
||||
* Convert a raw filesystem inode (bigint from stat with { bigint: true }) to a number
|
||||
* safe to store in PostgreSQL's signed bigint column.
|
||||
*
|
||||
* MergerFS and other synthetic filesystems produce unsigned 64-bit inodes that exceed
|
||||
* PostgreSQL's bigint range (2^63 - 1). Those are clamped to 0, which is the existing
|
||||
* sentinel value meaning "rename detection disabled for this file".
|
||||
*/
|
||||
export function clampIno(ino: bigint): number {
|
||||
if (ino > PG_BIGINT_MAX) return 0;
|
||||
return Number(ino);
|
||||
}
|
||||
|
||||
export interface FileStat {
|
||||
absolutePath: string;
|
||||
relPath: string; // relative to library folder root
|
||||
@@ -85,7 +100,7 @@ async function statFilesIntoAcc(
|
||||
): Promise<void> {
|
||||
const statResults = await Promise.all(
|
||||
filePaths.map(async (full) => {
|
||||
const s = await stat(full).catch((err: NodeJS.ErrnoException) => {
|
||||
const s = await stat(full, { bigint: true }).catch((err: NodeJS.ErrnoException) => {
|
||||
if (err.code === 'ENOENT') return null;
|
||||
throw err;
|
||||
});
|
||||
@@ -95,9 +110,11 @@ async function statFilesIntoAcc(
|
||||
for (const entry of statResults) {
|
||||
if (!entry) continue;
|
||||
const { full, s } = entry;
|
||||
const ino = Number(s.ino);
|
||||
if (ino === 0) {
|
||||
const ino = clampIno(s.ino);
|
||||
if (s.ino === 0n) {
|
||||
logger?.(`File has inode 0 (likely network mount), rename detection disabled: ${full}`);
|
||||
} else if (ino === 0) {
|
||||
logger?.(`File inode out of range for storage (MergerFS/synthetic inode ${s.ino}), rename detection disabled: ${full}`);
|
||||
}
|
||||
if (!acc.has(dir)) acc.set(dir, []);
|
||||
const { format, role } = classifyFile(full);
|
||||
@@ -105,7 +122,7 @@ async function statFilesIntoAcc(
|
||||
absolutePath: full,
|
||||
relPath: relative(libraryRoot, full),
|
||||
ino,
|
||||
sizeBytes: s.size,
|
||||
sizeBytes: Number(s.size),
|
||||
mtime: s.mtime,
|
||||
format,
|
||||
role,
|
||||
@@ -423,14 +440,18 @@ export async function buildSingleBookCandidate(
|
||||
|
||||
const stats = await Promise.all(
|
||||
filePaths.map(async (full) => {
|
||||
const s = await stat(full).catch(() => null);
|
||||
const s = await stat(full, { bigint: true }).catch(() => null);
|
||||
if (!s) return null;
|
||||
const { format, role } = classifyFile(full);
|
||||
const ino = clampIno(s.ino);
|
||||
if (s.ino > PG_BIGINT_MAX) {
|
||||
logger?.(`File inode out of range for storage (MergerFS/synthetic inode ${s.ino}), rename detection disabled: ${full}`);
|
||||
}
|
||||
return {
|
||||
absolutePath: full,
|
||||
relPath: relative(libraryFolderPath, full),
|
||||
ino: Number(s.ino),
|
||||
sizeBytes: s.size,
|
||||
ino,
|
||||
sizeBytes: Number(s.size),
|
||||
mtime: s.mtime,
|
||||
format,
|
||||
role,
|
||||
|
||||
@@ -16,7 +16,15 @@ import { readdir, stat } from 'fs/promises';
|
||||
import { classifyFile, DEFAULT_FORMAT_PRIORITY, FileRole, isAudioFormat } from './lib/classify';
|
||||
import { computeFileHash } from './lib/hash';
|
||||
import { waitForStability } from './lib/stability';
|
||||
import { BookCandidate, FileStat, findBookCandidates, findLooseFileCandidates, buildSingleBookCandidate, type WalkResult } from './lib/walk';
|
||||
import {
|
||||
BookCandidate,
|
||||
FileStat,
|
||||
findBookCandidates,
|
||||
findLooseFileCandidates,
|
||||
buildSingleBookCandidate,
|
||||
clampIno,
|
||||
type WalkResult,
|
||||
} from './lib/walk';
|
||||
import { ScannerRepository } from './scanner.repository';
|
||||
import { assembleBookCards } from '../book/utils/assemble-book-cards';
|
||||
|
||||
@@ -751,7 +759,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
return;
|
||||
}
|
||||
|
||||
const fileStat = await stat(filePath).catch(() => null);
|
||||
const fileStat = await stat(filePath, { bigint: true }).catch(() => null);
|
||||
if (!fileStat || !fileStat.isFile()) {
|
||||
this.logger.log(
|
||||
`[${event}] [end] libraryId=${libraryId} path="${sanitizeLogValue(filePath)}" durationMs=${Date.now() - startedAt} candidateFound=false scanScope=file - targeted book scan completed`,
|
||||
@@ -765,7 +773,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
{
|
||||
absolutePath: filePath,
|
||||
relPath: relative(libraryFolder.path, filePath),
|
||||
ino: Number(fileStat.ino),
|
||||
ino: clampIno(fileStat.ino),
|
||||
sizeBytes: Number(fileStat.size),
|
||||
mtime: fileStat.mtime,
|
||||
format,
|
||||
@@ -794,7 +802,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
event: string,
|
||||
startedAt: number,
|
||||
): Promise<void> {
|
||||
const fileStat = await stat(filePath).catch(() => null);
|
||||
const fileStat = await stat(filePath, { bigint: true }).catch(() => null);
|
||||
if (!fileStat || !fileStat.isFile()) {
|
||||
this.logger.log(
|
||||
`[${event}] [end] libraryId=${libraryId} path="${sanitizeLogValue(filePath)}" durationMs=${Date.now() - startedAt} candidateFound=false scanScope=root_file - targeted book scan completed`,
|
||||
@@ -824,7 +832,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
{
|
||||
absolutePath: filePath,
|
||||
relPath: relative(libraryFolder.path, filePath),
|
||||
ino: Number(fileStat.ino),
|
||||
ino: clampIno(fileStat.ino),
|
||||
sizeBytes: Number(fileStat.size),
|
||||
mtime: fileStat.mtime,
|
||||
format,
|
||||
|
||||
Reference in New Issue
Block a user