mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 08:21:39 -06:00
fix(scanner): clamp oversized MergerFS inodes before storage (#30)
* fix(scanner): clamp oversized MergerFS inodes before storage
MergerFS generates synthetic unsigned 64-bit inodes by hashing file paths.
Values above 2^63-1 overflow PostgreSQL's signed bigint column, crashing
the scanner with 'value out of range for type bigint'.
Switch all file stat() calls to stat(path, { bigint: true }) to read
inodes as precise BigInt values instead of lossy JS numbers. Introduce
clampIno(ino: bigint): number which returns 0 for any inode exceeding
PostgreSQL's bigint max - 0 is the existing sentinel meaning 'rename
detection disabled', so no other code needs to change.
The fix covers all four code paths that write ino to a FileStat:
statFilesIntoAcc and buildSingleBookCandidate in walk.ts, plus the two
targeted-scan paths in scanner.service.ts.
Fixes #19
* ci: add maintainer-gated PR GHCR image publish
---------
Signed-off-by: neon <18559658+neonsolstice@users.noreply.github.com>
This commit is contained in:
+109
-3
@@ -23,6 +23,8 @@ env:
|
|||||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||||
NODE_VERSION: "24"
|
NODE_VERSION: "24"
|
||||||
PNPM_VERSION: "10"
|
PNPM_VERSION: "10"
|
||||||
|
REGISTRY: ghcr.io
|
||||||
|
IMAGE_NAME: ${{ github.repository }}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
dependency-review:
|
dependency-review:
|
||||||
@@ -246,7 +248,7 @@ jobs:
|
|||||||
(needs.dependency-review.result == 'success' || needs.dependency-review.result == 'skipped') &&
|
(needs.dependency-review.result == 'success' || needs.dependency-review.result == 'skipped') &&
|
||||||
needs.detect-changes.result == 'success'
|
needs.detect-changes.result == 'success'
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: ${{ github.event_name == 'pull_request' }}
|
||||||
matrix:
|
matrix:
|
||||||
package: [server, client]
|
package: [server, client]
|
||||||
include:
|
include:
|
||||||
@@ -318,7 +320,7 @@ jobs:
|
|||||||
(needs.dependency-review.result == 'success' || needs.dependency-review.result == 'skipped') &&
|
(needs.dependency-review.result == 'success' || needs.dependency-review.result == 'skipped') &&
|
||||||
needs.detect-changes.result == 'success'
|
needs.detect-changes.result == 'success'
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: ${{ github.event_name == 'pull_request' }}
|
||||||
matrix:
|
matrix:
|
||||||
package: [server, client]
|
package: [server, client]
|
||||||
include:
|
include:
|
||||||
@@ -379,7 +381,7 @@ jobs:
|
|||||||
(needs.dependency-review.result == 'success' || needs.dependency-review.result == 'skipped') &&
|
(needs.dependency-review.result == 'success' || needs.dependency-review.result == 'skipped') &&
|
||||||
needs.detect-changes.result == 'success'
|
needs.detect-changes.result == 'success'
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: ${{ github.event_name == 'pull_request' }}
|
||||||
matrix:
|
matrix:
|
||||||
package: [server, client]
|
package: [server, client]
|
||||||
include:
|
include:
|
||||||
@@ -556,6 +558,110 @@ jobs:
|
|||||||
echo "e2e-setup=${{ needs.e2e-setup.result }}"
|
echo "e2e-setup=${{ needs.e2e-setup.result }}"
|
||||||
echo "e2e=${{ needs.e2e.result }}"
|
echo "e2e=${{ needs.e2e.result }}"
|
||||||
|
|
||||||
|
pr-image-scan:
|
||||||
|
name: PR Image Scan
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: [ci-gate]
|
||||||
|
timeout-minutes: 45
|
||||||
|
if: >
|
||||||
|
always() &&
|
||||||
|
github.event_name == 'pull_request' &&
|
||||||
|
needs.ci-gate.result == 'success'
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||||
|
|
||||||
|
- name: Prepare PR image tag
|
||||||
|
id: pr-meta
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||||
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
short_sha="${HEAD_SHA:0:12}"
|
||||||
|
scan_tag="pr-scan-${PR_NUMBER}-${short_sha}"
|
||||||
|
echo "scan_tag=${scan_tag}" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||||
|
|
||||||
|
- name: Build image
|
||||||
|
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./Dockerfile
|
||||||
|
push: false
|
||||||
|
load: true
|
||||||
|
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.pr-meta.outputs.scan_tag }}
|
||||||
|
platforms: linux/amd64
|
||||||
|
cache-from: type=gha,scope=ci-yml-pr-image
|
||||||
|
cache-to: type=gha,mode=max,scope=ci-yml-pr-image
|
||||||
|
|
||||||
|
- name: Scan image with Trivy
|
||||||
|
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||||
|
with:
|
||||||
|
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.pr-meta.outputs.scan_tag }}
|
||||||
|
format: table
|
||||||
|
vuln-type: os,library
|
||||||
|
severity: CRITICAL,HIGH
|
||||||
|
ignore-unfixed: true
|
||||||
|
exit-code: "1"
|
||||||
|
|
||||||
|
pr-image-publish:
|
||||||
|
name: PR Image Publish
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
needs: [ci-gate, pr-image-scan]
|
||||||
|
timeout-minutes: 45
|
||||||
|
if: >
|
||||||
|
always() &&
|
||||||
|
github.event_name == 'pull_request' &&
|
||||||
|
needs.ci-gate.result == 'success' &&
|
||||||
|
needs.pr-image-scan.result == 'success' &&
|
||||||
|
github.event.pull_request.head.repo.full_name == github.repository &&
|
||||||
|
contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association)
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
packages: write
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
||||||
|
|
||||||
|
- name: Prepare PR publish tag
|
||||||
|
id: pr-meta
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||||
|
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
short_sha="${HEAD_SHA:0:12}"
|
||||||
|
publish_tag="pr-${PR_NUMBER}-${short_sha}"
|
||||||
|
echo "publish_tag=${publish_tag}" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||||
|
|
||||||
|
- name: Log in to GHCR
|
||||||
|
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||||
|
with:
|
||||||
|
registry: ${{ env.REGISTRY }}
|
||||||
|
username: ${{ github.repository_owner }}
|
||||||
|
password: ${{ secrets.GHCR_TOKEN != '' && secrets.GHCR_TOKEN || github.token }}
|
||||||
|
|
||||||
|
- name: Build and push PR image
|
||||||
|
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./Dockerfile
|
||||||
|
push: true
|
||||||
|
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.pr-meta.outputs.publish_tag }}
|
||||||
|
platforms: linux/amd64
|
||||||
|
cache-from: type=gha,scope=ci-yml-pr-image
|
||||||
|
cache-to: type=gha,mode=max,scope=ci-yml-pr-image
|
||||||
|
|
||||||
ci-performance:
|
ci-performance:
|
||||||
name: CI Performance
|
name: CI Performance
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
|||||||
@@ -1,10 +1,8 @@
|
|||||||
name: Container Image
|
name: Container Image
|
||||||
run-name: >-
|
run-name: >-
|
||||||
${{ github.event_name == 'workflow_run' && format('Image - {0} ({1})', github.event.workflow_run.head_commit.message, github.event.workflow_run.head_branch) || github.event_name == 'pull_request' && format('Image check - PR #{0}', github.event.pull_request.number) || format('Image - {0}', github.ref_name) }}
|
${{ github.event_name == 'workflow_run' && format('Image - {0} ({1})', github.event.workflow_run.head_commit.message, github.event.workflow_run.head_branch) || format('Image - {0}', github.ref_name) }}
|
||||||
|
|
||||||
on:
|
on:
|
||||||
pull_request:
|
|
||||||
branches: [main]
|
|
||||||
# Break-glass: bypasses CI/E2E gate. Use only when CI is broken and a hotfix image is urgently needed.
|
# Break-glass: bypasses CI/E2E gate. Use only when CI is broken and a hotfix image is urgently needed.
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
@@ -18,7 +16,7 @@ on:
|
|||||||
branches: [main]
|
branches: [main]
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: ${{ github.workflow }}-${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.sha }}
|
group: ${{ github.workflow }}-${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }}
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
@@ -30,43 +28,6 @@ env:
|
|||||||
IMAGE_NAME: ${{ github.repository }}
|
IMAGE_NAME: ${{ github.repository }}
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# PR path: amd64-only build and scan for fast feedback. No push.
|
|
||||||
scan:
|
|
||||||
name: Build and Scan Image
|
|
||||||
if: github.event_name == 'pull_request'
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
timeout-minutes: 45
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
steps:
|
|
||||||
- name: Checkout
|
|
||||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
|
||||||
|
|
||||||
- name: Build image
|
|
||||||
uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: ./Dockerfile
|
|
||||||
push: false
|
|
||||||
load: true
|
|
||||||
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:pr-scan
|
|
||||||
platforms: linux/amd64
|
|
||||||
cache-from: type=gha,scope=container-image-yml-pr
|
|
||||||
cache-to: type=gha,mode=max,scope=container-image-yml-pr
|
|
||||||
|
|
||||||
- name: Scan image with Trivy
|
|
||||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
|
||||||
with:
|
|
||||||
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:pr-scan
|
|
||||||
format: table
|
|
||||||
vuln-type: os,library
|
|
||||||
severity: CRITICAL,HIGH
|
|
||||||
ignore-unfixed: true
|
|
||||||
exit-code: "1"
|
|
||||||
|
|
||||||
# Main push path: resolve context once, then build multi-arch natively in parallel.
|
# Main push path: resolve context once, then build multi-arch natively in parallel.
|
||||||
prepare:
|
prepare:
|
||||||
name: Resolve Build Context
|
name: Resolve Build Context
|
||||||
|
|||||||
@@ -0,0 +1,257 @@
|
|||||||
|
/**
|
||||||
|
* Integration tests verifying that oversized MergerFS inodes are clamped before storage.
|
||||||
|
*
|
||||||
|
* These tests mock `fs/promises.stat` to inject synthetic BigInt inode values that would
|
||||||
|
* otherwise overflow PostgreSQL's bigint column and crash the scanner.
|
||||||
|
*/
|
||||||
|
|
||||||
|
vi.mock('fs/promises', async () => {
|
||||||
|
const actual = await vi.importActual<typeof import('fs/promises')>('fs/promises');
|
||||||
|
return { ...actual, stat: vi.fn() };
|
||||||
|
});
|
||||||
|
|
||||||
|
import { stat } from 'fs/promises';
|
||||||
|
import { mkdir, mkdtemp, rm, writeFile } from 'fs/promises';
|
||||||
|
import { join } from 'path';
|
||||||
|
import { tmpdir } from 'os';
|
||||||
|
import type { MockedFunction } from 'vitest';
|
||||||
|
|
||||||
|
import { findBookCandidates, findLooseFileCandidates, buildSingleBookCandidate } from './walk';
|
||||||
|
|
||||||
|
const mockStat = stat as MockedFunction<typeof stat>;
|
||||||
|
|
||||||
|
// Real stat from fs/promises (before the mock replaces it) is not directly accessible here,
|
||||||
|
// so we keep FS setup minimal and return fully-shaped mocks.
|
||||||
|
function makeBigIntStatsMock(ino: bigint, size = 100n): import('fs').BigIntStats {
|
||||||
|
return {
|
||||||
|
ino,
|
||||||
|
size,
|
||||||
|
mtime: new Date('2024-01-01T00:00:00Z'),
|
||||||
|
atime: new Date(),
|
||||||
|
ctime: new Date(),
|
||||||
|
birthtime: new Date(),
|
||||||
|
atimeMs: 0n,
|
||||||
|
mtimeMs: BigInt(new Date('2024-01-01T00:00:00Z').getTime()),
|
||||||
|
ctimeMs: 0n,
|
||||||
|
birthtimeMs: 0n,
|
||||||
|
blksize: 4096n,
|
||||||
|
blocks: 8n,
|
||||||
|
dev: 0n,
|
||||||
|
gid: 0n,
|
||||||
|
mode: 0o100644n,
|
||||||
|
nlink: 1n,
|
||||||
|
rdev: 0n,
|
||||||
|
uid: 0n,
|
||||||
|
isFile: () => true,
|
||||||
|
isDirectory: () => false,
|
||||||
|
isBlockDevice: () => false,
|
||||||
|
isCharacterDevice: () => false,
|
||||||
|
isFIFO: () => false,
|
||||||
|
isSocket: () => false,
|
||||||
|
isSymbolicLink: () => false,
|
||||||
|
} as unknown as import('fs').BigIntStats;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeDirStatsMock(): import('fs').Stats {
|
||||||
|
return {
|
||||||
|
mtimeMs: Date.now(),
|
||||||
|
isFile: () => false,
|
||||||
|
isDirectory: () => true,
|
||||||
|
isBlockDevice: () => false,
|
||||||
|
isCharacterDevice: () => false,
|
||||||
|
isFIFO: () => false,
|
||||||
|
isSocket: () => false,
|
||||||
|
isSymbolicLink: () => false,
|
||||||
|
} as unknown as import('fs').Stats;
|
||||||
|
}
|
||||||
|
|
||||||
|
const OVERSIZED_INO = 17237992710316634000n; // exact value from the bug report
|
||||||
|
const NORMAL_INO = 12345n;
|
||||||
|
const PG_BIGINT_MAX_INO = 9223372036854775807n;
|
||||||
|
|
||||||
|
let suiteRoot: string;
|
||||||
|
let root: string;
|
||||||
|
let caseId = 0;
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
suiteRoot = await mkdtemp(join(tmpdir(), 'walk-ino-clamp-'));
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
root = join(suiteRoot, `case-${caseId++}`);
|
||||||
|
await mkdir(root, { recursive: true });
|
||||||
|
vi.clearAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
await rm(root, { recursive: true, force: true });
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterAll(async () => {
|
||||||
|
await rm(suiteRoot, { recursive: true, force: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
async function touchFile(relPath: string): Promise<string> {
|
||||||
|
const p = join(root, relPath);
|
||||||
|
await mkdir(join(p, '..'), { recursive: true });
|
||||||
|
await writeFile(p, 'x');
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Configure the stat mock: return BigIntStats (with given ino) for file calls,
|
||||||
|
// and a Stats-like object (for dir mtime tracking) for directory calls.
|
||||||
|
function setupStatMock(fileIno: bigint): void {
|
||||||
|
mockStat.mockImplementation((filePath: unknown, opts?: unknown) => {
|
||||||
|
if ((opts as { bigint?: boolean } | undefined)?.bigint === true) {
|
||||||
|
return Promise.resolve(makeBigIntStatsMock(fileIno));
|
||||||
|
}
|
||||||
|
return Promise.resolve(makeDirStatsMock());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── findBookCandidates ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
describe('findBookCandidates — oversized MergerFS inode', () => {
|
||||||
|
it('sets ino=0 when stat returns an inode exceeding PostgreSQL bigint range', async () => {
|
||||||
|
await touchFile('Book/book.epub');
|
||||||
|
setupStatMock(OVERSIZED_INO);
|
||||||
|
|
||||||
|
const { candidates } = await findBookCandidates(root);
|
||||||
|
|
||||||
|
expect(candidates).toHaveLength(1);
|
||||||
|
expect(candidates[0].files[0].ino).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('emits a logger warning that mentions the oversized inode value', async () => {
|
||||||
|
await touchFile('Book/book.epub');
|
||||||
|
setupStatMock(OVERSIZED_INO);
|
||||||
|
|
||||||
|
const warnings: string[] = [];
|
||||||
|
await findBookCandidates(root, [], (msg) => warnings.push(msg));
|
||||||
|
|
||||||
|
expect(warnings.some((w) => w.includes(String(OVERSIZED_INO)))).toBe(true);
|
||||||
|
expect(warnings.some((w) => w.includes('rename detection disabled'))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('preserves normal inodes that are within PostgreSQL bigint range', async () => {
|
||||||
|
await touchFile('Book/book.epub');
|
||||||
|
setupStatMock(NORMAL_INO);
|
||||||
|
|
||||||
|
const { candidates } = await findBookCandidates(root);
|
||||||
|
|
||||||
|
expect(candidates[0].files[0].ino).toBe(Number(NORMAL_INO));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('preserves the PostgreSQL bigint maximum inode without clamping', async () => {
|
||||||
|
await touchFile('Book/book.epub');
|
||||||
|
setupStatMock(PG_BIGINT_MAX_INO);
|
||||||
|
|
||||||
|
const { candidates } = await findBookCandidates(root);
|
||||||
|
|
||||||
|
expect(candidates[0].files[0].ino).not.toBe(0);
|
||||||
|
expect(candidates[0].files[0].ino).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not emit a warning for normal inodes', async () => {
|
||||||
|
await touchFile('Book/book.epub');
|
||||||
|
setupStatMock(NORMAL_INO);
|
||||||
|
|
||||||
|
const warnings: string[] = [];
|
||||||
|
await findBookCandidates(root, [], (msg) => warnings.push(msg));
|
||||||
|
|
||||||
|
expect(warnings).toHaveLength(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('still discovers books correctly even when all inodes are oversized', async () => {
|
||||||
|
await touchFile('AuthorA/Book1/book.epub');
|
||||||
|
await touchFile('AuthorB/Book2/book.epub');
|
||||||
|
setupStatMock(OVERSIZED_INO);
|
||||||
|
|
||||||
|
const { candidates } = await findBookCandidates(root);
|
||||||
|
|
||||||
|
expect(candidates).toHaveLength(2);
|
||||||
|
for (const c of candidates) {
|
||||||
|
expect(c.files[0].ino).toBe(0);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('sets ino=0 for the maximum possible unsigned 64-bit inode', async () => {
|
||||||
|
await touchFile('Book/book.epub');
|
||||||
|
setupStatMock(18446744073709551615n);
|
||||||
|
|
||||||
|
const { candidates } = await findBookCandidates(root);
|
||||||
|
|
||||||
|
expect(candidates[0].files[0].ino).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── findLooseFileCandidates ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
describe('findLooseFileCandidates — oversized MergerFS inode', () => {
|
||||||
|
it('sets ino=0 for oversized inodes in book_per_file mode', async () => {
|
||||||
|
await touchFile('book.epub');
|
||||||
|
setupStatMock(OVERSIZED_INO);
|
||||||
|
|
||||||
|
const { candidates } = await findLooseFileCandidates(root);
|
||||||
|
|
||||||
|
expect(candidates).toHaveLength(1);
|
||||||
|
expect(candidates[0].files[0].ino).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('passes normal inodes through in book_per_file mode', async () => {
|
||||||
|
await touchFile('book.epub');
|
||||||
|
setupStatMock(NORMAL_INO);
|
||||||
|
|
||||||
|
const { candidates } = await findLooseFileCandidates(root);
|
||||||
|
|
||||||
|
expect(candidates[0].files[0].ino).toBe(Number(NORMAL_INO));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── buildSingleBookCandidate ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
describe('buildSingleBookCandidate — oversized MergerFS inode', () => {
|
||||||
|
it('sets ino=0 when stat returns an inode exceeding PostgreSQL bigint range', async () => {
|
||||||
|
await touchFile('Book/book.epub');
|
||||||
|
setupStatMock(OVERSIZED_INO);
|
||||||
|
|
||||||
|
const result = await buildSingleBookCandidate(join(root, 'Book'), root);
|
||||||
|
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result!.files[0].ino).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('emits a logger warning for oversized inodes', async () => {
|
||||||
|
await touchFile('Book/book.epub');
|
||||||
|
setupStatMock(OVERSIZED_INO);
|
||||||
|
|
||||||
|
const warnings: string[] = [];
|
||||||
|
await buildSingleBookCandidate(join(root, 'Book'), root, [], (msg) => warnings.push(msg));
|
||||||
|
|
||||||
|
expect(warnings.some((w) => w.includes(String(OVERSIZED_INO)))).toBe(true);
|
||||||
|
expect(warnings.some((w) => w.includes('rename detection disabled'))).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('passes normal inodes through unchanged', async () => {
|
||||||
|
await touchFile('Book/book.epub');
|
||||||
|
setupStatMock(NORMAL_INO);
|
||||||
|
|
||||||
|
const result = await buildSingleBookCandidate(join(root, 'Book'), root);
|
||||||
|
|
||||||
|
expect(result!.files[0].ino).toBe(Number(NORMAL_INO));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('still returns correct folderPath and file list with oversized inodes', async () => {
|
||||||
|
await touchFile('Book/book.epub');
|
||||||
|
await touchFile('Book/cover.jpg');
|
||||||
|
setupStatMock(OVERSIZED_INO);
|
||||||
|
|
||||||
|
const result = await buildSingleBookCandidate(join(root, 'Book'), root);
|
||||||
|
|
||||||
|
expect(result).not.toBeNull();
|
||||||
|
expect(result!.folderPath).toBe(join(root, 'Book'));
|
||||||
|
expect(result!.files).toHaveLength(2);
|
||||||
|
expect(result!.files.every((f) => f.ino === 0)).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -2,7 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile, chmod } from 'fs/promises';
|
|||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
import { tmpdir } from 'os';
|
import { tmpdir } from 'os';
|
||||||
|
|
||||||
import { findBookCandidates, buildSingleBookCandidate, findLooseFileCandidates, BookCandidate } from './walk';
|
import { findBookCandidates, buildSingleBookCandidate, findLooseFileCandidates, clampIno, BookCandidate } from './walk';
|
||||||
|
|
||||||
let suiteRoot: string;
|
let suiteRoot: string;
|
||||||
let root: string;
|
let root: string;
|
||||||
@@ -925,3 +925,46 @@ describe('incremental scan — findLooseFileCandidates', () => {
|
|||||||
expect(second.unchangedDirs.size).toBeGreaterThan(0);
|
expect(second.unchangedDirs.size).toBeGreaterThan(0);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── clampIno ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
describe('clampIno', () => {
|
||||||
|
it('passes through zero unchanged', () => {
|
||||||
|
expect(clampIno(0n)).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('passes through typical small filesystem inodes', () => {
|
||||||
|
expect(clampIno(1n)).toBe(1);
|
||||||
|
expect(clampIno(12345n)).toBe(12345);
|
||||||
|
expect(clampIno(1000000n)).toBe(1000000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('passes through the maximum safe JS integer', () => {
|
||||||
|
expect(clampIno(BigInt(Number.MAX_SAFE_INTEGER))).toBe(Number.MAX_SAFE_INTEGER);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not clamp values at the PostgreSQL bigint maximum', () => {
|
||||||
|
// 2^63 - 1: largest value PostgreSQL bigint can store
|
||||||
|
const pgMax = 9223372036854775807n;
|
||||||
|
expect(clampIno(pgMax)).not.toBe(0);
|
||||||
|
expect(clampIno(pgMax)).toBeGreaterThan(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clamps values one above the PostgreSQL bigint maximum to 0', () => {
|
||||||
|
expect(clampIno(9223372036854775808n)).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clamps the exact MergerFS inode from the bug report to 0', () => {
|
||||||
|
// 17237992710316634000 is the unsigned 64-bit inode reported in the MergerFS issue
|
||||||
|
expect(clampIno(17237992710316634000n)).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('clamps the maximum unsigned 64-bit inode (2^64 - 1) to 0', () => {
|
||||||
|
expect(clampIno(18446744073709551615n)).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('returns a number type', () => {
|
||||||
|
expect(typeof clampIno(1n)).toBe('number');
|
||||||
|
expect(typeof clampIno(17237992710316634000n)).toBe('number');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -3,6 +3,21 @@ import { basename, dirname, join, relative } from 'path';
|
|||||||
|
|
||||||
import { classifyFile, isPrimaryFormat, isAudioFormat, type FileRole } from './classify';
|
import { classifyFile, isPrimaryFormat, isAudioFormat, type FileRole } from './classify';
|
||||||
|
|
||||||
|
const PG_BIGINT_MAX = 9223372036854775807n; // 2^63 - 1: upper bound of PostgreSQL bigint
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert a raw filesystem inode (bigint from stat with { bigint: true }) to a number
|
||||||
|
* safe to store in PostgreSQL's signed bigint column.
|
||||||
|
*
|
||||||
|
* MergerFS and other synthetic filesystems produce unsigned 64-bit inodes that exceed
|
||||||
|
* PostgreSQL's bigint range (2^63 - 1). Those are clamped to 0, which is the existing
|
||||||
|
* sentinel value meaning "rename detection disabled for this file".
|
||||||
|
*/
|
||||||
|
export function clampIno(ino: bigint): number {
|
||||||
|
if (ino > PG_BIGINT_MAX) return 0;
|
||||||
|
return Number(ino);
|
||||||
|
}
|
||||||
|
|
||||||
export interface FileStat {
|
export interface FileStat {
|
||||||
absolutePath: string;
|
absolutePath: string;
|
||||||
relPath: string; // relative to library folder root
|
relPath: string; // relative to library folder root
|
||||||
@@ -85,7 +100,7 @@ async function statFilesIntoAcc(
|
|||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const statResults = await Promise.all(
|
const statResults = await Promise.all(
|
||||||
filePaths.map(async (full) => {
|
filePaths.map(async (full) => {
|
||||||
const s = await stat(full).catch((err: NodeJS.ErrnoException) => {
|
const s = await stat(full, { bigint: true }).catch((err: NodeJS.ErrnoException) => {
|
||||||
if (err.code === 'ENOENT') return null;
|
if (err.code === 'ENOENT') return null;
|
||||||
throw err;
|
throw err;
|
||||||
});
|
});
|
||||||
@@ -95,9 +110,11 @@ async function statFilesIntoAcc(
|
|||||||
for (const entry of statResults) {
|
for (const entry of statResults) {
|
||||||
if (!entry) continue;
|
if (!entry) continue;
|
||||||
const { full, s } = entry;
|
const { full, s } = entry;
|
||||||
const ino = Number(s.ino);
|
const ino = clampIno(s.ino);
|
||||||
if (ino === 0) {
|
if (s.ino === 0n) {
|
||||||
logger?.(`File has inode 0 (likely network mount), rename detection disabled: ${full}`);
|
logger?.(`File has inode 0 (likely network mount), rename detection disabled: ${full}`);
|
||||||
|
} else if (ino === 0) {
|
||||||
|
logger?.(`File inode out of range for storage (MergerFS/synthetic inode ${s.ino}), rename detection disabled: ${full}`);
|
||||||
}
|
}
|
||||||
if (!acc.has(dir)) acc.set(dir, []);
|
if (!acc.has(dir)) acc.set(dir, []);
|
||||||
const { format, role } = classifyFile(full);
|
const { format, role } = classifyFile(full);
|
||||||
@@ -105,7 +122,7 @@ async function statFilesIntoAcc(
|
|||||||
absolutePath: full,
|
absolutePath: full,
|
||||||
relPath: relative(libraryRoot, full),
|
relPath: relative(libraryRoot, full),
|
||||||
ino,
|
ino,
|
||||||
sizeBytes: s.size,
|
sizeBytes: Number(s.size),
|
||||||
mtime: s.mtime,
|
mtime: s.mtime,
|
||||||
format,
|
format,
|
||||||
role,
|
role,
|
||||||
@@ -423,14 +440,18 @@ export async function buildSingleBookCandidate(
|
|||||||
|
|
||||||
const stats = await Promise.all(
|
const stats = await Promise.all(
|
||||||
filePaths.map(async (full) => {
|
filePaths.map(async (full) => {
|
||||||
const s = await stat(full).catch(() => null);
|
const s = await stat(full, { bigint: true }).catch(() => null);
|
||||||
if (!s) return null;
|
if (!s) return null;
|
||||||
const { format, role } = classifyFile(full);
|
const { format, role } = classifyFile(full);
|
||||||
|
const ino = clampIno(s.ino);
|
||||||
|
if (s.ino > PG_BIGINT_MAX) {
|
||||||
|
logger?.(`File inode out of range for storage (MergerFS/synthetic inode ${s.ino}), rename detection disabled: ${full}`);
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
absolutePath: full,
|
absolutePath: full,
|
||||||
relPath: relative(libraryFolderPath, full),
|
relPath: relative(libraryFolderPath, full),
|
||||||
ino: Number(s.ino),
|
ino,
|
||||||
sizeBytes: s.size,
|
sizeBytes: Number(s.size),
|
||||||
mtime: s.mtime,
|
mtime: s.mtime,
|
||||||
format,
|
format,
|
||||||
role,
|
role,
|
||||||
|
|||||||
@@ -16,7 +16,15 @@ import { readdir, stat } from 'fs/promises';
|
|||||||
import { classifyFile, DEFAULT_FORMAT_PRIORITY, FileRole, isAudioFormat } from './lib/classify';
|
import { classifyFile, DEFAULT_FORMAT_PRIORITY, FileRole, isAudioFormat } from './lib/classify';
|
||||||
import { computeFileHash } from './lib/hash';
|
import { computeFileHash } from './lib/hash';
|
||||||
import { waitForStability } from './lib/stability';
|
import { waitForStability } from './lib/stability';
|
||||||
import { BookCandidate, FileStat, findBookCandidates, findLooseFileCandidates, buildSingleBookCandidate, type WalkResult } from './lib/walk';
|
import {
|
||||||
|
BookCandidate,
|
||||||
|
FileStat,
|
||||||
|
findBookCandidates,
|
||||||
|
findLooseFileCandidates,
|
||||||
|
buildSingleBookCandidate,
|
||||||
|
clampIno,
|
||||||
|
type WalkResult,
|
||||||
|
} from './lib/walk';
|
||||||
import { ScannerRepository } from './scanner.repository';
|
import { ScannerRepository } from './scanner.repository';
|
||||||
import { assembleBookCards } from '../book/utils/assemble-book-cards';
|
import { assembleBookCards } from '../book/utils/assemble-book-cards';
|
||||||
|
|
||||||
@@ -751,7 +759,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const fileStat = await stat(filePath).catch(() => null);
|
const fileStat = await stat(filePath, { bigint: true }).catch(() => null);
|
||||||
if (!fileStat || !fileStat.isFile()) {
|
if (!fileStat || !fileStat.isFile()) {
|
||||||
this.logger.log(
|
this.logger.log(
|
||||||
`[${event}] [end] libraryId=${libraryId} path="${sanitizeLogValue(filePath)}" durationMs=${Date.now() - startedAt} candidateFound=false scanScope=file - targeted book scan completed`,
|
`[${event}] [end] libraryId=${libraryId} path="${sanitizeLogValue(filePath)}" durationMs=${Date.now() - startedAt} candidateFound=false scanScope=file - targeted book scan completed`,
|
||||||
@@ -765,7 +773,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
|||||||
{
|
{
|
||||||
absolutePath: filePath,
|
absolutePath: filePath,
|
||||||
relPath: relative(libraryFolder.path, filePath),
|
relPath: relative(libraryFolder.path, filePath),
|
||||||
ino: Number(fileStat.ino),
|
ino: clampIno(fileStat.ino),
|
||||||
sizeBytes: Number(fileStat.size),
|
sizeBytes: Number(fileStat.size),
|
||||||
mtime: fileStat.mtime,
|
mtime: fileStat.mtime,
|
||||||
format,
|
format,
|
||||||
@@ -794,7 +802,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
|||||||
event: string,
|
event: string,
|
||||||
startedAt: number,
|
startedAt: number,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const fileStat = await stat(filePath).catch(() => null);
|
const fileStat = await stat(filePath, { bigint: true }).catch(() => null);
|
||||||
if (!fileStat || !fileStat.isFile()) {
|
if (!fileStat || !fileStat.isFile()) {
|
||||||
this.logger.log(
|
this.logger.log(
|
||||||
`[${event}] [end] libraryId=${libraryId} path="${sanitizeLogValue(filePath)}" durationMs=${Date.now() - startedAt} candidateFound=false scanScope=root_file - targeted book scan completed`,
|
`[${event}] [end] libraryId=${libraryId} path="${sanitizeLogValue(filePath)}" durationMs=${Date.now() - startedAt} candidateFound=false scanScope=root_file - targeted book scan completed`,
|
||||||
@@ -824,7 +832,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
|||||||
{
|
{
|
||||||
absolutePath: filePath,
|
absolutePath: filePath,
|
||||||
relPath: relative(libraryFolder.path, filePath),
|
relPath: relative(libraryFolder.path, filePath),
|
||||||
ino: Number(fileStat.ino),
|
ino: clampIno(fileStat.ino),
|
||||||
sizeBytes: Number(fileStat.size),
|
sizeBytes: Number(fileStat.size),
|
||||||
mtime: fileStat.mtime,
|
mtime: fileStat.mtime,
|
||||||
format,
|
format,
|
||||||
|
|||||||
Reference in New Issue
Block a user