mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
feat: overhaul scanner module with 14 improvements and 147 unit tests
This commit is contained in:
neon
@@ -27,6 +27,7 @@ const mockRepo: Mocked<
|
||||
| 'markBooksAsPresent'
|
||||
| 'findBookFileWithContextByIno'
|
||||
| 'updateBookFolderPath'
|
||||
| 'findLibraryFolderPath'
|
||||
>
|
||||
> = {
|
||||
findBookFileByAbsolutePath: vi.fn(),
|
||||
@@ -41,6 +42,7 @@ const mockRepo: Mocked<
|
||||
markBooksAsPresent: vi.fn(),
|
||||
findBookFileWithContextByIno: vi.fn(),
|
||||
updateBookFolderPath: vi.fn(),
|
||||
findLibraryFolderPath: vi.fn(),
|
||||
};
|
||||
|
||||
function makeService() {
|
||||
@@ -65,6 +67,7 @@ beforeEach(() => {
|
||||
mockRepo.updateBookFile.mockResolvedValue({} as any);
|
||||
mockRepo.findBookFileWithContextByIno.mockResolvedValue(null);
|
||||
mockRepo.updateBookFolderPath.mockResolvedValue(undefined);
|
||||
mockRepo.findLibraryFolderPath.mockResolvedValue('/books');
|
||||
mockReaddir.mockResolvedValue([]);
|
||||
});
|
||||
|
||||
|
||||
@@ -56,11 +56,7 @@ export class FileEventProcessorService {
|
||||
(await this.scannerRepo.findMissingBookByFolderPath(absolutePath));
|
||||
if (!book) return { type: 'noop' };
|
||||
|
||||
await this.scannerRepo.updateBookFile(existing.file.id, {
|
||||
ino: Number(fileStat.ino),
|
||||
sizeBytes: Number(fileStat.size),
|
||||
mtime: fileStat.mtime,
|
||||
});
|
||||
await this.scannerRepo.updateBookFile(existing.file.id, this.statToFileInfo(fileStat));
|
||||
await this.scannerRepo.markBooksAsPresent([book.id]);
|
||||
this.logger.log(`Book ${book.id} restored — file returned: ${absolutePath}`);
|
||||
return { type: 'book-restored', libraryId: book.libraryId, bookIds: [book.id] };
|
||||
@@ -70,13 +66,13 @@ export class FileEventProcessorService {
|
||||
const book =
|
||||
(await this.scannerRepo.findMissingBookByFolderPath(folderPath)) ?? (await this.scannerRepo.findMissingBookByFolderPath(absolutePath));
|
||||
if (book) {
|
||||
const libraryFolderPath = await this.scannerRepo.findLibraryFolderPath(book.libraryFolderId);
|
||||
await this.scannerRepo.createBookFile({
|
||||
bookId: book.id,
|
||||
libraryFolderId: book.libraryFolderId,
|
||||
absolutePath,
|
||||
ino: Number(fileStat.ino),
|
||||
sizeBytes: Number(fileStat.size),
|
||||
mtime: fileStat.mtime,
|
||||
relPath: libraryFolderPath ? relative(libraryFolderPath, absolutePath) : undefined,
|
||||
...this.statToFileInfo(fileStat),
|
||||
format,
|
||||
role: 'primary',
|
||||
});
|
||||
@@ -110,11 +106,7 @@ export class FileEventProcessorService {
|
||||
const s = await stat(file.absolutePath).catch(() => null);
|
||||
if (!s || !s.isFile()) continue;
|
||||
|
||||
await this.scannerRepo.updateBookFile(file.id, {
|
||||
ino: Number(s.ino),
|
||||
sizeBytes: Number(s.size),
|
||||
mtime: s.mtime,
|
||||
});
|
||||
await this.scannerRepo.updateBookFile(file.id, this.statToFileInfo(s));
|
||||
await this.scannerRepo.markBooksAsPresent([book.id]);
|
||||
this.logger.log(`Book ${book.id} restored — file returned: ${file.absolutePath}`);
|
||||
return { type: 'book-restored', libraryId: book.libraryId, bookIds: [book.id] };
|
||||
@@ -133,9 +125,7 @@ export class FileEventProcessorService {
|
||||
await this.scannerRepo.updateBookFile(file.id, {
|
||||
absolutePath: newAbsolutePath,
|
||||
relPath: relative(libraryFolderPath, newAbsolutePath),
|
||||
ino: Number(fileStat.ino),
|
||||
sizeBytes: Number(fileStat.size),
|
||||
mtime: fileStat.mtime,
|
||||
...this.statToFileInfo(fileStat),
|
||||
});
|
||||
|
||||
if (newFolderPath !== oldFolderPath) {
|
||||
@@ -147,6 +137,10 @@ export class FileEventProcessorService {
|
||||
return { type: 'book-moved', libraryId, bookIds: [file.bookId] };
|
||||
}
|
||||
|
||||
private statToFileInfo(s: Awaited<ReturnType<typeof stat>>): { ino: number; sizeBytes: number; mtime: Date } {
|
||||
return { ino: Number(s.ino), sizeBytes: Number(s.size), mtime: s.mtime };
|
||||
}
|
||||
|
||||
private async detectMovedBooksInDir(dirPath: string): Promise<FileEventResult> {
|
||||
const entries = await readdir(dirPath, { recursive: true, withFileTypes: true }).catch(() => []);
|
||||
const movedBookIds: number[] = [];
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import { classifyFile, isPrimaryFormat, DEFAULT_FORMAT_PRIORITY } from './classify';
|
||||
|
||||
// ── PRIMARY FORMATS ───────────────────────────────────────────────────────────
|
||||
|
||||
describe('classifyFile — primary formats', () => {
|
||||
it.each(['epub', 'pdf', 'cbz', 'cbr', 'cb7', 'mobi', 'azw3', 'azw', 'fb2'])('classifies .%s as primary', (ext) => {
|
||||
const result = classifyFile(`/books/Author/Book/book.${ext}`);
|
||||
expect(result).toEqual({ format: ext, role: 'primary' });
|
||||
});
|
||||
|
||||
it('classifies extensions case-insensitively', () => {
|
||||
expect(classifyFile('/books/Book.EPUB')).toEqual({ format: 'epub', role: 'primary' });
|
||||
expect(classifyFile('/books/Book.PDF')).toEqual({ format: 'pdf', role: 'primary' });
|
||||
expect(classifyFile('/books/Book.CBZ')).toEqual({ format: 'cbz', role: 'primary' });
|
||||
});
|
||||
|
||||
it('format matches the lowercased extension, not the original', () => {
|
||||
const { format } = classifyFile('/books/Book.EPUB');
|
||||
expect(format).toBe('epub');
|
||||
});
|
||||
});
|
||||
|
||||
// ── COVER FILES ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe('classifyFile — cover images', () => {
|
||||
it.each(['cover', 'folder', 'thumbnail', 'artwork', 'front'])('classifies %s.jpg as cover', (stem) => {
|
||||
const result = classifyFile(`/books/Book/${stem}.jpg`);
|
||||
expect(result.role).toBe('cover');
|
||||
});
|
||||
|
||||
it.each(['jpg', 'jpeg', 'png', 'webp', 'gif', 'bmp'])('classifies cover.%s as cover', (ext) => {
|
||||
const result = classifyFile(`/books/Book/cover.${ext}`);
|
||||
expect(result.role).toBe('cover');
|
||||
expect(result.format).toBe(ext);
|
||||
});
|
||||
|
||||
it('classifies image with unrecognised basename as supplementary', () => {
|
||||
const result = classifyFile('/books/Book/myimage.jpg');
|
||||
expect(result.role).toBe('supplementary');
|
||||
expect(result.format).toBe('jpg');
|
||||
});
|
||||
|
||||
it('cover basename check is case-insensitive', () => {
|
||||
expect(classifyFile('/books/Book/Cover.jpg').role).toBe('cover');
|
||||
expect(classifyFile('/books/Book/COVER.PNG').role).toBe('cover');
|
||||
expect(classifyFile('/books/Book/Thumbnail.WEBP').role).toBe('cover');
|
||||
});
|
||||
});
|
||||
|
||||
// ── METADATA FILES ────────────────────────────────────────────────────────────
|
||||
|
||||
describe('classifyFile — metadata files', () => {
|
||||
it('classifies .opf as metadata', () => {
|
||||
expect(classifyFile('/books/Book/book.opf')).toEqual({ format: 'opf', role: 'metadata' });
|
||||
});
|
||||
|
||||
it('classifies .nfo as metadata', () => {
|
||||
expect(classifyFile('/books/Book/book.nfo')).toEqual({ format: 'nfo', role: 'metadata' });
|
||||
});
|
||||
});
|
||||
|
||||
// ── SUPPLEMENTARY ─────────────────────────────────────────────────────────────
|
||||
|
||||
describe('classifyFile — supplementary', () => {
|
||||
it('classifies unknown extensions as supplementary', () => {
|
||||
expect(classifyFile('/books/Book/readme.txt').role).toBe('supplementary');
|
||||
expect(classifyFile('/books/Book/notes.md').role).toBe('supplementary');
|
||||
expect(classifyFile('/books/Book/data.xml').role).toBe('supplementary');
|
||||
});
|
||||
|
||||
it('returns null format for files with no extension', () => {
|
||||
const result = classifyFile('/books/Book/RELEASE');
|
||||
expect(result.role).toBe('supplementary');
|
||||
expect(result.format).toBeNull();
|
||||
});
|
||||
|
||||
it('handles files with dots in the name but unknown final extension', () => {
|
||||
const result = classifyFile('/books/Book/my.archive.rar');
|
||||
expect(result.role).toBe('supplementary');
|
||||
expect(result.format).toBe('rar');
|
||||
});
|
||||
});
|
||||
|
||||
// ── isPrimaryFormat ───────────────────────────────────────────────────────────
|
||||
|
||||
describe('isPrimaryFormat', () => {
|
||||
it('returns true for all primary format extensions', () => {
|
||||
for (const ext of DEFAULT_FORMAT_PRIORITY) {
|
||||
expect(isPrimaryFormat(`/path/file.${ext}`)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('returns false for non-primary files', () => {
|
||||
expect(isPrimaryFormat('/path/cover.jpg')).toBe(false);
|
||||
expect(isPrimaryFormat('/path/book.opf')).toBe(false);
|
||||
expect(isPrimaryFormat('/path/readme.txt')).toBe(false);
|
||||
});
|
||||
|
||||
it('is case-insensitive', () => {
|
||||
expect(isPrimaryFormat('/path/book.EPUB')).toBe(true);
|
||||
expect(isPrimaryFormat('/path/book.PDF')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ── DEFAULT_FORMAT_PRIORITY ───────────────────────────────────────────────────
|
||||
|
||||
describe('DEFAULT_FORMAT_PRIORITY', () => {
|
||||
it('epub is first (highest priority)', () => {
|
||||
expect(DEFAULT_FORMAT_PRIORITY[0]).toBe('epub');
|
||||
});
|
||||
|
||||
it('every format in DEFAULT_FORMAT_PRIORITY is a primary format', () => {
|
||||
for (const fmt of DEFAULT_FORMAT_PRIORITY) {
|
||||
expect(isPrimaryFormat(`/path/file.${fmt}`)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import { extname, basename } from 'path';
|
||||
|
||||
const PRIMARY_FORMATS = new Set(['epub', 'pdf', 'cbz', 'cbr', 'cb7', 'mobi', 'azw3', 'azw', 'fb2']);
|
||||
export const DEFAULT_FORMAT_PRIORITY = ['epub', 'pdf', 'cbz', 'cbr', 'cb7', 'mobi', 'azw3', 'azw', 'fb2'];
|
||||
const PRIMARY_FORMATS = new Set(DEFAULT_FORMAT_PRIORITY);
|
||||
|
||||
const COVER_EXTENSIONS = new Set(['jpg', 'jpeg', 'png', 'webp', 'gif', 'bmp']);
|
||||
const COVER_BASENAMES = new Set(['cover', 'folder', 'thumbnail', 'artwork', 'front']);
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { mkdtemp, rm, writeFile } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
import { fingerprintFile } from './hash';
|
||||
|
||||
const EMPTY_MD5 = 'd41d8cd98f00b204e9800998ecf8427e';
|
||||
// 1 KB = 1024 bytes. The first read position is BASE (1024), so files <= 1024 bytes produce empty hash.
|
||||
const FIRST_READ_POSITION = 1024;
|
||||
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await mkdtemp(join(tmpdir(), 'scanner-hash-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// ── SMALL FILES ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe('files smaller than the first read position', () => {
|
||||
it('returns MD5 of empty string for a file with 0 bytes', async () => {
|
||||
await writeFile(join(tmpDir, 'empty.epub'), Buffer.alloc(0));
|
||||
const hash = await fingerprintFile(join(tmpDir, 'empty.epub'));
|
||||
expect(hash).toBe(EMPTY_MD5);
|
||||
});
|
||||
|
||||
it('returns MD5 of empty string for a file with exactly 1024 bytes', async () => {
|
||||
// Position 1024 >= size 1024 → no read
|
||||
await writeFile(join(tmpDir, 'exact.epub'), Buffer.alloc(FIRST_READ_POSITION, 'A'));
|
||||
const hash = await fingerprintFile(join(tmpDir, 'exact.epub'));
|
||||
expect(hash).toBe(EMPTY_MD5);
|
||||
});
|
||||
});
|
||||
|
||||
// ── FILES LARGE ENOUGH TO BE SAMPLED ─────────────────────────────────────────
|
||||
|
||||
describe('files large enough for sampling', () => {
|
||||
it('returns a 32-char hex string (MD5)', async () => {
|
||||
await writeFile(join(tmpDir, 'book.epub'), Buffer.alloc(2048, 'A'));
|
||||
const hash = await fingerprintFile(join(tmpDir, 'book.epub'));
|
||||
expect(hash).toMatch(/^[0-9a-f]{32}$/);
|
||||
});
|
||||
|
||||
it('identical content produces identical fingerprint', async () => {
|
||||
const content = Buffer.alloc(4096, 'Z');
|
||||
await writeFile(join(tmpDir, 'a.epub'), content);
|
||||
await writeFile(join(tmpDir, 'b.epub'), content);
|
||||
|
||||
const [h1, h2] = await Promise.all([fingerprintFile(join(tmpDir, 'a.epub')), fingerprintFile(join(tmpDir, 'b.epub'))]);
|
||||
expect(h1).toBe(h2);
|
||||
});
|
||||
|
||||
it('different content at sampled position produces different fingerprint', async () => {
|
||||
const base = Buffer.alloc(4096, 'A');
|
||||
const different = Buffer.from(base);
|
||||
different.write('Z', FIRST_READ_POSITION, 'ascii'); // change byte at the sampled position
|
||||
|
||||
await writeFile(join(tmpDir, 'a.epub'), base);
|
||||
await writeFile(join(tmpDir, 'b.epub'), different);
|
||||
|
||||
const [h1, h2] = await Promise.all([fingerprintFile(join(tmpDir, 'a.epub')), fingerprintFile(join(tmpDir, 'b.epub'))]);
|
||||
expect(h1).not.toBe(h2);
|
||||
});
|
||||
|
||||
it('fingerprint is deterministic across calls', async () => {
|
||||
await writeFile(join(tmpDir, 'book.epub'), Buffer.alloc(8192, 'X'));
|
||||
const [h1, h2] = await Promise.all([fingerprintFile(join(tmpDir, 'book.epub')), fingerprintFile(join(tmpDir, 'book.epub'))]);
|
||||
expect(h1).toBe(h2);
|
||||
});
|
||||
});
|
||||
|
||||
// ── ERROR HANDLING ────────────────────────────────────────────────────────────
|
||||
|
||||
describe('error handling', () => {
|
||||
it('throws when the file does not exist', async () => {
|
||||
await expect(fingerprintFile(join(tmpDir, 'nonexistent.epub'))).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
vi.mock('fs/promises', () => ({ stat: vi.fn() }));
|
||||
|
||||
import { stat } from 'fs/promises';
|
||||
import type { MockedFunction } from 'vitest';
|
||||
import { waitForStability } from './stability';
|
||||
|
||||
const mockStat = stat as MockedFunction<typeof stat>;
|
||||
|
||||
// Constants mirrored from stability.ts to keep assertions meaningful
|
||||
const POLL_INTERVAL_MS = 3_000;
|
||||
const STABLE_DURATION_MS = 10_000;
|
||||
const MAX_WAIT_MS = 60_000;
|
||||
const RECENTLY_MODIFIED_THRESHOLD_MS = 60_000;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
// ── FAST PATHS ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('fast paths (no polling)', () => {
|
||||
it('returns immediately for files whose mtime is older than the threshold', async () => {
|
||||
mockStat.mockResolvedValue({ mtimeMs: Date.now() - RECENTLY_MODIFIED_THRESHOLD_MS - 1 } as any);
|
||||
|
||||
await waitForStability('/path/old-book.epub');
|
||||
|
||||
expect(mockStat).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('returns immediately when the file does not exist', async () => {
|
||||
mockStat.mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' }));
|
||||
|
||||
await waitForStability('/path/gone.epub');
|
||||
|
||||
expect(mockStat).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── POLLING UNTIL STABLE ──────────────────────────────────────────────────────
|
||||
|
||||
describe('polling until stable', () => {
|
||||
it('returns once mtime has not changed for STABLE_DURATION_MS', async () => {
|
||||
const recentMtime = Date.now() - 1_000; // recently modified
|
||||
// Initial stat check
|
||||
mockStat.mockResolvedValueOnce({ mtimeMs: recentMtime } as any);
|
||||
// All subsequent polls return the same mtime (stable)
|
||||
mockStat.mockResolvedValue({ mtimeMs: recentMtime } as any);
|
||||
|
||||
const promise = waitForStability('/path/book.epub');
|
||||
|
||||
// Drain the initial stat and first loop iteration, then advance well past STABLE_DURATION_MS
|
||||
await vi.advanceTimersByTimeAsync(POLL_INTERVAL_MS * 4 + STABLE_DURATION_MS);
|
||||
await promise;
|
||||
|
||||
// Should have polled at least a few times before returning
|
||||
expect(mockStat.mock.calls.length).toBeGreaterThan(2);
|
||||
});
|
||||
|
||||
it('resets the stable timer when mtime changes mid-poll', async () => {
|
||||
const recentMtime = Date.now() - 1_000;
|
||||
let callCount = 0;
|
||||
mockStat.mockImplementation(() => {
|
||||
callCount++;
|
||||
// Mtime changes on the 2nd poll call (inside the loop), then stays stable
|
||||
return Promise.resolve({ mtimeMs: callCount < 3 ? recentMtime + callCount * 100 : recentMtime + 200 } as any);
|
||||
});
|
||||
|
||||
const promise = waitForStability('/path/book.epub');
|
||||
|
||||
// Advance far enough for the file to stabilise after the change
|
||||
await vi.advanceTimersByTimeAsync(POLL_INTERVAL_MS * 6 + STABLE_DURATION_MS);
|
||||
await promise;
|
||||
|
||||
// More iterations were needed because the timer reset after the mtime change
|
||||
expect(callCount).toBeGreaterThan(4);
|
||||
});
|
||||
});
|
||||
|
||||
// ── DEADLINE HIT ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe('deadline exceeded', () => {
|
||||
it('returns without throwing when mtime keeps changing past the deadline', async () => {
|
||||
let t = Date.now() - 1_000;
|
||||
mockStat.mockImplementation(() => {
|
||||
t += 100;
|
||||
return Promise.resolve({ mtimeMs: t } as any);
|
||||
});
|
||||
|
||||
const promise = waitForStability('/path/active-download.epub');
|
||||
|
||||
// Advance past the full deadline
|
||||
await vi.advanceTimersByTimeAsync(MAX_WAIT_MS + POLL_INTERVAL_MS * 2);
|
||||
await expect(promise).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── FILE DISAPPEARS DURING POLLING ────────────────────────────────────────────
|
||||
|
||||
describe('file disappears mid-poll', () => {
|
||||
it('returns silently when stat throws ENOENT inside the polling loop', async () => {
|
||||
const recentMtime = Date.now() - 1_000;
|
||||
mockStat
|
||||
.mockResolvedValueOnce({ mtimeMs: recentMtime } as any) // initial check
|
||||
.mockResolvedValueOnce({ mtimeMs: recentMtime } as any) // first poll
|
||||
.mockRejectedValueOnce(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })); // disappears
|
||||
|
||||
const promise = waitForStability('/path/book.epub');
|
||||
|
||||
await vi.advanceTimersByTimeAsync(POLL_INTERVAL_MS * 3);
|
||||
await expect(promise).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,300 @@
|
||||
import { mkdir, mkdtemp, rm, writeFile, chmod } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
import { findBookCandidates } from './walk';
|
||||
|
||||
let root: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), 'scanner-walk-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// Helpers
|
||||
async function dir(...parts: string[]): Promise<string> {
|
||||
const p = join(root, ...parts);
|
||||
await mkdir(p, { recursive: true });
|
||||
return p;
|
||||
}
|
||||
|
||||
async function file(relPath: string, content = 'x'): Promise<string> {
|
||||
const p = join(root, relPath);
|
||||
await mkdir(join(root, relPath, '..'), { recursive: true });
|
||||
await writeFile(p, content);
|
||||
return p;
|
||||
}
|
||||
|
||||
function candidatePaths(candidates: Awaited<ReturnType<typeof findBookCandidates>>) {
|
||||
return candidates.map((c) => c.folderPath).sort();
|
||||
}
|
||||
|
||||
function filePaths(candidate: Awaited<ReturnType<typeof findBookCandidates>>[0]) {
|
||||
return candidate.files.map((f) => f.absolutePath).sort();
|
||||
}
|
||||
|
||||
// ── EMPTY / NO PRIMARIES ──────────────────────────────────────────────────────
|
||||
|
||||
describe('empty and non-primary folders', () => {
|
||||
it('returns no candidates for an empty library', async () => {
|
||||
const candidates = await findBookCandidates(root);
|
||||
expect(candidates).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('returns no candidates when only non-primary files exist', async () => {
|
||||
await file('Author/Book/cover.jpg');
|
||||
await file('Author/Book/book.opf');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
expect(candidates).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('skips grouping folders that contain no primaries directly', async () => {
|
||||
await file('Author/Book/book.epub');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].folderPath).toBe(join(root, 'Author', 'Book'));
|
||||
});
|
||||
});
|
||||
|
||||
// ── ROOT-LEVEL FILES ──────────────────────────────────────────────────────────
|
||||
|
||||
describe('root-level primary files', () => {
|
||||
it('each root-level primary becomes its own candidate with folderPath = absolutePath', async () => {
|
||||
await file('book1.epub');
|
||||
await file('book2.pdf');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
expect(candidates).toHaveLength(2);
|
||||
|
||||
const paths = candidatePaths(candidates);
|
||||
expect(paths).toContain(join(root, 'book1.epub'));
|
||||
expect(paths).toContain(join(root, 'book2.pdf'));
|
||||
});
|
||||
|
||||
it('root-level candidate contains only the primary file', async () => {
|
||||
await file('book.epub');
|
||||
const candidates = await findBookCandidates(root);
|
||||
expect(candidates[0].files).toHaveLength(1);
|
||||
expect(candidates[0].files[0].absolutePath).toBe(join(root, 'book.epub'));
|
||||
});
|
||||
});
|
||||
|
||||
// ── SINGLE BOOK PER FOLDER ────────────────────────────────────────────────────
|
||||
|
||||
describe('single-book folder', () => {
|
||||
it('folderPath is the directory, all files are included', async () => {
|
||||
await file('Author/Book/book.epub');
|
||||
await file('Author/Book/cover.jpg');
|
||||
await file('Author/Book/book.opf');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].folderPath).toBe(join(root, 'Author', 'Book'));
|
||||
expect(candidates[0].files).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('multiple formats with the same stem → one candidate', async () => {
|
||||
await file('Author/Book/book.epub');
|
||||
await file('Author/Book/book.mobi');
|
||||
await file('Author/Book/book.pdf');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].files).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('relPath is relative to the library root', async () => {
|
||||
await file('Author/Book/book.epub');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
const epubFile = candidates[0].files.find((f) => f.absolutePath.endsWith('book.epub'))!;
|
||||
expect(epubFile.relPath).toBe('Author/Book/book.epub');
|
||||
});
|
||||
|
||||
it('FileStat fields are populated from the real filesystem', async () => {
|
||||
await file('Book/book.epub', 'epub content');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
const f = candidates[0].files[0];
|
||||
expect(f.ino).toBeGreaterThan(0);
|
||||
expect(f.sizeBytes).toBe('epub content'.length);
|
||||
expect(f.mtime).toBeInstanceOf(Date);
|
||||
});
|
||||
});
|
||||
|
||||
// ── SERIES FOLDER (MULTIPLE STEMS) ───────────────────────────────────────────
|
||||
|
||||
describe('series folder', () => {
|
||||
it('creates one candidate per primary stem', async () => {
|
||||
await file('Series/book1.epub');
|
||||
await file('Series/book2.epub');
|
||||
await file('Series/book3.pdf');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
expect(candidates).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('virtual folderPath is join(dir, origStem), preserving original casing', async () => {
|
||||
await file('Series/Book1.epub');
|
||||
await file('Series/Book2.epub');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
const paths = candidatePaths(candidates);
|
||||
expect(paths).toContain(join(root, 'Series', 'Book1'));
|
||||
expect(paths).toContain(join(root, 'Series', 'Book2'));
|
||||
});
|
||||
|
||||
it('sidecar files are matched to their primary by stem', async () => {
|
||||
await file('Series/book1.epub');
|
||||
await file('Series/book1.jpg'); // not a cover basename — supplementary matched by stem
|
||||
await file('Series/book2.epub');
|
||||
await file('Series/book2.jpg');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
const book1 = candidates.find((c) => c.folderPath.endsWith('book1'))!;
|
||||
expect(book1).toBeDefined();
|
||||
expect(filePaths(book1)).toContain(join(root, 'Series', 'book1.epub'));
|
||||
expect(filePaths(book1)).toContain(join(root, 'Series', 'book1.jpg'));
|
||||
|
||||
const book2 = candidates.find((c) => c.folderPath.endsWith('book2'))!;
|
||||
expect(filePaths(book2)).not.toContain(join(root, 'Series', 'book1.jpg'));
|
||||
});
|
||||
|
||||
it('case-insensitive stem matching links cover.jpg to Book.epub in series folder', async () => {
|
||||
await file('Series/Book1.epub');
|
||||
await file('Series/book1.jpg'); // lowercase stem matches uppercase primary stem
|
||||
await file('Series/Book2.epub');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
const book1 = candidates.find((c) => c.folderPath.endsWith('Book1'))!;
|
||||
expect(book1).toBeDefined();
|
||||
const paths = filePaths(book1);
|
||||
expect(paths).toContain(join(root, 'Series', 'book1.jpg'));
|
||||
});
|
||||
|
||||
it('original casing of folderPath is stable across scans (regression guard)', async () => {
|
||||
// Ensures the lowercase-for-comparison change does not alter the DB key
|
||||
await file('Series/MyBook.epub');
|
||||
await file('Series/OtherBook.epub');
|
||||
|
||||
const first = await findBookCandidates(root);
|
||||
const second = await findBookCandidates(root);
|
||||
|
||||
const firstPaths = candidatePaths(first);
|
||||
const secondPaths = candidatePaths(second);
|
||||
expect(firstPaths).toEqual(secondPaths);
|
||||
expect(firstPaths).toContain(join(root, 'Series', 'MyBook'));
|
||||
expect(firstPaths).toContain(join(root, 'Series', 'OtherBook'));
|
||||
});
|
||||
});
|
||||
|
||||
// ── DEEP NESTING ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe('deep nesting', () => {
|
||||
it('discovers books in arbitrarily deep subdirectories', async () => {
|
||||
await file('A/B/C/D/E/book.epub');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].folderPath).toBe(join(root, 'A', 'B', 'C', 'D', 'E'));
|
||||
});
|
||||
});
|
||||
|
||||
// ── HIDDEN FILES / FOLDERS ────────────────────────────────────────────────────
|
||||
|
||||
describe('hidden files and folders', () => {
|
||||
it('skips files starting with a dot', async () => {
|
||||
await file('Book/.DS_Store');
|
||||
await file('Book/book.epub');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
const f = candidates[0].files;
|
||||
expect(f.some((x) => x.absolutePath.includes('.DS_Store'))).toBe(false);
|
||||
});
|
||||
|
||||
it('does not recurse into dot-prefixed directories', async () => {
|
||||
await file('.calibre/book.epub');
|
||||
await file('Book/book.epub');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].folderPath).toBe(join(root, 'Book'));
|
||||
});
|
||||
});
|
||||
|
||||
// ── EXCLUDE PATTERNS ─────────────────────────────────────────────────────────
|
||||
|
||||
describe('excludePatterns', () => {
|
||||
it('excludes directories matching an exact pattern', async () => {
|
||||
await file('#recycle/book.epub');
|
||||
await file('Books/book.epub');
|
||||
|
||||
const candidates = await findBookCandidates(root, ['#recycle']);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].folderPath).toBe(join(root, 'Books'));
|
||||
});
|
||||
|
||||
it('excludes files matching a wildcard pattern', async () => {
|
||||
await file('Book/book.epub');
|
||||
await file('Book/backup.bak');
|
||||
|
||||
const candidates = await findBookCandidates(root, ['*.bak']);
|
||||
const files = candidates[0].files;
|
||||
expect(files.some((f) => f.absolutePath.endsWith('.bak'))).toBe(false);
|
||||
expect(files).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('excludes directories matching a wildcard pattern', async () => {
|
||||
await file('@eaDir/book.epub');
|
||||
await file('Normal/book.epub');
|
||||
|
||||
const candidates = await findBookCandidates(root, ['@*']);
|
||||
expect(candidates).toHaveLength(1);
|
||||
expect(candidates[0].folderPath).toBe(join(root, 'Normal'));
|
||||
});
|
||||
|
||||
it('returns all candidates when excludePatterns is empty', async () => {
|
||||
await file('#recycle/book.epub');
|
||||
await file('Books/book.epub');
|
||||
|
||||
const candidates = await findBookCandidates(root, []);
|
||||
expect(candidates).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('passes logger warnings on permission-denied directories', async () => {
|
||||
if (process.getuid?.() === 0) return; // skip when running as root
|
||||
|
||||
await dir('restricted');
|
||||
await file('Books/book.epub');
|
||||
|
||||
// Make restricted unreadable
|
||||
await chmod(join(root, 'restricted'), 0o000);
|
||||
|
||||
const warnings: string[] = [];
|
||||
const candidates = await findBookCandidates(root, [], (msg) => warnings.push(msg));
|
||||
|
||||
// Restore before assert so cleanup works
|
||||
await chmod(join(root, 'restricted'), 0o755);
|
||||
|
||||
expect(warnings.length).toBeGreaterThan(0);
|
||||
expect(warnings.some((w) => w.includes('restricted'))).toBe(true);
|
||||
expect(candidates).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ── MULTIPLE LIBRARIES IN SAME SCAN ──────────────────────────────────────────
|
||||
|
||||
describe('mixed root structure', () => {
|
||||
it('handles root-level files and subdirectory books together', async () => {
|
||||
await file('standalone.epub');
|
||||
await file('Author/Book/book.epub');
|
||||
|
||||
const candidates = await findBookCandidates(root);
|
||||
expect(candidates).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -16,18 +16,54 @@ export interface BookCandidate {
|
||||
files: FileStat[]; // all files in this folder
|
||||
}
|
||||
|
||||
const MAX_PATH_LENGTH = 4096;
|
||||
|
||||
function matchesExcludePattern(name: string, patterns: string[]): boolean {
|
||||
for (const pattern of patterns) {
|
||||
if (!pattern.includes('*')) {
|
||||
if (name === pattern) return true;
|
||||
} else {
|
||||
const escaped = pattern.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*');
|
||||
if (new RegExp(`^${escaped}$`).test(name)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Recursively collect files, grouped by their parent directory.
|
||||
async function collectByDir(dir: string, libraryRoot: string, acc: Map<string, FileStat[]>): Promise<void> {
|
||||
const entries = await readdir(dir, { withFileTypes: true });
|
||||
async function collectByDir(
|
||||
dir: string,
|
||||
libraryRoot: string,
|
||||
acc: Map<string, FileStat[]>,
|
||||
excludePatterns: string[],
|
||||
logger?: (msg: string) => void,
|
||||
): Promise<void> {
|
||||
let entries;
|
||||
try {
|
||||
entries = await readdir(dir, { withFileTypes: true });
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === 'EACCES' || code === 'EPERM') {
|
||||
logger?.(`Permission denied reading folder, skipping: ${dir}`);
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
const full = join(dir, entry.name);
|
||||
|
||||
if (entry.name.startsWith('.')) continue;
|
||||
if (excludePatterns.length > 0 && matchesExcludePattern(entry.name, excludePatterns)) continue;
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await collectByDir(full, libraryRoot, acc);
|
||||
await collectByDir(full, libraryRoot, acc, excludePatterns, logger);
|
||||
} else if (entry.isFile()) {
|
||||
if (full.length > MAX_PATH_LENGTH) {
|
||||
logger?.(`Path exceeds ${MAX_PATH_LENGTH} characters, skipping: ${full}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const s = await stat(full);
|
||||
const fileInfo: FileStat = {
|
||||
absolutePath: full,
|
||||
@@ -49,6 +85,13 @@ function stemOf(absolutePath: string): string {
|
||||
return dot !== -1 ? name.substring(0, dot) : name;
|
||||
}
|
||||
|
||||
// Lowercase-only version used for grouping comparisons so that Book.epub and
|
||||
// book.jpg pair correctly on case-sensitive filesystems, without changing the
|
||||
// original stem used for the DB folderPath key.
|
||||
function normStemOf(absolutePath: string): string {
|
||||
return stemOf(absolutePath).toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk a library folder and return book candidates.
|
||||
*
|
||||
@@ -62,9 +105,13 @@ function stemOf(absolutePath: string): string {
|
||||
* files = primary files for that stem + any non-primary files with matching stem.
|
||||
* - Directories with no primary files are skipped (author/grouping folders).
|
||||
*/
|
||||
export async function findBookCandidates(libraryFolderPath: string): Promise<BookCandidate[]> {
|
||||
export async function findBookCandidates(
|
||||
libraryFolderPath: string,
|
||||
excludePatterns: string[] = [],
|
||||
logger?: (msg: string) => void,
|
||||
): Promise<BookCandidate[]> {
|
||||
const byDir = new Map<string, FileStat[]>();
|
||||
await collectByDir(libraryFolderPath, libraryFolderPath, byDir);
|
||||
await collectByDir(libraryFolderPath, libraryFolderPath, byDir, excludePatterns, logger);
|
||||
|
||||
const candidates: BookCandidate[] = [];
|
||||
|
||||
@@ -80,12 +127,17 @@ export async function findBookCandidates(libraryFolderPath: string): Promise<Boo
|
||||
continue;
|
||||
}
|
||||
|
||||
// Group primary files by stem — same stem = same book in multiple formats.
|
||||
const stemGroups = new Map<string, FileStat[]>();
|
||||
// Group primary files by normalised stem so that Book.epub and book.mobi
|
||||
// are treated as the same book on case-sensitive filesystems. The original
|
||||
// stem (from the first primary found) is preserved for the folderPath key
|
||||
// so existing DB records are not invalidated.
|
||||
const stemGroups = new Map<string, { origStem: string; files: FileStat[] }>();
|
||||
for (const f of primaryFiles) {
|
||||
const stem = stemOf(f.absolutePath);
|
||||
if (!stemGroups.has(stem)) stemGroups.set(stem, []);
|
||||
stemGroups.get(stem)!.push(f);
|
||||
const norm = normStemOf(f.absolutePath);
|
||||
if (!stemGroups.has(norm)) {
|
||||
stemGroups.set(norm, { origStem: stemOf(f.absolutePath), files: [] });
|
||||
}
|
||||
stemGroups.get(norm)!.files.push(f);
|
||||
}
|
||||
|
||||
if (stemGroups.size === 1) {
|
||||
@@ -94,10 +146,10 @@ export async function findBookCandidates(libraryFolderPath: string): Promise<Boo
|
||||
} else {
|
||||
// Multiple books sharing a folder (series folder pattern).
|
||||
const nonPrimary = files.filter((f) => !isPrimaryFormat(f.absolutePath));
|
||||
for (const [stem, stemPrimaries] of stemGroups) {
|
||||
const sidecar = nonPrimary.filter((f) => stemOf(f.absolutePath) === stem);
|
||||
for (const [normStem, { origStem, files: stemPrimaries }] of stemGroups) {
|
||||
const sidecar = nonPrimary.filter((f) => normStemOf(f.absolutePath) === normStem);
|
||||
candidates.push({
|
||||
folderPath: join(dir, stem), // virtual unique key, not a real path
|
||||
folderPath: join(dir, origStem), // virtual unique key, not a real path
|
||||
files: [...stemPrimaries, ...sidecar],
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
import { ScanJobStore } from './scan-job-store.service';
|
||||
|
||||
function make() {
|
||||
return new ScanJobStore();
|
||||
}
|
||||
|
||||
// ── isRunning / create / delete ───────────────────────────────────────────────
|
||||
|
||||
describe('lifecycle', () => {
|
||||
it('is not running before a job is created', () => {
|
||||
expect(make().isRunning(1)).toBe(false);
|
||||
});
|
||||
|
||||
it('is running after create and not running after delete', () => {
|
||||
const store = make();
|
||||
store.create(100, 1, 50);
|
||||
expect(store.isRunning(1)).toBe(true);
|
||||
store.delete(1);
|
||||
expect(store.isRunning(1)).toBe(false);
|
||||
});
|
||||
|
||||
it('create initialises all counters to zero and lastEmitPct to -1', () => {
|
||||
const entry = make().create(1, 10, 200);
|
||||
expect(entry).toMatchObject({ jobId: 1, libraryId: 10, total: 200, processed: 0, added: 0, updated: 0, missing: 0, lastEmitPct: -1 });
|
||||
});
|
||||
|
||||
it('get returns the entry after create', () => {
|
||||
const store = make();
|
||||
const entry = store.create(5, 2, 0);
|
||||
expect(store.get(2)).toBe(entry);
|
||||
});
|
||||
|
||||
it('get returns undefined for unknown libraryId', () => {
|
||||
expect(make().get(99)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ── setTotal / increment ──────────────────────────────────────────────────────
|
||||
|
||||
describe('setTotal', () => {
|
||||
it('updates the total on an existing entry', () => {
|
||||
const store = make();
|
||||
store.create(1, 1, 0);
|
||||
store.setTotal(1, 500);
|
||||
expect(store.get(1)!.total).toBe(500);
|
||||
});
|
||||
|
||||
it('does nothing for unknown libraryId', () => {
|
||||
expect(() => make().setTotal(99, 100)).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('increment', () => {
|
||||
it('accumulates processed, added, updated, missing independently', () => {
|
||||
const store = make();
|
||||
store.create(1, 1, 100);
|
||||
store.increment(1, { processed: 5 });
|
||||
store.increment(1, { added: 3, updated: 1 });
|
||||
store.increment(1, { missing: 2 });
|
||||
|
||||
const entry = store.get(1)!;
|
||||
expect(entry.processed).toBe(5);
|
||||
expect(entry.added).toBe(3);
|
||||
expect(entry.updated).toBe(1);
|
||||
expect(entry.missing).toBe(2);
|
||||
});
|
||||
|
||||
it('returns the entry after incrementing', () => {
|
||||
const store = make();
|
||||
store.create(1, 1, 10);
|
||||
const result = store.increment(1, { processed: 1 });
|
||||
expect(result).toBeDefined();
|
||||
expect(result!.processed).toBe(1);
|
||||
});
|
||||
|
||||
it('returns undefined for unknown libraryId', () => {
|
||||
expect(make().increment(99, { processed: 1 })).toBeUndefined();
|
||||
});
|
||||
|
||||
it('ignores undefined delta fields', () => {
|
||||
const store = make();
|
||||
store.create(1, 1, 10);
|
||||
store.increment(1, {}); // no-op
|
||||
expect(store.get(1)!.processed).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ── shouldEmit ────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('shouldEmit', () => {
|
||||
it('emits when processed equals total (100%)', () => {
|
||||
const store = make();
|
||||
const entry = store.create(1, 1, 10);
|
||||
store.increment(1, { processed: 10 });
|
||||
expect(store.shouldEmit(entry)).toBe(true);
|
||||
});
|
||||
|
||||
it('emits when percentage advances by at least 1 point', () => {
|
||||
const store = make();
|
||||
const entry = store.create(1, 1, 100);
|
||||
store.markEmitted(entry); // sets lastEmitPct to 0
|
||||
store.increment(1, { processed: 1 }); // 1%
|
||||
expect(store.shouldEmit(entry)).toBe(true);
|
||||
});
|
||||
|
||||
it('does not emit when percentage has not moved', () => {
|
||||
const store = make();
|
||||
const entry = store.create(1, 1, 1000);
|
||||
store.markEmitted(entry); // lastEmitPct = 0, lastEmitMs = now
|
||||
store.increment(1, { processed: 1 }); // 0% (rounds down from 0.1%)
|
||||
// Neither pct nor time condition met (no fake timers, so ms check may vary —
|
||||
// we reset lastEmitMs to future to isolate the pct check)
|
||||
entry.lastEmitMs = Date.now() + 100_000;
|
||||
expect(store.shouldEmit(entry)).toBe(false);
|
||||
});
|
||||
|
||||
it('emits when at least 1 second has elapsed since last emit', () => {
|
||||
const store = make();
|
||||
const entry = store.create(1, 1, 1000);
|
||||
entry.lastEmitMs = Date.now() - 1100; // last emitted > 1s ago
|
||||
expect(store.shouldEmit(entry)).toBe(true);
|
||||
});
|
||||
|
||||
it('emits immediately (lastEmitMs = 0) because time threshold is always exceeded', () => {
|
||||
const store = make();
|
||||
const entry = store.create(1, 1, 10);
|
||||
// lastEmitMs starts at 0, so Date.now() - 0 is always >= 1000
|
||||
expect(store.shouldEmit(entry)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ── markEmitted ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe('markEmitted', () => {
|
||||
it('updates lastEmitMs to approximately now', () => {
|
||||
const before = Date.now();
|
||||
const store = make();
|
||||
const entry = store.create(1, 1, 100);
|
||||
store.markEmitted(entry);
|
||||
expect(entry.lastEmitMs).toBeGreaterThanOrEqual(before);
|
||||
});
|
||||
|
||||
it('updates lastEmitPct based on current progress', () => {
|
||||
const store = make();
|
||||
const entry = store.create(1, 1, 100);
|
||||
store.increment(1, { processed: 42 });
|
||||
store.markEmitted(entry);
|
||||
expect(entry.lastEmitPct).toBe(42);
|
||||
});
|
||||
|
||||
it('sets lastEmitPct to 0 when total is 0', () => {
|
||||
const store = make();
|
||||
const entry = store.create(1, 1, 0);
|
||||
store.markEmitted(entry);
|
||||
expect(entry.lastEmitPct).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -42,12 +42,17 @@ export class ScannerRepository {
|
||||
|
||||
async findLibrarySettings(libraryId: number) {
|
||||
const [row] = await this.db
|
||||
.select({ allowedFormats: libraries.allowedFormats, formatPriority: libraries.formatPriority })
|
||||
.select({ allowedFormats: libraries.allowedFormats, formatPriority: libraries.formatPriority, excludePatterns: libraries.excludePatterns })
|
||||
.from(libraries)
|
||||
.where(eq(libraries.id, libraryId));
|
||||
return row ?? null;
|
||||
}
|
||||
|
||||
async findLibraryFolderPath(libraryFolderId: number): Promise<string | null> {
|
||||
const [row] = await this.db.select({ path: libraryFolders.path }).from(libraryFolders).where(eq(libraryFolders.id, libraryFolderId)).limit(1);
|
||||
return row?.path ?? null;
|
||||
}
|
||||
|
||||
// ── Books ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async findBooksByLibraryFolder(libraryFolderId: number) {
|
||||
|
||||
@@ -0,0 +1,457 @@
|
||||
vi.mock('./lib/walk');
|
||||
vi.mock('./lib/hash');
|
||||
vi.mock('./lib/stability', () => ({ waitForStability: vi.fn().mockResolvedValue(undefined) }));
|
||||
|
||||
import { ConflictException, NotFoundException } from '@nestjs/common';
|
||||
import type { MockedFunction } from 'vitest';
|
||||
|
||||
import { ScannerService } from './scanner.service';
|
||||
import { ScanJobStore } from './scan-job-store.service';
|
||||
import { DEFAULT_FORMAT_PRIORITY } from './lib/classify';
|
||||
import type { BookCandidate, FileStat } from './lib/walk';
|
||||
import { findBookCandidates } from './lib/walk';
|
||||
import { fingerprintFile } from './lib/hash';
|
||||
|
||||
const mockFindCandidates = findBookCandidates as MockedFunction<typeof findBookCandidates>;
|
||||
const mockFingerprint = fingerprintFile as MockedFunction<typeof fingerprintFile>;
|
||||
|
||||
// ── Fixtures ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function makeFileStat(overrides: Partial<FileStat> = {}): FileStat {
|
||||
return {
|
||||
absolutePath: '/library/Author/Book/book.epub',
|
||||
relPath: 'Author/Book/book.epub',
|
||||
ino: 1001,
|
||||
sizeBytes: 1024,
|
||||
mtime: new Date('2024-01-01'),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeCandidate(folderPath: string, files: FileStat[]): BookCandidate {
|
||||
return { folderPath, files };
|
||||
}
|
||||
|
||||
function makeBookFile(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: 1,
|
||||
bookId: 1,
|
||||
libraryFolderId: 1,
|
||||
absolutePath: '/library/Author/Book/book.epub',
|
||||
relPath: 'Author/Book/book.epub',
|
||||
ino: 1001,
|
||||
sizeBytes: 1024,
|
||||
mtime: new Date('2024-01-01'),
|
||||
hash: 'abc123',
|
||||
format: 'epub',
|
||||
role: 'primary',
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeRepo(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
failAllRunningJobs: vi.fn().mockResolvedValue(undefined),
|
||||
findLibraryFolders: vi.fn().mockResolvedValue([{ id: 1, path: '/library', libraryId: 1 }]),
|
||||
findLibrarySettings: vi.fn().mockResolvedValue({ allowedFormats: [], formatPriority: DEFAULT_FORMAT_PRIORITY, excludePatterns: [] }),
|
||||
createScanJob: vi.fn().mockResolvedValue({ id: 100 }),
|
||||
completeScanJob: vi.fn().mockResolvedValue(undefined),
|
||||
failScanJob: vi.fn().mockResolvedValue(undefined),
|
||||
findBooksByLibraryFolder: vi.fn().mockResolvedValue([]),
|
||||
findBookFilesByLibraryFolder: vi.fn().mockResolvedValue([]),
|
||||
createBook: vi.fn().mockResolvedValue({ id: 1, status: 'present', libraryFolderId: 1, folderPath: '/library/Author/Book', libraryId: 1 }),
|
||||
updateBookStatus: vi.fn().mockResolvedValue(undefined),
|
||||
markBooksAsMissing: vi.fn().mockResolvedValue(undefined),
|
||||
createBookFile: vi.fn().mockResolvedValue(makeBookFile()),
|
||||
updateBookFile: vi.fn().mockResolvedValue(makeBookFile()),
|
||||
findBookFileByHash: vi.fn().mockResolvedValue(null),
|
||||
findPrimaryBookFilesByLibrary: vi.fn().mockResolvedValue([]),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
const mockGateway = {
|
||||
emitProgress: vi.fn(),
|
||||
emitBookMissing: vi.fn(),
|
||||
emitBookRestored: vi.fn(),
|
||||
emitBookMoved: vi.fn(),
|
||||
emitCoverRefreshed: vi.fn(),
|
||||
emitCoverRefreshProgress: vi.fn(),
|
||||
};
|
||||
|
||||
const mockMetadata = {
|
||||
extractAndSave: vi.fn().mockResolvedValue(undefined),
|
||||
refreshCoverForBook: vi.fn().mockResolvedValue(false),
|
||||
};
|
||||
|
||||
function makeService(repo: ReturnType<typeof makeRepo>) {
|
||||
const jobStore = new ScanJobStore();
|
||||
const service = new ScannerService(repo as any, mockMetadata as any, jobStore, mockGateway as any);
|
||||
return { service, jobStore };
|
||||
}
|
||||
|
||||
/**
|
||||
* Await the async scan by hooking into completeScanJob/failScanJob.
|
||||
* Must be called before startScan so the mock is set up in time.
|
||||
*/
|
||||
function awaitScan(repo: ReturnType<typeof makeRepo>): Promise<void> {
|
||||
return new Promise<void>((resolve) => {
|
||||
repo.completeScanJob.mockImplementationOnce(() => {
|
||||
resolve();
|
||||
return Promise.resolve(undefined);
|
||||
});
|
||||
repo.failScanJob.mockImplementationOnce(() => {
|
||||
resolve();
|
||||
return Promise.resolve(undefined);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockFindCandidates.mockResolvedValue([]);
|
||||
mockFingerprint.mockResolvedValue('hash-abc');
|
||||
});
|
||||
|
||||
// ── startScan — precondition checks ──────────────────────────────────────────
|
||||
|
||||
describe('startScan — preconditions', () => {
|
||||
it('throws ConflictException when a scan is already running for the library', async () => {
|
||||
const repo = makeRepo();
|
||||
const { service, jobStore } = makeService(repo);
|
||||
jobStore.create(99, 1, 0); // simulate running scan for library 1
|
||||
|
||||
await expect(service.startScan(1, 'manual')).rejects.toThrow(ConflictException);
|
||||
});
|
||||
|
||||
it('throws NotFoundException when library has no configured folders', async () => {
|
||||
const repo = makeRepo({ findLibraryFolders: vi.fn().mockResolvedValue([]) });
|
||||
const { service } = makeService(repo);
|
||||
|
||||
await expect(service.startScan(1, 'manual')).rejects.toThrow(NotFoundException);
|
||||
});
|
||||
|
||||
it('returns a jobId immediately without waiting for scan to finish', async () => {
|
||||
const repo = makeRepo();
|
||||
void awaitScan(repo); // set up so it doesn't hang
|
||||
const { service } = makeService(repo);
|
||||
|
||||
const result = await service.startScan(1, 'manual');
|
||||
expect(result).toHaveProperty('jobId', 100);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Missing book detection ────────────────────────────────────────────────────
|
||||
|
||||
describe('missing book detection', () => {
|
||||
it('marks books as missing when they are in the DB but not found in candidates', async () => {
|
||||
const repo = makeRepo({
|
||||
findBooksByLibraryFolder: vi.fn().mockResolvedValue([
|
||||
{ id: 5, libraryId: 1, libraryFolderId: 1, folderPath: '/library/old/book', status: 'present' },
|
||||
{ id: 6, libraryId: 1, libraryFolderId: 1, folderPath: '/library/old/other', status: 'present' },
|
||||
]),
|
||||
});
|
||||
mockFindCandidates.mockResolvedValue([]); // nothing found on disk
|
||||
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.markBooksAsMissing).toHaveBeenCalledWith([5, 6]);
|
||||
});
|
||||
|
||||
it('does not mark books as missing when they appear in candidates', async () => {
|
||||
const folderPath = '/library/Author/Book';
|
||||
const repo = makeRepo({
|
||||
findBooksByLibraryFolder: vi.fn().mockResolvedValue([{ id: 5, libraryId: 1, libraryFolderId: 1, folderPath, status: 'present' }]),
|
||||
});
|
||||
mockFindCandidates.mockResolvedValue([makeCandidate(folderPath, [makeFileStat()])]);
|
||||
repo.createBook.mockResolvedValue({ id: 5, status: 'present', libraryFolderId: 1, folderPath, libraryId: 1 });
|
||||
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.markBooksAsMissing).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── excludePatterns wiring ────────────────────────────────────────────────────
|
||||
|
||||
describe('excludePatterns', () => {
|
||||
it('passes excludePatterns from library settings to findBookCandidates', async () => {
|
||||
const repo = makeRepo({
|
||||
findLibrarySettings: vi
|
||||
.fn()
|
||||
.mockResolvedValue({ allowedFormats: [], formatPriority: DEFAULT_FORMAT_PRIORITY, excludePatterns: ['#recycle', '*.bak'] }),
|
||||
});
|
||||
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(mockFindCandidates).toHaveBeenCalledWith('/library', ['#recycle', '*.bak'], expect.any(Function));
|
||||
});
|
||||
});
|
||||
|
||||
// ── New file — happy path ─────────────────────────────────────────────────────
|
||||
|
||||
describe('genuinely new primary file', () => {
|
||||
it('creates a book record and a book file record', async () => {
|
||||
const candidate = makeCandidate('/library/Author/Book', [makeFileStat()]);
|
||||
mockFindCandidates.mockResolvedValue([candidate]);
|
||||
|
||||
const repo = makeRepo();
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.createBook).toHaveBeenCalled();
|
||||
expect(repo.createBookFile).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ absolutePath: '/library/Author/Book/book.epub', format: 'epub', role: 'primary' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('extracts metadata for new primary files in supported formats', async () => {
|
||||
const candidate = makeCandidate('/library/Author/Book', [makeFileStat()]);
|
||||
mockFindCandidates.mockResolvedValue([candidate]);
|
||||
|
||||
const repo = makeRepo();
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(mockMetadata.extractAndSave).toHaveBeenCalledWith(expect.any(Number), '/library/Author/Book/book.epub', 'epub');
|
||||
});
|
||||
|
||||
it('does not extract metadata for non-primary files', async () => {
|
||||
const primary = makeFileStat({ absolutePath: '/library/Book/book.epub', relPath: 'Book/book.epub' });
|
||||
const cover = makeFileStat({ absolutePath: '/library/Book/cover.jpg', relPath: 'Book/cover.jpg', sizeBytes: 512 });
|
||||
const candidate = makeCandidate('/library/Book', [primary, cover]);
|
||||
mockFindCandidates.mockResolvedValue([candidate]);
|
||||
|
||||
const repo = makeRepo();
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
// extractAndSave called once for epub, not for cover.jpg
|
||||
expect(mockMetadata.extractAndSave).toHaveBeenCalledTimes(1);
|
||||
expect(mockMetadata.extractAndSave).toHaveBeenCalledWith(expect.any(Number), '/library/Book/book.epub', 'epub');
|
||||
});
|
||||
|
||||
it('continues scanning when metadata extraction fails', async () => {
|
||||
mockMetadata.extractAndSave.mockRejectedValueOnce(new Error('parse error'));
|
||||
const candidate = makeCandidate('/library/Book', [makeFileStat()]);
|
||||
mockFindCandidates.mockResolvedValue([candidate]);
|
||||
|
||||
const repo = makeRepo();
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.completeScanJob).toHaveBeenCalled();
|
||||
expect(repo.failScanJob).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Zero-byte file handling ───────────────────────────────────────────────────
|
||||
|
||||
describe('zero-byte primary files', () => {
|
||||
it('skips zero-byte primary files — no book file created, no metadata extracted', async () => {
|
||||
const zeroByte = makeFileStat({ sizeBytes: 0 });
|
||||
const candidate = makeCandidate('/library/Book', [zeroByte]);
|
||||
mockFindCandidates.mockResolvedValue([candidate]);
|
||||
|
||||
const repo = makeRepo();
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.createBookFile).not.toHaveBeenCalled();
|
||||
expect(mockMetadata.extractAndSave).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('zero-byte primary does not win format election — valid sibling format gets primary role', async () => {
|
||||
// epub is first in formatPriority but is zero-byte → should NOT win
|
||||
// pdf is second and valid → should get primary role
|
||||
const zeroByte = makeFileStat({ absolutePath: '/library/Book/book.epub', relPath: 'Book/book.epub', sizeBytes: 0 });
|
||||
const valid = makeFileStat({ absolutePath: '/library/Book/book.pdf', relPath: 'Book/book.pdf', format: 'pdf' } as any);
|
||||
const candidate = makeCandidate('/library/Book', [zeroByte, valid]);
|
||||
mockFindCandidates.mockResolvedValue([candidate]);
|
||||
|
||||
const repo = makeRepo();
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.createBookFile).toHaveBeenCalledWith(expect.objectContaining({ absolutePath: '/library/Book/book.pdf', role: 'primary' }));
|
||||
});
|
||||
});
|
||||
|
||||
// ── File identity resolution ──────────────────────────────────────────────────
|
||||
|
||||
describe('file identity resolution', () => {
|
||||
it('updates the book file record when path matches but mtime changed', async () => {
|
||||
const oldMtime = new Date('2023-01-01');
|
||||
const newMtime = new Date('2024-06-01');
|
||||
const fileStat = makeFileStat({ mtime: newMtime });
|
||||
|
||||
const repo = makeRepo({
|
||||
findBookFilesByLibraryFolder: vi.fn().mockResolvedValue([makeBookFile({ absolutePath: fileStat.absolutePath, mtime: oldMtime })]),
|
||||
findBooksByLibraryFolder: vi
|
||||
.fn()
|
||||
.mockResolvedValue([{ id: 1, libraryId: 1, libraryFolderId: 1, folderPath: '/library/Author/Book', status: 'present' }]),
|
||||
});
|
||||
mockFindCandidates.mockResolvedValue([makeCandidate('/library/Author/Book', [fileStat])]);
|
||||
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.updateBookFile).toHaveBeenCalledWith(1, expect.objectContaining({ mtime: newMtime }));
|
||||
expect(repo.createBookFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not update when path matches and file is unchanged', async () => {
|
||||
const mtime = new Date('2024-01-01');
|
||||
const fileStat = makeFileStat({ mtime });
|
||||
|
||||
const repo = makeRepo({
|
||||
findBookFilesByLibraryFolder: vi.fn().mockResolvedValue([makeBookFile({ mtime, sizeBytes: fileStat.sizeBytes })]),
|
||||
findBooksByLibraryFolder: vi
|
||||
.fn()
|
||||
.mockResolvedValue([{ id: 1, libraryId: 1, libraryFolderId: 1, folderPath: '/library/Author/Book', status: 'present' }]),
|
||||
});
|
||||
mockFindCandidates.mockResolvedValue([makeCandidate('/library/Author/Book', [fileStat])]);
|
||||
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.updateBookFile).not.toHaveBeenCalled();
|
||||
expect(repo.createBookFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('updates path when inode matches a known file at a different path (renamed file)', async () => {
|
||||
const fileStat = makeFileStat({ absolutePath: '/library/Author/Book/renamed.epub', relPath: 'Author/Book/renamed.epub', ino: 9999 });
|
||||
|
||||
const repo = makeRepo({
|
||||
findBookFilesByLibraryFolder: vi.fn().mockResolvedValue([makeBookFile({ absolutePath: '/library/Author/Book/old-name.epub', ino: 9999 })]),
|
||||
findBooksByLibraryFolder: vi
|
||||
.fn()
|
||||
.mockResolvedValue([{ id: 1, libraryId: 1, libraryFolderId: 1, folderPath: '/library/Author/Book', status: 'present' }]),
|
||||
});
|
||||
mockFindCandidates.mockResolvedValue([makeCandidate('/library/Author/Book', [fileStat])]);
|
||||
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.updateBookFile).toHaveBeenCalledWith(1, expect.objectContaining({ absolutePath: '/library/Author/Book/renamed.epub' }));
|
||||
expect(repo.createBookFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('gracefully skips a file that disappears during fingerprinting (ENOENT) — scan still completes', async () => {
|
||||
const fileStat = makeFileStat({ ino: 7777 }); // different ino so inode match fails
|
||||
mockFindCandidates.mockResolvedValue([makeCandidate('/library/Author/Book', [fileStat])]);
|
||||
mockFingerprint.mockRejectedValueOnce(Object.assign(new Error('ENOENT'), { code: 'ENOENT' }));
|
||||
|
||||
const repo = makeRepo();
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.createBookFile).not.toHaveBeenCalled();
|
||||
expect(repo.completeScanJob).toHaveBeenCalled();
|
||||
expect(repo.failScanJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('updates path when hash matches a known file from a different filesystem (cross-fs move)', async () => {
|
||||
const fileStat = makeFileStat({ absolutePath: '/library/moved.epub', relPath: 'moved.epub', ino: 8888 });
|
||||
const existingFile = makeBookFile({ absolutePath: '/old-library/book.epub', ino: 1111, hash: 'fixed-hash' });
|
||||
|
||||
const repo = makeRepo({
|
||||
findBookFilesByLibraryFolder: vi.fn().mockResolvedValue([existingFile]),
|
||||
findBooksByLibraryFolder: vi
|
||||
.fn()
|
||||
.mockResolvedValue([{ id: 1, libraryId: 1, libraryFolderId: 1, folderPath: '/library/Author/Book', status: 'present' }]),
|
||||
findBookFileByHash: vi.fn().mockResolvedValue(existingFile),
|
||||
});
|
||||
mockFindCandidates.mockResolvedValue([makeCandidate('/library/Author/Book', [fileStat])]);
|
||||
mockFingerprint.mockResolvedValue('fixed-hash');
|
||||
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.updateBookFile).toHaveBeenCalledWith(existingFile.id, expect.objectContaining({ absolutePath: '/library/moved.epub' }));
|
||||
expect(repo.createBookFile).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// ── Format priority ───────────────────────────────────────────────────────────
|
||||
|
||||
describe('format priority', () => {
|
||||
it('assigns primary role to the highest-priority format when multiple primaries exist', async () => {
|
||||
const epub = makeFileStat({ absolutePath: '/library/Book/book.epub', relPath: 'Book/book.epub' });
|
||||
const mobi = makeFileStat({ absolutePath: '/library/Book/book.mobi', relPath: 'Book/book.mobi' });
|
||||
const candidate = makeCandidate('/library/Book', [epub, mobi]);
|
||||
mockFindCandidates.mockResolvedValue([candidate]);
|
||||
|
||||
const repo = makeRepo();
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
// epub comes before mobi in DEFAULT_FORMAT_PRIORITY
|
||||
const calls = repo.createBookFile.mock.calls.map((c: any) => ({ path: c[0].absolutePath, role: c[0].role }));
|
||||
const epubCall = calls.find((c: any) => c.path.endsWith('.epub'));
|
||||
const mobiCall = calls.find((c: any) => c.path.endsWith('.mobi'));
|
||||
expect(epubCall?.role).toBe('primary');
|
||||
expect(mobiCall?.role).toBe('supplementary');
|
||||
});
|
||||
});
|
||||
|
||||
// ── allowedFormats filtering ──────────────────────────────────────────────────
|
||||
|
||||
describe('allowedFormats filtering', () => {
|
||||
it('excludes primary files whose format is not in allowedFormats', async () => {
|
||||
const repo = makeRepo({
|
||||
findLibrarySettings: vi.fn().mockResolvedValue({ allowedFormats: ['epub'], formatPriority: DEFAULT_FORMAT_PRIORITY, excludePatterns: [] }),
|
||||
});
|
||||
|
||||
// findBookCandidates returns both, but the service filters before processing
|
||||
mockFindCandidates.mockResolvedValue([
|
||||
makeCandidate('/library/Book', [
|
||||
makeFileStat({ absolutePath: '/library/Book/book.epub', relPath: 'Book/book.epub' }),
|
||||
makeFileStat({ absolutePath: '/library/Book/book.cbz', relPath: 'Book/book.cbz' }),
|
||||
]),
|
||||
]);
|
||||
|
||||
const done = awaitScan(repo);
|
||||
const { service } = makeService(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
const createdPaths = repo.createBookFile.mock.calls.map((c: any) => c[0].absolutePath);
|
||||
expect(createdPaths.some((p: string) => p.endsWith('.cbz'))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -5,7 +5,7 @@ import { BookMetadataFetchOrchestratorService } from '../book-metadata-fetch/boo
|
||||
import { MetadataService } from '../metadata/metadata.service';
|
||||
import { ScanGateway } from './scan.gateway';
|
||||
import { ScanJobStore } from './scan-job-store.service';
|
||||
import { classifyFile, FileRole } from './lib/classify';
|
||||
import { classifyFile, DEFAULT_FORMAT_PRIORITY, FileRole } from './lib/classify';
|
||||
import { fingerprintFile } from './lib/hash';
|
||||
import { waitForStability } from './lib/stability';
|
||||
import { BookCandidate, FileStat, findBookCandidates } from './lib/walk';
|
||||
@@ -45,14 +45,15 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
if (folders.length === 0) throw new NotFoundException(`Library ${libraryId} has no folders`);
|
||||
|
||||
const allowedFormats = settings?.allowedFormats ?? [];
|
||||
const formatPriority = settings?.formatPriority ?? ['epub', 'pdf', 'cbz', 'cbr', 'cb7', 'mobi', 'azw3', 'azw', 'fb2'];
|
||||
const formatPriority = settings?.formatPriority ?? DEFAULT_FORMAT_PRIORITY;
|
||||
const excludePatterns = settings?.excludePatterns ?? [];
|
||||
|
||||
const job = await this.scannerRepo.createScanJob(libraryId, triggeredBy);
|
||||
|
||||
this.scanJobStore.create(job.id, libraryId, 0);
|
||||
this.emitFromStore(libraryId, job.id, 'running');
|
||||
|
||||
this.runScan(libraryId, job.id, folders, allowedFormats, formatPriority).catch((err) =>
|
||||
this.runScan(libraryId, job.id, folders, allowedFormats, formatPriority, excludePatterns).catch((err) =>
|
||||
this.logger.error(`Scan job ${job.id} crashed unexpectedly: ${(err as Error).message}`),
|
||||
);
|
||||
|
||||
@@ -99,6 +100,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
folders: Awaited<ReturnType<ScannerRepository['findLibraryFolders']>>,
|
||||
allowedFormats: string[],
|
||||
formatPriority: string[],
|
||||
excludePatterns: string[],
|
||||
): Promise<void> {
|
||||
this.logger.log(`Scan job ${jobId} started for library ${libraryId}`);
|
||||
|
||||
@@ -119,7 +121,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
for (const folder of folders) {
|
||||
let candidates: BookCandidate[] = [];
|
||||
try {
|
||||
candidates = await findBookCandidates(folder.path);
|
||||
candidates = await findBookCandidates(folder.path, excludePatterns, (msg) => this.logger.warn(msg));
|
||||
} catch (err) {
|
||||
this.logger.warn(`Cannot walk ${folder.path}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
@@ -151,8 +153,8 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
const totals: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
|
||||
|
||||
try {
|
||||
for (const { id: folderId, path: folderPath, candidates, knownBooks, knownFiles } of folderWork) {
|
||||
const counts = await this.scanFolderCandidates(folderId, libraryId, folderPath, candidates, knownBooks, knownFiles, jobId, formatPriority);
|
||||
for (const { id: folderId, candidates, knownBooks, knownFiles } of folderWork) {
|
||||
const counts = await this.scanFolderCandidates(folderId, libraryId, candidates, knownBooks, knownFiles, jobId, formatPriority);
|
||||
totals.addedCount += counts.addedCount;
|
||||
totals.updatedCount += counts.updatedCount;
|
||||
totals.missingCount += counts.missingCount;
|
||||
@@ -177,7 +179,6 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
private async scanFolderCandidates(
|
||||
libraryFolderId: number,
|
||||
libraryId: number,
|
||||
_folderPath: string,
|
||||
candidates: BookCandidate[],
|
||||
knownBooks: Awaited<ReturnType<ScannerRepository['findBooksByLibraryFolder']>>,
|
||||
knownFiles: Awaited<ReturnType<ScannerRepository['findBookFilesByLibraryFolder']>>,
|
||||
@@ -239,12 +240,19 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
counts.updated += fileCounts.updatedCount;
|
||||
|
||||
// Determine which format gets the 'primary' role when multiple primary-format files exist.
|
||||
const primaryFiles = candidate.files.filter((f) => classifyFile(f.absolutePath).role === 'primary');
|
||||
// Exclude zero-byte files from the election so a corrupt/empty file doesn't shadow a valid one.
|
||||
const primaryFiles = candidate.files.filter((f) => classifyFile(f.absolutePath).role === 'primary' && f.sizeBytes > 0);
|
||||
const chosenFormat =
|
||||
primaryFiles.length > 1 ? (formatPriority.find((fmt) => primaryFiles.some((f) => classifyFile(f.absolutePath).format === fmt)) ?? null) : null;
|
||||
|
||||
for (const fileStat of candidate.files) {
|
||||
const { format, role: classifiedRole } = classifyFile(fileStat.absolutePath);
|
||||
|
||||
if (classifiedRole === 'primary' && fileStat.sizeBytes === 0) {
|
||||
this.logger.warn(`Zero-byte primary file skipped: ${fileStat.absolutePath}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const role: FileRole =
|
||||
chosenFormat !== null && classifiedRole === 'primary' ? (format === chosenFormat ? 'primary' : 'supplementary') : classifiedRole;
|
||||
|
||||
@@ -349,7 +357,17 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
}
|
||||
|
||||
// 3. Hash match — cross-filesystem copy (expensive, last resort).
|
||||
const hash = await fingerprintFile(fileStat.absolutePath);
|
||||
let hash: string;
|
||||
try {
|
||||
hash = await fingerprintFile(fileStat.absolutePath);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === 'ENOENT' || code === 'EACCES') {
|
||||
this.logger.debug(`File no longer accessible, skipping: ${fileStat.absolutePath}`);
|
||||
return false;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const byHash = await this.scannerRepo.findBookFileByHash(hash, libraryFolderId);
|
||||
if (byHash) {
|
||||
await this.scannerRepo.updateBookFile(byHash.id, {
|
||||
|
||||
Reference in New Issue
Block a user