fix(scanner): invalidate dir scan state on book deletion (#1226)

* fix(scanner): invalidate dir scan state on book deletion

Deleting a book row left its library_dir_scan_state entries intact, so the incremental scanner kept skipping the unchanged directory and never re-created the book on rescan. Invalidate the deleted books' folder paths and ancestors after deleteByIds so the next scan re-reads them and re-discovers the records.

* fix(scanner): make scan-state invalidation atomic

Prevent deleted books and concurrent scans from restoring stale directory state by versioning each library folder and coordinating writes under a database transaction.

* test(server): repair full-suite failures

Update the Book Dock expectation to the unit-record processor contract and bound the watchdog cutoff against timestamps captured around the sweep.

---------

Co-authored-by: neon <18559658+neonsolstice@users.noreply.github.com>
This commit is contained in:
Gary J. Espitia S.
2026-08-31 23:23:57 +02:00
committed by GitHub
parent 6583fd696e
commit 2498e09083
16 changed files with 18760 additions and 64 deletions
@@ -0,0 +1,17 @@
import { scanStateInvalidationPaths } from './scan-state-paths.utils';
describe('scanStateInvalidationPaths', () => {
it('covers a folder-based book: the folder plus every ancestor to the filesystem root', () => {
expect(scanStateInvalidationPaths('/books/Series/Book')).toEqual(['/books/Series/Book', '/books/Series', '/books', '/']);
});
it('covers a loose-file book: the file path, containing directory, and every ancestor', () => {
expect(scanStateInvalidationPaths('/books/Book.epub')).toEqual(['/books/Book.epub', '/books', '/']);
});
it('includes the filesystem root exactly once', () => {
expect(scanStateInvalidationPaths('/only')).toEqual(['/only', '/']);
expect(scanStateInvalidationPaths('/book.epub')).toEqual(['/book.epub', '/']);
expect(scanStateInvalidationPaths('/')).toEqual(['/']);
});
});
@@ -0,0 +1,13 @@
import { dirname } from 'path';
export function scanStateInvalidationPaths(folderPath: string): string[] {
const paths = new Set<string>([folderPath]);
let current = dirname(folderPath);
while (!paths.has(current)) {
paths.add(current);
current = dirname(current);
}
return [...paths];
}
@@ -0,0 +1,2 @@
ALTER TABLE "library_folders" ADD COLUMN "scan_state_version" bigint DEFAULT 0 NOT NULL;--> statement-breakpoint
ALTER TABLE "library_folders" ADD CONSTRAINT "library_folders_scan_state_version_nonnegative_chk" CHECK ("library_folders"."scan_state_version" >= 0);
File diff suppressed because it is too large Load Diff
@@ -589,6 +589,13 @@
"when": 1788194498208,
"tag": "0083_add_book_requests",
"breakpoints": true
},
{
"idx": 84,
"version": "7",
"when": 1788206528114,
"tag": "0084_add_scan_state_version",
"breakpoints": true
}
]
}
+3
View File
@@ -1,5 +1,6 @@
import { sql } from 'drizzle-orm';
import {
bigint,
boolean,
check,
doublePrecision,
@@ -110,12 +111,14 @@ export const libraryFolders = pgTable(
.notNull()
.references(() => libraries.id, { onDelete: 'cascade' }),
path: varchar('path', { length: 4096 }).notNull(),
scanStateVersion: bigint('scan_state_version', { mode: 'number' }).notNull().default(0),
createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(),
},
(t) => [
index('library_folders_library_id_idx').on(t.libraryId),
uniqueIndex('library_folders_library_path_uidx').on(t.libraryId, t.path),
unique('library_folders_id_library_id_unique').on(t.id, t.libraryId),
check('library_folders_scan_state_version_nonnegative_chk', sql`${t.scanStateVersion} >= 0`),
],
);
@@ -698,7 +698,7 @@ describe('BookDockFinalizeService', () => {
vi.spyOn(service as never, 'applyMetadata').mockResolvedValue(undefined as never);
vi.spyOn(service as never, 'cleanupBookDockRecord').mockResolvedValue(undefined as never);
mockStat.mockResolvedValueOnce({ size: 100 } as never);
processor.createBookRecord.mockResolvedValueOnce({ bookId: 557 });
processor.createUnitBookRecords.mockResolvedValueOnce({ bookIds: [557], createdBookIds: [557], attachedFileIds: [] });
const row = makeRow({ targetLibraryId: 5, targetFolderId: 9, selectedMetadata: defaultPatternMetadata() });
const destPath = '/library/Frank Herbert/Dune/01. Dune (1965).epub';
@@ -710,7 +710,17 @@ describe('BookDockFinalizeService', () => {
bookId: 557,
});
expect(storage.moveToPath).toHaveBeenCalledWith('/tmp/book.epub', destPath);
expect(processor.createBookRecord).toHaveBeenCalledWith(5, 9, destPath, destPath, 'Frank Herbert/Dune/01. Dune (1965).epub', 'epub', 100);
expect(processor.createUnitBookRecords).toHaveBeenCalledWith(5, 9, [
{
folderPath: destPath,
absolutePath: destPath,
relPath: 'Frank Herbert/Dune/01. Dune (1965).epub',
format: 'epub',
sizeBytes: 100,
role: 'content',
sortOrder: 0,
},
]);
});
it('uses the file path as bookFolderPath in book_per_file mode', async () => {
@@ -115,9 +115,10 @@ describe('RequestWatchdogService stranded searches', () => {
await service.sweep();
const after = Date.now();
const [cutoff] = requests.findStrandedSearching.mock.calls[0] as [Date, number];
expect(cutoff.getTime()).toBeLessThanOrEqual(before - 30 * 60 * 1000);
expect(cutoff.getTime()).toBeGreaterThan(before - 31 * 60 * 1000);
expect(cutoff.getTime()).toBeGreaterThanOrEqual(before - 30 * 60 * 1000);
expect(cutoff.getTime()).toBeLessThanOrEqual(after - 30 * 60 * 1000);
});
/** No in-memory task survives a restart, so every one of these is stranded, however recent. */
@@ -12,6 +12,7 @@ function makeSelectChain<T>(terminalMethod: string, terminalResult: T) {
orderBy: vi.fn(),
limit: vi.fn(),
offset: vi.fn(),
for: vi.fn(),
};
chain.from.mockReturnValue(chain);
@@ -19,7 +20,11 @@ function makeSelectChain<T>(terminalMethod: string, terminalResult: T) {
chain.innerJoin.mockReturnValue(chain);
chain.offset.mockReturnValue(chain);
if (terminalMethod === 'where') {
if (terminalMethod === 'for') {
chain.where.mockReturnValue(chain);
chain.orderBy.mockReturnValue(chain);
chain.for.mockResolvedValue(terminalResult);
} else if (terminalMethod === 'where') {
chain.where.mockResolvedValue(terminalResult);
chain.orderBy.mockReturnValue(chain);
chain.limit.mockReturnValue(chain);
@@ -113,6 +118,73 @@ describe('BookRepository', () => {
expect(selectChain.leftJoin).toHaveBeenCalledTimes(1);
});
it('deletes books and invalidates their exact scan-state paths in one transaction', async () => {
const bookRows = [{ id: 10, libraryFolderId: 7, folderPath: '/books/Series/Book' }];
const bookSelect = makeSelectChain('for', bookRows);
const folderSelect = makeSelectChain('for', [{ id: 7 }]);
const updateWhere = vi.fn().mockResolvedValue(undefined);
const updateSet = vi.fn().mockReturnValue({ where: updateWhere });
const stateDeleteWhere = vi.fn().mockResolvedValue(undefined);
const bookDeleteWhere = vi.fn().mockResolvedValue(undefined);
const tx = {
select: vi.fn().mockReturnValueOnce(bookSelect).mockReturnValueOnce(folderSelect),
update: vi.fn().mockReturnValue({ set: updateSet }),
delete: vi.fn().mockReturnValueOnce({ where: stateDeleteWhere }).mockReturnValueOnce({ where: bookDeleteWhere }),
};
const db = { transaction: vi.fn((callback: (executor: typeof tx) => Promise<void>) => callback(tx)) };
const repo = new BookRepository(db as never);
await repo.deleteByIdsAndInvalidateScanState([10, 10]);
expect(db.transaction).toHaveBeenCalledTimes(1);
expect(tx.update).toHaveBeenCalledTimes(1);
expect(tx.delete).toHaveBeenCalledTimes(2);
const invalidationQuery = new PgDialect().sqlToQuery(stateDeleteWhere.mock.calls[0]![0]);
expect(invalidationQuery.sql).toContain('"library_dir_scan_state"."library_folder_id" = $1');
expect(invalidationQuery.sql).toContain('"library_dir_scan_state"."dir_path" in');
expect(invalidationQuery.params).toEqual([7, '/books/Series/Book', '/books/Series', '/books', '/']);
expect(invalidationQuery.params).not.toContain('/books/Sibling');
});
it('does not reach the book delete when scan-state invalidation fails', async () => {
const bookSelect = makeSelectChain('for', [{ id: 10, libraryFolderId: 7, folderPath: '/books/Book' }]);
const folderSelect = makeSelectChain('for', [{ id: 7 }]);
const updateWhere = vi.fn().mockResolvedValue(undefined);
const stateDeleteWhere = vi.fn().mockRejectedValue(new Error('invalidation failed'));
const tx = {
select: vi.fn().mockReturnValueOnce(bookSelect).mockReturnValueOnce(folderSelect),
update: vi.fn().mockReturnValue({ set: vi.fn().mockReturnValue({ where: updateWhere }) }),
delete: vi.fn().mockReturnValue({ where: stateDeleteWhere }),
};
const db = { transaction: vi.fn((callback: (executor: typeof tx) => Promise<void>) => callback(tx)) };
const repo = new BookRepository(db as never);
await expect(repo.deleteByIdsAndInvalidateScanState([10])).rejects.toThrow('invalidation failed');
expect(tx.delete).toHaveBeenCalledTimes(1);
});
it('chunks scan-state invalidation paths for large deletions', async () => {
const bookRows = Array.from({ length: 501 }, (_, index) => ({
id: index + 1,
libraryFolderId: 7,
folderPath: `/books/book-${index + 1}.epub`,
}));
const bookSelect = makeSelectChain('for', bookRows);
const folderSelect = makeSelectChain('for', [{ id: 7 }]);
const tx = {
select: vi.fn().mockReturnValueOnce(bookSelect).mockReturnValueOnce(folderSelect),
update: vi.fn().mockReturnValue({ set: vi.fn().mockReturnValue({ where: vi.fn().mockResolvedValue(undefined) }) }),
delete: vi.fn().mockReturnValue({ where: vi.fn().mockResolvedValue(undefined) }),
};
const db = { transaction: vi.fn((callback: (executor: typeof tx) => Promise<void>) => callback(tx)) };
const repo = new BookRepository(db as never);
await repo.deleteByIdsAndInvalidateScanState(bookRows.map((row) => row.id));
expect(tx.delete).toHaveBeenCalledTimes(3);
});
it('findCards loads card rows and related collections for the current user', async () => {
const rows = [{ id: 10, primaryFileId: 1001, _total: 1 }];
const authorRows = [{ bookId: 10, name: 'Frank Herbert' }];
@@ -19,6 +19,7 @@ import { buildContentFilterClauses } from '../../common/utils/content-filter-sql
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
import { advanceIsoTimestamp } from '../../common/utils/iso-timestamp.utils';
import { parsePgTimestamptz } from '../../common/utils/pg-timestamp.utils';
import { scanStateInvalidationPaths } from '../../common/utils/scan-state-paths.utils';
import { seriesIndexSortKeySql } from '../../common/utils/series-index-sql.utils';
import { SeriesIdentityService } from '../../common/services/series-identity.service';
import { SeriesMembershipService } from '../../common/services/series-membership.service';
@@ -49,6 +50,7 @@ import {
koreaderDeviceProgress,
koreaderProgressResets,
libraries,
libraryFolders,
narrators,
audiobookProgress,
readingProgress,
@@ -1893,6 +1895,66 @@ export class BookRepository {
await this.db.delete(books).where(inArray(books.id, bookIds));
}
async deleteByIdsAndInvalidateScanState(bookIds: number[]): Promise<void> {
const uniqueBookIds = [...new Set(bookIds)].sort((a, b) => a - b);
if (uniqueBookIds.length === 0) return;
await this.db.transaction(async (tx) => {
const rows = await tx
.select({ id: books.id, libraryFolderId: books.libraryFolderId, folderPath: books.folderPath })
.from(books)
.where(inArray(books.id, uniqueBookIds))
.orderBy(asc(books.id))
.for('update');
if (rows.length === 0) return;
const libraryFolderIds = [...new Set(rows.map((row) => row.libraryFolderId))].sort((a, b) => a - b);
await tx
.select({ id: libraryFolders.id })
.from(libraryFolders)
.where(inArray(libraryFolders.id, libraryFolderIds))
.orderBy(asc(libraryFolders.id))
.for('update');
await tx
.update(libraryFolders)
.set({ scanStateVersion: sql`${libraryFolders.scanStateVersion} + 1` })
.where(inArray(libraryFolders.id, libraryFolderIds));
const pathsByLibraryFolder = new Map<number, Set<string>>();
for (const row of rows) {
let paths = pathsByLibraryFolder.get(row.libraryFolderId);
if (!paths) {
paths = new Set<string>();
pathsByLibraryFolder.set(row.libraryFolderId, paths);
}
for (const path of scanStateInvalidationPaths(row.folderPath)) paths.add(path);
}
const chunkSize = 500;
for (const [libraryFolderId, paths] of pathsByLibraryFolder) {
const pathList = [...paths];
for (let offset = 0; offset < pathList.length; offset += chunkSize) {
await tx
.delete(schema.libraryDirScanState)
.where(
and(
eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId),
inArray(schema.libraryDirScanState.dirPath, pathList.slice(offset, offset + chunkSize)),
),
);
}
}
await tx.delete(books).where(
inArray(
books.id,
rows.map((row) => row.id),
),
);
});
}
async bulkSetRating(bookIds: number[], rating: number | null, userId: number): Promise<void> {
if (bookIds.length === 0) return;
+33 -3
View File
@@ -142,6 +142,7 @@ function makeService(overrides: { bookMetadataLockService?: unknown } = {}) {
replaceCommunityRatings: vi.fn(),
withTransaction: vi.fn(),
deleteByIds: vi.fn(),
deleteByIdsAndInvalidateScanState: vi.fn(),
findAllIds: vi.fn(),
findIdsByWhere: vi.fn(),
findCardsCollapsed: vi.fn(),
@@ -2240,13 +2241,13 @@ describe('BookService', () => {
{ id: 3, title: 'Dune' },
{ id: 4, title: null },
]);
bookRepo.deleteByIds.mockResolvedValue(undefined);
bookRepo.deleteByIdsAndInvalidateScanState.mockResolvedValue(undefined);
mockRm.mockRejectedValue(new Error('cannot delete'));
const result = await service.deleteBooks([3, 4], user);
expect(libraryService.verifyUserAccess).toHaveBeenCalledTimes(2);
expect(bookRepo.deleteByIds).toHaveBeenCalledWith([3, 4]);
expect(bookRepo.deleteByIdsAndInvalidateScanState).toHaveBeenCalledWith([3, 4]);
expect(mockRm).toHaveBeenCalledWith('/tmp/books/covers/3', { recursive: true, force: true });
expect(mockRm).toHaveBeenCalledWith('/tmp/books/covers/4', { recursive: true, force: true });
expect(mockRm).toHaveBeenCalledWith('/tmp/library/book3.epub', { force: true });
@@ -2272,7 +2273,7 @@ describe('BookService', () => {
Promise.resolve([...ids].reverse().map((id) => ({ id, title: `Book ${id}` }))),
);
bookRepo.findAllFilesByBookIds.mockResolvedValue([]);
bookRepo.deleteByIds.mockResolvedValue(undefined);
bookRepo.deleteByIdsAndInvalidateScanState.mockResolvedValue(undefined);
const result = await service.deleteBooks(bookIds, user);
@@ -2284,6 +2285,35 @@ describe('BookService', () => {
expect(result.omitted).toBe(5);
});
it('deletes books through the scan-state-aware repository transaction', async () => {
const { service, bookRepo } = makeService();
const user = makeUser();
bookRepo.findLibraryIdsByBookIds.mockResolvedValue([{ id: 3, libraryId: 7 }]);
bookRepo.findDeletionAuditBooksByIds.mockResolvedValue([{ id: 3, title: 'Dune' }]);
bookRepo.findAllFilesByBookIds.mockResolvedValue([]);
bookRepo.deleteByIdsAndInvalidateScanState.mockResolvedValue(undefined);
await service.deleteBooks([3], user);
expect(bookRepo.deleteByIdsAndInvalidateScanState).toHaveBeenCalledWith([3]);
});
it('does not commit book deletion when scan state invalidation fails', async () => {
const { service, bookRepo } = makeService();
const user = makeUser();
const warnSpy = vi.spyOn((service as unknown as { logger: { warn: (message: string) => void } }).logger, 'warn').mockImplementation();
bookRepo.findLibraryIdsByBookIds.mockResolvedValue([{ id: 3, libraryId: 7 }]);
bookRepo.findDeletionAuditBooksByIds.mockResolvedValue([{ id: 3, title: 'Dune' }]);
bookRepo.findAllFilesByBookIds.mockResolvedValue([]);
bookRepo.deleteByIdsAndInvalidateScanState.mockRejectedValue(new Error('db down'));
await expect(service.deleteBooks([3], user)).rejects.toThrow('db down');
expect(warnSpy).toHaveBeenCalled();
expect(mockRm).not.toHaveBeenCalled();
});
it('returns queued=0 when embed-all is already running', async () => {
const { service, bookRepo, embedder } = makeService();
let resolveEmbed: (() => void) | null = null;
+1 -1
View File
@@ -1548,7 +1548,7 @@ export class BookService {
return row ? [row] : [];
});
const files = await this.bookRepo.findAllFilesByBookIds(bookIds);
await this.bookRepo.deleteByIds(bookIds);
await this.bookRepo.deleteByIdsAndInvalidateScanState(deletedBookIds);
const deleteTargets = [
...rows.map((row) => ({
path: join(this.appDataPath, 'covers', String(row.id)),
@@ -330,51 +330,81 @@ describe('ScannerRepository', () => {
});
});
it('handles directory scan state lookups and upserts', async () => {
const { repo, queues, db } = makeRepo();
it('loads a versioned directory scan-state snapshot', async () => {
const { repo, queues } = makeRepo();
queues.select.push([
{ dirPath: '/books/a', lastSeenMtimeMs: 100 },
{ dirPath: '/books/b', lastSeenMtimeMs: 200 },
{ version: 7, dirPath: '/books/a', lastSeenMtimeMs: 100 },
{ version: 7, dirPath: '/books/b', lastSeenMtimeMs: 200 },
]);
const snapshot = await repo.findDirScanStateSnapshot(7);
expect(snapshot?.version).toBe(7);
expect(snapshot?.mtimes).toEqual(
new Map([
['/books/a', 100],
['/books/b', 200],
]),
);
});
it('returns an empty snapshot for a folder without state and null for a missing folder', async () => {
const { repo, queues } = makeRepo();
queues.select.push([{ version: 3, dirPath: null, lastSeenMtimeMs: null }]);
queues.select.push([]);
await expect(repo.findDirScanStateSnapshot(7)).resolves.toEqual({ version: 3, mtimes: new Map() });
await expect(repo.findDirScanStateSnapshot(8)).resolves.toBeNull();
});
it('persists directory scan state atomically when the version still matches', async () => {
const { repo, queues, db } = makeRepo();
queues.select.push([{ version: 7 }]);
queues.insert.push([]);
queues.insert.push([]);
const map = await repo.findDirScanState(7);
expect(map.get('/books/a')).toBe(100);
expect(map.get('/books/b')).toBe(200);
await repo.upsertDirScanState(7, []);
expect(db.insert).toHaveBeenCalledTimes(0);
queues.select.push([
{ id: 1, dirPath: '/books/0' },
{ id: 2, dirPath: '/stale' },
]);
const entries = Array.from({ length: 501 }, (_, i) => ({ dirPath: `/books/${i}`, mtimeMs: i + 0.4 }));
await repo.upsertDirScanState(7, entries);
await expect(repo.persistDirScanState(7, 7, entries)).resolves.toBe(true);
expect(db.transaction).toHaveBeenCalledTimes(1);
expect(db.insert).toHaveBeenCalledTimes(2);
expect(db.delete).toHaveBeenCalledTimes(1);
});
it('deletes stale directory scan state across all branch paths', async () => {
it('skips directory scan-state persistence after the version changes', async () => {
const { repo, queues, db } = makeRepo();
queues.select.push([{ version: 8 }]);
await repo.deleteStaleDirScanState(7, new Set());
expect(db.delete).toHaveBeenCalledTimes(1);
await expect(repo.persistDirScanState(7, 7, [{ dirPath: '/books', mtimeMs: 100 }])).resolves.toBe(false);
queues.select.push([
{ id: 1, dirPath: '/keep' },
{ id: 2, dirPath: '/drop' },
]);
await repo.deleteStaleDirScanState(7, new Set(['/keep']));
expect(db.delete).toHaveBeenCalledTimes(2);
queues.select.push([{ id: 3, dirPath: '/keep-only' }]);
await repo.deleteStaleDirScanState(7, new Set(['/keep-only']));
expect(db.delete).toHaveBeenCalledTimes(2);
expect(db.insert).not.toHaveBeenCalled();
expect(db.delete).not.toHaveBeenCalled();
});
it('clears directory scan state for a library folder', async () => {
const { repo, db } = makeRepo();
await repo.clearDirScanState(21);
it('clears directory scan state under the folder lock and returns its version', async () => {
const { repo, queues, db } = makeRepo();
queues.update.push([{ version: 9 }]);
await expect(repo.clearDirScanState(21)).resolves.toBe(9);
expect(db.transaction).toHaveBeenCalledTimes(1);
expect(db.update).toHaveBeenCalledTimes(1);
expect(db.delete).toHaveBeenCalledTimes(1);
});
it('does not clear directory scan state when the library folder no longer exists', async () => {
const { repo, queues, db } = makeRepo();
queues.update.push([]);
await expect(repo.clearDirScanState(21)).resolves.toBeNull();
expect(db.delete).not.toHaveBeenCalled();
});
it('findRecentScanJobs reads the newest jobs for one library', async () => {
const { repo, queues, db } = makeRepo();
queues.select.push([{ id: 2 }, { id: 1 }]);
@@ -21,6 +21,7 @@ import {
} from '../../db/schema';
type Db = NodePgDatabase<typeof schema>;
type DbTransaction = Parameters<Parameters<Db['transaction']>[0]>[0];
type MoveBookToLibraryResult = Pick<typeof books.$inferSelect, 'id' | 'libraryId' | 'libraryFolderId' | 'folderPath' | 'status'> & {
previousLibraryId: number;
libraryChanged: boolean;
@@ -619,20 +620,51 @@ export class ScannerRepository {
// ── Dir Scan State (Incremental Scan) ────────────────────────────────────
async findDirScanState(libraryFolderId: number): Promise<Map<string, number>> {
async findDirScanStateSnapshot(libraryFolderId: number): Promise<{ version: number; mtimes: Map<string, number> } | null> {
const rows = await this.db
.select({ dirPath: schema.libraryDirScanState.dirPath, lastSeenMtimeMs: schema.libraryDirScanState.lastSeenMtimeMs })
.from(schema.libraryDirScanState)
.where(eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId));
return new Map(rows.map((r) => [r.dirPath, r.lastSeenMtimeMs]));
.select({
version: libraryFolders.scanStateVersion,
dirPath: schema.libraryDirScanState.dirPath,
lastSeenMtimeMs: schema.libraryDirScanState.lastSeenMtimeMs,
})
.from(libraryFolders)
.leftJoin(schema.libraryDirScanState, eq(schema.libraryDirScanState.libraryFolderId, libraryFolders.id))
.where(eq(libraryFolders.id, libraryFolderId));
const first = rows[0];
if (!first) return null;
const mtimes = new Map<string, number>();
for (const row of rows) {
if (row.dirPath !== null && row.lastSeenMtimeMs !== null) mtimes.set(row.dirPath, row.lastSeenMtimeMs);
}
return { version: first.version, mtimes };
}
async upsertDirScanState(libraryFolderId: number, entries: Array<{ dirPath: string; mtimeMs: number }>): Promise<void> {
async persistDirScanState(
libraryFolderId: number,
expectedVersion: number,
entries: Array<{ dirPath: string; mtimeMs: number }>,
): Promise<boolean> {
return this.db.transaction(async (tx) => {
const [folder] = await tx
.select({ version: libraryFolders.scanStateVersion })
.from(libraryFolders)
.where(eq(libraryFolders.id, libraryFolderId))
.for('update');
if (!folder || folder.version !== expectedVersion) return false;
await this.upsertDirScanState(tx, libraryFolderId, entries);
await this.deleteStaleDirScanState(tx, libraryFolderId, new Set(entries.map((entry) => entry.dirPath)));
return true;
});
}
private async upsertDirScanState(tx: DbTransaction, libraryFolderId: number, entries: Array<{ dirPath: string; mtimeMs: number }>): Promise<void> {
if (entries.length === 0) return;
const CHUNK = 500;
for (let i = 0; i < entries.length; i += CHUNK) {
const chunk = entries.slice(i, i + CHUNK);
await this.db
await tx
.insert(schema.libraryDirScanState)
.values(chunk.map((e) => ({ libraryFolderId, dirPath: e.dirPath, lastSeenMtimeMs: Math.round(e.mtimeMs) })))
.onConflictDoUpdate({
@@ -642,12 +674,12 @@ export class ScannerRepository {
}
}
async deleteStaleDirScanState(libraryFolderId: number, validPaths: Set<string>): Promise<void> {
private async deleteStaleDirScanState(tx: DbTransaction, libraryFolderId: number, validPaths: Set<string>): Promise<void> {
if (validPaths.size === 0) {
await this.db.delete(schema.libraryDirScanState).where(eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId));
await tx.delete(schema.libraryDirScanState).where(eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId));
return;
}
const allRows = await this.db
const allRows = await tx
.select({ id: schema.libraryDirScanState.id, dirPath: schema.libraryDirScanState.dirPath })
.from(schema.libraryDirScanState)
.where(eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId));
@@ -655,11 +687,21 @@ export class ScannerRepository {
if (staleIds.length === 0) return;
const CHUNK = 500;
for (let i = 0; i < staleIds.length; i += CHUNK) {
await this.db.delete(schema.libraryDirScanState).where(inArray(schema.libraryDirScanState.id, staleIds.slice(i, i + CHUNK)));
await tx.delete(schema.libraryDirScanState).where(inArray(schema.libraryDirScanState.id, staleIds.slice(i, i + CHUNK)));
}
}
async clearDirScanState(libraryFolderId: number): Promise<void> {
await this.db.delete(schema.libraryDirScanState).where(eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId));
async clearDirScanState(libraryFolderId: number): Promise<number | null> {
return this.db.transaction(async (tx) => {
const [folder] = await tx
.update(libraryFolders)
.set({ scanStateVersion: sql`${libraryFolders.scanStateVersion} + 1` })
.where(eq(libraryFolders.id, libraryFolderId))
.returning({ version: libraryFolders.scanStateVersion });
if (!folder) return null;
await tx.delete(schema.libraryDirScanState).where(eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId));
return folder.version;
});
}
}
@@ -123,10 +123,9 @@ function makeRepo(overrides: Record<string, unknown> = {}) {
findBookCardData: vi.fn().mockResolvedValue({ rows: [], authorRows: [], fileRows: [], genreRows: [] }),
deleteBookFile: vi.fn().mockResolvedValue(undefined),
updateBookFolderPath: vi.fn().mockResolvedValue(undefined),
findDirScanState: vi.fn().mockResolvedValue(new Map()),
upsertDirScanState: vi.fn().mockResolvedValue(undefined),
deleteStaleDirScanState: vi.fn().mockResolvedValue(undefined),
clearDirScanState: vi.fn().mockResolvedValue(undefined),
findDirScanStateSnapshot: vi.fn().mockResolvedValue({ version: 0, mtimes: new Map() }),
persistDirScanState: vi.fn().mockResolvedValue(true),
clearDirScanState: vi.fn().mockResolvedValue(1),
...overrides,
};
}
@@ -3501,7 +3500,7 @@ describe('incremental scan — dir state', () => {
['/library/Author/Book', 2000],
]);
const repo = makeRepo({
findDirScanState: vi.fn().mockResolvedValue(storedMtimes),
findDirScanStateSnapshot: vi.fn().mockResolvedValue({ version: 4, mtimes: storedMtimes }),
});
const { service } = makeService(repo);
@@ -3517,7 +3516,7 @@ describe('incremental scan — dir state', () => {
await service.startScan(1, 'manual');
await done;
expect(repo.findDirScanState).toHaveBeenCalledWith(1);
expect(repo.findDirScanStateSnapshot).toHaveBeenCalledWith(1);
expect(mockFindCandidates).toHaveBeenCalledWith('/library', [], expect.any(Function), storedMtimes);
});
@@ -3538,14 +3537,71 @@ describe('incremental scan — dir state', () => {
await service.startScan(1, 'manual');
await done;
expect(repo.upsertDirScanState).toHaveBeenCalledWith(
expect(repo.persistDirScanState).toHaveBeenCalledWith(
1,
1,
expect.arrayContaining([
{ dirPath: '/library/Author', mtimeMs: 1000 },
{ dirPath: '/library/Author/Book', mtimeMs: 2000 },
]),
);
expect(repo.deleteStaleDirScanState).toHaveBeenCalledWith(1, new Set(['/library/Author', '/library/Author/Book']));
});
it('does not restore directory state invalidated while a scan is in flight', async () => {
const state = new Map<string, number>([['/library/Author/Book', 2000]]);
let version = 0;
let releaseWalk: (() => void) | undefined;
let markWalkStarted: (() => void) | undefined;
const walkStarted = new Promise<void>((resolve) => {
markWalkStarted = resolve;
});
const walkReleased = new Promise<void>((resolve) => {
releaseWalk = resolve;
});
const repo = makeRepo({
findDirScanStateSnapshot: vi.fn().mockImplementation(() => Promise.resolve({ version, mtimes: new Map(state) })),
clearDirScanState: vi.fn().mockImplementation(() => {
state.clear();
version += 1;
return Promise.resolve(version);
}),
persistDirScanState: vi
.fn()
.mockImplementation((_folderId: number, expectedVersion: number, entries: Array<{ dirPath: string; mtimeMs: number }>) => {
if (expectedVersion !== version) return Promise.resolve(false);
state.clear();
for (const entry of entries) state.set(entry.dirPath, entry.mtimeMs);
return Promise.resolve(true);
}),
});
const { service, jobStore } = makeService(repo);
let done = awaitScan(repo);
await service.startScan(1, 'manual');
await done;
await vi.waitFor(() => expect(jobStore.isRunning(1)).toBe(false));
mockFindCandidates.mockImplementationOnce(async () => {
markWalkStarted?.();
await walkReleased;
return {
candidates: [],
skippedDirs: new Set(),
unchangedDirs: new Set(['/library/Author/Book']),
dirMtimes: new Map([['/library/Author/Book', 2000]]),
};
});
repo.createScanJob.mockResolvedValueOnce({ id: 101 });
done = awaitScan(repo);
await service.startScan(1, 'manual');
await walkStarted;
state.clear();
version += 1;
releaseWalk?.();
await done;
expect(state.size).toBe(0);
});
it('clears dir state when forceFullScan is true', async () => {
@@ -3556,7 +3612,7 @@ describe('incremental scan — dir state', () => {
await done;
expect(repo.clearDirScanState).toHaveBeenCalledWith(1);
expect(repo.findDirScanState).not.toHaveBeenCalled();
expect(repo.findDirScanStateSnapshot).not.toHaveBeenCalled();
});
it('excludes books in unchanged dirs from missing detection', async () => {
@@ -1236,14 +1236,19 @@ export class ScannerService implements OnApplicationBootstrap {
// Load stored dir mtimes for incremental scanning (unless forced full)
let knownDirMtimes: Map<string, number> | undefined;
let scanStateVersion: number | undefined;
if (!forceFullScan) {
try {
knownDirMtimes = await this.scannerRepo.findDirScanState(folder.id);
const snapshot = await this.scannerRepo.findDirScanStateSnapshot(folder.id);
if (snapshot) {
knownDirMtimes = snapshot.mtimes;
scanStateVersion = snapshot.version;
}
} catch {
// If loading fails, fall back to full scan for this folder
}
} else {
await this.scannerRepo.clearDirScanState(folder.id).catch(() => {});
scanStateVersion = (await this.scannerRepo.clearDirScanState(folder.id).catch(() => null)) ?? undefined;
}
try {
@@ -1296,11 +1301,10 @@ export class ScannerService implements OnApplicationBootstrap {
totals.missingCount += counts.missingCount;
// Persist dir scan state after successful folder processing
if (dirMtimes.size > 0) {
if (dirMtimes.size > 0 && scanStateVersion !== undefined) {
try {
const entries = [...dirMtimes].map(([dirPath, mtimeMs]) => ({ dirPath, mtimeMs }));
await this.scannerRepo.upsertDirScanState(folder.id, entries);
await this.scannerRepo.deleteStaleDirScanState(folder.id, new Set(dirMtimes.keys()));
await this.scannerRepo.persistDirScanState(folder.id, scanStateVersion, entries);
} catch (err) {
this.logger.warn(
`[${event}] [fail] libraryId=${libraryId} jobId=${jobId} libraryFolderId=${folder.id} errorClass=${err instanceof Error ? err.name : 'Error'} error="${sanitizeLogValue(err instanceof Error ? err.message : String(err))}" - dir scan state persistence failed`,