mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 08:21:39 -06:00
fix(scanner): invalidate dir scan state on book deletion (#1226)
* fix(scanner): invalidate dir scan state on book deletion Deleting a book row left its library_dir_scan_state entries intact, so the incremental scanner kept skipping the unchanged directory and never re-created the book on rescan. Invalidate the deleted books' folder paths and ancestors after deleteByIds so the next scan re-reads them and re-discovers the records. * fix(scanner): make scan-state invalidation atomic Prevent deleted books and concurrent scans from restoring stale directory state by versioning each library folder and coordinating writes under a database transaction. * test(server): repair full-suite failures Update the Book Dock expectation to the unit-record processor contract and bound the watchdog cutoff against timestamps captured around the sweep. --------- Co-authored-by: neon <18559658+neonsolstice@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
parent
6583fd696e
commit
2498e09083
@@ -0,0 +1,17 @@
|
||||
import { scanStateInvalidationPaths } from './scan-state-paths.utils';
|
||||
|
||||
describe('scanStateInvalidationPaths', () => {
|
||||
it('covers a folder-based book: the folder plus every ancestor to the filesystem root', () => {
|
||||
expect(scanStateInvalidationPaths('/books/Series/Book')).toEqual(['/books/Series/Book', '/books/Series', '/books', '/']);
|
||||
});
|
||||
|
||||
it('covers a loose-file book: the file path, containing directory, and every ancestor', () => {
|
||||
expect(scanStateInvalidationPaths('/books/Book.epub')).toEqual(['/books/Book.epub', '/books', '/']);
|
||||
});
|
||||
|
||||
it('includes the filesystem root exactly once', () => {
|
||||
expect(scanStateInvalidationPaths('/only')).toEqual(['/only', '/']);
|
||||
expect(scanStateInvalidationPaths('/book.epub')).toEqual(['/book.epub', '/']);
|
||||
expect(scanStateInvalidationPaths('/')).toEqual(['/']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,13 @@
|
||||
import { dirname } from 'path';
|
||||
|
||||
export function scanStateInvalidationPaths(folderPath: string): string[] {
|
||||
const paths = new Set<string>([folderPath]);
|
||||
let current = dirname(folderPath);
|
||||
|
||||
while (!paths.has(current)) {
|
||||
paths.add(current);
|
||||
current = dirname(current);
|
||||
}
|
||||
|
||||
return [...paths];
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE "library_folders" ADD COLUMN "scan_state_version" bigint DEFAULT 0 NOT NULL;--> statement-breakpoint
|
||||
ALTER TABLE "library_folders" ADD CONSTRAINT "library_folders_scan_state_version_nonnegative_chk" CHECK ("library_folders"."scan_state_version" >= 0);
|
||||
File diff suppressed because it is too large
Load Diff
@@ -589,6 +589,13 @@
|
||||
"when": 1788194498208,
|
||||
"tag": "0083_add_book_requests",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 84,
|
||||
"version": "7",
|
||||
"when": 1788206528114,
|
||||
"tag": "0084_add_scan_state_version",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { sql } from 'drizzle-orm';
|
||||
import {
|
||||
bigint,
|
||||
boolean,
|
||||
check,
|
||||
doublePrecision,
|
||||
@@ -110,12 +111,14 @@ export const libraryFolders = pgTable(
|
||||
.notNull()
|
||||
.references(() => libraries.id, { onDelete: 'cascade' }),
|
||||
path: varchar('path', { length: 4096 }).notNull(),
|
||||
scanStateVersion: bigint('scan_state_version', { mode: 'number' }).notNull().default(0),
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(),
|
||||
},
|
||||
(t) => [
|
||||
index('library_folders_library_id_idx').on(t.libraryId),
|
||||
uniqueIndex('library_folders_library_path_uidx').on(t.libraryId, t.path),
|
||||
unique('library_folders_id_library_id_unique').on(t.id, t.libraryId),
|
||||
check('library_folders_scan_state_version_nonnegative_chk', sql`${t.scanStateVersion} >= 0`),
|
||||
],
|
||||
);
|
||||
|
||||
|
||||
@@ -698,7 +698,7 @@ describe('BookDockFinalizeService', () => {
|
||||
vi.spyOn(service as never, 'applyMetadata').mockResolvedValue(undefined as never);
|
||||
vi.spyOn(service as never, 'cleanupBookDockRecord').mockResolvedValue(undefined as never);
|
||||
mockStat.mockResolvedValueOnce({ size: 100 } as never);
|
||||
processor.createBookRecord.mockResolvedValueOnce({ bookId: 557 });
|
||||
processor.createUnitBookRecords.mockResolvedValueOnce({ bookIds: [557], createdBookIds: [557], attachedFileIds: [] });
|
||||
const row = makeRow({ targetLibraryId: 5, targetFolderId: 9, selectedMetadata: defaultPatternMetadata() });
|
||||
const destPath = '/library/Frank Herbert/Dune/01. Dune (1965).epub';
|
||||
|
||||
@@ -710,7 +710,17 @@ describe('BookDockFinalizeService', () => {
|
||||
bookId: 557,
|
||||
});
|
||||
expect(storage.moveToPath).toHaveBeenCalledWith('/tmp/book.epub', destPath);
|
||||
expect(processor.createBookRecord).toHaveBeenCalledWith(5, 9, destPath, destPath, 'Frank Herbert/Dune/01. Dune (1965).epub', 'epub', 100);
|
||||
expect(processor.createUnitBookRecords).toHaveBeenCalledWith(5, 9, [
|
||||
{
|
||||
folderPath: destPath,
|
||||
absolutePath: destPath,
|
||||
relPath: 'Frank Herbert/Dune/01. Dune (1965).epub',
|
||||
format: 'epub',
|
||||
sizeBytes: 100,
|
||||
role: 'content',
|
||||
sortOrder: 0,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('uses the file path as bookFolderPath in book_per_file mode', async () => {
|
||||
|
||||
@@ -115,9 +115,10 @@ describe('RequestWatchdogService stranded searches', () => {
|
||||
|
||||
await service.sweep();
|
||||
|
||||
const after = Date.now();
|
||||
const [cutoff] = requests.findStrandedSearching.mock.calls[0] as [Date, number];
|
||||
expect(cutoff.getTime()).toBeLessThanOrEqual(before - 30 * 60 * 1000);
|
||||
expect(cutoff.getTime()).toBeGreaterThan(before - 31 * 60 * 1000);
|
||||
expect(cutoff.getTime()).toBeGreaterThanOrEqual(before - 30 * 60 * 1000);
|
||||
expect(cutoff.getTime()).toBeLessThanOrEqual(after - 30 * 60 * 1000);
|
||||
});
|
||||
|
||||
/** No in-memory task survives a restart, so every one of these is stranded, however recent. */
|
||||
|
||||
@@ -12,6 +12,7 @@ function makeSelectChain<T>(terminalMethod: string, terminalResult: T) {
|
||||
orderBy: vi.fn(),
|
||||
limit: vi.fn(),
|
||||
offset: vi.fn(),
|
||||
for: vi.fn(),
|
||||
};
|
||||
|
||||
chain.from.mockReturnValue(chain);
|
||||
@@ -19,7 +20,11 @@ function makeSelectChain<T>(terminalMethod: string, terminalResult: T) {
|
||||
chain.innerJoin.mockReturnValue(chain);
|
||||
chain.offset.mockReturnValue(chain);
|
||||
|
||||
if (terminalMethod === 'where') {
|
||||
if (terminalMethod === 'for') {
|
||||
chain.where.mockReturnValue(chain);
|
||||
chain.orderBy.mockReturnValue(chain);
|
||||
chain.for.mockResolvedValue(terminalResult);
|
||||
} else if (terminalMethod === 'where') {
|
||||
chain.where.mockResolvedValue(terminalResult);
|
||||
chain.orderBy.mockReturnValue(chain);
|
||||
chain.limit.mockReturnValue(chain);
|
||||
@@ -113,6 +118,73 @@ describe('BookRepository', () => {
|
||||
expect(selectChain.leftJoin).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('deletes books and invalidates their exact scan-state paths in one transaction', async () => {
|
||||
const bookRows = [{ id: 10, libraryFolderId: 7, folderPath: '/books/Series/Book' }];
|
||||
const bookSelect = makeSelectChain('for', bookRows);
|
||||
const folderSelect = makeSelectChain('for', [{ id: 7 }]);
|
||||
const updateWhere = vi.fn().mockResolvedValue(undefined);
|
||||
const updateSet = vi.fn().mockReturnValue({ where: updateWhere });
|
||||
const stateDeleteWhere = vi.fn().mockResolvedValue(undefined);
|
||||
const bookDeleteWhere = vi.fn().mockResolvedValue(undefined);
|
||||
const tx = {
|
||||
select: vi.fn().mockReturnValueOnce(bookSelect).mockReturnValueOnce(folderSelect),
|
||||
update: vi.fn().mockReturnValue({ set: updateSet }),
|
||||
delete: vi.fn().mockReturnValueOnce({ where: stateDeleteWhere }).mockReturnValueOnce({ where: bookDeleteWhere }),
|
||||
};
|
||||
const db = { transaction: vi.fn((callback: (executor: typeof tx) => Promise<void>) => callback(tx)) };
|
||||
const repo = new BookRepository(db as never);
|
||||
|
||||
await repo.deleteByIdsAndInvalidateScanState([10, 10]);
|
||||
|
||||
expect(db.transaction).toHaveBeenCalledTimes(1);
|
||||
expect(tx.update).toHaveBeenCalledTimes(1);
|
||||
expect(tx.delete).toHaveBeenCalledTimes(2);
|
||||
const invalidationQuery = new PgDialect().sqlToQuery(stateDeleteWhere.mock.calls[0]![0]);
|
||||
expect(invalidationQuery.sql).toContain('"library_dir_scan_state"."library_folder_id" = $1');
|
||||
expect(invalidationQuery.sql).toContain('"library_dir_scan_state"."dir_path" in');
|
||||
expect(invalidationQuery.params).toEqual([7, '/books/Series/Book', '/books/Series', '/books', '/']);
|
||||
expect(invalidationQuery.params).not.toContain('/books/Sibling');
|
||||
});
|
||||
|
||||
it('does not reach the book delete when scan-state invalidation fails', async () => {
|
||||
const bookSelect = makeSelectChain('for', [{ id: 10, libraryFolderId: 7, folderPath: '/books/Book' }]);
|
||||
const folderSelect = makeSelectChain('for', [{ id: 7 }]);
|
||||
const updateWhere = vi.fn().mockResolvedValue(undefined);
|
||||
const stateDeleteWhere = vi.fn().mockRejectedValue(new Error('invalidation failed'));
|
||||
const tx = {
|
||||
select: vi.fn().mockReturnValueOnce(bookSelect).mockReturnValueOnce(folderSelect),
|
||||
update: vi.fn().mockReturnValue({ set: vi.fn().mockReturnValue({ where: updateWhere }) }),
|
||||
delete: vi.fn().mockReturnValue({ where: stateDeleteWhere }),
|
||||
};
|
||||
const db = { transaction: vi.fn((callback: (executor: typeof tx) => Promise<void>) => callback(tx)) };
|
||||
const repo = new BookRepository(db as never);
|
||||
|
||||
await expect(repo.deleteByIdsAndInvalidateScanState([10])).rejects.toThrow('invalidation failed');
|
||||
|
||||
expect(tx.delete).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('chunks scan-state invalidation paths for large deletions', async () => {
|
||||
const bookRows = Array.from({ length: 501 }, (_, index) => ({
|
||||
id: index + 1,
|
||||
libraryFolderId: 7,
|
||||
folderPath: `/books/book-${index + 1}.epub`,
|
||||
}));
|
||||
const bookSelect = makeSelectChain('for', bookRows);
|
||||
const folderSelect = makeSelectChain('for', [{ id: 7 }]);
|
||||
const tx = {
|
||||
select: vi.fn().mockReturnValueOnce(bookSelect).mockReturnValueOnce(folderSelect),
|
||||
update: vi.fn().mockReturnValue({ set: vi.fn().mockReturnValue({ where: vi.fn().mockResolvedValue(undefined) }) }),
|
||||
delete: vi.fn().mockReturnValue({ where: vi.fn().mockResolvedValue(undefined) }),
|
||||
};
|
||||
const db = { transaction: vi.fn((callback: (executor: typeof tx) => Promise<void>) => callback(tx)) };
|
||||
const repo = new BookRepository(db as never);
|
||||
|
||||
await repo.deleteByIdsAndInvalidateScanState(bookRows.map((row) => row.id));
|
||||
|
||||
expect(tx.delete).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('findCards loads card rows and related collections for the current user', async () => {
|
||||
const rows = [{ id: 10, primaryFileId: 1001, _total: 1 }];
|
||||
const authorRows = [{ bookId: 10, name: 'Frank Herbert' }];
|
||||
|
||||
@@ -19,6 +19,7 @@ import { buildContentFilterClauses } from '../../common/utils/content-filter-sql
|
||||
import { accentInsensitiveIlike } from '../../common/utils/accent-insensitive-search.utils';
|
||||
import { advanceIsoTimestamp } from '../../common/utils/iso-timestamp.utils';
|
||||
import { parsePgTimestamptz } from '../../common/utils/pg-timestamp.utils';
|
||||
import { scanStateInvalidationPaths } from '../../common/utils/scan-state-paths.utils';
|
||||
import { seriesIndexSortKeySql } from '../../common/utils/series-index-sql.utils';
|
||||
import { SeriesIdentityService } from '../../common/services/series-identity.service';
|
||||
import { SeriesMembershipService } from '../../common/services/series-membership.service';
|
||||
@@ -49,6 +50,7 @@ import {
|
||||
koreaderDeviceProgress,
|
||||
koreaderProgressResets,
|
||||
libraries,
|
||||
libraryFolders,
|
||||
narrators,
|
||||
audiobookProgress,
|
||||
readingProgress,
|
||||
@@ -1893,6 +1895,66 @@ export class BookRepository {
|
||||
await this.db.delete(books).where(inArray(books.id, bookIds));
|
||||
}
|
||||
|
||||
async deleteByIdsAndInvalidateScanState(bookIds: number[]): Promise<void> {
|
||||
const uniqueBookIds = [...new Set(bookIds)].sort((a, b) => a - b);
|
||||
if (uniqueBookIds.length === 0) return;
|
||||
|
||||
await this.db.transaction(async (tx) => {
|
||||
const rows = await tx
|
||||
.select({ id: books.id, libraryFolderId: books.libraryFolderId, folderPath: books.folderPath })
|
||||
.from(books)
|
||||
.where(inArray(books.id, uniqueBookIds))
|
||||
.orderBy(asc(books.id))
|
||||
.for('update');
|
||||
if (rows.length === 0) return;
|
||||
|
||||
const libraryFolderIds = [...new Set(rows.map((row) => row.libraryFolderId))].sort((a, b) => a - b);
|
||||
await tx
|
||||
.select({ id: libraryFolders.id })
|
||||
.from(libraryFolders)
|
||||
.where(inArray(libraryFolders.id, libraryFolderIds))
|
||||
.orderBy(asc(libraryFolders.id))
|
||||
.for('update');
|
||||
|
||||
await tx
|
||||
.update(libraryFolders)
|
||||
.set({ scanStateVersion: sql`${libraryFolders.scanStateVersion} + 1` })
|
||||
.where(inArray(libraryFolders.id, libraryFolderIds));
|
||||
|
||||
const pathsByLibraryFolder = new Map<number, Set<string>>();
|
||||
for (const row of rows) {
|
||||
let paths = pathsByLibraryFolder.get(row.libraryFolderId);
|
||||
if (!paths) {
|
||||
paths = new Set<string>();
|
||||
pathsByLibraryFolder.set(row.libraryFolderId, paths);
|
||||
}
|
||||
for (const path of scanStateInvalidationPaths(row.folderPath)) paths.add(path);
|
||||
}
|
||||
|
||||
const chunkSize = 500;
|
||||
for (const [libraryFolderId, paths] of pathsByLibraryFolder) {
|
||||
const pathList = [...paths];
|
||||
for (let offset = 0; offset < pathList.length; offset += chunkSize) {
|
||||
await tx
|
||||
.delete(schema.libraryDirScanState)
|
||||
.where(
|
||||
and(
|
||||
eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId),
|
||||
inArray(schema.libraryDirScanState.dirPath, pathList.slice(offset, offset + chunkSize)),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await tx.delete(books).where(
|
||||
inArray(
|
||||
books.id,
|
||||
rows.map((row) => row.id),
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async bulkSetRating(bookIds: number[], rating: number | null, userId: number): Promise<void> {
|
||||
if (bookIds.length === 0) return;
|
||||
|
||||
|
||||
@@ -142,6 +142,7 @@ function makeService(overrides: { bookMetadataLockService?: unknown } = {}) {
|
||||
replaceCommunityRatings: vi.fn(),
|
||||
withTransaction: vi.fn(),
|
||||
deleteByIds: vi.fn(),
|
||||
deleteByIdsAndInvalidateScanState: vi.fn(),
|
||||
findAllIds: vi.fn(),
|
||||
findIdsByWhere: vi.fn(),
|
||||
findCardsCollapsed: vi.fn(),
|
||||
@@ -2240,13 +2241,13 @@ describe('BookService', () => {
|
||||
{ id: 3, title: 'Dune' },
|
||||
{ id: 4, title: null },
|
||||
]);
|
||||
bookRepo.deleteByIds.mockResolvedValue(undefined);
|
||||
bookRepo.deleteByIdsAndInvalidateScanState.mockResolvedValue(undefined);
|
||||
mockRm.mockRejectedValue(new Error('cannot delete'));
|
||||
|
||||
const result = await service.deleteBooks([3, 4], user);
|
||||
|
||||
expect(libraryService.verifyUserAccess).toHaveBeenCalledTimes(2);
|
||||
expect(bookRepo.deleteByIds).toHaveBeenCalledWith([3, 4]);
|
||||
expect(bookRepo.deleteByIdsAndInvalidateScanState).toHaveBeenCalledWith([3, 4]);
|
||||
expect(mockRm).toHaveBeenCalledWith('/tmp/books/covers/3', { recursive: true, force: true });
|
||||
expect(mockRm).toHaveBeenCalledWith('/tmp/books/covers/4', { recursive: true, force: true });
|
||||
expect(mockRm).toHaveBeenCalledWith('/tmp/library/book3.epub', { force: true });
|
||||
@@ -2272,7 +2273,7 @@ describe('BookService', () => {
|
||||
Promise.resolve([...ids].reverse().map((id) => ({ id, title: `Book ${id}` }))),
|
||||
);
|
||||
bookRepo.findAllFilesByBookIds.mockResolvedValue([]);
|
||||
bookRepo.deleteByIds.mockResolvedValue(undefined);
|
||||
bookRepo.deleteByIdsAndInvalidateScanState.mockResolvedValue(undefined);
|
||||
|
||||
const result = await service.deleteBooks(bookIds, user);
|
||||
|
||||
@@ -2284,6 +2285,35 @@ describe('BookService', () => {
|
||||
expect(result.omitted).toBe(5);
|
||||
});
|
||||
|
||||
it('deletes books through the scan-state-aware repository transaction', async () => {
|
||||
const { service, bookRepo } = makeService();
|
||||
const user = makeUser();
|
||||
|
||||
bookRepo.findLibraryIdsByBookIds.mockResolvedValue([{ id: 3, libraryId: 7 }]);
|
||||
bookRepo.findDeletionAuditBooksByIds.mockResolvedValue([{ id: 3, title: 'Dune' }]);
|
||||
bookRepo.findAllFilesByBookIds.mockResolvedValue([]);
|
||||
bookRepo.deleteByIdsAndInvalidateScanState.mockResolvedValue(undefined);
|
||||
|
||||
await service.deleteBooks([3], user);
|
||||
|
||||
expect(bookRepo.deleteByIdsAndInvalidateScanState).toHaveBeenCalledWith([3]);
|
||||
});
|
||||
|
||||
it('does not commit book deletion when scan state invalidation fails', async () => {
|
||||
const { service, bookRepo } = makeService();
|
||||
const user = makeUser();
|
||||
const warnSpy = vi.spyOn((service as unknown as { logger: { warn: (message: string) => void } }).logger, 'warn').mockImplementation();
|
||||
|
||||
bookRepo.findLibraryIdsByBookIds.mockResolvedValue([{ id: 3, libraryId: 7 }]);
|
||||
bookRepo.findDeletionAuditBooksByIds.mockResolvedValue([{ id: 3, title: 'Dune' }]);
|
||||
bookRepo.findAllFilesByBookIds.mockResolvedValue([]);
|
||||
bookRepo.deleteByIdsAndInvalidateScanState.mockRejectedValue(new Error('db down'));
|
||||
|
||||
await expect(service.deleteBooks([3], user)).rejects.toThrow('db down');
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
expect(mockRm).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns queued=0 when embed-all is already running', async () => {
|
||||
const { service, bookRepo, embedder } = makeService();
|
||||
let resolveEmbed: (() => void) | null = null;
|
||||
|
||||
@@ -1548,7 +1548,7 @@ export class BookService {
|
||||
return row ? [row] : [];
|
||||
});
|
||||
const files = await this.bookRepo.findAllFilesByBookIds(bookIds);
|
||||
await this.bookRepo.deleteByIds(bookIds);
|
||||
await this.bookRepo.deleteByIdsAndInvalidateScanState(deletedBookIds);
|
||||
const deleteTargets = [
|
||||
...rows.map((row) => ({
|
||||
path: join(this.appDataPath, 'covers', String(row.id)),
|
||||
|
||||
@@ -330,51 +330,81 @@ describe('ScannerRepository', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('handles directory scan state lookups and upserts', async () => {
|
||||
const { repo, queues, db } = makeRepo();
|
||||
it('loads a versioned directory scan-state snapshot', async () => {
|
||||
const { repo, queues } = makeRepo();
|
||||
queues.select.push([
|
||||
{ dirPath: '/books/a', lastSeenMtimeMs: 100 },
|
||||
{ dirPath: '/books/b', lastSeenMtimeMs: 200 },
|
||||
{ version: 7, dirPath: '/books/a', lastSeenMtimeMs: 100 },
|
||||
{ version: 7, dirPath: '/books/b', lastSeenMtimeMs: 200 },
|
||||
]);
|
||||
|
||||
const snapshot = await repo.findDirScanStateSnapshot(7);
|
||||
|
||||
expect(snapshot?.version).toBe(7);
|
||||
expect(snapshot?.mtimes).toEqual(
|
||||
new Map([
|
||||
['/books/a', 100],
|
||||
['/books/b', 200],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns an empty snapshot for a folder without state and null for a missing folder', async () => {
|
||||
const { repo, queues } = makeRepo();
|
||||
queues.select.push([{ version: 3, dirPath: null, lastSeenMtimeMs: null }]);
|
||||
queues.select.push([]);
|
||||
|
||||
await expect(repo.findDirScanStateSnapshot(7)).resolves.toEqual({ version: 3, mtimes: new Map() });
|
||||
await expect(repo.findDirScanStateSnapshot(8)).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('persists directory scan state atomically when the version still matches', async () => {
|
||||
const { repo, queues, db } = makeRepo();
|
||||
queues.select.push([{ version: 7 }]);
|
||||
queues.insert.push([]);
|
||||
queues.insert.push([]);
|
||||
|
||||
const map = await repo.findDirScanState(7);
|
||||
expect(map.get('/books/a')).toBe(100);
|
||||
expect(map.get('/books/b')).toBe(200);
|
||||
|
||||
await repo.upsertDirScanState(7, []);
|
||||
expect(db.insert).toHaveBeenCalledTimes(0);
|
||||
queues.select.push([
|
||||
{ id: 1, dirPath: '/books/0' },
|
||||
{ id: 2, dirPath: '/stale' },
|
||||
]);
|
||||
|
||||
const entries = Array.from({ length: 501 }, (_, i) => ({ dirPath: `/books/${i}`, mtimeMs: i + 0.4 }));
|
||||
await repo.upsertDirScanState(7, entries);
|
||||
await expect(repo.persistDirScanState(7, 7, entries)).resolves.toBe(true);
|
||||
|
||||
expect(db.transaction).toHaveBeenCalledTimes(1);
|
||||
expect(db.insert).toHaveBeenCalledTimes(2);
|
||||
expect(db.delete).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('deletes stale directory scan state across all branch paths', async () => {
|
||||
it('skips directory scan-state persistence after the version changes', async () => {
|
||||
const { repo, queues, db } = makeRepo();
|
||||
queues.select.push([{ version: 8 }]);
|
||||
|
||||
await repo.deleteStaleDirScanState(7, new Set());
|
||||
expect(db.delete).toHaveBeenCalledTimes(1);
|
||||
await expect(repo.persistDirScanState(7, 7, [{ dirPath: '/books', mtimeMs: 100 }])).resolves.toBe(false);
|
||||
|
||||
queues.select.push([
|
||||
{ id: 1, dirPath: '/keep' },
|
||||
{ id: 2, dirPath: '/drop' },
|
||||
]);
|
||||
await repo.deleteStaleDirScanState(7, new Set(['/keep']));
|
||||
expect(db.delete).toHaveBeenCalledTimes(2);
|
||||
|
||||
queues.select.push([{ id: 3, dirPath: '/keep-only' }]);
|
||||
await repo.deleteStaleDirScanState(7, new Set(['/keep-only']));
|
||||
expect(db.delete).toHaveBeenCalledTimes(2);
|
||||
expect(db.insert).not.toHaveBeenCalled();
|
||||
expect(db.delete).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('clears directory scan state for a library folder', async () => {
|
||||
const { repo, db } = makeRepo();
|
||||
await repo.clearDirScanState(21);
|
||||
it('clears directory scan state under the folder lock and returns its version', async () => {
|
||||
const { repo, queues, db } = makeRepo();
|
||||
queues.update.push([{ version: 9 }]);
|
||||
|
||||
await expect(repo.clearDirScanState(21)).resolves.toBe(9);
|
||||
|
||||
expect(db.transaction).toHaveBeenCalledTimes(1);
|
||||
expect(db.update).toHaveBeenCalledTimes(1);
|
||||
expect(db.delete).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not clear directory scan state when the library folder no longer exists', async () => {
|
||||
const { repo, queues, db } = makeRepo();
|
||||
queues.update.push([]);
|
||||
|
||||
await expect(repo.clearDirScanState(21)).resolves.toBeNull();
|
||||
|
||||
expect(db.delete).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('findRecentScanJobs reads the newest jobs for one library', async () => {
|
||||
const { repo, queues, db } = makeRepo();
|
||||
queues.select.push([{ id: 2 }, { id: 1 }]);
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
} from '../../db/schema';
|
||||
|
||||
type Db = NodePgDatabase<typeof schema>;
|
||||
type DbTransaction = Parameters<Parameters<Db['transaction']>[0]>[0];
|
||||
type MoveBookToLibraryResult = Pick<typeof books.$inferSelect, 'id' | 'libraryId' | 'libraryFolderId' | 'folderPath' | 'status'> & {
|
||||
previousLibraryId: number;
|
||||
libraryChanged: boolean;
|
||||
@@ -619,20 +620,51 @@ export class ScannerRepository {
|
||||
|
||||
// ── Dir Scan State (Incremental Scan) ────────────────────────────────────
|
||||
|
||||
async findDirScanState(libraryFolderId: number): Promise<Map<string, number>> {
|
||||
async findDirScanStateSnapshot(libraryFolderId: number): Promise<{ version: number; mtimes: Map<string, number> } | null> {
|
||||
const rows = await this.db
|
||||
.select({ dirPath: schema.libraryDirScanState.dirPath, lastSeenMtimeMs: schema.libraryDirScanState.lastSeenMtimeMs })
|
||||
.from(schema.libraryDirScanState)
|
||||
.where(eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId));
|
||||
return new Map(rows.map((r) => [r.dirPath, r.lastSeenMtimeMs]));
|
||||
.select({
|
||||
version: libraryFolders.scanStateVersion,
|
||||
dirPath: schema.libraryDirScanState.dirPath,
|
||||
lastSeenMtimeMs: schema.libraryDirScanState.lastSeenMtimeMs,
|
||||
})
|
||||
.from(libraryFolders)
|
||||
.leftJoin(schema.libraryDirScanState, eq(schema.libraryDirScanState.libraryFolderId, libraryFolders.id))
|
||||
.where(eq(libraryFolders.id, libraryFolderId));
|
||||
const first = rows[0];
|
||||
if (!first) return null;
|
||||
|
||||
const mtimes = new Map<string, number>();
|
||||
for (const row of rows) {
|
||||
if (row.dirPath !== null && row.lastSeenMtimeMs !== null) mtimes.set(row.dirPath, row.lastSeenMtimeMs);
|
||||
}
|
||||
return { version: first.version, mtimes };
|
||||
}
|
||||
|
||||
async upsertDirScanState(libraryFolderId: number, entries: Array<{ dirPath: string; mtimeMs: number }>): Promise<void> {
|
||||
async persistDirScanState(
|
||||
libraryFolderId: number,
|
||||
expectedVersion: number,
|
||||
entries: Array<{ dirPath: string; mtimeMs: number }>,
|
||||
): Promise<boolean> {
|
||||
return this.db.transaction(async (tx) => {
|
||||
const [folder] = await tx
|
||||
.select({ version: libraryFolders.scanStateVersion })
|
||||
.from(libraryFolders)
|
||||
.where(eq(libraryFolders.id, libraryFolderId))
|
||||
.for('update');
|
||||
if (!folder || folder.version !== expectedVersion) return false;
|
||||
|
||||
await this.upsertDirScanState(tx, libraryFolderId, entries);
|
||||
await this.deleteStaleDirScanState(tx, libraryFolderId, new Set(entries.map((entry) => entry.dirPath)));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
private async upsertDirScanState(tx: DbTransaction, libraryFolderId: number, entries: Array<{ dirPath: string; mtimeMs: number }>): Promise<void> {
|
||||
if (entries.length === 0) return;
|
||||
const CHUNK = 500;
|
||||
for (let i = 0; i < entries.length; i += CHUNK) {
|
||||
const chunk = entries.slice(i, i + CHUNK);
|
||||
await this.db
|
||||
await tx
|
||||
.insert(schema.libraryDirScanState)
|
||||
.values(chunk.map((e) => ({ libraryFolderId, dirPath: e.dirPath, lastSeenMtimeMs: Math.round(e.mtimeMs) })))
|
||||
.onConflictDoUpdate({
|
||||
@@ -642,12 +674,12 @@ export class ScannerRepository {
|
||||
}
|
||||
}
|
||||
|
||||
async deleteStaleDirScanState(libraryFolderId: number, validPaths: Set<string>): Promise<void> {
|
||||
private async deleteStaleDirScanState(tx: DbTransaction, libraryFolderId: number, validPaths: Set<string>): Promise<void> {
|
||||
if (validPaths.size === 0) {
|
||||
await this.db.delete(schema.libraryDirScanState).where(eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId));
|
||||
await tx.delete(schema.libraryDirScanState).where(eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId));
|
||||
return;
|
||||
}
|
||||
const allRows = await this.db
|
||||
const allRows = await tx
|
||||
.select({ id: schema.libraryDirScanState.id, dirPath: schema.libraryDirScanState.dirPath })
|
||||
.from(schema.libraryDirScanState)
|
||||
.where(eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId));
|
||||
@@ -655,11 +687,21 @@ export class ScannerRepository {
|
||||
if (staleIds.length === 0) return;
|
||||
const CHUNK = 500;
|
||||
for (let i = 0; i < staleIds.length; i += CHUNK) {
|
||||
await this.db.delete(schema.libraryDirScanState).where(inArray(schema.libraryDirScanState.id, staleIds.slice(i, i + CHUNK)));
|
||||
await tx.delete(schema.libraryDirScanState).where(inArray(schema.libraryDirScanState.id, staleIds.slice(i, i + CHUNK)));
|
||||
}
|
||||
}
|
||||
|
||||
async clearDirScanState(libraryFolderId: number): Promise<void> {
|
||||
await this.db.delete(schema.libraryDirScanState).where(eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId));
|
||||
async clearDirScanState(libraryFolderId: number): Promise<number | null> {
|
||||
return this.db.transaction(async (tx) => {
|
||||
const [folder] = await tx
|
||||
.update(libraryFolders)
|
||||
.set({ scanStateVersion: sql`${libraryFolders.scanStateVersion} + 1` })
|
||||
.where(eq(libraryFolders.id, libraryFolderId))
|
||||
.returning({ version: libraryFolders.scanStateVersion });
|
||||
if (!folder) return null;
|
||||
|
||||
await tx.delete(schema.libraryDirScanState).where(eq(schema.libraryDirScanState.libraryFolderId, libraryFolderId));
|
||||
return folder.version;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,10 +123,9 @@ function makeRepo(overrides: Record<string, unknown> = {}) {
|
||||
findBookCardData: vi.fn().mockResolvedValue({ rows: [], authorRows: [], fileRows: [], genreRows: [] }),
|
||||
deleteBookFile: vi.fn().mockResolvedValue(undefined),
|
||||
updateBookFolderPath: vi.fn().mockResolvedValue(undefined),
|
||||
findDirScanState: vi.fn().mockResolvedValue(new Map()),
|
||||
upsertDirScanState: vi.fn().mockResolvedValue(undefined),
|
||||
deleteStaleDirScanState: vi.fn().mockResolvedValue(undefined),
|
||||
clearDirScanState: vi.fn().mockResolvedValue(undefined),
|
||||
findDirScanStateSnapshot: vi.fn().mockResolvedValue({ version: 0, mtimes: new Map() }),
|
||||
persistDirScanState: vi.fn().mockResolvedValue(true),
|
||||
clearDirScanState: vi.fn().mockResolvedValue(1),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
@@ -3501,7 +3500,7 @@ describe('incremental scan — dir state', () => {
|
||||
['/library/Author/Book', 2000],
|
||||
]);
|
||||
const repo = makeRepo({
|
||||
findDirScanState: vi.fn().mockResolvedValue(storedMtimes),
|
||||
findDirScanStateSnapshot: vi.fn().mockResolvedValue({ version: 4, mtimes: storedMtimes }),
|
||||
});
|
||||
const { service } = makeService(repo);
|
||||
|
||||
@@ -3517,7 +3516,7 @@ describe('incremental scan — dir state', () => {
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.findDirScanState).toHaveBeenCalledWith(1);
|
||||
expect(repo.findDirScanStateSnapshot).toHaveBeenCalledWith(1);
|
||||
expect(mockFindCandidates).toHaveBeenCalledWith('/library', [], expect.any(Function), storedMtimes);
|
||||
});
|
||||
|
||||
@@ -3538,14 +3537,71 @@ describe('incremental scan — dir state', () => {
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
|
||||
expect(repo.upsertDirScanState).toHaveBeenCalledWith(
|
||||
expect(repo.persistDirScanState).toHaveBeenCalledWith(
|
||||
1,
|
||||
1,
|
||||
expect.arrayContaining([
|
||||
{ dirPath: '/library/Author', mtimeMs: 1000 },
|
||||
{ dirPath: '/library/Author/Book', mtimeMs: 2000 },
|
||||
]),
|
||||
);
|
||||
expect(repo.deleteStaleDirScanState).toHaveBeenCalledWith(1, new Set(['/library/Author', '/library/Author/Book']));
|
||||
});
|
||||
|
||||
it('does not restore directory state invalidated while a scan is in flight', async () => {
|
||||
const state = new Map<string, number>([['/library/Author/Book', 2000]]);
|
||||
let version = 0;
|
||||
let releaseWalk: (() => void) | undefined;
|
||||
let markWalkStarted: (() => void) | undefined;
|
||||
const walkStarted = new Promise<void>((resolve) => {
|
||||
markWalkStarted = resolve;
|
||||
});
|
||||
const walkReleased = new Promise<void>((resolve) => {
|
||||
releaseWalk = resolve;
|
||||
});
|
||||
const repo = makeRepo({
|
||||
findDirScanStateSnapshot: vi.fn().mockImplementation(() => Promise.resolve({ version, mtimes: new Map(state) })),
|
||||
clearDirScanState: vi.fn().mockImplementation(() => {
|
||||
state.clear();
|
||||
version += 1;
|
||||
return Promise.resolve(version);
|
||||
}),
|
||||
persistDirScanState: vi
|
||||
.fn()
|
||||
.mockImplementation((_folderId: number, expectedVersion: number, entries: Array<{ dirPath: string; mtimeMs: number }>) => {
|
||||
if (expectedVersion !== version) return Promise.resolve(false);
|
||||
state.clear();
|
||||
for (const entry of entries) state.set(entry.dirPath, entry.mtimeMs);
|
||||
return Promise.resolve(true);
|
||||
}),
|
||||
});
|
||||
const { service, jobStore } = makeService(repo);
|
||||
|
||||
let done = awaitScan(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await done;
|
||||
await vi.waitFor(() => expect(jobStore.isRunning(1)).toBe(false));
|
||||
|
||||
mockFindCandidates.mockImplementationOnce(async () => {
|
||||
markWalkStarted?.();
|
||||
await walkReleased;
|
||||
return {
|
||||
candidates: [],
|
||||
skippedDirs: new Set(),
|
||||
unchangedDirs: new Set(['/library/Author/Book']),
|
||||
dirMtimes: new Map([['/library/Author/Book', 2000]]),
|
||||
};
|
||||
});
|
||||
repo.createScanJob.mockResolvedValueOnce({ id: 101 });
|
||||
done = awaitScan(repo);
|
||||
await service.startScan(1, 'manual');
|
||||
await walkStarted;
|
||||
|
||||
state.clear();
|
||||
version += 1;
|
||||
releaseWalk?.();
|
||||
await done;
|
||||
|
||||
expect(state.size).toBe(0);
|
||||
});
|
||||
|
||||
it('clears dir state when forceFullScan is true', async () => {
|
||||
@@ -3556,7 +3612,7 @@ describe('incremental scan — dir state', () => {
|
||||
await done;
|
||||
|
||||
expect(repo.clearDirScanState).toHaveBeenCalledWith(1);
|
||||
expect(repo.findDirScanState).not.toHaveBeenCalled();
|
||||
expect(repo.findDirScanStateSnapshot).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('excludes books in unchanged dirs from missing detection', async () => {
|
||||
|
||||
@@ -1236,14 +1236,19 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
|
||||
// Load stored dir mtimes for incremental scanning (unless forced full)
|
||||
let knownDirMtimes: Map<string, number> | undefined;
|
||||
let scanStateVersion: number | undefined;
|
||||
if (!forceFullScan) {
|
||||
try {
|
||||
knownDirMtimes = await this.scannerRepo.findDirScanState(folder.id);
|
||||
const snapshot = await this.scannerRepo.findDirScanStateSnapshot(folder.id);
|
||||
if (snapshot) {
|
||||
knownDirMtimes = snapshot.mtimes;
|
||||
scanStateVersion = snapshot.version;
|
||||
}
|
||||
} catch {
|
||||
// If loading fails, fall back to full scan for this folder
|
||||
}
|
||||
} else {
|
||||
await this.scannerRepo.clearDirScanState(folder.id).catch(() => {});
|
||||
scanStateVersion = (await this.scannerRepo.clearDirScanState(folder.id).catch(() => null)) ?? undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -1296,11 +1301,10 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
totals.missingCount += counts.missingCount;
|
||||
|
||||
// Persist dir scan state after successful folder processing
|
||||
if (dirMtimes.size > 0) {
|
||||
if (dirMtimes.size > 0 && scanStateVersion !== undefined) {
|
||||
try {
|
||||
const entries = [...dirMtimes].map(([dirPath, mtimeMs]) => ({ dirPath, mtimeMs }));
|
||||
await this.scannerRepo.upsertDirScanState(folder.id, entries);
|
||||
await this.scannerRepo.deleteStaleDirScanState(folder.id, new Set(dirMtimes.keys()));
|
||||
await this.scannerRepo.persistDirScanState(folder.id, scanStateVersion, entries);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`[${event}] [fail] libraryId=${libraryId} jobId=${jobId} libraryFolderId=${folder.id} errorClass=${err instanceof Error ? err.name : 'Error'} error="${sanitizeLogValue(err instanceof Error ? err.message : String(err))}" - dir scan state persistence failed`,
|
||||
|
||||
Reference in New Issue
Block a user