mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
feat(library): rebuild the folder picker as a tree with a live plan
Pick folders from a lazily loaded tree with find-by-name and typed paths, beside a plan showing the books each organization mode would import, overlaps with other libraries, and folders no library reads yet. New ManageLibraries /path routes walk folders within entry and time bounds. Fix the scanner filing a disc-named library folder such as /mnt/disk1 under the folder above it, and move books already filed there onto their primary file on the next scan.
This commit is contained in:
neon
@@ -0,0 +1,8 @@
|
||||
import { IsOptional, IsString, MaxLength } from 'class-validator';
|
||||
|
||||
export class FolderPathQueryDto {
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
@MaxLength(4096)
|
||||
path?: string;
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { ArrayMaxSize, ArrayMinSize, IsArray, IsString, MaxLength } from 'class-validator';
|
||||
|
||||
export class FolderSummariesDto {
|
||||
@IsArray()
|
||||
@ArrayMinSize(1)
|
||||
@ArrayMaxSize(100)
|
||||
@IsString({ each: true })
|
||||
@MaxLength(4096, { each: true })
|
||||
paths: string[];
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { IsIn, IsNotEmpty, IsOptional, IsString, MaxLength } from 'class-validator';
|
||||
|
||||
export class SearchFoldersQueryDto {
|
||||
@IsString()
|
||||
@IsNotEmpty()
|
||||
@MaxLength(200)
|
||||
q: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
@MaxLength(4096)
|
||||
path?: string;
|
||||
|
||||
/** "true" also searches the operating-system folders under `/`. */
|
||||
@IsOptional()
|
||||
@IsIn(['true', 'false'])
|
||||
system?: string;
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
import { mkdir, mkdtemp, rm, symlink, writeFile } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
|
||||
import { FolderInsightService } from './folder-insight.service';
|
||||
|
||||
async function files(dir: string, names: string[]) {
|
||||
await mkdir(dir, { recursive: true });
|
||||
for (const name of names) await writeFile(join(dir, name), 'xxxx');
|
||||
}
|
||||
|
||||
describe('FolderInsightService', () => {
|
||||
let root: string;
|
||||
let service: FolderInsightService;
|
||||
const pathPolicy = { resolveBrowsePath: vi.fn(), getAppDataPath: vi.fn(), getCanonicalAppDataPath: vi.fn() };
|
||||
const pathService = { getPlaces: vi.fn() };
|
||||
const pathRepository = { findLibraryFolders: vi.fn() };
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), 'folder-insight-'));
|
||||
// Novels reads Fiction; Science Fiction, Cookbooks and Unsorted belong to no library.
|
||||
await files(join(root, 'books', 'Ebooks', 'Fiction', 'Jane Austen', 'Emma (1815)'), ['Emma.epub']);
|
||||
await files(join(root, 'books', 'Ebooks', 'Science Fiction', 'Ann Leckie', 'Ancillary Justice (2013)'), ['Ancillary Justice.epub']);
|
||||
await files(join(root, 'books', 'Ebooks', 'Science Fiction', 'Ann Leckie', 'Provenance (2017)'), ['Provenance.epub']);
|
||||
await files(join(root, 'books', 'Ebooks', 'Science Fiction', 'Martha Wells', 'All Systems Red (2017)'), ['All Systems Red.epub']);
|
||||
await files(join(root, 'books', 'Ebooks', 'Cookbooks'), ['Zahav.pdf', 'Jerusalem.pdf']);
|
||||
await files(join(root, 'books', 'Unsorted'), ['a.epub', 'b.epub', 'c.mobi']);
|
||||
await files(join(root, 'data', 'covers'), ['cover.epub']);
|
||||
|
||||
pathPolicy.resolveBrowsePath.mockImplementation((path?: string) => Promise.resolve(path || root));
|
||||
pathPolicy.getAppDataPath.mockReturnValue(join(root, 'data'));
|
||||
pathPolicy.getCanonicalAppDataPath.mockResolvedValue(join(root, 'data'));
|
||||
pathService.getPlaces.mockResolvedValue({
|
||||
root,
|
||||
mounts: [join(root, 'books'), join(root, 'data')],
|
||||
appDataPath: join(root, 'data'),
|
||||
libraries: [],
|
||||
});
|
||||
pathRepository.findLibraryFolders.mockResolvedValue([
|
||||
{ libraryId: 1, name: 'Novels', icon: null, type: 'books', path: join(root, 'books', 'Ebooks', 'Fiction') },
|
||||
]);
|
||||
service = new FolderInsightService(pathPolicy as never, pathService as never, pathRepository as never);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('summarize', () => {
|
||||
it('answers for each path in the order asked, keeping the caller’s spelling', async () => {
|
||||
const scifi = join(root, 'books', 'Ebooks', 'Science Fiction');
|
||||
const cookbooks = join(root, 'books', 'Ebooks', 'Cookbooks');
|
||||
|
||||
const [first, second] = await service.summarize([scifi, cookbooks]);
|
||||
|
||||
expect(first).toMatchObject({ path: scifi, state: 'ok', files: 3, folderBooks: 3, folders: 2, mergedFolders: 0, truncated: false });
|
||||
expect(second).toMatchObject({ path: cookbooks, files: 2, folderBooks: 2, formats: { pdf: 2 } });
|
||||
});
|
||||
|
||||
it('sees the merge a parent folder would cause', async () => {
|
||||
const [ebooks] = await service.summarize([join(root, 'books', 'Ebooks')]);
|
||||
|
||||
expect(ebooks).toMatchObject({ files: 6, folderBooks: 5, mergedFolders: 1 });
|
||||
});
|
||||
|
||||
it('marks blocked and out-of-root paths without walking them', async () => {
|
||||
pathPolicy.resolveBrowsePath.mockImplementation((path: string) =>
|
||||
path === '/outside' ? Promise.reject(new Error('outside')) : Promise.resolve(path),
|
||||
);
|
||||
|
||||
const [proc, outside] = await service.summarize(['/proc/1', '/outside']);
|
||||
|
||||
expect(proc).toMatchObject({ path: '/proc/1', state: 'blocked', files: 0 });
|
||||
expect(outside).toMatchObject({ path: '/outside', state: 'blocked', files: 0 });
|
||||
});
|
||||
|
||||
it('reports folders the batch had no time left for as pending, never as empty, and does not cache them', async () => {
|
||||
const unsorted = join(root, 'books', 'Unsorted');
|
||||
const cookbooks = join(root, 'books', 'Ebooks', 'Cookbooks');
|
||||
await service.summarize([cookbooks]);
|
||||
|
||||
const [late, cached] = await service.summarize([unsorted, cookbooks], -1);
|
||||
expect(late).toMatchObject({ path: unsorted, state: 'pending', files: 0 });
|
||||
expect(cached).toMatchObject({ path: cookbooks, state: 'ok', files: 2 });
|
||||
|
||||
await expect(service.summarize([unsorted])).resolves.toMatchObject([{ state: 'ok', files: 3 }]);
|
||||
});
|
||||
|
||||
it('refuses a symlink into a blocked system path without walking it', async () => {
|
||||
await symlink('/dev', join(root, 'dev-link'));
|
||||
|
||||
await expect(service.summarize([join(root, 'dev-link')])).resolves.toMatchObject([{ path: join(root, 'dev-link'), state: 'blocked' }]);
|
||||
});
|
||||
|
||||
it('does not keep a failed answer: a folder that appears later is counted', async () => {
|
||||
const later = join(root, 'books', 'Later');
|
||||
await expect(service.summarize([later])).resolves.toMatchObject([{ state: 'missing' }]);
|
||||
|
||||
await files(later, ['Later.epub']);
|
||||
|
||||
await expect(service.summarize([later])).resolves.toMatchObject([{ state: 'ok', files: 1 }]);
|
||||
});
|
||||
|
||||
it('answers a repeat from the cache, and walks again once forgotten', async () => {
|
||||
const unsorted = join(root, 'books', 'Unsorted');
|
||||
await service.summarize([unsorted]);
|
||||
await writeFile(join(unsorted, 'd.epub'), 'x');
|
||||
|
||||
await expect(service.summarize([unsorted])).resolves.toMatchObject([{ files: 3 }]);
|
||||
service.forget();
|
||||
await expect(service.summarize([unsorted])).resolves.toMatchObject([{ files: 4 }]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('inspect', () => {
|
||||
it('adds the size and the first two book files', async () => {
|
||||
await expect(service.inspect(join(root, 'books', 'Ebooks', 'Science Fiction'))).resolves.toMatchObject({
|
||||
files: 3,
|
||||
bytes: 12,
|
||||
samples: ['Ann Leckie/Ancillary Justice (2013)/Ancillary Justice.epub', 'Ann Leckie/Provenance (2017)/Provenance.epub'],
|
||||
});
|
||||
});
|
||||
|
||||
it('does not walk a blocked path', async () => {
|
||||
pathPolicy.resolveBrowsePath.mockResolvedValue('/proc');
|
||||
|
||||
await expect(service.inspect('/proc')).resolves.toMatchObject({ state: 'blocked', bytes: 0, samples: [] });
|
||||
});
|
||||
});
|
||||
|
||||
describe('unclaimed', () => {
|
||||
it('lists library-shaped folders no library reads, biggest first, skipping the data folder', async () => {
|
||||
const result = await service.unclaimed();
|
||||
|
||||
expect(result.truncated).toBe(false);
|
||||
expect(result.folders.map((folder) => [folder.path.slice(root.length), folder.files])).toEqual([
|
||||
['/books/Ebooks/Science Fiction', 3],
|
||||
['/books/Unsorted', 3],
|
||||
['/books/Ebooks/Cookbooks', 2],
|
||||
]);
|
||||
});
|
||||
|
||||
it('looks around the libraries’ folders when nothing is mounted separately', async () => {
|
||||
pathService.getPlaces.mockResolvedValue({ root, mounts: [], appDataPath: join(root, 'data'), libraries: [] });
|
||||
|
||||
const result = await service.unclaimed();
|
||||
|
||||
expect(result.folders.map((folder) => folder.path.slice(root.length))).toEqual(['/books/Ebooks/Science Fiction', '/books/Ebooks/Cookbooks']);
|
||||
});
|
||||
|
||||
it('shares one scan between pickers asking at the same time', async () => {
|
||||
const [first, second] = await Promise.all([service.unclaimed(), service.unclaimed()]);
|
||||
|
||||
expect(first).toBe(second);
|
||||
expect(pathService.getPlaces).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('scans again once a library reads a different set of folders', async () => {
|
||||
await service.unclaimed();
|
||||
pathRepository.findLibraryFolders.mockResolvedValue([
|
||||
{ libraryId: 1, name: 'Novels', icon: null, type: 'books', path: join(root, 'books', 'Ebooks', 'Fiction') },
|
||||
{ libraryId: 2, name: 'Inbox', icon: null, type: 'books', path: join(root, 'books', 'Unsorted') },
|
||||
]);
|
||||
|
||||
const result = await service.unclaimed();
|
||||
|
||||
expect(pathService.getPlaces).toHaveBeenCalledTimes(2);
|
||||
expect(result.folders.map((folder) => folder.path.slice(root.length))).not.toContain('/books/Unsorted');
|
||||
});
|
||||
|
||||
it('does not keep a scan that was running when the cache was cleared', async () => {
|
||||
let release!: () => void;
|
||||
const places = await pathService.getPlaces();
|
||||
pathService.getPlaces.mockImplementationOnce(() => new Promise((resolve) => (release = () => resolve(places))));
|
||||
const running = service.unclaimed();
|
||||
await vi.waitFor(() => expect(release).toBeTypeOf('function'));
|
||||
|
||||
service.forget();
|
||||
release();
|
||||
await running;
|
||||
await service.unclaimed();
|
||||
|
||||
expect(pathService.getPlaces).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it('caches the scan until forgotten', async () => {
|
||||
await service.unclaimed();
|
||||
await service.unclaimed();
|
||||
expect(pathService.getPlaces).toHaveBeenCalledTimes(1);
|
||||
|
||||
service.forget();
|
||||
await service.unclaimed();
|
||||
expect(pathService.getPlaces).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,309 @@
|
||||
import type { FolderInspection, FolderSummary, UnclaimedFolder, UnclaimedFolders } from '@bookorbit/types';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import type { Dirent } from 'fs';
|
||||
import { readdir, realpath } from 'fs/promises';
|
||||
import { dirname, join, resolve, sep } from 'path';
|
||||
|
||||
import { sanitizeLogValue } from '../../common/utils/log-sanitize.utils';
|
||||
import { isPrimaryFormat } from '../scanner/lib/classify';
|
||||
import { isBlockedPath, isInsideOrAt, isListableDirectory, mapLimit, realTarget, SYSTEM_FOLDERS, walkFolder, type WalkOutcome } from './folder-walk';
|
||||
import { PathPolicyService } from './path-policy.service';
|
||||
import { PathRepository } from './path.repository';
|
||||
import { PathService } from './path.service';
|
||||
|
||||
const CACHE_TTL_MS = 10 * 60_000;
|
||||
const CACHE_MAX = 2_000;
|
||||
const SUMMARY_MAX_ENTRIES = 10_000;
|
||||
const SUMMARY_BATCH_BUDGET_MS = 3_000;
|
||||
const INSPECT_MAX_ENTRIES = 20_000;
|
||||
const INSPECT_BUDGET_MS = 3_000;
|
||||
const UNCLAIMED_MAX_ENTRIES = 30_000;
|
||||
const UNCLAIMED_BUDGET_MS = 4_000;
|
||||
const UNCLAIMED_MAX_DEPTH = 5;
|
||||
const UNCLAIMED_RESULTS = 8;
|
||||
|
||||
class TtlCache<V> {
|
||||
private readonly entries = new Map<string, { value: V; expires: number }>();
|
||||
|
||||
get(key: string): V | undefined {
|
||||
const hit = this.entries.get(key);
|
||||
if (!hit) return undefined;
|
||||
if (hit.expires < Date.now()) {
|
||||
this.entries.delete(key);
|
||||
return undefined;
|
||||
}
|
||||
return hit.value;
|
||||
}
|
||||
|
||||
set(key: string, value: V) {
|
||||
this.entries.delete(key);
|
||||
this.entries.set(key, { value, expires: Date.now() + CACHE_TTL_MS });
|
||||
if (this.entries.size > CACHE_MAX) this.entries.delete(this.entries.keys().next().value!);
|
||||
}
|
||||
|
||||
clear() {
|
||||
this.entries.clear();
|
||||
}
|
||||
}
|
||||
|
||||
interface ScanNode {
|
||||
own: number;
|
||||
formats: Record<string, number>;
|
||||
children: string[];
|
||||
}
|
||||
|
||||
function toSummary(path: string, outcome: WalkOutcome): FolderSummary {
|
||||
return {
|
||||
path,
|
||||
state: outcome.state,
|
||||
files: outcome.files,
|
||||
folderBooks: outcome.folderBooks,
|
||||
formats: outcome.formats,
|
||||
folders: outcome.folders,
|
||||
mergedFolders: outcome.mergedFolders,
|
||||
splitAudioFolders: outcome.splitAudioFolders,
|
||||
truncated: outcome.truncated,
|
||||
};
|
||||
}
|
||||
|
||||
const emptySummary = (path: string, state: FolderSummary['state']): FolderSummary => ({
|
||||
path,
|
||||
state,
|
||||
files: 0,
|
||||
folderBooks: 0,
|
||||
formats: {},
|
||||
folders: 0,
|
||||
mergedFolders: 0,
|
||||
splitAudioFolders: 0,
|
||||
truncated: false,
|
||||
});
|
||||
|
||||
/**
|
||||
* What folders hold, for the folder picker. Every walk is bounded by entries and time and says when it
|
||||
* stopped, and answers are cached briefly so scrolling back and forth costs nothing.
|
||||
*/
|
||||
@Injectable()
|
||||
export class FolderInsightService {
|
||||
private readonly logger = new Logger(FolderInsightService.name);
|
||||
private readonly summaries = new TtlCache<FolderSummary>();
|
||||
private readonly inspections = new TtlCache<FolderInspection>();
|
||||
/** Keyed by the folders libraries read, so a library added or removed elsewhere is seen at once. */
|
||||
private unclaimedCache: { claimedKey: string; value: UnclaimedFolders; expires: number } | null = null;
|
||||
private unclaimedRun: { claimedKey: string; promise: Promise<UnclaimedFolders> } | null = null;
|
||||
/** Bumped by `forget()`, so a walk that started before it does not store its stale answer. */
|
||||
private generation = 0;
|
||||
|
||||
constructor(
|
||||
private readonly pathPolicy: PathPolicyService,
|
||||
private readonly pathService: PathService,
|
||||
private readonly pathRepository: PathRepository,
|
||||
) {}
|
||||
|
||||
async summarize(paths: string[], budgetMs = SUMMARY_BATCH_BUDGET_MS): Promise<FolderSummary[]> {
|
||||
const startedAt = Date.now();
|
||||
const deadline = startedAt + budgetMs;
|
||||
let walked = 0;
|
||||
const generation = this.generation;
|
||||
const results = await mapLimit(paths, 4, async (path): Promise<FolderSummary> => {
|
||||
const resolved = await this.resolveOrNull(path);
|
||||
// Walked with symlinks followed, so every folder below is checked against the blocked list by its real path.
|
||||
const target = resolved && (await realTarget(resolved));
|
||||
if (!resolved || !target || isBlockedPath(resolved) || isBlockedPath(target)) return emptySummary(path, 'blocked');
|
||||
const cached = this.summaries.get(target);
|
||||
if (cached) return { ...cached, path };
|
||||
// Folders the batch had no time left for are not reported as empty: the client asks again.
|
||||
if (Date.now() > deadline) return emptySummary(path, 'pending');
|
||||
walked += 1;
|
||||
const outcome = await walkFolder(target, { maxEntries: SUMMARY_MAX_ENTRIES, deadline });
|
||||
const summary = toSummary(path, outcome);
|
||||
if (this.cacheable(outcome, deadline, generation)) this.summaries.set(target, summary);
|
||||
return summary;
|
||||
});
|
||||
if (walked > 0) {
|
||||
this.logger.log(
|
||||
`[path.summaries] [end] paths=${paths.length} walked=${walked} truncated=${results.filter((r) => r.truncated).length} durationMs=${Date.now() - startedAt} - folder summaries completed`,
|
||||
);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
async inspect(path: string): Promise<FolderInspection> {
|
||||
const resolved = await this.pathPolicy.resolveBrowsePath(path);
|
||||
const target = await realTarget(resolved);
|
||||
if (isBlockedPath(resolved) || isBlockedPath(target)) return { ...emptySummary(resolved, 'blocked'), bytes: 0, samples: [] };
|
||||
const cached = this.inspections.get(target);
|
||||
if (cached) return { ...cached, path: resolved };
|
||||
const generation = this.generation;
|
||||
const deadline = Date.now() + INSPECT_BUDGET_MS;
|
||||
const outcome = await walkFolder(target, { maxEntries: INSPECT_MAX_ENTRIES, deadline, withSizes: true, samples: 2 });
|
||||
const inspection: FolderInspection = { ...toSummary(resolved, outcome), bytes: outcome.bytes, samples: outcome.samples };
|
||||
if (this.cacheable(outcome, deadline, generation)) this.inspections.set(target, inspection);
|
||||
return inspection;
|
||||
}
|
||||
|
||||
/** A new folder changes what every folder above it holds, so cached answers are dropped. */
|
||||
forget() {
|
||||
this.generation += 1;
|
||||
this.summaries.clear();
|
||||
this.inspections.clear();
|
||||
this.unclaimedCache = null;
|
||||
this.unclaimedRun = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Only a readable folder's answer is kept: a failure may be a passing one (a share reconnecting),
|
||||
* and a walk cut short by its time budget is finished by the next request instead.
|
||||
*/
|
||||
private cacheable(outcome: WalkOutcome, deadline: number, generation: number): boolean {
|
||||
return outcome.state === 'ok' && generation === this.generation && (!outcome.truncated || Date.now() <= deadline);
|
||||
}
|
||||
|
||||
/**
|
||||
* Folders holding books that no library reads, biggest first. Stops at the first folder shaped
|
||||
* like a library (one folder per book, loose files, or series folders) rather than listing every
|
||||
* author folder below it.
|
||||
*/
|
||||
async unclaimed(): Promise<UnclaimedFolders> {
|
||||
const claimed = [...new Set((await this.pathRepository.findLibraryFolders()).map((row) => resolve(row.path)))].sort();
|
||||
const claimedKey = claimed.join('\0');
|
||||
if (this.unclaimedCache?.claimedKey === claimedKey && this.unclaimedCache.expires > Date.now()) return this.unclaimedCache.value;
|
||||
// Pickers opened together share one scan rather than walking the mounts once each.
|
||||
let run = this.unclaimedRun;
|
||||
if (run?.claimedKey !== claimedKey) {
|
||||
const started = { claimedKey, promise: this.scanUnclaimed(claimed, claimedKey) };
|
||||
this.unclaimedRun = run = started;
|
||||
started.promise
|
||||
.finally(() => {
|
||||
if (this.unclaimedRun === started) this.unclaimedRun = null;
|
||||
})
|
||||
.catch(() => undefined);
|
||||
}
|
||||
return run.promise;
|
||||
}
|
||||
|
||||
private async scanUnclaimed(claimed: string[], claimedKey: string): Promise<UnclaimedFolders> {
|
||||
const startedAt = Date.now();
|
||||
const generation = this.generation;
|
||||
const places = await this.pathService.getPlaces();
|
||||
const rawAppData = this.pathPolicy.getAppDataPath();
|
||||
const appDataPath = await this.pathPolicy.getCanonicalAppDataPath();
|
||||
// Paths are compared as typed and with symlinks followed: the data folder may be reached through a link.
|
||||
const inAppData = (path: string, real: string) =>
|
||||
(!!rawAppData && isInsideOrAt(path, rawAppData)) || (!!appDataPath && (isInsideOrAt(path, appDataPath) || isInsideOrAt(real, appDataPath)));
|
||||
// Without separate mounts (everything on one filesystem), look around the folders libraries already read.
|
||||
const starts = places.mounts.length > 0 ? places.mounts : [...new Set(claimed.map((folder) => dirname(folder)))];
|
||||
const reals = new Map(await Promise.all(starts.map(async (root) => [root, await realpath(root).catch(() => root)] as const)));
|
||||
const roots = starts.filter((root) => !inAppData(root, reals.get(root)!) && !starts.some((other) => other !== root && isInsideOrAt(root, other)));
|
||||
this.logger.log(`[path.unclaimed] [start] roots=${roots.length} claimed=${claimed.length} - unclaimed folder scan started`);
|
||||
|
||||
try {
|
||||
const nodes = new Map<string, ScanNode>();
|
||||
const deadline = startedAt + UNCLAIMED_BUDGET_MS;
|
||||
let entries = 0;
|
||||
let truncated = false;
|
||||
const queue: Array<{ dir: string; real: string; depth: number }> = roots.map((dir) => ({ dir, real: reals.get(dir)!, depth: 0 }));
|
||||
while (queue.length > 0) {
|
||||
if (entries >= UNCLAIMED_MAX_ENTRIES || Date.now() > deadline) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
const { dir, real, depth } = queue.shift()!;
|
||||
let dirents: Dirent[];
|
||||
try {
|
||||
dirents = await readdir(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
entries += dirents.length;
|
||||
const node: ScanNode = { own: 0, formats: {}, children: [] };
|
||||
for (const entry of dirents) {
|
||||
const full = join(dir, entry.name);
|
||||
if (isListableDirectory(entry, full)) {
|
||||
if (dir === sep && SYSTEM_FOLDERS.has(entry.name)) continue;
|
||||
const childReal = join(real, entry.name);
|
||||
if (inAppData(full, childReal) || isBlockedPath(childReal)) continue;
|
||||
node.children.push(full);
|
||||
// Deeper folders are left out of the totals, which are then a lower bound.
|
||||
if (depth < UNCLAIMED_MAX_DEPTH) queue.push({ dir: full, real: childReal, depth: depth + 1 });
|
||||
else truncated = true;
|
||||
} else if (entry.isFile() && isPrimaryFormat(full)) {
|
||||
const format = entry.name.split('.').pop()!.toLowerCase();
|
||||
node.own += 1;
|
||||
node.formats[format] = (node.formats[format] ?? 0) + 1;
|
||||
}
|
||||
}
|
||||
nodes.set(dir, node);
|
||||
}
|
||||
|
||||
const totals = new Map<string, { files: number; formats: Record<string, number> }>();
|
||||
const total = (dir: string): { files: number; formats: Record<string, number> } => {
|
||||
const known = totals.get(dir);
|
||||
if (known) return known;
|
||||
const node = nodes.get(dir);
|
||||
const sum = { files: node?.own ?? 0, formats: { ...(node?.formats ?? {}) } };
|
||||
for (const child of node?.children ?? []) {
|
||||
const sub = total(child);
|
||||
sum.files += sub.files;
|
||||
for (const [format, count] of Object.entries(sub.formats)) sum.formats[format] = (sum.formats[format] ?? 0) + count;
|
||||
}
|
||||
totals.set(dir, sum);
|
||||
return sum;
|
||||
};
|
||||
|
||||
const found: UnclaimedFolder[] = [];
|
||||
const pick = (dir: string): boolean => {
|
||||
if (claimed.some((folder) => isInsideOrAt(dir, folder))) return false;
|
||||
const sum = total(dir);
|
||||
if (sum.files === 0) return false;
|
||||
const holdsLibrary = claimed.some((folder) => folder !== dir && isInsideOrAt(folder, dir));
|
||||
if (!holdsLibrary && libraryShaped(dir, nodes)) {
|
||||
found.push({ path: dir, files: sum.files, formats: sum.formats });
|
||||
return true;
|
||||
}
|
||||
const below = (nodes.get(dir)?.children ?? []).map(pick).some(Boolean);
|
||||
if (!below && !holdsLibrary) {
|
||||
found.push({ path: dir, files: sum.files, formats: sum.formats });
|
||||
return true;
|
||||
}
|
||||
return below;
|
||||
};
|
||||
roots.forEach(pick);
|
||||
|
||||
const value: UnclaimedFolders = { folders: found.sort((a, b) => b.files - a.files).slice(0, UNCLAIMED_RESULTS), truncated };
|
||||
if (generation === this.generation) this.unclaimedCache = { claimedKey, value, expires: Date.now() + CACHE_TTL_MS };
|
||||
this.logger.log(
|
||||
`[path.unclaimed] [end] found=${found.length} entries=${entries} truncated=${truncated} durationMs=${Date.now() - startedAt} - unclaimed folder scan completed`,
|
||||
);
|
||||
return value;
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`[path.unclaimed] [fail] durationMs=${Date.now() - startedAt} errorClass=${error instanceof Error ? error.constructor.name : 'Error'} error="${sanitizeLogValue(error instanceof Error ? error.message : String(error))}" - unclaimed folder scan failed`,
|
||||
);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async resolveOrNull(path: string): Promise<string | null> {
|
||||
try {
|
||||
return await this.pathPolicy.resolveBrowsePath(path);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Loose files, series folders, or one folder per book (directly or under author folders). */
|
||||
function libraryShaped(dir: string, nodes: Map<string, ScanNode>): boolean {
|
||||
const node = nodes.get(dir);
|
||||
if (!node) return false;
|
||||
const kids = node.children.map((child) => nodes.get(child)).filter((child): child is ScanNode => !!child);
|
||||
if (node.own > 0 && node.own >= kids.length) return true;
|
||||
if (kids.length === 0) return false;
|
||||
const most = (test: (kid: ScanNode) => boolean) => kids.filter(test).length >= Math.max(1, kids.length * 0.6);
|
||||
const leafy = (kid: ScanNode) => kid.children.length === 0;
|
||||
if (most((kid) => leafy(kid) && kid.own > 0)) return true;
|
||||
return most((kid) => {
|
||||
const titles = kid.children.map((child) => nodes.get(child)).filter((child): child is ScanNode => !!child);
|
||||
return titles.length > 0 && kid.own === 0 && titles.every((title) => title.children.length === 0 && title.own > 0 && title.own <= 2);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
import { chmod, mkdir, mkdtemp, rm, symlink, writeFile } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
|
||||
import { findBookCandidates, findLooseFileCandidates } from '../scanner/lib/walk';
|
||||
import { hasListableChild, isBlockedPath, isBlockedTarget, isInsideOrAt, mapLimit, walkFolder } from './folder-walk';
|
||||
|
||||
const runningAsRoot = typeof process.getuid === 'function' && process.getuid() === 0;
|
||||
|
||||
async function files(dir: string, names: string[], size = 10) {
|
||||
await mkdir(dir, { recursive: true });
|
||||
for (const name of names) await writeFile(join(dir, name), 'x'.repeat(size));
|
||||
}
|
||||
|
||||
const unlimited = () => ({ maxEntries: 100_000, deadline: Date.now() + 10_000 });
|
||||
|
||||
describe('walkFolder', () => {
|
||||
let root: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), 'folder-walk-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await chmod(root, 0o755).catch(() => undefined);
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('counts every book file directly in the folder as its own book, as the scanner does', async () => {
|
||||
await files(root, ['Dune.epub', 'Emma.pdf', 'cover.jpg', 'notes.txt.bak']);
|
||||
|
||||
const outcome = await walkFolder(root, unlimited());
|
||||
|
||||
expect(outcome).toMatchObject({ state: 'ok', files: 2, folderBooks: 2, formats: { epub: 1, pdf: 1 }, folders: 0, mergedFolders: 0 });
|
||||
});
|
||||
|
||||
it('reads one book per folder below, through author folders', async () => {
|
||||
await files(join(root, 'Ann Leckie', 'Ancillary Justice (2013)'), ['Ancillary Justice.epub', 'Ancillary Justice.mobi']);
|
||||
await files(join(root, 'Ann Leckie', 'Provenance (2017)'), ['Provenance.epub']);
|
||||
await files(join(root, 'Martha Wells', 'All Systems Red (2017)'), ['All Systems Red.azw3']);
|
||||
|
||||
const outcome = await walkFolder(root, unlimited());
|
||||
|
||||
expect(outcome).toMatchObject({ files: 4, folderBooks: 3, folders: 2, mergedFolders: 0, formats: { epub: 2, mobi: 1, azw3: 1 } });
|
||||
});
|
||||
|
||||
it('flags folders below that hold several differently named books, which Folder as Book merges', async () => {
|
||||
await files(join(root, 'Saga (2012)'), ['Saga 001.cbz', 'Saga 002.cbz', 'Saga 003.cbz']);
|
||||
await files(join(root, 'Watchmen (1986)'), ['Watchmen 001.cbz', 'Watchmen 002.cbz']);
|
||||
|
||||
const outcome = await walkFolder(root, unlimited());
|
||||
|
||||
expect(outcome).toMatchObject({ files: 5, folderBooks: 2, mergedFolders: 2, splitAudioFolders: 0 });
|
||||
});
|
||||
|
||||
it('flags audiobooks in parts, which File as Book splits into one book per file', async () => {
|
||||
await files(join(root, 'Andy Weir', 'Project Hail Mary'), ['01.mp3', '02.mp3', '03.mp3']);
|
||||
await files(join(root, 'Andy Weir', 'Artemis'), ['Artemis.m4b']);
|
||||
|
||||
const outcome = await walkFolder(root, unlimited());
|
||||
|
||||
expect(outcome).toMatchObject({ files: 4, folderBooks: 2, mergedFolders: 0, splitAudioFolders: 1 });
|
||||
});
|
||||
|
||||
it('folds disc folders and folders named after a sibling file into their parent', async () => {
|
||||
await files(join(root, 'Author', 'Long Book', 'CD 1'), ['01.mp3']);
|
||||
await files(join(root, 'Author', 'Long Book', 'CD 2'), ['01.mp3']);
|
||||
await files(join(root, 'Author', 'Story'), ['Story.epub']);
|
||||
await files(join(root, 'Author', 'Story', 'Story'), ['Story.mp3']);
|
||||
|
||||
const outcome = await walkFolder(root, unlimited());
|
||||
|
||||
expect(outcome.folderBooks).toBe(2);
|
||||
});
|
||||
|
||||
it('folds disc folders inside a folder named after a sibling file into one book, as the scanner does', async () => {
|
||||
await files(join(root, 'Author', 'Title'), ['Title.epub']);
|
||||
await files(join(root, 'Author', 'Title', 'Title', 'CD 1'), ['01.mp3']);
|
||||
await files(join(root, 'Author', 'Title', 'Title', 'CD 2'), ['01.mp3']);
|
||||
|
||||
const outcome = await walkFolder(root, unlimited());
|
||||
|
||||
expect(outcome.folderBooks).toBe(1);
|
||||
});
|
||||
|
||||
it('folds a folder named after a sidecar file, not only after a book file', async () => {
|
||||
await files(join(root, 'Series'), ['A.epub', 'B.opf']);
|
||||
await files(join(root, 'Series', 'B'), ['01.mp3']);
|
||||
|
||||
const outcome = await walkFolder(root, unlimited());
|
||||
|
||||
expect(outcome.folderBooks).toBe(1);
|
||||
});
|
||||
|
||||
it('agrees with the scanner on every layout it groups', async () => {
|
||||
await files(root, ['Loose.epub', 'Loose.opf', 'Other.pdf']);
|
||||
await files(join(root, 'CD 1'), ['01.mp3', '02.mp3']);
|
||||
await files(join(root, 'Ann Leckie', 'Ancillary Justice (2013)'), ['Ancillary Justice.epub', 'Ancillary Justice.mobi', 'cover.jpg']);
|
||||
await files(join(root, 'Saga (2012)'), ['Saga 001.cbz', 'Saga 002.cbz']);
|
||||
await files(join(root, 'Andy Weir', 'Project Hail Mary', 'Disc 1'), ['01.mp3']);
|
||||
await files(join(root, 'Andy Weir', 'Project Hail Mary', 'Disc 2'), ['01.mp3']);
|
||||
await files(join(root, 'Author', 'Title'), ['Title.epub']);
|
||||
await files(join(root, 'Author', 'Title', 'Title', 'CD 1'), ['01.mp3']);
|
||||
await files(join(root, 'Author', 'Title', 'Title', 'CD 2'), ['01.mp3']);
|
||||
await files(join(root, 'Series'), ['A.epub', 'B.opf']);
|
||||
await files(join(root, 'Series', 'B'), ['01.mp3']);
|
||||
await files(join(root, 'Notes'), ['readme.txt']);
|
||||
|
||||
const outcome = await walkFolder(root, unlimited());
|
||||
|
||||
expect(outcome.folderBooks).toBe((await findBookCandidates(root)).candidates.length);
|
||||
expect(outcome.files).toBe((await findLooseFileCandidates(root)).candidates.length);
|
||||
});
|
||||
|
||||
it('skips hidden folders and symlinks', async () => {
|
||||
await files(join(root, '.cache'), ['Hidden.epub']);
|
||||
await files(join(root, 'Real'), ['Real.epub']);
|
||||
await symlink(join(root, 'Real'), join(root, 'Linked'));
|
||||
|
||||
const outcome = await walkFolder(root, unlimited());
|
||||
|
||||
expect(outcome).toMatchObject({ files: 1, folders: 1 });
|
||||
});
|
||||
|
||||
it('stops at the entry cap and says the counts are a lower bound', async () => {
|
||||
for (let i = 0; i < 5; i++) await files(join(root, `Author ${i}`), [`Book ${i}.epub`]);
|
||||
|
||||
const outcome = await walkFolder(root, { maxEntries: 3, deadline: Date.now() + 10_000 });
|
||||
|
||||
expect(outcome.truncated).toBe(true);
|
||||
expect(outcome.files).toBeLessThan(5);
|
||||
});
|
||||
|
||||
it('reads no more of a huge flat folder than the entry budget allows', async () => {
|
||||
await files(
|
||||
root,
|
||||
Array.from({ length: 50 }, (_, i) => `Book ${String(i).padStart(2, '0')}.epub`),
|
||||
);
|
||||
|
||||
const outcome = await walkFolder(root, { maxEntries: 10, deadline: Date.now() + 10_000 });
|
||||
|
||||
expect(outcome).toMatchObject({ truncated: true, files: 10, folderBooks: 10 });
|
||||
});
|
||||
|
||||
it('stops sizing files once the deadline passes mid-folder and says the answer is partial', async () => {
|
||||
await files(root, ['A.epub', 'B.epub'], 100);
|
||||
// Time for the folder and the first file, then the deadline is behind.
|
||||
const now = vi.spyOn(Date, 'now').mockReturnValueOnce(0).mockReturnValueOnce(0).mockReturnValue(100);
|
||||
|
||||
try {
|
||||
const outcome = await walkFolder(root, { maxEntries: 100, deadline: 50, withSizes: true });
|
||||
|
||||
expect(outcome).toMatchObject({ truncated: true, files: 2, bytes: 100 });
|
||||
} finally {
|
||||
now.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it.skipIf(runningAsRoot)('says the counts are a lower bound when a folder below cannot be opened', async () => {
|
||||
await files(join(root, 'Open'), ['Open.epub']);
|
||||
await files(join(root, 'Closed'), ['Closed.epub']);
|
||||
await chmod(join(root, 'Closed'), 0o000);
|
||||
|
||||
try {
|
||||
const outcome = await walkFolder(root, unlimited());
|
||||
|
||||
expect(outcome).toMatchObject({ state: 'ok', files: 1, truncated: true });
|
||||
} finally {
|
||||
await chmod(join(root, 'Closed'), 0o755);
|
||||
}
|
||||
});
|
||||
|
||||
it('sums sizes and returns the first files in name order when asked', async () => {
|
||||
await files(join(root, 'B'), ['Second.epub'], 30);
|
||||
await files(join(root, 'A'), ['First.epub'], 20);
|
||||
await files(root, ['Loose.pdf'], 50);
|
||||
|
||||
const outcome = await walkFolder(root, { ...unlimited(), withSizes: true, samples: 2 });
|
||||
|
||||
expect(outcome.bytes).toBe(100);
|
||||
expect(outcome.samples).toEqual(['Loose.pdf', 'A/First.epub']);
|
||||
});
|
||||
|
||||
it('reports a missing folder instead of an empty one', async () => {
|
||||
const outcome = await walkFolder(join(root, 'nope'), unlimited());
|
||||
|
||||
expect(outcome).toMatchObject({ state: 'missing', files: 0 });
|
||||
});
|
||||
|
||||
it.skipIf(runningAsRoot)('reports a folder it may not read as denied', async () => {
|
||||
await chmod(root, 0o000);
|
||||
|
||||
const outcome = await walkFolder(root, unlimited());
|
||||
|
||||
expect(outcome.state).toBe('denied');
|
||||
});
|
||||
});
|
||||
|
||||
describe('folder-walk helpers', () => {
|
||||
it('knows the blocked system paths and their children', () => {
|
||||
expect(isBlockedPath('/proc')).toBe(true);
|
||||
expect(isBlockedPath('/proc/1')).toBe(true);
|
||||
expect(isBlockedPath('/procedures')).toBe(false);
|
||||
});
|
||||
|
||||
it('refuses a symlink into a blocked system path like the path itself', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'folder-walk-'));
|
||||
try {
|
||||
await symlink('/dev', join(root, 'dev-link'));
|
||||
await mkdir(join(root, 'plain'));
|
||||
|
||||
expect(await isBlockedTarget(join(root, 'dev-link'))).toBe(true);
|
||||
expect(await isBlockedTarget(join(root, 'plain'))).toBe(false);
|
||||
expect(await isBlockedTarget(join(root, 'missing'))).toBe(false);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('treats a folder as inside itself and its ancestors only', () => {
|
||||
expect(isInsideOrAt('/books/Ebooks', '/books')).toBe(true);
|
||||
expect(isInsideOrAt('/books', '/books')).toBe(true);
|
||||
expect(isInsideOrAt('/bookshelf', '/books')).toBe(false);
|
||||
expect(isInsideOrAt('/anything', '/')).toBe(true);
|
||||
});
|
||||
|
||||
it('finds whether a folder has a listable folder inside', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'folder-walk-'));
|
||||
try {
|
||||
await files(join(root, 'empty'), []);
|
||||
await files(join(root, 'files-only'), ['a.epub']);
|
||||
await files(join(root, 'nested', 'inner'), []);
|
||||
await files(join(root, 'hidden-only', '.git'), []);
|
||||
|
||||
expect(await hasListableChild(join(root, 'empty'))).toBe(false);
|
||||
expect(await hasListableChild(join(root, 'files-only'))).toBe(false);
|
||||
expect(await hasListableChild(join(root, 'nested'))).toBe(true);
|
||||
expect(await hasListableChild(join(root, 'hidden-only'))).toBe(false);
|
||||
expect(await hasListableChild(join(root, 'missing'))).toBe(false);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps the input order while limiting concurrency', async () => {
|
||||
let inFlight = 0;
|
||||
let peak = 0;
|
||||
const results = await mapLimit([30, 10, 20, 5], 2, async (ms) => {
|
||||
inFlight += 1;
|
||||
peak = Math.max(peak, inFlight);
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
inFlight -= 1;
|
||||
return ms * 2;
|
||||
});
|
||||
|
||||
expect(results).toEqual([60, 20, 40, 10]);
|
||||
expect(peak).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,269 @@
|
||||
import type { Dirent } from 'fs';
|
||||
import { lstat, readdir, realpath } from 'fs/promises';
|
||||
import { basename, dirname, extname, join, relative, sep } from 'path';
|
||||
|
||||
import { isAudioFormat, isPrimaryFormat } from '../scanner/lib/classify';
|
||||
import { isDiscDirectory, stemOf } from '../scanner/lib/walk';
|
||||
|
||||
export const BLOCKED_PATHS = ['/proc', '/sys', '/dev', '/run', '/var/run', '/etc', '/root'];
|
||||
|
||||
/** Operating-system folders directly under `/`, hidden from the picker unless asked for. */
|
||||
export const SYSTEM_FOLDERS = new Set([
|
||||
'app',
|
||||
'bin',
|
||||
'boot',
|
||||
'home',
|
||||
'lib',
|
||||
'lib32',
|
||||
'lib64',
|
||||
'libx32',
|
||||
'lost+found',
|
||||
'opt',
|
||||
'sbin',
|
||||
'snap',
|
||||
'srv',
|
||||
'tmp',
|
||||
'usr',
|
||||
'var',
|
||||
'Applications',
|
||||
'Library',
|
||||
'System',
|
||||
'Volumes',
|
||||
'cores',
|
||||
'private',
|
||||
]);
|
||||
|
||||
export function isBlockedPath(path: string): boolean {
|
||||
return BLOCKED_PATHS.some((blocked) => path === blocked || path.startsWith(blocked + sep));
|
||||
}
|
||||
|
||||
/** `path` with symlinks followed, or as given when it cannot be resolved (a missing folder). */
|
||||
export async function realTarget(path: string): Promise<string> {
|
||||
try {
|
||||
return await realpath(path);
|
||||
} catch {
|
||||
return path;
|
||||
}
|
||||
}
|
||||
|
||||
/** Blocked as typed or once symlinks are followed, so a link to `/etc` is refused like `/etc`. */
|
||||
export async function isBlockedTarget(path: string): Promise<boolean> {
|
||||
return isBlockedPath(path) || isBlockedPath(await realTarget(path));
|
||||
}
|
||||
|
||||
export function isInsideOrAt(path: string, parent: string): boolean {
|
||||
if (path === parent) return true;
|
||||
return parent === sep ? path.startsWith(sep) : path.startsWith(parent + sep);
|
||||
}
|
||||
|
||||
/** A folder BookOrbit lists: a real directory, not hidden, not a symlink, not a blocked system path. */
|
||||
export function isListableDirectory(entry: Dirent, fullPath: string): boolean {
|
||||
return entry.isDirectory() && !entry.isSymbolicLink() && !entry.name.startsWith('.') && !isBlockedPath(fullPath);
|
||||
}
|
||||
|
||||
export interface WalkLimits {
|
||||
/** Directory entries to read before stopping; the result is then a lower bound. */
|
||||
maxEntries: number;
|
||||
/** Epoch milliseconds after which the walk stops. */
|
||||
deadline: number;
|
||||
/** Sum the size of every book file counted. */
|
||||
withSizes?: boolean;
|
||||
/** How many book files to return as samples, relative to the root. */
|
||||
samples?: number;
|
||||
}
|
||||
|
||||
export interface WalkOutcome {
|
||||
state: 'ok' | 'denied' | 'missing';
|
||||
files: number;
|
||||
folderBooks: number;
|
||||
formats: Record<string, number>;
|
||||
folders: number;
|
||||
mergedFolders: number;
|
||||
splitAudioFolders: number;
|
||||
truncated: boolean;
|
||||
bytes: number;
|
||||
samples: string[];
|
||||
}
|
||||
|
||||
function errorCode(error: unknown): string | undefined {
|
||||
if (typeof error !== 'object' || error === null || !('code' in error)) return undefined;
|
||||
const code = (error as { code?: unknown }).code;
|
||||
return typeof code === 'string' ? code : undefined;
|
||||
}
|
||||
|
||||
export function stateForError(error: unknown): 'denied' | 'missing' {
|
||||
const code = errorCode(error);
|
||||
return code === 'EACCES' || code === 'EPERM' ? 'denied' : 'missing';
|
||||
}
|
||||
|
||||
/** What one folder holds, as the scanner groups it. */
|
||||
interface DirFiles {
|
||||
books: number;
|
||||
audio: number;
|
||||
/** Stems of the non-audio book files: several mean several titles. */
|
||||
bookStems: Set<string>;
|
||||
/** Stems of every file, sidecars included: a subfolder named after one folds into this folder. */
|
||||
stems: Set<string>;
|
||||
}
|
||||
|
||||
function mergeInto(groups: Map<string, DirFiles>, from: string, to: string) {
|
||||
const source = groups.get(from)!;
|
||||
const target = groups.get(to);
|
||||
groups.delete(from);
|
||||
if (!target) {
|
||||
groups.set(to, source);
|
||||
return;
|
||||
}
|
||||
target.books += source.books;
|
||||
target.audio += source.audio;
|
||||
for (const stem of source.bookStems) target.bookStems.add(stem);
|
||||
for (const stem of source.stems) target.stems.add(stem);
|
||||
}
|
||||
|
||||
/**
|
||||
* Counts what a library would import from `root` by the scanner's grouping rules
|
||||
* (`findBookCandidates`): every book file directly in the root is its own book; below it, Folder as
|
||||
* Book reads each folder holding book files as one book, after folding disc folders ("CD 1") and then
|
||||
* folders named after a sibling file into their parent. Depth-first in name order, so samples are stable.
|
||||
*/
|
||||
export async function walkFolder(root: string, limits: WalkLimits): Promise<WalkOutcome> {
|
||||
const outcome: WalkOutcome = {
|
||||
state: 'ok',
|
||||
files: 0,
|
||||
folderBooks: 0,
|
||||
formats: {},
|
||||
folders: 0,
|
||||
mergedFolders: 0,
|
||||
splitAudioFolders: 0,
|
||||
truncated: false,
|
||||
bytes: 0,
|
||||
samples: [],
|
||||
};
|
||||
const sampleLimit = limits.samples ?? 0;
|
||||
const groups = new Map<string, DirFiles>();
|
||||
const stack = [root];
|
||||
let entries = 0;
|
||||
|
||||
while (stack.length > 0) {
|
||||
const dir = stack.pop()!;
|
||||
if (entries >= limits.maxEntries || Date.now() > limits.deadline) {
|
||||
outcome.truncated = true;
|
||||
break;
|
||||
}
|
||||
|
||||
let read: { entries: Dirent[]; cut: boolean };
|
||||
try {
|
||||
read = await readEntries(dir, limits.maxEntries - entries);
|
||||
} catch (error) {
|
||||
if (dir === root) {
|
||||
outcome.state = stateForError(error);
|
||||
return outcome;
|
||||
}
|
||||
// Books in a folder that cannot be opened are not counted, so the counts are a lower bound.
|
||||
outcome.truncated = true;
|
||||
continue;
|
||||
}
|
||||
const dirents = read.entries.sort(byName);
|
||||
entries += dirents.length;
|
||||
if (read.cut) outcome.truncated = true;
|
||||
|
||||
const children: string[] = [];
|
||||
const bookFiles: string[] = [];
|
||||
const stems = new Set<string>();
|
||||
for (const entry of dirents) {
|
||||
if (entry.name.startsWith('.')) continue;
|
||||
const full = join(dir, entry.name);
|
||||
if (isListableDirectory(entry, full)) children.push(full);
|
||||
else if (entry.isFile()) {
|
||||
stems.add(stemOf(entry.name));
|
||||
if (isPrimaryFormat(full)) bookFiles.push(full);
|
||||
}
|
||||
}
|
||||
if (dir === root) outcome.folders = children.length;
|
||||
|
||||
const group: DirFiles = { books: bookFiles.length, audio: 0, bookStems: new Set(), stems };
|
||||
for (const file of bookFiles) {
|
||||
const format = extname(file).slice(1).toLowerCase();
|
||||
outcome.files += 1;
|
||||
outcome.formats[format] = (outcome.formats[format] ?? 0) + 1;
|
||||
if (isAudioFormat(format)) group.audio += 1;
|
||||
else group.bookStems.add(stemOf(basename(file)));
|
||||
if (outcome.samples.length < sampleLimit) outcome.samples.push(relative(root, file).split(sep).join('/'));
|
||||
if (limits.withSizes && !outcome.truncated) {
|
||||
if (Date.now() > limits.deadline) {
|
||||
outcome.truncated = true;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
outcome.bytes += (await lstat(file)).size;
|
||||
} catch {
|
||||
// A file that vanished mid-walk adds nothing.
|
||||
}
|
||||
}
|
||||
}
|
||||
if (stems.size > 0) groups.set(dir, group);
|
||||
|
||||
if (read.cut) break;
|
||||
for (let i = children.length - 1; i >= 0; i--) stack.push(children[i]);
|
||||
}
|
||||
|
||||
// The scanner's two folding passes, in its order. The root never folds: its files are each a book.
|
||||
for (const dir of [...groups.keys()]) if (dir !== root && isDiscDirectory(basename(dir))) mergeInto(groups, dir, dirname(dir));
|
||||
const stemNamed = [...groups.keys()].filter((dir) => {
|
||||
const parent = dirname(dir);
|
||||
return dir !== root && parent !== root && !!groups.get(parent)?.stems.has(basename(dir));
|
||||
});
|
||||
for (const dir of stemNamed) if (groups.has(dir)) mergeInto(groups, dir, dirname(dir));
|
||||
|
||||
for (const [dir, group] of groups) {
|
||||
if (group.books === 0) continue;
|
||||
if (dir === root) {
|
||||
outcome.folderBooks += group.books;
|
||||
continue;
|
||||
}
|
||||
outcome.folderBooks += 1;
|
||||
if (group.audio === 0 && group.bookStems.size > 1) outcome.mergedFolders += 1;
|
||||
if (group.audio > 1) outcome.splitAudioFolders += 1;
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
|
||||
/** Whether `dir` holds at least one listable folder. */
|
||||
export async function hasListableChild(dir: string): Promise<boolean> {
|
||||
try {
|
||||
return (await readdir(dir, { withFileTypes: true })).some((entry) => isListableDirectory(entry, join(dir, entry.name)));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The first `budget` entries of `dir` in name order. `readdir` rather than a streamed `opendir`: on
|
||||
* filesystems that do not report entry types (some NFS and FUSE mounts) `opendir` falls back to a
|
||||
* synchronous lstat per entry, which would stall the server for every user.
|
||||
*/
|
||||
async function readEntries(dir: string, budget: number): Promise<{ entries: Dirent[]; cut: boolean }> {
|
||||
const entries = await readdir(dir, { withFileTypes: true });
|
||||
const limit = Math.max(0, budget);
|
||||
if (entries.length <= limit) return { entries, cut: false };
|
||||
return { entries: entries.sort(byName).slice(0, limit), cut: true };
|
||||
}
|
||||
|
||||
/** Code-unit order: stable and cheap on folders of tens of thousands of entries. */
|
||||
function byName(a: Dirent, b: Dirent): number {
|
||||
return a.name < b.name ? -1 : a.name > b.name ? 1 : 0;
|
||||
}
|
||||
|
||||
/** Runs `task` over `items` with at most `limit` in flight, keeping the input order. */
|
||||
export async function mapLimit<T, R>(items: T[], limit: number, task: (item: T) => Promise<R>): Promise<R[]> {
|
||||
const results = new Array<R>(items.length);
|
||||
let next = 0;
|
||||
const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
||||
while (next < items.length) {
|
||||
const index = next++;
|
||||
results[index] = await task(items[index]);
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { BadRequestException, ForbiddenException } from '@nestjs/common';
|
||||
import { mkdtemp, mkdir, rm, symlink } from 'fs/promises';
|
||||
import { mkdtemp, mkdir, realpath, rm, symlink } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join, resolve } from 'path';
|
||||
|
||||
@@ -83,6 +83,22 @@ describe('PathPolicyService', () => {
|
||||
await expect(service.isWithinBrowseRoot(join(root, 'loop'))).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it('resolves a symlinked data folder to the real path listings report', async () => {
|
||||
const root = await makeTempRoot();
|
||||
await mkdir(join(root, 'real-data'));
|
||||
await symlink(join(root, 'real-data'), join(root, 'data'));
|
||||
const service = new PathPolicyService({ libraryBrowseRoot: root, appDataPath: join(root, 'data') } as never);
|
||||
|
||||
expect(service.getAppDataPath()).toBe(join(root, 'data'));
|
||||
await expect(service.getCanonicalAppDataPath()).resolves.toBe(await realpath(join(root, 'real-data')));
|
||||
});
|
||||
|
||||
it('has no data folder when none is configured', async () => {
|
||||
const service = makeService(await makeTempRoot());
|
||||
|
||||
await expect(service.getCanonicalAppDataPath()).resolves.toBeNull();
|
||||
});
|
||||
|
||||
async function makeTempRoot(): Promise<string> {
|
||||
const path = await mkdtemp(join(tmpdir(), 'bookorbit-path-policy-'));
|
||||
tempRoots.push(path);
|
||||
|
||||
@@ -10,16 +10,30 @@ const MAX_BROWSE_PATH_LENGTH = 4096;
|
||||
@Injectable()
|
||||
export class PathPolicyService {
|
||||
private readonly browseRoot: string;
|
||||
private readonly appDataPath: string | null;
|
||||
private canonicalBrowseRootPromise: Promise<string> | null = null;
|
||||
private canonicalAppDataPathPromise: Promise<string | null> | null = null;
|
||||
|
||||
constructor(@Inject(storageConfig.KEY) storage: ConfigType<typeof storageConfig>) {
|
||||
this.browseRoot = resolve(storage.libraryBrowseRoot);
|
||||
this.appDataPath = storage.appDataPath ? resolve(storage.appDataPath) : null;
|
||||
}
|
||||
|
||||
getBrowseRoot(): string {
|
||||
return this.browseRoot;
|
||||
}
|
||||
|
||||
getAppDataPath(): string | null {
|
||||
return this.appDataPath;
|
||||
}
|
||||
|
||||
/** The data folder with symlinks resolved, so it matches paths read from the disk. */
|
||||
getCanonicalAppDataPath(): Promise<string | null> {
|
||||
if (!this.appDataPath) return Promise.resolve(null);
|
||||
this.canonicalAppDataPathPromise ??= realpath(this.appDataPath).catch(() => this.appDataPath);
|
||||
return this.canonicalAppDataPathPromise;
|
||||
}
|
||||
|
||||
async resolveBrowsePath(rawPath?: string | null): Promise<string> {
|
||||
const inputPath = typeof rawPath === 'string' ? rawPath.trim() : '';
|
||||
if (inputPath.length > MAX_BROWSE_PATH_LENGTH) throw new BadRequestException('Path is too long');
|
||||
|
||||
@@ -5,9 +5,18 @@ describe('PathController', () => {
|
||||
getConfig: vi.fn(),
|
||||
listDirectories: vi.fn(),
|
||||
createDirectory: vi.fn(),
|
||||
listFolder: vi.fn(),
|
||||
searchFolders: vi.fn(),
|
||||
getPlaces: vi.fn(),
|
||||
};
|
||||
const folderInsight = {
|
||||
summarize: vi.fn(),
|
||||
inspect: vi.fn(),
|
||||
unclaimed: vi.fn(),
|
||||
forget: vi.fn(),
|
||||
};
|
||||
|
||||
const controller = new PathController(pathService as never);
|
||||
const controller = new PathController(pathService as never, folderInsight as never);
|
||||
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
@@ -31,9 +40,54 @@ describe('PathController', () => {
|
||||
expect(pathService.listDirectories).toHaveBeenCalledWith('/books');
|
||||
});
|
||||
|
||||
it('forwards folder creation to the service', async () => {
|
||||
await controller.createFolder({ parentPath: '/books', name: 'scifi' });
|
||||
it('forwards folder creation to the service and drops cached folder counts', async () => {
|
||||
pathService.createDirectory.mockResolvedValue({ name: 'scifi', path: '/books/scifi' });
|
||||
|
||||
await expect(controller.createFolder({ parentPath: '/books', name: 'scifi' })).resolves.toEqual({ name: 'scifi', path: '/books/scifi' });
|
||||
|
||||
expect(pathService.createDirectory).toHaveBeenCalledWith('/books', 'scifi');
|
||||
expect(folderInsight.forget).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('keeps cached counts when folder creation fails', async () => {
|
||||
pathService.createDirectory.mockRejectedValue(new Error('exists'));
|
||||
|
||||
await expect(controller.createFolder({ parentPath: '/books', name: 'scifi' })).rejects.toThrow('exists');
|
||||
|
||||
expect(folderInsight.forget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('passes the listing path through to the service', async () => {
|
||||
await controller.listFolder({ path: '/books' });
|
||||
|
||||
expect(pathService.listFolder).toHaveBeenCalledWith('/books');
|
||||
});
|
||||
|
||||
it('summarizes the requested paths in one batch', async () => {
|
||||
await controller.summarize({ paths: ['/books/a', '/books/b'] });
|
||||
|
||||
expect(folderInsight.summarize).toHaveBeenCalledWith(['/books/a', '/books/b']);
|
||||
});
|
||||
|
||||
it('inspects the browse root when no path is given', async () => {
|
||||
await controller.inspect({});
|
||||
|
||||
expect(folderInsight.inspect).toHaveBeenCalledWith('');
|
||||
});
|
||||
|
||||
it('searches system folders only when asked', async () => {
|
||||
await controller.search({ q: 'wells', path: '/books' });
|
||||
await controller.search({ q: 'usr', system: 'true' });
|
||||
|
||||
expect(pathService.searchFolders).toHaveBeenNthCalledWith(1, 'wells', '/books', false);
|
||||
expect(pathService.searchFolders).toHaveBeenNthCalledWith(2, 'usr', undefined, true);
|
||||
});
|
||||
|
||||
it('returns places and unclaimed folders from their services', async () => {
|
||||
pathService.getPlaces.mockResolvedValue({ root: '/', mounts: ['/books'], appDataPath: '/data', libraries: [] });
|
||||
folderInsight.unclaimed.mockResolvedValue({ folders: [], truncated: false });
|
||||
|
||||
await expect(controller.places()).resolves.toEqual({ root: '/', mounts: ['/books'], appDataPath: '/data', libraries: [] });
|
||||
await expect(controller.unclaimed()).resolves.toEqual({ folders: [], truncated: false });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,14 +1,29 @@
|
||||
import type { PathConfig } from '@bookorbit/types';
|
||||
import type {
|
||||
FolderInspection,
|
||||
FolderListing,
|
||||
FolderPlaces,
|
||||
FolderSearchResult,
|
||||
FolderSummary,
|
||||
PathConfig,
|
||||
UnclaimedFolders,
|
||||
} from '@bookorbit/types';
|
||||
import { Permission } from '@bookorbit/types';
|
||||
import { Body, Controller, Get, Post, Query } from '@nestjs/common';
|
||||
import { Body, Controller, Get, HttpCode, Post, Query } from '@nestjs/common';
|
||||
|
||||
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
|
||||
import { CreateFolderDto } from './dto/create-folder.dto';
|
||||
import { FolderPathQueryDto } from './dto/folder-path-query.dto';
|
||||
import { FolderSummariesDto } from './dto/folder-summaries.dto';
|
||||
import { SearchFoldersQueryDto } from './dto/search-folders-query.dto';
|
||||
import { FolderInsightService } from './folder-insight.service';
|
||||
import { PathService } from './path.service';
|
||||
|
||||
@Controller('path')
|
||||
export class PathController {
|
||||
constructor(private readonly pathService: PathService) {}
|
||||
constructor(
|
||||
private readonly pathService: PathService,
|
||||
private readonly folderInsight: FolderInsightService,
|
||||
) {}
|
||||
|
||||
@Get('config')
|
||||
@RequirePermission(Permission.ManageLibraries)
|
||||
@@ -22,9 +37,48 @@ export class PathController {
|
||||
return this.pathService.listDirectories(path);
|
||||
}
|
||||
|
||||
@Get('listing')
|
||||
@RequirePermission(Permission.ManageLibraries)
|
||||
listFolder(@Query() query: FolderPathQueryDto): Promise<FolderListing> {
|
||||
return this.pathService.listFolder(query.path);
|
||||
}
|
||||
|
||||
@Post('summaries')
|
||||
@HttpCode(200)
|
||||
@RequirePermission(Permission.ManageLibraries)
|
||||
summarize(@Body() dto: FolderSummariesDto): Promise<FolderSummary[]> {
|
||||
return this.folderInsight.summarize(dto.paths);
|
||||
}
|
||||
|
||||
@Get('inspect')
|
||||
@RequirePermission(Permission.ManageLibraries)
|
||||
inspect(@Query() query: FolderPathQueryDto): Promise<FolderInspection> {
|
||||
return this.folderInsight.inspect(query.path ?? '');
|
||||
}
|
||||
|
||||
@Get('search')
|
||||
@RequirePermission(Permission.ManageLibraries)
|
||||
search(@Query() query: SearchFoldersQueryDto): Promise<FolderSearchResult> {
|
||||
return this.pathService.searchFolders(query.q, query.path, query.system === 'true');
|
||||
}
|
||||
|
||||
@Get('places')
|
||||
@RequirePermission(Permission.ManageLibraries)
|
||||
places(): Promise<FolderPlaces> {
|
||||
return this.pathService.getPlaces();
|
||||
}
|
||||
|
||||
@Get('unclaimed')
|
||||
@RequirePermission(Permission.ManageLibraries)
|
||||
unclaimed(): Promise<UnclaimedFolders> {
|
||||
return this.folderInsight.unclaimed();
|
||||
}
|
||||
|
||||
@Post()
|
||||
@RequirePermission(Permission.ManageLibraries)
|
||||
createFolder(@Body() dto: CreateFolderDto) {
|
||||
return this.pathService.createDirectory(dto.parentPath, dto.name);
|
||||
async createFolder(@Body() dto: CreateFolderDto) {
|
||||
const created = await this.pathService.createDirectory(dto.parentPath, dto.name);
|
||||
this.folderInsight.forget();
|
||||
return created;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
import 'reflect-metadata';
|
||||
|
||||
import { FolderInsightService } from './folder-insight.service';
|
||||
import { PathController } from './path.controller';
|
||||
import { PathModule } from './path.module';
|
||||
import { PathPolicyService } from './path-policy.service';
|
||||
import { PathRepository } from './path.repository';
|
||||
import { PathService } from './path.service';
|
||||
|
||||
describe('PathModule', () => {
|
||||
it('registers path controller and service', () => {
|
||||
it('registers the path controller, its services and the library folder repository', () => {
|
||||
expect(Reflect.getMetadata('controllers', PathModule)).toEqual([PathController]);
|
||||
expect(Reflect.getMetadata('providers', PathModule)).toEqual([PathPolicyService, PathService]);
|
||||
expect(Reflect.getMetadata('providers', PathModule)).toEqual([PathPolicyService, PathService, PathRepository, FolderInsightService]);
|
||||
expect(Reflect.getMetadata('exports', PathModule)).toEqual([PathPolicyService]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { FolderInsightService } from './folder-insight.service';
|
||||
import { PathController } from './path.controller';
|
||||
import { PathPolicyService } from './path-policy.service';
|
||||
import { PathRepository } from './path.repository';
|
||||
import { PathService } from './path.service';
|
||||
|
||||
@Module({
|
||||
controllers: [PathController],
|
||||
providers: [PathPolicyService, PathService],
|
||||
providers: [PathPolicyService, PathService, PathRepository, FolderInsightService],
|
||||
exports: [PathPolicyService],
|
||||
})
|
||||
export class PathModule {}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { APP_FEATURES } from '@bookorbit/types';
|
||||
import { asc, eq } from 'drizzle-orm';
|
||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
|
||||
import { DB } from '../../db';
|
||||
import * as schema from '../../db/schema';
|
||||
import { libraries, libraryFolders } from '../../db/schema';
|
||||
|
||||
type Db = NodePgDatabase<typeof schema>;
|
||||
|
||||
export interface LibraryFolderRow {
|
||||
libraryId: number;
|
||||
name: string;
|
||||
icon: string | null;
|
||||
type: string;
|
||||
path: string;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class PathRepository {
|
||||
constructor(@Inject(DB) private readonly db: Db) {}
|
||||
|
||||
/** Every library's folders, for landmarks and overlap checks. Callers hold ManageLibraries. */
|
||||
findLibraryFolders(): Promise<LibraryFolderRow[]> {
|
||||
return this.db
|
||||
.select({ libraryId: libraries.id, name: libraries.name, icon: libraries.icon, type: libraries.type, path: libraryFolders.path })
|
||||
.from(libraryFolders)
|
||||
.innerJoin(libraries, eq(libraries.id, libraryFolders.libraryId))
|
||||
.where(APP_FEATURES.podcasts ? undefined : eq(libraries.type, 'books'))
|
||||
.orderBy(asc(libraries.displayOrder), asc(libraries.name), asc(libraryFolders.createdAt), asc(libraryFolders.id));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
import { chmod, mkdir, mkdtemp, realpath, rm, symlink, writeFile } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
|
||||
import { PathService } from './path.service';
|
||||
|
||||
const runningAsRoot = typeof process.getuid === 'function' && process.getuid() === 0;
|
||||
|
||||
describe('PathService folder picker routes', () => {
|
||||
let root: string;
|
||||
let service: PathService;
|
||||
const pathPolicy = {
|
||||
getBrowseRoot: vi.fn(),
|
||||
getAppDataPath: vi.fn(),
|
||||
getCanonicalAppDataPath: vi.fn(),
|
||||
resolveBrowsePath: vi.fn(),
|
||||
};
|
||||
const pathRepository = { findLibraryFolders: vi.fn() };
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await mkdtemp(join(tmpdir(), 'path-folders-'));
|
||||
await mkdir(join(root, 'books', 'Ebooks', 'Fiction'), { recursive: true });
|
||||
await mkdir(join(root, 'books', 'Podcasts'), { recursive: true });
|
||||
await mkdir(join(root, 'books', '.hidden'), { recursive: true });
|
||||
await mkdir(join(root, 'data', 'covers'), { recursive: true });
|
||||
await writeFile(join(root, 'books', 'Loose.epub'), 'x');
|
||||
await symlink(join(root, 'books', 'Ebooks'), join(root, 'books', 'Linked'));
|
||||
|
||||
pathPolicy.getBrowseRoot.mockReturnValue(root);
|
||||
pathPolicy.getAppDataPath.mockReturnValue(join(root, 'data'));
|
||||
pathPolicy.getCanonicalAppDataPath.mockResolvedValue(join(root, 'data'));
|
||||
pathPolicy.resolveBrowsePath.mockImplementation((path?: string) => Promise.resolve(path || root));
|
||||
pathRepository.findLibraryFolders.mockResolvedValue([]);
|
||||
service = new PathService(pathPolicy as never, pathRepository as never);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await chmod(join(root, 'books'), 0o755).catch(() => undefined);
|
||||
await rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('listFolder', () => {
|
||||
it('lists real folders with whether each has folders of its own, skipping files, hidden folders and symlinks', async () => {
|
||||
const listing = await service.listFolder(join(root, 'books'));
|
||||
|
||||
expect(listing.state).toBe('ok');
|
||||
expect(listing.entries.map((entry) => [entry.name, entry.hasChildren, entry.system, entry.appData])).toEqual([
|
||||
['Ebooks', true, false, false],
|
||||
['Podcasts', false, false, false],
|
||||
]);
|
||||
});
|
||||
|
||||
it('lists a folder of hundreds of folders by name alone, without peeking inside each one', async () => {
|
||||
for (let i = 0; i < 401; i++) await mkdir(join(root, 'books', 'Ebooks', 'Fiction', `Author ${i}`));
|
||||
|
||||
const listing = await service.listFolder(join(root, 'books', 'Ebooks', 'Fiction'));
|
||||
|
||||
expect(listing.entries).toHaveLength(401);
|
||||
expect(listing.entries.every((entry) => entry.hasChildren === null)).toBe(true);
|
||||
});
|
||||
|
||||
it('lists only the first folders by name from a folder of thousands, and says so', async () => {
|
||||
const crowded = join(root, 'books', 'Ebooks', 'Fiction');
|
||||
for (let i = 0; i <= 5_000; i++) await mkdir(join(crowded, `Author ${String(i).padStart(4, '0')}`));
|
||||
|
||||
const listing = await service.listFolder(crowded);
|
||||
|
||||
expect(listing.truncated).toBe(true);
|
||||
expect(listing.entries).toHaveLength(5_000);
|
||||
expect(listing.entries.at(-1)?.name).toBe('Author 4999');
|
||||
});
|
||||
|
||||
it('marks BookOrbit’s data folder', async () => {
|
||||
const listing = await service.listFolder(root);
|
||||
|
||||
expect(listing.truncated).toBe(false);
|
||||
expect(listing.entries.find((entry) => entry.name === 'data')?.appData).toBe(true);
|
||||
expect(listing.entries.find((entry) => entry.name === 'books')?.appData).toBe(false);
|
||||
});
|
||||
|
||||
it('marks the data folder when the listing reaches it through a symlink', async () => {
|
||||
await mkdir(join(root, 'real', 'inner', 'data'), { recursive: true });
|
||||
await symlink(join(root, 'real'), join(root, 'via'));
|
||||
pathPolicy.getAppDataPath.mockReturnValue(join(root, 'real', 'inner', 'data'));
|
||||
pathPolicy.getCanonicalAppDataPath.mockResolvedValue(await realpath(join(root, 'real', 'inner', 'data')));
|
||||
|
||||
const listing = await service.listFolder(join(root, 'via', 'inner'));
|
||||
|
||||
expect(listing.entries).toMatchObject([{ name: 'data', appData: true }]);
|
||||
});
|
||||
|
||||
it('says a missing folder is missing rather than empty', async () => {
|
||||
await expect(service.listFolder(join(root, 'gone'))).resolves.toEqual({
|
||||
path: join(root, 'gone'),
|
||||
state: 'missing',
|
||||
entries: [],
|
||||
truncated: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('never opens blocked system paths', async () => {
|
||||
pathPolicy.resolveBrowsePath.mockResolvedValue('/proc');
|
||||
|
||||
await expect(service.listFolder('/proc')).resolves.toEqual({ path: '/proc', state: 'blocked', entries: [], truncated: false });
|
||||
});
|
||||
|
||||
it('refuses a symlinked folder', async () => {
|
||||
await expect(service.listFolder(join(root, 'books', 'Linked'))).resolves.toMatchObject({ state: 'blocked', entries: [] });
|
||||
});
|
||||
|
||||
it('refuses a folder reached through a symlink into a blocked system path', async () => {
|
||||
await symlink('/dev', join(root, 'dev-link'));
|
||||
|
||||
await expect(service.listFolder(join(root, 'dev-link', 'fd'))).resolves.toMatchObject({ state: 'blocked', entries: [] });
|
||||
});
|
||||
|
||||
it.skipIf(runningAsRoot)('says a folder it may not read is denied', async () => {
|
||||
await chmod(join(root, 'books'), 0o000);
|
||||
|
||||
await expect(service.listFolder(join(root, 'books'))).resolves.toMatchObject({ state: 'denied', entries: [] });
|
||||
});
|
||||
});
|
||||
|
||||
describe('searchFolders', () => {
|
||||
it('finds folders by name anywhere below, ignoring case', async () => {
|
||||
await expect(service.searchFolders('FICT')).resolves.toEqual({ results: [join(root, 'books', 'Ebooks', 'Fiction')], truncated: false });
|
||||
});
|
||||
|
||||
it('returns nothing for a blank query', async () => {
|
||||
await expect(service.searchFolders(' ')).resolves.toEqual({ results: [], truncated: false });
|
||||
});
|
||||
|
||||
it('does not follow symlinks or hidden folders', async () => {
|
||||
const { results } = await service.searchFolders('e');
|
||||
|
||||
expect(results.some((path) => path.includes('Linked') || path.includes('.hidden'))).toBe(false);
|
||||
});
|
||||
|
||||
it('does not search below a symlink into a blocked system path', async () => {
|
||||
await symlink('/dev', join(root, 'dev-link'));
|
||||
|
||||
await expect(service.searchFolders('fd', join(root, 'dev-link'))).resolves.toEqual({ results: [], truncated: false });
|
||||
});
|
||||
|
||||
it('matches a name stored decomposed when the query is typed composed', async () => {
|
||||
await mkdir(join(root, 'books', 'Cafe\u0301 Stories'));
|
||||
|
||||
const { results } = await service.searchFolders('café');
|
||||
|
||||
expect(results).toEqual([join(root, 'books', 'Cafe\u0301 Stories')]);
|
||||
});
|
||||
|
||||
it('says the results may be incomplete when folders lie deeper than it looks', async () => {
|
||||
await mkdir(join(root, 'books', 'a', 'b', 'c', 'd', 'e', 'f', 'Deep Find'), { recursive: true });
|
||||
|
||||
await expect(service.searchFolders('deep find')).resolves.toEqual({ results: [], truncated: true });
|
||||
});
|
||||
});
|
||||
|
||||
describe('getPlaces', () => {
|
||||
it('groups every library’s folders and offers the root when nothing is mounted separately', async () => {
|
||||
pathRepository.findLibraryFolders.mockResolvedValue([
|
||||
{ libraryId: 1, name: 'Novels', icon: 'BookOpen', type: 'books', path: join(root, 'books', 'Ebooks', 'Fiction') },
|
||||
{ libraryId: 2, name: 'Podcasts', icon: null, type: 'podcasts', path: join(root, 'books', 'Podcasts') },
|
||||
{ libraryId: 1, name: 'Novels', icon: 'BookOpen', type: 'books', path: join(root, 'books', 'Extra') },
|
||||
]);
|
||||
|
||||
await expect(service.getPlaces()).resolves.toEqual({
|
||||
root,
|
||||
mounts: [root],
|
||||
appDataPath: join(root, 'data'),
|
||||
libraries: [
|
||||
{
|
||||
id: 1,
|
||||
name: 'Novels',
|
||||
icon: 'BookOpen',
|
||||
type: 'books',
|
||||
folders: [join(root, 'books', 'Ebooks', 'Fiction'), join(root, 'books', 'Extra')],
|
||||
},
|
||||
{ id: 2, name: 'Podcasts', icon: null, type: 'podcasts', folders: [join(root, 'books', 'Podcasts')] },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it('leaves out a data folder that sits outside the browse root', async () => {
|
||||
pathPolicy.getCanonicalAppDataPath.mockResolvedValue('/elsewhere/data');
|
||||
|
||||
await expect(service.getPlaces()).resolves.toMatchObject({ appDataPath: null });
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -6,7 +6,7 @@ vi.mock('fs/promises', () => ({
|
||||
}));
|
||||
|
||||
import { BadRequestException, ConflictException, ForbiddenException } from '@nestjs/common';
|
||||
import { readdir, lstat, mkdir } from 'fs/promises';
|
||||
import { readdir, lstat, mkdir, realpath } from 'fs/promises';
|
||||
|
||||
import { PathService } from './path.service';
|
||||
|
||||
@@ -39,6 +39,7 @@ describe('PathService', () => {
|
||||
pathPolicy.getBrowseRoot.mockReturnValue('/tmp/books');
|
||||
pathPolicy.resolveBrowsePath.mockImplementation((path?: string) => Promise.resolve(path || '/tmp/books'));
|
||||
pathPolicy.assertWithinBrowseRoot.mockImplementation((path: string) => Promise.resolve(path));
|
||||
vi.mocked(realpath).mockImplementation((path) => Promise.resolve(String(path)));
|
||||
service = new PathService(pathPolicy as never);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
import type { CreateFolderResult, DirectoryEntry, PathConfig } from '@bookorbit/types';
|
||||
import type {
|
||||
CreateFolderResult,
|
||||
DirectoryEntry,
|
||||
FolderListing,
|
||||
FolderListingEntry,
|
||||
FolderPlaces,
|
||||
FolderPickerLibrary,
|
||||
FolderSearchResult,
|
||||
PathConfig,
|
||||
} from '@bookorbit/types';
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
@@ -8,19 +17,46 @@ import {
|
||||
InternalServerErrorException,
|
||||
Logger,
|
||||
} from '@nestjs/common';
|
||||
import { lstat, mkdir, readdir } from 'fs/promises';
|
||||
import type { Dirent } from 'fs';
|
||||
import { lstat, mkdir, readdir, realpath, stat } from 'fs/promises';
|
||||
import { basename, isAbsolute, join, relative, resolve, sep } from 'path';
|
||||
|
||||
import { sanitizeLogValue } from '../../common/utils/log-sanitize.utils';
|
||||
import {
|
||||
hasListableChild,
|
||||
isBlockedPath,
|
||||
isBlockedTarget,
|
||||
isInsideOrAt,
|
||||
isListableDirectory,
|
||||
mapLimit,
|
||||
realTarget,
|
||||
stateForError,
|
||||
SYSTEM_FOLDERS,
|
||||
} from './folder-walk';
|
||||
import { PathPolicyService } from './path-policy.service';
|
||||
import { PathRepository } from './path.repository';
|
||||
|
||||
const BLOCKED = ['/proc', '/sys', '/dev', '/run', '/var/run', '/etc', '/root'];
|
||||
/** Past this many folders, children are not opened to learn whether they have their own. */
|
||||
const HAS_CHILDREN_CHECK_LIMIT = 400;
|
||||
/** Folders listed from one folder; the rest are reached by typing a path or finding by name. */
|
||||
const LISTING_MAX_ENTRIES = 5_000;
|
||||
const SEARCH_MAX_DEPTH = 6;
|
||||
const SEARCH_MAX_RESULTS = 200;
|
||||
const SEARCH_MAX_ENTRIES = 20_000;
|
||||
const SEARCH_BUDGET_MS = 2_000;
|
||||
const MOUNT_SCAN_FOLDERS_PER_LEVEL = 50;
|
||||
const MOUNT_SCAN_MAX_STATS = 400;
|
||||
const MOUNT_CACHE_TTL_MS = 5 * 60_000;
|
||||
|
||||
@Injectable()
|
||||
export class PathService {
|
||||
private readonly logger = new Logger(PathService.name);
|
||||
private mountCache: { root: string; mounts: string[]; expires: number } | null = null;
|
||||
|
||||
constructor(private readonly pathPolicy: PathPolicyService) {}
|
||||
constructor(
|
||||
private readonly pathPolicy: PathPolicyService,
|
||||
private readonly pathRepository?: PathRepository,
|
||||
) {}
|
||||
|
||||
getConfig(): PathConfig {
|
||||
return { root: this.pathPolicy.getBrowseRoot() };
|
||||
@@ -32,7 +68,7 @@ export class PathService {
|
||||
const relativePath = relative(browseRoot, resolved);
|
||||
if (relativePath === '..' || relativePath.startsWith(`..${sep}`) || isAbsolute(relativePath)) return [];
|
||||
|
||||
if (this.isBlocked(resolved)) {
|
||||
if (await this.isBlocked(resolved)) {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
@@ -62,7 +98,7 @@ export class PathService {
|
||||
throw new ForbiddenException('Cannot create a folder outside the configured library browse root');
|
||||
}
|
||||
|
||||
if (this.isBlocked(parent)) {
|
||||
if (await this.isBlocked(parent)) {
|
||||
throw new ForbiddenException('Cannot create a folder in this location');
|
||||
}
|
||||
|
||||
@@ -94,8 +130,168 @@ export class PathService {
|
||||
}
|
||||
}
|
||||
|
||||
private isBlocked(resolved: string): boolean {
|
||||
return BLOCKED.some((b) => resolved === b || resolved.startsWith(b + sep));
|
||||
/**
|
||||
* One folder's subfolders with what the picker shows beside them. Unlike `listDirectories`, a folder
|
||||
* that cannot be read says so instead of looking empty.
|
||||
*/
|
||||
async listFolder(rawPath?: string): Promise<FolderListing> {
|
||||
const resolved = resolve(await this.pathPolicy.resolveBrowsePath(rawPath));
|
||||
const blocked: FolderListing = { path: resolved, state: 'blocked', entries: [], truncated: false };
|
||||
if (await this.isBlocked(resolved)) return blocked;
|
||||
|
||||
let dirents: Dirent[];
|
||||
let parentDev: number;
|
||||
let realDir: string;
|
||||
try {
|
||||
const rootStat = await lstat(resolved);
|
||||
if (rootStat.isSymbolicLink()) return blocked;
|
||||
if (!rootStat.isDirectory()) return { ...blocked, state: 'missing' };
|
||||
parentDev = rootStat.dev;
|
||||
dirents = await readdir(resolved, { withFileTypes: true });
|
||||
realDir = await realpath(resolved);
|
||||
} catch (error) {
|
||||
return { ...blocked, state: stateForError(error) };
|
||||
}
|
||||
|
||||
const rawAppData = this.pathPolicy.getAppDataPath();
|
||||
const appDataPath = await this.pathPolicy.getCanonicalAppDataPath();
|
||||
// The data folder may be reached through a symlink in either path, so both spellings are compared.
|
||||
const isAppData = (full: string, name: string) =>
|
||||
(!!rawAppData && isInsideOrAt(full, rawAppData)) ||
|
||||
(!!appDataPath && (isInsideOrAt(full, appDataPath) || isInsideOrAt(join(realDir, name), appDataPath)));
|
||||
const listable = dirents.filter((entry) => isListableDirectory(entry, join(resolved, entry.name)) && !isBlockedPath(join(realDir, entry.name)));
|
||||
const truncated = listable.length > LISTING_MAX_ENTRIES;
|
||||
const dirs = truncated ? listable.sort((a, b) => a.name.localeCompare(b.name)).slice(0, LISTING_MAX_ENTRIES) : listable;
|
||||
// A folder of thousands of author folders is listed by name alone: no per-child stat or peek.
|
||||
const inspectChildren = dirs.length <= HAS_CHILDREN_CHECK_LIMIT;
|
||||
const entries = await mapLimit(dirs, 16, async (entry): Promise<FolderListingEntry> => {
|
||||
const full = join(resolved, entry.name);
|
||||
let mount = false;
|
||||
if (inspectChildren) {
|
||||
try {
|
||||
mount = (await stat(full)).dev !== parentDev;
|
||||
} catch {
|
||||
// An unreadable folder is still listed; opening it reports why.
|
||||
}
|
||||
}
|
||||
return {
|
||||
name: entry.name,
|
||||
path: full,
|
||||
mount,
|
||||
system: resolved === sep && SYSTEM_FOLDERS.has(entry.name),
|
||||
appData: isAppData(full, entry.name),
|
||||
hasChildren: inspectChildren ? await hasListableChild(full) : null,
|
||||
};
|
||||
});
|
||||
return { path: resolved, state: 'ok', entries: entries.sort((a, b) => a.name.localeCompare(b.name)), truncated };
|
||||
}
|
||||
|
||||
/** Folders whose name contains `query`, breadth-first below `rawPath`, within fixed bounds. */
|
||||
async searchFolders(query: string, rawPath?: string, includeSystem = false): Promise<FolderSearchResult> {
|
||||
const root = resolve(await this.pathPolicy.resolveBrowsePath(rawPath));
|
||||
// Names are compared composed and case-folded the same way on every server, whatever its locale.
|
||||
const fold = (name: string) => name.normalize('NFC').toLowerCase();
|
||||
const needle = fold(query.trim());
|
||||
const results: string[] = [];
|
||||
if (!needle || (await this.isBlocked(root))) return { results, truncated: false };
|
||||
|
||||
const deadline = Date.now() + SEARCH_BUDGET_MS;
|
||||
// Each folder travels with its real path, so a symlink in the start path cannot lead into a blocked folder.
|
||||
let queue = [{ dir: root, real: await realTarget(root) }];
|
||||
let entries = 0;
|
||||
for (let depth = 0; depth < SEARCH_MAX_DEPTH && queue.length > 0; depth++) {
|
||||
const next: typeof queue = [];
|
||||
for (const { dir, real } of queue) {
|
||||
if (results.length >= SEARCH_MAX_RESULTS || entries >= SEARCH_MAX_ENTRIES || Date.now() > deadline) return { results, truncated: true };
|
||||
let dirents: Dirent[];
|
||||
try {
|
||||
dirents = await readdir(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
entries += dirents.length;
|
||||
dirents.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
||||
for (const entry of dirents) {
|
||||
const full = join(dir, entry.name);
|
||||
const childReal = join(real, entry.name);
|
||||
if (!isListableDirectory(entry, full) || isBlockedPath(childReal)) continue;
|
||||
if (!includeSystem && dir === sep && SYSTEM_FOLDERS.has(entry.name)) continue;
|
||||
if (fold(entry.name).includes(needle)) {
|
||||
results.push(full);
|
||||
if (results.length >= SEARCH_MAX_RESULTS) return { results, truncated: true };
|
||||
}
|
||||
next.push({ dir: full, real: childReal });
|
||||
}
|
||||
}
|
||||
queue = next;
|
||||
}
|
||||
// Folders deeper than the search goes were never looked at.
|
||||
return { results, truncated: queue.length > 0 };
|
||||
}
|
||||
|
||||
/** Where a picker starts: mounted folders near the browse root, the data folder, every library's folders. */
|
||||
async getPlaces(): Promise<FolderPlaces> {
|
||||
const root = resolve(this.pathPolicy.getBrowseRoot());
|
||||
const appDataPath = await this.pathPolicy.getCanonicalAppDataPath();
|
||||
const mounts = await this.findMounts(root);
|
||||
const rows = (await this.pathRepository?.findLibraryFolders()) ?? [];
|
||||
const libraries = new Map<number, FolderPickerLibrary>();
|
||||
for (const row of rows) {
|
||||
const library = libraries.get(row.libraryId) ?? { id: row.libraryId, name: row.name, icon: row.icon, type: row.type, folders: [] };
|
||||
library.folders.push(row.path);
|
||||
libraries.set(row.libraryId, library);
|
||||
}
|
||||
|
||||
return {
|
||||
root,
|
||||
mounts: mounts.length > 0 || root === sep ? mounts : [root],
|
||||
appDataPath: appDataPath && isInsideOrAt(appDataPath, root) ? appDataPath : null,
|
||||
libraries: [...libraries.values()],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Folders on their own filesystem within two levels of the root. Mounts change with a container
|
||||
* restart, not between picker opens, so the answer is cached; the walk reads only the first folders
|
||||
* of each level, so a root holding thousands of author folders costs the same as `/`.
|
||||
*/
|
||||
private async findMounts(root: string): Promise<string[]> {
|
||||
if (this.mountCache && this.mountCache.root === root && this.mountCache.expires > Date.now()) return this.mountCache.mounts;
|
||||
const mounts: string[] = [];
|
||||
let stats = 0;
|
||||
const visit = async (dir: string, dev: number, depth: number) => {
|
||||
let dirents: Dirent[];
|
||||
try {
|
||||
dirents = await readdir(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
dirents.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
||||
const dirs = dirents.filter((entry) => isListableDirectory(entry, join(dir, entry.name)) && !(dir === sep && SYSTEM_FOLDERS.has(entry.name)));
|
||||
for (const entry of dirs.slice(0, MOUNT_SCAN_FOLDERS_PER_LEVEL)) {
|
||||
if (stats++ >= MOUNT_SCAN_MAX_STATS) return;
|
||||
const full = join(dir, entry.name);
|
||||
let childDev: number;
|
||||
try {
|
||||
childDev = (await stat(full)).dev;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (childDev !== dev) mounts.push(full);
|
||||
else if (depth < 2) await visit(full, dev, depth + 1);
|
||||
}
|
||||
};
|
||||
try {
|
||||
await visit(root, (await stat(root)).dev, 1);
|
||||
} catch {
|
||||
// An unreadable root has no places to offer.
|
||||
}
|
||||
this.mountCache = { root, mounts, expires: Date.now() + MOUNT_CACHE_TTL_MS };
|
||||
return mounts;
|
||||
}
|
||||
|
||||
private isBlocked(resolved: string): Promise<boolean> {
|
||||
return isBlockedTarget(resolved);
|
||||
}
|
||||
|
||||
private assertSafeName(rawName: string): string {
|
||||
|
||||
@@ -414,6 +414,28 @@ describe('disc folder flattening', () => {
|
||||
expect(candidates).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("never folds a library folder whose own name looks like a disc folder, such as unRAID's disk1", async () => {
|
||||
await file('disk1/Dune.epub');
|
||||
await file('disk1/Emma.epub');
|
||||
await file('disk1/Saga/Saga 001.cbz');
|
||||
const library = join(root, 'disk1');
|
||||
|
||||
const { candidates } = await findBookCandidates(library);
|
||||
|
||||
expect(candidatePaths(candidates)).toEqual([join(library, 'Dune.epub'), join(library, 'Emma.epub'), join(library, 'Saga')]);
|
||||
});
|
||||
|
||||
it('still reads disc folders directly inside a disc-named library folder as loose files', async () => {
|
||||
await file('Part 1/Intro.mp3');
|
||||
await file('Part 1/CD 1/01.mp3');
|
||||
await file('Part 1/CD 1/02.mp3');
|
||||
const library = join(root, 'Part 1');
|
||||
|
||||
const { candidates } = await findBookCandidates(library);
|
||||
|
||||
expect(candidatePaths(candidates)).toEqual([join(library, 'CD 1', '01.mp3'), join(library, 'CD 1', '02.mp3'), join(library, 'Intro.mp3')]);
|
||||
});
|
||||
|
||||
it('does NOT treat names like "Discography" as disc directories', async () => {
|
||||
await file('AudioBook/Discography/01.mp3');
|
||||
await file('AudioBook/cover.jpg');
|
||||
|
||||
@@ -146,6 +146,8 @@ async function collectByDir(
|
||||
// that turn this dir into a stem-named subdir needing flattening).
|
||||
// Root is exempt: root-level files are never stem-merged into a parent (see
|
||||
// buildBookCandidates: parent === libraryFolderPath is explicitly skipped).
|
||||
// A disc-named library folder is no longer flattened, but it stays unskippable: libraries
|
||||
// scanned while it was are regrouped by their next scan, whatever its mtime.
|
||||
const hasDiscSubdirs = subdirectoryPaths.some((s) => isDiscDirectory(basename(s)));
|
||||
const canSkip = knownDirMtimes && unchangedDirs && dirMtimes && filePaths.length > 0 && !isDiscDirectory(basename(dir)) && !hasDiscSubdirs;
|
||||
|
||||
@@ -207,9 +209,11 @@ export async function findBookCandidates(
|
||||
|
||||
// Flatten disc subdirectories (e.g. "CD 1", "Disc 2") into their parent.
|
||||
// Collect disc dirs first to avoid mutating the map while iterating.
|
||||
// The library folder itself is never flattened, whatever its name: unRAID mounts its drives
|
||||
// as /mnt/disk1, and folding one would file its books under the folder above the library.
|
||||
const discDirs: string[] = [];
|
||||
for (const [dir] of byDir) {
|
||||
if (isDiscDirectory(basename(dir))) {
|
||||
if (dir !== libraryFolderPath && isDiscDirectory(basename(dir))) {
|
||||
discDirs.push(dir);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -165,6 +165,38 @@ export class ScannerRepository {
|
||||
.where(eq(books.libraryFolderId, libraryFolderId));
|
||||
}
|
||||
|
||||
/** Books filed under a library folder but located at `folderPath`, with their content files. */
|
||||
async findBooksAtFolderPath(libraryFolderId: number, folderPath: string) {
|
||||
const rows = await this.db
|
||||
.select({ id: books.id, libraryId: books.libraryId, primaryFileId: books.primaryFileId })
|
||||
.from(books)
|
||||
.where(and(eq(books.libraryFolderId, libraryFolderId), eq(books.folderPath, folderPath)));
|
||||
if (rows.length === 0) return [];
|
||||
const files = await this.db
|
||||
.select({ id: bookFiles.id, bookId: bookFiles.bookId, absolutePath: bookFiles.absolutePath })
|
||||
.from(bookFiles)
|
||||
.where(
|
||||
and(
|
||||
inArray(
|
||||
bookFiles.bookId,
|
||||
rows.map((row) => row.id),
|
||||
),
|
||||
eq(bookFiles.role, 'content'),
|
||||
),
|
||||
)
|
||||
.orderBy(asc(bookFiles.sortOrder), asc(bookFiles.id));
|
||||
return rows.map((row) => ({ ...row, contentFiles: files.filter((file) => file.bookId === row.id) }));
|
||||
}
|
||||
|
||||
async bookExistsAtFolderPath(libraryId: number, folderPath: string): Promise<boolean> {
|
||||
const [row] = await this.db
|
||||
.select({ id: books.id })
|
||||
.from(books)
|
||||
.where(and(eq(books.libraryId, libraryId), eq(books.folderPath, folderPath)))
|
||||
.limit(1);
|
||||
return !!row;
|
||||
}
|
||||
|
||||
async findPrimaryBookFilesByLibrary(libraryId: number) {
|
||||
return this.db
|
||||
.select({ bookId: books.id, absolutePath: bookFiles.absolutePath, format: bookFiles.format })
|
||||
|
||||
@@ -46,6 +46,7 @@ import {
|
||||
findBookCandidates,
|
||||
findLooseFileCandidates,
|
||||
buildSingleBookCandidate,
|
||||
isDiscDirectory,
|
||||
type WalkResult,
|
||||
} from './lib/walk';
|
||||
import { ScannerRepository } from './scanner.repository';
|
||||
@@ -1132,6 +1133,38 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* A library folder named like a disc folder ("disk1", "CD 1") used to be flattened into the folder
|
||||
* above it, so its loose files became one book filed there. Each is now its own book: the old book
|
||||
* moves to its primary file, keeping its progress, notes and edits, and the scan that follows gives
|
||||
* the other files books of their own.
|
||||
*/
|
||||
private async repairFlattenedLibraryFolder(libraryId: number, folder: { id: number; path: string }): Promise<void> {
|
||||
if (!isDiscDirectory(basename(folder.path))) return;
|
||||
const event = 'scanner.repair_flattened_library_folder';
|
||||
try {
|
||||
const stranded = await this.scannerRepo.findBooksAtFolderPath(folder.id, dirname(folder.path));
|
||||
for (const book of stranded) {
|
||||
const inside = book.contentFiles.filter((file) => file.absolutePath.startsWith(folder.path + sep));
|
||||
const target = inside.find((file) => file.id === book.primaryFileId) ?? inside[0];
|
||||
if (!target || (await this.scannerRepo.bookExistsAtFolderPath(book.libraryId, target.absolutePath))) {
|
||||
this.logger.warn(
|
||||
`[${event}] [fail] libraryId=${libraryId} bookId=${book.id} fileCount=${book.contentFiles.length} - no free file left to move the book to`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
await this.scannerRepo.updateBookFolderPath(book.id, target.absolutePath);
|
||||
this.logger.log(
|
||||
`[${event}] [end] libraryId=${libraryId} bookId=${book.id} folder="${sanitizeLogValue(target.absolutePath)}" fileCount=${book.contentFiles.length} - book moved from above its library folder to its primary file`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`[${event}] [fail] libraryId=${libraryId} libraryFolderId=${folder.id} errorClass=${err instanceof Error ? err.name : 'Error'} error="${sanitizeLogValue(err instanceof Error ? err.message : String(err))}" - flattened library folder repair failed`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async loadCandidateMaps(folderPath: string, libraryId: number): Promise<ScanLookupMaps> {
|
||||
const knownBooks = await this.scannerRepo.findBooksByFolderPath(folderPath, libraryId);
|
||||
const knownFiles = await this.scannerRepo.findBookFilesByBookIds(knownBooks.map((b) => b.id));
|
||||
@@ -1438,6 +1471,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
||||
this.emitFromStore(libraryId, jobId, 'running');
|
||||
|
||||
for (const folder of folders) {
|
||||
await this.repairFlattenedLibraryFolder(libraryId, folder);
|
||||
let candidates: BookCandidate[] = [];
|
||||
let skippedDirs = new Set<string>();
|
||||
let unchangedDirs = new Set<string>();
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"totalRoutes": 712,
|
||||
"totalRoutes": 719,
|
||||
"byPermission": {
|
||||
"(none)": 298,
|
||||
"Permission.ManageMetadataConfig": 47,
|
||||
@@ -7,7 +7,7 @@
|
||||
"Permission.LibraryEditMetadata": 54,
|
||||
"Permission.BookDockAccess": 17,
|
||||
"Permission.ManageAppSettings": 62,
|
||||
"Permission.ManageLibraries": 24,
|
||||
"Permission.ManageLibraries": 31,
|
||||
"Permission.ManageUsers": 14,
|
||||
"Permission.ManageEmail": 8,
|
||||
"Permission.KoboSync": 13,
|
||||
|
||||
@@ -443,6 +443,90 @@
|
||||
"allowDefault": false,
|
||||
"useGuards": []
|
||||
},
|
||||
{
|
||||
"file": "server/src/modules/path/path.controller.ts",
|
||||
"className": "PathController",
|
||||
"methodName": "listFolder",
|
||||
"httpMethod": "GET",
|
||||
"path": "/path/listing",
|
||||
"permissions": ["Permission.ManageLibraries"],
|
||||
"libraryAccess": [],
|
||||
"isPublic": false,
|
||||
"allowDefault": false,
|
||||
"useGuards": []
|
||||
},
|
||||
{
|
||||
"file": "server/src/modules/path/path.controller.ts",
|
||||
"className": "PathController",
|
||||
"methodName": "summarize",
|
||||
"httpMethod": "POST",
|
||||
"path": "/path/summaries",
|
||||
"permissions": ["Permission.ManageLibraries"],
|
||||
"libraryAccess": [],
|
||||
"isPublic": false,
|
||||
"allowDefault": false,
|
||||
"useGuards": []
|
||||
},
|
||||
{
|
||||
"file": "server/src/modules/path/path.controller.ts",
|
||||
"className": "PathController",
|
||||
"methodName": "inspect",
|
||||
"httpMethod": "GET",
|
||||
"path": "/path/inspect",
|
||||
"permissions": ["Permission.ManageLibraries"],
|
||||
"libraryAccess": [],
|
||||
"isPublic": false,
|
||||
"allowDefault": false,
|
||||
"useGuards": []
|
||||
},
|
||||
{
|
||||
"file": "server/src/modules/path/path.controller.ts",
|
||||
"className": "PathController",
|
||||
"methodName": "search",
|
||||
"httpMethod": "GET",
|
||||
"path": "/path/search",
|
||||
"permissions": ["Permission.ManageLibraries"],
|
||||
"libraryAccess": [],
|
||||
"isPublic": false,
|
||||
"allowDefault": false,
|
||||
"useGuards": []
|
||||
},
|
||||
{
|
||||
"file": "server/src/modules/path/path.controller.ts",
|
||||
"className": "PathController",
|
||||
"methodName": "places",
|
||||
"httpMethod": "GET",
|
||||
"path": "/path/places",
|
||||
"permissions": ["Permission.ManageLibraries"],
|
||||
"libraryAccess": [],
|
||||
"isPublic": false,
|
||||
"allowDefault": false,
|
||||
"useGuards": []
|
||||
},
|
||||
{
|
||||
"file": "server/src/modules/path/path.controller.ts",
|
||||
"className": "PathController",
|
||||
"methodName": "unclaimed",
|
||||
"httpMethod": "GET",
|
||||
"path": "/path/unclaimed",
|
||||
"permissions": ["Permission.ManageLibraries"],
|
||||
"libraryAccess": [],
|
||||
"isPublic": false,
|
||||
"allowDefault": false,
|
||||
"useGuards": []
|
||||
},
|
||||
{
|
||||
"file": "server/src/modules/path/path.controller.ts",
|
||||
"className": "PathController",
|
||||
"methodName": "createFolder",
|
||||
"httpMethod": "POST",
|
||||
"path": "/path",
|
||||
"permissions": ["Permission.ManageLibraries"],
|
||||
"libraryAccess": [],
|
||||
"isPublic": false,
|
||||
"allowDefault": false,
|
||||
"useGuards": []
|
||||
},
|
||||
{
|
||||
"file": "server/src/modules/reader/cbz/cbz.controller.ts",
|
||||
"className": "CbzController",
|
||||
|
||||
@@ -269,7 +269,6 @@
|
||||
"POST /migration/sources/:id/path-mappings/validate",
|
||||
"POST /migration/sources/:id/validate",
|
||||
"POST /migration/sources/test",
|
||||
"POST /path",
|
||||
"POST /readwise/validate-token",
|
||||
"POST /server-fonts/upload",
|
||||
"POST /smart-scopes/:id/books/jump-buckets",
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
import { mkdir, writeFile } from 'fs/promises';
|
||||
import { dirname, join } from 'path';
|
||||
|
||||
import { eq } from 'drizzle-orm';
|
||||
|
||||
import { bookFiles, books } from '../src/db/schema';
|
||||
|
||||
import type { FixtureEntry } from './e2e/scanner/scanner-fixture-builder';
|
||||
import { createFixtureTree, file } from './e2e/scanner/scanner-fixture-builder';
|
||||
import {
|
||||
@@ -460,6 +464,71 @@ describe('Scanner structures (e2e)', () => {
|
||||
}
|
||||
});
|
||||
|
||||
describe('a library folder named like a disc folder', () => {
|
||||
// unRAID mounts its drives as /mnt/disk1; such a folder was once flattened into the folder above it.
|
||||
const entries = [file('disk1/Dune.epub'), file('disk1/Emma.epub'), file('disk1/Saga/Saga 001.cbz')];
|
||||
const expected = (library: string) =>
|
||||
[join(library, 'Dune.epub'), join(library, 'Emma.epub'), join(library, 'Saga')].map((folderPath) => ({ folderPath, status: 'present' }));
|
||||
|
||||
it(
|
||||
'reads its loose files as books of their own, inside the library',
|
||||
async () => {
|
||||
if (!context) throw new Error('E2E context not initialized');
|
||||
const fixture = await createFixtureTree(entries, 'scanner-e2e-disc-named-root-');
|
||||
try {
|
||||
const library = join(fixture.rootPath, 'disk1');
|
||||
const { libraryId } = await seedLibrary(context.db, { rootPath: library, mode: 'book_per_folder' });
|
||||
|
||||
await waitForScanCompletion(context.db, await triggerLibraryScan(context, libraryId));
|
||||
|
||||
const state = await loadLibraryBookState(context.db, libraryId);
|
||||
expect(state.map(({ folderPath, status }) => ({ folderPath, status })).sort((a, b) => a.folderPath.localeCompare(b.folderPath))).toEqual(
|
||||
expected(library),
|
||||
);
|
||||
await assertNoIntegrityViolations(context.db);
|
||||
} finally {
|
||||
await fixture.cleanup();
|
||||
}
|
||||
},
|
||||
SCENARIO_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
it(
|
||||
'regroups a library scanned while it was flattened, keeping the old book on its primary file',
|
||||
async () => {
|
||||
if (!context) throw new Error('E2E context not initialized');
|
||||
const fixture = await createFixtureTree(entries, 'scanner-e2e-disc-named-root-repair-');
|
||||
try {
|
||||
const library = join(fixture.rootPath, 'disk1');
|
||||
const { libraryId } = await seedLibrary(context.db, { rootPath: library, mode: 'book_per_folder' });
|
||||
await waitForScanCompletion(context.db, await triggerLibraryScan(context, libraryId));
|
||||
|
||||
// Rebuild what the flattening scan stored: one book above the library holding every loose file.
|
||||
const rows = await context.db.select({ id: books.id, folderPath: books.folderPath }).from(books).where(eq(books.libraryId, libraryId));
|
||||
const dune = rows.find((row) => row.folderPath === join(library, 'Dune.epub'))!;
|
||||
const emma = rows.find((row) => row.folderPath === join(library, 'Emma.epub'))!;
|
||||
await context.db.update(bookFiles).set({ bookId: dune.id }).where(eq(bookFiles.bookId, emma.id));
|
||||
await context.db.delete(books).where(eq(books.id, emma.id));
|
||||
await context.db.update(books).set({ folderPath: fixture.rootPath }).where(eq(books.id, dune.id));
|
||||
|
||||
await waitForScanCompletion(context.db, await triggerLibraryScan(context, libraryId));
|
||||
|
||||
const state = await loadLibraryBookState(context.db, libraryId);
|
||||
expect(state.map(({ folderPath, status }) => ({ folderPath, status })).sort((a, b) => a.folderPath.localeCompare(b.folderPath))).toEqual(
|
||||
expected(library),
|
||||
);
|
||||
const [kept] = await context.db.select({ folderPath: books.folderPath }).from(books).where(eq(books.id, dune.id));
|
||||
expect(kept?.folderPath).toBe(join(library, 'Dune.epub'));
|
||||
expect(state.find((book) => book.folderPath === join(library, 'Emma.epub'))?.filePaths).toEqual([join(library, 'Emma.epub')]);
|
||||
await assertNoIntegrityViolations(context.db);
|
||||
} finally {
|
||||
await fixture.cleanup();
|
||||
}
|
||||
},
|
||||
SCENARIO_TIMEOUT_MS,
|
||||
);
|
||||
});
|
||||
|
||||
describe('organizationMode=book_per_file', () => {
|
||||
for (const scenario of bookPerFileScenarios) {
|
||||
it(
|
||||
|
||||
Reference in New Issue
Block a user