feat(scanner): add live scan progress via WebSocket and auto-scan

This commit is contained in:
neon
2026-01-19 16:48:11 -06:00
parent 2033a3d719
commit 73d66b75e3
17 changed files with 541 additions and 73 deletions
+7 -6
View File
@@ -27,17 +27,18 @@
"pdfjs-dist": "^5.4.530", "pdfjs-dist": "^5.4.530",
"pinia": "^3.0.4", "pinia": "^3.0.4",
"reka-ui": "^2.7.0", "reka-ui": "^2.7.0",
"socket.io-client": "^4.8.3",
"tailwind-merge": "^3.4.0", "tailwind-merge": "^3.4.0",
"tailwindcss": "^4.1.18", "tailwindcss": "^4.1.18",
"vue": "^3.5.26", "vue": "^3.5.27",
"vue-router": "^4.6.4", "vue-router": "^4.6.4",
"vue-sonner": "^2.0.9", "vue-sonner": "^2.0.9",
"vue-virtual-scroller": "^2.0.0-beta.8" "vue-virtual-scroller": "^2.0.0-beta.8"
}, },
"devDependencies": { "devDependencies": {
"@tsconfig/node24": "^24.0.3", "@tsconfig/node24": "^24.0.4",
"@types/jsdom": "^27.0.0", "@types/jsdom": "^27.0.0",
"@types/node": "^22.19.6", "@types/node": "^22.19.7",
"@vitejs/plugin-vue": "^6.0.3", "@vitejs/plugin-vue": "^6.0.3",
"@vitest/eslint-plugin": "^1.6.6", "@vitest/eslint-plugin": "^1.6.6",
"@vue/eslint-config-typescript": "^14.6.0", "@vue/eslint-config-typescript": "^14.6.0",
@@ -45,12 +46,12 @@
"@vue/tsconfig": "^0.8.1", "@vue/tsconfig": "^0.8.1",
"eslint": "^9.39.2", "eslint": "^9.39.2",
"eslint-config-prettier": "^10.1.8", "eslint-config-prettier": "^10.1.8",
"eslint-plugin-oxlint": "~1.39.0", "eslint-plugin-oxlint": "~1.41.0",
"eslint-plugin-vue": "~10.6.2", "eslint-plugin-vue": "~10.7.0",
"jiti": "^2.6.1", "jiti": "^2.6.1",
"jsdom": "^27.4.0", "jsdom": "^27.4.0",
"npm-run-all2": "^8.0.4", "npm-run-all2": "^8.0.4",
"oxlint": "~1.39.0", "oxlint": "~1.41.0",
"prettier": "3.8.0", "prettier": "3.8.0",
"tw-animate-css": "^1.4.0", "tw-animate-css": "^1.4.0",
"typescript": "~5.9.3", "typescript": "~5.9.3",
@@ -0,0 +1,64 @@
import { ref } from 'vue'
import { io, Socket } from 'socket.io-client'
import type { ScanProgressEvent } from '@projectx/types'
import { getAccessToken } from '@/lib/api'
let socket: Socket | null = null
const progressMap = ref<Map<number, ScanProgressEvent>>(new Map())
const subscribedLibraries = new Set<number>()
function getSocket(): Socket {
if (!socket) {
// Use a callback so getAccessToken() is called at (re)connect time, not just once at creation.
socket = io('/scan', {
auth: (cb: (data: object) => void) => cb({ token: getAccessToken() }),
transports: ['websocket'],
autoConnect: true,
})
socket.on('scan:progress', (event: ScanProgressEvent) => {
progressMap.value.set(event.libraryId, event)
progressMap.value = new Map(progressMap.value)
if (event.status !== 'running') {
// Keep the final event for a short time so UI can show completion, then clear.
setTimeout(() => {
progressMap.value.delete(event.libraryId)
progressMap.value = new Map(progressMap.value)
}, 3000)
}
})
socket.on('disconnect', () => {
// Clear stale progress bars but keep subscribedLibraries so we can re-subscribe on reconnect.
progressMap.value = new Map()
})
socket.on('connect', () => {
// Re-subscribe to all libraries after reconnect (server-side rooms are per-socket).
for (const id of subscribedLibraries) {
socket!.emit('subscribe:library', id)
}
})
}
return socket
}
export { getSocket }
export function useScanProgress() {
function subscribeLibrary(libraryId: number): void {
if (subscribedLibraries.has(libraryId)) return
subscribedLibraries.add(libraryId)
getSocket().emit('subscribe:library', libraryId)
}
function getProgress(libraryId: number): ScanProgressEvent | undefined {
return progressMap.value.get(libraryId)
}
function isScanning(libraryId: number): boolean {
return progressMap.value.get(libraryId)?.status === 'running'
}
return { subscribeLibrary, getProgress, isScanning, progressMap }
}
@@ -1,5 +1,5 @@
<script setup lang="ts"> <script setup lang="ts">
import { onMounted, ref } from 'vue' import { onMounted, ref, watch } from 'vue'
import { useRoute, useRouter } from 'vue-router' import { useRoute, useRouter } from 'vue-router'
import { FolderOpen, Plus, RefreshCw, Pencil, Trash2, Library } from 'lucide-vue-next' import { FolderOpen, Plus, RefreshCw, Pencil, Trash2, Library } from 'lucide-vue-next'
import { toast } from 'vue-sonner' import { toast } from 'vue-sonner'
@@ -7,16 +7,18 @@ import { api } from '@/lib/api'
import type { Library as LibraryType, LibraryStats } from '@projectx/types' import type { Library as LibraryType, LibraryStats } from '@projectx/types'
import LibraryCreatorModal from '@/features/library/components/LibraryCreatorModal.vue' import LibraryCreatorModal from '@/features/library/components/LibraryCreatorModal.vue'
import { useLibraries } from '@/features/library/composables/useLibraries' import { useLibraries } from '@/features/library/composables/useLibraries'
import { useScanProgress, getSocket } from '@/features/scanner/composables/useScanProgress'
const route = useRoute() const route = useRoute()
const router = useRouter() const router = useRouter()
const { libraries, fetchLibraries } = useLibraries() const { libraries, fetchLibraries } = useLibraries()
const { subscribeLibrary, getProgress, isScanning, progressMap } = useScanProgress()
const stats = ref<Record<number, LibraryStats>>({}) const stats = ref<Record<number, LibraryStats>>({})
const scanning = ref<Record<number, boolean>>({})
const scanningAll = ref(false) const scanningAll = ref(false)
const creatorOpen = ref(false) const creatorOpen = ref(false)
const editingLibrary = ref<LibraryType | null>(null) const editingLibrary = ref<LibraryType | null>(null)
const pendingNavigateLibraryId = ref<number | null>(null)
const deletingLibrary = ref<LibraryType | null>(null) const deletingLibrary = ref<LibraryType | null>(null)
const deleteConfirmName = ref('') const deleteConfirmName = ref('')
const deleting = ref(false) const deleting = ref(false)
@@ -30,24 +32,48 @@ async function loadAllStats() {
) )
} }
function subscribeAll() {
for (const lib of libraries.value) {
subscribeLibrary(lib.id)
}
}
onMounted(async () => { onMounted(async () => {
getSocket()
await fetchLibraries() await fetchLibraries()
subscribeAll()
loadAllStats() loadAllStats()
}) })
const statsReloadedFor = new Set<number>()
watch(progressMap, (map) => {
for (const [libraryId, event] of map) {
if (event.status === 'completed' && !statsReloadedFor.has(libraryId)) {
statsReloadedFor.add(libraryId)
api(`/api/libraries/${libraryId}/stats`).then(async (res) => {
if (res.ok) stats.value[libraryId] = await res.json()
setTimeout(() => statsReloadedFor.delete(libraryId), 5000)
})
if (pendingNavigateLibraryId.value === libraryId) {
pendingNavigateLibraryId.value = null
router.push({ name: 'library', params: { id: libraryId } })
}
}
}
})
async function scan(lib: LibraryType) { async function scan(lib: LibraryType) {
scanning.value[lib.id] = true
try { try {
const res = await api(`/api/scanner/libraries/${lib.id}/scan`, { method: 'POST' }) const res = await api(`/api/scanner/libraries/${lib.id}/scan`, { method: 'POST' })
if (res.ok) { if (res.ok) {
toast.success(`Scan started for "${lib.name}"`) toast.success(`Scan started for "${lib.name}"`)
subscribeLibrary(lib.id)
} else { } else {
toast.error(`Failed to start scan for "${lib.name}"`) toast.error(`Failed to start scan for "${lib.name}"`)
} }
} catch { } catch {
toast.error(`Failed to start scan for "${lib.name}"`) toast.error(`Failed to start scan for "${lib.name}"`)
} finally {
scanning.value[lib.id] = false
} }
} }
@@ -58,6 +84,7 @@ async function scanAll() {
const failed = results.filter((r) => !r.ok).length const failed = results.filter((r) => !r.ok).length
if (failed === 0) { if (failed === 0) {
toast.success('Scan started for all libraries') toast.success('Scan started for all libraries')
subscribeAll()
} else { } else {
toast.error(`${failed} librar${failed === 1 ? 'y' : 'ies'} failed to start`) toast.error(`${failed} librar${failed === 1 ? 'y' : 'ies'} failed to start`)
} }
@@ -79,8 +106,11 @@ function openEdit(lib: LibraryType) {
} }
async function onSaved(library: LibraryType) { async function onSaved(library: LibraryType) {
const isNew = !editingLibrary.value
creatorOpen.value = false creatorOpen.value = false
editingLibrary.value = null editingLibrary.value = null
subscribeLibrary(library.id)
if (isNew) pendingNavigateLibraryId.value = library.id
await fetchLibraries() await fetchLibraries()
loadAllStats() loadAllStats()
} }
@@ -129,8 +159,20 @@ function formatBytes(bytes: number): string {
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}` return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`
} }
function scanProgressLabel(libraryId: number): string {
const p = getProgress(libraryId)
if (!p) return ''
if (p.status === 'running') {
if (p.total === 0) return 'Scanning...'
const pct = Math.floor((p.processed / p.total) * 100)
return `Scanning ${pct}% (${p.processed}/${p.total})`
}
if (p.status === 'completed') return `Done - ${p.added} added, ${p.updated} updated`
if (p.status === 'failed') return p.errorMessage ? `Failed: ${p.errorMessage}` : 'Scan failed'
return ''
}
function iconComponent(iconName: string | null | undefined) { function iconComponent(iconName: string | null | undefined) {
// Dynamic Lucide icon lookup via pre-imported default
return iconName ? null : Library return iconName ? null : Library
} }
</script> </script>
@@ -150,7 +192,7 @@ function iconComponent(iconName: string | null | undefined) {
@click="scanAll" @click="scanAll"
> >
<RefreshCw :size="12" :class="scanningAll ? 'animate-spin' : ''" /> <RefreshCw :size="12" :class="scanningAll ? 'animate-spin' : ''" />
{{ scanningAll ? 'Scanning…' : 'Scan All' }} {{ scanningAll ? 'Scanning...' : 'Scan All' }}
</button> </button>
<button <button
class="flex items-center gap-1.5 text-xs font-medium px-3 py-1.5 rounded-md bg-primary text-primary-foreground hover:opacity-90 transition-opacity" class="flex items-center gap-1.5 text-xs font-medium px-3 py-1.5 rounded-md bg-primary text-primary-foreground hover:opacity-90 transition-opacity"
@@ -166,11 +208,18 @@ function iconComponent(iconName: string | null | undefined) {
<div class="border border-border rounded-lg overflow-hidden divide-y divide-border"> <div class="border border-border rounded-lg overflow-hidden divide-y divide-border">
<div v-for="lib in libraries" :key="lib.id" class="bg-card px-5 py-4"> <div v-for="lib in libraries" :key="lib.id" class="bg-card px-5 py-4">
<div class="flex items-center gap-4"> <div class="flex items-center gap-4">
<div class="flex h-10 w-10 shrink-0 items-center justify-center rounded-lg bg-primary/10"> <RouterLink
:to="{ name: 'library', params: { id: lib.id } }"
class="flex h-10 w-10 shrink-0 items-center justify-center rounded-lg bg-primary/10 hover:bg-primary/20 transition-colors"
>
<FolderOpen :size="17" class="text-primary" /> <FolderOpen :size="17" class="text-primary" />
</div> </RouterLink>
<div class="flex-1 min-w-0"> <div class="flex-1 min-w-0">
<p class="text-sm font-medium text-foreground truncate">{{ lib.name }}</p> <RouterLink
:to="{ name: 'library', params: { id: lib.id } }"
class="text-sm font-medium text-foreground hover:text-primary transition-colors truncate block"
>{{ lib.name }}</RouterLink
>
<div class="flex items-center gap-3 mt-0.5"> <div class="flex items-center gap-3 mt-0.5">
<span v-if="stats[lib.id]" class="text-xs text-muted-foreground"> <span v-if="stats[lib.id]" class="text-xs text-muted-foreground">
{{ stats[lib.id].totalBooks }} book{{ stats[lib.id].totalBooks === 1 ? '' : 's' }} {{ stats[lib.id].totalBooks }} book{{ stats[lib.id].totalBooks === 1 ? '' : 's' }}
@@ -181,6 +230,24 @@ function iconComponent(iconName: string | null | undefined) {
</span> </span>
<span v-if="lib.watch" class="text-xs text-primary/70">Watching</span> <span v-if="lib.watch" class="text-xs text-primary/70">Watching</span>
</div> </div>
<!-- Scan progress bar -->
<div v-if="getProgress(lib.id)" class="mt-2">
<div class="flex items-center justify-between mb-1">
<span class="text-xs" :class="getProgress(lib.id)?.status === 'failed' ? 'text-destructive' : 'text-muted-foreground'">
{{ scanProgressLabel(lib.id) }}
</span>
</div>
<div v-if="getProgress(lib.id)?.status === 'running'" class="h-1 w-full rounded-full bg-muted overflow-hidden">
<div
class="h-full rounded-full bg-primary transition-all duration-300"
:style="{
width:
getProgress(lib.id)!.total > 0 ? `${Math.floor((getProgress(lib.id)!.processed / getProgress(lib.id)!.total) * 100)}%` : '100%',
animation: getProgress(lib.id)!.total === 0 ? 'pulse 1.5s ease-in-out infinite' : 'none',
}"
/>
</div>
</div>
</div> </div>
<div class="flex items-center gap-1.5 shrink-0"> <div class="flex items-center gap-1.5 shrink-0">
<button <button
@@ -199,11 +266,11 @@ function iconComponent(iconName: string | null | undefined) {
</button> </button>
<button <button
class="flex items-center gap-1.5 text-xs font-medium px-3 py-1.5 rounded-md border border-border hover:bg-muted transition-colors text-muted-foreground hover:text-foreground disabled:opacity-50" class="flex items-center gap-1.5 text-xs font-medium px-3 py-1.5 rounded-md border border-border hover:bg-muted transition-colors text-muted-foreground hover:text-foreground disabled:opacity-50"
:disabled="scanning[lib.id]" :disabled="isScanning(lib.id)"
@click="scan(lib)" @click="scan(lib)"
> >
<RefreshCw :size="12" :class="scanning[lib.id] ? 'animate-spin' : ''" /> <RefreshCw :size="12" :class="isScanning(lib.id) ? 'animate-spin' : ''" />
{{ scanning[lib.id] ? 'Scanning…' : 'Scan' }} {{ isScanning(lib.id) ? 'Scanning...' : 'Scan' }}
</button> </button>
</div> </div>
</div> </div>
+5
View File
@@ -16,6 +16,11 @@ export default defineConfig({
server: { server: {
proxy: { proxy: {
'/api': 'http://localhost:3000', '/api': 'http://localhost:3000',
'/socket.io': {
target: 'http://localhost:3000',
ws: true,
changeOrigin: true,
},
}, },
}, },
}) })
+1
View File
@@ -1,3 +1,4 @@
export * from "./auth"; export * from "./auth";
export * from "./library"; export * from "./library";
export * from "./reader-settings"; export * from "./reader-settings";
export * from "./scanner";
+11
View File
@@ -0,0 +1,11 @@
export interface ScanProgressEvent {
jobId: number;
libraryId: number;
status: 'running' | 'completed' | 'failed';
processed: number;
total: number;
added: number;
updated: number;
missing: number;
errorMessage?: string;
}
+8 -4
View File
@@ -24,6 +24,7 @@
"test:e2e": "jest --config ./test/jest-e2e.json" "test:e2e": "jest --config ./test/jest-e2e.json"
}, },
"dependencies": { "dependencies": {
"@projectx/types": "workspace:*",
"7z-wasm": "^1.2.0", "7z-wasm": "^1.2.0",
"@fastify/cookie": "^11.0.2", "@fastify/cookie": "^11.0.2",
"@fastify/formbody": "^8.0.2", "@fastify/formbody": "^8.0.2",
@@ -34,8 +35,10 @@
"@nestjs/core": "^11.0.1", "@nestjs/core": "^11.0.1",
"@nestjs/jwt": "^11.0.2", "@nestjs/jwt": "^11.0.2",
"@nestjs/passport": "^11.0.5", "@nestjs/passport": "^11.0.5",
"@nestjs/platform-fastify": "^11.1.11", "@nestjs/platform-fastify": "^11.1.12",
"@nestjs/platform-socket.io": "^11.1.12",
"@nestjs/schedule": "^6.1.0", "@nestjs/schedule": "^6.1.0",
"@nestjs/websockets": "^11.1.12",
"bcryptjs": "^3.0.3", "bcryptjs": "^3.0.3",
"chokidar": "^4.0.3", "chokidar": "^4.0.3",
"class-transformer": "^0.5.1", "class-transformer": "^0.5.1",
@@ -48,10 +51,11 @@
"passport": "^0.7.0", "passport": "^0.7.0",
"passport-jwt": "^4.0.1", "passport-jwt": "^4.0.1",
"pdf-lib": "^1.17.1", "pdf-lib": "^1.17.1",
"pg": "^8.16.3", "pg": "^8.17.1",
"reflect-metadata": "^0.2.2", "reflect-metadata": "^0.2.2",
"rxjs": "^7.8.1", "rxjs": "^7.8.1",
"sharp": "^0.34.5", "sharp": "^0.34.5",
"socket.io": "^4.8.3",
"unzipper": "^0.12.3" "unzipper": "^0.12.3"
}, },
"devDependencies": { "devDependencies": {
@@ -60,8 +64,8 @@
"@nestjs/cli": "^11.0.0", "@nestjs/cli": "^11.0.0",
"@nestjs/schematics": "^11.0.0", "@nestjs/schematics": "^11.0.0",
"@nestjs/testing": "^11.0.1", "@nestjs/testing": "^11.0.1",
"@swc/cli": "^0.7.9", "@swc/cli": "^0.7.10",
"@swc/core": "^1.15.8", "@swc/core": "^1.15.10",
"@types/bcrypt": "^6.0.0", "@types/bcrypt": "^6.0.0",
"@types/bcryptjs": "^3.0.0", "@types/bcryptjs": "^3.0.0",
"@types/chokidar": "^1.7.5", "@types/chokidar": "^1.7.5",
+2 -2
View File
@@ -5,14 +5,14 @@
{ {
"idx": 0, "idx": 0,
"version": "7", "version": "7",
"when": 1768599599999, "when": 1768909427779,
"tag": "0000_baseline", "tag": "0000_baseline",
"breakpoints": true "breakpoints": true
}, },
{ {
"idx": 1, "idx": 1,
"version": "7", "version": "7",
"when": 1768599599999, "when": 1768909427779,
"tag": "0001_add_scan_jobs_cascade", "tag": "0001_add_scan_jobs_cascade",
"breakpoints": true "breakpoints": true
} }
+3
View File
@@ -1,6 +1,7 @@
import { NestFactory } from '@nestjs/core'; import { NestFactory } from '@nestjs/core';
import { FastifyAdapter, NestFastifyApplication } from '@nestjs/platform-fastify'; import { FastifyAdapter, NestFastifyApplication } from '@nestjs/platform-fastify';
import { ValidationPipe } from '@nestjs/common'; import { ValidationPipe } from '@nestjs/common';
import { IoAdapter } from '@nestjs/platform-socket.io';
import { AppModule } from './app.module'; import { AppModule } from './app.module';
import { GlobalExceptionFilter } from './common/filters/http-exception.filter'; import { GlobalExceptionFilter } from './common/filters/http-exception.filter';
import { join } from 'path'; import { join } from 'path';
@@ -14,6 +15,8 @@ async function bootstrap() {
await pool.end(); await pool.end();
const app = await NestFactory.create<NestFastifyApplication>(AppModule, new FastifyAdapter({ logger: true })); const app = await NestFactory.create<NestFastifyApplication>(AppModule, new FastifyAdapter({ logger: true }));
app.useWebSocketAdapter(new IoAdapter(app));
app.setGlobalPrefix('api'); app.setGlobalPrefix('api');
app.useGlobalPipes( app.useGlobalPipes(
@@ -1,10 +1,12 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { ScannerModule } from '../scanner/scanner.module';
import { LibraryController } from './library.controller'; import { LibraryController } from './library.controller';
import { LibraryRepository } from './library.repository'; import { LibraryRepository } from './library.repository';
import { LibraryService } from './library.service'; import { LibraryService } from './library.service';
@Module({ @Module({
imports: [ScannerModule],
controllers: [LibraryController], controllers: [LibraryController],
providers: [LibraryService, LibraryRepository], providers: [LibraryService, LibraryRepository],
exports: [LibraryService], exports: [LibraryService],
@@ -5,6 +5,7 @@ import { join } from 'path';
import type { RequestUser } from '../../common/types/request-user'; import type { RequestUser } from '../../common/types/request-user';
import { isPrimaryFormat } from '../scanner/lib/classify'; import { isPrimaryFormat } from '../scanner/lib/classify';
import { ScannerService } from '../scanner/scanner.service';
import { CreateLibraryDto } from './dto/create-library.dto'; import { CreateLibraryDto } from './dto/create-library.dto';
import { GrantLibraryAccessDto } from './dto/grant-library-access.dto'; import { GrantLibraryAccessDto } from './dto/grant-library-access.dto';
import { PrescanLibraryDto } from './dto/prescan-library.dto'; import { PrescanLibraryDto } from './dto/prescan-library.dto';
@@ -20,6 +21,7 @@ export class LibraryService {
constructor( constructor(
private readonly libraryRepo: LibraryRepository, private readonly libraryRepo: LibraryRepository,
private readonly config: ConfigService, private readonly config: ConfigService,
private readonly scannerService: ScannerService,
) { ) {
this.booksPath = this.config.get<string>('storage.booksPath')!; this.booksPath = this.config.get<string>('storage.booksPath')!;
} }
@@ -62,6 +64,8 @@ export class LibraryService {
const folders = await Promise.all(dto.folders.map((path) => this.libraryRepo.insertFolder({ libraryId: library.id, path }))); const folders = await Promise.all(dto.folders.map((path) => this.libraryRepo.insertFolder({ libraryId: library.id, path })));
this.scannerService.startScanAsync(library.id);
return { ...library, folders: folders.map(([f]) => f) }; return { ...library, folders: folders.map(([f]) => f) };
} }
@@ -0,0 +1,74 @@
import { Injectable } from '@nestjs/common';
export interface ScanEntry {
jobId: number;
libraryId: number;
processed: number;
total: number;
added: number;
updated: number;
missing: number;
lastEmitMs: number;
lastEmitPct: number;
}
@Injectable()
export class ScanJobStore {
private readonly byLibrary = new Map<number, ScanEntry>();
isRunning(libraryId: number): boolean {
return this.byLibrary.has(libraryId);
}
create(jobId: number, libraryId: number, total: number): ScanEntry {
const entry: ScanEntry = {
jobId,
libraryId,
processed: 0,
total,
added: 0,
updated: 0,
missing: 0,
lastEmitMs: 0,
lastEmitPct: -1,
};
this.byLibrary.set(libraryId, entry);
return entry;
}
setTotal(libraryId: number, total: number): void {
const entry = this.byLibrary.get(libraryId);
if (entry) entry.total = total;
}
increment(libraryId: number, delta: { processed?: number; added?: number; updated?: number; missing?: number }): ScanEntry | undefined {
const entry = this.byLibrary.get(libraryId);
if (!entry) return undefined;
if (delta.processed !== undefined) entry.processed += delta.processed;
if (delta.added !== undefined) entry.added += delta.added;
if (delta.updated !== undefined) entry.updated += delta.updated;
if (delta.missing !== undefined) entry.missing += delta.missing;
return entry;
}
get(libraryId: number): ScanEntry | undefined {
return this.byLibrary.get(libraryId);
}
shouldEmit(entry: ScanEntry): boolean {
if (entry.total > 0 && entry.processed >= entry.total) return true;
const pct = entry.total > 0 ? Math.floor((entry.processed / entry.total) * 100) : 0;
if (pct >= entry.lastEmitPct + 1) return true;
if (Date.now() - entry.lastEmitMs >= 1000) return true;
return false;
}
markEmitted(entry: ScanEntry): void {
entry.lastEmitMs = Date.now();
entry.lastEmitPct = entry.total > 0 ? Math.floor((entry.processed / entry.total) * 100) : 0;
}
delete(libraryId: number): void {
this.byLibrary.delete(libraryId);
}
}
@@ -0,0 +1,70 @@
import { Logger } from '@nestjs/common';
import { JwtService } from '@nestjs/jwt';
import {
OnGatewayConnection,
OnGatewayDisconnect,
SubscribeMessage,
WebSocketGateway,
WebSocketServer,
} from '@nestjs/websockets';
import { Server, Socket } from 'socket.io';
import type { ScanProgressEvent } from '@projectx/types';
import { AuthService } from '../auth/auth.service';
import { ScanJobStore } from './scan-job-store.service';
@WebSocketGateway({ namespace: '/scan', cors: { origin: '*', credentials: true } })
export class ScanGateway implements OnGatewayConnection, OnGatewayDisconnect {
@WebSocketServer() server: Server;
private readonly logger = new Logger(ScanGateway.name);
constructor(
private readonly jwtService: JwtService,
private readonly authService: AuthService,
private readonly scanJobStore: ScanJobStore,
) {}
async handleConnection(client: Socket): Promise<void> {
try {
const token = client.handshake.auth?.token as string | undefined;
if (!token) throw new Error('No token provided');
const payload = this.jwtService.verify<{ sub: number; ver: number }>(token);
const user = await this.authService.validateUser(payload.sub, payload.ver);
if (!user) throw new Error('User not found or token revoked');
client.data.user = user;
this.logger.debug(`WS connected: user=${user.id} socket=${client.id}`);
} catch (err) {
this.logger.warn(`WS rejected: ${(err as Error).message} socket=${client.id}`);
client.disconnect();
}
}
handleDisconnect(client: Socket): void {
this.logger.debug(`WS disconnected: socket=${client.id}`);
}
@SubscribeMessage('subscribe:library')
handleSubscribeLibrary(client: Socket, libraryId: number): void {
client.join(`library:${libraryId}`);
// Send current snapshot immediately so reconnecting clients catch up.
const entry = this.scanJobStore.get(libraryId);
if (entry) {
const event: ScanProgressEvent = {
jobId: entry.jobId,
libraryId,
status: 'running',
processed: entry.processed,
total: entry.total,
added: entry.added,
updated: entry.updated,
missing: entry.missing,
};
client.emit('scan:progress', event);
}
}
// Called by ScannerService — not a WS message handler.
emitProgress(event: ScanProgressEvent): void {
this.server.to(`library:${event.libraryId}`).emit('scan:progress', event);
}
}
@@ -1,4 +1,4 @@
import { Controller, Param, ParseIntPipe, Post } from '@nestjs/common'; import { Controller, HttpCode, HttpStatus, Param, ParseIntPipe, Post } from '@nestjs/common';
import { RequirePermission } from '../../common/decorators/require-permission.decorator'; import { RequirePermission } from '../../common/decorators/require-permission.decorator';
import { ScannerService } from './scanner.service'; import { ScannerService } from './scanner.service';
@@ -9,7 +9,8 @@ export class ScannerController {
@Post('libraries/:id/scan') @Post('libraries/:id/scan')
@RequirePermission('manage_libraries') @RequirePermission('manage_libraries')
@HttpCode(HttpStatus.ACCEPTED)
scan(@Param('id', ParseIntPipe) libraryId: number) { scan(@Param('id', ParseIntPipe) libraryId: number) {
return this.scannerService.scan(libraryId, 'manual'); return this.scannerService.startScan(libraryId, 'manual');
} }
} }
+17 -2
View File
@@ -1,14 +1,29 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { JwtModule } from '@nestjs/jwt';
import { AuthModule } from '../auth/auth.module';
import { MetadataModule } from '../metadata/metadata.module'; import { MetadataModule } from '../metadata/metadata.module';
import { ScanGateway } from './scan.gateway';
import { ScanJobStore } from './scan-job-store.service';
import { ScannerController } from './scanner.controller'; import { ScannerController } from './scanner.controller';
import { ScannerRepository } from './scanner.repository'; import { ScannerRepository } from './scanner.repository';
import { ScannerService } from './scanner.service'; import { ScannerService } from './scanner.service';
@Module({ @Module({
imports: [MetadataModule], imports: [
MetadataModule,
AuthModule,
JwtModule.registerAsync({
inject: [ConfigService],
useFactory: (config: ConfigService) => ({
secret: config.get<string>('auth.jwtSecret'),
signOptions: { expiresIn: config.get('auth.jwtExpiresIn') as any },
}),
}),
],
controllers: [ScannerController], controllers: [ScannerController],
providers: [ScannerService, ScannerRepository], providers: [ScannerService, ScannerRepository, ScanGateway, ScanJobStore],
exports: [ScannerService], exports: [ScannerService],
}) })
export class ScannerModule {} export class ScannerModule {}
@@ -1,5 +1,5 @@
import { Inject, Injectable } from '@nestjs/common'; import { Inject, Injectable } from '@nestjs/common';
import { and, eq, inArray } from 'drizzle-orm'; import { and, eq, inArray, ne } from 'drizzle-orm';
import { NodePgDatabase } from 'drizzle-orm/node-postgres'; import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { DB } from '../../db'; import { DB } from '../../db';
@@ -30,6 +30,13 @@ export class ScannerRepository {
await this.db.update(scanJobs).set({ status: 'failed', errorMessage, completedAt: new Date() }).where(eq(scanJobs.id, id)); await this.db.update(scanJobs).set({ status: 'failed', errorMessage, completedAt: new Date() }).where(eq(scanJobs.id, id));
} }
async failAllRunningJobs(errorMessage: string): Promise<void> {
await this.db
.update(scanJobs)
.set({ status: 'failed', errorMessage, completedAt: new Date() })
.where(eq(scanJobs.status, 'running'));
}
// ── Library Folders ──────────────────────────────────────────────────────── // ── Library Folders ────────────────────────────────────────────────────────
async findLibraryFolders(libraryId: number) { async findLibraryFolders(libraryId: number) {
+182 -43
View File
@@ -1,14 +1,18 @@
import { Injectable, Logger, NotFoundException } from '@nestjs/common'; import { ConflictException, Injectable, Logger, NotFoundException, OnApplicationBootstrap } from '@nestjs/common';
const METADATA_FORMATS = new Set(['epub', 'mobi', 'azw3', 'azw', 'cbz', 'cbr', 'cb7', 'pdf']);
import type { ScanProgressEvent } from '@projectx/types';
import { MetadataService } from '../metadata/metadata.service'; import { MetadataService } from '../metadata/metadata.service';
import { ScanGateway } from './scan.gateway';
import { ScanJobStore } from './scan-job-store.service';
import { classifyFile } from './lib/classify'; import { classifyFile } from './lib/classify';
import { sha256File } from './lib/hash'; import { sha256File } from './lib/hash';
import { waitForStability } from './lib/stability'; import { waitForStability } from './lib/stability';
import { BookCandidate, FileStat, findBookCandidates } from './lib/walk'; import { BookCandidate, FileStat, findBookCandidates } from './lib/walk';
import { ScannerRepository } from './scanner.repository'; import { ScannerRepository } from './scanner.repository';
const METADATA_FORMATS = new Set(['epub', 'mobi', 'azw3', 'azw', 'cbz', 'cbr', 'cb7', 'pdf']);
const BATCH_SIZE = 5;
interface ScanCounts { interface ScanCounts {
addedCount: number; addedCount: number;
updatedCount: number; updatedCount: number;
@@ -16,86 +20,207 @@ interface ScanCounts {
} }
@Injectable() @Injectable()
export class ScannerService { export class ScannerService implements OnApplicationBootstrap {
private readonly logger = new Logger(ScannerService.name); private readonly logger = new Logger(ScannerService.name);
constructor( constructor(
private readonly scannerRepo: ScannerRepository, private readonly scannerRepo: ScannerRepository,
private readonly metadataService: MetadataService, private readonly metadataService: MetadataService,
private readonly scanJobStore: ScanJobStore,
private readonly scanGateway: ScanGateway,
) {} ) {}
async scan(libraryId: number, triggeredBy: 'manual' | 'watcher' | 'schedule' = 'manual') { async onApplicationBootstrap(): Promise<void> {
await this.scannerRepo.failAllRunningJobs('Server restarted during scan');
}
async startScan(libraryId: number, triggeredBy: 'manual' | 'watcher' | 'schedule'): Promise<{ jobId: number }> {
if (this.scanJobStore.isRunning(libraryId)) {
throw new ConflictException(`A scan is already running for library ${libraryId}`);
}
const folders = await this.scannerRepo.findLibraryFolders(libraryId); const folders = await this.scannerRepo.findLibraryFolders(libraryId);
if (folders.length === 0) throw new NotFoundException(`Library ${libraryId} has no folders`); if (folders.length === 0) throw new NotFoundException(`Library ${libraryId} has no folders`);
const job = await this.scannerRepo.createScanJob(libraryId, triggeredBy); const job = await this.scannerRepo.createScanJob(libraryId, triggeredBy);
this.logger.log(`Scan job ${job.id} started for library ${libraryId}`);
this.scanJobStore.create(job.id, libraryId, 0);
this.emitFromStore(libraryId, job.id, 'running');
this.runScan(libraryId, job.id, folders).catch((err) =>
this.logger.error(`Scan job ${job.id} crashed unexpectedly: ${(err as Error).message}`),
);
return { jobId: job.id };
}
startScanAsync(libraryId: number): void {
if (this.scanJobStore.isRunning(libraryId)) return;
this.startScan(libraryId, 'manual').catch((err) =>
this.logger.error(`Auto-scan failed to start for library ${libraryId}: ${(err as Error).message}`),
);
}
private async runScan(
libraryId: number,
jobId: number,
folders: Awaited<ReturnType<ScannerRepository['findLibraryFolders']>>,
): Promise<void> {
this.logger.log(`Scan job ${jobId} started for library ${libraryId}`);
type FolderWork = {
id: number;
libraryId: number;
path: string;
candidates: BookCandidate[];
knownBooks: Awaited<ReturnType<ScannerRepository['findBooksByLibraryFolder']>>;
knownFiles: Awaited<ReturnType<ScannerRepository['findBookFilesByLibraryFolder']>>;
};
const folderWork: FolderWork[] = [];
let totalCandidates = 0;
for (const folder of folders) {
let candidates: BookCandidate[] = [];
try {
candidates = await findBookCandidates(folder.path);
} catch (err) {
this.logger.warn(`Cannot walk ${folder.path}: ${err}`);
}
const [knownBooks, knownFiles] = await Promise.all([
this.scannerRepo.findBooksByLibraryFolder(folder.id),
this.scannerRepo.findBookFilesByLibraryFolder(folder.id),
]);
folderWork.push({ ...folder, candidates, knownBooks, knownFiles });
totalCandidates += candidates.length;
}
this.scanJobStore.setTotal(libraryId, totalCandidates);
this.emitFromStore(libraryId, jobId, 'running');
const totals: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 }; const totals: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
try { try {
for (const folder of folders) { for (const { id: folderId, path: folderPath, candidates, knownBooks, knownFiles } of folderWork) {
this.logger.log(`Scanning folder: ${folder.path}`); const counts = await this.scanFolderCandidates(
const counts = await this.scanFolder(folder.id, folder.libraryId, folder.path); folderId,
libraryId,
folderPath,
candidates,
knownBooks,
knownFiles,
jobId,
);
totals.addedCount += counts.addedCount; totals.addedCount += counts.addedCount;
totals.updatedCount += counts.updatedCount; totals.updatedCount += counts.updatedCount;
totals.missingCount += counts.missingCount; totals.missingCount += counts.missingCount;
} }
await this.scannerRepo.completeScanJob(job.id, totals); await this.scannerRepo.completeScanJob(jobId, totals);
this.logger.log(`Scan job ${job.id} completed — ${JSON.stringify(totals)}`); this.logger.log(`Scan job ${jobId} completed — ${JSON.stringify(totals)}`);
this.emitFromStore(libraryId, jobId, 'completed');
} catch (err) { } catch (err) {
const message = err instanceof Error ? err.message : String(err); const message = err instanceof Error ? err.message : String(err);
await this.scannerRepo.failScanJob(job.id, message); await this.scannerRepo.failScanJob(jobId, message).catch(() => {
this.logger.error(`Scan job ${job.id} failed: ${message}`); // Job row may have been cascade-deleted if library was deleted.
throw err; });
this.logger.error(`Scan job ${jobId} failed: ${message}`);
this.emitFromStore(libraryId, jobId, 'failed', message);
} finally {
this.scanJobStore.delete(libraryId);
} }
return { jobId: job.id, ...totals };
} }
private async scanFolder(libraryFolderId: number, libraryId: number, folderPath: string): Promise<ScanCounts> { private async scanFolderCandidates(
libraryFolderId: number,
libraryId: number,
folderPath: string,
candidates: BookCandidate[],
knownBooks: Awaited<ReturnType<ScannerRepository['findBooksByLibraryFolder']>>,
knownFiles: Awaited<ReturnType<ScannerRepository['findBookFilesByLibraryFolder']>>,
jobId: number,
): Promise<ScanCounts> {
const counts: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 }; const counts: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
const [knownBooks, knownFiles] = await Promise.all([
this.scannerRepo.findBooksByLibraryFolder(libraryFolderId),
this.scannerRepo.findBookFilesByLibraryFolder(libraryFolderId),
]);
const bookByFolderPath = new Map(knownBooks.map((b) => [b.folderPath, b])); const bookByFolderPath = new Map(knownBooks.map((b) => [b.folderPath, b]));
const fileByPath = new Map(knownFiles.map((f) => [f.absolutePath, f])); const fileByPath = new Map(knownFiles.map((f) => [f.absolutePath, f]));
const fileByIno = new Map(knownFiles.map((f) => [f.ino, f])); const fileByIno = new Map(knownFiles.map((f) => [f.ino, f]));
const seenBookIds = new Set<number>(); const seenBookIds = new Set<number>();
let candidates: BookCandidate[]; for (let i = 0; i < candidates.length; i += BATCH_SIZE) {
try { const batch = candidates.slice(i, i + BATCH_SIZE);
candidates = await findBookCandidates(folderPath);
} catch (err) {
this.logger.warn(`Cannot walk ${folderPath}: ${err}`);
return counts;
}
for (const candidate of candidates) { const results = await Promise.all(
const book = await this.upsertBook(candidate, libraryId, libraryFolderId, bookByFolderPath, counts); batch.map((c) => this.processCandidate(c, libraryId, libraryFolderId, bookByFolderPath, fileByPath, fileByIno)),
seenBookIds.add(book.id); );
for (const fileStat of candidate.files) { for (const r of results) {
const { format } = classifyFile(fileStat.absolutePath); seenBookIds.add(r.bookId);
const isNew = await this.processFile(fileStat, book.id, libraryFolderId, fileByPath, fileByIno, counts); counts.addedCount += r.added;
if (isNew && format && METADATA_FORMATS.has(format)) { counts.updatedCount += r.updated;
await this.metadataService.extractAndSave(book.id, fileStat.absolutePath, format); }
}
const entry = this.scanJobStore.increment(libraryId, { processed: batch.length });
if (entry && this.scanJobStore.shouldEmit(entry)) {
this.emitFromStore(libraryId, jobId, 'running');
this.scanJobStore.markEmitted(entry);
} }
} }
const missingIds = knownBooks.filter((b) => !seenBookIds.has(b.id)).map((b) => b.id); const missingIds = knownBooks.filter((b) => !seenBookIds.has(b.id)).map((b) => b.id);
if (missingIds.length > 0) {
await this.scannerRepo.markBooksAsMissing(missingIds); await this.scannerRepo.markBooksAsMissing(missingIds);
counts.missingCount += missingIds.length; counts.missingCount += missingIds.length;
this.scanJobStore.increment(libraryId, { missing: missingIds.length });
}
return counts; return counts;
} }
private async processCandidate(
candidate: BookCandidate,
libraryId: number,
libraryFolderId: number,
bookByFolderPath: Map<string, { id: number; status: string; folderPath: string }>,
fileByPath: Map<string, { id: number; ino: number; sizeBytes: number | null; mtime: Date | null; hash: string | null }>,
fileByIno: Map<number, { id: number; absolutePath: string }>,
): Promise<{ bookId: number; added: number; updated: number }> {
const counts = { added: 0, updated: 0 };
const fileCounts: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
const book = await this.upsertBook(candidate, libraryId, libraryFolderId, bookByFolderPath, fileCounts);
counts.added += fileCounts.addedCount;
counts.updated += fileCounts.updatedCount;
for (const fileStat of candidate.files) {
const { format } = classifyFile(fileStat.absolutePath);
const fileCount: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
let isNew: boolean;
try {
isNew = await this.processFile(fileStat, book.id, libraryFolderId, fileByPath, fileByIno, fileCount);
} catch (err) {
this.logger.warn(`Failed to process file ${fileStat.absolutePath}: ${err}`);
continue;
}
counts.added += fileCount.addedCount;
counts.updated += fileCount.updatedCount;
if (isNew && format && METADATA_FORMATS.has(format)) {
try {
await this.metadataService.extractAndSave(book.id, fileStat.absolutePath, format);
} catch (err) {
this.logger.warn(`Metadata extraction failed for ${fileStat.absolutePath}: ${err}`);
}
}
}
return { bookId: book.id, ...counts };
}
private async upsertBook( private async upsertBook(
candidate: BookCandidate, candidate: BookCandidate,
libraryId: number, libraryId: number,
@@ -124,7 +249,6 @@ export class ScannerService {
return existing; return existing;
} }
// Returns true if a new file record was created (triggers metadata extraction).
private async processFile( private async processFile(
fileStat: FileStat, fileStat: FileStat,
bookId: number, bookId: number,
@@ -141,7 +265,6 @@ export class ScannerService {
const byPath = fileByPath.get(fileStat.absolutePath); const byPath = fileByPath.get(fileStat.absolutePath);
if (byPath) { if (byPath) {
const changed = fileStat.sizeBytes !== byPath.sizeBytes || fileStat.mtime.getTime() !== byPath.mtime?.getTime(); const changed = fileStat.sizeBytes !== byPath.sizeBytes || fileStat.mtime.getTime() !== byPath.mtime?.getTime();
if (changed) { if (changed) {
await this.scannerRepo.updateBookFile(byPath.id, { await this.scannerRepo.updateBookFile(byPath.id, {
ino: fileStat.ino, ino: fileStat.ino,
@@ -203,4 +326,20 @@ export class ScannerService {
counts.addedCount++; counts.addedCount++;
return true; return true;
} }
private emitFromStore(libraryId: number, jobId: number, status: 'running' | 'completed' | 'failed', errorMessage?: string): void {
const entry = this.scanJobStore.get(libraryId);
const event: ScanProgressEvent = {
jobId,
libraryId,
status,
processed: entry?.processed ?? 0,
total: entry?.total ?? 0,
added: entry?.added ?? 0,
updated: entry?.updated ?? 0,
missing: entry?.missing ?? 0,
errorMessage,
};
this.scanGateway.emitProgress(event);
}
} }