mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
feat(scanner): add live scan progress via WebSocket and auto-scan
This commit is contained in:
+7
-6
@@ -27,17 +27,18 @@
|
|||||||
"pdfjs-dist": "^5.4.530",
|
"pdfjs-dist": "^5.4.530",
|
||||||
"pinia": "^3.0.4",
|
"pinia": "^3.0.4",
|
||||||
"reka-ui": "^2.7.0",
|
"reka-ui": "^2.7.0",
|
||||||
|
"socket.io-client": "^4.8.3",
|
||||||
"tailwind-merge": "^3.4.0",
|
"tailwind-merge": "^3.4.0",
|
||||||
"tailwindcss": "^4.1.18",
|
"tailwindcss": "^4.1.18",
|
||||||
"vue": "^3.5.26",
|
"vue": "^3.5.27",
|
||||||
"vue-router": "^4.6.4",
|
"vue-router": "^4.6.4",
|
||||||
"vue-sonner": "^2.0.9",
|
"vue-sonner": "^2.0.9",
|
||||||
"vue-virtual-scroller": "^2.0.0-beta.8"
|
"vue-virtual-scroller": "^2.0.0-beta.8"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@tsconfig/node24": "^24.0.3",
|
"@tsconfig/node24": "^24.0.4",
|
||||||
"@types/jsdom": "^27.0.0",
|
"@types/jsdom": "^27.0.0",
|
||||||
"@types/node": "^22.19.6",
|
"@types/node": "^22.19.7",
|
||||||
"@vitejs/plugin-vue": "^6.0.3",
|
"@vitejs/plugin-vue": "^6.0.3",
|
||||||
"@vitest/eslint-plugin": "^1.6.6",
|
"@vitest/eslint-plugin": "^1.6.6",
|
||||||
"@vue/eslint-config-typescript": "^14.6.0",
|
"@vue/eslint-config-typescript": "^14.6.0",
|
||||||
@@ -45,12 +46,12 @@
|
|||||||
"@vue/tsconfig": "^0.8.1",
|
"@vue/tsconfig": "^0.8.1",
|
||||||
"eslint": "^9.39.2",
|
"eslint": "^9.39.2",
|
||||||
"eslint-config-prettier": "^10.1.8",
|
"eslint-config-prettier": "^10.1.8",
|
||||||
"eslint-plugin-oxlint": "~1.39.0",
|
"eslint-plugin-oxlint": "~1.41.0",
|
||||||
"eslint-plugin-vue": "~10.6.2",
|
"eslint-plugin-vue": "~10.7.0",
|
||||||
"jiti": "^2.6.1",
|
"jiti": "^2.6.1",
|
||||||
"jsdom": "^27.4.0",
|
"jsdom": "^27.4.0",
|
||||||
"npm-run-all2": "^8.0.4",
|
"npm-run-all2": "^8.0.4",
|
||||||
"oxlint": "~1.39.0",
|
"oxlint": "~1.41.0",
|
||||||
"prettier": "3.8.0",
|
"prettier": "3.8.0",
|
||||||
"tw-animate-css": "^1.4.0",
|
"tw-animate-css": "^1.4.0",
|
||||||
"typescript": "~5.9.3",
|
"typescript": "~5.9.3",
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
import { ref } from 'vue'
|
||||||
|
import { io, Socket } from 'socket.io-client'
|
||||||
|
import type { ScanProgressEvent } from '@projectx/types'
|
||||||
|
import { getAccessToken } from '@/lib/api'
|
||||||
|
|
||||||
|
let socket: Socket | null = null
|
||||||
|
const progressMap = ref<Map<number, ScanProgressEvent>>(new Map())
|
||||||
|
const subscribedLibraries = new Set<number>()
|
||||||
|
|
||||||
|
function getSocket(): Socket {
|
||||||
|
if (!socket) {
|
||||||
|
// Use a callback so getAccessToken() is called at (re)connect time, not just once at creation.
|
||||||
|
socket = io('/scan', {
|
||||||
|
auth: (cb: (data: object) => void) => cb({ token: getAccessToken() }),
|
||||||
|
transports: ['websocket'],
|
||||||
|
autoConnect: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
socket.on('scan:progress', (event: ScanProgressEvent) => {
|
||||||
|
progressMap.value.set(event.libraryId, event)
|
||||||
|
progressMap.value = new Map(progressMap.value)
|
||||||
|
if (event.status !== 'running') {
|
||||||
|
// Keep the final event for a short time so UI can show completion, then clear.
|
||||||
|
setTimeout(() => {
|
||||||
|
progressMap.value.delete(event.libraryId)
|
||||||
|
progressMap.value = new Map(progressMap.value)
|
||||||
|
}, 3000)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
socket.on('disconnect', () => {
|
||||||
|
// Clear stale progress bars but keep subscribedLibraries so we can re-subscribe on reconnect.
|
||||||
|
progressMap.value = new Map()
|
||||||
|
})
|
||||||
|
|
||||||
|
socket.on('connect', () => {
|
||||||
|
// Re-subscribe to all libraries after reconnect (server-side rooms are per-socket).
|
||||||
|
for (const id of subscribedLibraries) {
|
||||||
|
socket!.emit('subscribe:library', id)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return socket
|
||||||
|
}
|
||||||
|
|
||||||
|
export { getSocket }
|
||||||
|
|
||||||
|
export function useScanProgress() {
|
||||||
|
function subscribeLibrary(libraryId: number): void {
|
||||||
|
if (subscribedLibraries.has(libraryId)) return
|
||||||
|
subscribedLibraries.add(libraryId)
|
||||||
|
getSocket().emit('subscribe:library', libraryId)
|
||||||
|
}
|
||||||
|
|
||||||
|
function getProgress(libraryId: number): ScanProgressEvent | undefined {
|
||||||
|
return progressMap.value.get(libraryId)
|
||||||
|
}
|
||||||
|
|
||||||
|
function isScanning(libraryId: number): boolean {
|
||||||
|
return progressMap.value.get(libraryId)?.status === 'running'
|
||||||
|
}
|
||||||
|
|
||||||
|
return { subscribeLibrary, getProgress, isScanning, progressMap }
|
||||||
|
}
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { onMounted, ref } from 'vue'
|
import { onMounted, ref, watch } from 'vue'
|
||||||
import { useRoute, useRouter } from 'vue-router'
|
import { useRoute, useRouter } from 'vue-router'
|
||||||
import { FolderOpen, Plus, RefreshCw, Pencil, Trash2, Library } from 'lucide-vue-next'
|
import { FolderOpen, Plus, RefreshCw, Pencil, Trash2, Library } from 'lucide-vue-next'
|
||||||
import { toast } from 'vue-sonner'
|
import { toast } from 'vue-sonner'
|
||||||
@@ -7,16 +7,18 @@ import { api } from '@/lib/api'
|
|||||||
import type { Library as LibraryType, LibraryStats } from '@projectx/types'
|
import type { Library as LibraryType, LibraryStats } from '@projectx/types'
|
||||||
import LibraryCreatorModal from '@/features/library/components/LibraryCreatorModal.vue'
|
import LibraryCreatorModal from '@/features/library/components/LibraryCreatorModal.vue'
|
||||||
import { useLibraries } from '@/features/library/composables/useLibraries'
|
import { useLibraries } from '@/features/library/composables/useLibraries'
|
||||||
|
import { useScanProgress, getSocket } from '@/features/scanner/composables/useScanProgress'
|
||||||
|
|
||||||
const route = useRoute()
|
const route = useRoute()
|
||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
const { libraries, fetchLibraries } = useLibraries()
|
const { libraries, fetchLibraries } = useLibraries()
|
||||||
|
const { subscribeLibrary, getProgress, isScanning, progressMap } = useScanProgress()
|
||||||
|
|
||||||
const stats = ref<Record<number, LibraryStats>>({})
|
const stats = ref<Record<number, LibraryStats>>({})
|
||||||
const scanning = ref<Record<number, boolean>>({})
|
|
||||||
const scanningAll = ref(false)
|
const scanningAll = ref(false)
|
||||||
const creatorOpen = ref(false)
|
const creatorOpen = ref(false)
|
||||||
const editingLibrary = ref<LibraryType | null>(null)
|
const editingLibrary = ref<LibraryType | null>(null)
|
||||||
|
const pendingNavigateLibraryId = ref<number | null>(null)
|
||||||
const deletingLibrary = ref<LibraryType | null>(null)
|
const deletingLibrary = ref<LibraryType | null>(null)
|
||||||
const deleteConfirmName = ref('')
|
const deleteConfirmName = ref('')
|
||||||
const deleting = ref(false)
|
const deleting = ref(false)
|
||||||
@@ -30,24 +32,48 @@ async function loadAllStats() {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function subscribeAll() {
|
||||||
|
for (const lib of libraries.value) {
|
||||||
|
subscribeLibrary(lib.id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
|
getSocket()
|
||||||
await fetchLibraries()
|
await fetchLibraries()
|
||||||
|
subscribeAll()
|
||||||
loadAllStats()
|
loadAllStats()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const statsReloadedFor = new Set<number>()
|
||||||
|
|
||||||
|
watch(progressMap, (map) => {
|
||||||
|
for (const [libraryId, event] of map) {
|
||||||
|
if (event.status === 'completed' && !statsReloadedFor.has(libraryId)) {
|
||||||
|
statsReloadedFor.add(libraryId)
|
||||||
|
api(`/api/libraries/${libraryId}/stats`).then(async (res) => {
|
||||||
|
if (res.ok) stats.value[libraryId] = await res.json()
|
||||||
|
setTimeout(() => statsReloadedFor.delete(libraryId), 5000)
|
||||||
|
})
|
||||||
|
if (pendingNavigateLibraryId.value === libraryId) {
|
||||||
|
pendingNavigateLibraryId.value = null
|
||||||
|
router.push({ name: 'library', params: { id: libraryId } })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
async function scan(lib: LibraryType) {
|
async function scan(lib: LibraryType) {
|
||||||
scanning.value[lib.id] = true
|
|
||||||
try {
|
try {
|
||||||
const res = await api(`/api/scanner/libraries/${lib.id}/scan`, { method: 'POST' })
|
const res = await api(`/api/scanner/libraries/${lib.id}/scan`, { method: 'POST' })
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
toast.success(`Scan started for "${lib.name}"`)
|
toast.success(`Scan started for "${lib.name}"`)
|
||||||
|
subscribeLibrary(lib.id)
|
||||||
} else {
|
} else {
|
||||||
toast.error(`Failed to start scan for "${lib.name}"`)
|
toast.error(`Failed to start scan for "${lib.name}"`)
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
toast.error(`Failed to start scan for "${lib.name}"`)
|
toast.error(`Failed to start scan for "${lib.name}"`)
|
||||||
} finally {
|
|
||||||
scanning.value[lib.id] = false
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,6 +84,7 @@ async function scanAll() {
|
|||||||
const failed = results.filter((r) => !r.ok).length
|
const failed = results.filter((r) => !r.ok).length
|
||||||
if (failed === 0) {
|
if (failed === 0) {
|
||||||
toast.success('Scan started for all libraries')
|
toast.success('Scan started for all libraries')
|
||||||
|
subscribeAll()
|
||||||
} else {
|
} else {
|
||||||
toast.error(`${failed} librar${failed === 1 ? 'y' : 'ies'} failed to start`)
|
toast.error(`${failed} librar${failed === 1 ? 'y' : 'ies'} failed to start`)
|
||||||
}
|
}
|
||||||
@@ -79,8 +106,11 @@ function openEdit(lib: LibraryType) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function onSaved(library: LibraryType) {
|
async function onSaved(library: LibraryType) {
|
||||||
|
const isNew = !editingLibrary.value
|
||||||
creatorOpen.value = false
|
creatorOpen.value = false
|
||||||
editingLibrary.value = null
|
editingLibrary.value = null
|
||||||
|
subscribeLibrary(library.id)
|
||||||
|
if (isNew) pendingNavigateLibraryId.value = library.id
|
||||||
await fetchLibraries()
|
await fetchLibraries()
|
||||||
loadAllStats()
|
loadAllStats()
|
||||||
}
|
}
|
||||||
@@ -129,8 +159,20 @@ function formatBytes(bytes: number): string {
|
|||||||
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`
|
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function scanProgressLabel(libraryId: number): string {
|
||||||
|
const p = getProgress(libraryId)
|
||||||
|
if (!p) return ''
|
||||||
|
if (p.status === 'running') {
|
||||||
|
if (p.total === 0) return 'Scanning...'
|
||||||
|
const pct = Math.floor((p.processed / p.total) * 100)
|
||||||
|
return `Scanning ${pct}% (${p.processed}/${p.total})`
|
||||||
|
}
|
||||||
|
if (p.status === 'completed') return `Done - ${p.added} added, ${p.updated} updated`
|
||||||
|
if (p.status === 'failed') return p.errorMessage ? `Failed: ${p.errorMessage}` : 'Scan failed'
|
||||||
|
return ''
|
||||||
|
}
|
||||||
|
|
||||||
function iconComponent(iconName: string | null | undefined) {
|
function iconComponent(iconName: string | null | undefined) {
|
||||||
// Dynamic Lucide icon lookup via pre-imported default
|
|
||||||
return iconName ? null : Library
|
return iconName ? null : Library
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
@@ -150,7 +192,7 @@ function iconComponent(iconName: string | null | undefined) {
|
|||||||
@click="scanAll"
|
@click="scanAll"
|
||||||
>
|
>
|
||||||
<RefreshCw :size="12" :class="scanningAll ? 'animate-spin' : ''" />
|
<RefreshCw :size="12" :class="scanningAll ? 'animate-spin' : ''" />
|
||||||
{{ scanningAll ? 'Scanning…' : 'Scan All' }}
|
{{ scanningAll ? 'Scanning...' : 'Scan All' }}
|
||||||
</button>
|
</button>
|
||||||
<button
|
<button
|
||||||
class="flex items-center gap-1.5 text-xs font-medium px-3 py-1.5 rounded-md bg-primary text-primary-foreground hover:opacity-90 transition-opacity"
|
class="flex items-center gap-1.5 text-xs font-medium px-3 py-1.5 rounded-md bg-primary text-primary-foreground hover:opacity-90 transition-opacity"
|
||||||
@@ -166,11 +208,18 @@ function iconComponent(iconName: string | null | undefined) {
|
|||||||
<div class="border border-border rounded-lg overflow-hidden divide-y divide-border">
|
<div class="border border-border rounded-lg overflow-hidden divide-y divide-border">
|
||||||
<div v-for="lib in libraries" :key="lib.id" class="bg-card px-5 py-4">
|
<div v-for="lib in libraries" :key="lib.id" class="bg-card px-5 py-4">
|
||||||
<div class="flex items-center gap-4">
|
<div class="flex items-center gap-4">
|
||||||
<div class="flex h-10 w-10 shrink-0 items-center justify-center rounded-lg bg-primary/10">
|
<RouterLink
|
||||||
|
:to="{ name: 'library', params: { id: lib.id } }"
|
||||||
|
class="flex h-10 w-10 shrink-0 items-center justify-center rounded-lg bg-primary/10 hover:bg-primary/20 transition-colors"
|
||||||
|
>
|
||||||
<FolderOpen :size="17" class="text-primary" />
|
<FolderOpen :size="17" class="text-primary" />
|
||||||
</div>
|
</RouterLink>
|
||||||
<div class="flex-1 min-w-0">
|
<div class="flex-1 min-w-0">
|
||||||
<p class="text-sm font-medium text-foreground truncate">{{ lib.name }}</p>
|
<RouterLink
|
||||||
|
:to="{ name: 'library', params: { id: lib.id } }"
|
||||||
|
class="text-sm font-medium text-foreground hover:text-primary transition-colors truncate block"
|
||||||
|
>{{ lib.name }}</RouterLink
|
||||||
|
>
|
||||||
<div class="flex items-center gap-3 mt-0.5">
|
<div class="flex items-center gap-3 mt-0.5">
|
||||||
<span v-if="stats[lib.id]" class="text-xs text-muted-foreground">
|
<span v-if="stats[lib.id]" class="text-xs text-muted-foreground">
|
||||||
{{ stats[lib.id].totalBooks }} book{{ stats[lib.id].totalBooks === 1 ? '' : 's' }}
|
{{ stats[lib.id].totalBooks }} book{{ stats[lib.id].totalBooks === 1 ? '' : 's' }}
|
||||||
@@ -181,6 +230,24 @@ function iconComponent(iconName: string | null | undefined) {
|
|||||||
</span>
|
</span>
|
||||||
<span v-if="lib.watch" class="text-xs text-primary/70">Watching</span>
|
<span v-if="lib.watch" class="text-xs text-primary/70">Watching</span>
|
||||||
</div>
|
</div>
|
||||||
|
<!-- Scan progress bar -->
|
||||||
|
<div v-if="getProgress(lib.id)" class="mt-2">
|
||||||
|
<div class="flex items-center justify-between mb-1">
|
||||||
|
<span class="text-xs" :class="getProgress(lib.id)?.status === 'failed' ? 'text-destructive' : 'text-muted-foreground'">
|
||||||
|
{{ scanProgressLabel(lib.id) }}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<div v-if="getProgress(lib.id)?.status === 'running'" class="h-1 w-full rounded-full bg-muted overflow-hidden">
|
||||||
|
<div
|
||||||
|
class="h-full rounded-full bg-primary transition-all duration-300"
|
||||||
|
:style="{
|
||||||
|
width:
|
||||||
|
getProgress(lib.id)!.total > 0 ? `${Math.floor((getProgress(lib.id)!.processed / getProgress(lib.id)!.total) * 100)}%` : '100%',
|
||||||
|
animation: getProgress(lib.id)!.total === 0 ? 'pulse 1.5s ease-in-out infinite' : 'none',
|
||||||
|
}"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="flex items-center gap-1.5 shrink-0">
|
<div class="flex items-center gap-1.5 shrink-0">
|
||||||
<button
|
<button
|
||||||
@@ -199,11 +266,11 @@ function iconComponent(iconName: string | null | undefined) {
|
|||||||
</button>
|
</button>
|
||||||
<button
|
<button
|
||||||
class="flex items-center gap-1.5 text-xs font-medium px-3 py-1.5 rounded-md border border-border hover:bg-muted transition-colors text-muted-foreground hover:text-foreground disabled:opacity-50"
|
class="flex items-center gap-1.5 text-xs font-medium px-3 py-1.5 rounded-md border border-border hover:bg-muted transition-colors text-muted-foreground hover:text-foreground disabled:opacity-50"
|
||||||
:disabled="scanning[lib.id]"
|
:disabled="isScanning(lib.id)"
|
||||||
@click="scan(lib)"
|
@click="scan(lib)"
|
||||||
>
|
>
|
||||||
<RefreshCw :size="12" :class="scanning[lib.id] ? 'animate-spin' : ''" />
|
<RefreshCw :size="12" :class="isScanning(lib.id) ? 'animate-spin' : ''" />
|
||||||
{{ scanning[lib.id] ? 'Scanning…' : 'Scan' }}
|
{{ isScanning(lib.id) ? 'Scanning...' : 'Scan' }}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -16,6 +16,11 @@ export default defineConfig({
|
|||||||
server: {
|
server: {
|
||||||
proxy: {
|
proxy: {
|
||||||
'/api': 'http://localhost:3000',
|
'/api': 'http://localhost:3000',
|
||||||
|
'/socket.io': {
|
||||||
|
target: 'http://localhost:3000',
|
||||||
|
ws: true,
|
||||||
|
changeOrigin: true,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
export * from "./auth";
|
export * from "./auth";
|
||||||
export * from "./library";
|
export * from "./library";
|
||||||
export * from "./reader-settings";
|
export * from "./reader-settings";
|
||||||
|
export * from "./scanner";
|
||||||
|
|||||||
@@ -0,0 +1,11 @@
|
|||||||
|
export interface ScanProgressEvent {
|
||||||
|
jobId: number;
|
||||||
|
libraryId: number;
|
||||||
|
status: 'running' | 'completed' | 'failed';
|
||||||
|
processed: number;
|
||||||
|
total: number;
|
||||||
|
added: number;
|
||||||
|
updated: number;
|
||||||
|
missing: number;
|
||||||
|
errorMessage?: string;
|
||||||
|
}
|
||||||
+8
-4
@@ -24,6 +24,7 @@
|
|||||||
"test:e2e": "jest --config ./test/jest-e2e.json"
|
"test:e2e": "jest --config ./test/jest-e2e.json"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@projectx/types": "workspace:*",
|
||||||
"7z-wasm": "^1.2.0",
|
"7z-wasm": "^1.2.0",
|
||||||
"@fastify/cookie": "^11.0.2",
|
"@fastify/cookie": "^11.0.2",
|
||||||
"@fastify/formbody": "^8.0.2",
|
"@fastify/formbody": "^8.0.2",
|
||||||
@@ -34,8 +35,10 @@
|
|||||||
"@nestjs/core": "^11.0.1",
|
"@nestjs/core": "^11.0.1",
|
||||||
"@nestjs/jwt": "^11.0.2",
|
"@nestjs/jwt": "^11.0.2",
|
||||||
"@nestjs/passport": "^11.0.5",
|
"@nestjs/passport": "^11.0.5",
|
||||||
"@nestjs/platform-fastify": "^11.1.11",
|
"@nestjs/platform-fastify": "^11.1.12",
|
||||||
|
"@nestjs/platform-socket.io": "^11.1.12",
|
||||||
"@nestjs/schedule": "^6.1.0",
|
"@nestjs/schedule": "^6.1.0",
|
||||||
|
"@nestjs/websockets": "^11.1.12",
|
||||||
"bcryptjs": "^3.0.3",
|
"bcryptjs": "^3.0.3",
|
||||||
"chokidar": "^4.0.3",
|
"chokidar": "^4.0.3",
|
||||||
"class-transformer": "^0.5.1",
|
"class-transformer": "^0.5.1",
|
||||||
@@ -48,10 +51,11 @@
|
|||||||
"passport": "^0.7.0",
|
"passport": "^0.7.0",
|
||||||
"passport-jwt": "^4.0.1",
|
"passport-jwt": "^4.0.1",
|
||||||
"pdf-lib": "^1.17.1",
|
"pdf-lib": "^1.17.1",
|
||||||
"pg": "^8.16.3",
|
"pg": "^8.17.1",
|
||||||
"reflect-metadata": "^0.2.2",
|
"reflect-metadata": "^0.2.2",
|
||||||
"rxjs": "^7.8.1",
|
"rxjs": "^7.8.1",
|
||||||
"sharp": "^0.34.5",
|
"sharp": "^0.34.5",
|
||||||
|
"socket.io": "^4.8.3",
|
||||||
"unzipper": "^0.12.3"
|
"unzipper": "^0.12.3"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
@@ -60,8 +64,8 @@
|
|||||||
"@nestjs/cli": "^11.0.0",
|
"@nestjs/cli": "^11.0.0",
|
||||||
"@nestjs/schematics": "^11.0.0",
|
"@nestjs/schematics": "^11.0.0",
|
||||||
"@nestjs/testing": "^11.0.1",
|
"@nestjs/testing": "^11.0.1",
|
||||||
"@swc/cli": "^0.7.9",
|
"@swc/cli": "^0.7.10",
|
||||||
"@swc/core": "^1.15.8",
|
"@swc/core": "^1.15.10",
|
||||||
"@types/bcrypt": "^6.0.0",
|
"@types/bcrypt": "^6.0.0",
|
||||||
"@types/bcryptjs": "^3.0.0",
|
"@types/bcryptjs": "^3.0.0",
|
||||||
"@types/chokidar": "^1.7.5",
|
"@types/chokidar": "^1.7.5",
|
||||||
|
|||||||
@@ -5,14 +5,14 @@
|
|||||||
{
|
{
|
||||||
"idx": 0,
|
"idx": 0,
|
||||||
"version": "7",
|
"version": "7",
|
||||||
"when": 1768599599999,
|
"when": 1768909427779,
|
||||||
"tag": "0000_baseline",
|
"tag": "0000_baseline",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"idx": 1,
|
"idx": 1,
|
||||||
"version": "7",
|
"version": "7",
|
||||||
"when": 1768599599999,
|
"when": 1768909427779,
|
||||||
"tag": "0001_add_scan_jobs_cascade",
|
"tag": "0001_add_scan_jobs_cascade",
|
||||||
"breakpoints": true
|
"breakpoints": true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { NestFactory } from '@nestjs/core';
|
import { NestFactory } from '@nestjs/core';
|
||||||
import { FastifyAdapter, NestFastifyApplication } from '@nestjs/platform-fastify';
|
import { FastifyAdapter, NestFastifyApplication } from '@nestjs/platform-fastify';
|
||||||
import { ValidationPipe } from '@nestjs/common';
|
import { ValidationPipe } from '@nestjs/common';
|
||||||
|
import { IoAdapter } from '@nestjs/platform-socket.io';
|
||||||
import { AppModule } from './app.module';
|
import { AppModule } from './app.module';
|
||||||
import { GlobalExceptionFilter } from './common/filters/http-exception.filter';
|
import { GlobalExceptionFilter } from './common/filters/http-exception.filter';
|
||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
@@ -14,6 +15,8 @@ async function bootstrap() {
|
|||||||
await pool.end();
|
await pool.end();
|
||||||
const app = await NestFactory.create<NestFastifyApplication>(AppModule, new FastifyAdapter({ logger: true }));
|
const app = await NestFactory.create<NestFastifyApplication>(AppModule, new FastifyAdapter({ logger: true }));
|
||||||
|
|
||||||
|
app.useWebSocketAdapter(new IoAdapter(app));
|
||||||
|
|
||||||
app.setGlobalPrefix('api');
|
app.setGlobalPrefix('api');
|
||||||
|
|
||||||
app.useGlobalPipes(
|
app.useGlobalPipes(
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
import { Module } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
|
|
||||||
|
import { ScannerModule } from '../scanner/scanner.module';
|
||||||
import { LibraryController } from './library.controller';
|
import { LibraryController } from './library.controller';
|
||||||
import { LibraryRepository } from './library.repository';
|
import { LibraryRepository } from './library.repository';
|
||||||
import { LibraryService } from './library.service';
|
import { LibraryService } from './library.service';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
|
imports: [ScannerModule],
|
||||||
controllers: [LibraryController],
|
controllers: [LibraryController],
|
||||||
providers: [LibraryService, LibraryRepository],
|
providers: [LibraryService, LibraryRepository],
|
||||||
exports: [LibraryService],
|
exports: [LibraryService],
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { join } from 'path';
|
|||||||
|
|
||||||
import type { RequestUser } from '../../common/types/request-user';
|
import type { RequestUser } from '../../common/types/request-user';
|
||||||
import { isPrimaryFormat } from '../scanner/lib/classify';
|
import { isPrimaryFormat } from '../scanner/lib/classify';
|
||||||
|
import { ScannerService } from '../scanner/scanner.service';
|
||||||
import { CreateLibraryDto } from './dto/create-library.dto';
|
import { CreateLibraryDto } from './dto/create-library.dto';
|
||||||
import { GrantLibraryAccessDto } from './dto/grant-library-access.dto';
|
import { GrantLibraryAccessDto } from './dto/grant-library-access.dto';
|
||||||
import { PrescanLibraryDto } from './dto/prescan-library.dto';
|
import { PrescanLibraryDto } from './dto/prescan-library.dto';
|
||||||
@@ -20,6 +21,7 @@ export class LibraryService {
|
|||||||
constructor(
|
constructor(
|
||||||
private readonly libraryRepo: LibraryRepository,
|
private readonly libraryRepo: LibraryRepository,
|
||||||
private readonly config: ConfigService,
|
private readonly config: ConfigService,
|
||||||
|
private readonly scannerService: ScannerService,
|
||||||
) {
|
) {
|
||||||
this.booksPath = this.config.get<string>('storage.booksPath')!;
|
this.booksPath = this.config.get<string>('storage.booksPath')!;
|
||||||
}
|
}
|
||||||
@@ -62,6 +64,8 @@ export class LibraryService {
|
|||||||
|
|
||||||
const folders = await Promise.all(dto.folders.map((path) => this.libraryRepo.insertFolder({ libraryId: library.id, path })));
|
const folders = await Promise.all(dto.folders.map((path) => this.libraryRepo.insertFolder({ libraryId: library.id, path })));
|
||||||
|
|
||||||
|
this.scannerService.startScanAsync(library.id);
|
||||||
|
|
||||||
return { ...library, folders: folders.map(([f]) => f) };
|
return { ...library, folders: folders.map(([f]) => f) };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
import { Injectable } from '@nestjs/common';
|
||||||
|
|
||||||
|
export interface ScanEntry {
|
||||||
|
jobId: number;
|
||||||
|
libraryId: number;
|
||||||
|
processed: number;
|
||||||
|
total: number;
|
||||||
|
added: number;
|
||||||
|
updated: number;
|
||||||
|
missing: number;
|
||||||
|
lastEmitMs: number;
|
||||||
|
lastEmitPct: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class ScanJobStore {
|
||||||
|
private readonly byLibrary = new Map<number, ScanEntry>();
|
||||||
|
|
||||||
|
isRunning(libraryId: number): boolean {
|
||||||
|
return this.byLibrary.has(libraryId);
|
||||||
|
}
|
||||||
|
|
||||||
|
create(jobId: number, libraryId: number, total: number): ScanEntry {
|
||||||
|
const entry: ScanEntry = {
|
||||||
|
jobId,
|
||||||
|
libraryId,
|
||||||
|
processed: 0,
|
||||||
|
total,
|
||||||
|
added: 0,
|
||||||
|
updated: 0,
|
||||||
|
missing: 0,
|
||||||
|
lastEmitMs: 0,
|
||||||
|
lastEmitPct: -1,
|
||||||
|
};
|
||||||
|
this.byLibrary.set(libraryId, entry);
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
setTotal(libraryId: number, total: number): void {
|
||||||
|
const entry = this.byLibrary.get(libraryId);
|
||||||
|
if (entry) entry.total = total;
|
||||||
|
}
|
||||||
|
|
||||||
|
increment(libraryId: number, delta: { processed?: number; added?: number; updated?: number; missing?: number }): ScanEntry | undefined {
|
||||||
|
const entry = this.byLibrary.get(libraryId);
|
||||||
|
if (!entry) return undefined;
|
||||||
|
if (delta.processed !== undefined) entry.processed += delta.processed;
|
||||||
|
if (delta.added !== undefined) entry.added += delta.added;
|
||||||
|
if (delta.updated !== undefined) entry.updated += delta.updated;
|
||||||
|
if (delta.missing !== undefined) entry.missing += delta.missing;
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
get(libraryId: number): ScanEntry | undefined {
|
||||||
|
return this.byLibrary.get(libraryId);
|
||||||
|
}
|
||||||
|
|
||||||
|
shouldEmit(entry: ScanEntry): boolean {
|
||||||
|
if (entry.total > 0 && entry.processed >= entry.total) return true;
|
||||||
|
const pct = entry.total > 0 ? Math.floor((entry.processed / entry.total) * 100) : 0;
|
||||||
|
if (pct >= entry.lastEmitPct + 1) return true;
|
||||||
|
if (Date.now() - entry.lastEmitMs >= 1000) return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
markEmitted(entry: ScanEntry): void {
|
||||||
|
entry.lastEmitMs = Date.now();
|
||||||
|
entry.lastEmitPct = entry.total > 0 ? Math.floor((entry.processed / entry.total) * 100) : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
delete(libraryId: number): void {
|
||||||
|
this.byLibrary.delete(libraryId);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
import { Logger } from '@nestjs/common';
|
||||||
|
import { JwtService } from '@nestjs/jwt';
|
||||||
|
import {
|
||||||
|
OnGatewayConnection,
|
||||||
|
OnGatewayDisconnect,
|
||||||
|
SubscribeMessage,
|
||||||
|
WebSocketGateway,
|
||||||
|
WebSocketServer,
|
||||||
|
} from '@nestjs/websockets';
|
||||||
|
import { Server, Socket } from 'socket.io';
|
||||||
|
|
||||||
|
import type { ScanProgressEvent } from '@projectx/types';
|
||||||
|
import { AuthService } from '../auth/auth.service';
|
||||||
|
import { ScanJobStore } from './scan-job-store.service';
|
||||||
|
|
||||||
|
@WebSocketGateway({ namespace: '/scan', cors: { origin: '*', credentials: true } })
|
||||||
|
export class ScanGateway implements OnGatewayConnection, OnGatewayDisconnect {
|
||||||
|
@WebSocketServer() server: Server;
|
||||||
|
private readonly logger = new Logger(ScanGateway.name);
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly jwtService: JwtService,
|
||||||
|
private readonly authService: AuthService,
|
||||||
|
private readonly scanJobStore: ScanJobStore,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
async handleConnection(client: Socket): Promise<void> {
|
||||||
|
try {
|
||||||
|
const token = client.handshake.auth?.token as string | undefined;
|
||||||
|
if (!token) throw new Error('No token provided');
|
||||||
|
const payload = this.jwtService.verify<{ sub: number; ver: number }>(token);
|
||||||
|
const user = await this.authService.validateUser(payload.sub, payload.ver);
|
||||||
|
if (!user) throw new Error('User not found or token revoked');
|
||||||
|
client.data.user = user;
|
||||||
|
this.logger.debug(`WS connected: user=${user.id} socket=${client.id}`);
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.warn(`WS rejected: ${(err as Error).message} socket=${client.id}`);
|
||||||
|
client.disconnect();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
handleDisconnect(client: Socket): void {
|
||||||
|
this.logger.debug(`WS disconnected: socket=${client.id}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
@SubscribeMessage('subscribe:library')
|
||||||
|
handleSubscribeLibrary(client: Socket, libraryId: number): void {
|
||||||
|
client.join(`library:${libraryId}`);
|
||||||
|
// Send current snapshot immediately so reconnecting clients catch up.
|
||||||
|
const entry = this.scanJobStore.get(libraryId);
|
||||||
|
if (entry) {
|
||||||
|
const event: ScanProgressEvent = {
|
||||||
|
jobId: entry.jobId,
|
||||||
|
libraryId,
|
||||||
|
status: 'running',
|
||||||
|
processed: entry.processed,
|
||||||
|
total: entry.total,
|
||||||
|
added: entry.added,
|
||||||
|
updated: entry.updated,
|
||||||
|
missing: entry.missing,
|
||||||
|
};
|
||||||
|
client.emit('scan:progress', event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Called by ScannerService — not a WS message handler.
|
||||||
|
emitProgress(event: ScanProgressEvent): void {
|
||||||
|
this.server.to(`library:${event.libraryId}`).emit('scan:progress', event);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
import { Controller, Param, ParseIntPipe, Post } from '@nestjs/common';
|
import { Controller, HttpCode, HttpStatus, Param, ParseIntPipe, Post } from '@nestjs/common';
|
||||||
|
|
||||||
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
|
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
|
||||||
import { ScannerService } from './scanner.service';
|
import { ScannerService } from './scanner.service';
|
||||||
@@ -9,7 +9,8 @@ export class ScannerController {
|
|||||||
|
|
||||||
@Post('libraries/:id/scan')
|
@Post('libraries/:id/scan')
|
||||||
@RequirePermission('manage_libraries')
|
@RequirePermission('manage_libraries')
|
||||||
|
@HttpCode(HttpStatus.ACCEPTED)
|
||||||
scan(@Param('id', ParseIntPipe) libraryId: number) {
|
scan(@Param('id', ParseIntPipe) libraryId: number) {
|
||||||
return this.scannerService.scan(libraryId, 'manual');
|
return this.scannerService.startScan(libraryId, 'manual');
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,14 +1,29 @@
|
|||||||
import { Module } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
|
import { ConfigService } from '@nestjs/config';
|
||||||
|
import { JwtModule } from '@nestjs/jwt';
|
||||||
|
|
||||||
|
import { AuthModule } from '../auth/auth.module';
|
||||||
import { MetadataModule } from '../metadata/metadata.module';
|
import { MetadataModule } from '../metadata/metadata.module';
|
||||||
|
import { ScanGateway } from './scan.gateway';
|
||||||
|
import { ScanJobStore } from './scan-job-store.service';
|
||||||
import { ScannerController } from './scanner.controller';
|
import { ScannerController } from './scanner.controller';
|
||||||
import { ScannerRepository } from './scanner.repository';
|
import { ScannerRepository } from './scanner.repository';
|
||||||
import { ScannerService } from './scanner.service';
|
import { ScannerService } from './scanner.service';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [MetadataModule],
|
imports: [
|
||||||
|
MetadataModule,
|
||||||
|
AuthModule,
|
||||||
|
JwtModule.registerAsync({
|
||||||
|
inject: [ConfigService],
|
||||||
|
useFactory: (config: ConfigService) => ({
|
||||||
|
secret: config.get<string>('auth.jwtSecret'),
|
||||||
|
signOptions: { expiresIn: config.get('auth.jwtExpiresIn') as any },
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
],
|
||||||
controllers: [ScannerController],
|
controllers: [ScannerController],
|
||||||
providers: [ScannerService, ScannerRepository],
|
providers: [ScannerService, ScannerRepository, ScanGateway, ScanJobStore],
|
||||||
exports: [ScannerService],
|
exports: [ScannerService],
|
||||||
})
|
})
|
||||||
export class ScannerModule {}
|
export class ScannerModule {}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { Inject, Injectable } from '@nestjs/common';
|
import { Inject, Injectable } from '@nestjs/common';
|
||||||
import { and, eq, inArray } from 'drizzle-orm';
|
import { and, eq, inArray, ne } from 'drizzle-orm';
|
||||||
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||||
|
|
||||||
import { DB } from '../../db';
|
import { DB } from '../../db';
|
||||||
@@ -30,6 +30,13 @@ export class ScannerRepository {
|
|||||||
await this.db.update(scanJobs).set({ status: 'failed', errorMessage, completedAt: new Date() }).where(eq(scanJobs.id, id));
|
await this.db.update(scanJobs).set({ status: 'failed', errorMessage, completedAt: new Date() }).where(eq(scanJobs.id, id));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async failAllRunningJobs(errorMessage: string): Promise<void> {
|
||||||
|
await this.db
|
||||||
|
.update(scanJobs)
|
||||||
|
.set({ status: 'failed', errorMessage, completedAt: new Date() })
|
||||||
|
.where(eq(scanJobs.status, 'running'));
|
||||||
|
}
|
||||||
|
|
||||||
// ── Library Folders ────────────────────────────────────────────────────────
|
// ── Library Folders ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async findLibraryFolders(libraryId: number) {
|
async findLibraryFolders(libraryId: number) {
|
||||||
|
|||||||
@@ -1,14 +1,18 @@
|
|||||||
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
|
import { ConflictException, Injectable, Logger, NotFoundException, OnApplicationBootstrap } from '@nestjs/common';
|
||||||
|
|
||||||
const METADATA_FORMATS = new Set(['epub', 'mobi', 'azw3', 'azw', 'cbz', 'cbr', 'cb7', 'pdf']);
|
|
||||||
|
|
||||||
|
import type { ScanProgressEvent } from '@projectx/types';
|
||||||
import { MetadataService } from '../metadata/metadata.service';
|
import { MetadataService } from '../metadata/metadata.service';
|
||||||
|
import { ScanGateway } from './scan.gateway';
|
||||||
|
import { ScanJobStore } from './scan-job-store.service';
|
||||||
import { classifyFile } from './lib/classify';
|
import { classifyFile } from './lib/classify';
|
||||||
import { sha256File } from './lib/hash';
|
import { sha256File } from './lib/hash';
|
||||||
import { waitForStability } from './lib/stability';
|
import { waitForStability } from './lib/stability';
|
||||||
import { BookCandidate, FileStat, findBookCandidates } from './lib/walk';
|
import { BookCandidate, FileStat, findBookCandidates } from './lib/walk';
|
||||||
import { ScannerRepository } from './scanner.repository';
|
import { ScannerRepository } from './scanner.repository';
|
||||||
|
|
||||||
|
const METADATA_FORMATS = new Set(['epub', 'mobi', 'azw3', 'azw', 'cbz', 'cbr', 'cb7', 'pdf']);
|
||||||
|
const BATCH_SIZE = 5;
|
||||||
|
|
||||||
interface ScanCounts {
|
interface ScanCounts {
|
||||||
addedCount: number;
|
addedCount: number;
|
||||||
updatedCount: number;
|
updatedCount: number;
|
||||||
@@ -16,86 +20,207 @@ interface ScanCounts {
|
|||||||
}
|
}
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class ScannerService {
|
export class ScannerService implements OnApplicationBootstrap {
|
||||||
private readonly logger = new Logger(ScannerService.name);
|
private readonly logger = new Logger(ScannerService.name);
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
private readonly scannerRepo: ScannerRepository,
|
private readonly scannerRepo: ScannerRepository,
|
||||||
private readonly metadataService: MetadataService,
|
private readonly metadataService: MetadataService,
|
||||||
|
private readonly scanJobStore: ScanJobStore,
|
||||||
|
private readonly scanGateway: ScanGateway,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
async scan(libraryId: number, triggeredBy: 'manual' | 'watcher' | 'schedule' = 'manual') {
|
async onApplicationBootstrap(): Promise<void> {
|
||||||
|
await this.scannerRepo.failAllRunningJobs('Server restarted during scan');
|
||||||
|
}
|
||||||
|
|
||||||
|
async startScan(libraryId: number, triggeredBy: 'manual' | 'watcher' | 'schedule'): Promise<{ jobId: number }> {
|
||||||
|
if (this.scanJobStore.isRunning(libraryId)) {
|
||||||
|
throw new ConflictException(`A scan is already running for library ${libraryId}`);
|
||||||
|
}
|
||||||
|
|
||||||
const folders = await this.scannerRepo.findLibraryFolders(libraryId);
|
const folders = await this.scannerRepo.findLibraryFolders(libraryId);
|
||||||
if (folders.length === 0) throw new NotFoundException(`Library ${libraryId} has no folders`);
|
if (folders.length === 0) throw new NotFoundException(`Library ${libraryId} has no folders`);
|
||||||
|
|
||||||
const job = await this.scannerRepo.createScanJob(libraryId, triggeredBy);
|
const job = await this.scannerRepo.createScanJob(libraryId, triggeredBy);
|
||||||
this.logger.log(`Scan job ${job.id} started for library ${libraryId}`);
|
|
||||||
|
this.scanJobStore.create(job.id, libraryId, 0);
|
||||||
|
this.emitFromStore(libraryId, job.id, 'running');
|
||||||
|
|
||||||
|
this.runScan(libraryId, job.id, folders).catch((err) =>
|
||||||
|
this.logger.error(`Scan job ${job.id} crashed unexpectedly: ${(err as Error).message}`),
|
||||||
|
);
|
||||||
|
|
||||||
|
return { jobId: job.id };
|
||||||
|
}
|
||||||
|
|
||||||
|
startScanAsync(libraryId: number): void {
|
||||||
|
if (this.scanJobStore.isRunning(libraryId)) return;
|
||||||
|
this.startScan(libraryId, 'manual').catch((err) =>
|
||||||
|
this.logger.error(`Auto-scan failed to start for library ${libraryId}: ${(err as Error).message}`),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async runScan(
|
||||||
|
libraryId: number,
|
||||||
|
jobId: number,
|
||||||
|
folders: Awaited<ReturnType<ScannerRepository['findLibraryFolders']>>,
|
||||||
|
): Promise<void> {
|
||||||
|
this.logger.log(`Scan job ${jobId} started for library ${libraryId}`);
|
||||||
|
|
||||||
|
type FolderWork = {
|
||||||
|
id: number;
|
||||||
|
libraryId: number;
|
||||||
|
path: string;
|
||||||
|
candidates: BookCandidate[];
|
||||||
|
knownBooks: Awaited<ReturnType<ScannerRepository['findBooksByLibraryFolder']>>;
|
||||||
|
knownFiles: Awaited<ReturnType<ScannerRepository['findBookFilesByLibraryFolder']>>;
|
||||||
|
};
|
||||||
|
|
||||||
|
const folderWork: FolderWork[] = [];
|
||||||
|
let totalCandidates = 0;
|
||||||
|
|
||||||
|
for (const folder of folders) {
|
||||||
|
let candidates: BookCandidate[] = [];
|
||||||
|
try {
|
||||||
|
candidates = await findBookCandidates(folder.path);
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.warn(`Cannot walk ${folder.path}: ${err}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const [knownBooks, knownFiles] = await Promise.all([
|
||||||
|
this.scannerRepo.findBooksByLibraryFolder(folder.id),
|
||||||
|
this.scannerRepo.findBookFilesByLibraryFolder(folder.id),
|
||||||
|
]);
|
||||||
|
|
||||||
|
folderWork.push({ ...folder, candidates, knownBooks, knownFiles });
|
||||||
|
totalCandidates += candidates.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.scanJobStore.setTotal(libraryId, totalCandidates);
|
||||||
|
this.emitFromStore(libraryId, jobId, 'running');
|
||||||
|
|
||||||
const totals: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
|
const totals: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
|
||||||
|
|
||||||
try {
|
try {
|
||||||
for (const folder of folders) {
|
for (const { id: folderId, path: folderPath, candidates, knownBooks, knownFiles } of folderWork) {
|
||||||
this.logger.log(`Scanning folder: ${folder.path}`);
|
const counts = await this.scanFolderCandidates(
|
||||||
const counts = await this.scanFolder(folder.id, folder.libraryId, folder.path);
|
folderId,
|
||||||
|
libraryId,
|
||||||
|
folderPath,
|
||||||
|
candidates,
|
||||||
|
knownBooks,
|
||||||
|
knownFiles,
|
||||||
|
jobId,
|
||||||
|
);
|
||||||
totals.addedCount += counts.addedCount;
|
totals.addedCount += counts.addedCount;
|
||||||
totals.updatedCount += counts.updatedCount;
|
totals.updatedCount += counts.updatedCount;
|
||||||
totals.missingCount += counts.missingCount;
|
totals.missingCount += counts.missingCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.scannerRepo.completeScanJob(job.id, totals);
|
await this.scannerRepo.completeScanJob(jobId, totals);
|
||||||
this.logger.log(`Scan job ${job.id} completed — ${JSON.stringify(totals)}`);
|
this.logger.log(`Scan job ${jobId} completed — ${JSON.stringify(totals)}`);
|
||||||
|
this.emitFromStore(libraryId, jobId, 'completed');
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
await this.scannerRepo.failScanJob(job.id, message);
|
await this.scannerRepo.failScanJob(jobId, message).catch(() => {
|
||||||
this.logger.error(`Scan job ${job.id} failed: ${message}`);
|
// Job row may have been cascade-deleted if library was deleted.
|
||||||
throw err;
|
});
|
||||||
|
this.logger.error(`Scan job ${jobId} failed: ${message}`);
|
||||||
|
this.emitFromStore(libraryId, jobId, 'failed', message);
|
||||||
|
} finally {
|
||||||
|
this.scanJobStore.delete(libraryId);
|
||||||
}
|
}
|
||||||
|
|
||||||
return { jobId: job.id, ...totals };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async scanFolder(libraryFolderId: number, libraryId: number, folderPath: string): Promise<ScanCounts> {
|
private async scanFolderCandidates(
|
||||||
|
libraryFolderId: number,
|
||||||
|
libraryId: number,
|
||||||
|
folderPath: string,
|
||||||
|
candidates: BookCandidate[],
|
||||||
|
knownBooks: Awaited<ReturnType<ScannerRepository['findBooksByLibraryFolder']>>,
|
||||||
|
knownFiles: Awaited<ReturnType<ScannerRepository['findBookFilesByLibraryFolder']>>,
|
||||||
|
jobId: number,
|
||||||
|
): Promise<ScanCounts> {
|
||||||
const counts: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
|
const counts: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
|
||||||
|
|
||||||
const [knownBooks, knownFiles] = await Promise.all([
|
|
||||||
this.scannerRepo.findBooksByLibraryFolder(libraryFolderId),
|
|
||||||
this.scannerRepo.findBookFilesByLibraryFolder(libraryFolderId),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const bookByFolderPath = new Map(knownBooks.map((b) => [b.folderPath, b]));
|
const bookByFolderPath = new Map(knownBooks.map((b) => [b.folderPath, b]));
|
||||||
const fileByPath = new Map(knownFiles.map((f) => [f.absolutePath, f]));
|
const fileByPath = new Map(knownFiles.map((f) => [f.absolutePath, f]));
|
||||||
const fileByIno = new Map(knownFiles.map((f) => [f.ino, f]));
|
const fileByIno = new Map(knownFiles.map((f) => [f.ino, f]));
|
||||||
const seenBookIds = new Set<number>();
|
const seenBookIds = new Set<number>();
|
||||||
|
|
||||||
let candidates: BookCandidate[];
|
for (let i = 0; i < candidates.length; i += BATCH_SIZE) {
|
||||||
try {
|
const batch = candidates.slice(i, i + BATCH_SIZE);
|
||||||
candidates = await findBookCandidates(folderPath);
|
|
||||||
} catch (err) {
|
|
||||||
this.logger.warn(`Cannot walk ${folderPath}: ${err}`);
|
|
||||||
return counts;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const candidate of candidates) {
|
const results = await Promise.all(
|
||||||
const book = await this.upsertBook(candidate, libraryId, libraryFolderId, bookByFolderPath, counts);
|
batch.map((c) => this.processCandidate(c, libraryId, libraryFolderId, bookByFolderPath, fileByPath, fileByIno)),
|
||||||
seenBookIds.add(book.id);
|
);
|
||||||
|
|
||||||
for (const fileStat of candidate.files) {
|
for (const r of results) {
|
||||||
const { format } = classifyFile(fileStat.absolutePath);
|
seenBookIds.add(r.bookId);
|
||||||
const isNew = await this.processFile(fileStat, book.id, libraryFolderId, fileByPath, fileByIno, counts);
|
counts.addedCount += r.added;
|
||||||
if (isNew && format && METADATA_FORMATS.has(format)) {
|
counts.updatedCount += r.updated;
|
||||||
await this.metadataService.extractAndSave(book.id, fileStat.absolutePath, format);
|
}
|
||||||
}
|
|
||||||
|
const entry = this.scanJobStore.increment(libraryId, { processed: batch.length });
|
||||||
|
if (entry && this.scanJobStore.shouldEmit(entry)) {
|
||||||
|
this.emitFromStore(libraryId, jobId, 'running');
|
||||||
|
this.scanJobStore.markEmitted(entry);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const missingIds = knownBooks.filter((b) => !seenBookIds.has(b.id)).map((b) => b.id);
|
const missingIds = knownBooks.filter((b) => !seenBookIds.has(b.id)).map((b) => b.id);
|
||||||
|
if (missingIds.length > 0) {
|
||||||
await this.scannerRepo.markBooksAsMissing(missingIds);
|
await this.scannerRepo.markBooksAsMissing(missingIds);
|
||||||
counts.missingCount += missingIds.length;
|
counts.missingCount += missingIds.length;
|
||||||
|
this.scanJobStore.increment(libraryId, { missing: missingIds.length });
|
||||||
|
}
|
||||||
|
|
||||||
return counts;
|
return counts;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async processCandidate(
|
||||||
|
candidate: BookCandidate,
|
||||||
|
libraryId: number,
|
||||||
|
libraryFolderId: number,
|
||||||
|
bookByFolderPath: Map<string, { id: number; status: string; folderPath: string }>,
|
||||||
|
fileByPath: Map<string, { id: number; ino: number; sizeBytes: number | null; mtime: Date | null; hash: string | null }>,
|
||||||
|
fileByIno: Map<number, { id: number; absolutePath: string }>,
|
||||||
|
): Promise<{ bookId: number; added: number; updated: number }> {
|
||||||
|
const counts = { added: 0, updated: 0 };
|
||||||
|
const fileCounts: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
|
||||||
|
|
||||||
|
const book = await this.upsertBook(candidate, libraryId, libraryFolderId, bookByFolderPath, fileCounts);
|
||||||
|
counts.added += fileCounts.addedCount;
|
||||||
|
counts.updated += fileCounts.updatedCount;
|
||||||
|
|
||||||
|
for (const fileStat of candidate.files) {
|
||||||
|
const { format } = classifyFile(fileStat.absolutePath);
|
||||||
|
const fileCount: ScanCounts = { addedCount: 0, updatedCount: 0, missingCount: 0 };
|
||||||
|
let isNew: boolean;
|
||||||
|
|
||||||
|
try {
|
||||||
|
isNew = await this.processFile(fileStat, book.id, libraryFolderId, fileByPath, fileByIno, fileCount);
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.warn(`Failed to process file ${fileStat.absolutePath}: ${err}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
counts.added += fileCount.addedCount;
|
||||||
|
counts.updated += fileCount.updatedCount;
|
||||||
|
|
||||||
|
if (isNew && format && METADATA_FORMATS.has(format)) {
|
||||||
|
try {
|
||||||
|
await this.metadataService.extractAndSave(book.id, fileStat.absolutePath, format);
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.warn(`Metadata extraction failed for ${fileStat.absolutePath}: ${err}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { bookId: book.id, ...counts };
|
||||||
|
}
|
||||||
|
|
||||||
private async upsertBook(
|
private async upsertBook(
|
||||||
candidate: BookCandidate,
|
candidate: BookCandidate,
|
||||||
libraryId: number,
|
libraryId: number,
|
||||||
@@ -124,7 +249,6 @@ export class ScannerService {
|
|||||||
return existing;
|
return existing;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Returns true if a new file record was created (triggers metadata extraction).
|
|
||||||
private async processFile(
|
private async processFile(
|
||||||
fileStat: FileStat,
|
fileStat: FileStat,
|
||||||
bookId: number,
|
bookId: number,
|
||||||
@@ -141,7 +265,6 @@ export class ScannerService {
|
|||||||
const byPath = fileByPath.get(fileStat.absolutePath);
|
const byPath = fileByPath.get(fileStat.absolutePath);
|
||||||
if (byPath) {
|
if (byPath) {
|
||||||
const changed = fileStat.sizeBytes !== byPath.sizeBytes || fileStat.mtime.getTime() !== byPath.mtime?.getTime();
|
const changed = fileStat.sizeBytes !== byPath.sizeBytes || fileStat.mtime.getTime() !== byPath.mtime?.getTime();
|
||||||
|
|
||||||
if (changed) {
|
if (changed) {
|
||||||
await this.scannerRepo.updateBookFile(byPath.id, {
|
await this.scannerRepo.updateBookFile(byPath.id, {
|
||||||
ino: fileStat.ino,
|
ino: fileStat.ino,
|
||||||
@@ -203,4 +326,20 @@ export class ScannerService {
|
|||||||
counts.addedCount++;
|
counts.addedCount++;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private emitFromStore(libraryId: number, jobId: number, status: 'running' | 'completed' | 'failed', errorMessage?: string): void {
|
||||||
|
const entry = this.scanJobStore.get(libraryId);
|
||||||
|
const event: ScanProgressEvent = {
|
||||||
|
jobId,
|
||||||
|
libraryId,
|
||||||
|
status,
|
||||||
|
processed: entry?.processed ?? 0,
|
||||||
|
total: entry?.total ?? 0,
|
||||||
|
added: entry?.added ?? 0,
|
||||||
|
updated: entry?.updated ?? 0,
|
||||||
|
missing: entry?.missing ?? 0,
|
||||||
|
errorMessage,
|
||||||
|
};
|
||||||
|
this.scanGateway.emitProgress(event);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user