feat: add epub streaming with server-side file serving

This commit is contained in:
neon
2026-03-04 04:53:52 -06:00
parent 94a0d51584
commit 38a601675e
15 changed files with 471 additions and 61 deletions
-1
View File
@@ -871,7 +871,6 @@ class Loader {
const directUrl = this.#getDirectUrl(path) const directUrl = this.#getDirectUrl(path)
if (directUrl) { if (directUrl) {
el.setAttribute(attr, directUrl) el.setAttribute(attr, directUrl)
el.setAttribute('loading', 'lazy') // Native lazy loading hint
return true return true
} }
return false return false
@@ -35,24 +35,6 @@ export const makeStreamingLoader = (bookId, baseUrl, bookInfo, authToken = null,
} }
} }
// Build URL with token for browser-initiated requests (fonts, images in CSS)
const getDirectFileUrl = (name) => {
if (!name) return null
const encodedPath = name.split('/').map(encodeURIComponent).join('/')
let url = `${baseUrl}/${bookId}/file/${encodedPath}`
const params = []
if (bookType) {
params.push(`bookType=${encodeURIComponent(bookType)}`)
}
if (authToken) {
params.push(`token=${encodeURIComponent(authToken)}`)
}
if (params.length > 0) {
url += '?' + params.join('&')
}
return url
}
/** /**
* Load file as text * Load file as text
*/ */
@@ -105,24 +87,13 @@ export const makeStreamingLoader = (bookId, baseUrl, bookInfo, authToken = null,
return item?.size ?? 0 return item?.size ?? 0
} }
/**
* Get direct URL for a file without fetching it.
* This allows the browser to fetch resources on-demand (fonts, images).
* Note: Uses token in URL since browser-initiated requests can't use headers.
* @param {string} name - File path within the EPUB
* @returns {string|null} Direct URL to the file
*/
const getDirectUrl = (name) => {
if (!name) return null
return getDirectFileUrl(name)
}
return { return {
loadText, loadText,
loadBlob, loadBlob,
getSize, getSize,
getDirectUrl, // For lazy loading of fonts/images in CSS // getDirectUrl intentionally omitted: browser-initiated fetches from inside a
// Expose for debugging // sandboxed blob-URL iframe are blocked by Cross-Origin-Resource-Policy headers.
// Images go through loadBlob (JS fetch with Authorization header) instead.
_bookInfo: bookInfo, _bookInfo: bookInfo,
_manifestMap: manifestMap, _manifestMap: manifestMap,
} }
+1 -1
View File
@@ -145,7 +145,7 @@ onMounted(async () => {
} }
} }
await open(fileId, fileFormat, progress.cfi.value) await open(bookId, fileId, fileFormat, progress.cfi.value)
setChapters(getChapters()) setChapters(getChapters())
sectionFractions.value = getSectionFractions() sectionFractions.value = getSectionFractions()
await bookmarks.load(bookId) await bookmarks.load(bookId)
@@ -1,5 +1,5 @@
import { onUnmounted, ref } from 'vue' import { onUnmounted, ref } from 'vue'
import { api } from '@/lib/api' import { api, getAccessToken } from '@/lib/api'
import { useFoliateAnnotations } from './useFoliateAnnotations' import { useFoliateAnnotations } from './useFoliateAnnotations'
import { useFoliateSelection } from './useFoliateSelection' import { useFoliateSelection } from './useFoliateSelection'
import { useFoliateInput } from './useFoliateInput' import { useFoliateInput } from './useFoliateInput'
@@ -38,10 +38,11 @@ export function useFoliate(
async function loadScript() { async function loadScript() {
if (customElements.get('foliate-view')) return if (customElements.get('foliate-view')) return
const src = import.meta.env.DEV ? `/assets/foliate/view.js?v=${Date.now()}` : '/assets/foliate/view.js'
await new Promise<void>((resolve, reject) => { await new Promise<void>((resolve, reject) => {
const script = document.createElement('script') const script = document.createElement('script')
script.type = 'module' script.type = 'module'
script.src = '/assets/foliate/view.js' script.src = src
script.onload = () => setTimeout(resolve, 100) script.onload = () => setTimeout(resolve, 100)
script.onerror = () => reject(new Error('Failed to load foliate/view.js')) script.onerror = () => reject(new Error('Failed to load foliate/view.js'))
document.head.appendChild(script) document.head.appendChild(script)
@@ -49,7 +50,7 @@ export function useFoliate(
await customElements.whenDefined('foliate-view') await customElements.whenDefined('foliate-view')
} }
async function open(fileId: number, format: string, cfi?: string | null) { async function open(bookId: number, fileId: number, format: string, cfi?: string | null) {
const el = container() const el = container()
if (!el) return if (!el) return
@@ -104,13 +105,25 @@ export function useFoliate(
loading.value = false loading.value = false
}) })
const mimeType = format === 'pdf' ? 'application/pdf' : format === 'cbz' ? 'application/zip' : 'application/epub+zip' if (format === 'epub') {
const ext = format === 'pdf' ? 'pdf' : format === 'cbz' ? 'cbz' : 'epub' const infoRes = await api(`/api/v1/epub/${bookId}/info`)
const res = await api(`/api/v1/books/files/${fileId}/serve`) if (!infoRes.ok) throw new Error(`Failed to fetch EPUB info: ${infoRes.status}`)
if (!res.ok) throw new Error(`Failed to fetch book file: ${res.status}`) const bookInfo = await infoRes.json()
const blob = await res.blob() const makeStreamingBook = (window as Record<string, unknown>).makeStreamingBook as
const file = new File([blob], `book-file-${fileId}.${ext}`, { type: mimeType }) | ((id: number, base: string, info: unknown, token: string | null) => Promise<unknown>)
await view.open(file) | undefined
if (!makeStreamingBook) throw new Error('makeStreamingBook not available')
const book = await makeStreamingBook(bookId, '/api/v1/epub', bookInfo, getAccessToken())
await view.open(book as never)
} else {
const mimeType = format === 'pdf' ? 'application/pdf' : 'application/zip'
const ext = format === 'pdf' ? 'pdf' : format === 'cbz' ? 'cbz' : format
const res = await api(`/api/v1/books/files/${fileId}/serve`)
if (!res.ok) throw new Error(`Failed to fetch book file: ${res.status}`)
const blob = await res.blob()
const file = new File([blob], `book-file-${fileId}.${ext}`, { type: mimeType })
await view.open(file)
}
if (onApplyStyles) onApplyStyles(view.renderer) if (onApplyStyles) onApplyStyles(view.renderer)
await view.goTo(cfi ?? 0).catch(() => {}) await view.goTo(cfi ?? 0).catch(() => {})
} catch (e) { } catch (e) {
@@ -146,7 +159,7 @@ export function useFoliate(
error, error,
fraction, fraction,
view: viewRef, view: viewRef,
open, open: (bookId: number, fileId: number, format: string, cfi?: string | null) => open(bookId, fileId, format, cfi),
prev: () => getViewEl()?.prev?.(), prev: () => getViewEl()?.prev?.(),
next: () => getViewEl()?.next?.(), next: () => getViewEl()?.next?.(),
goTo: (t: string | number) => getViewEl()?.goTo?.(t), goTo: (t: string | number) => getViewEl()?.goTo?.(t),
+30
View File
@@ -0,0 +1,30 @@
export interface EpubManifestItem {
id: string
href: string
mediaType: string
properties?: string[]
size: number
}
export interface EpubSpineItem {
idref: string
href: string
mediaType: string
linear: boolean
}
export interface EpubTocItem {
label: string
href?: string
children?: EpubTocItem[]
}
export interface EpubBookInfo {
containerPath: string
rootPath: string
spine: EpubSpineItem[]
manifest: EpubManifestItem[]
toc: EpubTocItem | null
metadata: Record<string, unknown>
coverPath: string | null
}
+1
View File
@@ -14,3 +14,4 @@ export * from "./opds";
export * from "./kobo"; export * from "./kobo";
export * from "./staging"; export * from "./staging";
export * from "./file-write"; export * from "./file-write";
export * from "./epub";
+2
View File
@@ -37,6 +37,7 @@ import { RoleModule } from './modules/role/role.module';
import { ScannerModule } from './modules/scanner/scanner.module'; import { ScannerModule } from './modules/scanner/scanner.module';
import { SeedModule } from './modules/seed/seed.module'; import { SeedModule } from './modules/seed/seed.module';
import { EmailModule } from './modules/email/email.module'; import { EmailModule } from './modules/email/email.module';
import { EpubModule } from './modules/epub/epub.module';
import { StagingModule } from './modules/staging/staging.module'; import { StagingModule } from './modules/staging/staging.module';
import { UploadModule } from './modules/upload/upload.module'; import { UploadModule } from './modules/upload/upload.module';
import { UserModule } from './modules/user/user.module'; import { UserModule } from './modules/user/user.module';
@@ -78,6 +79,7 @@ import { UserModule } from './modules/user/user.module';
HealthModule, HealthModule,
CbzModule, CbzModule,
ReaderPreferencesModule, ReaderPreferencesModule,
EpubModule,
StagingModule, StagingModule,
EmailModule, EmailModule,
], ],
+4 -1
View File
@@ -101,7 +101,10 @@ async function bootstrap() {
max: 100, max: 100,
timeWindow: '1 minute', timeWindow: '1 minute',
allowList: (req: { cookies?: { access_token?: string }; headers?: { authorization?: string }; url?: string }) => allowList: (req: { cookies?: { access_token?: string }; headers?: { authorization?: string }; url?: string }) =>
!!req.cookies?.access_token || !!req.headers?.authorization || /^\/api\/v1\/kobo\//.test(req.url ?? ''), !!req.cookies?.access_token ||
!!req.headers?.authorization ||
/^\/api\/v1\/kobo\//.test(req.url ?? '') ||
/^\/api\/v1\/epub\/\d+\/file\//.test(req.url ?? ''),
}); });
if (process.env.NODE_ENV !== 'production') { if (process.env.NODE_ENV !== 'production') {
+1
View File
@@ -15,6 +15,7 @@ export class JwtStrategy extends PassportStrategy(Strategy) {
jwtFromRequest: ExtractJwt.fromExtractors([ jwtFromRequest: ExtractJwt.fromExtractors([
ExtractJwt.fromAuthHeaderAsBearerToken(), ExtractJwt.fromAuthHeaderAsBearerToken(),
(req: { cookies?: Record<string, string | undefined> }) => req?.cookies?.access_token ?? null, (req: { cookies?: Record<string, string | undefined> }) => req?.cookies?.access_token ?? null,
(req: { query?: Record<string, string | undefined> }) => req?.query?.token ?? null,
]), ]),
secretOrKey: config.get<string>('auth.jwtSecret')!, secretOrKey: config.get<string>('auth.jwtSecret')!,
}); });
@@ -0,0 +1,39 @@
import { Controller, Get, Param, ParseIntPipe, Res } from '@nestjs/common';
import type { FastifyReply } from 'fastify';
import { CurrentUser } from '../../common/decorators/current-user.decorator';
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
import type { RequestUser } from '../../common/types/request-user';
import { EpubService } from './epub.service';
@Controller('epub')
export class EpubController {
constructor(private readonly epubService: EpubService) {}
@Get(':bookId/info')
@RequirePermission('library_download')
getBookInfo(@Param('bookId', ParseIntPipe) bookId: number, @CurrentUser() user: RequestUser) {
return this.epubService.getBookInfo(bookId, user);
}
@Get(':bookId/file/*')
@RequirePermission('library_download')
async getFile(
@Param('bookId', ParseIntPipe) bookId: number,
@Param('*') encodedPath: string,
@CurrentUser() user: RequestUser,
@Res() reply: FastifyReply,
) {
const filePath = encodedPath
.split('/')
.map((s) => decodeURIComponent(s))
.join('/');
const { stream, contentType, size } = await this.epubService.streamFile(bookId, filePath, user);
reply.header('Content-Type', contentType);
if (size > 0) reply.header('Content-Length', size);
reply.header('Cache-Control', 'public, max-age=3600');
reply.send(stream);
}
}
+13
View File
@@ -0,0 +1,13 @@
import { Module } from '@nestjs/common';
import { BookModule } from '../book/book.module';
import { LibraryModule } from '../library/library.module';
import { EpubController } from './epub.controller';
import { EpubService } from './epub.service';
@Module({
imports: [BookModule, LibraryModule],
controllers: [EpubController],
providers: [EpubService],
})
export class EpubModule {}
+324
View File
@@ -0,0 +1,324 @@
import { ForbiddenException, Injectable, Logger, NotFoundException } from '@nestjs/common';
import { stat } from 'fs/promises';
import * as unzipper from 'unzipper';
import { XMLParser } from 'fast-xml-parser';
import type { EpubBookInfo, EpubManifestItem, EpubSpineItem, EpubTocItem } from '@projectx/types';
import { BookRepository } from '../book/book.repository';
import { LibraryService } from '../library/library.service';
import type { RequestUser } from '../../common/types/request-user';
const CONTENT_TYPES: Record<string, string> = {
'.xhtml': 'application/xhtml+xml',
'.html': 'application/xhtml+xml',
'.htm': 'application/xhtml+xml',
'.css': 'text/css',
'.js': 'application/javascript',
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.png': 'image/png',
'.gif': 'image/gif',
'.svg': 'image/svg+xml',
'.webp': 'image/webp',
'.woff': 'font/woff',
'.woff2': 'font/woff2',
'.ttf': 'font/ttf',
'.otf': 'font/otf',
'.eot': 'application/vnd.ms-fontobject',
'.xml': 'application/xml',
'.ncx': 'application/x-dtbncx+xml',
'.opf': 'application/oebps-package+xml',
'.smil': 'application/smil+xml',
'.mp3': 'audio/mpeg',
'.mp4': 'video/mp4',
'.webm': 'video/webm',
};
const MAX_CACHE = 50;
interface CacheEntry {
info: EpubBookInfo;
mtime: number;
validPaths: Set<string>;
lastAccessed: number;
}
const xmlParser = new XMLParser({
ignoreAttributes: false,
attributeNamePrefix: '@_',
textNodeName: '#text',
});
function toArray<T>(v: T | T[] | undefined | null): T[] {
if (v == null) return [];
return Array.isArray(v) ? v : [v];
}
function getText(v: unknown): string | null {
if (typeof v === 'string') return v.trim() || null;
if (typeof v === 'number') return String(v);
if (v != null && typeof v === 'object') {
const text = (v as Record<string, unknown>)['#text'];
if (typeof text === 'string') return text.trim() || null;
if (typeof text === 'number') return String(text);
}
return null;
}
function findInZip(files: unzipper.File[], path: string): unzipper.File | undefined {
const clean = path.startsWith('/') ? path.slice(1) : path;
return files.find((f) => {
const fp = f.path.startsWith('/') ? f.path.slice(1) : f.path;
return fp === clean;
});
}
function resolveHref(href: string, basePath: string): string {
if (!href || href.startsWith('http://') || href.startsWith('https://')) return href;
if (href.startsWith('/')) return href.slice(1);
let resolved = basePath + href;
while (resolved.includes('/../')) {
const idx = resolved.indexOf('/../');
const prev = resolved.lastIndexOf('/', idx - 1);
if (prev >= 0) resolved = resolved.slice(0, prev + 1) + resolved.slice(idx + 4);
else break;
}
return resolved;
}
function guessContentType(path: string): string {
const dot = path.lastIndexOf('.');
if (dot < 0) return 'application/octet-stream';
return CONTENT_TYPES[path.slice(dot).toLowerCase()] ?? 'application/octet-stream';
}
function parseNavOl(ol: Record<string, unknown>, basePath: string): EpubTocItem[] {
return toArray(ol?.li)
.map((li: Record<string, unknown>) => {
const a = li?.a as Record<string, unknown> | string | undefined;
let label = '';
let href: string | undefined;
if (typeof a === 'string') {
label = a.trim();
} else if (a != null) {
label = getText(a) ?? getText(a.span) ?? '';
const rawHref = a['@_href'] as string | undefined;
if (rawHref && !rawHref.startsWith('http')) href = resolveHref(rawHref, basePath);
else href = rawHref;
}
const nestedOl = li?.ol as Record<string, unknown> | undefined;
const children = nestedOl ? parseNavOl(nestedOl, basePath) : undefined;
return { label, href, children: children?.length ? children : undefined } satisfies EpubTocItem;
})
.filter((item) => item.label.length > 0);
}
function parseNcxNavPoints(parent: Record<string, unknown>, basePath: string): EpubTocItem[] {
return toArray(parent?.navPoint)
.map((np: Record<string, unknown>) => {
const navLabel = np?.navLabel as Record<string, unknown> | undefined;
const label = getText(navLabel?.text) ?? '';
const content = np?.content as Record<string, string> | undefined;
let href = content?.['@_src'];
if (href && !href.startsWith('http')) href = resolveHref(href, basePath);
const children = parseNcxNavPoints(np, basePath);
return { label, href, children: children.length ? children : undefined } satisfies EpubTocItem;
})
.filter((item) => item.label.length > 0);
}
async function parseEpub(epubPath: string): Promise<EpubBookInfo> {
const zip = await unzipper.Open.file(epubPath);
const containerEntry = findInZip(zip.files, 'META-INF/container.xml');
if (!containerEntry) throw new Error('Missing META-INF/container.xml');
const containerDoc = xmlParser.parse(await containerEntry.buffer()) as Record<string, unknown>;
const container = containerDoc['container'] as Record<string, unknown>;
const rootfiles = (container?.rootfiles as Record<string, unknown>)?.rootfile;
const rootfile: unknown = Array.isArray(rootfiles) ? rootfiles[0] : rootfiles;
const opfPath = (rootfile as Record<string, string>)?.['@_full-path'];
if (!opfPath) throw new Error('Cannot find OPF path in container.xml');
const rootPath = opfPath.includes('/') ? opfPath.slice(0, opfPath.lastIndexOf('/') + 1) : '';
const opfEntry = findInZip(zip.files, opfPath);
if (!opfEntry) throw new Error(`OPF not found: ${opfPath}`);
const opfDoc = xmlParser.parse(await opfEntry.buffer()) as Record<string, unknown>;
const pkg = (opfDoc['package'] ?? opfDoc) as Record<string, unknown>;
const manifestEl = pkg['manifest'] as Record<string, unknown> | undefined;
const spineEl = pkg['spine'] as Record<string, unknown> | undefined;
const metadataEl = pkg['metadata'] as Record<string, unknown> | undefined;
const manifestById = new Map<string, EpubManifestItem>();
const manifest: EpubManifestItem[] = toArray(manifestEl?.item).map((item: Record<string, string>) => {
const id = item['@_id'];
const relHref = item['@_href'];
const mediaType = item['@_media-type'] ?? 'application/octet-stream';
const propertiesStr = item['@_properties'];
const properties = propertiesStr ? propertiesStr.split(/\s+/) : undefined;
const fullHref = rootPath + relHref;
const size = findInZip(zip.files, fullHref)?.uncompressedSize ?? 0;
const manifestItem: EpubManifestItem = { id, href: fullHref, mediaType, size, ...(properties ? { properties } : {}) };
manifestById.set(id, manifestItem);
return manifestItem;
});
const spine: EpubSpineItem[] = toArray(spineEl?.itemref).reduce<EpubSpineItem[]>((acc, itemref: Record<string, string>) => {
const idref = itemref['@_idref'];
const m = manifestById.get(idref);
if (m) acc.push({ idref, href: m.href, mediaType: m.mediaType, linear: itemref['@_linear'] !== 'no' });
return acc;
}, []);
const metadata: Record<string, unknown> = {};
if (metadataEl) {
const title = getText(metadataEl['dc:title']);
const creator = getText(toArray(metadataEl['dc:creator'])[0]);
const language = getText(metadataEl['dc:language']);
const publisher = getText(metadataEl['dc:publisher']);
const description = getText(metadataEl['dc:description']);
const identifier = getText(toArray(metadataEl['dc:identifier'])[0]);
if (title) metadata.title = title;
if (creator) metadata.creator = creator;
if (language) metadata.language = language;
if (publisher) metadata.publisher = publisher;
if (description) metadata.description = description;
if (identifier) metadata.identifier = identifier;
}
let coverPath: string | null = manifest.find((m) => m.properties?.includes('cover-image'))?.href ?? null;
if (!coverPath && metadataEl) {
const coverMeta = toArray(metadataEl['meta']).find((m: Record<string, string>) => m['@_name'] === 'cover');
if (coverMeta) coverPath = manifestById.get((coverMeta as Record<string, string>)['@_content'])?.href ?? null;
}
let toc: EpubTocItem | null = null;
const navItem = manifest.find((m) => m.properties?.includes('nav'));
if (navItem) {
try {
const navEntry = findInZip(zip.files, navItem.href);
if (navEntry) {
const navDoc = xmlParser.parse(await navEntry.buffer()) as Record<string, unknown>;
const navDir = navItem.href.includes('/') ? navItem.href.slice(0, navItem.href.lastIndexOf('/') + 1) : rootPath;
const html = (navDoc['html'] ?? navDoc) as Record<string, unknown>;
const body = html['body'] as Record<string, unknown> | undefined;
const navs = toArray(body?.nav);
const tocNav = (navs.find((n: Record<string, unknown>) => {
const type = n['@_epub:type'] ?? n['@_type'];
return typeof type === 'string' && type.split(/\s+/).includes('toc');
}) ?? navs[0]) as Record<string, unknown> | undefined;
if (tocNav) {
const ol = tocNav['ol'] as Record<string, unknown> | undefined;
if (ol) toc = { label: 'Table of Contents', children: parseNavOl(ol, navDir) };
}
}
} catch {
// fall through to NCX
}
}
if (!toc) {
const ncxItem = manifest.find((m) => m.mediaType === 'application/x-dtbncx+xml');
if (ncxItem) {
try {
const ncxEntry = findInZip(zip.files, ncxItem.href);
if (ncxEntry) {
const ncxDoc = xmlParser.parse(await ncxEntry.buffer()) as Record<string, unknown>;
const ncxDir = ncxItem.href.includes('/') ? ncxItem.href.slice(0, ncxItem.href.lastIndexOf('/') + 1) : rootPath;
const ncx = (ncxDoc['ncx'] ?? ncxDoc) as Record<string, unknown>;
const navMap = ncx['navMap'] as Record<string, unknown> | undefined;
if (navMap) toc = { label: 'Table of Contents', children: parseNcxNavPoints(navMap, ncxDir) };
}
} catch {
// TOC unavailable
}
}
}
return { containerPath: opfPath, rootPath, spine, manifest, toc, metadata, coverPath };
}
@Injectable()
export class EpubService {
private readonly logger = new Logger(EpubService.name);
private readonly cache = new Map<string, CacheEntry>();
constructor(
private readonly bookRepo: BookRepository,
private readonly libraryService: LibraryService,
) {}
async getBookInfo(bookId: number, user: RequestUser): Promise<EpubBookInfo> {
const epubPath = await this.resolveEpubPath(bookId, user);
return (await this.getCachedEntry(epubPath)).info;
}
async streamFile(
bookId: number,
filePath: string,
user: RequestUser,
): Promise<{ stream: NodeJS.ReadableStream; contentType: string; size: number }> {
if (filePath.includes('..')) throw new ForbiddenException('Invalid path');
const epubPath = await this.resolveEpubPath(bookId, user);
const cached = await this.getCachedEntry(epubPath);
if (!cached.validPaths.has(filePath)) throw new NotFoundException(`File not found in EPUB: ${filePath}`);
const zip = await unzipper.Open.file(epubPath);
const entry = findInZip(zip.files, filePath);
if (!entry) throw new NotFoundException(`Entry not in archive: ${filePath}`);
const manifestItem = cached.info.manifest.find((m) => m.href === filePath);
const contentType = manifestItem?.mediaType ?? guessContentType(filePath);
return { stream: entry.stream(), contentType, size: entry.uncompressedSize };
}
private async resolveEpubPath(bookId: number, user: RequestUser): Promise<string> {
const libraryId = await this.bookRepo.findLibraryIdByBookId(bookId);
if (libraryId === null) throw new NotFoundException(`Book ${bookId} not found`);
await this.libraryService.verifyUserAccess(
user.id,
libraryId,
user.roles.some((r) => r.isSuperuser),
);
const [file] = await this.bookRepo.findPrimaryFilesByBookIds([bookId]);
if (!file || file.format !== 'epub') throw new NotFoundException(`No primary EPUB file for book ${bookId}`);
return file.absolutePath;
}
private async getCachedEntry(epubPath: string): Promise<CacheEntry> {
const { mtimeMs } = await stat(epubPath);
const cached = this.cache.get(epubPath);
if (cached && cached.mtime === mtimeMs) {
cached.lastAccessed = Date.now();
return cached;
}
this.logger.debug(`Parsing EPUB: ${epubPath}`);
const info = await parseEpub(epubPath);
const validPaths = new Set<string>(['META-INF/container.xml', info.containerPath]);
for (const item of info.manifest) validPaths.add(item.href);
const entry: CacheEntry = { info, mtime: mtimeMs, validPaths, lastAccessed: Date.now() };
this.evict();
this.cache.set(epubPath, entry);
return entry;
}
private evict(): void {
if (this.cache.size < MAX_CACHE) return;
const oldest = [...this.cache.entries()].sort((a, b) => a[1].lastAccessed - b[1].lastAccessed)[0];
if (oldest) this.cache.delete(oldest[0]);
}
}
+25 -11
View File
@@ -1,16 +1,30 @@
import { createHash } from 'crypto'; import { createHash } from 'crypto';
import { createReadStream } from 'fs'; import { open } from 'fs/promises';
const BASE = 1024;
const BLOCK_SIZE = 1024;
/** /**
* SHA-256 a file by streaming it — avoids loading large files into memory. * Partial MD5 fingerprint — reads 1 KB blocks from exponentially spaced positions
* Only called when path and inode lookups both fail (cross-filesystem move). * (1 KB, 4 KB, 16 KB, … up to 1 GB) rather than the entire file.
* Used only when path and inode lookups both fail (cross-filesystem move detection).
*/ */
export function sha256File(absolutePath: string): Promise<string> { export async function fingerprintFile(absolutePath: string): Promise<string> {
return new Promise((resolve, reject) => { const fh = await open(absolutePath, 'r');
const hash = createHash('sha256'); try {
const stream = createReadStream(absolutePath); const { size } = await fh.stat();
stream.on('data', (chunk) => hash.update(chunk)); const hash = createHash('md5');
stream.on('end', () => resolve(hash.digest('hex'))); const buf = Buffer.allocUnsafe(BLOCK_SIZE);
stream.on('error', reject);
}); for (let i = 0; i <= 10; i++) {
const position = BASE << (2 * i);
if (position >= size) break;
const { bytesRead } = await fh.read(buf, 0, BLOCK_SIZE, position);
if (bytesRead > 0) hash.update(buf.subarray(0, bytesRead));
}
return hash.digest('hex');
} finally {
await fh.close();
}
} }
@@ -5,7 +5,7 @@ import { MetadataService } from '../metadata/metadata.service';
import { ScanGateway } from './scan.gateway'; import { ScanGateway } from './scan.gateway';
import { ScanJobStore } from './scan-job-store.service'; import { ScanJobStore } from './scan-job-store.service';
import { classifyFile, FileRole } from './lib/classify'; import { classifyFile, FileRole } from './lib/classify';
import { sha256File } from './lib/hash'; import { fingerprintFile } from './lib/hash';
import { waitForStability } from './lib/stability'; import { waitForStability } from './lib/stability';
import { BookCandidate, FileStat, findBookCandidates } from './lib/walk'; import { BookCandidate, FileStat, findBookCandidates } from './lib/walk';
import { ScannerRepository } from './scanner.repository'; import { ScannerRepository } from './scanner.repository';
@@ -343,7 +343,7 @@ export class ScannerService implements OnApplicationBootstrap {
} }
// 3. Hash match — cross-filesystem copy (expensive, last resort). // 3. Hash match — cross-filesystem copy (expensive, last resort).
const hash = await sha256File(fileStat.absolutePath); const hash = await fingerprintFile(fileStat.absolutePath);
const byHash = await this.scannerRepo.findBookFileByHash(hash); const byHash = await this.scannerRepo.findBookFileByHash(hash);
if (byHash) { if (byHash) {
await this.scannerRepo.updateBookFile(byHash.id, { await this.scannerRepo.updateBookFile(byHash.id, {
@@ -6,7 +6,7 @@ import { DB } from '../../db';
import * as schema from '../../db/schema'; import * as schema from '../../db/schema';
import { bookFiles, bookMetadata, books } from '../../db/schema'; import { bookFiles, bookMetadata, books } from '../../db/schema';
import { MetadataService } from '../metadata/metadata.service'; import { MetadataService } from '../metadata/metadata.service';
import { sha256File } from '../scanner/lib/hash'; import { fingerprintFile } from '../scanner/lib/hash';
type Db = NodePgDatabase<typeof schema>; type Db = NodePgDatabase<typeof schema>;
@@ -30,7 +30,7 @@ export class UploadProcessorService {
format: string, format: string,
sizeBytes: number, sizeBytes: number,
): Promise<{ bookId: number }> { ): Promise<{ bookId: number }> {
const [fileStat, hash] = await Promise.all([stat(absolutePath), sha256File(absolutePath)]); const [fileStat, hash] = await Promise.all([stat(absolutePath), fingerprintFile(absolutePath)]);
const [book] = await this.db.insert(books).values({ libraryId, libraryFolderId, folderPath, status: 'present' }).returning(); const [book] = await this.db.insert(books).values({ libraryId, libraryFolderId, folderPath, status: 'present' }).returning();