mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 08:21:39 -06:00
feat: add epub streaming with server-side file serving
This commit is contained in:
@@ -871,7 +871,6 @@ class Loader {
|
|||||||
const directUrl = this.#getDirectUrl(path)
|
const directUrl = this.#getDirectUrl(path)
|
||||||
if (directUrl) {
|
if (directUrl) {
|
||||||
el.setAttribute(attr, directUrl)
|
el.setAttribute(attr, directUrl)
|
||||||
el.setAttribute('loading', 'lazy') // Native lazy loading hint
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
|
|||||||
@@ -35,24 +35,6 @@ export const makeStreamingLoader = (bookId, baseUrl, bookInfo, authToken = null,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Build URL with token for browser-initiated requests (fonts, images in CSS)
|
|
||||||
const getDirectFileUrl = (name) => {
|
|
||||||
if (!name) return null
|
|
||||||
const encodedPath = name.split('/').map(encodeURIComponent).join('/')
|
|
||||||
let url = `${baseUrl}/${bookId}/file/${encodedPath}`
|
|
||||||
const params = []
|
|
||||||
if (bookType) {
|
|
||||||
params.push(`bookType=${encodeURIComponent(bookType)}`)
|
|
||||||
}
|
|
||||||
if (authToken) {
|
|
||||||
params.push(`token=${encodeURIComponent(authToken)}`)
|
|
||||||
}
|
|
||||||
if (params.length > 0) {
|
|
||||||
url += '?' + params.join('&')
|
|
||||||
}
|
|
||||||
return url
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Load file as text
|
* Load file as text
|
||||||
*/
|
*/
|
||||||
@@ -105,24 +87,13 @@ export const makeStreamingLoader = (bookId, baseUrl, bookInfo, authToken = null,
|
|||||||
return item?.size ?? 0
|
return item?.size ?? 0
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Get direct URL for a file without fetching it.
|
|
||||||
* This allows the browser to fetch resources on-demand (fonts, images).
|
|
||||||
* Note: Uses token in URL since browser-initiated requests can't use headers.
|
|
||||||
* @param {string} name - File path within the EPUB
|
|
||||||
* @returns {string|null} Direct URL to the file
|
|
||||||
*/
|
|
||||||
const getDirectUrl = (name) => {
|
|
||||||
if (!name) return null
|
|
||||||
return getDirectFileUrl(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
loadText,
|
loadText,
|
||||||
loadBlob,
|
loadBlob,
|
||||||
getSize,
|
getSize,
|
||||||
getDirectUrl, // For lazy loading of fonts/images in CSS
|
// getDirectUrl intentionally omitted: browser-initiated fetches from inside a
|
||||||
// Expose for debugging
|
// sandboxed blob-URL iframe are blocked by Cross-Origin-Resource-Policy headers.
|
||||||
|
// Images go through loadBlob (JS fetch with Authorization header) instead.
|
||||||
_bookInfo: bookInfo,
|
_bookInfo: bookInfo,
|
||||||
_manifestMap: manifestMap,
|
_manifestMap: manifestMap,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -145,7 +145,7 @@ onMounted(async () => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
await open(fileId, fileFormat, progress.cfi.value)
|
await open(bookId, fileId, fileFormat, progress.cfi.value)
|
||||||
setChapters(getChapters())
|
setChapters(getChapters())
|
||||||
sectionFractions.value = getSectionFractions()
|
sectionFractions.value = getSectionFractions()
|
||||||
await bookmarks.load(bookId)
|
await bookmarks.load(bookId)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { onUnmounted, ref } from 'vue'
|
import { onUnmounted, ref } from 'vue'
|
||||||
import { api } from '@/lib/api'
|
import { api, getAccessToken } from '@/lib/api'
|
||||||
import { useFoliateAnnotations } from './useFoliateAnnotations'
|
import { useFoliateAnnotations } from './useFoliateAnnotations'
|
||||||
import { useFoliateSelection } from './useFoliateSelection'
|
import { useFoliateSelection } from './useFoliateSelection'
|
||||||
import { useFoliateInput } from './useFoliateInput'
|
import { useFoliateInput } from './useFoliateInput'
|
||||||
@@ -38,10 +38,11 @@ export function useFoliate(
|
|||||||
|
|
||||||
async function loadScript() {
|
async function loadScript() {
|
||||||
if (customElements.get('foliate-view')) return
|
if (customElements.get('foliate-view')) return
|
||||||
|
const src = import.meta.env.DEV ? `/assets/foliate/view.js?v=${Date.now()}` : '/assets/foliate/view.js'
|
||||||
await new Promise<void>((resolve, reject) => {
|
await new Promise<void>((resolve, reject) => {
|
||||||
const script = document.createElement('script')
|
const script = document.createElement('script')
|
||||||
script.type = 'module'
|
script.type = 'module'
|
||||||
script.src = '/assets/foliate/view.js'
|
script.src = src
|
||||||
script.onload = () => setTimeout(resolve, 100)
|
script.onload = () => setTimeout(resolve, 100)
|
||||||
script.onerror = () => reject(new Error('Failed to load foliate/view.js'))
|
script.onerror = () => reject(new Error('Failed to load foliate/view.js'))
|
||||||
document.head.appendChild(script)
|
document.head.appendChild(script)
|
||||||
@@ -49,7 +50,7 @@ export function useFoliate(
|
|||||||
await customElements.whenDefined('foliate-view')
|
await customElements.whenDefined('foliate-view')
|
||||||
}
|
}
|
||||||
|
|
||||||
async function open(fileId: number, format: string, cfi?: string | null) {
|
async function open(bookId: number, fileId: number, format: string, cfi?: string | null) {
|
||||||
const el = container()
|
const el = container()
|
||||||
if (!el) return
|
if (!el) return
|
||||||
|
|
||||||
@@ -104,13 +105,25 @@ export function useFoliate(
|
|||||||
loading.value = false
|
loading.value = false
|
||||||
})
|
})
|
||||||
|
|
||||||
const mimeType = format === 'pdf' ? 'application/pdf' : format === 'cbz' ? 'application/zip' : 'application/epub+zip'
|
if (format === 'epub') {
|
||||||
const ext = format === 'pdf' ? 'pdf' : format === 'cbz' ? 'cbz' : 'epub'
|
const infoRes = await api(`/api/v1/epub/${bookId}/info`)
|
||||||
const res = await api(`/api/v1/books/files/${fileId}/serve`)
|
if (!infoRes.ok) throw new Error(`Failed to fetch EPUB info: ${infoRes.status}`)
|
||||||
if (!res.ok) throw new Error(`Failed to fetch book file: ${res.status}`)
|
const bookInfo = await infoRes.json()
|
||||||
const blob = await res.blob()
|
const makeStreamingBook = (window as Record<string, unknown>).makeStreamingBook as
|
||||||
const file = new File([blob], `book-file-${fileId}.${ext}`, { type: mimeType })
|
| ((id: number, base: string, info: unknown, token: string | null) => Promise<unknown>)
|
||||||
await view.open(file)
|
| undefined
|
||||||
|
if (!makeStreamingBook) throw new Error('makeStreamingBook not available')
|
||||||
|
const book = await makeStreamingBook(bookId, '/api/v1/epub', bookInfo, getAccessToken())
|
||||||
|
await view.open(book as never)
|
||||||
|
} else {
|
||||||
|
const mimeType = format === 'pdf' ? 'application/pdf' : 'application/zip'
|
||||||
|
const ext = format === 'pdf' ? 'pdf' : format === 'cbz' ? 'cbz' : format
|
||||||
|
const res = await api(`/api/v1/books/files/${fileId}/serve`)
|
||||||
|
if (!res.ok) throw new Error(`Failed to fetch book file: ${res.status}`)
|
||||||
|
const blob = await res.blob()
|
||||||
|
const file = new File([blob], `book-file-${fileId}.${ext}`, { type: mimeType })
|
||||||
|
await view.open(file)
|
||||||
|
}
|
||||||
if (onApplyStyles) onApplyStyles(view.renderer)
|
if (onApplyStyles) onApplyStyles(view.renderer)
|
||||||
await view.goTo(cfi ?? 0).catch(() => {})
|
await view.goTo(cfi ?? 0).catch(() => {})
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@@ -146,7 +159,7 @@ export function useFoliate(
|
|||||||
error,
|
error,
|
||||||
fraction,
|
fraction,
|
||||||
view: viewRef,
|
view: viewRef,
|
||||||
open,
|
open: (bookId: number, fileId: number, format: string, cfi?: string | null) => open(bookId, fileId, format, cfi),
|
||||||
prev: () => getViewEl()?.prev?.(),
|
prev: () => getViewEl()?.prev?.(),
|
||||||
next: () => getViewEl()?.next?.(),
|
next: () => getViewEl()?.next?.(),
|
||||||
goTo: (t: string | number) => getViewEl()?.goTo?.(t),
|
goTo: (t: string | number) => getViewEl()?.goTo?.(t),
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
export interface EpubManifestItem {
|
||||||
|
id: string
|
||||||
|
href: string
|
||||||
|
mediaType: string
|
||||||
|
properties?: string[]
|
||||||
|
size: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EpubSpineItem {
|
||||||
|
idref: string
|
||||||
|
href: string
|
||||||
|
mediaType: string
|
||||||
|
linear: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EpubTocItem {
|
||||||
|
label: string
|
||||||
|
href?: string
|
||||||
|
children?: EpubTocItem[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface EpubBookInfo {
|
||||||
|
containerPath: string
|
||||||
|
rootPath: string
|
||||||
|
spine: EpubSpineItem[]
|
||||||
|
manifest: EpubManifestItem[]
|
||||||
|
toc: EpubTocItem | null
|
||||||
|
metadata: Record<string, unknown>
|
||||||
|
coverPath: string | null
|
||||||
|
}
|
||||||
@@ -14,3 +14,4 @@ export * from "./opds";
|
|||||||
export * from "./kobo";
|
export * from "./kobo";
|
||||||
export * from "./staging";
|
export * from "./staging";
|
||||||
export * from "./file-write";
|
export * from "./file-write";
|
||||||
|
export * from "./epub";
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ import { RoleModule } from './modules/role/role.module';
|
|||||||
import { ScannerModule } from './modules/scanner/scanner.module';
|
import { ScannerModule } from './modules/scanner/scanner.module';
|
||||||
import { SeedModule } from './modules/seed/seed.module';
|
import { SeedModule } from './modules/seed/seed.module';
|
||||||
import { EmailModule } from './modules/email/email.module';
|
import { EmailModule } from './modules/email/email.module';
|
||||||
|
import { EpubModule } from './modules/epub/epub.module';
|
||||||
import { StagingModule } from './modules/staging/staging.module';
|
import { StagingModule } from './modules/staging/staging.module';
|
||||||
import { UploadModule } from './modules/upload/upload.module';
|
import { UploadModule } from './modules/upload/upload.module';
|
||||||
import { UserModule } from './modules/user/user.module';
|
import { UserModule } from './modules/user/user.module';
|
||||||
@@ -78,6 +79,7 @@ import { UserModule } from './modules/user/user.module';
|
|||||||
HealthModule,
|
HealthModule,
|
||||||
CbzModule,
|
CbzModule,
|
||||||
ReaderPreferencesModule,
|
ReaderPreferencesModule,
|
||||||
|
EpubModule,
|
||||||
StagingModule,
|
StagingModule,
|
||||||
EmailModule,
|
EmailModule,
|
||||||
],
|
],
|
||||||
|
|||||||
+4
-1
@@ -101,7 +101,10 @@ async function bootstrap() {
|
|||||||
max: 100,
|
max: 100,
|
||||||
timeWindow: '1 minute',
|
timeWindow: '1 minute',
|
||||||
allowList: (req: { cookies?: { access_token?: string }; headers?: { authorization?: string }; url?: string }) =>
|
allowList: (req: { cookies?: { access_token?: string }; headers?: { authorization?: string }; url?: string }) =>
|
||||||
!!req.cookies?.access_token || !!req.headers?.authorization || /^\/api\/v1\/kobo\//.test(req.url ?? ''),
|
!!req.cookies?.access_token ||
|
||||||
|
!!req.headers?.authorization ||
|
||||||
|
/^\/api\/v1\/kobo\//.test(req.url ?? '') ||
|
||||||
|
/^\/api\/v1\/epub\/\d+\/file\//.test(req.url ?? ''),
|
||||||
});
|
});
|
||||||
|
|
||||||
if (process.env.NODE_ENV !== 'production') {
|
if (process.env.NODE_ENV !== 'production') {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ export class JwtStrategy extends PassportStrategy(Strategy) {
|
|||||||
jwtFromRequest: ExtractJwt.fromExtractors([
|
jwtFromRequest: ExtractJwt.fromExtractors([
|
||||||
ExtractJwt.fromAuthHeaderAsBearerToken(),
|
ExtractJwt.fromAuthHeaderAsBearerToken(),
|
||||||
(req: { cookies?: Record<string, string | undefined> }) => req?.cookies?.access_token ?? null,
|
(req: { cookies?: Record<string, string | undefined> }) => req?.cookies?.access_token ?? null,
|
||||||
|
(req: { query?: Record<string, string | undefined> }) => req?.query?.token ?? null,
|
||||||
]),
|
]),
|
||||||
secretOrKey: config.get<string>('auth.jwtSecret')!,
|
secretOrKey: config.get<string>('auth.jwtSecret')!,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import { Controller, Get, Param, ParseIntPipe, Res } from '@nestjs/common';
|
||||||
|
import type { FastifyReply } from 'fastify';
|
||||||
|
|
||||||
|
import { CurrentUser } from '../../common/decorators/current-user.decorator';
|
||||||
|
import { RequirePermission } from '../../common/decorators/require-permission.decorator';
|
||||||
|
import type { RequestUser } from '../../common/types/request-user';
|
||||||
|
import { EpubService } from './epub.service';
|
||||||
|
|
||||||
|
@Controller('epub')
|
||||||
|
export class EpubController {
|
||||||
|
constructor(private readonly epubService: EpubService) {}
|
||||||
|
|
||||||
|
@Get(':bookId/info')
|
||||||
|
@RequirePermission('library_download')
|
||||||
|
getBookInfo(@Param('bookId', ParseIntPipe) bookId: number, @CurrentUser() user: RequestUser) {
|
||||||
|
return this.epubService.getBookInfo(bookId, user);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Get(':bookId/file/*')
|
||||||
|
@RequirePermission('library_download')
|
||||||
|
async getFile(
|
||||||
|
@Param('bookId', ParseIntPipe) bookId: number,
|
||||||
|
@Param('*') encodedPath: string,
|
||||||
|
@CurrentUser() user: RequestUser,
|
||||||
|
@Res() reply: FastifyReply,
|
||||||
|
) {
|
||||||
|
const filePath = encodedPath
|
||||||
|
.split('/')
|
||||||
|
.map((s) => decodeURIComponent(s))
|
||||||
|
.join('/');
|
||||||
|
|
||||||
|
const { stream, contentType, size } = await this.epubService.streamFile(bookId, filePath, user);
|
||||||
|
|
||||||
|
reply.header('Content-Type', contentType);
|
||||||
|
if (size > 0) reply.header('Content-Length', size);
|
||||||
|
reply.header('Cache-Control', 'public, max-age=3600');
|
||||||
|
reply.send(stream);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
|
||||||
|
import { BookModule } from '../book/book.module';
|
||||||
|
import { LibraryModule } from '../library/library.module';
|
||||||
|
import { EpubController } from './epub.controller';
|
||||||
|
import { EpubService } from './epub.service';
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
imports: [BookModule, LibraryModule],
|
||||||
|
controllers: [EpubController],
|
||||||
|
providers: [EpubService],
|
||||||
|
})
|
||||||
|
export class EpubModule {}
|
||||||
@@ -0,0 +1,324 @@
|
|||||||
|
import { ForbiddenException, Injectable, Logger, NotFoundException } from '@nestjs/common';
|
||||||
|
import { stat } from 'fs/promises';
|
||||||
|
import * as unzipper from 'unzipper';
|
||||||
|
import { XMLParser } from 'fast-xml-parser';
|
||||||
|
|
||||||
|
import type { EpubBookInfo, EpubManifestItem, EpubSpineItem, EpubTocItem } from '@projectx/types';
|
||||||
|
import { BookRepository } from '../book/book.repository';
|
||||||
|
import { LibraryService } from '../library/library.service';
|
||||||
|
import type { RequestUser } from '../../common/types/request-user';
|
||||||
|
|
||||||
|
const CONTENT_TYPES: Record<string, string> = {
|
||||||
|
'.xhtml': 'application/xhtml+xml',
|
||||||
|
'.html': 'application/xhtml+xml',
|
||||||
|
'.htm': 'application/xhtml+xml',
|
||||||
|
'.css': 'text/css',
|
||||||
|
'.js': 'application/javascript',
|
||||||
|
'.jpg': 'image/jpeg',
|
||||||
|
'.jpeg': 'image/jpeg',
|
||||||
|
'.png': 'image/png',
|
||||||
|
'.gif': 'image/gif',
|
||||||
|
'.svg': 'image/svg+xml',
|
||||||
|
'.webp': 'image/webp',
|
||||||
|
'.woff': 'font/woff',
|
||||||
|
'.woff2': 'font/woff2',
|
||||||
|
'.ttf': 'font/ttf',
|
||||||
|
'.otf': 'font/otf',
|
||||||
|
'.eot': 'application/vnd.ms-fontobject',
|
||||||
|
'.xml': 'application/xml',
|
||||||
|
'.ncx': 'application/x-dtbncx+xml',
|
||||||
|
'.opf': 'application/oebps-package+xml',
|
||||||
|
'.smil': 'application/smil+xml',
|
||||||
|
'.mp3': 'audio/mpeg',
|
||||||
|
'.mp4': 'video/mp4',
|
||||||
|
'.webm': 'video/webm',
|
||||||
|
};
|
||||||
|
|
||||||
|
const MAX_CACHE = 50;
|
||||||
|
|
||||||
|
interface CacheEntry {
|
||||||
|
info: EpubBookInfo;
|
||||||
|
mtime: number;
|
||||||
|
validPaths: Set<string>;
|
||||||
|
lastAccessed: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const xmlParser = new XMLParser({
|
||||||
|
ignoreAttributes: false,
|
||||||
|
attributeNamePrefix: '@_',
|
||||||
|
textNodeName: '#text',
|
||||||
|
});
|
||||||
|
|
||||||
|
function toArray<T>(v: T | T[] | undefined | null): T[] {
|
||||||
|
if (v == null) return [];
|
||||||
|
return Array.isArray(v) ? v : [v];
|
||||||
|
}
|
||||||
|
|
||||||
|
function getText(v: unknown): string | null {
|
||||||
|
if (typeof v === 'string') return v.trim() || null;
|
||||||
|
if (typeof v === 'number') return String(v);
|
||||||
|
if (v != null && typeof v === 'object') {
|
||||||
|
const text = (v as Record<string, unknown>)['#text'];
|
||||||
|
if (typeof text === 'string') return text.trim() || null;
|
||||||
|
if (typeof text === 'number') return String(text);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function findInZip(files: unzipper.File[], path: string): unzipper.File | undefined {
|
||||||
|
const clean = path.startsWith('/') ? path.slice(1) : path;
|
||||||
|
return files.find((f) => {
|
||||||
|
const fp = f.path.startsWith('/') ? f.path.slice(1) : f.path;
|
||||||
|
return fp === clean;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveHref(href: string, basePath: string): string {
|
||||||
|
if (!href || href.startsWith('http://') || href.startsWith('https://')) return href;
|
||||||
|
if (href.startsWith('/')) return href.slice(1);
|
||||||
|
let resolved = basePath + href;
|
||||||
|
while (resolved.includes('/../')) {
|
||||||
|
const idx = resolved.indexOf('/../');
|
||||||
|
const prev = resolved.lastIndexOf('/', idx - 1);
|
||||||
|
if (prev >= 0) resolved = resolved.slice(0, prev + 1) + resolved.slice(idx + 4);
|
||||||
|
else break;
|
||||||
|
}
|
||||||
|
return resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
function guessContentType(path: string): string {
|
||||||
|
const dot = path.lastIndexOf('.');
|
||||||
|
if (dot < 0) return 'application/octet-stream';
|
||||||
|
return CONTENT_TYPES[path.slice(dot).toLowerCase()] ?? 'application/octet-stream';
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseNavOl(ol: Record<string, unknown>, basePath: string): EpubTocItem[] {
|
||||||
|
return toArray(ol?.li)
|
||||||
|
.map((li: Record<string, unknown>) => {
|
||||||
|
const a = li?.a as Record<string, unknown> | string | undefined;
|
||||||
|
let label = '';
|
||||||
|
let href: string | undefined;
|
||||||
|
|
||||||
|
if (typeof a === 'string') {
|
||||||
|
label = a.trim();
|
||||||
|
} else if (a != null) {
|
||||||
|
label = getText(a) ?? getText(a.span) ?? '';
|
||||||
|
const rawHref = a['@_href'] as string | undefined;
|
||||||
|
if (rawHref && !rawHref.startsWith('http')) href = resolveHref(rawHref, basePath);
|
||||||
|
else href = rawHref;
|
||||||
|
}
|
||||||
|
|
||||||
|
const nestedOl = li?.ol as Record<string, unknown> | undefined;
|
||||||
|
const children = nestedOl ? parseNavOl(nestedOl, basePath) : undefined;
|
||||||
|
|
||||||
|
return { label, href, children: children?.length ? children : undefined } satisfies EpubTocItem;
|
||||||
|
})
|
||||||
|
.filter((item) => item.label.length > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseNcxNavPoints(parent: Record<string, unknown>, basePath: string): EpubTocItem[] {
|
||||||
|
return toArray(parent?.navPoint)
|
||||||
|
.map((np: Record<string, unknown>) => {
|
||||||
|
const navLabel = np?.navLabel as Record<string, unknown> | undefined;
|
||||||
|
const label = getText(navLabel?.text) ?? '';
|
||||||
|
|
||||||
|
const content = np?.content as Record<string, string> | undefined;
|
||||||
|
let href = content?.['@_src'];
|
||||||
|
if (href && !href.startsWith('http')) href = resolveHref(href, basePath);
|
||||||
|
|
||||||
|
const children = parseNcxNavPoints(np, basePath);
|
||||||
|
return { label, href, children: children.length ? children : undefined } satisfies EpubTocItem;
|
||||||
|
})
|
||||||
|
.filter((item) => item.label.length > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function parseEpub(epubPath: string): Promise<EpubBookInfo> {
|
||||||
|
const zip = await unzipper.Open.file(epubPath);
|
||||||
|
|
||||||
|
const containerEntry = findInZip(zip.files, 'META-INF/container.xml');
|
||||||
|
if (!containerEntry) throw new Error('Missing META-INF/container.xml');
|
||||||
|
const containerDoc = xmlParser.parse(await containerEntry.buffer()) as Record<string, unknown>;
|
||||||
|
|
||||||
|
const container = containerDoc['container'] as Record<string, unknown>;
|
||||||
|
const rootfiles = (container?.rootfiles as Record<string, unknown>)?.rootfile;
|
||||||
|
const rootfile: unknown = Array.isArray(rootfiles) ? rootfiles[0] : rootfiles;
|
||||||
|
const opfPath = (rootfile as Record<string, string>)?.['@_full-path'];
|
||||||
|
if (!opfPath) throw new Error('Cannot find OPF path in container.xml');
|
||||||
|
|
||||||
|
const rootPath = opfPath.includes('/') ? opfPath.slice(0, opfPath.lastIndexOf('/') + 1) : '';
|
||||||
|
|
||||||
|
const opfEntry = findInZip(zip.files, opfPath);
|
||||||
|
if (!opfEntry) throw new Error(`OPF not found: ${opfPath}`);
|
||||||
|
const opfDoc = xmlParser.parse(await opfEntry.buffer()) as Record<string, unknown>;
|
||||||
|
const pkg = (opfDoc['package'] ?? opfDoc) as Record<string, unknown>;
|
||||||
|
const manifestEl = pkg['manifest'] as Record<string, unknown> | undefined;
|
||||||
|
const spineEl = pkg['spine'] as Record<string, unknown> | undefined;
|
||||||
|
const metadataEl = pkg['metadata'] as Record<string, unknown> | undefined;
|
||||||
|
|
||||||
|
const manifestById = new Map<string, EpubManifestItem>();
|
||||||
|
const manifest: EpubManifestItem[] = toArray(manifestEl?.item).map((item: Record<string, string>) => {
|
||||||
|
const id = item['@_id'];
|
||||||
|
const relHref = item['@_href'];
|
||||||
|
const mediaType = item['@_media-type'] ?? 'application/octet-stream';
|
||||||
|
const propertiesStr = item['@_properties'];
|
||||||
|
const properties = propertiesStr ? propertiesStr.split(/\s+/) : undefined;
|
||||||
|
const fullHref = rootPath + relHref;
|
||||||
|
const size = findInZip(zip.files, fullHref)?.uncompressedSize ?? 0;
|
||||||
|
const manifestItem: EpubManifestItem = { id, href: fullHref, mediaType, size, ...(properties ? { properties } : {}) };
|
||||||
|
manifestById.set(id, manifestItem);
|
||||||
|
return manifestItem;
|
||||||
|
});
|
||||||
|
|
||||||
|
const spine: EpubSpineItem[] = toArray(spineEl?.itemref).reduce<EpubSpineItem[]>((acc, itemref: Record<string, string>) => {
|
||||||
|
const idref = itemref['@_idref'];
|
||||||
|
const m = manifestById.get(idref);
|
||||||
|
if (m) acc.push({ idref, href: m.href, mediaType: m.mediaType, linear: itemref['@_linear'] !== 'no' });
|
||||||
|
return acc;
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const metadata: Record<string, unknown> = {};
|
||||||
|
if (metadataEl) {
|
||||||
|
const title = getText(metadataEl['dc:title']);
|
||||||
|
const creator = getText(toArray(metadataEl['dc:creator'])[0]);
|
||||||
|
const language = getText(metadataEl['dc:language']);
|
||||||
|
const publisher = getText(metadataEl['dc:publisher']);
|
||||||
|
const description = getText(metadataEl['dc:description']);
|
||||||
|
const identifier = getText(toArray(metadataEl['dc:identifier'])[0]);
|
||||||
|
if (title) metadata.title = title;
|
||||||
|
if (creator) metadata.creator = creator;
|
||||||
|
if (language) metadata.language = language;
|
||||||
|
if (publisher) metadata.publisher = publisher;
|
||||||
|
if (description) metadata.description = description;
|
||||||
|
if (identifier) metadata.identifier = identifier;
|
||||||
|
}
|
||||||
|
|
||||||
|
let coverPath: string | null = manifest.find((m) => m.properties?.includes('cover-image'))?.href ?? null;
|
||||||
|
if (!coverPath && metadataEl) {
|
||||||
|
const coverMeta = toArray(metadataEl['meta']).find((m: Record<string, string>) => m['@_name'] === 'cover');
|
||||||
|
if (coverMeta) coverPath = manifestById.get((coverMeta as Record<string, string>)['@_content'])?.href ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
let toc: EpubTocItem | null = null;
|
||||||
|
|
||||||
|
const navItem = manifest.find((m) => m.properties?.includes('nav'));
|
||||||
|
if (navItem) {
|
||||||
|
try {
|
||||||
|
const navEntry = findInZip(zip.files, navItem.href);
|
||||||
|
if (navEntry) {
|
||||||
|
const navDoc = xmlParser.parse(await navEntry.buffer()) as Record<string, unknown>;
|
||||||
|
const navDir = navItem.href.includes('/') ? navItem.href.slice(0, navItem.href.lastIndexOf('/') + 1) : rootPath;
|
||||||
|
const html = (navDoc['html'] ?? navDoc) as Record<string, unknown>;
|
||||||
|
const body = html['body'] as Record<string, unknown> | undefined;
|
||||||
|
const navs = toArray(body?.nav);
|
||||||
|
const tocNav = (navs.find((n: Record<string, unknown>) => {
|
||||||
|
const type = n['@_epub:type'] ?? n['@_type'];
|
||||||
|
return typeof type === 'string' && type.split(/\s+/).includes('toc');
|
||||||
|
}) ?? navs[0]) as Record<string, unknown> | undefined;
|
||||||
|
if (tocNav) {
|
||||||
|
const ol = tocNav['ol'] as Record<string, unknown> | undefined;
|
||||||
|
if (ol) toc = { label: 'Table of Contents', children: parseNavOl(ol, navDir) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// fall through to NCX
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!toc) {
|
||||||
|
const ncxItem = manifest.find((m) => m.mediaType === 'application/x-dtbncx+xml');
|
||||||
|
if (ncxItem) {
|
||||||
|
try {
|
||||||
|
const ncxEntry = findInZip(zip.files, ncxItem.href);
|
||||||
|
if (ncxEntry) {
|
||||||
|
const ncxDoc = xmlParser.parse(await ncxEntry.buffer()) as Record<string, unknown>;
|
||||||
|
const ncxDir = ncxItem.href.includes('/') ? ncxItem.href.slice(0, ncxItem.href.lastIndexOf('/') + 1) : rootPath;
|
||||||
|
const ncx = (ncxDoc['ncx'] ?? ncxDoc) as Record<string, unknown>;
|
||||||
|
const navMap = ncx['navMap'] as Record<string, unknown> | undefined;
|
||||||
|
if (navMap) toc = { label: 'Table of Contents', children: parseNcxNavPoints(navMap, ncxDir) };
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// TOC unavailable
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { containerPath: opfPath, rootPath, spine, manifest, toc, metadata, coverPath };
|
||||||
|
}
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class EpubService {
|
||||||
|
private readonly logger = new Logger(EpubService.name);
|
||||||
|
private readonly cache = new Map<string, CacheEntry>();
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly bookRepo: BookRepository,
|
||||||
|
private readonly libraryService: LibraryService,
|
||||||
|
) {}
|
||||||
|
|
||||||
|
async getBookInfo(bookId: number, user: RequestUser): Promise<EpubBookInfo> {
|
||||||
|
const epubPath = await this.resolveEpubPath(bookId, user);
|
||||||
|
return (await this.getCachedEntry(epubPath)).info;
|
||||||
|
}
|
||||||
|
|
||||||
|
async streamFile(
|
||||||
|
bookId: number,
|
||||||
|
filePath: string,
|
||||||
|
user: RequestUser,
|
||||||
|
): Promise<{ stream: NodeJS.ReadableStream; contentType: string; size: number }> {
|
||||||
|
if (filePath.includes('..')) throw new ForbiddenException('Invalid path');
|
||||||
|
|
||||||
|
const epubPath = await this.resolveEpubPath(bookId, user);
|
||||||
|
const cached = await this.getCachedEntry(epubPath);
|
||||||
|
|
||||||
|
if (!cached.validPaths.has(filePath)) throw new NotFoundException(`File not found in EPUB: ${filePath}`);
|
||||||
|
|
||||||
|
const zip = await unzipper.Open.file(epubPath);
|
||||||
|
const entry = findInZip(zip.files, filePath);
|
||||||
|
if (!entry) throw new NotFoundException(`Entry not in archive: ${filePath}`);
|
||||||
|
|
||||||
|
const manifestItem = cached.info.manifest.find((m) => m.href === filePath);
|
||||||
|
const contentType = manifestItem?.mediaType ?? guessContentType(filePath);
|
||||||
|
|
||||||
|
return { stream: entry.stream(), contentType, size: entry.uncompressedSize };
|
||||||
|
}
|
||||||
|
|
||||||
|
private async resolveEpubPath(bookId: number, user: RequestUser): Promise<string> {
|
||||||
|
const libraryId = await this.bookRepo.findLibraryIdByBookId(bookId);
|
||||||
|
if (libraryId === null) throw new NotFoundException(`Book ${bookId} not found`);
|
||||||
|
await this.libraryService.verifyUserAccess(
|
||||||
|
user.id,
|
||||||
|
libraryId,
|
||||||
|
user.roles.some((r) => r.isSuperuser),
|
||||||
|
);
|
||||||
|
|
||||||
|
const [file] = await this.bookRepo.findPrimaryFilesByBookIds([bookId]);
|
||||||
|
if (!file || file.format !== 'epub') throw new NotFoundException(`No primary EPUB file for book ${bookId}`);
|
||||||
|
return file.absolutePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getCachedEntry(epubPath: string): Promise<CacheEntry> {
|
||||||
|
const { mtimeMs } = await stat(epubPath);
|
||||||
|
const cached = this.cache.get(epubPath);
|
||||||
|
if (cached && cached.mtime === mtimeMs) {
|
||||||
|
cached.lastAccessed = Date.now();
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.logger.debug(`Parsing EPUB: ${epubPath}`);
|
||||||
|
const info = await parseEpub(epubPath);
|
||||||
|
|
||||||
|
const validPaths = new Set<string>(['META-INF/container.xml', info.containerPath]);
|
||||||
|
for (const item of info.manifest) validPaths.add(item.href);
|
||||||
|
|
||||||
|
const entry: CacheEntry = { info, mtime: mtimeMs, validPaths, lastAccessed: Date.now() };
|
||||||
|
this.evict();
|
||||||
|
this.cache.set(epubPath, entry);
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
|
||||||
|
private evict(): void {
|
||||||
|
if (this.cache.size < MAX_CACHE) return;
|
||||||
|
const oldest = [...this.cache.entries()].sort((a, b) => a[1].lastAccessed - b[1].lastAccessed)[0];
|
||||||
|
if (oldest) this.cache.delete(oldest[0]);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,16 +1,30 @@
|
|||||||
import { createHash } from 'crypto';
|
import { createHash } from 'crypto';
|
||||||
import { createReadStream } from 'fs';
|
import { open } from 'fs/promises';
|
||||||
|
|
||||||
|
const BASE = 1024;
|
||||||
|
const BLOCK_SIZE = 1024;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* SHA-256 a file by streaming it — avoids loading large files into memory.
|
* Partial MD5 fingerprint — reads 1 KB blocks from exponentially spaced positions
|
||||||
* Only called when path and inode lookups both fail (cross-filesystem move).
|
* (1 KB, 4 KB, 16 KB, … up to 1 GB) rather than the entire file.
|
||||||
|
* Used only when path and inode lookups both fail (cross-filesystem move detection).
|
||||||
*/
|
*/
|
||||||
export function sha256File(absolutePath: string): Promise<string> {
|
export async function fingerprintFile(absolutePath: string): Promise<string> {
|
||||||
return new Promise((resolve, reject) => {
|
const fh = await open(absolutePath, 'r');
|
||||||
const hash = createHash('sha256');
|
try {
|
||||||
const stream = createReadStream(absolutePath);
|
const { size } = await fh.stat();
|
||||||
stream.on('data', (chunk) => hash.update(chunk));
|
const hash = createHash('md5');
|
||||||
stream.on('end', () => resolve(hash.digest('hex')));
|
const buf = Buffer.allocUnsafe(BLOCK_SIZE);
|
||||||
stream.on('error', reject);
|
|
||||||
});
|
for (let i = 0; i <= 10; i++) {
|
||||||
|
const position = BASE << (2 * i);
|
||||||
|
if (position >= size) break;
|
||||||
|
const { bytesRead } = await fh.read(buf, 0, BLOCK_SIZE, position);
|
||||||
|
if (bytesRead > 0) hash.update(buf.subarray(0, bytesRead));
|
||||||
|
}
|
||||||
|
|
||||||
|
return hash.digest('hex');
|
||||||
|
} finally {
|
||||||
|
await fh.close();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import { MetadataService } from '../metadata/metadata.service';
|
|||||||
import { ScanGateway } from './scan.gateway';
|
import { ScanGateway } from './scan.gateway';
|
||||||
import { ScanJobStore } from './scan-job-store.service';
|
import { ScanJobStore } from './scan-job-store.service';
|
||||||
import { classifyFile, FileRole } from './lib/classify';
|
import { classifyFile, FileRole } from './lib/classify';
|
||||||
import { sha256File } from './lib/hash';
|
import { fingerprintFile } from './lib/hash';
|
||||||
import { waitForStability } from './lib/stability';
|
import { waitForStability } from './lib/stability';
|
||||||
import { BookCandidate, FileStat, findBookCandidates } from './lib/walk';
|
import { BookCandidate, FileStat, findBookCandidates } from './lib/walk';
|
||||||
import { ScannerRepository } from './scanner.repository';
|
import { ScannerRepository } from './scanner.repository';
|
||||||
@@ -343,7 +343,7 @@ export class ScannerService implements OnApplicationBootstrap {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 3. Hash match — cross-filesystem copy (expensive, last resort).
|
// 3. Hash match — cross-filesystem copy (expensive, last resort).
|
||||||
const hash = await sha256File(fileStat.absolutePath);
|
const hash = await fingerprintFile(fileStat.absolutePath);
|
||||||
const byHash = await this.scannerRepo.findBookFileByHash(hash);
|
const byHash = await this.scannerRepo.findBookFileByHash(hash);
|
||||||
if (byHash) {
|
if (byHash) {
|
||||||
await this.scannerRepo.updateBookFile(byHash.id, {
|
await this.scannerRepo.updateBookFile(byHash.id, {
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import { DB } from '../../db';
|
|||||||
import * as schema from '../../db/schema';
|
import * as schema from '../../db/schema';
|
||||||
import { bookFiles, bookMetadata, books } from '../../db/schema';
|
import { bookFiles, bookMetadata, books } from '../../db/schema';
|
||||||
import { MetadataService } from '../metadata/metadata.service';
|
import { MetadataService } from '../metadata/metadata.service';
|
||||||
import { sha256File } from '../scanner/lib/hash';
|
import { fingerprintFile } from '../scanner/lib/hash';
|
||||||
|
|
||||||
type Db = NodePgDatabase<typeof schema>;
|
type Db = NodePgDatabase<typeof schema>;
|
||||||
|
|
||||||
@@ -30,7 +30,7 @@ export class UploadProcessorService {
|
|||||||
format: string,
|
format: string,
|
||||||
sizeBytes: number,
|
sizeBytes: number,
|
||||||
): Promise<{ bookId: number }> {
|
): Promise<{ bookId: number }> {
|
||||||
const [fileStat, hash] = await Promise.all([stat(absolutePath), sha256File(absolutePath)]);
|
const [fileStat, hash] = await Promise.all([stat(absolutePath), fingerprintFile(absolutePath)]);
|
||||||
|
|
||||||
const [book] = await this.db.insert(books).values({ libraryId, libraryFolderId, folderPath, status: 'present' }).returning();
|
const [book] = await this.db.insert(books).values({ libraryId, libraryFolderId, folderPath, status: 'present' }).returning();
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user