mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
perf(metadata): preserve unchanged epub entries during write-back
Copy unchanged ZIP records without inflating and recompressing book content so metadata saves avoid processing every image, chapter, and font. Use bounded file I/O and validate archive structure while preserving atomic replacement and original entry compression. Cover ZIP64, descriptor variants, byte preservation, repeated saves, and filesystem failures in regression tests. Closes #1798
This commit is contained in:
neon
@@ -75,6 +75,7 @@
|
||||
"@nestjs/throttler": "^6.7.1",
|
||||
"@nestjs/websockets": "^12.1.2",
|
||||
"@types/archiver": "^8.0.0",
|
||||
"@zip.js/zip.js": "^2.23.0",
|
||||
"archiver": "~8.0.0",
|
||||
"bcryptjs": "^3.0.3",
|
||||
"cheerio": "^1.2.0",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { ZipArchive } from 'archiver';
|
||||
import { randomBytes } from 'crypto';
|
||||
import { createWriteStream } from 'fs';
|
||||
import { mkdtemp, rm } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
@@ -26,7 +27,11 @@ afterEach(async () => {
|
||||
await rm(testRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function writeArchive(filePath: string, opfXml: string, entries: Array<{ path: string; content: string | Buffer }>): Promise<void> {
|
||||
async function writeArchive(
|
||||
filePath: string,
|
||||
opfXml: string,
|
||||
entries: Array<{ path: string; content: string | Buffer; store?: boolean }>,
|
||||
): Promise<void> {
|
||||
const output = createWriteStream(filePath);
|
||||
const archive = new ZipArchive({ zlib: { level: 6 } });
|
||||
|
||||
@@ -38,7 +43,7 @@ async function writeArchive(filePath: string, opfXml: string, entries: Array<{ p
|
||||
archive.append(Buffer.from('application/epub+zip'), { name: 'mimetype', store: true });
|
||||
archive.append(Buffer.from(containerXml), { name: 'META-INF/container.xml' });
|
||||
archive.append(Buffer.from(opfXml), { name: 'OPS/content.opf' });
|
||||
for (const entry of entries) archive.append(entry.content, { name: entry.path });
|
||||
for (const entry of entries) archive.append(entry.content, { name: entry.path, store: entry.store });
|
||||
void archive.finalize();
|
||||
});
|
||||
}
|
||||
@@ -225,6 +230,51 @@ describe('EPUB cover write round trip', () => {
|
||||
});
|
||||
|
||||
describe('EPUB metadata write round trip', () => {
|
||||
it('preserves original entry compression and content across repeated metadata saves', async () => {
|
||||
const filePath = join(testRoot, 'image-heavy.epub');
|
||||
const opf = `
|
||||
<package version="3.0" unique-identifier="uid" xmlns="http://www.idpf.org/2007/opf">
|
||||
<metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
|
||||
<dc:identifier id="uid">urn:test:image-heavy</dc:identifier>
|
||||
<dc:title>Old title</dc:title>
|
||||
<dc:language>en</dc:language>
|
||||
</metadata>
|
||||
<manifest>
|
||||
<item id="image" href="images/page.JPG" media-type="image/jpeg" />
|
||||
<item id="chapter" href="chapter.xhtml" media-type="application/xhtml+xml" />
|
||||
</manifest>
|
||||
<spine><itemref idref="chapter" /></spine>
|
||||
</package>`;
|
||||
const image = await sharp(randomBytes(256 * 256 * 3), { raw: { width: 256, height: 256, channels: 3 } })
|
||||
.jpeg({ quality: 90 })
|
||||
.toBuffer();
|
||||
const compressible = Buffer.alloc(100_000);
|
||||
const chapter = `<html><body>${'chapter text '.repeat(1000)}</body></html>`;
|
||||
await writeArchive(filePath, opf, [
|
||||
{ path: 'OPS/images/page.JPG', content: image, store: true },
|
||||
{ path: 'OPS/images/mislabeled.png', content: compressible },
|
||||
{ path: 'OPS/chapter.xhtml', content: chapter },
|
||||
]);
|
||||
|
||||
for (const title of ['First title', 'Second title']) {
|
||||
const result = await new EpubFormatWriter().write(filePath, { title }, { fieldMask: new Set(['title']), dryRun: false });
|
||||
expect(result.status).toBe('success');
|
||||
const zip = await unzipper.Open.file(filePath);
|
||||
const byPath = new Map(zip.files.map((entry) => [entry.path, entry]));
|
||||
expect(zip.files[0]!.path).toBe('mimetype');
|
||||
expect(zip.files[0]!.compressionMethod).toBe(0);
|
||||
expect(byPath.get('OPS/images/page.JPG')!.compressionMethod).toBe(0);
|
||||
expect(byPath.get('OPS/images/mislabeled.png')!.compressionMethod).toBe(8);
|
||||
expect(byPath.get('OPS/chapter.xhtml')!.compressionMethod).toBe(8);
|
||||
expect(byPath.get('OPS/content.opf')!.compressionMethod).toBe(8);
|
||||
await expect(byPath.get('OPS/images/page.JPG')!.buffer()).resolves.toEqual(image);
|
||||
await expect(byPath.get('OPS/images/mislabeled.png')!.buffer()).resolves.toEqual(compressible);
|
||||
await expect(byPath.get('OPS/chapter.xhtml')!.buffer()).resolves.toEqual(Buffer.from(chapter));
|
||||
const pkg = objectParser.parse((await byPath.get('OPS/content.opf')!.buffer()).toString()).package;
|
||||
expect(pkg.metadata.title['#text']).toBe(title);
|
||||
}
|
||||
});
|
||||
|
||||
async function listEntryPaths(filePath: string): Promise<string[]> {
|
||||
const zip = await unzipper.Open.file(filePath);
|
||||
return zip.files.map((entry) => entry.path).sort();
|
||||
|
||||
@@ -1,186 +1,70 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import { PassThrough, Readable } from 'stream';
|
||||
import type { MockedFunction } from 'vitest';
|
||||
import { createWriteStream } from 'fs';
|
||||
import * as fs from 'fs/promises';
|
||||
import * as unzipper from 'unzipper';
|
||||
import { ZipArchive } from 'archiver';
|
||||
|
||||
import { replaceFileAtomically } from '../shared/atomic-file-replace';
|
||||
import { rewriteEpubZip } from './epub-zip-rewrite';
|
||||
import { listEntryPaths, patch, readEntry } from './epub-zip-patcher';
|
||||
|
||||
vi.mock('fs', async () => {
|
||||
const actual = await vi.importActual('fs');
|
||||
return {
|
||||
...actual,
|
||||
createWriteStream: vi.fn(),
|
||||
};
|
||||
});
|
||||
vi.mock('unzipper', () => ({ Open: { file: vi.fn() } }));
|
||||
vi.mock('./epub-zip-rewrite', () => ({ rewriteEpubZip: vi.fn() }));
|
||||
vi.mock('../shared/atomic-file-replace', () => ({ replaceFileAtomically: vi.fn() }));
|
||||
|
||||
vi.mock('fs/promises', async () => {
|
||||
const actual = await vi.importActual('fs/promises');
|
||||
return {
|
||||
...actual,
|
||||
rename: vi.fn(),
|
||||
unlink: vi.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('unzipper', () => ({
|
||||
Open: {
|
||||
file: vi.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
let lastArchive: EventEmitter & {
|
||||
append: vi.Mock;
|
||||
pipe: vi.Mock;
|
||||
finalize: vi.Mock;
|
||||
};
|
||||
|
||||
const processedEntries: string[] = [];
|
||||
|
||||
vi.mock('archiver', () => ({
|
||||
ZipArchive: vi.fn(function () {
|
||||
let output: EventEmitter | null = null;
|
||||
const emitter = new EventEmitter() as EventEmitter & {
|
||||
append: vi.Mock;
|
||||
pipe: vi.Mock;
|
||||
finalize: vi.Mock;
|
||||
};
|
||||
// The real archiver emits `entry` once it has consumed and written an appended source.
|
||||
emitter.append = vi.fn((source: unknown, data: { name: string }) => {
|
||||
if (source instanceof Readable) source.resume();
|
||||
setImmediate(() => {
|
||||
processedEntries.push(data.name);
|
||||
emitter.emit('entry', data);
|
||||
});
|
||||
return emitter;
|
||||
});
|
||||
emitter.pipe = vi.fn((out: EventEmitter) => {
|
||||
output = out;
|
||||
});
|
||||
emitter.finalize = vi.fn(() => {
|
||||
setImmediate(() => output?.emit('close'));
|
||||
});
|
||||
lastArchive = emitter;
|
||||
return emitter;
|
||||
}),
|
||||
}));
|
||||
|
||||
const mockCreateWriteStream = createWriteStream as MockedFunction<typeof createWriteStream>;
|
||||
const mockRename = fs.rename as MockedFunction<typeof fs.rename>;
|
||||
const mockUnlink = fs.unlink as MockedFunction<typeof fs.unlink>;
|
||||
const mockOpenFile = unzipper.Open.file as MockedFunction<typeof unzipper.Open.file>;
|
||||
const streamFrom = (value: string) => Readable.from([Buffer.from(value)]);
|
||||
const mockOpenFile = vi.mocked(unzipper.Open.file);
|
||||
const mockRewrite = vi.mocked(rewriteEpubZip);
|
||||
const mockReplace = vi.mocked(replaceFileAtomically);
|
||||
|
||||
describe('epub-zip-patcher', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
processedEntries.length = 0;
|
||||
mockCreateWriteStream.mockImplementation(() => new PassThrough() as never);
|
||||
mockRename.mockResolvedValue(undefined);
|
||||
mockUnlink.mockResolvedValue(undefined);
|
||||
vi.resetAllMocks();
|
||||
mockRewrite.mockResolvedValue(undefined);
|
||||
mockReplace.mockResolvedValue(undefined);
|
||||
});
|
||||
|
||||
it('readEntry returns UTF-8 text for existing entry', async () => {
|
||||
it('readEntry returns UTF-8 text for an existing entry', async () => {
|
||||
mockOpenFile.mockResolvedValue({
|
||||
files: [{ path: 'OPS/content.opf', buffer: vi.fn().mockResolvedValue(Buffer.from('<package/>')) }],
|
||||
} as never);
|
||||
|
||||
await expect(readEntry('/book.epub', 'OPS/content.opf')).resolves.toBe('<package/>');
|
||||
await expect(readEntry('/path/to/book.epub', 'OPS/content.opf')).resolves.toBe('<package/>');
|
||||
});
|
||||
|
||||
it('readEntry throws for missing entry', async () => {
|
||||
it('readEntry rejects a missing entry', async () => {
|
||||
mockOpenFile.mockResolvedValue({ files: [] } as never);
|
||||
|
||||
await expect(readEntry('/book.epub', 'OPS/content.opf')).rejects.toThrow('Entry not found in EPUB: OPS/content.opf');
|
||||
await expect(readEntry('/path/to/book.epub', 'OPS/content.opf')).rejects.toThrow('Entry not found in EPUB: OPS/content.opf');
|
||||
});
|
||||
|
||||
it('lists every occupied archive entry path', async () => {
|
||||
mockOpenFile.mockResolvedValue({
|
||||
files: [{ path: 'mimetype' }, { path: 'OPS/content.opf' }, { path: 'OPS/images/cover.webp' }],
|
||||
} as never);
|
||||
|
||||
await expect(listEntryPaths('/book.epub')).resolves.toEqual(['mimetype', 'OPS/content.opf', 'OPS/images/cover.webp']);
|
||||
mockOpenFile.mockResolvedValue({ files: [{ path: 'mimetype' }, { path: 'OPS/content.opf' }, { path: 'OPS/images/cover.webp' }] } as never);
|
||||
await expect(listEntryPaths('/path/to/book.epub')).resolves.toEqual(['mimetype', 'OPS/content.opf', 'OPS/images/cover.webp']);
|
||||
});
|
||||
|
||||
it('patch writes mimetype first, patches existing entries, and appends new ones', async () => {
|
||||
const opfEntry = { path: 'OPS/content.opf', stream: vi.fn(() => Buffer.from('old-opf')) };
|
||||
const chapterEntry = { path: 'OPS/ch1.xhtml', stream: vi.fn(() => streamFrom('old-ch1')) };
|
||||
const mimetypeEntry = { path: 'mimetype', stream: vi.fn(() => streamFrom('old-mimetype')) };
|
||||
|
||||
mockOpenFile.mockResolvedValue({ files: [mimetypeEntry, opfEntry, chapterEntry] } as never);
|
||||
|
||||
const patches = new Map<string, Buffer>([
|
||||
['OPS/content.opf', Buffer.from('new-opf')],
|
||||
['OPS/images/cover.jpg', Buffer.from('new-cover')],
|
||||
]);
|
||||
|
||||
await patch('/book.epub', patches);
|
||||
|
||||
expect(ZipArchive).toHaveBeenCalledWith({ zlib: { level: 6 } });
|
||||
expect(lastArchive.append).toHaveBeenNthCalledWith(1, Buffer.from('application/epub+zip'), { name: 'mimetype', store: true });
|
||||
expect(lastArchive.append).toHaveBeenCalledWith(Buffer.from('new-opf'), { name: 'OPS/content.opf' });
|
||||
expect(chapterEntry.stream).toHaveBeenCalledTimes(1);
|
||||
expect(lastArchive.append).toHaveBeenCalledWith(expect.objectContaining({ pipe: expect.any(Function) }), { name: 'OPS/ch1.xhtml' });
|
||||
expect(lastArchive.append).toHaveBeenCalledWith(Buffer.from('new-cover'), { name: 'OPS/images/cover.jpg' });
|
||||
expect(mockRename).toHaveBeenCalledWith(expect.stringMatching(/^\/\.epub-write-[0-9a-f-]{36}$/), '/book.epub');
|
||||
it('replaces the original only after the rewrite completes', async () => {
|
||||
const patches = new Map([['OPS/content.opf', Buffer.from('new-opf')]]);
|
||||
mockRewrite.mockImplementation(() => {
|
||||
expect(mockReplace).not.toHaveBeenCalled();
|
||||
return Promise.resolve();
|
||||
});
|
||||
await patch('/path/to/book.epub', patches);
|
||||
const tempPath = mockRewrite.mock.calls[0]![1];
|
||||
expect(tempPath).toMatch(/^\/path\/to\/\.epub-write-[0-9a-f-]{36}$/);
|
||||
expect(mockRewrite).toHaveBeenCalledWith('/path/to/book.epub', tempPath, patches);
|
||||
expect(mockReplace).toHaveBeenCalledWith(tempPath, '/path/to/book.epub');
|
||||
});
|
||||
|
||||
it('writes beside the book under a hidden random name that never grows with the file name', async () => {
|
||||
mockOpenFile.mockResolvedValue({ files: [] } as never);
|
||||
const longName = `/library/${'L'.repeat(250)}.epub`;
|
||||
|
||||
it('uses distinct short temporary names even for a long book filename', async () => {
|
||||
const longName = `/path/to/${'L'.repeat(250)}.epub`;
|
||||
await patch(longName, new Map());
|
||||
|
||||
const [tempPath, target] = mockRename.mock.calls[0]!;
|
||||
expect(target).toBe(longName);
|
||||
expect(String(tempPath)).toMatch(/^\/library\/\.epub-write-[0-9a-f-]{36}$/);
|
||||
await patch(longName, new Map());
|
||||
const paths = mockRewrite.mock.calls.map((call) => call[1]);
|
||||
expect(paths[0]).not.toBe(paths[1]);
|
||||
for (const path of paths) expect(path).toMatch(/^\/path\/to\/\.epub-write-[0-9a-f-]{36}$/);
|
||||
});
|
||||
|
||||
it('opens each carried-over entry stream only after the previous entry was written', async () => {
|
||||
const openedAfter: string[][] = [];
|
||||
const carried = ['OPS/ch1.xhtml', 'OPS/ch2.xhtml', 'OPS/ch3.xhtml'].map((path) => ({
|
||||
path,
|
||||
stream: vi.fn(() => {
|
||||
openedAfter.push([...processedEntries]);
|
||||
return streamFrom(path);
|
||||
}),
|
||||
}));
|
||||
|
||||
mockOpenFile.mockResolvedValue({ files: carried } as never);
|
||||
|
||||
await patch('/book.epub', new Map());
|
||||
|
||||
expect(openedAfter).toEqual([['mimetype'], ['mimetype', 'OPS/ch1.xhtml'], ['mimetype', 'OPS/ch1.xhtml', 'OPS/ch2.xhtml']]);
|
||||
expect(processedEntries).toEqual(['mimetype', 'OPS/ch1.xhtml', 'OPS/ch2.xhtml', 'OPS/ch3.xhtml']);
|
||||
for (const entry of carried) expect(entry.stream).toHaveBeenCalledTimes(1);
|
||||
it('never replaces the original when the rewrite fails', async () => {
|
||||
mockRewrite.mockRejectedValue(new Error('read failed'));
|
||||
await expect(patch('/path/to/book.epub', new Map())).rejects.toThrow('read failed');
|
||||
expect(mockReplace).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects when a source entry stream errors while patching', async () => {
|
||||
const sourceError = Object.assign(new Error('ENOENT: no such file or directory, open /book.epub'), { code: 'ENOENT' });
|
||||
const missingEntry = {
|
||||
path: 'OPS/ch1.xhtml',
|
||||
stream: vi.fn(() => {
|
||||
const source = new PassThrough();
|
||||
process.nextTick(() => source.emit('error', sourceError));
|
||||
return source;
|
||||
}),
|
||||
};
|
||||
mockOpenFile.mockResolvedValue({ files: [missingEntry] } as never);
|
||||
|
||||
await expect(patch('/book.epub', new Map())).rejects.toThrow('ENOENT');
|
||||
|
||||
expect(missingEntry.stream).toHaveBeenCalledTimes(1);
|
||||
expect(mockRename).not.toHaveBeenCalled();
|
||||
expect(mockUnlink).toHaveBeenCalledWith(expect.stringMatching(/^\/\.epub-write-/));
|
||||
});
|
||||
|
||||
it('deletes temp file when rename fails', async () => {
|
||||
mockOpenFile.mockResolvedValue({ files: [] } as never);
|
||||
mockRename.mockRejectedValue(new Error('rename failed'));
|
||||
|
||||
await expect(patch('/book.epub', new Map())).rejects.toThrow('rename failed');
|
||||
expect(mockUnlink).toHaveBeenCalledWith(mockRename.mock.calls[0]![0]);
|
||||
it('propagates atomic replacement failures', async () => {
|
||||
mockReplace.mockRejectedValue(new Error('rename failed'));
|
||||
await expect(patch('/path/to/book.epub', new Map())).rejects.toThrow('rename failed');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { unlink } from 'fs/promises';
|
||||
import { dirname, join } from 'path';
|
||||
import * as unzipper from 'unzipper';
|
||||
import { replaceFileAtomically } from '../shared/atomic-file-replace';
|
||||
import { writeZipArchive, type ZipRewriteEntry } from '../shared/zip-rewrite';
|
||||
|
||||
const MIMETYPE_ENTRY_PATH = 'mimetype';
|
||||
import { rewriteEpubZip } from './epub-zip-rewrite';
|
||||
|
||||
export async function readEntry(filePath: string, entryPath: string): Promise<string> {
|
||||
const zip = await unzipper.Open.file(filePath);
|
||||
@@ -23,30 +20,6 @@ export async function patch(filePath: string, patches: Map<string, Buffer>): Pro
|
||||
// A hidden, random name like every other writer: appending ".tmp" overflowed long file names and
|
||||
// would have overwritten a user's own file of that name.
|
||||
const tmpPath = join(dirname(filePath), `.epub-write-${randomUUID()}`);
|
||||
const zip = await unzipper.Open.file(filePath);
|
||||
|
||||
try {
|
||||
await writeZipArchive(tmpPath, rewriteEntries(zip.files, patches));
|
||||
} catch (error) {
|
||||
await unlink(tmpPath).catch(() => {});
|
||||
throw error;
|
||||
}
|
||||
|
||||
await rewriteEpubZip(filePath, tmpPath, patches);
|
||||
await replaceFileAtomically(tmpPath, filePath);
|
||||
}
|
||||
|
||||
function* rewriteEntries(files: readonly unzipper.File[], patches: Map<string, Buffer>): Generator<ZipRewriteEntry> {
|
||||
yield { name: MIMETYPE_ENTRY_PATH, source: Buffer.from('application/epub+zip'), store: true };
|
||||
|
||||
for (const entry of files) {
|
||||
if (entry.path === MIMETYPE_ENTRY_PATH) continue;
|
||||
const patched = patches.get(entry.path);
|
||||
yield { name: entry.path, source: patched ?? (() => entry.stream()) };
|
||||
}
|
||||
|
||||
for (const [path, content] of patches) {
|
||||
if (!files.some((entry) => entry.path === path)) {
|
||||
yield { name: path, source: content };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
import * as fs from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { crc32, deflateRawSync, inflateRawSync } from 'zlib';
|
||||
import { Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter, type ZipWriterConstructorOptions } from '@zip.js/zip.js';
|
||||
import * as unzipper from 'unzipper';
|
||||
import { patch } from './epub-zip-patcher';
|
||||
import { rewriteEpubZip } from './epub-zip-rewrite';
|
||||
|
||||
vi.mock('fs/promises', async () => ({ ...(await vi.importActual<typeof fs>('fs/promises')) }));
|
||||
|
||||
let root: string;
|
||||
let filePath: string;
|
||||
let tmpPath: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await fs.mkdtemp(join(tmpdir(), 'epub-raw-copy-'));
|
||||
filePath = join(root, 'book.epub');
|
||||
tmpPath = join(root, 'rewrite.tmp');
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const contents = new Map([
|
||||
['mimetype', Buffer.from('application/epub+zip')],
|
||||
['OPS/content.opf', Buffer.from('<package><title>Old</title></package>')],
|
||||
['OPS/images/stored.jpg', Buffer.from('original JPEG bytes'.repeat(100))],
|
||||
['OPS/images/packed.png', Buffer.alloc(10_000)],
|
||||
['OPS/chapter.xhtml', Buffer.from('<html>chapter</html>'.repeat(1000))],
|
||||
['OPS/fonts/文書.otf', Buffer.from('obfuscated font bytes'.repeat(100))],
|
||||
['OPS/empty', Buffer.alloc(0)],
|
||||
['OPS/directory/', Buffer.alloc(0)],
|
||||
]);
|
||||
|
||||
async function fixture(options: ZipWriterConstructorOptions = {}): Promise<Buffer> {
|
||||
const writer = new ZipWriter(new Uint8ArrayWriter(), { useWebWorkers: false, extendedTimestamp: false, ...options });
|
||||
for (const [name, content] of contents) {
|
||||
const stored = name === 'mimetype' || name.endsWith('.jpg') || name.endsWith('/') || name.endsWith('empty');
|
||||
await writer.add(name, new Uint8ArrayReader(stored ? content : deflateRawSync(content, { level: 1 })), {
|
||||
passThrough: true,
|
||||
compressionMethod: stored ? 0 : 8,
|
||||
uncompressedSize: content.length,
|
||||
crc32: crc32(content),
|
||||
directory: name.endsWith('/'),
|
||||
lastModDate: new Date('2020-01-02T03:04:06Z'),
|
||||
comment: 'entry comment',
|
||||
extraField: new Map([[0xcafe, new Uint8Array([1, 2, 3])]]),
|
||||
externalFileAttributes: 0x81a40000,
|
||||
internalFileAttributes: 1,
|
||||
});
|
||||
}
|
||||
return Buffer.from(await writer.close(new Uint8Array([0xff, 0, 0x42])));
|
||||
}
|
||||
|
||||
async function snapshot(bytes: Buffer) {
|
||||
const reader = new ZipReader(new Uint8ArrayReader(bytes));
|
||||
const entries = await reader.getEntries();
|
||||
const ordered = [...entries].sort((a, b) => a.offset - b.offset);
|
||||
const records = new Map(
|
||||
ordered.map((entry, index) => [entry.filename, bytes.subarray(entry.offset, ordered[index + 1]?.offset ?? reader.directoryOffset)]),
|
||||
);
|
||||
await reader.close();
|
||||
return { entries: new Map(entries.map((entry) => [entry.filename, entry])), records, comment: reader.comment };
|
||||
}
|
||||
|
||||
function centralRecords(bytes: Buffer): number[] {
|
||||
const eocd = bytes.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06]));
|
||||
const offsets: number[] = [];
|
||||
let offset = bytes.readUInt32LE(eocd + 16);
|
||||
while (bytes.readUInt32LE(offset) === 0x02014b50) {
|
||||
offsets.push(offset);
|
||||
offset += 46 + bytes.readUInt16LE(offset + 28) + bytes.readUInt16LE(offset + 30) + bytes.readUInt16LE(offset + 32);
|
||||
}
|
||||
return offsets;
|
||||
}
|
||||
|
||||
describe('raw EPUB ZIP rewrite', () => {
|
||||
it.each([
|
||||
{ dataDescriptor: false },
|
||||
{ dataDescriptor: true, dataDescriptorSignature: true },
|
||||
{ dataDescriptor: true, dataDescriptorSignature: false },
|
||||
{ zip64: true, dataDescriptor: true, dataDescriptorSignature: true },
|
||||
{ zip64: true, dataDescriptor: true, dataDescriptorSignature: false },
|
||||
])('preserves complete unchanged local records with %j', async (options) => {
|
||||
const original = await fixture(options);
|
||||
await fs.writeFile(filePath, original);
|
||||
await fs.chmod(filePath, 0o640);
|
||||
const before = await snapshot(original);
|
||||
const newOpf = Buffer.from('<package><title>Updated title</title></package>');
|
||||
const cover = Buffer.from('new cover image bytes');
|
||||
for (let save = 0; save < 2; save++) {
|
||||
await patch(
|
||||
filePath,
|
||||
new Map([
|
||||
['OPS/content.opf', newOpf],
|
||||
['OPS/images/new-cover.jpg', cover],
|
||||
]),
|
||||
);
|
||||
const bytes = await fs.readFile(filePath);
|
||||
const after = await snapshot(bytes);
|
||||
const independent = await unzipper.Open.file(filePath);
|
||||
const decoded = new Map(independent.files.map((entry) => [entry.path, entry]));
|
||||
expect(after.comment).toEqual(before.comment);
|
||||
expect(after.entries.size).toBe(contents.size + 1);
|
||||
for (const [name, content] of contents) {
|
||||
if (name === 'mimetype' || name.endsWith('.opf')) continue;
|
||||
const oldEntry = before.entries.get(name)!;
|
||||
const newEntry = after.entries.get(name)!;
|
||||
expect(after.records.get(name), name).toEqual(before.records.get(name));
|
||||
expect(newEntry.compressionMethod).toBe(oldEntry.compressionMethod);
|
||||
expect(newEntry.compressedSize).toBe(oldEntry.compressedSize);
|
||||
expect(newEntry.crc32).toBe(oldEntry.crc32);
|
||||
expect(newEntry.rawExtraField).toEqual(oldEntry.rawExtraField);
|
||||
expect(newEntry.rawComment).toEqual(oldEntry.rawComment);
|
||||
expect(newEntry.externalFileAttributes).toBe(oldEntry.externalFileAttributes);
|
||||
expect(newEntry.internalFileAttributes).toBe(oldEntry.internalFileAttributes);
|
||||
const record = after.records.get(name)!;
|
||||
const dataOffset = 30 + record.readUInt16LE(26) + record.readUInt16LE(28);
|
||||
const payload = record.subarray(dataOffset, dataOffset + newEntry.compressedSize);
|
||||
const decodedBytes = newEntry.compressionMethod === 0 ? payload : inflateRawSync(payload);
|
||||
expect(decodedBytes).toEqual(content);
|
||||
expect(crc32(decodedBytes)).toBe(newEntry.crc32);
|
||||
// unzipper cannot read empty stored entries with unsigned descriptors; native decoding covers them.
|
||||
if (content.length > 0) await expect(decoded.get(name)!.buffer()).resolves.toEqual(content);
|
||||
}
|
||||
await expect(decoded.get('OPS/content.opf')!.buffer()).resolves.toEqual(newOpf);
|
||||
await expect(decoded.get('OPS/images/new-cover.jpg')!.buffer()).resolves.toEqual(cover);
|
||||
expect(bytes.readUInt32LE(0)).toBe(0x04034b50);
|
||||
expect(bytes.readUInt16LE(8)).toBe(0);
|
||||
expect(bytes.readUInt16LE(6) & 9).toBe(0);
|
||||
expect(bytes.readUInt16LE(28)).toBe(0);
|
||||
expect(bytes.subarray(30, 38).toString()).toBe('mimetype');
|
||||
expect(bytes.subarray(38, 58).toString()).toBe('application/epub+zip');
|
||||
expect((await fs.stat(filePath)).mode & 0o777).toBe(0o640);
|
||||
}
|
||||
expect(await fs.readdir(root)).toEqual(['book.epub']);
|
||||
});
|
||||
|
||||
it('retains UTF-8 filenames from producers that omit the UTF-8 flag', async () => {
|
||||
const original = await fixture();
|
||||
const before = await snapshot(original);
|
||||
const name = 'OPS/fonts/文書.otf';
|
||||
const entry = before.entries.get(name)!;
|
||||
original.writeUInt16LE(original.readUInt16LE(entry.offset + 6) & ~0x800, entry.offset + 6);
|
||||
const central = centralRecords(original).find(
|
||||
(offset) => original.subarray(offset + 46, offset + 46 + original.readUInt16LE(offset + 28)).toString() === name,
|
||||
)!;
|
||||
original.writeUInt16LE(original.readUInt16LE(central + 8) & ~0x800, central + 8);
|
||||
await fs.writeFile(filePath, original);
|
||||
await patch(filePath, new Map());
|
||||
const after = await unzipper.Open.file(filePath);
|
||||
await expect(after.files.find((entry) => entry.path === name)!.buffer()).resolves.toEqual(contents.get(name));
|
||||
});
|
||||
|
||||
it.each([
|
||||
'local signature',
|
||||
'local filename',
|
||||
'local compression',
|
||||
'local flags',
|
||||
'overlap',
|
||||
'duplicate name',
|
||||
'duplicate offset',
|
||||
'truncated directory',
|
||||
])('keeps the original intact for %s corruption', async (kind) => {
|
||||
const original = await fixture({ dataDescriptor: false });
|
||||
const records = centralRecords(original);
|
||||
const before = await snapshot(original);
|
||||
const chapterOffset = before.entries.get('OPS/chapter.xhtml')!.offset;
|
||||
if (kind === 'local signature') original.writeUInt32LE(0, chapterOffset);
|
||||
if (kind === 'local filename') original[chapterOffset + 30] = 0x2e;
|
||||
if (kind === 'local compression') original.writeUInt16LE(0, chapterOffset + 8);
|
||||
if (kind === 'local flags') original.writeUInt16LE(1, chapterOffset + 6);
|
||||
if (kind === 'overlap') original.writeUInt32LE(original.length, records[4]! + 20);
|
||||
if (kind === 'duplicate name') {
|
||||
const first = records[2]!;
|
||||
const second = records[3]!;
|
||||
const name = original.subarray(first + 46, first + 46 + original.readUInt16LE(first + 28));
|
||||
expect(name.length).toBe(original.readUInt16LE(second + 28));
|
||||
name.copy(original, second + 46);
|
||||
}
|
||||
if (kind === 'duplicate offset') original.writeUInt32LE(original.readUInt32LE(records[2]! + 42), records[3]! + 42);
|
||||
const input = kind === 'truncated directory' ? original.subarray(0, original.length - 10) : original;
|
||||
await fs.writeFile(filePath, input);
|
||||
await expect(patch(filePath, new Map([['OPS/content.opf', Buffer.from('updated')]]))).rejects.toThrow();
|
||||
await expect(fs.readFile(filePath)).resolves.toEqual(input);
|
||||
expect(await fs.readdir(root)).toEqual(['book.epub']);
|
||||
});
|
||||
|
||||
it.each(['../escape', '/path/to/escape', 'file:escape', 'OPS/../../escape', 'OPS\\escape', 'OPS/\0escape'])(
|
||||
'rejects unsafe replacement path %j',
|
||||
async (name) => {
|
||||
const original = await fixture();
|
||||
await fs.writeFile(filePath, original);
|
||||
await expect(patch(filePath, new Map([[name, Buffer.from('unsafe')]]))).rejects.toThrow('replacement path');
|
||||
await expect(fs.readFile(filePath)).resolves.toEqual(original);
|
||||
expect(await fs.readdir(root)).toEqual(['book.epub']);
|
||||
},
|
||||
);
|
||||
|
||||
it('never overwrites or removes an existing temporary file', async () => {
|
||||
await fs.writeFile(filePath, await fixture());
|
||||
await fs.writeFile(tmpPath, 'user-owned');
|
||||
await expect(rewriteEpubZip(filePath, tmpPath, new Map())).rejects.toMatchObject({ code: 'EEXIST' });
|
||||
await expect(fs.readFile(tmpPath, 'utf8')).resolves.toBe('user-owned');
|
||||
});
|
||||
|
||||
it.each(['read', 'write', 'sync', 'zero read', 'zero write', 'changed source', 'source close', 'output close'])(
|
||||
'preserves the source and closes handles after %s failure',
|
||||
async (failure) => {
|
||||
const original = await fixture();
|
||||
await fs.writeFile(filePath, original);
|
||||
const open = fs.open;
|
||||
const handles: fs.FileHandle[] = [];
|
||||
const closed: ReturnType<typeof vi.spyOn>[] = [];
|
||||
vi.spyOn(fs, 'open').mockImplementation(async (...args: Parameters<typeof fs.open>) => {
|
||||
const handle = await open(...args);
|
||||
handles.push(handle);
|
||||
const close = handle.close.bind(handle);
|
||||
const closeSpy = vi.spyOn(handle, 'close');
|
||||
closed.push(closeSpy);
|
||||
if ((failure === 'source close' && args[1] === 'r') || (failure === 'output close' && args[1] !== 'r')) {
|
||||
closeSpy.mockImplementation(async () => {
|
||||
await close();
|
||||
throw new Error('injected close failure');
|
||||
});
|
||||
}
|
||||
if (args[1] === 'r') {
|
||||
if (failure === 'read') vi.spyOn(handle, 'read').mockRejectedValue(new Error('injected read failure'));
|
||||
if (failure === 'zero read') vi.spyOn(handle, 'read').mockResolvedValue({ bytesRead: 0, buffer: Buffer.alloc(0) } as never);
|
||||
if (failure === 'changed source') {
|
||||
const stat = handle.stat.bind(handle);
|
||||
let calls = 0;
|
||||
vi.spyOn(handle, 'stat').mockImplementation(async () => {
|
||||
const result = await stat();
|
||||
if (++calls > 1) result.mtimeMs += 1000;
|
||||
return result;
|
||||
});
|
||||
}
|
||||
} else {
|
||||
if (failure === 'write') vi.spyOn(handle, 'write').mockRejectedValue(new Error('injected write failure'));
|
||||
if (failure === 'zero write') vi.spyOn(handle, 'write').mockResolvedValue({ bytesWritten: 0, buffer: Buffer.alloc(0) } as never);
|
||||
if (failure === 'sync') vi.spyOn(handle, 'sync').mockRejectedValue(new Error('injected sync failure'));
|
||||
}
|
||||
return handle;
|
||||
});
|
||||
await expect(patch(filePath, new Map())).rejects.toThrow();
|
||||
for (const close of closed) expect(close).toHaveBeenCalledOnce();
|
||||
vi.restoreAllMocks();
|
||||
await expect(fs.readFile(filePath)).resolves.toEqual(original);
|
||||
expect(await fs.readdir(root)).toEqual(['book.epub']);
|
||||
expect(handles.length).toBeGreaterThan(0);
|
||||
},
|
||||
);
|
||||
|
||||
it('handles short filesystem reads and writes without losing bytes', async () => {
|
||||
const original = await fixture();
|
||||
await fs.writeFile(filePath, original);
|
||||
const open = fs.open;
|
||||
vi.spyOn(fs, 'open').mockImplementation(async (...args: Parameters<typeof fs.open>) => {
|
||||
const handle = await open(...args);
|
||||
if (args[1] === 'r') {
|
||||
const read = handle.read.bind(handle);
|
||||
vi.spyOn(handle, 'read').mockImplementation(((buffer: Buffer, offset: number, length: number, position: number) =>
|
||||
read(buffer, offset, Math.min(length, 97), position)) as never);
|
||||
} else {
|
||||
const write = handle.write.bind(handle);
|
||||
vi.spyOn(handle, 'write').mockImplementation(((buffer: Buffer, offset: number, length: number) =>
|
||||
write(buffer, offset, Math.min(length, 71))) as never);
|
||||
}
|
||||
return handle;
|
||||
});
|
||||
await patch(filePath, new Map());
|
||||
vi.restoreAllMocks();
|
||||
const before = await snapshot(original);
|
||||
const after = await snapshot(await fs.readFile(filePath));
|
||||
for (const [name, bytes] of before.records) {
|
||||
if (name !== 'mimetype') expect(after.records.get(name)).toEqual(bytes);
|
||||
}
|
||||
});
|
||||
|
||||
it('bounds directory allocation before reading a large declared directory', async () => {
|
||||
const file = await fs.open(filePath, 'w');
|
||||
const directoryLength = 64 * 1024 * 1024 + 1;
|
||||
const eocd = Buffer.alloc(22);
|
||||
eocd.writeUInt32LE(0x06054b50, 0);
|
||||
eocd.writeUInt16LE(1, 8);
|
||||
eocd.writeUInt16LE(1, 10);
|
||||
eocd.writeUInt32LE(directoryLength, 12);
|
||||
await file.write(eocd, 0, eocd.length, directoryLength);
|
||||
await file.close();
|
||||
await expect(patch(filePath, new Map())).rejects.toThrow('read limit');
|
||||
expect((await fs.stat(filePath)).size).toBe(directoryLength + 22);
|
||||
expect(await fs.readdir(root)).toEqual(['book.epub']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
import { open, unlink, type FileHandle } from 'fs/promises';
|
||||
import { promisify } from 'util';
|
||||
import { crc32, deflateRaw } from 'zlib';
|
||||
import { Reader, Writer, Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter, type CreateReadableOptions, type Entry } from '@zip.js/zip.js';
|
||||
|
||||
const MAX_READ_BYTES = 64 * 1024 * 1024;
|
||||
const MAX_ENTRIES = 100_000;
|
||||
const COPY_CHUNK_BYTES = 256 * 1024;
|
||||
const compress = promisify(deflateRaw);
|
||||
|
||||
class FileReader extends Reader<FileHandle> {
|
||||
constructor(
|
||||
private readonly file: FileHandle,
|
||||
size: number,
|
||||
) {
|
||||
super(file);
|
||||
this.size = size;
|
||||
}
|
||||
|
||||
override createReadable(options: CreateReadableOptions = {}): ReadableStream<Uint8Array> {
|
||||
return super.createReadable({ ...options, chunkSize: COPY_CHUNK_BYTES });
|
||||
}
|
||||
|
||||
override async readUint8Array(index: number, length: number): Promise<Uint8Array> {
|
||||
if (!Number.isSafeInteger(index) || !Number.isSafeInteger(length) || index < 0 || index > this.size || length < 0) {
|
||||
throw new Error('Invalid EPUB ZIP read range');
|
||||
}
|
||||
const size = Math.min(length, this.size - index);
|
||||
// zip.js reads the central directory in one allocation, but streams unchanged entry records.
|
||||
if (size > MAX_READ_BYTES) throw new Error('EPUB ZIP directory exceeds the read limit');
|
||||
const buffer = Buffer.allocUnsafe(size);
|
||||
let offset = 0;
|
||||
while (offset < size) {
|
||||
const { bytesRead } = await this.file.read(buffer, offset, size - offset, index + offset);
|
||||
if (!bytesRead) throw new Error('EPUB ZIP was truncated during write-back');
|
||||
offset += bytesRead;
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
}
|
||||
|
||||
class FileWriter extends Writer<void> {
|
||||
constructor(private readonly file: FileHandle) {
|
||||
super();
|
||||
}
|
||||
|
||||
override async writeUint8Array(data: Uint8Array): Promise<void> {
|
||||
let offset = 0;
|
||||
while (offset < data.length) {
|
||||
const { bytesWritten } = await this.file.write(data, offset, data.length - offset);
|
||||
if (!bytesWritten) throw new Error('EPUB ZIP write made no progress');
|
||||
offset += bytesWritten;
|
||||
}
|
||||
}
|
||||
|
||||
override async getData(): Promise<void> {}
|
||||
}
|
||||
|
||||
export async function rewriteEpubZip(filePath: string, tmpPath: string, patches: ReadonlyMap<string, Buffer>): Promise<void> {
|
||||
for (const [name, content] of patches) {
|
||||
if (
|
||||
!name ||
|
||||
name.startsWith('/') ||
|
||||
name.includes('\\') ||
|
||||
name.includes('\0') ||
|
||||
/^[a-z][a-z\d+.-]*:/i.test(name) ||
|
||||
name.split('/').includes('..')
|
||||
) {
|
||||
throw new Error('Invalid EPUB ZIP replacement path');
|
||||
}
|
||||
if (content.length > MAX_READ_BYTES) throw new Error('EPUB ZIP replacement exceeds the size limit');
|
||||
}
|
||||
const source = await open(filePath, 'r');
|
||||
let output: FileHandle | undefined;
|
||||
let completed = false;
|
||||
try {
|
||||
try {
|
||||
const original = await source.stat();
|
||||
if (!Number.isSafeInteger(original.size)) throw new Error('EPUB ZIP size exceeds the supported range');
|
||||
const reader = new FileReader(source, original.size);
|
||||
const readerOptions = { filenameEncoding: 'utf-8' };
|
||||
const zipReader = new ZipReader(reader, readerOptions);
|
||||
const replacements = new Map<string, Entry>();
|
||||
const names = new Set<string>();
|
||||
const offsets = new Set<number>();
|
||||
try {
|
||||
for await (const entry of zipReader.getEntriesGenerator()) {
|
||||
if (names.size >= MAX_ENTRIES) throw new Error('EPUB ZIP entry count exceeds the limit');
|
||||
if (names.has(entry.filename) || offsets.has(entry.offset)) throw new Error('EPUB ZIP contains duplicate entries');
|
||||
names.add(entry.filename);
|
||||
offsets.add(entry.offset);
|
||||
if (!entry.directory) {
|
||||
// Validate header agreement and bounds without reading or inflating the entry payload.
|
||||
await entry.getData(new Uint8ArrayWriter(), {
|
||||
passThrough: true,
|
||||
checkLocalDirectory: true,
|
||||
checkLocalFilename: true,
|
||||
checkOverlappingEntryOnly: true,
|
||||
});
|
||||
}
|
||||
if (patches.has(entry.filename)) replacements.set(entry.filename, entry);
|
||||
}
|
||||
} finally {
|
||||
await zipReader.close();
|
||||
}
|
||||
const addedCount = [...patches.keys()].filter((name) => name !== 'mimetype' && !names.has(name)).length;
|
||||
if (names.size + addedCount + Number(!names.has('mimetype')) > MAX_ENTRIES) throw new Error('EPUB ZIP entry count exceeds the limit');
|
||||
|
||||
output = await open(tmpPath, 'wx');
|
||||
const writer = new ZipWriter(new FileWriter(output), { useWebWorkers: false, extendedTimestamp: false, dataDescriptor: false });
|
||||
const mimetype = Buffer.from('application/epub+zip');
|
||||
await writer.add('mimetype', new Uint8ArrayReader(mimetype), {
|
||||
passThrough: true,
|
||||
compressionMethod: 0,
|
||||
uncompressedSize: mimetype.length,
|
||||
crc32: crc32(mimetype),
|
||||
zip64: false,
|
||||
});
|
||||
// Copy local headers, compressed bytes and descriptors verbatim; only directory offsets change.
|
||||
await writer.appendZip(reader, { readerOptions, filter: (entry) => entry.filename !== 'mimetype' && !patches.has(entry.filename) });
|
||||
for (const [name, content] of patches) {
|
||||
if (name === 'mimetype') continue;
|
||||
const compressed = await compress(content, { level: 6 });
|
||||
const stored = compressed.length >= content.length;
|
||||
const previous = replacements.get(name);
|
||||
await writer.add(name, new Uint8ArrayReader(stored ? content : compressed), {
|
||||
passThrough: true,
|
||||
compressionMethod: stored ? 0 : 8,
|
||||
uncompressedSize: content.length,
|
||||
crc32: crc32(content),
|
||||
rawLastModDate: previous ? Number(previous.rawLastModDate) : undefined,
|
||||
externalFileAttributes: previous?.externalFileAttributes,
|
||||
internalFileAttributes: previous?.internalFileAttributes,
|
||||
versionMadeBy: previous?.versionMadeBy,
|
||||
comment: previous?.comment,
|
||||
});
|
||||
}
|
||||
await writer.close(zipReader.comment);
|
||||
const current = await source.stat();
|
||||
if (current.size !== original.size || current.mtimeMs !== original.mtimeMs || current.ctimeMs !== original.ctimeMs) {
|
||||
throw new Error('EPUB changed during write-back');
|
||||
}
|
||||
await output.sync();
|
||||
} finally {
|
||||
try {
|
||||
await output?.close();
|
||||
} finally {
|
||||
await source.close();
|
||||
}
|
||||
}
|
||||
completed = true;
|
||||
} finally {
|
||||
if (!completed && output) await unlink(tmpPath).catch(() => {});
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user