mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 08:21:39 -06:00
feat(metadata): add fallback scanner for EPUBs with missing ISBNs
Extract ISBNs from the EPUB spine contents if missing from the OPF metadata. Closes #350
This commit is contained in:
@@ -1,39 +1,10 @@
|
||||
import * as unzipper from 'unzipper';
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
|
||||
import { attr, findInZip, resolvePath, toRecordArray } from './epub-zip-utils';
|
||||
|
||||
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '@_', removeNSPrefix: true });
|
||||
|
||||
function attr(obj: Record<string, unknown>, key: string): string {
|
||||
const v = obj[key];
|
||||
return typeof v === 'string' ? v : '';
|
||||
}
|
||||
|
||||
function toRecordArray(val: unknown): Record<string, unknown>[] {
|
||||
if (Array.isArray(val)) return val as Record<string, unknown>[];
|
||||
if (val != null && typeof val === 'object') return [val as Record<string, unknown>];
|
||||
return [];
|
||||
}
|
||||
|
||||
function resolvePath(base: string, rel: string): string {
|
||||
const parts = (base + rel).split('/');
|
||||
const resolved: string[] = [];
|
||||
for (const part of parts) {
|
||||
if (part === '..') resolved.pop();
|
||||
else if (part !== '.') resolved.push(part);
|
||||
}
|
||||
return resolved.join('/');
|
||||
}
|
||||
|
||||
function findInZip(zip: unzipper.CentralDirectory, rawHref: string, opfDir: string) {
|
||||
const decoded = decodeURIComponent(rawHref);
|
||||
const candidates = [resolvePath(opfDir, decoded), resolvePath(opfDir, rawHref), decoded, rawHref];
|
||||
const lower = candidates.map((c) => c.toLowerCase());
|
||||
return zip.files.find((f) => {
|
||||
const p = f.path.startsWith('/') ? f.path.slice(1) : f.path;
|
||||
return candidates.includes(p) || lower.includes(p.toLowerCase());
|
||||
});
|
||||
}
|
||||
|
||||
/** Extract the first <img src> from an HTML cover page and resolve it to a zip entry. */
|
||||
async function imageFromHtmlCoverPage(zip: unzipper.CentralDirectory, htmlItem: Record<string, unknown>, opfDir: string): Promise<Buffer | null> {
|
||||
const htmlHref = attr(htmlItem, '@_href');
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
import { Readable } from 'stream';
|
||||
|
||||
import { scanEpubSpineForIsbn } from './epub-isbn-scan';
|
||||
|
||||
const ISBN_13 = '9780306406157';
|
||||
const ISBN_13_HUMAN = '978-0-306-40615-7';
|
||||
const ISBN_13_OTHER = '9781635766264';
|
||||
|
||||
interface FakeFile {
|
||||
path: string;
|
||||
uncompressedSize: number;
|
||||
buffer: () => Promise<Buffer>;
|
||||
stream: () => Readable;
|
||||
}
|
||||
|
||||
function file(path: string, content: string, sizeOverride?: number): FakeFile {
|
||||
const buf = Buffer.from(content);
|
||||
return {
|
||||
path,
|
||||
uncompressedSize: sizeOverride ?? buf.length,
|
||||
buffer: () => Promise.resolve(buf),
|
||||
stream: () => Readable.from([buf]),
|
||||
};
|
||||
}
|
||||
|
||||
function zip(files: FakeFile[]) {
|
||||
return { files } as never;
|
||||
}
|
||||
|
||||
function opf(manifestItems: string, spineRefs: string): string {
|
||||
return `<?xml version="1.0"?>
|
||||
<package xmlns="http://www.idpf.org/2007/opf">
|
||||
<manifest>${manifestItems}</manifest>
|
||||
<spine>${spineRefs}</spine>
|
||||
</package>`;
|
||||
}
|
||||
|
||||
function htmlDoc(items: { id: string; href: string }[]): string {
|
||||
return items.map((i) => `<item id="${i.id}" href="${i.href}" media-type="application/xhtml+xml"/>`).join('');
|
||||
}
|
||||
|
||||
function spine(ids: string[]): string {
|
||||
return ids.map((id) => `<itemref idref="${id}"/>`).join('');
|
||||
}
|
||||
|
||||
function copyrightHtml(body: string): string {
|
||||
return `<html><body><p>${body}</p></body></html>`;
|
||||
}
|
||||
|
||||
describe('scanEpubSpineForIsbn', () => {
|
||||
it('recovers a labeled ISBN from the copyright page', async () => {
|
||||
const manifest = htmlDoc([
|
||||
{ id: 'cover', href: 'cover.xhtml' },
|
||||
{ id: 'copy', href: 'copyright.xhtml' },
|
||||
{ id: 'ch1', href: 'ch1.xhtml' },
|
||||
]);
|
||||
const z = zip([
|
||||
file('cover.xhtml', copyrightHtml('A Novel')),
|
||||
file('copyright.xhtml', copyrightHtml(`ISBN: ${ISBN_13_HUMAN}`)),
|
||||
file('ch1.xhtml', copyrightHtml('Chapter One')),
|
||||
]);
|
||||
|
||||
const result = await scanEpubSpineForIsbn(z, opf(manifest, spine(['cover', 'copy', 'ch1'])), 'content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: ISBN_13 });
|
||||
});
|
||||
|
||||
it('prefers ISBN-13 when both forms appear on the page', async () => {
|
||||
const manifest = htmlDoc([{ id: 'copy', href: 'copyright.xhtml' }]);
|
||||
const z = zip([file('copyright.xhtml', copyrightHtml(`ISBN-13: ${ISBN_13_HUMAN} ISBN-10: 0-306-40615-2`))]);
|
||||
|
||||
const result = await scanEpubSpineForIsbn(z, opf(manifest, spine(['copy'])), 'content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: ISBN_13 });
|
||||
});
|
||||
|
||||
it('returns the front-matter ISBN first (document-level early exit)', async () => {
|
||||
const manifest = htmlDoc([
|
||||
{ id: 'copy', href: 'copyright.xhtml' },
|
||||
{ id: 'back', href: 'back.xhtml' },
|
||||
]);
|
||||
const z = zip([file('copyright.xhtml', copyrightHtml(`ISBN: ${ISBN_13_HUMAN}`)), file('back.xhtml', copyrightHtml(`ISBN: ${ISBN_13_OTHER}`))]);
|
||||
|
||||
const result = await scanEpubSpineForIsbn(z, opf(manifest, spine(['copy', 'back'])), 'content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: ISBN_13 });
|
||||
});
|
||||
|
||||
it('finds an ISBN that lives only in back matter (last-5 window)', async () => {
|
||||
const ids = Array.from({ length: 20 }, (_, i) => `d${i}`);
|
||||
const manifest = htmlDoc(ids.map((id) => ({ id, href: `${id}.xhtml` })));
|
||||
const files = ids.map((id, i) => file(`${id}.xhtml`, copyrightHtml(i === 17 ? `ISBN: ${ISBN_13_HUMAN}` : 'text')));
|
||||
|
||||
const result = await scanEpubSpineForIsbn(zip(files), opf(manifest, spine(ids)), 'content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: ISBN_13 });
|
||||
});
|
||||
|
||||
it('does not find an ISBN that lives outside the window', async () => {
|
||||
const ids = Array.from({ length: 20 }, (_, i) => `d${i}`);
|
||||
const manifest = htmlDoc(ids.map((id) => ({ id, href: `${id}.xhtml` })));
|
||||
const files = ids.map((id, i) => file(`${id}.xhtml`, copyrightHtml(i === 12 ? `ISBN: ${ISBN_13_HUMAN}` : 'text')));
|
||||
|
||||
const result = await scanEpubSpineForIsbn(zip(files), opf(manifest, spine(ids)), 'content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: null });
|
||||
});
|
||||
|
||||
it('skips non-HTML spine items', async () => {
|
||||
const manifest = `<item id="img" href="page.svg" media-type="image/svg+xml"/>`;
|
||||
const z = zip([file('page.svg', `<svg>ISBN: ${ISBN_13_HUMAN}</svg>`)]);
|
||||
|
||||
const result = await scanEpubSpineForIsbn(z, opf(manifest, spine(['img'])), 'content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: null });
|
||||
});
|
||||
|
||||
it('returns nulls for an empty spine', async () => {
|
||||
const result = await scanEpubSpineForIsbn(zip([]), opf('', ''), 'content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: null });
|
||||
});
|
||||
|
||||
it('returns nulls (and does not throw) when a document read fails', async () => {
|
||||
const manifest = htmlDoc([{ id: 'copy', href: 'copyright.xhtml' }]);
|
||||
const failing: FakeFile = {
|
||||
path: 'copyright.xhtml',
|
||||
uncompressedSize: 10,
|
||||
buffer: () => Promise.reject(new Error('boom')),
|
||||
stream: () => Readable.from([]),
|
||||
};
|
||||
|
||||
const result = await scanEpubSpineForIsbn(zip([failing]), opf(manifest, spine(['copy'])), 'content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: null });
|
||||
});
|
||||
|
||||
it('continues past a failing document to a later one', async () => {
|
||||
const manifest = htmlDoc([
|
||||
{ id: 'bad', href: 'bad.xhtml' },
|
||||
{ id: 'copy', href: 'copyright.xhtml' },
|
||||
]);
|
||||
const bad: FakeFile = {
|
||||
path: 'bad.xhtml',
|
||||
uncompressedSize: 10,
|
||||
buffer: () => Promise.reject(new Error('boom')),
|
||||
stream: () => Readable.from([]),
|
||||
};
|
||||
const z = zip([bad, file('copyright.xhtml', copyrightHtml(`ISBN: ${ISBN_13_HUMAN}`))]);
|
||||
|
||||
const result = await scanEpubSpineForIsbn(z, opf(manifest, spine(['bad', 'copy'])), 'content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: ISBN_13 });
|
||||
});
|
||||
|
||||
it('streams oversized documents and still finds a front-matter ISBN', async () => {
|
||||
const manifest = htmlDoc([{ id: 'big', href: 'big.xhtml' }]);
|
||||
const content = copyrightHtml(`ISBN: ${ISBN_13_HUMAN}`) + ' '.repeat(2_200_000);
|
||||
const big = file('big.xhtml', content, content.length);
|
||||
|
||||
const result = await scanEpubSpineForIsbn(zip([big]), opf(manifest, spine(['big'])), 'content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: ISBN_13 });
|
||||
});
|
||||
|
||||
it('resolves spine hrefs relative to a nested OPF directory', async () => {
|
||||
const manifest = htmlDoc([{ id: 'copy', href: 'text/copyright.xhtml' }]);
|
||||
const z = zip([file('OEBPS/text/copyright.xhtml', copyrightHtml(`ISBN: ${ISBN_13_HUMAN}`))]);
|
||||
|
||||
const result = await scanEpubSpineForIsbn(z, opf(manifest, spine(['copy'])), 'OEBPS/content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: ISBN_13 });
|
||||
});
|
||||
|
||||
it('returns nulls for malformed OPF XML', async () => {
|
||||
const result = await scanEpubSpineForIsbn(zip([file('a.xhtml', 'x')]), 'not really xml', 'content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: null });
|
||||
});
|
||||
|
||||
it('never throws even when the zip itself misbehaves', async () => {
|
||||
const manifest = htmlDoc([{ id: 'copy', href: 'copyright.xhtml' }]);
|
||||
const throwingZip = {
|
||||
get files() {
|
||||
throw new Error('zip exploded');
|
||||
},
|
||||
} as never;
|
||||
|
||||
const result = await scanEpubSpineForIsbn(throwingZip, opf(manifest, spine(['copy'])), 'content.opf');
|
||||
expect(result).toEqual({ isbn10: null, isbn13: null });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,106 @@
|
||||
import * as unzipper from 'unzipper';
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
|
||||
import { attr, findInZip, toRecordArray } from './epub-zip-utils';
|
||||
import { findIsbnInText, pickBestIsbn } from './isbn-detect';
|
||||
|
||||
const SPINE_WINDOW_FRONT = 10;
|
||||
const SPINE_WINDOW_BACK = 5;
|
||||
const PER_DOC_BYTE_CAP = 2 * 1024 * 1024;
|
||||
|
||||
const parser = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '@_',
|
||||
removeNSPrefix: true,
|
||||
isArray: (name) => name === 'item' || name === 'itemref',
|
||||
});
|
||||
|
||||
interface IsbnResult {
|
||||
isbn10: string | null;
|
||||
isbn13: string | null;
|
||||
}
|
||||
|
||||
const EMPTY: IsbnResult = { isbn10: null, isbn13: null };
|
||||
|
||||
function stripTags(html: string): string {
|
||||
return html
|
||||
.replace(/<script[\s\S]*?<\/script>/gi, ' ')
|
||||
.replace(/<style[\s\S]*?<\/style>/gi, ' ')
|
||||
.replace(/<[^>]+>/g, ' ');
|
||||
}
|
||||
|
||||
async function readEntryCapped(file: unzipper.File): Promise<string> {
|
||||
const size = file.uncompressedSize ?? 0;
|
||||
if (size === 0 || size <= PER_DOC_BYTE_CAP) {
|
||||
return (await file.buffer()).toString('utf-8');
|
||||
}
|
||||
return await new Promise<string>((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
let len = 0;
|
||||
const stream = file.stream();
|
||||
stream.on('data', (chunk: Buffer) => {
|
||||
chunks.push(chunk);
|
||||
len += chunk.length;
|
||||
if (len >= PER_DOC_BYTE_CAP) stream.destroy();
|
||||
});
|
||||
const finish = () => resolve(Buffer.concat(chunks).toString('utf-8'));
|
||||
stream.on('close', finish);
|
||||
stream.on('end', finish);
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function resolveContentDocs(opfXml: string, opfDir: string, zip: unzipper.CentralDirectory): unzipper.File[] {
|
||||
const opf = parser.parse(opfXml) as Record<string, unknown>;
|
||||
const pkg = opf['package'] as Record<string, unknown> | undefined;
|
||||
const manifest = pkg?.['manifest'] as Record<string, unknown> | undefined;
|
||||
const spine = pkg?.['spine'] as Record<string, unknown> | undefined;
|
||||
|
||||
const byId = new Map<string, Record<string, unknown>>();
|
||||
for (const item of toRecordArray(manifest?.['item'])) byId.set(attr(item, '@_id'), item);
|
||||
|
||||
const docs: unzipper.File[] = [];
|
||||
for (const ref of toRecordArray(spine?.['itemref'])) {
|
||||
const item = byId.get(attr(ref, '@_idref'));
|
||||
if (!item) continue;
|
||||
const mediaType = attr(item, '@_media-type').toLowerCase();
|
||||
const href = attr(item, '@_href');
|
||||
const isHtml = mediaType.includes('xhtml') || mediaType.includes('html') || /\.x?html?$/i.test(href);
|
||||
if (!isHtml || !href) continue;
|
||||
const entry = findInZip(zip, href, opfDir);
|
||||
if (entry) docs.push(entry);
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
|
||||
function windowDocs(docs: unzipper.File[]): unzipper.File[] {
|
||||
if (docs.length <= SPINE_WINDOW_FRONT + SPINE_WINDOW_BACK) return docs;
|
||||
return [...docs.slice(0, SPINE_WINDOW_FRONT), ...docs.slice(docs.length - SPINE_WINDOW_BACK)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback ISBN detection for EPUBs whose OPF metadata carries no ISBN. Scans a bounded
|
||||
* window of spine content documents (front + back matter, where the copyright/colophon
|
||||
* lives) for a labeled, checksum-valid ISBN. Reuses the already-open zip; never throws -
|
||||
* any failure yields { isbn10: null, isbn13: null } so the metadata path is unaffected.
|
||||
*/
|
||||
export async function scanEpubSpineForIsbn(zip: unzipper.CentralDirectory, opfXml: string, opfPath: string): Promise<IsbnResult> {
|
||||
try {
|
||||
const opfDir = opfPath.includes('/') ? opfPath.substring(0, opfPath.lastIndexOf('/') + 1) : '';
|
||||
const docs = windowDocs(resolveContentDocs(opfXml, opfDir, zip));
|
||||
|
||||
for (const doc of docs) {
|
||||
let text: string;
|
||||
try {
|
||||
text = stripTags(await readEntryCapped(doc));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const hits = findIsbnInText(text);
|
||||
if (hits.some((h) => h.labeled)) return pickBestIsbn(hits);
|
||||
}
|
||||
return EMPTY;
|
||||
} catch {
|
||||
return EMPTY;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { attr, findInZip, resolvePath, toRecordArray } from './epub-zip-utils';
|
||||
|
||||
describe('resolvePath', () => {
|
||||
it('joins base and relative href', () => {
|
||||
expect(resolvePath('OEBPS/', 'images/cover.jpg')).toBe('OEBPS/images/cover.jpg');
|
||||
});
|
||||
it('resolves parent segments', () => {
|
||||
expect(resolvePath('OEBPS/text/', '../images/cover.jpg')).toBe('OEBPS/images/cover.jpg');
|
||||
});
|
||||
it('drops current-dir segments', () => {
|
||||
expect(resolvePath('OEBPS/', './cover.xhtml')).toBe('OEBPS/cover.xhtml');
|
||||
});
|
||||
});
|
||||
|
||||
describe('attr', () => {
|
||||
it('returns string attribute values', () => {
|
||||
expect(attr({ '@_id': 'cover' }, '@_id')).toBe('cover');
|
||||
});
|
||||
it('returns empty string for non-string or missing values', () => {
|
||||
expect(attr({ '@_id': 5 }, '@_id')).toBe('');
|
||||
expect(attr({}, '@_id')).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('toRecordArray', () => {
|
||||
it('passes arrays through', () => {
|
||||
expect(toRecordArray([{ a: 1 }, { b: 2 }])).toEqual([{ a: 1 }, { b: 2 }]);
|
||||
});
|
||||
it('wraps a single object', () => {
|
||||
expect(toRecordArray({ a: 1 })).toEqual([{ a: 1 }]);
|
||||
});
|
||||
it('returns empty for null/undefined', () => {
|
||||
expect(toRecordArray(null)).toEqual([]);
|
||||
expect(toRecordArray(undefined)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findInZip', () => {
|
||||
it('finds an entry resolved against the OPF directory', () => {
|
||||
const zip = { files: [{ path: 'OEBPS/images/cover.jpg' }] } as never;
|
||||
expect(findInZip(zip, 'images/cover.jpg', 'OEBPS/')?.path).toBe('OEBPS/images/cover.jpg');
|
||||
});
|
||||
it('matches case-insensitively', () => {
|
||||
const zip = { files: [{ path: 'OEBPS/Cover.JPG' }] } as never;
|
||||
expect(findInZip(zip, 'cover.jpg', 'OEBPS/')?.path).toBe('OEBPS/Cover.JPG');
|
||||
});
|
||||
it('handles a leading slash in the zip path', () => {
|
||||
const zip = { files: [{ path: '/OEBPS/a.xhtml' }] } as never;
|
||||
expect(findInZip(zip, 'a.xhtml', 'OEBPS/')?.path).toBe('/OEBPS/a.xhtml');
|
||||
});
|
||||
it('decodes percent-encoded hrefs', () => {
|
||||
const zip = { files: [{ path: 'OEBPS/a b.xhtml' }] } as never;
|
||||
expect(findInZip(zip, 'a%20b.xhtml', 'OEBPS/')?.path).toBe('OEBPS/a b.xhtml');
|
||||
});
|
||||
it('returns undefined when nothing matches', () => {
|
||||
const zip = { files: [{ path: 'OEBPS/cover.jpg' }] } as never;
|
||||
expect(findInZip(zip, 'missing.jpg', 'OEBPS/')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import * as unzipper from 'unzipper';
|
||||
|
||||
export function attr(obj: Record<string, unknown>, key: string): string {
|
||||
const v = obj[key];
|
||||
return typeof v === 'string' ? v : '';
|
||||
}
|
||||
|
||||
export function toRecordArray(val: unknown): Record<string, unknown>[] {
|
||||
if (Array.isArray(val)) return val as Record<string, unknown>[];
|
||||
if (val != null && typeof val === 'object') return [val as Record<string, unknown>];
|
||||
return [];
|
||||
}
|
||||
|
||||
export function resolvePath(base: string, rel: string): string {
|
||||
const parts = (base + rel).split('/');
|
||||
const resolved: string[] = [];
|
||||
for (const part of parts) {
|
||||
if (part === '..') resolved.pop();
|
||||
else if (part !== '.') resolved.push(part);
|
||||
}
|
||||
return resolved.join('/');
|
||||
}
|
||||
|
||||
/** Case-insensitive lookup of a manifest href inside an EPUB zip, resolved against the OPF directory. */
|
||||
export function findInZip(zip: unzipper.CentralDirectory, rawHref: string, opfDir: string): unzipper.File | undefined {
|
||||
const decoded = decodeURIComponent(rawHref);
|
||||
const candidates = [resolvePath(opfDir, decoded), resolvePath(opfDir, rawHref), decoded, rawHref];
|
||||
const lower = candidates.map((c) => c.toLowerCase());
|
||||
return zip.files.find((f) => {
|
||||
const p = f.path.startsWith('/') ? f.path.slice(1) : f.path;
|
||||
return candidates.includes(p) || lower.includes(p.toLowerCase());
|
||||
});
|
||||
}
|
||||
@@ -81,4 +81,64 @@ describe('extractEpubMetadata', () => {
|
||||
|
||||
await expect(extractEpubMetadata('/broken.epub')).resolves.toBeNull();
|
||||
});
|
||||
|
||||
it('fills the ISBN from a content scan when the OPF has none', async () => {
|
||||
const containerXml = `<container><rootfiles><rootfile full-path="content.opf" /></rootfiles></container>`;
|
||||
const opfXml = `<package xmlns="http://www.idpf.org/2007/opf"><manifest><item id="c" href="copyright.xhtml" media-type="application/xhtml+xml"/></manifest><spine><itemref idref="c"/></spine></package>`;
|
||||
|
||||
mockOpenFile.mockResolvedValue({
|
||||
files: [
|
||||
zipFile('META-INF/container.xml', containerXml),
|
||||
zipFile('content.opf', opfXml),
|
||||
zipFile('copyright.xhtml', '<html><body><p>ISBN: 978-0-306-40615-7</p></body></html>'),
|
||||
],
|
||||
});
|
||||
|
||||
mockParseOpf.mockReturnValue({
|
||||
title: 'No Metadata ISBN',
|
||||
subtitle: null,
|
||||
description: null,
|
||||
isbn10: null,
|
||||
isbn13: null,
|
||||
publisher: null,
|
||||
publishedYear: null,
|
||||
language: null,
|
||||
seriesName: null,
|
||||
seriesIndex: null,
|
||||
authors: [],
|
||||
tags: [],
|
||||
});
|
||||
|
||||
await expect(extractEpubMetadata('/books/x.epub')).resolves.toEqual(expect.objectContaining({ isbn10: null, isbn13: '9780306406157' }));
|
||||
});
|
||||
|
||||
it('keeps the OPF ISBN and does not override it from content', async () => {
|
||||
const containerXml = `<container><rootfiles><rootfile full-path="content.opf" /></rootfiles></container>`;
|
||||
const opfXml = `<package xmlns="http://www.idpf.org/2007/opf"><manifest><item id="c" href="copyright.xhtml" media-type="application/xhtml+xml"/></manifest><spine><itemref idref="c"/></spine></package>`;
|
||||
|
||||
mockOpenFile.mockResolvedValue({
|
||||
files: [
|
||||
zipFile('META-INF/container.xml', containerXml),
|
||||
zipFile('content.opf', opfXml),
|
||||
zipFile('copyright.xhtml', '<html><body><p>ISBN: 978-1-63576-626-4</p></body></html>'),
|
||||
],
|
||||
});
|
||||
|
||||
mockParseOpf.mockReturnValue({
|
||||
title: 'Has Metadata ISBN',
|
||||
subtitle: null,
|
||||
description: null,
|
||||
isbn10: '0306406152',
|
||||
isbn13: null,
|
||||
publisher: null,
|
||||
publishedYear: null,
|
||||
language: null,
|
||||
seriesName: null,
|
||||
seriesIndex: null,
|
||||
authors: [],
|
||||
tags: [],
|
||||
});
|
||||
|
||||
await expect(extractEpubMetadata('/books/y.epub')).resolves.toEqual(expect.objectContaining({ isbn10: '0306406152', isbn13: null }));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import * as unzipper from 'unzipper';
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
|
||||
import { scanEpubSpineForIsbn } from './epub-isbn-scan';
|
||||
import { ParsedOpf, parseOpf } from './opf-parser';
|
||||
|
||||
const logger = new Logger('EpubMetadata');
|
||||
|
||||
const containerParser = new XMLParser({
|
||||
ignoreAttributes: false,
|
||||
attributeNamePrefix: '@_',
|
||||
@@ -40,7 +44,18 @@ export async function extractEpubMetadata(absolutePath: string): Promise<ParsedO
|
||||
const zip = await unzipper.Open.file(absolutePath);
|
||||
const opfPath = await findOpfPath(zip);
|
||||
const opfXml = await readFileFromZip(zip, opfPath);
|
||||
return parseOpf(opfXml);
|
||||
const opf = parseOpf(opfXml);
|
||||
|
||||
if (opf.isbn10 == null && opf.isbn13 == null) {
|
||||
const found = await scanEpubSpineForIsbn(zip, opfXml, opfPath);
|
||||
if (found.isbn10 || found.isbn13) {
|
||||
opf.isbn10 = found.isbn10;
|
||||
opf.isbn13 = found.isbn13;
|
||||
logger.debug(`[epub.isbn_fallback] isbn13=${found.isbn13 ?? ''} isbn10=${found.isbn10 ?? ''} - recovered isbn from content`);
|
||||
}
|
||||
}
|
||||
|
||||
return opf;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import { findIsbnInText, isValidIsbn10, isValidIsbn13, normalizeIsbn, pickBestIsbn } from './isbn-detect';
|
||||
|
||||
describe('normalizeIsbn', () => {
|
||||
it('strips separators and uppercases X', () => {
|
||||
expect(normalizeIsbn('978-0-306-40615-7')).toBe('9780306406157');
|
||||
expect(normalizeIsbn('0 306 40615 2')).toBe('0306406152');
|
||||
expect(normalizeIsbn('097522980x')).toBe('097522980X');
|
||||
});
|
||||
});
|
||||
|
||||
describe('isValidIsbn10', () => {
|
||||
it('accepts valid ISBN-10 incl. X check digit', () => {
|
||||
expect(isValidIsbn10('0306406152')).toBe(true);
|
||||
expect(isValidIsbn10('097522980X')).toBe(true);
|
||||
});
|
||||
it('rejects bad checksum, wrong length, lowercase x', () => {
|
||||
expect(isValidIsbn10('0306406153')).toBe(false);
|
||||
expect(isValidIsbn10('1234567890')).toBe(false);
|
||||
expect(isValidIsbn10('030640615')).toBe(false);
|
||||
expect(isValidIsbn10('097522980x')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isValidIsbn13', () => {
|
||||
it('accepts valid 978/979 ISBN-13', () => {
|
||||
expect(isValidIsbn13('9780306406157')).toBe(true);
|
||||
expect(isValidIsbn13('9781635766264')).toBe(true);
|
||||
});
|
||||
it('rejects bad checksum and non-978/979 prefix', () => {
|
||||
expect(isValidIsbn13('9780306406158')).toBe(false);
|
||||
expect(isValidIsbn13('1230306406157')).toBe(false);
|
||||
expect(isValidIsbn13('978030640615')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findIsbnInText', () => {
|
||||
it('finds a labeled ISBN-13', () => {
|
||||
const hits = findIsbnInText('Copyright page. ISBN: 978-0-306-40615-7. All rights reserved.');
|
||||
expect(hits).toEqual([{ value: '9780306406157', kind: 13, labeled: true }]);
|
||||
});
|
||||
|
||||
it('marks an unlabeled ISBN as not labeled', () => {
|
||||
const hits = findIsbnInText('Some long preamble text here 9780306406157 trailing');
|
||||
expect(hits).toEqual([{ value: '9780306406157', kind: 13, labeled: false }]);
|
||||
});
|
||||
|
||||
it('finds a labeled ISBN-10', () => {
|
||||
const hits = findIsbnInText('ISBN-10: 0-306-40615-2');
|
||||
expect(hits).toEqual([{ value: '0306406152', kind: 10, labeled: true }]);
|
||||
});
|
||||
|
||||
it('rejects phone numbers, dates and bad checksums', () => {
|
||||
expect(findIsbnInText('Call 1-800-555-0199 today')).toEqual([]);
|
||||
expect(findIsbnInText('Published 2020-01-01')).toEqual([]);
|
||||
expect(findIsbnInText('ISBN 978-0-306-40615-8')).toEqual([]);
|
||||
});
|
||||
|
||||
it('de-duplicates by value and upgrades to labeled', () => {
|
||||
const hits = findIsbnInText('9780306406157\nfoo bar baz\nISBN: 9780306406157');
|
||||
expect(hits).toEqual([{ value: '9780306406157', kind: 13, labeled: true }]);
|
||||
});
|
||||
|
||||
it('respects the label lookback window', () => {
|
||||
const far = findIsbnInText('ISBN is the worldwide standard, but this number 9780306406157');
|
||||
expect(far[0].labeled).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('pickBestIsbn', () => {
|
||||
it('prefers a labeled ISBN-13 over a labeled ISBN-10', () => {
|
||||
const result = pickBestIsbn([
|
||||
{ value: '0306406152', kind: 10, labeled: true },
|
||||
{ value: '9780306406157', kind: 13, labeled: true },
|
||||
]);
|
||||
expect(result).toEqual({ isbn10: null, isbn13: '9780306406157' });
|
||||
});
|
||||
|
||||
it('falls back to a labeled ISBN-10', () => {
|
||||
expect(pickBestIsbn([{ value: '0306406152', kind: 10, labeled: true }])).toEqual({ isbn10: '0306406152', isbn13: null });
|
||||
});
|
||||
|
||||
it('ignores unlabeled hits', () => {
|
||||
expect(pickBestIsbn([{ value: '9780306406157', kind: 13, labeled: false }])).toEqual({ isbn10: null, isbn13: null });
|
||||
});
|
||||
|
||||
it('returns nulls for an empty set', () => {
|
||||
expect(pickBestIsbn([])).toEqual({ isbn10: null, isbn13: null });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
export interface IsbnHit {
|
||||
value: string;
|
||||
kind: 10 | 13;
|
||||
labeled: boolean;
|
||||
}
|
||||
|
||||
const LABEL_LOOKBACK = 25;
|
||||
|
||||
export function normalizeIsbn(raw: string): string {
|
||||
return raw.replace(/[^0-9Xx]/g, '').toUpperCase();
|
||||
}
|
||||
|
||||
export function isValidIsbn10(n: string): boolean {
|
||||
if (!/^[0-9]{9}[0-9X]$/.test(n)) return false;
|
||||
let sum = 0;
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const c = n[i];
|
||||
const v = c === 'X' ? 10 : c.charCodeAt(0) - 48;
|
||||
sum += (i + 1) * v;
|
||||
}
|
||||
return sum % 11 === 0;
|
||||
}
|
||||
|
||||
export function isValidIsbn13(n: string): boolean {
|
||||
if (!/^97[89][0-9]{10}$/.test(n)) return false;
|
||||
let sum = 0;
|
||||
for (let i = 0; i < 13; i++) {
|
||||
const v = n.charCodeAt(i) - 48;
|
||||
sum += i % 2 === 0 ? v : v * 3;
|
||||
}
|
||||
return sum % 10 === 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find checksum-valid ISBN-10/13 strings inside free text. Each hit records whether
|
||||
* an "ISBN" label sits just before it. De-duplicates by value, upgrading to labeled
|
||||
* if any occurrence of the same value was labeled.
|
||||
*/
|
||||
export function findIsbnInText(text: string): IsbnHit[] {
|
||||
const candidate = /[0-9][0-9\s-]{7,16}[0-9Xx]/g;
|
||||
const byValue = new Map<string, IsbnHit>();
|
||||
|
||||
for (const match of text.matchAll(candidate)) {
|
||||
const norm = normalizeIsbn(match[0]);
|
||||
let kind: 10 | 13 | null = null;
|
||||
if (norm.length === 13 && isValidIsbn13(norm)) kind = 13;
|
||||
else if (norm.length === 10 && isValidIsbn10(norm)) kind = 10;
|
||||
if (kind === null) continue;
|
||||
|
||||
const idx = match.index ?? 0;
|
||||
const before = text.slice(Math.max(0, idx - LABEL_LOOKBACK), idx).toLowerCase();
|
||||
const labeled = before.includes('isbn');
|
||||
|
||||
const existing = byValue.get(norm);
|
||||
if (!existing) byValue.set(norm, { value: norm, kind, labeled });
|
||||
else if (labeled && !existing.labeled) existing.labeled = true;
|
||||
}
|
||||
|
||||
return [...byValue.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the most trustworthy ISBN from a set of hits. Only labeled hits are eligible
|
||||
* (precision); prefers ISBN-13 over ISBN-10. Returns a single form, mirroring the
|
||||
* OPF parser's parseIsbn (no ISBN-10 -> 13 conversion).
|
||||
*/
|
||||
export function pickBestIsbn(hits: IsbnHit[]): { isbn10: string | null; isbn13: string | null } {
|
||||
const labeled = hits.filter((h) => h.labeled);
|
||||
const best13 = labeled.find((h) => h.kind === 13);
|
||||
if (best13) return { isbn10: null, isbn13: best13.value };
|
||||
const best10 = labeled.find((h) => h.kind === 10);
|
||||
if (best10) return { isbn10: best10.value, isbn13: null };
|
||||
return { isbn10: null, isbn13: null };
|
||||
}
|
||||
Reference in New Issue
Block a user