Migrations/bookorbit
1
0
Fork 0
mirror of https://github.com/bookorbit/bookorbit.git synced 2026-11-03 00:11:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(kobo): sync normalized language and ISBN metadata

Browse Source 
Normalize heterogeneous language metadata at the Kobo boundary and include validated ISBN values so device details remain accurate. Version metadata hashes to refresh previously synchronized books.

Fixes #920
This commit is contained in:
neon
2026-08-08 04:58:52 +00:00
parent 27040d5dee
commit 494b59562f
6 changed files with 221 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/package.json
+2
View File
@@ -83,6 +83,7 @@
"fastify": "^5.10.0",
"iconv-lite": "^0.7.3",
"jose": "^6.2.4",
"langs": "^2.0.0",
"ms": "^2.1.3",
"mysql2": "^3.23.2",
"nestjs-pino": "^4.6.1",
@@ -113,6 +114,7 @@
"@swc/core": "^1.15.46",
"@types/bcrypt": "^6.0.0",
"@types/express": "^5.0.6",
"@types/langs": "^2.0.5",
"@types/node": "^26.1.1",
"@types/nodemailer": "^8.0.1",
"@types/passport-jwt": "^4.0.1",
server/src/modules/kobo/services/kobo-metadata.utils.test.ts
+44
View File
@@ -0,0 +1,44 @@
import { describe, expect, it } from 'vitest';
import { normalizeKoboLanguage, selectKoboIsbn } from './kobo-metadata.utils';
describe('normalizeKoboLanguage', () => {
it.each([
['English', 'en'],
['english', 'en'],
[' en ', 'en'],
['eng', 'en'],
['en-US', 'en'],
['EN_us', 'en'],
['Spanish; Castilian', 'es'],
['Espa\u00f1ol', 'es'],
['deu', 'de'],
['ger', 'de'],
['zh-Hant', 'zh'],
])('normalizes %j to %s', (value, expected) => {
expect(normalizeKoboLanguage(value)).toBe(expected);
});
it.each([null, '', ' ', 'Custom language'])('falls back to English for unsupported value %j', (value) => {
expect(normalizeKoboLanguage(value)).toBe('en');
});
});
describe('selectKoboIsbn', () => {
it('prefers a valid normalized ISBN-13 over ISBN-10', () => {
expect(selectKoboIsbn('978-0-306-40615-7', '0-306-40615-2')).toBe('9780306406157');
});
it('falls back to a valid ISBN-10, including an X check digit', () => {
expect(selectKoboIsbn(null, '0-9752298-0-X')).toBe('097522980X');
expect(selectKoboIsbn('9780306406158', '0306406152')).toBe('0306406152');
});
it.each([
[null, null],
['', ''],
['9780306406158', null],
[null, '0306406153'],
])('returns null when neither ISBN is valid', (isbn13, isbn10) => {
expect(selectKoboIsbn(isbn13, isbn10)).toBeNull();
});
});
server/src/modules/kobo/services/kobo-metadata.utils.ts
+64
View File
@@ -0,0 +1,64 @@
import langs from 'langs';
const DEFAULT_KOBO_LANGUAGE = 'en';
const iso6391ByLanguage = new Map<string, string>();
for (const language of langs.all()) {
const iso6391 = language['1'];
if (!iso6391) continue;
for (const value of [language['1'], language['2'], language['2B'], language['2T'], language['3'], language.name, language.local]) {
if (value) iso6391ByLanguage.set(value.trim().toLowerCase(), iso6391);
}
}
export function normalizeKoboLanguage(value: string | null): string {
const normalized = value?.trim().toLowerCase();
if (!normalized) return DEFAULT_KOBO_LANGUAGE;
const directMatch = iso6391ByLanguage.get(normalized);
if (directMatch) return directMatch;
for (const alias of normalized.split(';')) {
const aliasMatch = iso6391ByLanguage.get(alias.trim());
if (aliasMatch) return aliasMatch;
}
const primarySubtag = normalized.split(/[-_]/, 1)[0];
return iso6391ByLanguage.get(primarySubtag) ?? DEFAULT_KOBO_LANGUAGE;
}
function normalizeIsbn(value: string | null): string | null {
if (!value) return null;
const normalized = value.replace(/[^0-9Xx]/g, '').toUpperCase();
return normalized || null;
}
function isValidIsbn10(value: string): boolean {
if (!/^\d{9}[\dX]$/.test(value)) return false;
let sum = 0;
for (let index = 0; index < value.length; index += 1) {
const digit = index === 9 && value[index] === 'X' ? 10 : Number(value[index]);
sum += digit * (10 - index);
}
return sum % 11 === 0;
}
function isValidIsbn13(value: string): boolean {
if (!/^97[89]\d{10}$/.test(value)) return false;
let sum = 0;
for (let index = 0; index < value.length; index += 1) {
sum += Number(value[index]) * (index % 2 === 0 ? 1 : 3);
}
return sum % 10 === 0;
}
export function selectKoboIsbn(isbn13: string | null, isbn10: string | null): string | null {
const normalized13 = normalizeIsbn(isbn13);
if (normalized13 && isValidIsbn13(normalized13)) return normalized13;
const normalized10 = normalizeIsbn(isbn10);
return normalized10 && isValidIsbn10(normalized10) ? normalized10 : null;
}
server/src/modules/kobo/services/kobo-sync.service.test.ts
+62 -54
View File
@@ -1,3 +1,4 @@
import { createHash } from 'crypto';
import { sql } from 'drizzle-orm';
import * as schema from '../../../db/schema';
import { KoboSyncService } from './kobo-sync.service';
@@ -123,8 +124,10 @@ function makeBook(id: number, format = 'epub') {
authors: ['Author One'],
description: 'Description',
publisher: 'Publisher',
publishedDate: null,
publishedYear: 2022,
language: 'en',
isbn: '9780306406157',
seriesName: 'Series',
seriesIndex: 2,
fileFormat: format,
@@ -271,6 +274,8 @@ describe('KoboSyncService', () => {
expect(fetchSpy).toHaveBeenCalledWith(3, [12], true, expect.any(Map));
expect(metadata.Title).toBe('Book 12');
expect(metadata.Language).toBe('en');
expect(metadata.ISBN).toBe('9780306406157');
expect(metadata.DownloadUrls).toEqual([
{
Format: 'PDF',
@@ -282,6 +287,14 @@ describe('KoboSyncService', () => {
]);
});
it('omits ISBN from Kobo metadata when the book has no valid ISBN', () => {
const service = makeService(makeDb());
const metadata = (service as any).buildBookMetadata({ ...makeBook(12), isbn: null }, 'tok', 'https://base');
expect(metadata).not.toHaveProperty('ISBN');
});
it('removeBookFromSync handles missing snapshot/row and delete-vs-mark paths', async () => {
const db = makeDb();
const service = makeService(db);
@@ -955,60 +968,44 @@ describe('KoboSyncService', () => {
});
});
it('buildMetadataHash is deterministic and changes when metadata inputs change', () => {
it('buildMetadataHash covers normalized language, ISBN, identity, cover, and serializer changes', () => {
const service = makeService(makeDb());
const params = {
title: 'Dune',
authors: ['Frank Herbert'],
seriesName: 'Dune',
seriesIndex: 1,
metadataUpdatedAt: new Date('2026-01-01T00:00:00.000Z'),
entitlementId: 'entitlement-1',
coverImageId: 'cover-1',
language: 'en',
isbn: '9780306406157',
};
const hashA = (service as any).buildMetadataHash({
title: 'Dune',
authors: ['Frank Herbert'],
seriesName: 'Dune',
seriesIndex: 1,
metadataUpdatedAt: new Date('2026-01-01T00:00:00.000Z'),
entitlementId: 'entitlement-1',
coverImageId: 'cover-1',
});
const hashB = (service as any).buildMetadataHash({
title: 'Dune',
authors: ['Frank Herbert'],
seriesName: 'Dune',
seriesIndex: 1,
metadataUpdatedAt: new Date('2026-01-01T00:00:00.000Z'),
entitlementId: 'entitlement-1',
coverImageId: 'cover-1',
});
const hashC = (service as any).buildMetadataHash({
title: 'Dune Messiah',
authors: ['Frank Herbert'],
seriesName: 'Dune',
seriesIndex: 2,
metadataUpdatedAt: new Date('2026-01-01T00:00:00.000Z'),
entitlementId: 'entitlement-1',
coverImageId: 'cover-1',
});
const hashE = (service as any).buildMetadataHash({
title: 'Dune',
authors: ['Frank Herbert'],
seriesName: 'Dune',
seriesIndex: 1,
metadataUpdatedAt: new Date('2026-01-01T00:00:00.000Z'),
entitlementId: 'entitlement-2',
coverImageId: 'cover-1',
});
const hashF = (service as any).buildMetadataHash({
title: 'Dune',
authors: ['Frank Herbert'],
seriesName: 'Dune',
seriesIndex: 1,
metadataUpdatedAt: new Date('2026-01-01T00:00:00.000Z'),
entitlementId: 'entitlement-1',
coverImageId: 'cover-2',
});
const hash = (service as any).buildMetadataHash(params);
const legacyHash = createHash('sha256')
.update(
[
params.title,
params.authors.join(','),
params.seriesName,
String(params.seriesIndex),
String(params.metadataUpdatedAt),
params.entitlementId,
params.coverImageId,
].join('|'),
)
.digest('hex')
.slice(0, 16);
expect(hashA).toBe(hashB);
expect(hashA).not.toBe(hashC);
expect(hashA).not.toBe(hashE);
expect(hashA).not.toBe(hashF);
expect(hashA).toHaveLength(16);
expect((service as any).buildMetadataHash({ ...params })).toBe(hash);
expect((service as any).buildMetadataHash({ ...params, title: 'Dune Messiah' })).not.toBe(hash);
expect((service as any).buildMetadataHash({ ...params, entitlementId: 'entitlement-2' })).not.toBe(hash);
expect((service as any).buildMetadataHash({ ...params, coverImageId: 'cover-2' })).not.toBe(hash);
expect((service as any).buildMetadataHash({ ...params, language: 'de' })).not.toBe(hash);
expect((service as any).buildMetadataHash({ ...params, isbn: '9780441172719' })).not.toBe(hash);
expect(hash).not.toBe(legacyHash);
expect(hash).toHaveLength(16);
});
it('maps settings and file metadata into the actual Kobo delivery format', () => {
@@ -1064,6 +1061,9 @@ describe('KoboSyncService', () => {
{
bookId: 5,
title: 'Dune',
isbn10: '0306406152',
isbn13: '9780306406157',
language: 'English',
seriesName: 'Saga',
seriesIndex: 2,
metadataUpdatedAt: new Date('2026-01-01T00:00:00.000Z'),
@@ -1080,7 +1080,9 @@ describe('KoboSyncService', () => {
description: 'Desc',
publisher: 'Pub',
publishedYear: 1965,
language: 'en',
language: 'English',
isbn10: '0306406152',
isbn13: '9780306406157',
seriesName: 'Saga',
seriesIndex: 2,
fileFormat: 'epub',
@@ -1091,7 +1093,10 @@ describe('KoboSyncService', () => {
updatedAt: new Date('2026-01-02T00:00:00.000Z'),
},
],
[{ bookId: 5, name: 'Author A' }],
[
{ bookId: 5, name: 'Author A' },
{ bookId: 5, name: 'Author B' },
],
[{ bookId: 5, name: 'Collection A' }],
],
});
@@ -1113,10 +1118,13 @@ describe('KoboSyncService', () => {
expect(books.get(5)).toEqual(
expect.objectContaining({
title: 'Dune',
authors: ['Author A'],
authors: ['Author A', 'Author B'],
collectionNames: ['Collection A'],
language: 'en',
isbn: '9780306406157',
metadataHash: expect.any(String),
}),
);
expect(books.get(5)?.metadataHash).toBe(snapshotRows[0].metadataHash);
});
});
server/src/modules/kobo/services/kobo-sync.service.ts
+27 -3
View File
@@ -12,6 +12,7 @@ import { BookQueryBuilder } from '../../book/book-query-builder.service';
import { SmartScopeService } from '../../smart-scope/smart-scope.service';
import { KoboBookAccessService } from './kobo-book-access.service';
import { KoboBookIdentityService } from './kobo-book-identity.service';
import { normalizeKoboLanguage, selectKoboIsbn } from './kobo-metadata.utils';
import { KoboReadingStateService } from './kobo-reading-state.service';
type Db = NodePgDatabase<typeof schema>;
@@ -20,6 +21,7 @@ const SYNC_PAGE_SIZE = 5;
const SNAPSHOT_RECONCILE_BATCH_SIZE = 5000;
const SNAPSHOT_CREATE_BATCH_SIZE = 5000;
const TOKEN_PREFIX = 'PX.';
const METADATA_SERIALIZER_VERSION = 2;
type EligibleSnapshotRow = {
bookId: number;
@@ -75,7 +77,8 @@ export interface KoboBookEntry {
publisher: string | null;
publishedDate: string | null;
publishedYear: number | null;
language: string | null;
language: string;
isbn: string | null;
seriesName: string | null;
seriesIndex: number | null;
fileFormat: string;
@@ -800,7 +803,8 @@ export class KoboSyncService {
Description: book.description,
Publisher: publisher,
PublicationDate: publicationDate,
Language: book.language ?? 'en',
Language: book.language,
...(book.isbn ? { ISBN: book.isbn } : {}),
Genre: '00000000-0000-0000-0000-000000000001',
CoverImageId: book.koboCoverImageId,
Contributors: book.authors,
@@ -841,8 +845,11 @@ export class KoboSyncService {
metadataUpdatedAt: Date | null;
entitlementId: string;
coverImageId: string;
language: string;
isbn: string | null;
}): string {
const metaStr = [
METADATA_SERIALIZER_VERSION,
params.title ?? '',
params.authors.join(','),
params.seriesName ?? '',
@@ -850,6 +857,8 @@ export class KoboSyncService {
String(params.metadataUpdatedAt ?? ''),
params.entitlementId,
params.coverImageId,
params.language,
params.isbn ?? '',
].join('|');
return createHash('sha256').update(metaStr).digest('hex').slice(0, 16);
}
@@ -962,6 +971,9 @@ export class KoboSyncService {
.select({
bookId: schema.books.id,
title: schema.bookMetadata.title,
isbn10: schema.bookMetadata.isbn10,
isbn13: schema.bookMetadata.isbn13,
language: schema.bookMetadata.language,
seriesName: schema.bookMetadata.seriesName,
seriesIndex: schema.bookMetadata.seriesIndex,
metadataUpdatedAt: schema.bookMetadata.updatedAt,
@@ -979,6 +991,9 @@ export class KoboSyncService {
.groupBy(
schema.books.id,
schema.bookMetadata.title,
schema.bookMetadata.isbn10,
schema.bookMetadata.isbn13,
schema.bookMetadata.language,
schema.bookMetadata.seriesName,
schema.bookMetadata.seriesIndex,
schema.bookMetadata.updatedAt,
@@ -1011,6 +1026,8 @@ export class KoboSyncService {
metadataUpdatedAt: row.metadataUpdatedAt,
entitlementId: identity?.entitlementId ?? String(row.bookId),
coverImageId,
language: normalizeKoboLanguage(row.language),
isbn: selectKoboIsbn(row.isbn13, row.isbn10),
}),
needsLegacyNumericRemoval: identity?.needsLegacyNumericRemoval ?? false,
};
@@ -1039,6 +1056,8 @@ export class KoboSyncService {
publishedDate: schema.bookMetadata.publishedDate,
publishedYear: schema.bookMetadata.publishedYear,
language: schema.bookMetadata.language,
isbn10: schema.bookMetadata.isbn10,
isbn13: schema.bookMetadata.isbn13,
seriesName: schema.bookMetadata.seriesName,
seriesIndex: schema.bookMetadata.seriesIndex,
fileFormat: schema.bookFiles.format,
@@ -1099,6 +1118,8 @@ export class KoboSyncService {
const identity = identitiesById.get(row.bookId);
if (!identity) continue;
const coverImageId = this.bookIdentityService.buildVersionedCoverImageId(identity.coverImageId, row.metadataUpdatedAt);
const language = normalizeKoboLanguage(row.language);
const isbn = selectKoboIsbn(row.isbn13, row.isbn10);
byId.set(row.bookId, {
bookId: row.bookId,
koboEntitlementId: identity.entitlementId,
@@ -1109,7 +1130,8 @@ export class KoboSyncService {
publisher: row.publisher,
publishedDate: row.publishedDate,
publishedYear: row.publishedYear,
language: row.language,
language,
isbn,
seriesName: row.seriesName,
seriesIndex: row.seriesIndex,
fileFormat: row.fileFormat ?? 'epub',
@@ -1124,6 +1146,8 @@ export class KoboSyncService {
metadataUpdatedAt: row.metadataUpdatedAt,
entitlementId: identity.entitlementId,
coverImageId,
language,
isbn,
}),
metadataUpdatedAt: row.metadataUpdatedAt,
collectionNames: collectionsByBook.get(row.bookId) ?? [],