fix(kobo): sync Kobo bookmarks as bookmarks and keep annotation sync converging

A Kobo dogear was stored as an empty yellow highlight. Land it as a
BookOrbit bookmark linked to the device's id, serve it back as a dogear,
sync deletions both ways, and move dogears already stored as highlights.
Report only books on that Kobo from checkforchanges: a re-registered
device was told about books it never had, so real changes were fetched
outside its sync and never applied. Serve a known annotation at its last
location instead of failing the whole pull when this copy cannot place it.

Fixes #1374
This commit is contained in:
neon
2026-10-17 02:42:42 +00:00
parent 5397825003
commit 058cd6e7e4
16 changed files with 24703 additions and 18 deletions
@@ -0,0 +1,21 @@
CREATE TABLE "kobo_bookmark_links" (
"id" serial PRIMARY KEY NOT NULL,
"bookmark_id" integer NOT NULL,
"user_id" integer NOT NULL,
"device_id" integer NOT NULL,
"book_id" integer NOT NULL,
"external_key" varchar(64) NOT NULL,
"location" jsonb NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL,
"updated_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
ALTER TABLE "bookmarks" DROP CONSTRAINT "bookmarks_origin_chk";--> statement-breakpoint
ALTER TABLE "kobo_bookmark_links" ADD CONSTRAINT "kobo_bookmark_links_bookmark_id_bookmarks_id_fk" FOREIGN KEY ("bookmark_id") REFERENCES "public"."bookmarks"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "kobo_bookmark_links" ADD CONSTRAINT "kobo_bookmark_links_user_id_users_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."users"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "kobo_bookmark_links" ADD CONSTRAINT "kobo_bookmark_links_book_id_books_id_fk" FOREIGN KEY ("book_id") REFERENCES "public"."books"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "kobo_bookmark_links" ADD CONSTRAINT "kobo_bookmark_links_owner_fk" FOREIGN KEY ("device_id","user_id") REFERENCES "public"."kobo_devices"("id","user_id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
CREATE UNIQUE INDEX "kobo_bookmark_links_bookmark_device_uidx" ON "kobo_bookmark_links" USING btree ("bookmark_id","device_id");--> statement-breakpoint
CREATE UNIQUE INDEX "kobo_bookmark_links_device_key_uidx" ON "kobo_bookmark_links" USING btree ("device_id","external_key");--> statement-breakpoint
CREATE INDEX "kobo_bookmark_links_user_device_book_idx" ON "kobo_bookmark_links" USING btree ("user_id","device_id","book_id");--> statement-breakpoint
ALTER TABLE "bookmarks" ADD CONSTRAINT "bookmarks_origin_chk" CHECK ("bookmarks"."origin" in ('web', 'koreader', 'kobo'));
File diff suppressed because it is too large Load Diff
@@ -827,6 +827,13 @@
"when": 1792191772317,
"tag": "0117_add_reading_session_sync_interval",
"breakpoints": true
},
{
"idx": 118,
"version": "7",
"when": 1792201109071,
"tag": "0118_add_kobo_bookmark_links",
"breakpoints": true
}
]
}
+44
View File
@@ -19,6 +19,7 @@ import {
import { books, bookFiles } from './books';
import { users } from './auth';
import { bookmarks } from './reader';
export const koboDevices = pgTable(
'kobo_devices',
@@ -317,3 +318,46 @@ export const koboDeviceReadingStates = pgTable(
}).onDelete('cascade'),
],
);
/**
* One row per (bookmark, Kobo device): a bookmark the device holds as a dogear, under the
* annotation id it gave it, with the location it reported. The device reconciles its annotations
* against the set it is served, so this keeps the dogear served back as a dogear, and lets a
* bookmark deleted in BookOrbit leave the device by being left out.
*/
export const koboBookmarkLinks = pgTable(
'kobo_bookmark_links',
{
id: serial('id').primaryKey(),
bookmarkId: integer('bookmark_id')
.notNull()
.references(() => bookmarks.id, { onDelete: 'cascade' }),
userId: integer('user_id')
.notNull()
.references(() => users.id, { onDelete: 'cascade' }),
deviceId: integer('device_id').notNull(),
bookId: integer('book_id')
.notNull()
.references(() => books.id, { onDelete: 'cascade' }),
externalKey: varchar('external_key', { length: 64 }).notNull(),
location: jsonb('location').notNull(),
createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(),
updatedAt: timestamp('updated_at', { withTimezone: true })
.notNull()
.defaultNow()
.$onUpdateFn(() => new Date()),
},
(table) => [
foreignKey({
columns: [table.deviceId, table.userId],
foreignColumns: [koboDevices.id, koboDevices.userId],
name: 'kobo_bookmark_links_owner_fk',
}).onDelete('cascade'),
uniqueIndex('kobo_bookmark_links_bookmark_device_uidx').on(table.bookmarkId, table.deviceId),
uniqueIndex('kobo_bookmark_links_device_key_uidx').on(table.deviceId, table.externalKey),
index('kobo_bookmark_links_user_device_book_idx').on(table.userId, table.deviceId, table.bookId),
],
);
export type KoboBookmarkLink = typeof koboBookmarkLinks.$inferSelect;
export type NewKoboBookmarkLink = typeof koboBookmarkLinks.$inferInsert;
+2 -2
View File
@@ -426,7 +426,7 @@ export const audiobookProgress = pgTable(
export type AudiobookProgress = typeof audiobookProgress.$inferSelect;
export type NewAudiobookProgress = typeof audiobookProgress.$inferInsert;
export type BookmarkOrigin = 'web' | 'koreader';
export type BookmarkOrigin = 'web' | 'koreader' | 'kobo';
export const userBookReadAloudSyncSettings = pgTable(
'user_book_read_aloud_sync_settings',
@@ -498,7 +498,7 @@ export const bookmarks = pgTable(
uniqueIndex('bookmarks_user_book_pos_uidx')
.on(t.userId, t.bookId, t.positionSeconds)
.where(sql`${t.positionSeconds} is not null and ${t.cfi} is null and ${t.deletedAt} is null`),
check('bookmarks_origin_chk', sql`${t.origin} in ('web', 'koreader')`),
check('bookmarks_origin_chk', sql`${t.origin} in ('web', 'koreader', 'kobo')`),
],
);
@@ -35,7 +35,7 @@ export class BookmarkSyncService {
const restored = await this.bookmarkRepo.restoreAtLocation(userId, bookId, location, {
title: fields.title,
origin: 'koreader',
origin: fields.origin ?? 'koreader',
devicePos: fields.devicePos,
pageno: fields.pageno,
});
@@ -4,7 +4,7 @@ import { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { DB } from '../../db';
import * as schema from '../../db/schema';
import { bookmarks, type BookmarkRow, type NewBookmark } from '../../db/schema';
import { bookmarks, type BookmarkOrigin, type BookmarkRow, type NewBookmark } from '../../db/schema';
type Db = NodePgDatabase<typeof schema>;
@@ -14,6 +14,8 @@ export interface DeviceBookmarkFields {
title: string;
devicePos: string;
pageno: number | null;
/** The device family the bookmark came from; KOReader when omitted. */
origin?: BookmarkOrigin;
}
@Injectable()
@@ -175,7 +177,7 @@ export class BookmarkRepository {
bookId,
cfi: fields.cfi,
title: fields.title,
origin: 'koreader',
origin: fields.origin ?? 'koreader',
devicePos: fields.devicePos,
pageno: fields.pageno,
})
@@ -0,0 +1,116 @@
import { Inject, Injectable } from '@nestjs/common';
import { and, eq, inArray, isNotNull } from 'drizzle-orm';
import type { NodePgDatabase } from 'drizzle-orm/node-postgres';
import { isUniqueViolation } from '../../common/utils/db-error.utils';
import { DB } from '../../db';
import * as schema from '../../db/schema';
/** A device's dogear link with the state of the bookmark behind it. */
export interface KoboBookmarkLinkState {
bookmarkId: number;
externalKey: string;
location: unknown;
bookmarkUpdatedAt: Date;
bookmarkDeletedAt: Date | null;
}
const REMOVED_BOOK_LIMIT = 500;
@Injectable()
export class KoboBookmarkLinkRepository {
constructor(@Inject(DB) private readonly db: NodePgDatabase<typeof schema>) {}
findForBook(userId: number, deviceId: number, bookId: number): Promise<KoboBookmarkLinkState[]> {
return this.db
.select({
bookmarkId: schema.koboBookmarkLinks.bookmarkId,
externalKey: schema.koboBookmarkLinks.externalKey,
location: schema.koboBookmarkLinks.location,
bookmarkUpdatedAt: schema.bookmarks.updatedAt,
bookmarkDeletedAt: schema.bookmarks.deletedAt,
})
.from(schema.koboBookmarkLinks)
.innerJoin(schema.bookmarks, eq(schema.bookmarks.id, schema.koboBookmarkLinks.bookmarkId))
.where(
and(
eq(schema.koboBookmarkLinks.userId, userId),
eq(schema.koboBookmarkLinks.deviceId, deviceId),
eq(schema.koboBookmarkLinks.bookId, bookId),
),
);
}
findByExternalKeys(userId: number, deviceId: number, externalKeys: string[]) {
if (externalKeys.length === 0) return Promise.resolve([]);
return this.db
.select()
.from(schema.koboBookmarkLinks)
.where(
and(
eq(schema.koboBookmarkLinks.userId, userId),
eq(schema.koboBookmarkLinks.deviceId, deviceId),
inArray(schema.koboBookmarkLinks.externalKey, externalKeys),
),
);
}
/**
* Links a dogear to its bookmark. Two dogears the device placed at one spot land on the same
* bookmark, which keeps the first link; the device drops the other once it is no longer served.
*/
async link(values: typeof schema.koboBookmarkLinks.$inferInsert): Promise<boolean> {
try {
await this.db
.insert(schema.koboBookmarkLinks)
.values(values)
.onConflictDoUpdate({
target: [schema.koboBookmarkLinks.deviceId, schema.koboBookmarkLinks.externalKey],
set: { bookmarkId: values.bookmarkId, bookId: values.bookId, location: values.location, updatedAt: new Date() },
});
return true;
} catch (error) {
if (isUniqueViolation(error)) return false;
throw error;
}
}
async updateLocation(userId: number, deviceId: number, externalKey: string, location: unknown): Promise<void> {
await this.db
.update(schema.koboBookmarkLinks)
.set({ location, updatedAt: new Date() })
.where(
and(
eq(schema.koboBookmarkLinks.userId, userId),
eq(schema.koboBookmarkLinks.deviceId, deviceId),
eq(schema.koboBookmarkLinks.externalKey, externalKey),
),
);
}
async deleteForBookmarks(userId: number, deviceId: number, bookmarkIds: number[]): Promise<number> {
if (bookmarkIds.length === 0) return 0;
const deleted = await this.db
.delete(schema.koboBookmarkLinks)
.where(
and(
eq(schema.koboBookmarkLinks.userId, userId),
eq(schema.koboBookmarkLinks.deviceId, deviceId),
inArray(schema.koboBookmarkLinks.bookmarkId, bookmarkIds),
),
)
.returning({ id: schema.koboBookmarkLinks.id });
return deleted.length;
}
/** Books holding a dogear on this device whose bookmark was deleted in BookOrbit. */
async findBookIdsWithRemovedBookmarks(userId: number, deviceId: number): Promise<number[]> {
const rows = await this.db
.selectDistinct({ bookId: schema.koboBookmarkLinks.bookId })
.from(schema.koboBookmarkLinks)
.innerJoin(schema.bookmarks, eq(schema.bookmarks.id, schema.koboBookmarkLinks.bookmarkId))
.where(and(eq(schema.koboBookmarkLinks.userId, userId), eq(schema.koboBookmarkLinks.deviceId, deviceId), isNotNull(schema.bookmarks.deletedAt)))
.limit(REMOVED_BOOK_LIMIT);
return rows.map((row) => row.bookId);
}
}
@@ -18,6 +18,8 @@ import { KoboBookIdentityService } from './services/kobo-book-identity.service';
import { KoboDeviceService } from './services/kobo-device.service';
import { KoboDeliveryService } from './services/kobo-delivery.service';
import { KoboDeliveryRepository } from './kobo-delivery.repository';
import { KoboBookmarkLinkRepository } from './kobo-bookmark-link.repository';
import { BookmarkModule } from '../bookmark/bookmark.module';
import { KoboDownloadRepository } from './kobo-download.repository';
import { KoboDownloadHashBackfillService } from './services/kobo-download-hash-backfill.service';
import { AppSettingsModule } from '../app-settings/app-settings.module';
@@ -47,6 +49,7 @@ describe('KoboModule', () => {
const imports = Reflect.getMetadata(MODULE_METADATA.IMPORTS, KoboModule) as unknown[];
expect(imports).toContain(BookModule);
expect(imports).toContain(BookmarkModule);
expect(imports).toContain(AppSettingsModule);
expect(imports).toContain(ReadingSessionModule);
expect(imports).toContain(AnnotationModule);
@@ -72,6 +75,7 @@ describe('KoboModule', () => {
KoboDownloadService,
KoboDownloadRepository,
KoboDeliveryRepository,
KoboBookmarkLinkRepository,
KoboReadingProgressRepository,
KoboDeliveryService,
KoboDownloadHashBackfillService,
+4
View File
@@ -7,6 +7,7 @@ import { BookModule } from '../book/book.module';
import { BookCoverStoreModule } from '../book-cover-store/book-cover-store.module';
import { MetadataModule } from '../metadata/metadata.module';
import { PositionConverterModule } from '../position-converter/position-converter.module';
import { BookmarkModule } from '../bookmark/bookmark.module';
import { ReadingSessionModule } from '../reading-session/reading-session.module';
import { SmartScopeModule } from '../smart-scope/smart-scope.module';
import { UserModule } from '../user/user.module';
@@ -26,6 +27,7 @@ import { KoboDeviceService } from './services/kobo-device.service';
import { KoboDownloadService } from './services/kobo-download.service';
import { KoboReadingProgressRepository } from './kobo-reading-progress.repository';
import { KoboDeliveryRepository } from './kobo-delivery.repository';
import { KoboBookmarkLinkRepository } from './kobo-bookmark-link.repository';
import { KoboDeliveryService } from './services/kobo-delivery.service';
import { KoboDownloadRepository } from './kobo-download.repository';
import { KoboDownloadHashRegistrationService } from './services/kobo-download-hash-registration.service';
@@ -51,6 +53,7 @@ import { KoboAnalyticsService } from './services/kobo-analytics.service';
AchievementModule,
AnnotationModule,
BookModule,
BookmarkModule,
BookCoverStoreModule,
MetadataModule,
PositionConverterModule,
@@ -80,6 +83,7 @@ import { KoboAnalyticsService } from './services/kobo-analytics.service';
KoboDownloadService,
KoboDownloadRepository,
KoboDeliveryRepository,
KoboBookmarkLinkRepository,
KoboReadingProgressRepository,
KoboDeliveryService,
KoboDownloadHashBackfillService,
@@ -0,0 +1,48 @@
import { describe, expect, it, vi } from 'vitest';
import { KoboAnnotationExchangeService } from './kobo-annotation-exchange.service';
function makeService(onDevice: number[]) {
const annotationSync = { listPendingKoboChangeBookIds: vi.fn().mockResolvedValue([1, 2]) };
const bookIdentityService = {
findByBookIds: vi.fn((_userId: number, bookIds: number[]) =>
Promise.resolve(new Map(bookIds.map((id) => [id, { entitlementId: `content-${id}` }]))),
),
};
const bookmarkLinks = { findBookIdsWithRemovedBookmarks: vi.fn().mockResolvedValue([2, 3]) };
const syncService = {
findBookIdsOnDevice: vi.fn((_userId: number, _deviceId: number, bookIds: number[]) =>
Promise.resolve(bookIds.filter((id) => onDevice.includes(id))),
),
};
const service = new KoboAnnotationExchangeService(
annotationSync as never,
bookIdentityService as never,
{} as never,
{} as never,
{ version: 1 } as never,
{ getSettings: vi.fn().mockResolvedValue({ syncBookOrbitAnnotationsToKobo: false }) } as never,
{} as never,
bookmarkLinks as never,
syncService as never,
);
return { service, syncService, bookIdentityService };
}
describe('KoboAnnotationExchangeService.getChangedContentIds', () => {
it('reports only books the device holds, once each, from annotation and bookmark changes', async () => {
const { service, syncService } = makeService([2, 3]);
await expect(service.getChangedContentIds(7, 11)).resolves.toEqual(['content-2', 'content-3']);
expect(syncService.findBookIdsOnDevice).toHaveBeenCalledWith(7, 11, [1, 2, 3]);
});
it('reports nothing for changes on books the device never received', async () => {
const { service, bookIdentityService } = makeService([]);
await expect(service.getChangedContentIds(7, 11)).resolves.toEqual([]);
expect(bookIdentityService.findByBookIds).not.toHaveBeenCalled();
});
});
@@ -6,16 +6,21 @@ import type { AnnotationPosition, AnnotationRow, AnnotationSyncStateRow } from '
import { sanitizeLogValue } from '../../../common/utils/log-sanitize.utils';
import { AnnotationSyncService, formatDeviceDatetime, type IncomingDeviceAnnotation } from '../../annotation/annotation-sync.service';
import { AnnotationConversionService } from '../../annotation/annotation-conversion.service';
import { BookmarkSyncService } from '../../bookmark/bookmark-sync.service';
import { KoboSpanConverterService, serializeKoboSpanPos } from '../../position-converter/kobo-span-converter.service';
import { parseSpanSelector } from '../../position-converter/kobo-span.core';
import { KoboDeliveryService, isUnconfirmedReplacement } from './kobo-delivery.service';
import { KoboBookIdentityService } from './kobo-book-identity.service';
import { KoboSyncService } from './kobo-sync.service';
import { KoboAnnotationMaterializerService } from './kobo-annotation-materializer.service';
import { KoboKepubContextService } from './kobo-kepub-context.service';
import { KoboSettingsService } from './kobo-settings.service';
import type { KepubContextResult } from './kobo-kepub-context.service';
import { KoboBookmarkLinkRepository } from '../kobo-bookmark-link.repository';
import {
extractKoboAnnotationOperations,
toKoboReadingServiceAnnotation,
toKoboReadingServiceDogear,
type IncomingUpsert,
type KoboReadingServiceAnnotationsResponse,
type ServeAnnotation,
@@ -24,6 +29,9 @@ import {
const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const GET_EVENT = 'kobo.annotation_get';
const PATCH_EVENT = 'kobo.annotation_patch';
const BOOKMARK_TITLE_MAX_LENGTH = 500;
type ReadyKepub = Extract<KepubContextResult, { ok: true }>;
/**
* Raised when a content id the device asks about is not a BookOrbit book. Kobo devices route every
@@ -40,6 +48,8 @@ export class KoboContentNotFoundException extends NotFoundException {
export interface ServedAckPlan {
entries: { annotationId: number; version: number; externalKey: string; externalCreatedAt: string | null }[];
tombstoneStateIds: number[];
/** Bookmarks deleted in BookOrbit and left out of this response, which the device deletes. */
removedBookmarkIds?: number[];
}
export interface KoboContentAnnotationsResult {
@@ -72,6 +82,9 @@ export class KoboAnnotationExchangeService {
private readonly materializer: KoboAnnotationMaterializerService,
private readonly koboSpanConverter: KoboSpanConverterService,
private readonly settingsService: KoboSettingsService,
private readonly bookmarkSync: BookmarkSyncService,
private readonly bookmarkLinks: KoboBookmarkLinkRepository,
private readonly syncService: KoboSyncService,
@Optional() private readonly deliveries?: KoboDeliveryService,
@Optional() private readonly annotationConversion?: AnnotationConversionService,
) {}
@@ -91,6 +104,7 @@ export class KoboAnnotationExchangeService {
const includeAllOrigins = kepub.settings?.syncBookOrbitAnnotationsToKobo ?? false;
if (this.deliveries && !kepub.ok) throw new ConflictException({ code: 'KOBO_PRECISE_SYNC_UNAVAILABLE' });
if (this.deliveries && kepub.ok) await this.annotationConversion?.ensureCfiPositionsForBook(userId, bookId, 200);
if (kepub.ok) await this.convertLegacyDogears(userId, deviceId, bookId, kepub);
if (kepub.ok && !this.deliveries) {
await this.materializer.ensureKoboSpanPositionsForBook(userId, bookId, kepub.file, kepub.ctx, { includeAllOrigins });
}
@@ -129,7 +143,8 @@ export class KoboAnnotationExchangeService {
(candidate) => candidate.annotationId === annotation.id && candidate.format === 'cfi' && candidate.status !== 'failed',
);
const raw = allPositions.find((candidate) => candidate.annotationId === annotation.id && candidate.format === 'kobo_span');
location = raw && (raw.extras as { renditionHash?: string } | null)?.renditionHash === renditionHash ? this.usableLocation(raw) : null;
const placed = raw && (raw.extras as { renditionHash?: string } | null)?.renditionHash === renditionHash ? this.usableLocation(raw) : null;
location = placed;
if (canonical?.pos0 && (!location || (raw?.extras as { sourceCfi?: string } | null)?.sourceCfi !== canonical.pos0)) {
const converted = await this.koboSpanConverter.canonicalToKoboSpan({
bookFileId: kepub.file.id,
@@ -140,6 +155,10 @@ export class KoboAnnotationExchangeService {
});
location = converted.status === 'failed' ? null : (converted.location ?? null);
}
// The device deletes what it is not served, and clears a book's annotations when its download is
// removed, so refusing the pull would leave them gone from the device. Where this copy cannot
// place an annotation, the location already placed in it, then the one the device reported, stand in.
if (!location && known) location = placed ?? (raw?.extras as { koboLocation?: unknown } | null)?.koboLocation ?? null;
if (!location && known) throw new ConflictException({ code: 'KOBO_ANNOTATION_RENDITION_UNAVAILABLE' });
}
if (!location) {
@@ -171,9 +190,14 @@ export class KoboAnnotationExchangeService {
// flags do not work - the device keeps any annotation still present in the
// response and ignores deleted markers.
const tombstones = await this.annotationSync.listDeleteCandidates(userId, 'kobo', String(deviceId), bookId, 500);
const servedAck: ServedAckPlan = { entries: ackEntries, tombstoneStateIds: tombstones.map((row) => row.state.id) };
const dogears = await this.listServedDogears(userId, deviceId, bookId);
const servedAck: ServedAckPlan = {
entries: ackEntries,
tombstoneStateIds: tombstones.map((row) => row.state.id),
removedBookmarkIds: dogears.removedBookmarkIds,
};
const etag = this.buildEtag(served.length, ackEntries, renditionHash);
const etag = this.buildEtag(served.length, ackEntries, renditionHash, dogears.served);
const notModified = this.etagMatches(ifNoneMatch, etag);
this.logger.log(
@@ -191,7 +215,7 @@ export class KoboAnnotationExchangeService {
servedCount: served.length,
tombstoneCount: servedAck.tombstoneStateIds.length,
response: {
annotations: served.map((row) => toKoboReadingServiceAnnotation(row)),
annotations: [...served.map((row) => toKoboReadingServiceAnnotation(row)), ...dogears.served.map((row) => toKoboReadingServiceDogear(row))],
nextPageOffsetToken: null,
},
};
@@ -209,6 +233,8 @@ export class KoboAnnotationExchangeService {
entries: ack.entries,
tombstoneStateIds: ack.tombstoneStateIds,
});
// A dogear left out of the response is deleted on the device, so its link has nothing left to track.
await this.bookmarkLinks.deleteForBookmarks(userId, deviceId, ack.removedBookmarkIds ?? []);
}
async patchContentAnnotations(userId: number, contentId: string, body: unknown, deviceId: number): Promise<KoboPatchAnnotationsResult> {
@@ -227,8 +253,25 @@ export class KoboAnnotationExchangeService {
const kepub = await this.kepubContextService.resolveForBook(userId, bookId, deviceId);
try {
if (kepub.ok) await this.convertLegacyDogears(userId, deviceId, bookId, kepub);
const dogears = kepub.ok
? await this.ingestDogears(
userId,
deviceId,
bookId,
kepub,
upserts.filter((operation) => operation.type === 'dogear'),
)
: { created: 0, updated: 0, placed: new Set<string>() };
const annotationUpserts = upserts.filter((operation) => !dogears.placed.has(operation.id));
const removedBookmarks = await this.removeDeviceBookmarks(
userId,
deviceId,
deletes.map((operation) => operation.id),
);
const delivery = await this.deliveries?.current(userId, deviceId, bookId);
const incoming = upserts.map((operation) => {
const incoming = annotationUpserts.map((operation) => {
const annotation = this.toIncomingAnnotation(operation);
annotation.posExtras = {
...(annotation.posExtras ?? {}),
@@ -254,23 +297,30 @@ export class KoboAnnotationExchangeService {
source: 'kobo',
deviceId: String(deviceId),
bookId,
deletes: deletes.map((operation) => ({ externalKey: operation.id })),
deletes: deletes.filter((operation) => !removedBookmarks.keys.has(operation.id)).map((operation) => ({ externalKey: operation.id })),
});
if (kepub.ok && upserts.length > 0) {
if (kepub.ok && annotationUpserts.length > 0) {
await this.materializer.convertIncomingForBook(
userId,
bookId,
kepub.file,
kepub.ctx,
upserts.map((operation) => operation.id),
annotationUpserts.map((operation) => operation.id),
);
}
this.logger.log(
`[${PATCH_EVENT}] [end] userId=${userId} bookId=${bookId} deviceId=${deviceId} durationMs=${Date.now() - startedAtMs} created=${result.created} updated=${result.updated} unchanged=${result.unchanged} deleted=${deletedCount} kepubReady=${kepub.ok} - kobo annotation operations applied`,
);
return { bookId, created: result.created, updated: result.updated, unchanged: result.unchanged, deleted: deletedCount, kepubReady: kepub.ok };
return {
bookId,
created: result.created + dogears.created,
updated: result.updated + dogears.updated,
unchanged: result.unchanged,
deleted: deletedCount + removedBookmarks.removed,
kepubReady: kepub.ok,
};
} finally {
if (kepub.ok) await kepub.release?.();
}
@@ -278,15 +328,166 @@ export class KoboAnnotationExchangeService {
async getChangedContentIds(userId: number, deviceId: number): Promise<string[]> {
const settings = await this.settingsService.getSettings(userId);
const bookIds = await this.annotationSync.listPendingKoboChangeBookIds(userId, String(deviceId), {
const annotationBookIds = await this.annotationSync.listPendingKoboChangeBookIds(userId, String(deviceId), {
includeAllOrigins: settings.syncBookOrbitAnnotationsToKobo,
resolverVersion: this.koboSpanConverter.version,
});
const changed = [...new Set([...annotationBookIds, ...(await this.bookmarkLinks.findBookIdsWithRemovedBookmarks(userId, deviceId))])];
// A device can only fetch and acknowledge annotations for books it holds. Another device's
// annotations on a book this one never received stay unserved, so reporting them would list the
// book on every sync, and a device busy with those leaves the fetch for a real change detached
// from its sync, where it is never applied.
const bookIds = await this.syncService.findBookIdsOnDevice(userId, deviceId, changed);
if (bookIds.length === 0) return [];
const identities = await this.bookIdentityService.findByBookIds(userId, bookIds);
return [...identities.values()].map((identity) => identity.entitlementId);
}
/**
* Lands the device's dogears as bookmarks. One the book cannot place stays on the annotation path,
* where it is still served back, since the device deletes whatever it is not served.
*/
private async ingestDogears(
userId: number,
deviceId: number,
bookId: number,
kepub: ReadyKepub,
dogears: IncomingUpsert[],
): Promise<{ created: number; updated: number; placed: Set<string> }> {
const placed = new Set<string>();
let created = 0;
let updated = 0;
if (dogears.length === 0) return { created, updated, placed };
const known = new Set(
(
await this.bookmarkLinks.findByExternalKeys(
userId,
deviceId,
dogears.map((dogear) => dogear.id),
)
).map((link) => link.externalKey),
);
for (const dogear of dogears) {
if (known.has(dogear.id)) {
await this.bookmarkLinks.updateLocation(userId, deviceId, dogear.id, dogear.location);
placed.add(dogear.id);
updated += 1;
} else if (
await this.landBookmark(userId, deviceId, bookId, kepub, dogear.id, dogear.location, this.bookmarkTitle(dogear.note, dogear.chapterTitle))
) {
placed.add(dogear.id);
created += 1;
}
}
return { created, updated, placed };
}
/**
* Dogears uploaded before Kobo bookmarks existed were stored as empty highlights. Each one this
* device holds moves to a bookmark under the same id, so the device keeps it as a dogear.
*/
private async convertLegacyDogears(userId: number, deviceId: number, bookId: number, kepub: ReadyKepub): Promise<void> {
const legacy = (await this.annotationSync.listStatesBySourceForBook(userId, 'kobo', bookId)).filter(
({ state, annotation }) =>
state.deviceId === String(deviceId) && annotation.origin === 'kobo' && annotation.text === '' && annotation.deletedAt === null,
);
if (legacy.length === 0) return;
const positions = await this.annotationSync.findPositions(
legacy.map(({ annotation }) => annotation.id),
['kobo_span'],
);
const locations = new Map(
positions.map((position) => [position.annotationId, (position.extras as { koboLocation?: unknown } | null)?.koboLocation]),
);
const converted: string[] = [];
for (const { state, annotation } of legacy) {
const location = locations.get(annotation.id);
if (!location) continue;
const title = this.bookmarkTitle(annotation.note, annotation.chapterTitle);
if (await this.landBookmark(userId, deviceId, bookId, kepub, state.externalKey, location, title)) converted.push(state.externalKey);
}
if (converted.length === 0) return;
await this.annotationSync.applyDeviceDeletes({
userId,
source: 'kobo',
deviceId: String(deviceId),
bookId,
deletes: converted.map((externalKey) => ({ externalKey })),
});
this.logger.log(
`[${PATCH_EVENT}] userId=${userId} bookId=${bookId} deviceId=${deviceId} converted=${converted.length} - kobo dogears moved from highlights to bookmarks`,
);
}
private async landBookmark(
userId: number,
deviceId: number,
bookId: number,
kepub: ReadyKepub,
externalKey: string,
location: unknown,
title: string,
): Promise<boolean> {
const point = await this.dogearToCanonical(kepub, location);
if (!point) return false;
const landed = await this.bookmarkSync.upsertFromDevice(userId, bookId, {
cfi: point.cfi,
title,
devicePos: point.xpointer,
pageno: null,
origin: 'kobo',
});
if (!landed) return false;
await this.bookmarkLinks.link({ bookmarkId: landed.row.id, userId, deviceId, bookId, externalKey, location });
return true;
}
private async dogearToCanonical(kepub: ReadyKepub, location: unknown): Promise<{ cfi: string; xpointer: string } | null> {
const span = location && typeof location === 'object' ? (location as { span?: unknown }).span : null;
if (!span || typeof span !== 'object') return null;
const { chapterFilename, startPath } = span as { chapterFilename?: unknown; startPath?: unknown };
const spanId = typeof startPath === 'string' ? parseSpanSelector(startPath) : null;
if (typeof chapterFilename !== 'string' || !spanId) return null;
const outcome = await this.koboSpanConverter.koboBookmarkToPositions({ bookFileId: kepub.file.id, ctx: kepub.ctx, chapterFilename, spanId });
return outcome.status === 'failed' || !outcome.cfi || !outcome.xpointer ? null : { cfi: outcome.cfi, xpointer: outcome.xpointer };
}
/** A device delete of a dogear deletes its bookmark everywhere, as a device delete of a highlight does. */
private async removeDeviceBookmarks(userId: number, deviceId: number, externalKeys: string[]): Promise<{ removed: number; keys: Set<string> }> {
const links = await this.bookmarkLinks.findByExternalKeys(userId, deviceId, externalKeys);
if (links.length === 0) return { removed: 0, keys: new Set() };
const bookmarkIds = links.map((link) => link.bookmarkId);
const removed = await this.bookmarkSync.tombstone(userId, bookmarkIds);
await this.bookmarkLinks.deleteForBookmarks(userId, deviceId, bookmarkIds);
return { removed, keys: new Set(links.map((link) => link.externalKey)) };
}
private async listServedDogears(
userId: number,
deviceId: number,
bookId: number,
): Promise<{ served: { externalKey: string; clientLastModifiedUtc: string; location: unknown }[]; removedBookmarkIds: number[] }> {
const served: { externalKey: string; clientLastModifiedUtc: string; location: unknown }[] = [];
const removedBookmarkIds: number[] = [];
for (const link of await this.bookmarkLinks.findForBook(userId, deviceId, bookId)) {
if (link.bookmarkDeletedAt) {
removedBookmarkIds.push(link.bookmarkId);
continue;
}
served.push({
externalKey: link.externalKey,
clientLastModifiedUtc: link.bookmarkUpdatedAt.toISOString().replace(/\.\d{3}Z$/, 'Z'),
location: link.location,
});
}
return { served, removedBookmarkIds };
}
/** A dogear has no text of its own; its note, then its chapter, names it, as KOReader's do. */
private bookmarkTitle(note: string | null, chapterTitle: string | null): string {
return (note?.trim() || chapterTitle?.trim() || 'Bookmark').slice(0, BOOKMARK_TITLE_MAX_LENGTH);
}
private toIncomingAnnotation(operation: IncomingUpsert): IncomingDeviceAnnotation {
const datetime = formatDeviceDatetime(operation.clientModifiedAt);
const span = this.parseLocationSpans(operation.location);
@@ -351,9 +552,17 @@ export class KoboAnnotationExchangeService {
}
}
private buildEtag(count: number, entries: ServedAckPlan['entries'], renditionHash?: string | null): string {
private buildEtag(
count: number,
entries: ServedAckPlan['entries'],
renditionHash?: string | null,
dogears: { clientLastModifiedUtc: string }[] = [],
): string {
const maxVersion = entries.reduce((max, entry) => Math.max(max, entry.version), 0);
return `W/"${maxVersion}-${count}${renditionHash ? `-${renditionHash}` : ''}"`;
const bookmarks = dogears.length
? `-b${dogears.length}-${dogears.reduce((max, row) => (row.clientLastModifiedUtc > max ? row.clientLastModifiedUtc : max), '')}`
: '';
return `W/"${maxVersion}-${count}${renditionHash ? `-${renditionHash}` : ''}${bookmarks}"`;
}
private etagMatches(ifNoneMatch: string | undefined, etag: string): boolean {
@@ -0,0 +1,42 @@
import { describe, expect, it } from 'vitest';
import { extractKoboAnnotationOperations, toKoboReadingServiceDogear } from './kobo-annotation-payload';
const location = {
span: {
chapterFilename: 'OEBPS/Text/018.xhtml',
chapterProgress: 0.25,
chapterTitle: 'Chapter Eleven',
startPath: 'OEBPS/Text/018.xhtml#kobo.23.1',
endPath: 'OEBPS/Text/018.xhtml#kobo.23.1',
startChar: 0,
endChar: 0,
},
};
describe('kobo annotation payload', () => {
it('reads a device dogear with its type and chapter', () => {
const [operation] = extractKoboAnnotationOperations({
updatedAnnotations: [
{ clientLastModifiedUtc: '2026-10-17T01:28:39Z', highlightedText: '', id: '13b9bf63-c3d8-4f56-98f8-468f744f1423', location, type: 'dogear' },
],
});
expect(operation).toMatchObject({ kind: 'upsert', type: 'dogear', text: '', chapterTitle: 'Chapter Eleven', location });
});
it('serves a bookmark back in the shape the device uploads a dogear, with no text or colour', () => {
const served = toKoboReadingServiceDogear({ externalKey: 'dogear-1', clientLastModifiedUtc: '2026-10-17T01:28:39Z', location });
expect(served).toEqual({
attachments: {},
clientLastModifiedUtc: '2026-10-17T01:28:39Z',
context: null,
highlightedText: '',
id: 'dogear-1',
location,
type: 'dogear',
});
expect(served).not.toHaveProperty('highlightColor');
});
});
@@ -4,7 +4,7 @@ export interface KoboReadingServiceAnnotation {
attachments: Record<string, never>;
clientLastModifiedUtc: string;
context: null;
highlightColor: string;
highlightColor?: string;
highlightedText: string;
id: string;
location: unknown;
@@ -63,6 +63,23 @@ export function toKoboReadingServiceAnnotation(row: ServeAnnotation): KoboReadin
};
}
/** A bookmark, served back as the dogear the device created: no text and no colour, as it uploads one. */
export function toKoboReadingServiceDogear(row: {
externalKey: string;
clientLastModifiedUtc: string;
location: unknown;
}): KoboReadingServiceAnnotation {
return {
attachments: {},
clientLastModifiedUtc: row.clientLastModifiedUtc,
context: null,
highlightedText: '',
id: row.externalKey,
location: row.location,
type: 'dogear',
};
}
export function extractKoboAnnotationOperations(body: unknown): IncomingOperation[] {
if (!isRecord(body)) return [];
@@ -206,6 +206,25 @@ export class KoboSyncService {
return [this.buildBookMetadata(book, deviceToken, baseUrl)];
}
/** Of the given books, those this device holds: in its library snapshot, and not removed or being removed. */
async findBookIdsOnDevice(userId: number, deviceId: number, bookIds: number[]): Promise<number[]> {
if (bookIds.length === 0) return [];
const rows = await this.db
.select({ bookId: schema.koboSnapshotBooks.bookId })
.from(schema.koboSnapshotBooks)
.innerJoin(schema.koboLibrarySnapshots, eq(schema.koboLibrarySnapshots.id, schema.koboSnapshotBooks.snapshotId))
.where(
and(
eq(schema.koboLibrarySnapshots.userId, userId),
eq(schema.koboLibrarySnapshots.deviceId, deviceId),
inArray(schema.koboSnapshotBooks.bookId, bookIds),
eq(schema.koboSnapshotBooks.pendingDelete, false),
eq(schema.koboSnapshotBooks.removedByDevice, false),
),
);
return rows.map((row) => row.bookId);
}
async removeBookFromSync(userId: number, deviceId: number, bookId: number): Promise<void> {
const snapshot = await this.findDeviceSnapshot(userId, deviceId);
if (!snapshot) return;
@@ -40,6 +40,19 @@ const KOBO_LOCATION = {
},
};
const DOGEAR_LOCATION = {
span: {
startPath: 'span#kobo\\.1\\.1',
startChar: 0,
endPath: 'span#kobo\\.1\\.1',
endChar: 0,
chapterFilename: 'OPS/chapter.xhtml',
chapterProgress: 0,
chapterTitle: 'Chapter 1',
},
};
const DOGEAR_CFI = 'epubcfi(/6/2!/4/2/1:0)';
const KOREADER_USERNAME = `kobo-mesh-device-${randomUUID().slice(0, 8)}`;
const KOREADER_PASSWORD = 'KoboMeshDevicePass123';
@@ -523,4 +536,112 @@ describe.each(['epub', 'kepub'])('Kobo annotation sync with %s (e2e)', { timeout
expect(served?.location).toMatchObject({ span: { startChar: 0, endChar: 7, chapterFilename: 'OPS/chapter.xhtml' } });
expect((await findPositions(annotation.id)).get('cfi')).toMatchObject({ status: 'exact', pos0: FIXTURE_CFI });
});
async function findBookmarks() {
return ctx.db
.select()
.from(schema.bookmarks)
.where(and(eq(schema.bookmarks.userId, adminUserId), eq(schema.bookmarks.bookId, epub.bookId)));
}
async function findBookmarkLinks() {
return ctx.db.select().from(schema.koboBookmarkLinks).where(eq(schema.koboBookmarkLinks.deviceId, deviceDbId));
}
function dogear(id: string) {
return { id, type: 'dogear', highlightedText: '', location: DOGEAR_LOCATION, clientLastModifiedUtc: '2026-06-13T08:00:00Z' };
}
it('lands a device dogear as a bookmark and serves it back as a dogear', async () => {
const dogearId = randomUUID();
await patchAnnotations({ updatedAnnotations: [dogear(dogearId)] });
const live = (await findBookmarks()).filter((row) => row.deletedAt === null);
expect(live).toHaveLength(1);
expect(live[0]).toMatchObject({ origin: 'kobo', title: 'Chapter 1', cfi: DOGEAR_CFI });
expect((await findCanonicalAnnotations()).filter((row) => row.text === '' && row.deletedAt === null)).toEqual([]);
const served = (await getAnnotations()).annotations.find((row) => row.id === dogearId);
expect(served).toMatchObject({ type: 'dogear', highlightedText: '' });
expect(served).not.toHaveProperty('highlightColor');
expect(served?.location).toEqual(DOGEAR_LOCATION);
});
it('reports a bookmark deleted in BookOrbit and leaves it out of the next pull', async () => {
const [bookmark] = (await findBookmarks()).filter((row) => row.deletedAt === null);
const removed = await ctx.app.inject({
method: 'DELETE',
url: `/api/v1/books/${epub.bookId}/bookmarks/${bookmark!.id}`,
headers: authHeader(ctx.adminToken),
});
expect(removed.statusCode).toBe(204);
expect(await checkForChanges()).toContain(contentId);
expect((await getAnnotations()).annotations.some((row) => row.type === 'dogear')).toBe(false);
expect(await findBookmarkLinks()).toEqual([]);
expect(await checkForChanges()).not.toContain(contentId);
});
it('deletes the bookmark when the device deletes its dogear', async () => {
const dogearId = randomUUID();
await patchAnnotations({ updatedAnnotations: [dogear(dogearId)] });
expect((await findBookmarks()).filter((row) => row.deletedAt === null)).toHaveLength(1);
await patchAnnotations({ deletedAnnotationIds: [dogearId] });
expect((await findBookmarks()).filter((row) => row.deletedAt === null)).toEqual([]);
expect(await findBookmarkLinks()).toEqual([]);
});
it('moves a dogear stored as an empty highlight to a bookmark under the same id', async () => {
const legacyId = randomUUID();
await patchAnnotations({
updatedAnnotations: [{ ...dogear(legacyId), type: 'highlight', highlightColor: '#C6E09E' }],
});
const legacy = (await findCanonicalAnnotations()).find((row) => row.text === '' && row.deletedAt === null);
expect(legacy).toBeDefined();
const served = (await getAnnotations()).annotations.find((row) => row.id === legacyId);
expect(served).toMatchObject({ type: 'dogear' });
expect((await findCanonicalAnnotations()).find((row) => row.id === legacy!.id)?.deletedAt).not.toBeNull();
expect((await findBookmarks()).filter((row) => row.deletedAt === null)).toEqual([expect.objectContaining({ origin: 'kobo', cfi: DOGEAR_CFI })]);
expect(await findBookmarkLinks()).toEqual([expect.objectContaining({ externalKey: legacyId })]);
});
it('serves a known annotation at the location the device reported when this copy cannot place it', async () => {
const highlightId = randomUUID();
await patchAnnotations({
updatedAnnotations: [
{
id: highlightId,
type: 'highlight',
highlightColor: '#C6E09E',
highlightedText: 'fixture',
location: KOBO_LOCATION,
clientLastModifiedUtc: '2026-06-14T08:00:00Z',
},
],
});
await getAnnotations();
const annotation = (await findCanonicalAnnotations()).find((row) => row.deletedAt === null && row.deviceCreatedAt === '2026-06-14 08:00:00')!;
// Placement fails for every annotation below, so the others leave first; this one is the subject.
for (const other of (await findCanonicalAnnotations()).filter((row) => row.deletedAt === null && row.id !== annotation.id)) {
await ctx.db.update(schema.annotations).set({ deletedAt: new Date() }).where(eq(schema.annotations.id, other.id));
}
await ctx.db
.update(schema.annotationPositions)
.set({ extras: { koboLocation: KOBO_LOCATION, renditionHash: 'an-earlier-copy' } })
.where(and(eq(schema.annotationPositions.annotationId, annotation.id), eq(schema.annotationPositions.format, 'kobo_span')));
const placement = vi
.spyOn(ctx.app.get(KoboSpanConverterService), 'canonicalToKoboSpan')
.mockResolvedValue({ status: 'failed', reason: 'span_not_found', renditionHash: 'this-copy' });
try {
const served = (await getAnnotations()).annotations.find((row) => row.id === highlightId);
expect(served?.location).toEqual(KOBO_LOCATION);
} finally {
placement.mockRestore();
}
});
});