diff --git a/server/src/modules/kobo/kobo-sync.controller.test.ts b/server/src/modules/kobo/kobo-sync.controller.test.ts index f86114f44..c07fb9676 100644 --- a/server/src/modules/kobo/kobo-sync.controller.test.ts +++ b/server/src/modules/kobo/kobo-sync.controller.test.ts @@ -1,5 +1,6 @@ import { BadRequestException, HttpStatus } from '@nestjs/common'; +import { encodeSyncToken } from './services/kobo-sync-token'; import { KoboSyncController } from './kobo-sync.controller'; function makeReply() { @@ -231,7 +232,7 @@ describe('KoboSyncController', () => { }); describe('store sync', () => { - const SNAPSHOT_TOKEN = `PX.${Buffer.from(JSON.stringify({ snapshotId: 4 })).toString('base64')}`; + const SNAPSHOT_TOKEN = encodeSyncToken(); function syncRequest(incomingSyncToken?: string) { return { @@ -280,7 +281,7 @@ describe('KoboSyncController', () => { await runSync(reply, 'kobo-native-token'); expect(proxyService.request).not.toHaveBeenCalled(); - expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', 'PX2.4.kobo-native-token'); + expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', 'kobo-native-token'); }); it('appends store entitlements and stamps Kobo’s cursor into the sync token', async () => { @@ -295,7 +296,7 @@ describe('KoboSyncController', () => { expect.objectContaining({ extraHeaders: { 'x-kobo-synctoken': 'kobo-cursor-1' } }), ); expect(reply.send).toHaveBeenCalledWith([{ Local: 1 }, { Store: 1 }, { Store: 2 }]); - expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', 'PX2.4.kobo-cursor-2'); + expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', 'kobo-cursor-2'); expect(historyService.recordSuccess).toHaveBeenCalledWith( expect.objectContaining({ counts: { entitlements: 3, hasMore: false, storeEntitlements: 2 } }), ); @@ -414,7 +415,7 @@ describe('KoboSyncController', () => { await runSync(reply, SNAPSHOT_TOKEN); expect(reply.header).toHaveBeenCalledWith('x-kobo-sync', 'continue'); - expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', `PX2.4.${continuation}`); + expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', continuation); }); it('preserves a known Kobo cursor across a failed store call so paging resumes next sync', async () => { @@ -423,7 +424,7 @@ describe('KoboSyncController', () => { await runSync(reply, `PX.${Buffer.from(JSON.stringify({ snapshotId: 4, koboSyncToken: 'kobo-cursor-1' })).toString('base64')}`); - expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', 'PX2.4.kobo-cursor-1'); + expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', 'kobo-cursor-1'); }); it('forwards the Kobo cursor from a compact token upstream', async () => { diff --git a/server/src/modules/kobo/services/kobo-sync-token.test.ts b/server/src/modules/kobo/services/kobo-sync-token.test.ts index 68ff33b0b..365002469 100644 --- a/server/src/modules/kobo/services/kobo-sync-token.test.ts +++ b/server/src/modules/kobo/services/kobo-sync-token.test.ts @@ -1,42 +1,59 @@ import { decodeSyncToken, encodeSyncToken, isUsableKoboSyncToken, withKoboSyncToken } from './kobo-sync-token'; +function base64Json(value: unknown): string { + return Buffer.from(JSON.stringify(value)).toString('base64'); +} + describe('kobo sync token', () => { - it('keeps the pre-store-sync token shape when there is no upstream cursor', () => { - const token = encodeSyncToken(7); + it('hands the device the token Calibre-Web sends when there is no upstream cursor (#1175)', () => { + const token = encodeSyncToken(); - expect(token).toBe(`PX.${Buffer.from(JSON.stringify({ snapshotId: 7 })).toString('base64')}`); - expect(decodeSyncToken(token)).toEqual({ snapshotId: 7, koboSyncToken: undefined }); + expect(JSON.parse(Buffer.from(token, 'base64').toString('utf8'))).toEqual({ + version: '1-1-0', + data: { + raw_kobo_store_token: '', + books_last_modified: -62135596800, + books_last_created: -62135596800, + archive_last_modified: -62135596800, + reading_state_last_modified: -62135596800, + tags_last_modified: -62135596800, + }, + }); + expect(token).toMatch(/^[A-Za-z0-9+/]+={0,2}$/); + expect(decodeSyncToken(token)).toEqual({}); }); - it('round-trips both cursors', () => { - expect(decodeSyncToken(encodeSyncToken(7, 'kobo-cursor'))).toEqual({ snapshotId: 7, koboSyncToken: 'kobo-cursor' }); - }); - - it('carries Kobo’s cursor verbatim instead of re-encoding it', () => { + it('hands the device Kobo’s own cursor verbatim when there is one', () => { const cursor = 'eyJJbnRlcm5hbFN5bmNUb2tlbiI6ImV5SjAifQ.eyJ0eXAiOjF9'; - expect(encodeSyncToken(7, cursor)).toBe(`PX2.7.${cursor}`); - expect(decodeSyncToken(`PX2.7.${cursor}`)).toEqual({ snapshotId: 7, koboSyncToken: cursor }); + expect(encodeSyncToken(cursor)).toBe(cursor); + expect(decodeSyncToken(cursor)).toEqual({ koboSyncToken: cursor }); }); - it('still reads a Kobo cursor from tokens minted in the JSON format', () => { - const legacy = `PX.${Buffer.from(JSON.stringify({ snapshotId: 3, koboSyncToken: 'kobo-cursor' })).toString('base64')}`; + it('reads the store cursor out of a Calibre-Web token, ours or one Calibre-Web left behind', () => { + const token = base64Json({ data: { raw_kobo_store_token: 'kobo-cursor', books_last_modified: 1700000000 }, version: '1-1-0' }); - expect(decodeSyncToken(legacy)).toEqual({ snapshotId: 3, koboSyncToken: 'kobo-cursor' }); + expect(decodeSyncToken(token)).toEqual({ koboSyncToken: 'kobo-cursor' }); + }); + + it('still reads the cursor from compact PX2 tokens devices already hold', () => { + const cursor = 'eyJJbnRlcm5hbFN5bmNUb2tlbiI6ImV5SjAifQ.eyJ0eXAiOjF9'; + + expect(decodeSyncToken(`PX2.7.${cursor}`)).toEqual({ koboSyncToken: cursor }); + expect(decodeSyncToken('PX2.7')).toEqual({}); + }); + + it('still reads PX tokens, with and without a store cursor', () => { + expect(decodeSyncToken(`PX.${base64Json({ snapshotId: 3, koboSyncToken: 'kobo-cursor' })}`)).toEqual({ koboSyncToken: 'kobo-cursor' }); + expect(decodeSyncToken(`PX.${base64Json({ snapshotId: 3 })}`)).toEqual({}); + }); + + it('treats an unprefixed token it cannot parse as Kobo’s own, for devices that synced with Kobo first', () => { + expect(decodeSyncToken('kobo-opaque-token')).toEqual({ koboSyncToken: 'kobo-opaque-token' }); }); it('keeps the header within a proxy line limit for the largest cursor it carries', () => { - expect(encodeSyncToken(2_147_483_647, 'x'.repeat(7000)).length).toBeLessThan(8000); - }); - - it('reads tokens minted before store sync existed', () => { - const legacy = `PX.${Buffer.from(JSON.stringify({ snapshotId: 3 })).toString('base64')}`; - - expect(decodeSyncToken(legacy)).toEqual({ snapshotId: 3, koboSyncToken: undefined }); - }); - - it('treats an unprefixed token as Kobo’s own, for devices that synced with Kobo first', () => { - expect(decodeSyncToken('kobo-opaque-token')).toEqual({ koboSyncToken: 'kobo-opaque-token' }); + expect(encodeSyncToken('x'.repeat(7000)).length).toBeLessThan(8000); }); it.each([ @@ -44,13 +61,13 @@ describe('kobo sync token', () => { ['empty', ''], ['whitespace', ' '], ['malformed base64 payload', 'PX.!!!not-base64!!!'], - ['a JSON array payload', `PX.${Buffer.from(JSON.stringify([1, 2])).toString('base64')}`], - ['a non-object payload', `PX.${Buffer.from(JSON.stringify('nope')).toString('base64')}`], - ['a compact token with a non-numeric snapshot', 'PX2.abc.kobo-cursor'], + ['a JSON array payload', `PX.${base64Json([1, 2])}`], + ['a non-object payload', `PX.${base64Json('nope')}`], + ['a Calibre-Web token with no data', base64Json({ version: '1-1-0' })], + ['a Calibre-Web token with a non-string cursor', base64Json({ data: { raw_kobo_store_token: 42 } })], ['a compact token with nothing after the prefix', 'PX2.'], ])('never throws on %s', (_label, raw) => { expect(() => decodeSyncToken(raw)).not.toThrow(); - expect(decodeSyncToken(raw).snapshotId).toBeUndefined(); }); it('reports which upstream cursors can be carried, so callers can log the ones that cannot', () => { @@ -64,20 +81,17 @@ describe('kobo sync token', () => { const oversized = 'x'.repeat(7001); expect(decodeSyncToken(oversized)).toEqual({}); - expect(decodeSyncToken(encodeSyncToken(7, oversized))).toEqual({ snapshotId: 7, koboSyncToken: undefined }); + expect(encodeSyncToken(oversized)).toBe(encodeSyncToken()); + expect(withKoboSyncToken(encodeSyncToken(), oversized)).toBe(encodeSyncToken()); }); it('leaves the token untouched when there is no upstream cursor to stamp', () => { - const token = encodeSyncToken(7); + const token = encodeSyncToken(); expect(withKoboSyncToken(token, undefined)).toBe(token); }); - it('stamps the upstream cursor onto a token minted by the sync service', () => { - expect(withKoboSyncToken(encodeSyncToken(7), 'kobo-cursor')).toBe(encodeSyncToken(7, 'kobo-cursor')); - }); - - it('returns an unreadable token unchanged rather than inventing a snapshot', () => { - expect(withKoboSyncToken('PX.!!!', 'kobo-cursor')).toBe('PX.!!!'); + it('replaces the token minted by the sync service with the upstream cursor', () => { + expect(withKoboSyncToken(encodeSyncToken(), 'kobo-cursor')).toBe('kobo-cursor'); }); }); diff --git a/server/src/modules/kobo/services/kobo-sync-token.ts b/server/src/modules/kobo/services/kobo-sync-token.ts index 0660f0633..843dab3d4 100644 --- a/server/src/modules/kobo/services/kobo-sync-token.ts +++ b/server/src/modules/kobo/services/kobo-sync-token.ts @@ -1,17 +1,22 @@ +// Older BookOrbit tokens, still sent back by devices that last synced before the Kobo-standard shape. const TOKEN_PREFIX = 'PX.'; -// Carries Kobo's cursor verbatim after the snapshot id: `PX2..`. Kobo's token -// already travels in a header, and wrapping it in JSON and base64 again grew it by a third. const COMPACT_TOKEN_PREFIX = 'PX2.'; -// Kobo's own sync tokens are opaque and we round-trip them through the device inside ours, so a -// hostile or corrupted upstream value cannot be allowed to grow the response header without bound. -// A small store library already produces a 3,300 character cursor and continuation pages carry more, -// so this leaves room for those while the whole header stays under the common 8 KB proxy line limit. +// A Kobo on firmware 4.45 was reported failing every sync on BookOrbit's prefixed token and passing +// on Calibre-Web's (#1175), so a device is only ever handed a shape Kobo devices are known to take: +// Kobo's own cursor verbatim, or the base64 JSON token Calibre-Web has always sent. The token +// identifies no snapshot; the sync service finds that by device. +const CALIBRE_WEB_TOKEN_VERSION = '1-1-0'; +const CALIBRE_WEB_EPOCH_FLOOR = -62135596800; + +// Kobo's own sync tokens are opaque and we round-trip them through the device, so a hostile or +// corrupted upstream value cannot be allowed to grow the response header without bound. A small +// store library already produces a 3,300 character cursor and continuation pages carry more, so this +// leaves room for those while the whole header stays under the common 8 KB proxy line limit. // Anything larger is dropped, and store paging for that sync stops instead of restarting. const MAX_KOBO_SYNC_TOKEN_LENGTH = 7000; export type KoboSyncTokenParts = { - snapshotId?: number; koboSyncToken?: string; }; @@ -23,17 +28,28 @@ export function isUsableKoboSyncToken(koboSyncToken: string): boolean { return koboSyncToken.length > 0 && koboSyncToken.length <= MAX_KOBO_SYNC_TOKEN_LENGTH; } -export function encodeSyncToken(snapshotId: number, koboSyncToken?: string): string { - const usableKoboToken = koboSyncToken && isUsableKoboSyncToken(koboSyncToken) ? koboSyncToken : undefined; - if (usableKoboToken) return `${COMPACT_TOKEN_PREFIX}${snapshotId}.${usableKoboToken}`; - return TOKEN_PREFIX + Buffer.from(JSON.stringify({ snapshotId })).toString('base64'); +export function encodeSyncToken(koboSyncToken?: string): string { + if (koboSyncToken && isUsableKoboSyncToken(koboSyncToken)) return koboSyncToken; + return Buffer.from( + JSON.stringify({ + version: CALIBRE_WEB_TOKEN_VERSION, + data: { + raw_kobo_store_token: '', + books_last_modified: CALIBRE_WEB_EPOCH_FLOOR, + books_last_created: CALIBRE_WEB_EPOCH_FLOOR, + archive_last_modified: CALIBRE_WEB_EPOCH_FLOOR, + reading_state_last_modified: CALIBRE_WEB_EPOCH_FLOOR, + tags_last_modified: CALIBRE_WEB_EPOCH_FLOOR, + }, + }), + ).toString('base64'); } /** - * Tokens minted before store sync existed carry only {snapshotId}, and a device that synced against - * Kobo directly before BookOrbit was put in the path sends Kobo's own opaque token. Neither shape - * may throw: getDelta finds the snapshot by device rather than by token, so a token we cannot read - * costs a store page, never a broken sync. + * Reads the upstream cursor out of whatever token the device sends back: Kobo's own, a Calibre-Web + * shaped one (ours, or one left by Calibre-Web before BookOrbit was put in the path), or an older + * BookOrbit `PX.` / `PX2.` token. None of them may throw: a token we cannot read costs a store page, + * never a broken sync. */ export function decodeSyncToken(raw: string | undefined): KoboSyncTokenParts { const value = raw?.trim(); @@ -41,39 +57,39 @@ export function decodeSyncToken(raw: string | undefined): KoboSyncTokenParts { if (value.startsWith(COMPACT_TOKEN_PREFIX)) { const rest = value.slice(COMPACT_TOKEN_PREFIX.length); const separator = rest.indexOf('.'); - const id = separator === -1 ? rest : rest.slice(0, separator); - const koboSyncToken = separator === -1 ? '' : rest.slice(separator + 1); - return { - snapshotId: /^\d{1,15}$/.test(id) ? Number(id) : undefined, - koboSyncToken: isUsableKoboSyncToken(koboSyncToken) ? koboSyncToken : undefined, - }; + return usableCursor(separator === -1 ? '' : rest.slice(separator + 1)); } - if (!value.startsWith(TOKEN_PREFIX)) { - return isUsableKoboSyncToken(value) ? { koboSyncToken: value } : {}; + if (value.startsWith(TOKEN_PREFIX)) { + const decoded = parseBase64Json(value.slice(TOKEN_PREFIX.length)); + return usableCursor(decoded?.koboSyncToken); } - - try { - const decoded: unknown = JSON.parse(Buffer.from(value.slice(TOKEN_PREFIX.length), 'base64').toString('utf8')); - if (!decoded || typeof decoded !== 'object' || Array.isArray(decoded)) return {}; - const { snapshotId, koboSyncToken } = decoded as Record; - const usableKoboToken = typeof koboSyncToken === 'string' && isUsableKoboSyncToken(koboSyncToken) ? koboSyncToken : undefined; - return { - snapshotId: typeof snapshotId === 'number' && Number.isFinite(snapshotId) ? snapshotId : undefined, - koboSyncToken: usableKoboToken, - }; - } catch { - return {}; + const calibreWeb = parseBase64Json(value); + if (calibreWeb && typeof calibreWeb.data === 'object' && calibreWeb.data !== null && !Array.isArray(calibreWeb.data)) { + return usableCursor((calibreWeb.data as Record).raw_kobo_store_token); } + return usableCursor(value); } /** - * Re-stamps a token minted by KoboSyncService with the upstream cursor. Returns the token unchanged - * when there is no upstream cursor to carry, so a sync with store sync off is byte-identical to one - * from before store sync existed. + * Re-stamps a token minted by KoboSyncService with the upstream cursor, which then travels alone. + * Returns the token unchanged when there is no upstream cursor to carry. */ export function withKoboSyncToken(compositeToken: string, koboSyncToken: string | undefined): string { - if (!koboSyncToken) return compositeToken; - const { snapshotId } = decodeSyncToken(compositeToken); - if (snapshotId === undefined) return compositeToken; - return encodeSyncToken(snapshotId, koboSyncToken); + if (!koboSyncToken || !isUsableKoboSyncToken(koboSyncToken)) return compositeToken; + return encodeSyncToken(koboSyncToken); +} + +function usableCursor(value: unknown): KoboSyncTokenParts { + return typeof value === 'string' && isUsableKoboSyncToken(value) ? { koboSyncToken: value } : {}; +} + +// Kobo's own cursor is dotted base64url, which never parses to a JSON object here. +function parseBase64Json(value: string): Record | null { + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(value)) return null; + try { + const decoded: unknown = JSON.parse(Buffer.from(value, 'base64').toString('utf8')); + return decoded && typeof decoded === 'object' && !Array.isArray(decoded) ? (decoded as Record) : null; + } catch { + return null; + } } diff --git a/server/src/modules/kobo/services/kobo-sync.service.test.ts b/server/src/modules/kobo/services/kobo-sync.service.test.ts index 613b945a8..de036c1ef 100644 --- a/server/src/modules/kobo/services/kobo-sync.service.test.ts +++ b/server/src/modules/kobo/services/kobo-sync.service.test.ts @@ -1,6 +1,7 @@ import { createHash } from 'crypto'; import { sql } from 'drizzle-orm'; import * as schema from '../../../db/schema'; +import { encodeSyncToken } from './kobo-sync-token'; import { KoboSyncService } from './kobo-sync.service'; // Recursively counts bound query params (drizzle-orm's `Param` chunks) in a built SQL tree, @@ -511,7 +512,7 @@ describe('KoboSyncService', () => { expect(result).toEqual({ entitlements: [{ ChangedTag: {} }], hasMore: false, - syncToken: expect.stringMatching(/^PX\./), + syncToken: encodeSyncToken(), }); }); it('getPageFromSnapshot returns page entitlements for removed/new/changed books', async () => { diff --git a/server/src/modules/kobo/services/kobo-sync.service.ts b/server/src/modules/kobo/services/kobo-sync.service.ts index cb8e43a30..d1706b4b2 100644 --- a/server/src/modules/kobo/services/kobo-sync.service.ts +++ b/server/src/modules/kobo/services/kobo-sync.service.ts @@ -620,7 +620,7 @@ export class KoboSyncService { smartScopeMatchCache: SmartScopeMatchCache, resolveEligibleIds: EligibleIdsResolver, ): Promise<{ entitlements: unknown[]; hasMore: boolean; syncToken: string }> { - const syncToken = encodeSyncToken(snapshotId); + const syncToken = encodeSyncToken(); const pending = await this.db .select()