fix(kobo): hand the Kobo a sync token in a shape it recognises

Kobos on firmware 4.45 were reported failing every sync on BookOrbit's
prefixed PX tokens and passing on Calibre-Web's. Send Calibre-Web's base64
JSON token, or Kobo's own cursor verbatim when store titles carry one. The
token never located the snapshot, and PX and PX2 tokens still decode.

Fixes #1175
Fixes #435

server/src/modules/kobo/kobo-sync.controller.test.ts

@@ -1,5 +1,6 @@
 import { BadRequestException, HttpStatus } from '@nestjs/common';
 
+import { encodeSyncToken } from './services/kobo-sync-token';
 import { KoboSyncController } from './kobo-sync.controller';
 
 function makeReply() {
@@ -231,7 +232,7 @@ describe('KoboSyncController', () => {
   });
 
   describe('store sync', () => {
-    const SNAPSHOT_TOKEN = `PX.${Buffer.from(JSON.stringify({ snapshotId: 4 })).toString('base64')}`;
+    const SNAPSHOT_TOKEN = encodeSyncToken();
 
     function syncRequest(incomingSyncToken?: string) {
       return {
@@ -280,7 +281,7 @@ describe('KoboSyncController', () => {
       await runSync(reply, 'kobo-native-token');
 
       expect(proxyService.request).not.toHaveBeenCalled();
-      expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', 'PX2.4.kobo-native-token');
+      expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', 'kobo-native-token');
     });
 
     it('appends store entitlements and stamps Kobo’s cursor into the sync token', async () => {
@@ -295,7 +296,7 @@ describe('KoboSyncController', () => {
         expect.objectContaining({ extraHeaders: { 'x-kobo-synctoken': 'kobo-cursor-1' } }),
       );
       expect(reply.send).toHaveBeenCalledWith([{ Local: 1 }, { Store: 1 }, { Store: 2 }]);
-      expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', 'PX2.4.kobo-cursor-2');
+      expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', 'kobo-cursor-2');
       expect(historyService.recordSuccess).toHaveBeenCalledWith(
         expect.objectContaining({ counts: { entitlements: 3, hasMore: false, storeEntitlements: 2 } }),
       );
@@ -414,7 +415,7 @@ describe('KoboSyncController', () => {
       await runSync(reply, SNAPSHOT_TOKEN);
 
       expect(reply.header).toHaveBeenCalledWith('x-kobo-sync', 'continue');
-      expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', `PX2.4.${continuation}`);
+      expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', continuation);
     });
 
     it('preserves a known Kobo cursor across a failed store call so paging resumes next sync', async () => {
@@ -423,7 +424,7 @@ describe('KoboSyncController', () => {
 
       await runSync(reply, `PX.${Buffer.from(JSON.stringify({ snapshotId: 4, koboSyncToken: 'kobo-cursor-1' })).toString('base64')}`);
 
-      expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', 'PX2.4.kobo-cursor-1');
+      expect(reply.header).toHaveBeenCalledWith('x-kobo-synctoken', 'kobo-cursor-1');
     });
 
     it('forwards the Kobo cursor from a compact token upstream', async () => {

server/src/modules/kobo/services/kobo-sync-token.test.ts

@@ -1,42 +1,59 @@
 import { decodeSyncToken, encodeSyncToken, isUsableKoboSyncToken, withKoboSyncToken } from './kobo-sync-token';
 
+function base64Json(value: unknown): string {
+  return Buffer.from(JSON.stringify(value)).toString('base64');
+}
+
 describe('kobo sync token', () => {
-  it('keeps the pre-store-sync token shape when there is no upstream cursor', () => {
+  it('hands the device the token Calibre-Web sends when there is no upstream cursor (#1175)', () => {
-    const token = encodeSyncToken(7);
+    const token = encodeSyncToken();
 
-    expect(token).toBe(`PX.${Buffer.from(JSON.stringify({ snapshotId: 7 })).toString('base64')}`);
+    expect(JSON.parse(Buffer.from(token, 'base64').toString('utf8'))).toEqual({
-    expect(decodeSyncToken(token)).toEqual({ snapshotId: 7, koboSyncToken: undefined });
+      version: '1-1-0',
+      data: {
+        raw_kobo_store_token: '',
+        books_last_modified: -62135596800,
+        books_last_created: -62135596800,
+        archive_last_modified: -62135596800,
+        reading_state_last_modified: -62135596800,
+        tags_last_modified: -62135596800,
+      },
+    });
+    expect(token).toMatch(/^[A-Za-z0-9+/]+={0,2}$/);
+    expect(decodeSyncToken(token)).toEqual({});
   });
 
-  it('round-trips both cursors', () => {
+  it('hands the device Kobo’s own cursor verbatim when there is one', () => {
-    expect(decodeSyncToken(encodeSyncToken(7, 'kobo-cursor'))).toEqual({ snapshotId: 7, koboSyncToken: 'kobo-cursor' });
-  });
-
-  it('carries Kobo’s cursor verbatim instead of re-encoding it', () => {
     const cursor = 'eyJJbnRlcm5hbFN5bmNUb2tlbiI6ImV5SjAifQ.eyJ0eXAiOjF9';
 
-    expect(encodeSyncToken(7, cursor)).toBe(`PX2.7.${cursor}`);
+    expect(encodeSyncToken(cursor)).toBe(cursor);
-    expect(decodeSyncToken(`PX2.7.${cursor}`)).toEqual({ snapshotId: 7, koboSyncToken: cursor });
+    expect(decodeSyncToken(cursor)).toEqual({ koboSyncToken: cursor });
   });
 
-  it('still reads a Kobo cursor from tokens minted in the JSON format', () => {
+  it('reads the store cursor out of a Calibre-Web token, ours or one Calibre-Web left behind', () => {
-    const legacy = `PX.${Buffer.from(JSON.stringify({ snapshotId: 3, koboSyncToken: 'kobo-cursor' })).toString('base64')}`;
+    const token = base64Json({ data: { raw_kobo_store_token: 'kobo-cursor', books_last_modified: 1700000000 }, version: '1-1-0' });
 
-    expect(decodeSyncToken(legacy)).toEqual({ snapshotId: 3, koboSyncToken: 'kobo-cursor' });
+    expect(decodeSyncToken(token)).toEqual({ koboSyncToken: 'kobo-cursor' });
+  });
+
+  it('still reads the cursor from compact PX2 tokens devices already hold', () => {
+    const cursor = 'eyJJbnRlcm5hbFN5bmNUb2tlbiI6ImV5SjAifQ.eyJ0eXAiOjF9';
+
+    expect(decodeSyncToken(`PX2.7.${cursor}`)).toEqual({ koboSyncToken: cursor });
+    expect(decodeSyncToken('PX2.7')).toEqual({});
+  });
+
+  it('still reads PX tokens, with and without a store cursor', () => {
+    expect(decodeSyncToken(`PX.${base64Json({ snapshotId: 3, koboSyncToken: 'kobo-cursor' })}`)).toEqual({ koboSyncToken: 'kobo-cursor' });
+    expect(decodeSyncToken(`PX.${base64Json({ snapshotId: 3 })}`)).toEqual({});
+  });
+
+  it('treats an unprefixed token it cannot parse as Kobo’s own, for devices that synced with Kobo first', () => {
+    expect(decodeSyncToken('kobo-opaque-token')).toEqual({ koboSyncToken: 'kobo-opaque-token' });
   });
 
   it('keeps the header within a proxy line limit for the largest cursor it carries', () => {
-    expect(encodeSyncToken(2_147_483_647, 'x'.repeat(7000)).length).toBeLessThan(8000);
+    expect(encodeSyncToken('x'.repeat(7000)).length).toBeLessThan(8000);
-  });
-
-  it('reads tokens minted before store sync existed', () => {
-    const legacy = `PX.${Buffer.from(JSON.stringify({ snapshotId: 3 })).toString('base64')}`;
-
-    expect(decodeSyncToken(legacy)).toEqual({ snapshotId: 3, koboSyncToken: undefined });
-  });
-
-  it('treats an unprefixed token as Kobo’s own, for devices that synced with Kobo first', () => {
-    expect(decodeSyncToken('kobo-opaque-token')).toEqual({ koboSyncToken: 'kobo-opaque-token' });
   });
 
   it.each([
@@ -44,13 +61,13 @@ describe('kobo sync token', () => {
     ['empty', ''],
     ['whitespace', '   '],
     ['malformed base64 payload', 'PX.!!!not-base64!!!'],
-    ['a JSON array payload', `PX.${Buffer.from(JSON.stringify([1, 2])).toString('base64')}`],
+    ['a JSON array payload', `PX.${base64Json([1, 2])}`],
-    ['a non-object payload', `PX.${Buffer.from(JSON.stringify('nope')).toString('base64')}`],
+    ['a non-object payload', `PX.${base64Json('nope')}`],
-    ['a compact token with a non-numeric snapshot', 'PX2.abc.kobo-cursor'],
+    ['a Calibre-Web token with no data', base64Json({ version: '1-1-0' })],
+    ['a Calibre-Web token with a non-string cursor', base64Json({ data: { raw_kobo_store_token: 42 } })],
     ['a compact token with nothing after the prefix', 'PX2.'],
   ])('never throws on %s', (_label, raw) => {
     expect(() => decodeSyncToken(raw)).not.toThrow();
-    expect(decodeSyncToken(raw).snapshotId).toBeUndefined();
   });
 
   it('reports which upstream cursors can be carried, so callers can log the ones that cannot', () => {
@@ -64,20 +81,17 @@ describe('kobo sync token', () => {
     const oversized = 'x'.repeat(7001);
 
     expect(decodeSyncToken(oversized)).toEqual({});
-    expect(decodeSyncToken(encodeSyncToken(7, oversized))).toEqual({ snapshotId: 7, koboSyncToken: undefined });
+    expect(encodeSyncToken(oversized)).toBe(encodeSyncToken());
+    expect(withKoboSyncToken(encodeSyncToken(), oversized)).toBe(encodeSyncToken());
   });
 
   it('leaves the token untouched when there is no upstream cursor to stamp', () => {
-    const token = encodeSyncToken(7);
+    const token = encodeSyncToken();
 
     expect(withKoboSyncToken(token, undefined)).toBe(token);
   });
 
-  it('stamps the upstream cursor onto a token minted by the sync service', () => {
+  it('replaces the token minted by the sync service with the upstream cursor', () => {
-    expect(withKoboSyncToken(encodeSyncToken(7), 'kobo-cursor')).toBe(encodeSyncToken(7, 'kobo-cursor'));
+    expect(withKoboSyncToken(encodeSyncToken(), 'kobo-cursor')).toBe('kobo-cursor');
-  });
-
-  it('returns an unreadable token unchanged rather than inventing a snapshot', () => {
-    expect(withKoboSyncToken('PX.!!!', 'kobo-cursor')).toBe('PX.!!!');
   });
 });

server/src/modules/kobo/services/kobo-sync-token.ts

@@ -1,17 +1,22 @@
+// Older BookOrbit tokens, still sent back by devices that last synced before the Kobo-standard shape.
 const TOKEN_PREFIX = 'PX.';
-// Carries Kobo's cursor verbatim after the snapshot id: `PX2.<snapshotId>.<kobo token>`. Kobo's token
-// already travels in a header, and wrapping it in JSON and base64 again grew it by a third.
 const COMPACT_TOKEN_PREFIX = 'PX2.';
 
-// Kobo's own sync tokens are opaque and we round-trip them through the device inside ours, so a
+// A Kobo on firmware 4.45 was reported failing every sync on BookOrbit's prefixed token and passing
-// hostile or corrupted upstream value cannot be allowed to grow the response header without bound.
+// on Calibre-Web's (#1175), so a device is only ever handed a shape Kobo devices are known to take:
-// A small store library already produces a 3,300 character cursor and continuation pages carry more,
+// Kobo's own cursor verbatim, or the base64 JSON token Calibre-Web has always sent. The token
-// so this leaves room for those while the whole header stays under the common 8 KB proxy line limit.
+// identifies no snapshot; the sync service finds that by device.
+const CALIBRE_WEB_TOKEN_VERSION = '1-1-0';
+const CALIBRE_WEB_EPOCH_FLOOR = -62135596800;
+
+// Kobo's own sync tokens are opaque and we round-trip them through the device, so a hostile or
+// corrupted upstream value cannot be allowed to grow the response header without bound. A small
+// store library already produces a 3,300 character cursor and continuation pages carry more, so this
+// leaves room for those while the whole header stays under the common 8 KB proxy line limit.
 // Anything larger is dropped, and store paging for that sync stops instead of restarting.
 const MAX_KOBO_SYNC_TOKEN_LENGTH = 7000;
 
 export type KoboSyncTokenParts = {
-  snapshotId?: number;
   koboSyncToken?: string;
 };
 
@@ -23,17 +28,28 @@ export function isUsableKoboSyncToken(koboSyncToken: string): boolean {
   return koboSyncToken.length > 0 && koboSyncToken.length <= MAX_KOBO_SYNC_TOKEN_LENGTH;
 }
 
-export function encodeSyncToken(snapshotId: number, koboSyncToken?: string): string {
+export function encodeSyncToken(koboSyncToken?: string): string {
-  const usableKoboToken = koboSyncToken && isUsableKoboSyncToken(koboSyncToken) ? koboSyncToken : undefined;
+  if (koboSyncToken && isUsableKoboSyncToken(koboSyncToken)) return koboSyncToken;
-  if (usableKoboToken) return `${COMPACT_TOKEN_PREFIX}${snapshotId}.${usableKoboToken}`;
+  return Buffer.from(
-  return TOKEN_PREFIX + Buffer.from(JSON.stringify({ snapshotId })).toString('base64');
+    JSON.stringify({
+      version: CALIBRE_WEB_TOKEN_VERSION,
+      data: {
+        raw_kobo_store_token: '',
+        books_last_modified: CALIBRE_WEB_EPOCH_FLOOR,
+        books_last_created: CALIBRE_WEB_EPOCH_FLOOR,
+        archive_last_modified: CALIBRE_WEB_EPOCH_FLOOR,
+        reading_state_last_modified: CALIBRE_WEB_EPOCH_FLOOR,
+        tags_last_modified: CALIBRE_WEB_EPOCH_FLOOR,
+      },
+    }),
+  ).toString('base64');
 }
 
 /**
- * Tokens minted before store sync existed carry only {snapshotId}, and a device that synced against
+ * Reads the upstream cursor out of whatever token the device sends back: Kobo's own, a Calibre-Web
- * Kobo directly before BookOrbit was put in the path sends Kobo's own opaque token. Neither shape
+ * shaped one (ours, or one left by Calibre-Web before BookOrbit was put in the path), or an older
- * may throw: getDelta finds the snapshot by device rather than by token, so a token we cannot read
+ * BookOrbit `PX.` / `PX2.` token. None of them may throw: a token we cannot read costs a store page,
- * costs a store page, never a broken sync.
+ * never a broken sync.
  */
 export function decodeSyncToken(raw: string | undefined): KoboSyncTokenParts {
   const value = raw?.trim();
@@ -41,39 +57,39 @@ export function decodeSyncToken(raw: string | undefined): KoboSyncTokenParts {
   if (value.startsWith(COMPACT_TOKEN_PREFIX)) {
     const rest = value.slice(COMPACT_TOKEN_PREFIX.length);
     const separator = rest.indexOf('.');
-    const id = separator === -1 ? rest : rest.slice(0, separator);
+    return usableCursor(separator === -1 ? '' : rest.slice(separator + 1));
-    const koboSyncToken = separator === -1 ? '' : rest.slice(separator + 1);
-    return {
-      snapshotId: /^\d{1,15}$/.test(id) ? Number(id) : undefined,
-      koboSyncToken: isUsableKoboSyncToken(koboSyncToken) ? koboSyncToken : undefined,
-    };
   }
-  if (!value.startsWith(TOKEN_PREFIX)) {
+  if (value.startsWith(TOKEN_PREFIX)) {
-    return isUsableKoboSyncToken(value) ? { koboSyncToken: value } : {};
+    const decoded = parseBase64Json(value.slice(TOKEN_PREFIX.length));
+    return usableCursor(decoded?.koboSyncToken);
   }
-
+  const calibreWeb = parseBase64Json(value);
-  try {
+  if (calibreWeb && typeof calibreWeb.data === 'object' && calibreWeb.data !== null && !Array.isArray(calibreWeb.data)) {
-    const decoded: unknown = JSON.parse(Buffer.from(value.slice(TOKEN_PREFIX.length), 'base64').toString('utf8'));
+    return usableCursor((calibreWeb.data as Record<string, unknown>).raw_kobo_store_token);
-    if (!decoded || typeof decoded !== 'object' || Array.isArray(decoded)) return {};
-    const { snapshotId, koboSyncToken } = decoded as Record<string, unknown>;
-    const usableKoboToken = typeof koboSyncToken === 'string' && isUsableKoboSyncToken(koboSyncToken) ? koboSyncToken : undefined;
-    return {
-      snapshotId: typeof snapshotId === 'number' && Number.isFinite(snapshotId) ? snapshotId : undefined,
-      koboSyncToken: usableKoboToken,
-    };
-  } catch {
-    return {};
   }
+  return usableCursor(value);
 }
 
 /**
- * Re-stamps a token minted by KoboSyncService with the upstream cursor. Returns the token unchanged
+ * Re-stamps a token minted by KoboSyncService with the upstream cursor, which then travels alone.
- * when there is no upstream cursor to carry, so a sync with store sync off is byte-identical to one
+ * Returns the token unchanged when there is no upstream cursor to carry.
- * from before store sync existed.
  */
 export function withKoboSyncToken(compositeToken: string, koboSyncToken: string | undefined): string {
-  if (!koboSyncToken) return compositeToken;
+  if (!koboSyncToken || !isUsableKoboSyncToken(koboSyncToken)) return compositeToken;
-  const { snapshotId } = decodeSyncToken(compositeToken);
+  return encodeSyncToken(koboSyncToken);
-  if (snapshotId === undefined) return compositeToken;
+}
-  return encodeSyncToken(snapshotId, koboSyncToken);
+
+function usableCursor(value: unknown): KoboSyncTokenParts {
+  return typeof value === 'string' && isUsableKoboSyncToken(value) ? { koboSyncToken: value } : {};
+}
+
+// Kobo's own cursor is dotted base64url, which never parses to a JSON object here.
+function parseBase64Json(value: string): Record<string, unknown> | null {
+  if (!/^[A-Za-z0-9+/]+={0,2}$/.test(value)) return null;
+  try {
+    const decoded: unknown = JSON.parse(Buffer.from(value, 'base64').toString('utf8'));
+    return decoded && typeof decoded === 'object' && !Array.isArray(decoded) ? (decoded as Record<string, unknown>) : null;
+  } catch {
+    return null;
+  }
 }

server/src/modules/kobo/services/kobo-sync.service.test.ts

@@ -1,6 +1,7 @@
 import { createHash } from 'crypto';
 import { sql } from 'drizzle-orm';
 import * as schema from '../../../db/schema';
+import { encodeSyncToken } from './kobo-sync-token';
 import { KoboSyncService } from './kobo-sync.service';
 
 // Recursively counts bound query params (drizzle-orm's `Param` chunks) in a built SQL tree,
@@ -511,7 +512,7 @@ describe('KoboSyncService', () => {
     expect(result).toEqual({
       entitlements: [{ ChangedTag: {} }],
       hasMore: false,
-      syncToken: expect.stringMatching(/^PX\./),
+      syncToken: encodeSyncToken(),
     });
   });
   it('getPageFromSnapshot returns page entitlements for removed/new/changed books', async () => {

server/src/modules/kobo/services/kobo-sync.service.ts

@@ -620,7 +620,7 @@ export class KoboSyncService {
     smartScopeMatchCache: SmartScopeMatchCache,
     resolveEligibleIds: EligibleIdsResolver,
   ): Promise<{ entitlements: unknown[]; hasMore: boolean; syncToken: string }> {
-    const syncToken = encodeSyncToken(snapshotId);
+    const syncToken = encodeSyncToken();
 
     const pending = await this.db
       .select()