feat(sync): three way (kobo, koreader and bookorbit) progress and annotation sync (#338)

Introduces a custom KOReader plugin to synchronize reading progress, stats, and annotations. Adds backend controllers, services, and database schema updates to ingest plugin data. Updates the frontend to display KOReader annotations alongside built-in reader highlights.

Closes #337

packages/types/src/__tests__/reading-session-source-bucket.spec.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  READING_SESSION_SOURCE_BUCKETS,
+  READING_SESSION_SOURCE_BUCKET_LABELS,
+  emptySourceBucketRecord,
+  toReadingSessionSourceBucket,
+} from "../reading-session-source-bucket";
+
+describe("toReadingSessionSourceBucket", () => {
+  it("maps web to bookorbit", () => {
+    expect(toReadingSessionSourceBucket("web")).toBe("bookorbit");
+  });
+
+  it("maps manual to bookorbit", () => {
+    expect(toReadingSessionSourceBucket("manual")).toBe("bookorbit");
+  });
+
+  it("maps koreader to koreader", () => {
+    expect(toReadingSessionSourceBucket("koreader")).toBe("koreader");
+  });
+
+  it("maps kobo to kobo", () => {
+    expect(toReadingSessionSourceBucket("kobo")).toBe("kobo");
+  });
+
+  it("maps null/undefined to bookorbit", () => {
+    expect(toReadingSessionSourceBucket(null)).toBe("bookorbit");
+    expect(toReadingSessionSourceBucket(undefined)).toBe("bookorbit");
+  });
+});
+
+describe("reading session source bucket constants", () => {
+  it("exposes exactly three buckets", () => {
+    expect(READING_SESSION_SOURCE_BUCKETS).toEqual(["bookorbit", "koreader", "kobo"]);
+  });
+
+  it("labels every bucket", () => {
+    expect(READING_SESSION_SOURCE_BUCKET_LABELS).toEqual({
+      bookorbit: "BookOrbit",
+      koreader: "KOReader",
+      kobo: "Kobo",
+    });
+  });
+
+  it("builds a zero-filled record", () => {
+    expect(emptySourceBucketRecord()).toEqual({ bookorbit: 0, koreader: 0, kobo: 0 });
+  });
+});

packages/types/src/annotation.ts

@@ -1,18 +1,90 @@
+export type AnnotationPositionStatus = "exact" | "repaired" | "failed" | "pending";
+
+export const KOBO_HIGHLIGHT_COLORS = [
+  { name: "yellow", label: "Yellow", hex: "#F6F3B3" },
+  { name: "green", label: "Green", hex: "#C6E09E" },
+  { name: "blue", label: "Blue", hex: "#B2E1E8" },
+  { name: "pink", label: "Pink", hex: "#E8AFCF" },
+] as const;
+
+export type KoboHighlightColorName = (typeof KOBO_HIGHLIGHT_COLORS)[number]["name"];
+
+export const KOREADER_HIGHLIGHT_COLORS = [
+  { name: "red", label: "Red", hex: "#FF3300", appHex: "#F87171", koboFallback: "pink" },
+  { name: "orange", label: "Orange", hex: "#FF8800", appHex: "#FB923C", koboFallback: "yellow" },
+  { name: "yellow", label: "Yellow", hex: "#FFFF33", appHex: "#FACC15", koboFallback: "yellow" },
+  { name: "green", label: "Green", hex: "#00AA66", appHex: "#4ADE80", koboFallback: "green" },
+  { name: "olive", label: "Olive", hex: "#88FF77", appHex: "#84CC16", koboFallback: "green" },
+  { name: "cyan", label: "Cyan", hex: "#00FFEE", appHex: "#22D3EE", koboFallback: "blue" },
+  { name: "blue", label: "Blue", hex: "#0066FF", appHex: "#38BDF8", koboFallback: "blue" },
+  { name: "purple", label: "Purple", hex: "#EE00FF", appHex: "#C084FC", koboFallback: "pink" },
+  { name: "gray", label: "Gray", hex: "#808080", appHex: "#9CA3AF", koboFallback: "yellow" },
+] as const satisfies readonly {
+  name: string;
+  label: string;
+  hex: string;
+  appHex: string;
+  koboFallback: KoboHighlightColorName;
+}[];
+
+export type KoreaderHighlightColorName = (typeof KOREADER_HIGHLIGHT_COLORS)[number]["name"];
+
+export const ANNOTATION_HIGHLIGHT_COLORS = [
+  { name: "yellow", label: "Yellow", hex: "#FACC15", koreaderFallback: "yellow", koboFallback: "yellow" },
+  { name: "green", label: "Green", hex: "#4ADE80", koreaderFallback: "green", koboFallback: "green" },
+  { name: "blue", label: "Blue", hex: "#38BDF8", koreaderFallback: "blue", koboFallback: "blue" },
+  { name: "pink", label: "Pink", hex: "#F472B6", koreaderFallback: "purple", koboFallback: "pink" },
+  { name: "orange", label: "Orange", hex: "#FB923C", koreaderFallback: "orange", koboFallback: "yellow" },
+  { name: "red", label: "Red", hex: "#F87171", koreaderFallback: "red", koboFallback: "pink" },
+  { name: "olive", label: "Olive", hex: "#84CC16", koreaderFallback: "olive", koboFallback: "green" },
+  { name: "cyan", label: "Cyan", hex: "#22D3EE", koreaderFallback: "cyan", koboFallback: "blue" },
+  { name: "purple", label: "Purple", hex: "#C084FC", koreaderFallback: "purple", koboFallback: "pink" },
+  { name: "gray", label: "Gray", hex: "#9CA3AF", koreaderFallback: "gray", koboFallback: "yellow" },
+] as const satisfies readonly {
+  name: string;
+  label: string;
+  hex: string;
+  koreaderFallback: KoreaderHighlightColorName;
+  koboFallback: KoboHighlightColorName;
+}[];
+
+export type AnnotationHighlightColorName = (typeof ANNOTATION_HIGHLIGHT_COLORS)[number]["name"];
+
+export const KOREADER_EXACT_HIGHLIGHT_COLORS = [
+  { hex: "#FF3300", label: "KOReader Red" },
+  { hex: "#FF8800", label: "KOReader Orange" },
+  { hex: "#FFFF33", label: "KOReader Yellow" },
+  { hex: "#00AA66", label: "KOReader Green" },
+  { hex: "#88FF77", label: "KOReader Olive" },
+  { hex: "#00FFEE", label: "KOReader Cyan" },
+  { hex: "#0066FF", label: "KOReader Blue" },
+  { hex: "#EE00FF", label: "KOReader Purple" },
+  { hex: "#808080", label: "KOReader Gray" },
+] as const;
+
+export const ANNOTATION_COLOR_FILTER_OPTIONS = [...ANNOTATION_HIGHLIGHT_COLORS, ...KOREADER_EXACT_HIGHLIGHT_COLORS] as const;
+
 export interface AnnotationItem {
   id: number;
   bookId: number;
-  cfi: string;
+  cfi: string | null;
+  jumpFileId: number | null;
+  pageno: number | null;
   text: string;
   color: string;
   style: string;
   note: string | null;
   chapterTitle: string | null;
+  origin: "web" | "koreader" | "kobo";
+  positionStatus: AnnotationPositionStatus | null;
+  chapterIndex: number | null;
   createdAt: string;
 }
 
 export interface AnnotationStats {
   totalHighlights: number;
   colorBreakdown: { color: string; count: number }[];
+  originBreakdown: { origin: AnnotationItem["origin"]; count: number }[];
   chaptersWithHighlights: number;
   highlightsWithNotes: number;
   chapters: string[];
@@ -25,3 +97,58 @@ export interface AnnotationListResponse {
   pageSize: number;
   stats: AnnotationStats;
 }
+
+export interface AnnotationHubItem extends AnnotationItem {
+  bookTitle: string | null;
+  author: string | null;
+  deletedAt: string | null;
+}
+
+export interface AnnotationHubStats {
+  books: number;
+  withNotes: number;
+  originBreakdown: { origin: AnnotationItem["origin"]; count: number }[];
+}
+
+export interface AnnotationHubResponse {
+  items: AnnotationHubItem[];
+  total: number;
+  page: number;
+  pageSize: number;
+  stats: AnnotationHubStats;
+}
+
+export interface AnnotationHubBookFacet {
+  bookId: number;
+  bookTitle: string | null;
+  author: string | null;
+  count: number;
+}
+
+export type AnnotationPositionFormat = "cfi" | "xpointer" | "pdf" | "kobo_span";
+
+export interface AnnotationPositionInfo {
+  format: AnnotationPositionFormat;
+  status: AnnotationPositionStatus;
+  reason: string | null;
+  converterVersion: number | null;
+  updatedAt: string;
+}
+
+export interface AnnotationDeviceSyncInfo {
+  source: "koreader" | "kobo";
+  deviceId: string;
+  deviceName: string | null;
+  lastAppliedVersion: number;
+  upToDate: boolean;
+  deleteAckedAt: string | null;
+  lastSyncedAt: string;
+}
+
+export interface AnnotationSyncDetail {
+  annotationId: number;
+  origin: AnnotationItem["origin"];
+  version: number;
+  positions: AnnotationPositionInfo[];
+  devices: AnnotationDeviceSyncInfo[];
+}

packages/types/src/index.ts

@@ -44,6 +44,7 @@ export * from "./font";
 export * from "./dictionary";
 export * from "./achievement";
 export * from "./reading-session";
+export * from "./reading-session-source-bucket";
 export * from "./annotation";
 export * from "./hardcover";
 export * from "./path";

packages/types/src/kobo.ts

@@ -16,6 +16,7 @@ export interface KoboSyncSettings {
   forceEnableHyphenation: boolean;
   kepubConversionLimitMb: number;
   twoWayProgressSync: boolean;
+  syncBookOrbitAnnotationsToKobo: boolean;
 }
 
 export interface CreateKoboDeviceRequest {
@@ -33,4 +34,5 @@ export interface UpdateKoboSyncSettingsRequest {
   forceEnableHyphenation?: boolean;
   kepubConversionLimitMb?: number;
   twoWayProgressSync?: boolean;
+  syncBookOrbitAnnotationsToKobo?: boolean;
 }

packages/types/src/koreader.ts

@@ -20,11 +20,36 @@ export interface KoreaderBookProgress {
   updatedAt: string;
 }
 
+export interface KoreaderDeviceSweepInfo {
+  deviceId: string;
+  deviceModel: string;
+  pluginVersion: string | null;
+  latestPluginVersion: string | null;
+  updateAvailable: boolean | null;
+  lastSweepAt: string;
+  lastSweepBooksMatched: number;
+  lastSweepPageStats: number;
+  lastSweepAnnotations: number;
+}
+
+export interface KoreaderPluginTotals {
+  matchedBooks: number;
+  trashedAnnotations: number;
+  pendingDeletes: number;
+  failedPositions: number;
+  pageStatEvents: number;
+  annotations: number;
+}
+
 export interface KoreaderSyncStatus {
   credentials: KoreaderCredentials | null;
   devices: KoreaderDeviceInfo[];
   totalSyncedBooks: number;
   lastSyncAt: string | null;
+  latestPluginVersion: string | null;
+  pluginUpdateAvailable: boolean;
+  sweeps: KoreaderDeviceSweepInfo[];
+  pluginTotals: KoreaderPluginTotals;
 }
 
 export interface KoreaderBookSyncInfo {
@@ -55,3 +80,125 @@ export interface TestKoreaderConnectionResult {
   username: string;
   serverUrl: string;
 }
+
+export type KoreaderCatalogSection =
+  | "libraries"
+  | "collections"
+  | "smart-scopes"
+  | "authors"
+  | "series"
+  | "search"
+  | "recent"
+  | "all-books"
+  | "continue-reading";
+
+export type KoreaderCatalogSort = "title" | "author" | "recently_added" | "recently_updated" | "recently_read" | "series";
+
+export type KoreaderCatalogSortOrder = "asc" | "desc";
+
+export type KoreaderCatalogReadStatusFilter = "unread" | "reading" | "finished";
+
+// Read statuses the catalog detail page can set on a book. A subset of the
+// full ReadStatus enum, chosen for reading-device ergonomics.
+export type KoreaderCatalogSettableReadStatus = "want_to_read" | "reading" | "on_hold" | "read" | "abandoned";
+
+export interface KoreaderCatalogSeriesSummary {
+  total: number;
+  finished: number;
+}
+
+export interface KoreaderCatalogEntry {
+  id: string;
+  title: string;
+  section: KoreaderCatalogSection;
+  subtitle?: string | null;
+  count?: number;
+  icon?: string | null;
+  href?: string;
+  booksHref?: string;
+}
+
+export interface KoreaderCatalogFile {
+  id: number;
+  format: string;
+  role: string;
+  sizeBytes: number | null;
+  durationSeconds: number | null;
+  downloadUrl: string;
+}
+
+export interface KoreaderCatalogProgress {
+  fileId: number;
+  percentage: number;
+  koreaderProgress: string | null;
+  updatedAt: string;
+}
+
+export interface KoreaderCatalogBookListItem {
+  id: number;
+  title: string;
+  authors: string[];
+  seriesName: string | null;
+  seriesIndex: number | null;
+  progressPercentage: number | null;
+  readStatus: string | null;
+  formats: string[];
+  hasCover: boolean;
+  thumbnailUrl: string | null;
+  detailUrl: string;
+  addedAt: string;
+  updatedAt: string;
+}
+
+export interface KoreaderCatalogBookDetail extends KoreaderCatalogBookListItem {
+  subtitle: string | null;
+  description: string | null;
+  publisher: string | null;
+  publishedYear: number | null;
+  language: string | null;
+  isbn10: string | null;
+  isbn13: string | null;
+  libraryId: number;
+  libraryName: string;
+  rating: number | null;
+  pageCount: number | null;
+  collections: { id: number; name: string }[];
+  genres: string[];
+  tags: string[];
+  progress: KoreaderCatalogProgress | null;
+  files: KoreaderCatalogFile[];
+}
+
+export interface KoreaderCatalogPage<T> {
+  items: T[];
+  total: number;
+  page: number;
+  size: number;
+  hasNext: boolean;
+  hasPrevious: boolean;
+  nextUrl: string | null;
+  previousUrl: string | null;
+  // Present when the page is scoped to a single series, summarising
+  // read-through progress across the whole series.
+  seriesSummary?: KoreaderCatalogSeriesSummary | null;
+}
+
+export interface KoreaderCatalogSectionResponse {
+  section: KoreaderCatalogSection;
+  items: KoreaderCatalogEntry[];
+  // Pagination is only emitted for large sections (authors, series).
+  page?: number;
+  hasNext?: boolean;
+  hasPrevious?: boolean;
+  nextUrl?: string | null;
+  previousUrl?: string | null;
+  query?: string | null;
+}
+
+export interface KoreaderCatalogReadStatusResult {
+  readStatus: KoreaderCatalogSettableReadStatus;
+}
+
+export interface KoreaderCatalogRatingResult {
+  rating: number | null;
+}

packages/types/src/reading-session-source-bucket.ts

@@ -0,0 +1,22 @@
+import type { ReadingSessionSource } from "./reading-session";
+
+export const READING_SESSION_SOURCE_BUCKETS = ["bookorbit", "koreader", "kobo"] as const;
+export type ReadingSessionSourceBucket = (typeof READING_SESSION_SOURCE_BUCKETS)[number];
+
+export const READING_SESSION_SOURCE_BUCKET_LABELS: Record<ReadingSessionSourceBucket, string> = {
+  bookorbit: "BookOrbit",
+  koreader: "KOReader",
+  kobo: "Kobo",
+};
+
+// web/manual are both native BookOrbit surfaces; null/unknown (and historical Kobo-as-web
+// sessions that predate source tagging) collapse into the BookOrbit bucket by design.
+export function toReadingSessionSourceBucket(source: ReadingSessionSource | null | undefined): ReadingSessionSourceBucket {
+  if (source === "koreader") return "koreader";
+  if (source === "kobo") return "kobo";
+  return "bookorbit";
+}
+
+export function emptySourceBucketRecord(): Record<ReadingSessionSourceBucket, number> {
+  return { bookorbit: 0, koreader: 0, kobo: 0 };
+}

packages/types/src/reading-session.ts

@@ -1,3 +1,8 @@
+import type { ReadingSessionSourceBucket } from "./reading-session-source-bucket";
+
+export const READING_SESSION_SOURCES = ["web", "koreader", "manual", "kobo"] as const;
+export type ReadingSessionSource = (typeof READING_SESSION_SOURCES)[number];
+
 export interface BookReadingSession {
   id: number;
   startedAt: string;
@@ -6,6 +11,13 @@ export interface BookReadingSession {
   progressDelta: number | null;
   endProgress: number | null;
   format: string | null;
+  source: ReadingSessionSource | null;
+}
+
+export interface BookReadingSourceSlice {
+  bucket: ReadingSessionSourceBucket;
+  totalSeconds: number;
+  totalSessions: number;
 }
 
 export interface BookReadingSessionStats {
@@ -15,6 +27,12 @@ export interface BookReadingSessionStats {
   firstSessionAt: string | null;
   lastSessionAt: string | null;
   dailySummary: { day: string; totalMinutes: number }[];
+  paceProgressDelta: number;
+  paceDurationSeconds: number;
+  progressSummary: { day: string; endProgress: number }[];
+  // Reading time/sessions split across the 3 display buckets, ordered by
+  // READING_SESSION_SOURCE_BUCKETS; only buckets with activity are included.
+  bySource: BookReadingSourceSlice[];
 }
 
 export interface BookReadingSessionListResponse {

packages/types/src/statistics.ts

@@ -12,6 +12,7 @@ export type StatisticsChartId =
   | "format-share-over-time"
   | "page-count-distribution"
   | "reading-heatmap"
+  | "reading-source-distribution"
   | "peak-reading-hours"
   | "favorite-reading-days"
   | "completion-timeline"
@@ -78,6 +79,7 @@ export const DEFAULT_LIBRARY_CHART_ORDER: StatisticsChartId[] = [
 
 export const DEFAULT_USER_CHART_ORDER: StatisticsChartId[] = [
   "reading-heatmap",
+  "reading-source-distribution",
   "peak-reading-hours",
   "favorite-reading-days",
   "completion-timeline",

packages/types/src/user-statistics.ts

@@ -1,3 +1,5 @@
+import type { ReadingSessionSourceBucket } from "./reading-session-source-bucket";
+
 export interface UserStatisticsSummary {
   trackedBooks: number;
   startedBooks: number;
@@ -11,6 +13,19 @@ export interface UserDailyReadingStat {
   readingSeconds: number;
   progressDelta: number;
   eventsCount: number;
+  // Populated only by the reading heatmap (per-source tooltip); other daily-stat
+  // consumers leave it undefined.
+  bySource?: Record<ReadingSessionSourceBucket, number>;
+}
+
+export interface UserReadingSourceDistributionSlice {
+  bucket: ReadingSessionSourceBucket;
+  readingSeconds: number;
+}
+
+export interface UserReadingSourceDistribution {
+  totalSeconds: number;
+  slices: UserReadingSourceDistributionSlice[];
 }
 
 export interface UserPeakHourStat {
@@ -18,12 +33,15 @@ export interface UserPeakHourStat {
   readingSeconds: number;
   eventsCount: number;
   byFormat: Record<string, number>;
+  bySource: Record<ReadingSessionSourceBucket, number>;
 }
 
 export interface UserFavoriteDayStat {
   dayOfWeek: number;
   readingSeconds: number;
   eventsCount: number;
+  byFormat: Record<string, number>;
+  bySource: Record<ReadingSessionSourceBucket, number>;
 }
 
 export interface UserCompletionTimelinePoint {
@@ -76,6 +94,7 @@ export interface UserCompletionLatencyDistribution {
 export interface UserGenreReadingTimeItem {
   genre: string;
   readingSeconds: number;
+  bySource: Record<ReadingSessionSourceBucket, number>;
 }
 
 export interface UserReadingSessionTimelineItem {
@@ -83,6 +102,7 @@ export interface UserReadingSessionTimelineItem {
   bookId: number;
   bookTitle: string | null;
   bookFormat: string | null;
+  bookSource: ReadingSessionSourceBucket;
   startedAt: string;
   endedAt: string;
   durationSeconds: number;
@@ -99,6 +119,8 @@ export interface UserReadingSessionTimeline {
 export interface UserReadingPacePoint {
   durationSeconds: number;
   progressDelta: number;
+  bucket: ReadingSessionSourceBucket;
+  format: string;
 }
 
 export interface UserReadingSurvivalPoint {