Migrations/bookorbit
1
0
Fork 0
mirror of https://github.com/bookorbit/bookorbit.git synced 2026-11-03 00:11:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

feat(koreader): add bookmark sync and exchange support

Browse Source 
Add two-way bookmark sync support between KOReader plugin and server. Include migration 0056 for bookmark sync columns, server endpoints and services for bookmark exchange, and KOReader plugin bookmark management with comprehensive unit and E2E tests.
This commit is contained in:
neon
2026-07-28 04:34:46 +00:00
parent e0ca660f2e
commit dbb10b5c2f
47 changed files with 18917 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
koreader-plugin/README.md
+11 -4
View File
@@ -48,23 +48,30 @@ From the dashboard menu, the plugin update check is top-level; from the Tools me
A first sync on a device with years of history uploads in batches of 500 events (roughly 400 requests per 200k events, a few minutes of background work), and is safe to interrupt.
## Two-way highlight sync
## Two-way highlight and bookmark sync
Requires server 0.4+; against an older server the plugin falls back to upload-only automatically.
Requires server 0.4+; against an older server the plugin falls back to upload-only automatically. Bookmark sync additionally requires a server that advertises the `bookmarkSync` capability; on an older one the bookmark paths stay dormant and highlights sync exactly as before.
- Device to web: highlight positions (crengine xpointers) convert to reader CFIs, verified against the highlighted text, so they render as native, editable highlights on the web.
- Web to device: applied on book open (with auto sync on), on manual sync, and during the full sweep for closed books, with positions re-verified/re-anchored against the text.
- Deletions go both ways through a trash/restore flow, so nothing is lost outright.
- Identity is the highlight's creation datetime plus position, so extending a highlight on the device is recognized as a move, not a delete-and-recreate.
- Styles map across formats (e.g. squiggly <-> underline, invert, named colors <-> hex).
- Toggle via Settings > Sync > "Two-way highlight sync" (uploads keep working either way).
- Toggle via Settings > Sync > "Two-way highlights & bookmarks" (uploads keep working either way).
Position-only bookmarks (dogears) sync the same way:
- Device to web: the dogear xpointer converts to a reader CFI and shows up in the web reader's bookmark list. Its title is the note you attached, else the chapter, else the page.
- Web to device: applied on book open, on manual sync, and during the full sweep for closed books. The position is only placed when it resolves; an entry that does not is retried on the next exchange.
- Deletions and device-side renames propagate; the web has no rename surface yet, so edits flow device to web only.
- A dogear has no highlighted text to re-anchor against, so a converted position can sit slightly off where a highlight would have been repaired exactly.
## Limitations
- Web highlight changes reach a closed book only via a manual sweep or its next open.
- Web-created PDF highlights aren't supported (device PDF highlights sync up but aren't drawn over the PDF).
- Books with reading stats but no sidecar path sync reading time only, until the full sweep covers them.
- Position-only bookmarks (no highlighted text) don't sync.
- PDF bookmarks don't sync in either direction (the web has no PDF bookmark surface).
- A device clock far in the past can delay edit detection for web-modified highlights.
- The full sweep is manual-only; books you never reopen only sync when you run it.
- A cloned device with a different KOReader `device_id` double-counts reading time; clones keeping the same `device_id` deduplicate naturally.
koreader-plugin/bookorbit.koplugin/bookorbit_api.lua
+8
View File
@@ -517,6 +517,14 @@ function BookOrbitApi:exchangeAck(books)
return self:request("POST", "/koreader/plugin/annotations/exchange-ack", self:withDevice({ books = books }))
end
function BookOrbitApi:exchangeBookmarks(books)
return self:request("POST", "/koreader/plugin/bookmarks/exchange", self:withDevice({ books = books }))
end
function BookOrbitApi:exchangeBookmarksAck(books)
return self:request("POST", "/koreader/plugin/bookmarks/exchange-ack", self:withDevice({ books = books }))
end
function BookOrbitApi:uploadBookStates(books)
return self:request("POST", "/koreader/plugin/book-states", self:withDevice({ books = books }))
end
koreader-plugin/bookorbit.koplugin/bookorbit_book_sync.lua
+94 -10
View File
@@ -22,6 +22,7 @@ local _ = require("gettext")
local BookOrbitAnnotations = require("bookorbit_annotations")
local BookOrbitApi = require("bookorbit_api")
local BookOrbitBookmarks = require("bookorbit_bookmarks")
local BookOrbitHighlightSummary = require("bookorbit_highlight_summary")
local BookOrbitSidecar = require("bookorbit_sidecar")
local BookOrbitState = require("bookorbit_state")
@@ -73,8 +74,9 @@ function BookOrbitBookSync.capture(plugin)
if not digest then return nil end
local started_ms = nowMs()
local annotations, ann_max, ann_signature =
BookOrbitSidecar.normalizeAnnotations(ui.annotation and ui.annotation.annotations)
local live_annotations = ui.annotation and ui.annotation.annotations
local annotations, ann_max, ann_signature = BookOrbitSidecar.normalizeAnnotations(live_annotations)
local bookmarks, _bm_max, bm_signature = BookOrbitSidecar.normalizeBookmarks(live_annotations)
local summary = BookOrbitSidecar.normalizeSummary(ui.doc_settings:readSetting("summary"))
local stats = ui.statistics
@@ -119,6 +121,8 @@ function BookOrbitBookSync.capture(plugin)
ann_count = #annotations,
ann_max_datetime = ann_max,
ann_signature = ann_signature,
bookmarks = bookmarks,
bm_signature = bm_signature,
mtime_at_capture = file and BookOrbitSidecar.sidecarMtime(file) or nil,
ts = ts,
}
@@ -216,7 +220,7 @@ local function finish(ctx, err)
text = text .. "\n" .. _("BookOrbit server needs an update for two-way highlights.")
end
if ctx.highlight_disabled_reported then
text = text .. "\n" .. _("Two-way highlight sync is disabled.")
text = text .. "\n" .. _("Two-way highlight and bookmark sync is disabled.")
end
if ctx.skip_progress then
text = text .. "\n" .. _("Progress was not changed.")
@@ -266,7 +270,7 @@ local function step(ctx, fn)
end)
end
local stepMatch, stepStats, stepAnnotations, stepAnnotationsLegacy, stepState, stepProgress
local stepMatch, stepStats, stepAnnotations, stepAnnotationsLegacy, stepBookmarks, stepState, stepProgress
stepMatch = function(ctx)
if ctx.acknowledged.match then
@@ -374,7 +378,7 @@ end
-- while the book is open, sidecar on the close path) and acks them.
stepAnnotations = function(ctx)
if ctx.acknowledged.annotations then
return step(ctx, stepState)
return step(ctx, stepBookmarks)
end
local book = ctx.state:getBook(ctx.snap.digest)
if not book then return finish(ctx, "unmatched") end
@@ -401,7 +405,7 @@ stepAnnotations = function(ctx)
and BookOrbitAnnotations.canSkipExchange(book, ctx.snap.ann_signature) then
logger.dbg("BookOrbit: book sync annotation exchange skipped, local set unchanged")
if not acknowledge(ctx, "annotations", false) then return end
return step(ctx, stepState)
return step(ctx, stepBookmarks)
end
if pending and apply_mode == "sidecar" then
@@ -445,7 +449,7 @@ stepAnnotations = function(ctx)
end
ctx.had_errors = true
ctx.highlight_summary = BookOrbitHighlightSummary.add(ctx.highlight_summary, { had_errors = true })
return step(ctx, stepState)
return step(ctx, stepBookmarks)
end
if result.had_errors then ctx.had_errors = true end
@@ -469,7 +473,7 @@ stepAnnotations = function(ctx)
return
end
end
return step(ctx, stepState)
return step(ctx, stepBookmarks)
end
-- One-way upload for servers without the 0.4 exchange endpoint.
@@ -498,7 +502,7 @@ stepAnnotationsLegacy = function(ctx)
book.annCount = ctx.snap.ann_count
if not acknowledge(ctx, "annotations") then return end
end
return step(ctx, stepState)
return step(ctx, stepBookmarks)
end
-- Consumed through a head cursor: removing from the front of a Lua array
@@ -518,7 +522,7 @@ stepAnnotationsLegacy = function(ctx)
ctx.highlight_summary = BookOrbitHighlightSummary.add(ctx.highlight_summary, { had_errors = true })
if isTransportError(err) then return finish(ctx, "network") end
logger.dbg("BookOrbit: book sync annotations upload failed:", err)
return step(ctx, stepState)
return step(ctx, stepBookmarks)
end
if isUnmatched(body, ctx.snap.digest) then
ctx.state:setUnmatched(ctx.snap.digest)
@@ -530,6 +534,86 @@ stepAnnotationsLegacy = function(ctx)
return step(ctx, stepAnnotationsLegacy)
end
-- Two-way dogear exchange. It shares the annotations phase acknowledgement, so
-- an outbox entry queued before bookmark sync existed still drains: only the
-- free-form remote_pending kind is new.
stepBookmarks = function(ctx)
if not ctx.annotation_sync or not BookOrbitBookmarks.enabled(ctx.client, true) then
return step(ctx, stepState)
end
local book = ctx.state:getBook(ctx.snap.digest)
if not book then return finish(ctx, "unmatched") end
local apply_mode = "skip"
if (ctx.reason == "close" or ctx.reason == "suspend" or ctx.reason == "recovery")
and ctx.snap.file and lfs.attributes(ctx.snap.file, "mode") == "file" then
apply_mode = "sidecar"
elseif ctx.reason == "manual" and ctx.plugin and ctx.plugin.ui and ctx.plugin.ui.document then
apply_mode = "live"
end
local pending = ctx.remote_pending and ctx.remote_pending.bookmarks
if not pending and not ctx.interactive
and BookOrbitBookmarks.canSkipExchange(book, ctx.snap.bm_signature) then
logger.dbg("BookOrbit: book sync bookmark exchange skipped, local set unchanged")
return step(ctx, stepState)
end
if pending and apply_mode == "sidecar" then
local applied, deleted = BookOrbitBookmarks.applySidecar(ctx.snap.file, pending.to_apply or {})
local ack_body, ack_err = ctx.client:exchangeBookmarksAck({
{ hash = ctx.snap.digest, applied = applied, deleted = deleted },
})
if not ack_body then
if isAuthError(ack_err) then return finish(ctx, "auth") end
return finish(ctx, "network")
end
if ctx.on_remote_applied then
local ok, cleared = pcall(ctx.on_remote_applied, "bookmarks")
if not ok or cleared == false then return finish(ctx, "outbox") end
end
ctx.remote_pending.bookmarks = nil
end
local result, err = BookOrbitBookmarks.exchangeBook({
client = ctx.client,
state = ctx.state,
digest = ctx.snap.digest,
bookmarks = ctx.snap.bookmarks,
bm_signature = ctx.snap.bm_signature,
apply_mode = apply_mode,
ui = apply_mode == "live" and ctx.plugin.ui or nil,
file = ctx.snap.file,
})
if not result then
if err == "auth" then return finish(ctx, "auth") end
if err == "network" then
ctx.had_errors = true
return finish(ctx, "network")
end
if err == "unmatched" then return finish(ctx, "unmatched") end
if err == "unsupported_server" then
BookOrbitBookmarks.markUnsupported(ctx.client)
return step(ctx, stepState)
end
ctx.had_errors = true
ctx.highlight_summary = BookOrbitHighlightSummary.addBookmarks(ctx.highlight_summary, { had_errors = true })
return step(ctx, stepState)
end
if result.had_errors then ctx.had_errors = true end
ctx.counts.bookmarks = (ctx.counts.bookmarks or 0) + result.uploaded
ctx.highlight_summary = BookOrbitHighlightSummary.addBookmarks(ctx.highlight_summary, result)
if not result.had_errors and result.remote_pending and ctx.on_remote_pending then
local ok, recorded = pcall(ctx.on_remote_pending, "bookmarks", result.remote_pending)
if not ok or recorded == false then
ctx.had_errors = true
return finish(ctx, "outbox")
end
end
return step(ctx, stepState)
end
stepState = function(ctx)
if ctx.acknowledged.state then
return step(ctx, stepProgress)
koreader-plugin/bookorbit.koplugin/bookorbit_bookmarks.lua
+427
View File
@@ -0,0 +1,427 @@
--[[--
Two-way exchange of position-only bookmarks (dogears) with BookOrbit.
Mirrors bookorbit_annotations in miniature. A dogear has no highlighted text,
so there is nothing to verify a repaired position against: the device checks
that the server's xpointer resolves and otherwise reports the entry as failed.
Why the whole local set is uploaded instead of a datetime delta: KOReader does
not stamp datetime_updated when the user edits a dogear note, so a watermark
would silently swallow every rename. Dogears are rare per book, the per-book
skip stamp keeps unchanged books off the wire entirely, and the server dedupes
by identity key, so re-sending the set costs one request and loses nothing.
Apply modes:
- live: the book is open; entries go through ReaderAnnotation so the dogear
indicator, the bookmark list and the footer stay coherent.
- sidecar: the book is closed; entries are merged into the sidecar annotations
table and annotations_externally_modified is set, so KOReader re-validates
and re-sorts on next open.
- skip: upload-only (suspend path); pending server changes stay queued.
]]
local DocSettings = require("docsettings")
local Event = require("ui/event")
local UIManager = require("ui/uimanager")
local T = require("ffi/util").template
local _ = require("gettext")
local logger = require("logger")
local md5 = require("ffi/sha2").md5
local BookOrbitCapabilities = require("bookorbit_capabilities")
local BookOrbitSidecar = require("bookorbit_sidecar")
local MAX_KEYS_PER_BOOK = 500
local MAX_PULL_ROUNDS = 10
local UPLOAD_CHUNK = 50
local BookOrbitBookmarks = {}
BookOrbitBookmarks.CAPABILITY = "bookmarkSync"
-- Same bound as the highlight exchange: it is the only channel that delivers
-- web-created bookmarks, so a drained lifecycle entry may not skip forever.
BookOrbitBookmarks.EXCHANGE_MAX_AGE = 6 * 3600
-- Bookmark sync rides the existing two-way toggle and only runs against a
-- server that advertises the route, so a new plugin on an old server stays
-- dormant instead of collecting 404s.
function BookOrbitBookmarks.enabled(client, annotation_sync)
if not annotation_sync or not client then return false end
return BookOrbitCapabilities.supports(client, BookOrbitBookmarks.CAPABILITY) == true
end
-- Records a confirmed 404 on the bookmark route so the rest of the session
-- stops asking, matching how the other capability-gated routes downgrade.
function BookOrbitBookmarks.markUnsupported(client)
if not client then return end
BookOrbitCapabilities.markUnsupported(client, BookOrbitBookmarks.CAPABILITY)
end
function BookOrbitBookmarks.rememberExchanged(book, signature, now)
if not book or type(signature) ~= "string" or signature == "" then return end
book.bmSignature = signature
book.bmExchangedAt = now or os.time()
end
function BookOrbitBookmarks.canSkipExchange(book, signature, now)
if not book then return false end
if type(signature) ~= "string" or signature == "" then return false end
if book.bmSignature ~= signature then return false end
local exchanged_at = book.bmExchangedAt
if type(exchanged_at) ~= "number" then return false end
now = now or os.time()
if exchanged_at > now then return false end
return (now - exchanged_at) <= BookOrbitBookmarks.EXCHANGE_MAX_AGE
end
function BookOrbitBookmarks.buildKey(datetime, pos)
return md5(datetime .. "|" .. pos)
end
function BookOrbitBookmarks.collectKeys(normalized)
local keys = {}
for _, bookmark in ipairs(normalized) do
table.insert(keys, { k = BookOrbitBookmarks.buildKey(bookmark.datetime, bookmark.pos), dt = bookmark.datetime })
end
return keys
end
local function isDogear(annotation)
return type(annotation) == "table" and annotation.drawer == nil and type(annotation.page) == "string"
end
-- Local identity of a dogear: its position is the only stable handle, and the
-- server addresses deletes by the key it was acked with.
local function findByKey(annotations, key, datetime)
for index, annotation in ipairs(annotations) do
if isDogear(annotation) and type(annotation.datetime) == "string" then
if BookOrbitBookmarks.buildKey(annotation.datetime, annotation.page) == key then
return index, annotation
end
end
end
if type(datetime) ~= "string" or datetime == "" then return nil end
for index, annotation in ipairs(annotations) do
if isDogear(annotation) and annotation.datetime == datetime then
return index, annotation
end
end
end
-- A datetime is required as well as the position: the ack has to report a local
-- identity, and md5(datetime|pos) cannot be built without one.
local function findByPos(annotations, pos)
for index, annotation in ipairs(annotations) do
if isDogear(annotation) and annotation.page == pos and type(annotation.datetime) == "string" then
return index, annotation
end
end
end
local function ackApplied(entry, opts)
opts = opts or {}
return {
serverId = entry.serverId,
status = opts.failed and "failed" or "applied",
key = opts.key,
datetime = opts.datetime,
pos = opts.pos,
}
end
local function ackExisting(entry, annotation)
return ackApplied(entry, {
key = BookOrbitBookmarks.buildKey(annotation.datetime, annotation.page),
datetime = annotation.datetime,
pos = annotation.page,
})
end
-- The server stores one title per bookmark. A dogear shows its chapter and its
-- note, so a title that already matches the chapter needs no note; anything
-- else is the user's own label and belongs in the note. Both shapes upload back
-- as the same title, so devices and the web converge either way.
local function applyTitleFields(item, title, chapter)
if chapter and chapter ~= "" then
item.chapter = chapter
-- Matches what ReaderBookmark writes for a device-created dogear.
item.text = T(_("in %1"), chapter)
item.note = (title and title ~= "" and title ~= chapter) and title or nil
else
item.note = (title and title ~= "") and title or nil
end
end
local function chapterForPos(ui, pos)
if not ui or not ui.toc or not ui.toc.getTocTitleByPage then return nil end
local ok, chapter = pcall(ui.toc.getTocTitleByPage, ui.toc, pos)
if not ok or type(chapter) ~= "string" or chapter == "" then return nil end
return chapter
end
local function refreshDogear(ui)
if not ui or not ui.bookmark then return end
local ok, page = pcall(ui.bookmark.getCurrentPageNumber, ui.bookmark)
if ok and page ~= nil then
pcall(ui.bookmark.setDogearVisibility, ui.bookmark, page)
end
end
-- Applies server changes to the OPEN book through ReaderAnnotation.
function BookOrbitBookmarks.applyLive(ui, to_apply)
local applied, deleted = {}, {}
local document = ui.document
local annotations = ui.annotation.annotations
local touched, deleted_touched = 0, 0
for _, entry in ipairs(to_apply.add or {}) do
local existing_index, existing = findByPos(annotations, entry.pos)
if existing_index then
table.insert(applied, ackExisting(entry, existing))
else
local ok, resolves = pcall(document.isXPointerInDocument, document, entry.pos)
if not (ok and resolves) then
logger.dbg("BookOrbit: bookmark apply failed, position does not resolve:", entry.serverId)
table.insert(applied, ackApplied(entry, { failed = true }))
else
local item = { page = entry.pos }
applyTitleFields(item, entry.title, chapterForPos(ui, entry.pos))
local index = ui.annotation:addItem(item)
touched = touched + 1
ui:handleEvent(Event:new("AnnotationsModified", { item, index_modified = index }))
table.insert(applied, ackApplied(entry, {
key = BookOrbitBookmarks.buildKey(item.datetime, item.page),
datetime = item.datetime,
pos = item.page,
}))
end
end
end
for _, entry in ipairs(to_apply.delete or {}) do
local index = findByKey(annotations, entry.key, entry.datetime)
if index then
ui.bookmark:removeItemByIndex(index)
touched = touched + 1
deleted_touched = deleted_touched + 1
end
-- Missing entries were already deleted locally; ack either way.
table.insert(deleted, { serverId = entry.serverId, status = "applied" })
end
if touched > 0 then
refreshDogear(ui)
UIManager:setDirty("all", "ui")
end
return applied, deleted, touched, deleted_touched
end
-- Merges server changes into the sidecar of a CLOSED book and raises
-- annotations_externally_modified so KOReader re-validates and sorts on open.
function BookOrbitBookmarks.applySidecar(file, to_apply)
local applied, deleted = {}, {}
local doc_settings = DocSettings:open(file)
local annotations = doc_settings:readSetting("annotations") or {}
local touched, deleted_touched = 0, 0
for _, entry in ipairs(to_apply.add or {}) do
local existing_index, existing = findByPos(annotations, entry.pos)
if existing_index then
table.insert(applied, ackExisting(entry, existing))
else
-- No document is loaded, so the position cannot be verified and no
-- chapter can be looked up; KOReader drops an unresolvable entry on
-- the next open, and the server re-pushes it after that.
local item = { page = entry.pos, datetime = os.date("%Y-%m-%d %H:%M:%S") }
if type(entry.pageno) == "number" then item.pageno = entry.pageno end
applyTitleFields(item, entry.title, nil)
table.insert(annotations, item)
touched = touched + 1
table.insert(applied, ackApplied(entry, {
key = BookOrbitBookmarks.buildKey(item.datetime, item.page),
datetime = item.datetime,
pos = item.page,
}))
end
end
for _, entry in ipairs(to_apply.delete or {}) do
local index = findByKey(annotations, entry.key, entry.datetime)
if index then
table.remove(annotations, index)
touched = touched + 1
deleted_touched = deleted_touched + 1
end
table.insert(deleted, { serverId = entry.serverId, status = "applied" })
end
if touched > 0 then
doc_settings:saveSetting("annotations", annotations)
doc_settings:makeTrue("annotations_externally_modified")
doc_settings:flush()
end
return applied, deleted, touched, deleted_touched
end
local function hasPending(to_apply)
return to_apply and (#(to_apply.add or {}) > 0 or #(to_apply.delete or {}) > 0)
end
--[[--
Full bookmark exchange for one matched book. Blocking; callers run it from a
step machine or a scheduled task.
opts:
- client: BookOrbitApi instance
- state: BookOrbitState instance (book must already be matched)
- digest: partial md5 of the book file
- bookmarks: normalized bookmark list (live or sidecar source)
- bm_signature: change signal for that list, recorded on a complete exchange
- apply_mode: "live" | "sidecar" | "skip"
- ui: ReaderUI (required for apply_mode "live")
- file: book file path (required for apply_mode "sidecar")
Returns a result table { uploaded, applied, deleted, failed, had_errors } or
nil, err for auth/network level failures.
]]
function BookOrbitBookmarks.exchangeBook(opts)
local book = opts.state:getBook(opts.digest)
if not book then return nil, "unmatched" end
local bookmarks = opts.bookmarks or {}
local keys = BookOrbitBookmarks.collectKeys(bookmarks)
local keys_complete = #keys <= MAX_KEYS_PER_BOOK
if not keys_complete then keys = {} end
local result = { uploaded = 0, applied = 0, deleted = 0, failed = 0, had_errors = false }
local response
local first_request = true
local upload_failed = false
local cursor = 1
repeat
local chunk = {}
while cursor <= #bookmarks and #chunk < UPLOAD_CHUNK do
table.insert(chunk, bookmarks[cursor])
cursor = cursor + 1
end
local body, err = opts.client:exchangeBookmarks({
{
hash = opts.digest,
keys = first_request and keys or {},
keysComplete = first_request and keys_complete or false,
changes = chunk,
},
})
if not body then
if err == 401 or err == 403 then return nil, "auth" end
if err == 404 then return nil, "unsupported_server" end
if type(err) ~= "number" then return nil, "network" end
logger.dbg("BookOrbit: bookmark exchange failed:", err)
result.had_errors = true
upload_failed = true
break
end
for _, hash in ipairs(body.unmatched or {}) do
if hash == opts.digest then
opts.state:setUnmatched(opts.digest)
return nil, "unmatched"
end
end
response = body.results and body.results[1] or nil
result.uploaded = result.uploaded + #chunk
first_request = false
until cursor > #bookmarks
-- Recorded only when the complete key set went out and nothing was left
-- undelivered: the skip rule reads this stamp.
local function rememberComplete()
if upload_failed or result.had_errors or result.failed > 0 then return end
if not keys_complete then return end
BookOrbitBookmarks.rememberExchanged(book, opts.bm_signature)
end
if opts.apply_mode == "skip" then
if response and hasPending(response.toApply) then
result.remote_pending = {
to_apply = response.toApply,
more = response.more == true,
}
else
rememberComplete()
end
return result
end
local rounds = 0
local pull_complete = true
while response and hasPending(response.toApply) do
if rounds >= MAX_PULL_ROUNDS then
pull_complete = false
break
end
rounds = rounds + 1
local applied, deleted, touched, deleted_touched
if opts.apply_mode == "live" and opts.ui and opts.ui.document then
applied, deleted, touched, deleted_touched = BookOrbitBookmarks.applyLive(opts.ui, response.toApply)
elseif opts.apply_mode == "sidecar" and opts.file then
applied, deleted, touched, deleted_touched = BookOrbitBookmarks.applySidecar(opts.file, response.toApply)
else
pull_complete = false
break
end
deleted_touched = deleted_touched or 0
result.applied = result.applied + math.max(0, touched - deleted_touched)
result.deleted = result.deleted + deleted_touched
for _, entry in ipairs(applied) do
if entry.status == "failed" then result.failed = result.failed + 1 end
end
local ack_body, ack_err = opts.client:exchangeBookmarksAck({
{ hash = opts.digest, applied = applied, deleted = deleted },
})
if not ack_body then
logger.dbg("BookOrbit: bookmark exchange ack failed:", ack_err)
result.had_errors = true
break
end
if not response.more then break end
local body, err = opts.client:exchangeBookmarks({
{ hash = opts.digest, keys = {}, keysComplete = false, changes = {} },
})
if not body then
logger.dbg("BookOrbit: bookmark exchange follow-up failed:", err)
result.had_errors = true
break
end
response = body.results and body.results[1] or nil
end
if pull_complete then rememberComplete() end
return result
end
-- Convenience wrapper for the open book: collects live dogears, exchanges and
-- applies in place.
function BookOrbitBookmarks.exchangeOpenBook(opts)
local ui = opts.ui
if not ui or not ui.document then return nil, "no_document" end
-- Paging documents store page numbers, which normalizeBookmarks drops, so a
-- PDF simply produces an empty set instead of needing its own branch.
local bookmarks, _max, signature =
BookOrbitSidecar.normalizeBookmarks(ui.annotation and ui.annotation.annotations)
return BookOrbitBookmarks.exchangeBook({
client = opts.client,
state = opts.state,
digest = opts.digest,
bookmarks = bookmarks,
bm_signature = signature,
apply_mode = "live",
ui = ui,
})
end
return BookOrbitBookmarks
koreader-plugin/bookorbit.koplugin/bookorbit_highlight_diagnostics.lua
+16
View File
@@ -26,6 +26,22 @@ function HighlightDiagnostics.lastSyncText(summary, format_time)
return text
end
function HighlightDiagnostics.lastBookmarkSyncText(summary, format_time)
if type(summary) ~= "table" then return _("never") end
local uploaded = count(summary.bm_uploaded)
local applied = count(summary.bm_applied)
local deleted = count(summary.bm_deleted)
local failed = count(summary.bm_failed)
if uploaded + applied + deleted + failed == 0 then return _("none") end
local text = T(_("%1 uploaded, %2 applied, %3 deleted, %4 failed"), uploaded, applied, deleted, failed)
local at = tonumber(summary.at)
if at and at > 0 then
local when = format_time and format_time(at) or os.date("%H:%M", at)
text = T(_("%1 at %2"), text, when)
end
return text
end
function HighlightDiagnostics.retryText(status)
if type(status) == "table" and status.pending then
return _("pending")
koreader-plugin/bookorbit.koplugin/bookorbit_highlight_summary.lua
+49 -2
View File
@@ -19,6 +19,15 @@ local COUNT_FIELDS = {
"touched_books",
}
-- Bookmarks are counted separately from highlights: they are a different user
-- surface, and folding them together would make "3 applied" unreadable.
local BOOKMARK_COUNT_FIELDS = {
"bm_uploaded",
"bm_applied",
"bm_deleted",
"bm_failed",
}
local ACTIONABLE_ERRORS = {
auth = true,
network = true,
@@ -43,6 +52,9 @@ function HighlightSummary.normalize(summary)
for _, field in ipairs(COUNT_FIELDS) do
normalized[field] = numberValue(summary[field])
end
for _, field in ipairs(BOOKMARK_COUNT_FIELDS) do
normalized[field] = numberValue(summary[field])
end
return normalized
end
@@ -66,12 +78,35 @@ function HighlightSummary.add(summary, result, opts)
return summary
end
-- Bookmark exchange results share the highlight result shape, so they are
-- folded into their own counters rather than the highlight ones.
function HighlightSummary.addBookmarks(summary, result)
summary = HighlightSummary.normalize(summary)
result = result or {}
summary.bm_uploaded = summary.bm_uploaded + numberValue(result.uploaded)
summary.bm_applied = summary.bm_applied + numberValue(result.applied)
summary.bm_deleted = summary.bm_deleted + numberValue(result.deleted)
summary.bm_failed = summary.bm_failed + numberValue(result.failed)
if result.had_errors and numberValue(result.failed) == 0 then
summary.bm_failed = summary.bm_failed + 1
end
return summary
end
function HighlightSummary.hasBookmarkCounts(summary)
summary = HighlightSummary.normalize(summary)
for _, field in ipairs(BOOKMARK_COUNT_FIELDS) do
if summary[field] > 0 then return true end
end
return false
end
function HighlightSummary.hasCounts(summary)
summary = HighlightSummary.normalize(summary)
for _, field in ipairs(COUNT_FIELDS) do
if summary[field] > 0 then return true end
end
return false
return HighlightSummary.hasBookmarkCounts(summary)
end
function HighlightSummary.hasRemoteChanges(summary)
@@ -79,10 +114,15 @@ function HighlightSummary.hasRemoteChanges(summary)
return summary.applied + summary.deleted > 0
end
function HighlightSummary.hasRemoteBookmarkChanges(summary)
summary = HighlightSummary.normalize(summary)
return summary.bm_applied + summary.bm_deleted > 0
end
function HighlightSummary.actionableError(summary, err)
if ACTIONABLE_ERRORS[err] then return err end
summary = HighlightSummary.normalize(summary)
if summary.failed > 0 then return "partial_failure" end
if summary.failed > 0 or summary.bm_failed > 0 then return "partial_failure" end
end
function HighlightSummary.message(summary)
@@ -92,6 +132,13 @@ function HighlightSummary.message(summary)
if summary.failed > 0 or summary.skipped > 0 then
text = text .. "\n" .. T(_("Failed: %1. Skipped: %2."), summary.failed, summary.skipped)
end
if HighlightSummary.hasBookmarkCounts(summary) then
text = text .. "\n" .. T(_("Bookmarks: %1 uploaded, %2 applied, %3 deleted."),
summary.bm_uploaded, summary.bm_applied, summary.bm_deleted)
if summary.bm_failed > 0 then
text = text .. "\n" .. T(_("Bookmarks failed: %1."), summary.bm_failed)
end
end
if summary.touched_books > 0 then
text = text .. "\n" .. T(_("%1 closed book(s) updated."), summary.touched_books)
end
koreader-plugin/bookorbit.koplugin/bookorbit_main_menu.lua
+5 -3
View File
@@ -189,7 +189,7 @@ function MainMenu:diagnosticsRows()
table.insert(rows, "----------------------------")
table.insert(rows, { _("Auto sync"), formatBool(self.settings.auto_sync) })
table.insert(rows, { _("Two-way highlights"), formatBool(self.settings.annotation_sync) })
table.insert(rows, { _("Two-way highlights & bookmarks"), formatBool(self.settings.annotation_sync) })
table.insert(rows, { _("Open highlights"), open_highlight_text })
table.insert(rows, { _("Highlight apply retry"), BookOrbitHighlightDiagnostics.retryText(self.open_highlight_retry_status) })
table.insert(rows, { _("Skip offline auto-sync"), formatBool(self.settings.skip_sync_when_offline) })
@@ -206,6 +206,8 @@ function MainMenu:diagnosticsRows()
table.insert(rows, "----------------------------")
table.insert(rows, { _("Last sync"), recordText(self.settings.last_sync) })
table.insert(rows, { _("Last highlight sync"), highlightRecordText(self.settings.last_highlight_sync) })
table.insert(rows, { _("Last bookmark sync"),
BookOrbitHighlightDiagnostics.lastBookmarkSyncText(self.settings.last_highlight_sync, formatShortTime) })
table.insert(rows, { _("Last error"), recordText(self.settings.last_error) })
table.insert(rows, "----------------------------")
if has_open_book then
@@ -452,9 +454,9 @@ If set to 0, updating progress based on page turns will be disabled.]]),
end,
})
table.insert(items, {
text = _("Two-way highlight sync"),
text = _("Two-way highlights & bookmarks"),
checked_func = function() return self.settings.annotation_sync end,
help_text = _([[Also applies highlights, notes and deletions made in BookOrbit to this device: on book open, after the manual book sync, and during the full sweep for closed books. Turning this off keeps uploads only.]]),
help_text = _([[Also applies highlights, notes, bookmarks (dogears) and deletions made in BookOrbit to this device: on book open, after the manual book sync, and during the full sweep for closed books. Turning this off keeps uploads only. Bookmark sync needs a server that supports it, and PDF bookmarks never sync.]]),
callback = function()
self.settings.annotation_sync = not self.settings.annotation_sync
end,
koreader-plugin/bookorbit.koplugin/bookorbit_sidecar.lua
+54 -2
View File
@@ -117,6 +117,53 @@ function BookOrbitSidecar.normalizeAnnotations(raw)
string.format("%d:%s:%d:%d", #annotations, max_datetime, sum, mix)
end
-- Position-only bookmarks carry no text, so a rename is only visible in the
-- fields themselves. KOReader also leaves datetime_updated untouched when the
-- user edits a dogear note, which is why the note and chapter are hashed here
-- and why the exchange uploads the whole set rather than a datetime delta.
local function bookmarkHash(entry)
local pos = entry.pos or ""
local key = entry.datetime .. "|" .. #pos .. "|" .. pos:sub(1, 24) .. "|" .. pos:sub(-24)
.. "|" .. (entry.datetimeUpdated or "") .. "|" .. (entry.note or "") .. "|" .. (entry.chapter or "")
local hash = 5381
for index = 1, #key do
hash = (hash * 33 + key:byte(index)) % 4294967296
end
return hash
end
-- Returns normalized position-only bookmarks (dogears), the max effective
-- datetime and a signature that changes whenever the local set changes.
-- Paging documents store a page number instead of an xpointer; those are out of
-- scope, so only string positions are kept.
function BookOrbitSidecar.normalizeBookmarks(raw)
local bookmarks = {}
local max_datetime = ""
local sum, mix = 0, 0
for _, a in ipairs(raw or {}) do
if a.drawer == nil and isDeviceDatetime(a.datetime) and type(a.page) == "string" and a.page ~= "" then
local entry = {
datetime = a.datetime,
datetimeUpdated = isDeviceDatetime(a.datetime_updated) and a.datetime_updated or nil,
pos = truncate(a.page, 4000),
pageno = type(a.pageno) == "number" and math.floor(a.pageno) or nil,
chapter = truncate(a.chapter, 500),
note = truncate(a.note, 500),
}
table.insert(bookmarks, entry)
local effective = entry.datetimeUpdated or entry.datetime
if effective > max_datetime then
max_datetime = effective
end
local hash = bookmarkHash(entry)
sum = (sum + hash) % 4294967296
mix = (mix + hash * (hash % 8191 + 1)) % 4294967296
end
end
return bookmarks, max_datetime,
string.format("%d:%s:%d:%d", #bookmarks, max_datetime, sum, mix)
end
-- Validates a summary table (sidecar or live doc_settings reference) into a
-- fresh plain table; never returns the input reference.
function BookOrbitSidecar.normalizeSummary(summary)
@@ -299,8 +346,9 @@ function BookOrbitSidecar.extract(file)
local doc_settings = DocSettings:open(file)
local summary = BookOrbitSidecar.normalizeSummary(doc_settings:readSetting("summary"))
local annotations, max_datetime, signature =
BookOrbitSidecar.normalizeAnnotations(doc_settings:readSetting("annotations"))
local raw_annotations = doc_settings:readSetting("annotations")
local annotations, max_datetime, signature = BookOrbitSidecar.normalizeAnnotations(raw_annotations)
local bookmarks, bm_max_datetime, bm_signature = BookOrbitSidecar.normalizeBookmarks(raw_annotations)
local last_position = doc_settings:readSetting("last_xpointer")
if not last_position then
@@ -322,6 +370,10 @@ function BookOrbitSidecar.extract(file)
annotations_count = #annotations,
annotations_max_datetime = max_datetime,
annotations_signature = signature,
bookmarks = bookmarks,
bookmarks_count = #bookmarks,
bookmarks_max_datetime = bm_max_datetime,
bookmarks_signature = bm_signature,
}
end
koreader-plugin/bookorbit.koplugin/bookorbit_sweep.lua
+49 -1
View File
@@ -31,6 +31,7 @@ local T = ffiutil.template
local _ = require("gettext")
local BookOrbitAnnotations = require("bookorbit_annotations")
local BookOrbitBookmarks = require("bookorbit_bookmarks")
local BookOrbitApi = require("bookorbit_api")
local BookOrbitHighlightSummary = require("bookorbit_highlight_summary")
local BookOrbitQueue = require("bookorbit_queue")
@@ -253,7 +254,7 @@ local function finish(ctx, err)
text = text .. "\n" .. _("BookOrbit server needs an update for two-way highlights.")
end
if ctx.highlight_disabled_reported then
text = text .. "\n" .. _("Two-way highlight sync is disabled.")
text = text .. "\n" .. _("Two-way highlight and bookmark sync is disabled.")
end
if ctx.had_errors then
text = text .. "\n" .. _("Some books failed and will retry on the next sync.")
@@ -745,6 +746,8 @@ local function sidecarEntry(ctx, md5)
annotations = extract.annotations,
ann_max_datetime = extract.annotations_max_datetime,
ann_signature = extract.annotations_signature,
bookmarks = extract.bookmarks,
bm_signature = extract.bookmarks_signature,
})
end
@@ -815,6 +818,48 @@ local function currentlyOpenFile()
end
end
-- Dogears share the sidecar and the apply mode with highlights, so the sweep
-- exchanges both for the same book in one pass. A server without the route is
-- recorded once and skipped for the rest of the run.
local function exchangeBookmarksForEntry(ctx, entry, apply_mode, pending)
if not ctx.annotation_sync or ctx.bookmarks_unsupported then return end
if not BookOrbitBookmarks.enabled(ctx.client, true) then
ctx.bookmarks_unsupported = true
return
end
local book = ctx.state:getBook(entry.md5)
if not book then return end
if not ctx.interactive and BookOrbitBookmarks.canSkipExchange(book, entry.bm_signature) then return end
local result, err = BookOrbitBookmarks.exchangeBook({
client = ctx.client,
state = ctx.state,
digest = entry.md5,
bookmarks = entry.bookmarks,
bm_signature = entry.bm_signature,
apply_mode = apply_mode,
file = entry.file,
})
if not result then
if err == "unsupported_server" then
BookOrbitBookmarks.markUnsupported(ctx.client)
ctx.bookmarks_unsupported = true
return
end
ctx.had_errors = true
if pending then pending.failed = true end
ctx.highlight_summary = BookOrbitHighlightSummary.addBookmarks(ctx.highlight_summary, { had_errors = true })
logger.dbg("BookOrbit: bookmark exchange failed for", entry.md5, err)
return
end
if result.had_errors then
ctx.had_errors = true
if pending then pending.failed = true end
end
ctx.counts.bookmarks = (ctx.counts.bookmarks or 0) + result.uploaded
ctx.highlight_summary = BookOrbitHighlightSummary.addBookmarks(ctx.highlight_summary, result)
end
-- Phase 5: per-book annotation exchange. Uploads the local delta, reports the
-- key set for deletion detection and applies server-side changes into the
-- sidecar of closed books. Falls back to the legacy one-way upload when the
@@ -885,6 +930,9 @@ local function stepAnnotationsNext(ctx)
ctx.highlight_summary = BookOrbitHighlightSummary.add(ctx.highlight_summary, result, {
closed_book = apply_mode == "sidecar",
})
exchangeBookmarksForEntry(ctx, entry, apply_mode, pending)
if aborted(ctx) then return end
return step(ctx, ctx.steps.annotationsNext)
end
koreader-plugin/bookorbit.koplugin/main.lua
+49 -24
View File
@@ -30,6 +30,7 @@ local T = require("ffi/util").template
local _ = require("gettext")
local BookOrbitAnnotations = require("bookorbit_annotations")
local BookOrbitBookmarks = require("bookorbit_bookmarks")
local BookOrbitApi = require("bookorbit_api")
local BookOrbitBookSync = require("bookorbit_book_sync")
local BookOrbitCatalog = require("bookorbit_catalog")
@@ -1013,47 +1014,71 @@ function BookOrbit:exchangeAnnotationsForOpenBook(reason, retry_count)
end
self.annotation_exchange_running = true
local client = self:newClient()
local ok, result, err = pcall(BookOrbitAnnotations.exchangeOpenBook, {
client = self:newClient(),
client = client,
state = state,
digest = digest,
ui = self.ui,
})
local bm_ok, bm_result, bm_err
if BookOrbitBookmarks.enabled(client, self.settings.annotation_sync) then
bm_ok, bm_result, bm_err = pcall(BookOrbitBookmarks.exchangeOpenBook, {
client = client,
state = state,
digest = digest,
ui = self.ui,
})
if bm_ok and bm_err == "unsupported_server" then
BookOrbitBookmarks.markUnsupported(client)
end
end
state:flush()
self.annotation_exchange_running = false
local summary = { event = "open_book", reason = reason or "auto" }
local error_code
if not ok then
logger.err("BookOrbit: annotation exchange error:", result)
self:recordHighlightSync({
event = "open_book",
reason = reason or "auto",
failed = 1,
}, "partial_failure")
summary.failed = 1
error_code = "partial_failure"
elseif result then
local summary = BookOrbitHighlightSummary.add({
event = "open_book",
reason = reason or "auto",
}, result)
self:recordHighlightSync(summary)
if BookOrbitHighlightSummary.hasRemoteChanges(summary) then
Notification:notify(T(_("BookOrbit: %1 highlight(s) updated"),
summary.applied + summary.deleted))
summary = BookOrbitHighlightSummary.add(summary, result)
elseif err then
summary.skipped = err == "unmatched" and 1 or 0
summary.failed = (err == "network" or err == "unsupported_server") and 1 or 0
error_code = err
if err ~= "unmatched" and err ~= "unsupported_server" and err ~= "network" then
logger.dbg("BookOrbit: annotation exchange skipped:", err)
end
end
if bm_ok == false then
logger.err("BookOrbit: bookmark exchange error:", bm_result)
summary = BookOrbitHighlightSummary.addBookmarks(summary, { had_errors = true })
error_code = error_code or "partial_failure"
elseif bm_result then
summary = BookOrbitHighlightSummary.addBookmarks(summary, bm_result)
elseif bm_err and bm_err ~= "unmatched" and bm_err ~= "unsupported_server" then
summary = BookOrbitHighlightSummary.addBookmarks(summary, { had_errors = true })
error_code = error_code or bm_err
end
self:recordHighlightSync(summary, error_code)
if BookOrbitHighlightSummary.hasRemoteChanges(summary) then
Notification:notify(T(_("BookOrbit: %1 highlight(s) updated"),
summary.applied + summary.deleted))
end
if BookOrbitHighlightSummary.hasRemoteBookmarkChanges(summary) then
Notification:notify(T(_("BookOrbit: %1 bookmark(s) updated"),
summary.bm_applied + summary.bm_deleted))
end
if ok and result then
if reason == "annotation_open" and (summary.failed or 0) > 0 then
self:scheduleOpenHighlightRetry(reason, retry_count or 0)
elseif (summary.failed or 0) == 0 then
self.open_highlight_retry_status = nil
end
elseif err then
self:recordHighlightSync({
event = "open_book",
reason = reason or "auto",
skipped = err == "unmatched" and 1 or 0,
failed = (err == "network" or err == "unsupported_server") and 1 or 0,
}, err)
if err ~= "unmatched" and err ~= "unsupported_server" and err ~= "network" then
logger.dbg("BookOrbit: annotation exchange skipped:", err)
end
end
end
koreader-plugin/spec/bookorbit_book_sync_fast_path_test.lua
+7
View File
@@ -30,6 +30,13 @@ package.loaded["libs/libkoreader-lfs"] = {
package.loaded["ffi/util"] = { template = function(value) return value end }
package.loaded["gettext"] = function(value) return value end
package.loaded["bookorbit_bookmarks"] = {
enabled = function() return false end,
markUnsupported = function() end,
canSkipExchange = function() return false end,
rememberExchanged = function() end,
exchangeBook = function() error("bookmark sync is off in this fixture") end,
}
package.loaded["bookorbit_annotations"] = {
exchangeBook = function()
table.insert(calls, "annotations")
koreader-plugin/spec/bookorbit_book_sync_outbox_test.lua
+7
View File
@@ -37,6 +37,13 @@ package.loaded["ffi/util"] = {
}
package.loaded["gettext"] = function(value) return value end
package.loaded["bookorbit_bookmarks"] = {
enabled = function() return false end,
markUnsupported = function() end,
canSkipExchange = function() return false end,
rememberExchanged = function() end,
exchangeBook = function() error("bookmark sync is off in this fixture") end,
}
package.loaded["bookorbit_annotations"] = {
exchangeBook = function(opts)
table.insert(calls, "annotations")
koreader-plugin/spec/bookorbit_bookmark_exchange_test.lua
+245
View File
@@ -0,0 +1,245 @@
-- The bookmark exchange round trip: full-set upload with chunking, the pull
-- and ack loop, the skip stamp and the error classification the call sites
-- branch on.
package.path = "koreader-plugin/bookorbit.koplugin/?.lua;koreader-plugin/spec/?.lua;" .. package.path
local sidecar_annotations = {}
package.loaded["docsettings"] = {
open = function()
return {
readSetting = function(_, key)
if key == "annotations" then return sidecar_annotations end
end,
saveSetting = function(_, key, value)
if key == "annotations" then sidecar_annotations = value end
end,
makeTrue = function() end,
flush = function() end,
}
end,
hasSidecarFile = function() return false end,
findSidecarFile = function() return nil end,
}
package.loaded["ui/event"] = { new = function(_, name, payload) return { name = name, payload = payload } end }
package.loaded["ui/uimanager"] = { setDirty = function() end }
package.loaded["logger"] = { dbg = function() end }
package.loaded["ffi/sha2"] = { md5 = function(value) return value end }
package.loaded["ffi/util"] = { template = function(pattern, a) return (pattern:gsub("%%1", tostring(a))) end }
package.loaded["gettext"] = function(value) return value end
package.loaded["libs/libkoreader-lfs"] = { attributes = function() return nil end }
package.loaded["ui/widget/booklist"] = { setBookInfoCacheProperty = function() end }
package.loaded["bookorbit_capabilities"] = {
supports = function() return true end,
markUnsupported = function() end,
}
local BookOrbitBookmarks = require("bookorbit_bookmarks")
local function assertEqual(actual, expected, label)
if actual ~= expected then
error(string.format("%s: expected %s, got %s", label, tostring(expected), tostring(actual)), 2)
end
end
local function assertTrue(value, label)
if not value then error(label .. ": expected truthy", 2) end
end
local function makeState(book)
return {
getBook = function() return book end,
setUnmatched = function(self) self.unmatched = true end,
}
end
local function bookmark(datetime, pos)
return { datetime = datetime, pos = pos, pageno = 1 }
end
-- Upload: the whole local set goes out, chunked, with the key list only on the
-- first request. A watermark delta would drop renames, because KOReader leaves
-- datetime_updated alone when a dogear note is edited.
local requests = {}
local client = {
exchangeBookmarks = function(_, books)
table.insert(requests, books[1])
return { results = { { hash = "digest", toApply = { add = {}, delete = {} } } }, unmatched = {} }
end,
exchangeBookmarksAck = function() return { results = {} } end,
}
local many = {}
for index = 1, 120 do
table.insert(many, bookmark(string.format("2026-06-01 10:%02d:00", index % 60), "/body/p[" .. index .. "]"))
end
local book = {}
local result = BookOrbitBookmarks.exchangeBook({
client = client,
state = makeState(book),
digest = "digest",
bookmarks = many,
bm_signature = "sig-120",
apply_mode = "sidecar",
file = "/books/one.epub",
})
assertEqual(result.uploaded, 120, "every bookmark is uploaded")
assertEqual(#requests, 3, "120 bookmarks chunk into three requests")
assertEqual(#requests[1].keys, 120, "the full key set rides the first request")
assertEqual(requests[1].keysComplete, true, "the key set is complete")
assertEqual(#requests[2].keys, 0, "follow-up chunks carry no keys")
assertEqual(requests[2].keysComplete, false, "only the first request may drive deletion detection")
assertEqual(book.bmSignature, "sig-120", "a clean exchange records the skip stamp")
-- An empty local set still exchanges once, so the server can push web bookmarks
-- at a book the device has never bookmarked.
requests = {}
local empty_book = {}
local empty_result = BookOrbitBookmarks.exchangeBook({
client = client,
state = makeState(empty_book),
digest = "digest",
bookmarks = {},
bm_signature = "sig-empty",
apply_mode = "sidecar",
file = "/books/one.epub",
})
assertEqual(#requests, 1, "an empty set still asks the server")
assertEqual(empty_result.uploaded, 0, "nothing was uploaded")
-- Pull, apply and ack, then a follow-up round while the server reports more.
local rounds = 0
local acks = {}
sidecar_annotations = {}
local pull_client = {
exchangeBookmarks = function()
rounds = rounds + 1
if rounds == 1 then
return {
results = { {
hash = "digest",
toApply = { add = { { serverId = 1, pos = "/body/p[1]", title = "First" } }, delete = {} },
more = true,
} },
}
end
return {
results = { {
hash = "digest",
toApply = { add = { { serverId = 2, pos = "/body/p[2]", title = "Second" } }, delete = {} },
more = false,
} },
}
end,
exchangeBookmarksAck = function(_, books)
table.insert(acks, books[1])
return { results = {} }
end,
}
local pull_book = {}
local pull_result = BookOrbitBookmarks.exchangeBook({
client = pull_client,
state = makeState(pull_book),
digest = "digest",
bookmarks = {},
bm_signature = "sig-pull",
apply_mode = "sidecar",
file = "/books/one.epub",
})
assertEqual(pull_result.applied, 2, "both pushed bookmarks landed")
assertEqual(#acks, 2, "each round is acknowledged before the next pull")
assertEqual(acks[1].applied[1].serverId, 1, "the ack names the server id")
assertEqual(acks[1].applied[1].key ~= nil, true, "the ack carries the device identity")
assertEqual(#sidecar_annotations, 2, "both dogears are in the sidecar")
assertEqual(pull_book.bmSignature, "sig-pull", "a fully drained pull records the skip stamp")
-- Skip mode parks server changes instead of applying them, and must not record
-- the stamp: the next exchange has to come back for them.
local skip_book = {}
local skip_result = BookOrbitBookmarks.exchangeBook({
client = {
exchangeBookmarks = function()
return { results = { { hash = "digest", toApply = { add = { { serverId = 9, pos = "/body/p[9]", title = "Later" } } } } } }
end,
exchangeBookmarksAck = function() error("skip mode must not ack") end,
},
state = makeState(skip_book),
digest = "digest",
bookmarks = {},
bm_signature = "sig-skip",
apply_mode = "skip",
})
assertTrue(skip_result.remote_pending, "server changes are parked")
assertEqual(skip_result.remote_pending.to_apply.add[1].serverId, 9, "the parked payload is preserved")
assertEqual(skip_book.bmSignature, nil, "parked changes keep the next exchange mandatory")
-- Error classification the call sites branch on.
local function exchangeWithError(err)
return select(2, BookOrbitBookmarks.exchangeBook({
client = {
exchangeBookmarks = function() return nil, err end,
exchangeBookmarksAck = function() return { results = {} } end,
},
state = makeState({}),
digest = "digest",
bookmarks = {},
bm_signature = "sig-error",
apply_mode = "sidecar",
file = "/books/one.epub",
}))
end
assertEqual(exchangeWithError(401), "auth", "401 is an auth failure")
assertEqual(exchangeWithError(403), "auth", "403 is an auth failure")
assertEqual(exchangeWithError(404), "unsupported_server", "404 downgrades the capability")
assertEqual(exchangeWithError("timeout"), "network", "a transport failure is a network error")
-- An unmatched book stops the exchange and is remembered as unmatched.
local unmatched_state = makeState({})
local unmatched_result, unmatched_err = BookOrbitBookmarks.exchangeBook({
client = {
exchangeBookmarks = function() return { results = {}, unmatched = { "digest" } } end,
exchangeBookmarksAck = function() return { results = {} } end,
},
state = unmatched_state,
digest = "digest",
bookmarks = {},
bm_signature = "sig-unmatched",
apply_mode = "sidecar",
file = "/books/one.epub",
})
assertEqual(unmatched_result, nil, "an unmatched book produces no result")
assertEqual(unmatched_err, "unmatched", "the caller learns the book is unmatched")
assertTrue(unmatched_state.unmatched, "the unmatched digest is recorded")
-- A 500 mid-upload leaves the stamp unset so the next sync retries.
local failing_book = {}
local flaky_calls = 0
local two_chunks = {}
for index = 1, 60 do
table.insert(two_chunks, bookmark(string.format("2026-06-01 10:%02d:00", index % 60), "/body/p[" .. index .. "]"))
end
local flaky_result = BookOrbitBookmarks.exchangeBook({
client = {
exchangeBookmarks = function()
flaky_calls = flaky_calls + 1
if flaky_calls == 1 then
return { results = { { hash = "digest", toApply = { add = {}, delete = {} } } } }
end
return nil, 500
end,
exchangeBookmarksAck = function() return { results = {} } end,
},
state = makeState(failing_book),
digest = "digest",
bookmarks = two_chunks,
bm_signature = "sig-flaky",
apply_mode = "sidecar",
file = "/books/one.epub",
})
assertTrue(flaky_result.had_errors, "a 5xx is reported as an error")
assertEqual(failing_book.bmSignature, nil, "a failed upload does not record the skip stamp")
print("bookorbit_bookmark_exchange_test.lua: ok")
koreader-plugin/spec/bookorbit_bookmarks_test.lua
+234
View File
@@ -0,0 +1,234 @@
-- Dogear sync: normalization, skip logic, live apply, sidecar apply,
-- deletion detection and the ack identity the server links devices by.
package.path = "koreader-plugin/bookorbit.koplugin/?.lua;koreader-plugin/spec/?.lua;" .. package.path
local sidecar_store
local sidecar_flags
package.loaded["docsettings"] = {
open = function()
return {
readSetting = function(_, key)
if key == "annotations" then return sidecar_store end
end,
saveSetting = function(_, key, value)
if key == "annotations" then sidecar_store = value end
end,
makeTrue = function(_, key) sidecar_flags[key] = true end,
flush = function() sidecar_flags.flushed = true end,
}
end,
hasSidecarFile = function() return false end,
findSidecarFile = function() return nil end,
}
local handled_events = {}
package.loaded["ui/event"] = {
new = function(_, name, payload)
return { name = name, payload = payload }
end,
}
local dirty_calls = 0
package.loaded["ui/uimanager"] = {
setDirty = function() dirty_calls = dirty_calls + 1 end,
}
package.loaded["logger"] = { dbg = function() end }
-- Identity md5 is replaced by the input itself so assertions stay readable.
package.loaded["ffi/sha2"] = { md5 = function(value) return value end }
package.loaded["ffi/util"] = {
template = function(pattern, a) return (pattern:gsub("%%1", tostring(a))) end,
}
package.loaded["gettext"] = function(value) return value end
package.loaded["libs/libkoreader-lfs"] = { attributes = function() return nil end }
package.loaded["ui/widget/booklist"] = { setBookInfoCacheProperty = function() end }
local capability_answer = true
local marked_unsupported = false
package.loaded["bookorbit_capabilities"] = {
supports = function(_, feature)
if feature ~= "bookmarkSync" then return false end
return capability_answer
end,
markUnsupported = function() marked_unsupported = true end,
}
local BookOrbitSidecar = require("bookorbit_sidecar")
local BookOrbitBookmarks = require("bookorbit_bookmarks")
local function assertEqual(actual, expected, label)
if actual ~= expected then
error(string.format("%s: expected %s, got %s", label, tostring(expected), tostring(actual)), 2)
end
end
local function assertTrue(value, label)
if not value then error(label .. ": expected truthy", 2) end
end
-- normalizeBookmarks
local raw = {
{ page = "/body/DocFragment[2]/body/p[1]/text().0", datetime = "2026-06-01 10:00:00", chapter = "One", pageno = 4 },
{ page = "/body/DocFragment[3]", datetime = "2026-06-02 11:00:00", note = "Look", pageno = 9 },
-- A highlight: has a drawer, belongs to the annotation exchange.
{ page = "/body/DocFragment[4]", pos0 = "/body/DocFragment[4]", drawer = "lighten", datetime = "2026-06-03 12:00:00" },
-- A paging document dogear: the position is a page number, out of scope.
{ page = 12, datetime = "2026-06-04 13:00:00" },
-- Unusable datetime.
{ page = "/body/DocFragment[5]", datetime = "nonsense" },
}
local bookmarks, max_datetime, signature = BookOrbitSidecar.normalizeBookmarks(raw)
assertEqual(#bookmarks, 2, "only rolling dogears are normalized")
assertEqual(bookmarks[1].pos, "/body/DocFragment[2]/body/p[1]/text().0", "position comes from page")
assertEqual(bookmarks[1].chapter, "One", "chapter is carried")
assertEqual(bookmarks[1].pageno, 4, "pageno is carried")
assertEqual(bookmarks[2].note, "Look", "note is carried")
assertEqual(max_datetime, "2026-06-02 11:00:00", "max datetime ignores highlights")
local renamed = {
{ page = "/body/DocFragment[2]/body/p[1]/text().0", datetime = "2026-06-01 10:00:00", chapter = "One", pageno = 4 },
{ page = "/body/DocFragment[3]", datetime = "2026-06-02 11:00:00", note = "Renamed", pageno = 9 },
}
local _renamed, _renamed_max, renamed_signature = BookOrbitSidecar.normalizeBookmarks(renamed)
assertTrue(renamed_signature ~= signature,
"a note edit changes the signature even though KOReader leaves datetime_updated alone")
-- The highlight signature must not move, or every book re-exchanges after an update.
local annotations_before = select(3, BookOrbitSidecar.normalizeAnnotations(raw))
local annotations_after = select(3, BookOrbitSidecar.normalizeAnnotations(raw))
assertEqual(annotations_before, annotations_after, "highlight signature is stable")
-- keys and skip logic
local keys = BookOrbitBookmarks.collectKeys(bookmarks)
assertEqual(#keys, 2, "one key per bookmark")
assertEqual(keys[1].k, "2026-06-01 10:00:00|/body/DocFragment[2]/body/p[1]/text().0", "key is datetime|pos")
assertEqual(keys[1].dt, "2026-06-01 10:00:00", "key carries its datetime")
local book = {}
assertEqual(BookOrbitBookmarks.canSkipExchange(book, signature, 1000), false, "never exchanged, cannot skip")
BookOrbitBookmarks.rememberExchanged(book, signature, 1000)
assertEqual(BookOrbitBookmarks.canSkipExchange(book, signature, 1100), true, "unchanged set may skip")
assertEqual(BookOrbitBookmarks.canSkipExchange(book, renamed_signature, 1100), false, "changed set may not skip")
assertEqual(BookOrbitBookmarks.canSkipExchange(book, signature, 1000 + BookOrbitBookmarks.EXCHANGE_MAX_AGE + 1), false,
"a stale stamp may not skip")
assertEqual(BookOrbitBookmarks.canSkipExchange(book, signature, 500), false, "a stamp in the future may not skip")
-- capability gate
assertEqual(BookOrbitBookmarks.enabled({}, false), false, "the two-way toggle gates bookmark sync")
assertEqual(BookOrbitBookmarks.enabled({}, true), true, "an advertising server enables bookmark sync")
capability_answer = false
assertEqual(BookOrbitBookmarks.enabled({}, true), false, "a server without the capability stays dormant")
capability_answer = true
BookOrbitBookmarks.markUnsupported({})
assertTrue(marked_unsupported, "a confirmed 404 downgrades the capability")
-- live apply
local resolvable = {}
local annotations = {}
local added_items = {}
local removed_indexes = {}
local ui = {
document = {
isXPointerInDocument = function(_, pos) return resolvable[pos] == true end,
},
annotation = {
annotations = annotations,
addItem = function(self, item)
item.datetime = item.datetime or "2026-07-01 09:00:00"
table.insert(self.annotations, item)
table.insert(added_items, item)
return #self.annotations
end,
},
bookmark = {
getCurrentPageNumber = function() return "/body/DocFragment[1]" end,
setDogearVisibility = function() end,
removeItemByIndex = function(_, index)
table.insert(removed_indexes, index)
table.remove(annotations, index)
end,
},
toc = {
getTocTitleByPage = function(_, pos)
if pos == "/body/DocFragment[7]" then return "Chapter Seven" end
return ""
end,
},
handleEvent = function(_, event) table.insert(handled_events, event) end,
}
resolvable["/body/DocFragment[7]"] = true
resolvable["/body/DocFragment[8]"] = true
local applied, deleted, touched, deleted_touched = BookOrbitBookmarks.applyLive(ui, {
add = {
{ serverId = 1, pos = "/body/DocFragment[7]", title = "Chapter Seven" },
{ serverId = 2, pos = "/body/DocFragment[8]", title = "My own label" },
{ serverId = 3, pos = "/body/DocFragment[9]", title = "Broken" },
},
})
assertEqual(touched, 2, "two resolvable positions were added")
assertEqual(deleted_touched, 0, "nothing was deleted")
assertEqual(#deleted, 0, "no delete acks")
assertEqual(applied[1].status, "applied", "resolvable entry applied")
assertEqual(applied[1].key, "2026-07-01 09:00:00|/body/DocFragment[7]", "ack carries the device identity")
assertEqual(applied[1].pos, "/body/DocFragment[7]", "ack carries the device position")
assertEqual(added_items[1].chapter, "Chapter Seven", "the device fills the chapter from its own toc")
assertEqual(added_items[1].note, nil, "a title matching the chapter needs no note")
assertEqual(added_items[2].note, "My own label", "a title that is not the chapter becomes the note")
assertEqual(applied[3].status, "failed", "an unresolvable position fails instead of landing")
assertTrue(dirty_calls > 0, "the screen is refreshed after a change")
-- An entry already present locally is acked with the local identity, not re-added.
local before_count = #annotations
local applied_again = BookOrbitBookmarks.applyLive(ui, {
add = { { serverId = 1, pos = "/body/DocFragment[7]", title = "Chapter Seven" } },
})
assertEqual(#annotations, before_count, "an already present position is not duplicated")
assertEqual(applied_again[1].key, "2026-07-01 09:00:00|/body/DocFragment[7]", "the existing identity is acked")
-- Delete by the identity the server was acked with.
local _a, deletes, _t, delete_touched = BookOrbitBookmarks.applyLive(ui, {
delete = {
{ serverId = 1, key = "2026-07-01 09:00:00|/body/DocFragment[7]", datetime = "2026-07-01 09:00:00" },
{ serverId = 99, key = "never|seen", datetime = "2020-01-01 00:00:00" },
},
})
assertEqual(delete_touched, 1, "only the known entry was removed")
assertEqual(#deletes, 2, "an already absent entry is still acked")
assertEqual(deletes[2].status, "applied", "a missing local entry counts as applied")
-- sidecar apply
sidecar_store = {
{ page = "/body/DocFragment[2]", datetime = "2026-06-01 10:00:00" },
{ page = "/body/DocFragment[4]", drawer = "lighten", pos0 = "/body/DocFragment[4]", datetime = "2026-06-03 12:00:00" },
}
sidecar_flags = {}
local s_applied, s_deleted, s_touched, s_deleted_touched = BookOrbitBookmarks.applySidecar("/books/one.epub", {
add = { { serverId = 5, pos = "/body/DocFragment[6]", title = "From the web", pageno = 12 } },
delete = { { serverId = 6, key = "2026-06-01 10:00:00|/body/DocFragment[2]", datetime = "2026-06-01 10:00:00" } },
})
assertEqual(s_touched, 2, "one add and one delete touched the sidecar")
assertEqual(s_deleted_touched, 1, "one delete landed")
assertEqual(#s_applied, 1, "one apply ack")
assertEqual(#s_deleted, 1, "one delete ack")
assertEqual(#sidecar_store, 2, "the highlight stayed and the new dogear was appended")
assertEqual(sidecar_store[2].note, "From the web", "a closed book has no toc, so the title lands in the note")
assertEqual(sidecar_store[2].pageno, 12, "the server pageno is kept")
assertTrue(sidecar_flags.annotations_externally_modified, "KOReader is asked to re-validate on open")
assertTrue(sidecar_flags.flushed, "the sidecar was flushed")
-- A highlight must never be touched by the bookmark exchange.
assertEqual(sidecar_store[1].drawer, "lighten", "the highlight survived the dogear delete")
print("bookorbit_bookmarks_test.lua: ok")
koreader-plugin/spec/bookorbit_capture_identity_test.lua
+7
View File
@@ -48,6 +48,13 @@ package.loaded["libs/libkoreader-lfs"] = { attributes = function() return nil en
package.loaded["ffi/util"] = { template = function(value) return value end }
package.loaded["gettext"] = function(value) return value end
package.loaded["bookorbit_bookmarks"] = {
enabled = function() return false end,
markUnsupported = function() end,
canSkipExchange = function() return false end,
rememberExchanged = function() end,
exchangeBook = function() error("bookmark sync is off in this fixture") end,
}
package.loaded["bookorbit_annotations"] = {
canSkipExchange = function() return false end,
rememberExchanged = function() end,
koreader-plugin/spec/bookorbit_lifecycle_sidecar_marker_test.lua
+7
View File
@@ -32,6 +32,13 @@ package.loaded["ffi/util"] = {
}
package.loaded["gettext"] = function(value) return value end
package.loaded["bookorbit_bookmarks"] = {
enabled = function() return false end,
markUnsupported = function() end,
canSkipExchange = function() return false end,
rememberExchanged = function() end,
exchangeBook = function() error("bookmark sync is off in this fixture") end,
}
package.loaded["bookorbit_annotations"] = {
exchangeBook = function() error("every phase is already acknowledged") end,
canSkipExchange = function() return false end,
koreader-plugin/spec/bookorbit_menu_context_test.lua
+5 -5
View File
@@ -128,7 +128,7 @@ assertEqual(hasItem(items, "Current book"), false, "file manager menu hides curr
assertEqual(hasItem(items, "Sync current book now"), false, "file manager menu hides current book sync")
assertEqual(hasItem(items, "Auto sync current book"), false, "file manager menu hides current book auto sync")
assertEqual(hasItem(items, "Sync all books now"), true, "logged-in menu shows sync all")
assertEqual(hasItem(items, "Two-way highlight sync"), false, "logged-in top level hides highlight toggle")
assertEqual(hasItem(items, "Two-way highlights & bookmarks"), false, "logged-in top level hides the two-way toggle")
assertEqual(hasItem(items, "Skip auto-sync when offline"), false, "logged-in top level hides offline skip toggle")
assertEqual(hasItem(items, "Sync settings"), false, "logged-in menu replaces sync settings with settings")
assertEqual(hasItem(items, "Settings"), true, "logged-in menu shows settings")
@@ -142,7 +142,7 @@ assertEqual(hasItem(file_manager_dashboard_settings.sub_item_table, "Open dashbo
assertEqual(hasItem(file_manager_dashboard_settings.sub_item_table, "Show below Continue reading (Discover)"), true, "file manager dashboard settings has the section picker")
local file_manager_sync_settings = findItem(file_manager_settings.sub_item_table, "Sync")
assertEqual(hasItem(file_manager_sync_settings.sub_item_table, "Auto sync current book"), false, "file manager sync settings hides reader auto sync")
assertEqual(hasItem(file_manager_sync_settings.sub_item_table, "Two-way highlight sync"), true, "file manager sync settings has highlight toggle")
assertEqual(hasItem(file_manager_sync_settings.sub_item_table, "Two-way highlights & bookmarks"), true, "file manager sync settings has the two-way toggle")
assertEqual(hasItem(file_manager_sync_settings.sub_item_table, "Skip auto-sync when offline"), true, "file manager sync settings has offline skip toggle")
assertEqual(hasItem(file_manager_sync_settings.sub_item_table, "Open dashboard on startup (Off)"), false, "file manager sync settings excludes dashboard startup")
local file_manager_plugin_settings = findItem(file_manager_settings.sub_item_table, "Plugin")
@@ -167,7 +167,7 @@ assertEqual(hasItem(dashboard_dashboard_settings.sub_item_table, "Open dashboard
assertEqual(hasItem(dashboard_dashboard_settings.sub_item_table, "Show below Continue reading (Discover)"), true, "dashboard settings has the section picker")
assertEqual(hasItem(dashboard_items, "Show below Continue reading (Discover)"), false, "the section picker is not promoted to the dashboard menu top level")
local dashboard_sync_settings = findItem(dashboard_settings.sub_item_table, "Sync")
assertEqual(hasItem(dashboard_sync_settings.sub_item_table, "Two-way highlight sync"), true, "dashboard sync settings has highlight toggle")
assertEqual(hasItem(dashboard_sync_settings.sub_item_table, "Two-way highlights & bookmarks"), true, "dashboard sync settings has the two-way toggle")
assertEqual(hasItem(dashboard_sync_settings.sub_item_table, "Skip auto-sync when offline"), true, "dashboard sync settings has offline skip toggle")
assertEqual(hasItem(dashboard_sync_settings.sub_item_table, "Open dashboard on startup (Off)"), false, "dashboard sync settings excludes dashboard startup")
assertEqual(hasItem(dashboard_settings.sub_item_table, "Plugin"), false, "dashboard settings omits duplicate plugin group")
@@ -203,10 +203,10 @@ local reader_dashboard_settings = findItem(settings.sub_item_table, "Dashboard")
assertEqual(hasItem(reader_dashboard_settings.sub_item_table, "Open dashboard on startup (Off)"), true, "reader dashboard settings has startup option")
local sync_settings = findItem(settings.sub_item_table, "Sync")
assertEqual(hasItem(sync_settings.sub_item_table, "Auto sync current book"), false, "reader sync settings excludes top-level auto sync")
assertEqual(hasItem(sync_settings.sub_item_table, "Two-way highlight sync"), true, "sync settings has highlight toggle")
assertEqual(hasItem(sync_settings.sub_item_table, "Two-way highlights & bookmarks"), true, "sync settings has the two-way toggle")
assertEqual(hasItem(sync_settings.sub_item_table, "Skip auto-sync when offline"), true, "sync settings has offline toggle")
assertEqual(hasItem(sync_settings.sub_item_table, "Open dashboard on startup (Off)"), false, "sync settings excludes dashboard startup")
assertEqual(hasItem(items, "Two-way highlight sync"), false, "top level hides highlight toggle")
assertEqual(hasItem(items, "Two-way highlights & bookmarks"), false, "top level hides the two-way toggle")
assertEqual(hasItem(items, "Skip auto-sync when offline"), false, "top level hides offline toggle")
assertEqual(findItem(sync_settings.sub_item_table, "Skip auto-sync when offline").enabled_func(), true, "offline toggle is enabled when auto sync is on")
local reader_plugin_settings = findItem(settings.sub_item_table, "Plugin")
koreader-plugin/spec/helpers/sweep_harness.lua
+12
View File
@@ -155,6 +155,7 @@ function SweepHarness.install(opts)
sidecar_extracts = {},
history_resolved = {},
},
bookmark_exchanges = {},
flushes = 0,
}
@@ -254,9 +255,20 @@ function SweepHarness.install(opts)
readWatermark = function() return "" end,
advanceWatermark = function() end,
}
package.loaded["bookorbit_bookmarks"] = {
enabled = function() return opts.bookmark_sync == true end,
markUnsupported = function() end,
canSkipExchange = function() return false end,
rememberExchanged = function() end,
exchangeBook = function()
table.insert(handle.bookmark_exchanges, true)
return { uploaded = 0, applied = 0, deleted = 0, failed = 0, had_errors = false }
end,
}
package.loaded["bookorbit_highlight_summary"] = {
normalize = function(value) return value end,
add = function(value) return value end,
addBookmarks = function(value) return value end,
hasCounts = function() return false end,
hasRemoteChanges = function() return false end,
}