feat(reader): serialize and coordinate sync jobs in KOReader plugin

Introduce a sync coordinator to serialize and prioritize KOReader sync jobs (such as progress sync, highlight sync, library sweep, and update check). Coalesce pending jobs by family to prevent race conditions and redundant network requests. Use a dedicated scheduler for open annotations to handle highlight sync and matching asynchronously on book open. Add a diagnostics menu in the settings to track detailed sync status and provide clear error messages for debugging.
This commit is contained in:
neon
2026-07-08 17:55:40 +00:00
parent 512f16e288
commit 36bfc6e83d
26 changed files with 3140 additions and 473 deletions
@@ -1,9 +1,9 @@
--[[--
Main menu mixin for the BookOrbit plugin.
Builds the BookOrbit entry in KOReader's Tools menu (and the dashboard
hamburger menu that mirrors it), plus the account dialogs: server address,
login/logout. Installed onto the plugin controller as regular methods.
Builds the BookOrbit entry in KOReader's Tools menu and the dashboard menu that
mirrors it, plus the account dialogs: server address, login/logout. Installed
onto the plugin controller as regular methods.
]]
local Device = require("device")
@@ -19,7 +19,7 @@ local T = require("ffi/util").template
local _ = require("gettext")
local BookOrbitApi = require("bookorbit_api")
local BookOrbitBookSync = require("bookorbit_book_sync")
local BookOrbitHighlightDiagnostics = require("bookorbit_highlight_diagnostics")
local BookOrbitSweep = require("bookorbit_sweep")
-- Assigned from the plugin class on install so menu labels can name the
@@ -81,186 +81,407 @@ function MainMenu:catalogAutoOpenMenu()
}
end
local function formatBool(value)
return value and _("On") or _("Off")
end
local function formatTime(ts)
ts = tonumber(ts)
if not ts or ts <= 0 then return _("never") end
return os.date("%Y-%m-%d %H:%M", ts)
end
local function formatShortTime(ts)
ts = tonumber(ts)
if not ts or ts <= 0 then return _("never") end
return os.date("%H:%M", ts)
end
local function deviceIdPrefix(device_id)
if type(device_id) ~= "string" or device_id == "" then return _("unknown") end
return device_id:sub(1, 8)
end
local function versionText(version)
if type(version) ~= "string" or version == "" then return _("unknown") end
return "v" .. version:gsub("^v", "")
end
local function recordText(entry)
if type(entry) ~= "table" then return _("none") end
local when = formatTime(entry.at)
local message = entry.message or entry.event or _("unknown")
return T(_("%1 at %2"), tostring(message), when)
end
local function highlightRecordText(entry)
return BookOrbitHighlightDiagnostics.lastSyncText(entry, formatShortTime)
end
local function jobText(job)
if type(job) ~= "table" then return _("none") end
return tostring(job.label or job.family or _("unknown"))
end
local function jobStartedText(job)
if type(job) ~= "table" then return _("none") end
return formatTime(job.started_at)
end
local function safeDocumentDigest(plugin, has_open_book)
if not has_open_book or not plugin.getDocumentDigest then return nil end
local ok, digest = pcall(plugin.getDocumentDigest, plugin)
if ok then return digest end
end
local function hasReaderBook(plugin)
return plugin.ui and plugin.ui.document ~= nil
end
local function hasCredentials(settings)
return settings.server_url ~= nil and settings.username ~= nil and settings.userkey ~= nil
end
function MainMenu:diagnosticsRows()
local status = BookOrbitSweep.syncStatus()
local sync_status = self.getSyncCoordinatorStatus and self:getSyncCoordinatorStatus() or { pending_count = 0 }
local has_open_book = hasReaderBook(self)
local digest = safeDocumentDigest(self, has_open_book)
local scheduler = self.open_annotation_scheduler and self.open_annotation_scheduler:status() or nil
local logged_in = self:isLoggedIn()
local open_highlight_text = BookOrbitHighlightDiagnostics.openHighlightsText{
annotation_sync = self.settings.annotation_sync,
has_open_book = has_open_book,
matched = self.isOpenBookMatched and self:isOpenBookMatched(digest) or false,
scheduler_status = scheduler,
last_highlight_sync = self.settings.last_highlight_sync,
}
local rows = {
{ _("Server"), self.settings.server_url or _("not set") },
{ _("Username"), self.settings.username or _("not set") },
{ _("Login"), self:isLoggedIn() and _("configured") or _("not configured") },
{ _("Device"), Device.model or _("unknown") },
{ _("Device ID"), deviceIdPrefix(self.device_id) },
{ _("Plugin"), versionText(self.PLUGIN_VERSION) },
{ _("Latest plugin"), versionText(self.settings.update_latest_version) },
}
if not logged_in then
if self.settings.last_error then
table.insert(rows, "----------------------------")
table.insert(rows, { _("Last error"), recordText(self.settings.last_error) })
end
table.insert(rows, "----------------------------")
if hasCredentials(self.settings) then
table.insert(rows, {
_("Test connection"),
_("Run"),
callback = function()
self:testConnection()
end,
})
end
return rows
end
table.insert(rows, "----------------------------")
table.insert(rows, { _("Auto sync"), formatBool(self.settings.auto_sync) })
table.insert(rows, { _("Two-way highlights"), formatBool(self.settings.annotation_sync) })
table.insert(rows, { _("Open highlights"), open_highlight_text })
table.insert(rows, { _("Highlight apply retry"), BookOrbitHighlightDiagnostics.retryText(self.open_highlight_retry_status) })
table.insert(rows, { _("Skip offline auto-sync"), formatBool(self.settings.skip_sync_when_offline) })
table.insert(rows, { _("Current sync"), jobText(sync_status.current) })
table.insert(rows, { _("Current sync started"), jobStartedText(sync_status.current) })
table.insert(rows, { _("Pending syncs"), tostring(sync_status.pending_count or 0) })
table.insert(rows, { _("Next sync"), jobText(sync_status.next) })
table.insert(rows, { _("Last sweep"), formatTime(status.lastSweepAt) })
table.insert(rows, { _("Matched local books"), tostring(status.matched or 0) })
table.insert(rows, { _("Unmatched local books"), tostring(status.unmatched or 0) })
table.insert(rows, "----------------------------")
table.insert(rows, { _("Last sync"), recordText(self.settings.last_sync) })
table.insert(rows, { _("Last highlight sync"), highlightRecordText(self.settings.last_highlight_sync) })
table.insert(rows, { _("Last error"), recordText(self.settings.last_error) })
table.insert(rows, "----------------------------")
if has_open_book then
if self.settings.annotation_sync then
table.insert(rows, {
_("Retry highlight sync"),
_("Run"),
callback = function()
self:retryOpenHighlightSync()
end,
})
end
table.insert(rows, {
_("Retry open-book match"),
_("Run"),
callback = function()
self:retryOpenBookMatch()
end,
})
end
table.insert(rows, {
_("Test connection"),
_("Run"),
callback = function()
self:testConnection()
end,
})
table.insert(rows, {
_("Check for update"),
_("Run"),
callback = function()
self:checkForUpdate()
end,
})
return rows
end
function MainMenu:showDiagnostics()
local KeyValuePage = require("ui/widget/keyvaluepage")
UIManager:show(KeyValuePage:new{
title = _("BookOrbit diagnostics"),
kv_pairs = self:diagnosticsRows(),
value_overflow_align = "right",
})
end
function MainMenu:testConnection()
if not self.settings.server_url then
UIManager:show(InfoMessage:new{ text = _("Set the BookOrbit server address first."), timeout = 3 })
return
end
if not self:isLoggedIn() then
self:promptLogin()
return
end
NetworkMgr:runWhenConnected(function()
local checking = InfoMessage:new{ text = _("Testing BookOrbit connection...") }
UIManager:show(checking)
local body, err = self:newClient():auth()
UIManager:close(checking)
if body then
self:recordSyncSuccess("connection_test", _("Connection test succeeded"))
UIManager:show(InfoMessage:new{ text = _("BookOrbit connection works."), timeout = 3 })
else
self:recordSyncError("connection_test", err)
UIManager:show(InfoMessage:new{
text = T(_("BookOrbit connection failed: %1"), self:errorLabel(err)),
timeout = 4,
})
end
end)
end
function MainMenu:dashboardSettingsMenu()
return {
{
text_func = function()
return T(_("Open dashboard on startup (%1)"), self:catalogAutoOpenLabel())
end,
sub_item_table = self:catalogAutoOpenMenu(),
},
}
end
function MainMenu:syncSettingsMenu(has_open_book)
local items = {}
table.insert(items, {
text = _("Skip auto-sync when offline"),
checked_func = function() return self.settings.skip_sync_when_offline end,
enabled_func = function() return self.settings.auto_sync end,
help_text = _([[When enabled, automatic sync on book open is skipped if the device is not already online. Prevents the e-reader from stalling while trying to connect to Wi-Fi.]]),
callback = function()
self.settings.skip_sync_when_offline = not self.settings.skip_sync_when_offline
end,
})
table.insert(items, {
text_func = function()
return T(_("Periodically sync every # pages (%1)"), self:getSyncPeriod())
end,
enabled_func = function() return self.settings.auto_sync end,
keep_menu_open = true,
separator = true,
callback = function(touchmenu_instance)
local SpinWidget = require("ui/widget/spinwidget")
local spin = SpinWidget:new{
text = _([[This value determines how many page turns it takes to push book progress.
If set to 0, updating progress based on page turns will be disabled.]]),
value = self.settings.pages_before_update or 0,
value_min = 0,
value_max = 999,
value_step = 1,
value_hold_step = 10,
ok_text = _("Set"),
title_text = _("Number of pages before update"),
default_value = 10,
callback = function(widget)
self:setPagesBeforeUpdate(widget.value)
if touchmenu_instance then touchmenu_instance:updateItems() end
end,
}
UIManager:show(spin)
end,
})
table.insert(items, {
text = _("Two-way highlight sync"),
checked_func = function() return self.settings.annotation_sync end,
help_text = _([[Also applies highlights, notes and deletions made in BookOrbit to this device: on book open, after the manual book sync, and during the full sweep for closed books. Turning this off keeps uploads only.]]),
callback = function()
self.settings.annotation_sync = not self.settings.annotation_sync
end,
})
table.insert(items, {
text_func = function()
return T(_("Sync to a newer state (%1)"), getNameStrategy(self.settings.sync_forward))
end,
sub_item_table = self:strategyMenu(
function() return self.settings.sync_forward end,
function(value) self.settings.sync_forward = value end
),
})
table.insert(items, {
text_func = function()
return T(_("Sync to an older state (%1)"), getNameStrategy(self.settings.sync_backward))
end,
sub_item_table = self:strategyMenu(
function() return self.settings.sync_backward end,
function(value) self.settings.sync_backward = value end
),
})
return items
end
function MainMenu:pluginSettingsMenu()
return {
{
id = "plugin_update",
text_func = function()
return self:updateCheckMenuText()
end,
keep_menu_open = true,
callback = function()
self:checkForUpdate()
end,
},
}
end
function MainMenu:settingsMenu(has_open_book, opts)
opts = opts or {}
local items = {
{
text = _("Dashboard"),
sub_item_table = self:dashboardSettingsMenu(),
},
{
text = _("Sync"),
sub_item_table = self:syncSettingsMenu(has_open_book),
},
}
if opts.include_plugin ~= false then
table.insert(items, {
text = _("Plugin"),
sub_item_table = self:pluginSettingsMenu(),
})
end
return items
end
function MainMenu:addToMainMenu(menu_items)
menu_items.bookorbit = {
text = _("BookOrbit"),
-- Fallback placement only: BookOrbitMenuPin normally pins this entry
-- right below calibre on the first page of the Tools menu.
sorting_hint = "tools",
sub_item_table = {
{
text = _("Open dashboard"),
enabled_func = function()
return self:isLoggedIn()
end,
callback = function()
self:browseCatalog()
end,
},
{
text_func = function()
return self:updateCheckMenuText()
end,
enabled_func = function()
return self:isLoggedIn()
and not BookOrbitSweep.isRunning()
and not BookOrbitBookSync.isRunning()
end,
keep_menu_open = true,
callback = function()
self:checkForUpdate()
end,
},
{
text_func = function()
local status = BookOrbitSweep.syncStatus()
local when = (status.lastSweepAt == 0) and _("never")
or os.date("%Y-%m-%d %H:%M", status.lastSweepAt)
if status.unmatched > 0 then
return T(_("Last sync: %1 (%2 linked, %3 unmatched)"), when, status.matched, status.unmatched)
elseif status.matched > 0 then
return T(_("Last sync: %1 (%2 linked)"), when, status.matched)
end
return T(_("Last sync: %1"), when)
end,
enabled = false,
separator = true,
},
{
text = _("Sync this book now"),
enabled_func = function()
return self:isLoggedIn() and self.ui.document ~= nil
and not BookOrbitSweep.isRunning() and not BookOrbitBookSync.isRunning()
end,
local logged_in = self:isLoggedIn()
local has_open_book = hasReaderBook(self)
local items = {}
if logged_in then
table.insert(items, {
id = "open_dashboard",
text = _("Open dashboard"),
callback = function()
self:browseCatalog()
end,
separator = true,
})
if has_open_book then
table.insert(items, {
id = "sync_current_book",
text = _("Sync current book now"),
callback = function()
self:onBookOrbitSyncBook()
end,
},
{
text = _("Sync all books now"),
enabled_func = function()
return self:isLoggedIn() and not BookOrbitSweep.isRunning() and not BookOrbitBookSync.isRunning()
end,
callback = function()
self:startSweep()
end,
separator = true,
},
{
text = _("Auto sync this book"),
})
table.insert(items, {
id = "auto_sync_current_book",
text = _("Auto sync current book"),
checked_func = function() return self.settings.auto_sync end,
help_text = _([[Pulls progress when a book is opened; pushes progress, highlights, status, rating and reading time when it is closed and on suspend.]]),
callback = function()
self:onBookOrbitToggleAutoSync(nil, true)
end,
},
{
text = _("Two-way highlight sync"),
checked_func = function() return self.settings.annotation_sync end,
help_text = _([[Also applies highlights, notes and deletions made in BookOrbit to this device: on book open, after the manual book sync, and during the full sweep for closed books. Turning this off keeps uploads only.]]),
callback = function()
self.settings.annotation_sync = not self.settings.annotation_sync
end,
},
{
text = _("Skip auto-sync when offline"),
checked_func = function() return self.settings.skip_sync_when_offline end,
enabled_func = function() return self.settings.auto_sync end,
help_text = _([[When enabled, automatic sync on book open is skipped if the device is not already online. Prevents the e-reader from stalling while trying to connect to Wi-Fi.]]),
callback = function()
self.settings.skip_sync_when_offline = not self.settings.skip_sync_when_offline
end,
separator = true,
})
end
table.insert(items, {
text = _("Sync all books now"),
callback = function()
self:startSweep()
end,
separator = true,
})
table.insert(items, {
id = "settings",
text = _("Settings"),
separator = true,
sub_item_table = self:settingsMenu(has_open_book),
})
end
table.insert(items, {
text = _("Account & setup"),
sub_item_table = {
{
text = _("BookOrbit server address"),
keep_menu_open = true,
callback = function()
self:setServerAddress()
end,
},
{
text = _("Sync settings"),
sub_item_table = {
{
text_func = function()
return T(_("Open dashboard on startup (%1)"), self:catalogAutoOpenLabel())
end,
sub_item_table = self:catalogAutoOpenMenu(),
},
{
text_func = function()
return T(_("Periodically sync every # pages (%1)"), self:getSyncPeriod())
end,
enabled_func = function() return self.settings.auto_sync end,
keep_menu_open = true,
callback = function(touchmenu_instance)
local SpinWidget = require("ui/widget/spinwidget")
local items = SpinWidget:new{
text = _([[This value determines how many page turns it takes to push book progress.
If set to 0, updating progress based on page turns will be disabled.]]),
value = self.settings.pages_before_update or 0,
value_min = 0,
value_max = 999,
value_step = 1,
value_hold_step = 10,
ok_text = _("Set"),
title_text = _("Number of pages before update"),
default_value = 10,
callback = function(spin)
self:setPagesBeforeUpdate(spin.value)
if touchmenu_instance then touchmenu_instance:updateItems() end
end,
}
UIManager:show(items)
end,
},
{
text_func = function()
return T(_("Sync to a newer state (%1)"), getNameStrategy(self.settings.sync_forward))
end,
sub_item_table = self:strategyMenu(
function() return self.settings.sync_forward end,
function(value) self.settings.sync_forward = value end
),
},
{
text_func = function()
return T(_("Sync to an older state (%1)"), getNameStrategy(self.settings.sync_backward))
end,
sub_item_table = self:strategyMenu(
function() return self.settings.sync_backward end,
function(value) self.settings.sync_backward = value end
),
},
},
text_func = function()
return self.settings.userkey and _("Logout") or _("Login")
end,
enabled_func = function()
return self.settings.server_url ~= nil
end,
keep_menu_open = true,
callback_func = function()
if self.settings.userkey then
return function(menu)
self:logout(menu)
end
else
return function(menu)
self:login(menu)
end
end
end,
},
{
text = _("Account & setup"),
sub_item_table = {
{
text = _("BookOrbit server address"),
keep_menu_open = true,
callback = function()
self:setServerAddress()
end,
},
{
text_func = function()
return self.settings.userkey and _("Logout") or _("Login")
end,
enabled_func = function()
return self.settings.server_url ~= nil
end,
keep_menu_open = true,
callback_func = function()
if self.settings.userkey then
return function(menu)
self:logout(menu)
end
else
return function(menu)
self:login(menu)
end
end
end,
},
},
text = _("Diagnostics"),
callback = function()
self:showDiagnostics()
end,
},
},
})
menu_items.bookorbit = {
text = _("BookOrbit"),
-- Fallback placement only: BookOrbitMenuPin normally pins this entry
-- right below calibre on the first page of the Tools menu.
sorting_hint = "tools",
sub_item_table = items,
}
end
@@ -271,9 +492,23 @@ function MainMenu:dashboardMenuItems(catalog)
local items = {
icon = "appbar.menu",
}
for index, item in ipairs(bookorbit_items) do
if index ~= 1 then
table.insert(items, item)
if self:isLoggedIn() then
local plugin_update = self:pluginSettingsMenu()[1]
plugin_update.separator = true
table.insert(items, plugin_update)
end
for _index, item in ipairs(bookorbit_items) do
if item.id ~= "open_dashboard" and item.id ~= "sync_current_book" and item.id ~= "auto_sync_current_book" then
if item.id == "settings" then
table.insert(items, {
id = "settings",
text = _("Settings"),
separator = item.separator,
sub_item_table = self:settingsMenu(false, { include_plugin = false }),
})
else
table.insert(items, item)
end
end
end
if catalog then
@@ -427,7 +662,11 @@ function MainMenu:doLogin(username, password, menu)
if menu then menu:updateItems() end
UIManager:show(InfoMessage:new{ text = _("Logged in to BookOrbit.") })
UIManager:scheduleIn(1, function()
self:maybeCheckForUpdate(false)
if self.requestUpdateCheck then
self:requestUpdateCheck(false, "login")
else
self:maybeCheckForUpdate(false)
end
end)
elseif err == 401 or err == 403 then
UIManager:show(InfoMessage:new{