mirror of
https://github.com/bookorbit/bookorbit.git
synced 2026-11-03 00:11:37 -06:00
neon
1c27e5dd05
Report a linked copy whose bytes differ from the library file as positionIncompatible instead of unmatched, so the plugin keeps the match, skips highlights without a failure and still syncs progress. Treat a position-less place within a page as the same place and jump to the nearest page. Suggest library books with the same title and authors for books to review, with bulk linking. See docs/KOREADER_HASH_MATCHING.md. Closes #1872 Closes #1919
934 lines
38 KiB
Lua
934 lines
38 KiB
Lua
--[[--
|
|
Two-way annotation exchange with BookOrbit.
|
|
|
|
Uploads local highlight changes, applies server-side changes (web-created
|
|
highlights, note/style edits, deletions) and reports back an acknowledgment so
|
|
the server only marks entries delivered once they really landed on-device.
|
|
|
|
Apply modes:
|
|
- live: the book is open; entries are verified against crengine
|
|
(isXPointerInDocument + getTextFromXPointers) and repaired through
|
|
findAllText when the server-generated xpointer does not resolve to the
|
|
highlighted text. crengine has the final word; corrected positions are sent
|
|
back in the ack so the server stores them.
|
|
- sidecar: the book is closed; entries are merged into the sidecar annotations
|
|
table and annotations_externally_modified is set, the same path KOHighlights
|
|
uses, so KOReader re-validates and re-sorts on next open.
|
|
- skip: upload-only (suspend path); pending server changes stay queued because
|
|
no ack is sent for them.
|
|
]]
|
|
|
|
local DocSettings = require("docsettings")
|
|
local Event = require("ui/event")
|
|
local UIManager = require("ui/uimanager")
|
|
local logger = require("logger")
|
|
local md5 = require("ffi/sha2").md5
|
|
local util = require("util")
|
|
|
|
local BookOrbitSidecar = require("bookorbit_sidecar")
|
|
|
|
local MAX_KEYS_PER_BOOK = 5000
|
|
local MAX_PULL_ROUNDS = 10
|
|
local UPLOAD_CHUNK = 50
|
|
|
|
local BookOrbitAnnotations = {}
|
|
|
|
-- An exchange may be skipped only for a short while. It is the sole channel
|
|
-- that delivers server-side highlight changes to the device, and a drained
|
|
-- lifecycle entry can be hours old, so an unbounded skip would park web-created
|
|
-- highlights until the book's next open.
|
|
BookOrbitAnnotations.EXCHANGE_MAX_AGE = 6 * 3600
|
|
|
|
-- Records that the local set in `signature` was fully exchanged, which is what
|
|
-- lets the next drained lifecycle sync skip a book nothing happened to.
|
|
function BookOrbitAnnotations.rememberExchanged(book, signature, now)
|
|
if not book or type(signature) ~= "string" or signature == "" then return end
|
|
book.annSignature = signature
|
|
book.annExchangedAt = now or os.time()
|
|
end
|
|
|
|
function BookOrbitAnnotations.canSkipExchange(book, signature, now)
|
|
if not book then return false end
|
|
if type(signature) ~= "string" or signature == "" then return false end
|
|
if book.annSignature ~= signature then return false end
|
|
local exchanged_at = book.annExchangedAt
|
|
if type(exchanged_at) ~= "number" then return false end
|
|
now = now or os.time()
|
|
if exchanged_at > now then return false end
|
|
return (now - exchanged_at) <= BookOrbitAnnotations.EXCHANGE_MAX_AGE
|
|
end
|
|
|
|
function BookOrbitAnnotations.buildKey(datetime, pos0)
|
|
return md5(datetime .. "|" .. pos0)
|
|
end
|
|
|
|
-- Returns the stored upload watermark, resetting it when it sits in the
|
|
-- device's future: bad data there (e.g. a server that minted UTC datetimes)
|
|
-- would swallow every new annotation. The re-upload is a server-side no-op.
|
|
function BookOrbitAnnotations.readWatermark(book, device_now)
|
|
local watermark = book.annWatermark or ""
|
|
if watermark > device_now then
|
|
watermark = ""
|
|
book.annWatermark = ""
|
|
end
|
|
return watermark
|
|
end
|
|
|
|
-- Advances the watermark, capped at the device clock so a future-dated entry
|
|
-- cannot freeze it; such an entry just re-uploads until time passes it.
|
|
function BookOrbitAnnotations.advanceWatermark(book, ann_max_datetime, device_now)
|
|
if not ann_max_datetime or ann_max_datetime == "" then return end
|
|
local advance = ann_max_datetime
|
|
if advance > device_now then
|
|
advance = device_now
|
|
end
|
|
if advance > (book.annWatermark or "") then
|
|
book.annWatermark = advance
|
|
end
|
|
end
|
|
|
|
local function noteHash(note)
|
|
local hash = 5381
|
|
for index = 1, #note do
|
|
hash = (hash * 33 + note:byte(index)) % 4294967296
|
|
end
|
|
return hash
|
|
end
|
|
|
|
-- KOReader stamps datetime_updated only when a save flips an item between
|
|
-- highlight and note, so rewriting an existing note's text moves no timestamp
|
|
-- and the watermark never sees it. Each exchange remembers the notes it saw;
|
|
-- a note that differs from that record was edited since and goes up stamped
|
|
-- with the time its snapshot was taken, which is what the server judges edits
|
|
-- by. A queued snapshot can be older than the last exchange; stamping it with
|
|
-- the upload time would let its stale note overwrite a newer edit.
|
|
function BookOrbitAnnotations.noteEditedSince(book, annotation)
|
|
local hashes = book.annNoteHashes
|
|
if type(hashes) ~= "table" or not annotation.note then return false end
|
|
local remembered = hashes[BookOrbitAnnotations.buildKey(annotation.datetime, annotation.pos0)]
|
|
return remembered ~= nil and remembered ~= noteHash(annotation.note)
|
|
end
|
|
|
|
-- Remembers each note in the exchanged set and when. Only notes are kept, and
|
|
-- a book past the key cap keeps none, so the state stays small.
|
|
function BookOrbitAnnotations.rememberNotes(book, normalized, now)
|
|
if #normalized > MAX_KEYS_PER_BOOK then
|
|
book.annNoteHashes = nil
|
|
book.annNotesAt = nil
|
|
return
|
|
end
|
|
local hashes = {}
|
|
for _, annotation in ipairs(normalized) do
|
|
if annotation.note then
|
|
hashes[BookOrbitAnnotations.buildKey(annotation.datetime, annotation.pos0)] = noteHash(annotation.note)
|
|
end
|
|
end
|
|
book.annNoteHashes = hashes
|
|
book.annNotesAt = now
|
|
end
|
|
|
|
-- An item the server just applied carries the server's stamp, which is newer
|
|
-- than the upload watermark (read from the list taken before the pull), so the
|
|
-- next exchange would send the server its own copy back. Each applied item's
|
|
-- stamp is kept until the watermark passes it; a local change after the apply
|
|
-- restamps the item (style, color, a note save) or shows in the note record,
|
|
-- and goes up as usual.
|
|
local function appliedEntry(annotation)
|
|
local pos0 = annotation.pos0
|
|
if type(pos0) ~= "string" then pos0 = BookOrbitSidecar.serializePos(pos0) end
|
|
if type(annotation.datetime) ~= "string" or annotation.datetime == "" or not pos0 then return nil end
|
|
return {
|
|
key = BookOrbitAnnotations.buildKey(annotation.datetime, pos0),
|
|
stamp = annotation.datetime_updated or annotation.datetime,
|
|
}
|
|
end
|
|
|
|
local function addApplied(stamps, annotation)
|
|
local entry = appliedEntry(annotation)
|
|
if entry then table.insert(stamps, entry) end
|
|
end
|
|
|
|
local function pruneApplied(book)
|
|
local applied = book.annApplied
|
|
if type(applied) ~= "table" then return end
|
|
local watermark = book.annWatermark or ""
|
|
local kept = 0
|
|
for key, stamp in pairs(applied) do
|
|
if stamp <= watermark then applied[key] = nil else kept = kept + 1 end
|
|
end
|
|
if kept == 0 then book.annApplied = nil end
|
|
end
|
|
|
|
function BookOrbitAnnotations.rememberApplied(book, entries)
|
|
if not entries or #entries == 0 then return end
|
|
local applied = type(book.annApplied) == "table" and book.annApplied or {}
|
|
local count = 0
|
|
for _ in pairs(applied) do count = count + 1 end
|
|
for _, entry in ipairs(entries) do
|
|
if applied[entry.key] == nil then count = count + 1 end
|
|
applied[entry.key] = entry.stamp
|
|
end
|
|
-- Past the cap the record is dropped: the cost is one harmless echo per item.
|
|
book.annApplied = count <= MAX_KEYS_PER_BOOK and applied or nil
|
|
pruneApplied(book)
|
|
end
|
|
|
|
function BookOrbitAnnotations.isAppliedCopy(book, annotation, effective)
|
|
local applied = book.annApplied
|
|
if type(applied) ~= "table" then return false end
|
|
return applied[BookOrbitAnnotations.buildKey(annotation.datetime, annotation.pos0)] == effective
|
|
end
|
|
|
|
-- A note rewritten since the notes were remembered, whose own stamp is no
|
|
-- later than that moment: nothing dated the rewrite. A highlight just received
|
|
-- from the server carries a datetime past the upload watermark, so its first
|
|
-- rewrite would otherwise go up with that older datetime and be dropped.
|
|
local function needsEditStamp(book, annotation, effective, watermark)
|
|
if not BookOrbitAnnotations.noteEditedSince(book, annotation) then return false end
|
|
return effective <= (book.annNotesAt or watermark)
|
|
end
|
|
|
|
local function editedAt(annotation, datetime)
|
|
local change = {}
|
|
for field, value in pairs(annotation) do change[field] = value end
|
|
change.datetimeUpdated = datetime
|
|
return change
|
|
end
|
|
|
|
-- Key list for deletion detection, built from normalized entries so the
|
|
-- serialized pos0 matches what the server hashed at upload time.
|
|
function BookOrbitAnnotations.collectKeys(normalized)
|
|
local keys = {}
|
|
for _, a in ipairs(normalized) do
|
|
table.insert(keys, { k = BookOrbitAnnotations.buildKey(a.datetime, a.pos0), dt = a.datetime })
|
|
end
|
|
return keys
|
|
end
|
|
|
|
local function normalizeText(text)
|
|
if type(text) ~= "string" then return "" end
|
|
return util.trim(text:gsub("%s+", " "))
|
|
end
|
|
|
|
local ackApplied
|
|
local applyEditFields
|
|
|
|
local function normalizePos(pos)
|
|
if type(pos) ~= "string" then return "" end
|
|
return util.trim(pos)
|
|
end
|
|
|
|
local function sameRangeAndText(annotation, entry)
|
|
local text = normalizeText(entry.text)
|
|
if text == "" or normalizePos(entry.pos0) == "" then return false end
|
|
if normalizeText(annotation.text) ~= text then return false end
|
|
if normalizePos(annotation.pos0) ~= normalizePos(entry.pos0) then return false end
|
|
local entry_pos1 = normalizePos(entry.pos1)
|
|
if entry_pos1 ~= "" and normalizePos(annotation.pos1) ~= entry_pos1 then return false end
|
|
return true
|
|
end
|
|
|
|
local function rangeKey(item)
|
|
local text = normalizeText(item.text)
|
|
local pos0 = normalizePos(item.pos0)
|
|
if text == "" or pos0 == "" then return nil end
|
|
return text .. "\0" .. pos0
|
|
end
|
|
|
|
--[[--
|
|
Identity lookup over one annotation list.
|
|
|
|
A remote batch resolves every entry against the local list, and a plain scan
|
|
per entry makes that quadratic. The index keys both identities the resolver
|
|
uses: the exact datetime and the normalized range plus text. Ambiguous keys
|
|
(the same datetime or the same range on several entries) deliberately stay on
|
|
the scanning path, because that is what the ambiguous case has always resolved
|
|
to and the scan order defines which duplicate wins.
|
|
|
|
The list is mutated during the same pass, so the index tracks that: an append
|
|
patches in place, anything that shifts positions (an insert elsewhere, a
|
|
removal) marks the index stale and the next lookup rebuilds it.
|
|
|
|
Every entry of a batch is a different server row, so a local highlight an entry
|
|
was matched to or created for is claimed and never resolves another entry.
|
|
Without that, two highlights made in the same second arriving in one batch (a
|
|
fresh copy re-offered everything) collapse into one: the second matches the
|
|
first by datetime and is moved onto it.
|
|
]]
|
|
local IdentityIndex = {}
|
|
IdentityIndex.__index = IdentityIndex
|
|
|
|
local function addSlot(map, key, index, annotation)
|
|
if not key then return end
|
|
local slot = map[key]
|
|
if slot then
|
|
slot.count = slot.count + 1
|
|
return
|
|
end
|
|
map[key] = { index = index, annotation = annotation, count = 1 }
|
|
end
|
|
|
|
function IdentityIndex:rebuild()
|
|
local by_datetime, by_range = {}, {}
|
|
for index, annotation in ipairs(self.annotations) do
|
|
local datetime = annotation.datetime
|
|
if type(datetime) == "string" and datetime ~= "" then
|
|
addSlot(by_datetime, datetime, index, annotation)
|
|
end
|
|
addSlot(by_range, rangeKey(annotation), index, annotation)
|
|
end
|
|
self.by_datetime = by_datetime
|
|
self.by_range = by_range
|
|
self.stale = false
|
|
end
|
|
|
|
function IdentityIndex.new(annotations)
|
|
local self = setmetatable({ annotations = annotations or {}, claims = {} }, IdentityIndex)
|
|
self:rebuild()
|
|
return self
|
|
end
|
|
|
|
function IdentityIndex:claim(annotation, entry)
|
|
if annotation then self.claims[annotation] = entry end
|
|
end
|
|
|
|
function IdentityIndex:claimedByOther(annotation, entry)
|
|
local owner = self.claims[annotation]
|
|
return owner ~= nil and owner ~= entry
|
|
end
|
|
|
|
function IdentityIndex:invalidate()
|
|
self.stale = true
|
|
end
|
|
|
|
-- Only valid when the annotation really is the new last element; any other
|
|
-- insertion shifts later positions and must invalidate instead.
|
|
function IdentityIndex:noteAppended(annotation)
|
|
if self.stale then return end
|
|
local index = #self.annotations
|
|
if self.annotations[index] ~= annotation then
|
|
self.stale = true
|
|
return
|
|
end
|
|
local datetime = annotation.datetime
|
|
if type(datetime) == "string" and datetime ~= "" then
|
|
addSlot(self.by_datetime, datetime, index, annotation)
|
|
end
|
|
addSlot(self.by_range, rangeKey(annotation), index, annotation)
|
|
end
|
|
|
|
function IdentityIndex:identityOf(annotation)
|
|
return annotation.datetime, rangeKey(annotation)
|
|
end
|
|
|
|
-- An applied change can rewrite the datetime or the highlighted text, moving
|
|
-- the entry's keys. Repointing a key that may be shared with another entry is
|
|
-- not safe, so the index is dropped and rebuilt on the next lookup.
|
|
function IdentityIndex:noteMutated(annotation, previous_datetime, previous_range_key)
|
|
if self.stale then return end
|
|
if annotation.datetime ~= previous_datetime or rangeKey(annotation) ~= previous_range_key then
|
|
self.stale = true
|
|
end
|
|
end
|
|
|
|
function IdentityIndex:find(entry)
|
|
if self.stale then self:rebuild() end
|
|
|
|
local datetime = entry.datetime
|
|
if type(datetime) == "string" and datetime ~= "" then
|
|
local slot = self.by_datetime[datetime]
|
|
if slot and slot.count == 1 and not self:claimedByOther(slot.annotation, entry) then
|
|
return slot.index, slot.annotation
|
|
end
|
|
end
|
|
|
|
local key = rangeKey(entry)
|
|
if not key then return nil end
|
|
local slot = self.by_range[key]
|
|
if not slot then return nil end
|
|
if slot.count == 1 then
|
|
if sameRangeAndText(slot.annotation, entry) and not self:claimedByOther(slot.annotation, entry) then
|
|
return slot.index, slot.annotation
|
|
end
|
|
return nil
|
|
end
|
|
for index, annotation in ipairs(self.annotations) do
|
|
if sameRangeAndText(annotation, entry) and not self:claimedByOther(annotation, entry) then
|
|
return index, annotation
|
|
end
|
|
end
|
|
end
|
|
|
|
local function applyIdentityFields(annotation, entry)
|
|
if type(entry.datetime) == "string" and entry.datetime ~= "" then
|
|
annotation.datetime = entry.datetime
|
|
end
|
|
applyEditFields(annotation, entry)
|
|
end
|
|
|
|
local function ackExisting(entry, annotation, verified)
|
|
return ackApplied(entry, {
|
|
verified = verified == true,
|
|
pos0 = annotation.pos0 or entry.pos0,
|
|
pos1 = annotation.pos1 or entry.pos1,
|
|
pageno = type(annotation.pageno) == "number" and annotation.pageno or entry.pageno,
|
|
})
|
|
end
|
|
|
|
-- crengine verification: both endpoints must resolve and, when the server
|
|
-- knows the highlighted text, the extracted range must match it.
|
|
local function verifyRange(document, pos0, pos1, text)
|
|
if type(pos0) ~= "string" or type(pos1) ~= "string" then return false end
|
|
local ok0, in0 = pcall(document.isXPointerInDocument, document, pos0)
|
|
local ok1, in1 = pcall(document.isXPointerInDocument, document, pos1)
|
|
if not (ok0 and in0 and ok1 and in1) then return false end
|
|
local wanted = normalizeText(text)
|
|
if wanted == "" then return true end
|
|
local ok, extracted = pcall(document.getTextFromXPointers, document, pos0, pos1)
|
|
return ok and type(extracted) == "string" and normalizeText(extracted) == wanted
|
|
end
|
|
|
|
-- Re-anchors an entry by searching its highlighted text, picking the hit
|
|
-- nearest to the page the broken xpointer pointed at (when derivable).
|
|
local function repairRange(document, entry)
|
|
local needle = type(entry.text) == "string" and util.trim(entry.text) or ""
|
|
if needle == "" or not document.findAllText then return nil end
|
|
|
|
local page_hint
|
|
if type(entry.pos0) == "string" and verifyRange(document, entry.pos0, entry.pos0, "") then
|
|
local ok, page = pcall(document.getPageFromXPointer, document, entry.pos0)
|
|
if ok and type(page) == "number" then page_hint = page end
|
|
end
|
|
|
|
local ok, hits = pcall(document.findAllText, document, needle, true, 0, 20, false)
|
|
pcall(document.clearSelection, document)
|
|
if not ok or type(hits) ~= "table" or #hits == 0 then return nil end
|
|
if #hits > 1 and not page_hint then return nil end
|
|
|
|
local best = hits[1]
|
|
if page_hint and #hits > 1 then
|
|
local best_distance = math.huge
|
|
local ambiguous = false
|
|
for _, hit in ipairs(hits) do
|
|
local okp, page = pcall(document.getPageFromXPointer, document, hit.start)
|
|
if okp and type(page) == "number" then
|
|
local distance = math.abs(page - page_hint)
|
|
if distance < best_distance then
|
|
best_distance = distance
|
|
best = hit
|
|
ambiguous = false
|
|
elseif distance == best_distance then
|
|
ambiguous = true
|
|
end
|
|
end
|
|
end
|
|
if ambiguous then return nil end
|
|
end
|
|
if verifyRange(document, best.start, best["end"], entry.text) then
|
|
return best.start, best["end"]
|
|
end
|
|
return nil
|
|
end
|
|
|
|
local function updateVerifiedRange(ui, annotation, entry)
|
|
local pos0, pos1 = entry.pos0 or annotation.pos0, entry.pos1 or annotation.pos1
|
|
local corrected = false
|
|
if not verifyRange(ui.document, pos0, pos1, entry.text) then
|
|
pos0, pos1 = repairRange(ui.document, {
|
|
pos0 = pos0, pos1 = pos1, text = entry.text,
|
|
})
|
|
corrected = true
|
|
end
|
|
if not pos0 or not pos1 then return false end
|
|
annotation.page, annotation.pos0, annotation.pos1 = pos0, pos0, pos1
|
|
annotation.bookorbit_pending_verification = nil
|
|
if ui.annotation.updateAnnotations then ui.annotation:updateAnnotations(true, true) end
|
|
return true, corrected
|
|
end
|
|
|
|
ackApplied = function(entry, opts)
|
|
return {
|
|
serverId = entry.serverId,
|
|
version = entry.version,
|
|
status = opts.failed and "failed" or "applied",
|
|
verified = opts.verified or false,
|
|
corrected = opts.corrected or false,
|
|
pos0 = opts.pos0,
|
|
pos1 = opts.pos1,
|
|
pageno = opts.pageno,
|
|
datetimeUpdated = entry.datetimeUpdated,
|
|
}
|
|
end
|
|
|
|
applyEditFields = function(annotation, entry)
|
|
annotation.drawer = entry.drawer or annotation.drawer
|
|
annotation.color = entry.color or annotation.color
|
|
if entry.text and entry.text ~= "" then annotation.text = entry.text end
|
|
annotation.note = entry.note
|
|
if entry.chapter then annotation.chapter = entry.chapter end
|
|
if entry.datetimeUpdated then annotation.datetime_updated = entry.datetimeUpdated end
|
|
end
|
|
|
|
-- Applies server changes to the OPEN book through ReaderAnnotation, so the
|
|
-- view, footer and bookmark list stay coherent. Returns ack tables.
|
|
function BookOrbitAnnotations.applyLive(ui, to_apply)
|
|
local applied, deleted, stamps = {}, {}, {}
|
|
local document = ui.document
|
|
local annotations = ui.annotation.annotations
|
|
local lookup = IdentityIndex.new(annotations)
|
|
local touched = 0
|
|
local deleted_touched = 0
|
|
|
|
for _, entry in ipairs(to_apply.add or {}) do
|
|
local existing_index, existing_annotation = lookup:find(entry)
|
|
lookup:claim(existing_annotation, entry)
|
|
if existing_index then
|
|
local verified, corrected = updateVerifiedRange(ui, existing_annotation, entry)
|
|
if verified then
|
|
applyIdentityFields(existing_annotation, entry)
|
|
lookup:invalidate()
|
|
local index = lookup:find(entry)
|
|
ui:handleEvent(Event:new("AnnotationsModified", { existing_annotation, index_modified = index or existing_index }))
|
|
touched = touched + 1
|
|
addApplied(stamps, existing_annotation)
|
|
local ack = ackExisting(entry, existing_annotation, true)
|
|
ack.corrected = corrected
|
|
table.insert(applied, ack)
|
|
else
|
|
table.insert(applied, ackApplied(entry, { failed = true }))
|
|
end
|
|
elseif entry.posFormat ~= "xpointer" or not ui.rolling then
|
|
-- PDF adds from the web are out of scope; never expected here.
|
|
table.insert(applied, ackApplied(entry, { failed = true }))
|
|
else
|
|
local pos0, pos1 = entry.pos0, entry.pos1
|
|
local verified = verifyRange(document, pos0, pos1, entry.text)
|
|
local corrected = false
|
|
if not verified then
|
|
local repaired0, repaired1 = repairRange(document, entry)
|
|
if repaired0 then
|
|
pos0, pos1 = repaired0, repaired1
|
|
verified = true
|
|
corrected = true
|
|
end
|
|
end
|
|
if not verified then
|
|
logger.dbg("BookOrbit: annotation apply failed, no verifiable position:", entry.serverId)
|
|
table.insert(applied, ackApplied(entry, { failed = true }))
|
|
else
|
|
local item = {
|
|
datetime = entry.datetime,
|
|
datetime_updated = entry.datetimeUpdated,
|
|
drawer = entry.drawer,
|
|
color = entry.color,
|
|
text = entry.text,
|
|
note = entry.note,
|
|
chapter = entry.chapter,
|
|
page = pos0,
|
|
pos0 = pos0,
|
|
pos1 = pos1,
|
|
}
|
|
local index = ui.annotation:addItem(item)
|
|
lookup:noteAppended(item)
|
|
lookup:claim(item, entry)
|
|
if ui.view and ui.view.footer and ui.view.footer.maybeUpdateFooter then
|
|
pcall(ui.view.footer.maybeUpdateFooter, ui.view.footer)
|
|
end
|
|
ui:handleEvent(Event:new("AnnotationsModified", {
|
|
item,
|
|
nb_highlights_added = 1,
|
|
index_modified = index,
|
|
}))
|
|
touched = touched + 1
|
|
addApplied(stamps, item)
|
|
table.insert(applied, ackApplied(entry, {
|
|
verified = true,
|
|
corrected = corrected,
|
|
pos0 = pos0,
|
|
pos1 = pos1,
|
|
pageno = type(item.pageno) == "number" and item.pageno or nil,
|
|
}))
|
|
end
|
|
end
|
|
end
|
|
|
|
for _, entry in ipairs(to_apply.edit or {}) do
|
|
local index, annotation = lookup:find(entry)
|
|
lookup:claim(annotation, entry)
|
|
if not index then
|
|
logger.dbg("BookOrbit: annotation edit target missing:", entry.serverId)
|
|
table.insert(applied, ackApplied(entry, { failed = true }))
|
|
elseif type(annotation.pos0) ~= "string" then
|
|
-- A page-position (PDF) highlight: nothing elsewhere can move it, so only its note,
|
|
-- color and drawer change, and there is no xpointer range to verify.
|
|
applyEditFields(annotation, entry)
|
|
lookup:invalidate()
|
|
local updated_index = lookup:find(entry)
|
|
ui:handleEvent(Event:new("AnnotationsModified", { annotation, index_modified = updated_index or index }))
|
|
touched = touched + 1
|
|
addApplied(stamps, annotation)
|
|
table.insert(applied, ackApplied(entry, {
|
|
verified = true,
|
|
pos0 = BookOrbitSidecar.serializePos(annotation.pos0),
|
|
pos1 = BookOrbitSidecar.serializePos(annotation.pos1),
|
|
}))
|
|
else
|
|
local verified, corrected = updateVerifiedRange(ui, annotation, entry)
|
|
if verified then
|
|
applyEditFields(annotation, entry)
|
|
lookup:invalidate()
|
|
local updated_index = lookup:find(entry)
|
|
ui:handleEvent(Event:new("AnnotationsModified", { annotation, index_modified = updated_index or index }))
|
|
touched = touched + 1
|
|
addApplied(stamps, annotation)
|
|
local ack = ackExisting(entry, annotation, true)
|
|
ack.corrected = corrected
|
|
table.insert(applied, ack)
|
|
else
|
|
table.insert(applied, ackApplied(entry, { failed = true }))
|
|
end
|
|
end
|
|
end
|
|
|
|
for _, entry in ipairs(to_apply.delete or {}) do
|
|
local index, annotation = lookup:find(entry)
|
|
if index then
|
|
if ui.paging then
|
|
pcall(ui.highlight.writePdfAnnotation, ui.highlight, "delete", annotation)
|
|
end
|
|
ui.bookmark:removeItemByIndex(index)
|
|
lookup:invalidate()
|
|
touched = touched + 1
|
|
deleted_touched = deleted_touched + 1
|
|
end
|
|
-- Missing entries were already deleted locally; ack either way.
|
|
table.insert(deleted, { serverId = entry.serverId, status = "applied" })
|
|
end
|
|
|
|
if touched > 0 then
|
|
UIManager:setDirty("all", "ui")
|
|
end
|
|
return applied, deleted, touched, deleted_touched, stamps
|
|
end
|
|
|
|
-- Merges server changes into the sidecar of a CLOSED book and raises
|
|
-- annotations_externally_modified so KOReader re-validates and sorts on open.
|
|
function BookOrbitAnnotations.applySidecar(file, to_apply)
|
|
local applied, deleted, stamps = {}, {}, {}
|
|
local doc_settings = DocSettings:open(file)
|
|
local annotations = doc_settings:readSetting("annotations") or {}
|
|
local lookup = IdentityIndex.new(annotations)
|
|
local touched = 0
|
|
local deleted_touched = 0
|
|
|
|
for _, entry in ipairs(to_apply.add or {}) do
|
|
if entry.posFormat ~= "xpointer" then
|
|
table.insert(applied, ackApplied(entry, { failed = true }))
|
|
else
|
|
local existing_index, existing = lookup:find(entry)
|
|
lookup:claim(existing, entry)
|
|
if existing_index then
|
|
local previous_datetime, previous_range = lookup:identityOf(existing)
|
|
applyIdentityFields(existing, entry)
|
|
existing.bookorbit_pending_verification = true
|
|
lookup:noteMutated(existing, previous_datetime, previous_range)
|
|
touched = touched + 1
|
|
addApplied(stamps, existing)
|
|
else
|
|
local item = {
|
|
datetime = entry.datetime,
|
|
datetime_updated = entry.datetimeUpdated,
|
|
drawer = entry.drawer,
|
|
color = entry.color,
|
|
text = entry.text,
|
|
note = entry.note,
|
|
chapter = entry.chapter,
|
|
pageno = entry.pageno,
|
|
page = entry.pos0,
|
|
pos0 = entry.pos0,
|
|
pos1 = entry.pos1,
|
|
bookorbit_pending_verification = true,
|
|
}
|
|
table.insert(annotations, item)
|
|
lookup:noteAppended(item)
|
|
lookup:claim(item, entry)
|
|
touched = touched + 1
|
|
addApplied(stamps, item)
|
|
end
|
|
-- Unverified until the book is opened; the server keeps it pending.
|
|
table.insert(applied, ackExisting(entry, existing or {
|
|
pos0 = entry.pos0,
|
|
pos1 = entry.pos1,
|
|
pageno = entry.pageno,
|
|
}, false))
|
|
end
|
|
end
|
|
|
|
for _, entry in ipairs(to_apply.edit or {}) do
|
|
local index, annotation = lookup:find(entry)
|
|
lookup:claim(annotation, entry)
|
|
if not index then
|
|
table.insert(applied, ackApplied(entry, { failed = true }))
|
|
else
|
|
local previous_datetime, previous_range = lookup:identityOf(annotation)
|
|
applyEditFields(annotation, entry)
|
|
-- Page-position (PDF) entries have no xpointer range for the next open to verify.
|
|
if type(annotation.pos0) == "string" then
|
|
annotation.bookorbit_pending_verification = true
|
|
end
|
|
if entry.pos0 and entry.pos1 then
|
|
annotation.page, annotation.pos0, annotation.pos1 = entry.pos0, entry.pos0, entry.pos1
|
|
end
|
|
lookup:noteMutated(annotation, previous_datetime, previous_range)
|
|
touched = touched + 1
|
|
addApplied(stamps, annotation)
|
|
if type(annotation.pos0) == "string" then
|
|
table.insert(applied, ackApplied(entry, {}))
|
|
else
|
|
table.insert(applied, ackApplied(entry, {
|
|
verified = true,
|
|
pos0 = BookOrbitSidecar.serializePos(annotation.pos0),
|
|
pos1 = BookOrbitSidecar.serializePos(annotation.pos1),
|
|
}))
|
|
end
|
|
end
|
|
end
|
|
|
|
for _, entry in ipairs(to_apply.delete or {}) do
|
|
local index = lookup:find(entry)
|
|
if index then
|
|
table.remove(annotations, index)
|
|
lookup:invalidate()
|
|
touched = touched + 1
|
|
deleted_touched = deleted_touched + 1
|
|
end
|
|
table.insert(deleted, { serverId = entry.serverId, status = "applied" })
|
|
end
|
|
|
|
if touched > 0 then
|
|
doc_settings:saveSetting("annotations", annotations)
|
|
doc_settings:makeTrue("annotations_externally_modified")
|
|
doc_settings:flush()
|
|
end
|
|
return applied, deleted, touched, deleted_touched, stamps
|
|
end
|
|
|
|
local function hasPending(to_apply)
|
|
return to_apply and (#(to_apply.add or {}) > 0 or #(to_apply.edit or {}) > 0 or #(to_apply.delete or {}) > 0)
|
|
end
|
|
|
|
-- The server matches a copy whose bytes differ from the library file but cannot
|
|
-- place its highlights, so nothing is exchanged and nothing failed.
|
|
local function incompatibleResult()
|
|
return { uploaded = 0, applied = 0, deleted = 0, failed = 0, unverified = 0, had_errors = false, incompatible = 1 }
|
|
end
|
|
|
|
--[[--
|
|
Full annotation exchange for one matched book. Blocking; callers run it from a
|
|
step machine or a scheduled task.
|
|
|
|
opts:
|
|
- client: BookOrbitApi instance
|
|
- state: BookOrbitState instance (book must already be matched)
|
|
- digest: partial md5 of the book file
|
|
- annotations: normalized annotation list (live or sidecar source)
|
|
- ann_max_datetime: max effective datetime of that list
|
|
- ann_signature: change signal for that list, recorded on a complete exchange
|
|
- captured_at: os.time() the list was read; dates notes rewritten without a timestamp
|
|
- sidecar_state: "known", "new" or nil, from BookOrbitSidecar.sidecarState
|
|
- apply_mode: "live" | "sidecar" | "skip"
|
|
- ui: ReaderUI (required for apply_mode "live")
|
|
- file: book file path (required for apply_mode "sidecar")
|
|
|
|
Returns a result table { uploaded, applied, deleted, failed, had_errors } or
|
|
nil, err for auth/network level failures. A position-incompatible copy returns
|
|
a result with `incompatible = 1` and exchanges nothing.
|
|
]]
|
|
function BookOrbitAnnotations.exchangeBook(opts)
|
|
local book = opts.state:getBook(opts.digest)
|
|
if not book then return nil, "unmatched" end
|
|
if book.positionIncompatible then return incompatibleResult() end
|
|
|
|
local keys = BookOrbitAnnotations.collectKeys(opts.key_annotations or opts.annotations)
|
|
local keys_complete = opts.keys_complete ~= false and #keys <= MAX_KEYS_PER_BOOK
|
|
if opts.file and opts.state.hasOtherCopy and opts.state:hasOtherCopy(opts.digest, opts.file) then
|
|
keys_complete = false
|
|
end
|
|
local sidecar_state
|
|
sidecar_state, keys_complete = BookOrbitSidecar.resolveExchangeState(opts.client, opts.sidecar_state, keys_complete)
|
|
if not keys_complete then keys = {} end
|
|
|
|
local device_now = os.date("%Y-%m-%d %H:%M:%S")
|
|
local watermark = BookOrbitAnnotations.readWatermark(book, device_now)
|
|
local captured_at = type(opts.captured_at) == "number" and os.date("%Y-%m-%d %H:%M:%S", opts.captured_at) or device_now
|
|
if captured_at > device_now then captured_at = device_now end
|
|
-- A list read before the notes were last remembered (a queued snapshot, a
|
|
-- sidecar older than the open book) is older than what the device already
|
|
-- reported. Its notes are not edits, and must not replace the record.
|
|
local older_list = book.annNotesAt ~= nil and captured_at < book.annNotesAt
|
|
local delta = {}
|
|
for _, annotation in ipairs(opts.annotations) do
|
|
local effective = annotation.datetimeUpdated or annotation.datetime
|
|
if not older_list and needsEditStamp(book, annotation, effective, watermark) then
|
|
table.insert(delta, editedAt(annotation, captured_at))
|
|
elseif effective > watermark and not BookOrbitAnnotations.isAppliedCopy(book, annotation, effective) then
|
|
table.insert(delta, annotation)
|
|
end
|
|
end
|
|
|
|
local result = { uploaded = 0, applied = 0, deleted = 0, failed = 0, unverified = 0, had_errors = false }
|
|
local response
|
|
local first_request = true
|
|
local upload_failed = false
|
|
-- Consumed through a head cursor: removing from the front of a Lua array
|
|
-- shifts every remaining element, which turns a long delta into O(n^2).
|
|
local cursor = 1
|
|
|
|
repeat
|
|
local chunk = {}
|
|
while cursor <= #delta and #chunk < UPLOAD_CHUNK do
|
|
table.insert(chunk, delta[cursor])
|
|
cursor = cursor + 1
|
|
end
|
|
local body, err = opts.client:exchangeAnnotations({
|
|
{
|
|
hash = opts.digest,
|
|
keys = first_request and keys or {},
|
|
keysComplete = first_request and keys_complete or false,
|
|
sidecarState = first_request and sidecar_state or nil,
|
|
changes = chunk,
|
|
},
|
|
})
|
|
if not body then
|
|
if err == 401 or err == 403 then return nil, "auth" end
|
|
if err == 404 then return nil, "unsupported_server" end
|
|
if type(err) ~= "number" then return nil, "network" end
|
|
logger.dbg("BookOrbit: annotation exchange failed:", err)
|
|
result.had_errors = true
|
|
upload_failed = true
|
|
break
|
|
end
|
|
for _, hash in ipairs(body.unmatched or {}) do
|
|
if hash == opts.digest then
|
|
opts.state:setUnmatched(opts.digest)
|
|
return nil, "unmatched"
|
|
end
|
|
end
|
|
for _, hash in ipairs(body.positionIncompatible or {}) do
|
|
if hash == opts.digest then
|
|
opts.state:markPositionIncompatible(opts.digest)
|
|
return incompatibleResult()
|
|
end
|
|
end
|
|
response = body.results and body.results[1] or nil
|
|
result.uploaded = result.uploaded + #chunk
|
|
first_request = false
|
|
until cursor > #delta
|
|
|
|
if not upload_failed then
|
|
BookOrbitAnnotations.advanceWatermark(book, opts.ann_max_datetime, device_now)
|
|
pruneApplied(book)
|
|
if not older_list then BookOrbitAnnotations.rememberNotes(book, opts.annotations, captured_at) end
|
|
end
|
|
|
|
-- Recorded only when the complete key set went out and nothing was left
|
|
-- undelivered: the skip rule reads this stamp, so anything the server will
|
|
-- re-send must keep the next exchange mandatory.
|
|
local function rememberComplete()
|
|
if upload_failed or result.had_errors or result.failed > 0 or result.unverified > 0 then return end
|
|
if not keys_complete then return end
|
|
BookOrbitAnnotations.rememberExchanged(book, opts.ann_signature)
|
|
end
|
|
|
|
if opts.apply_mode == "skip" then
|
|
if response and hasPending(response.toApply) then
|
|
result.remote_pending = {
|
|
to_apply = response.toApply,
|
|
more = response.more == true,
|
|
}
|
|
else
|
|
rememberComplete()
|
|
end
|
|
return result
|
|
end
|
|
|
|
local rounds = 0
|
|
local pull_complete = true
|
|
while response and hasPending(response.toApply) do
|
|
if rounds >= MAX_PULL_ROUNDS then
|
|
pull_complete = false
|
|
break
|
|
end
|
|
rounds = rounds + 1
|
|
local applied, deleted, touched, deleted_touched, stamps
|
|
if opts.apply_mode == "live" and opts.ui and opts.ui.document then
|
|
applied, deleted, touched, deleted_touched, stamps = BookOrbitAnnotations.applyLive(opts.ui, response.toApply)
|
|
elseif opts.apply_mode == "sidecar" and opts.file then
|
|
applied, deleted, touched, deleted_touched, stamps = BookOrbitAnnotations.applySidecar(opts.file, response.toApply)
|
|
else
|
|
pull_complete = false
|
|
break
|
|
end
|
|
|
|
deleted_touched = deleted_touched or 0
|
|
result.applied = result.applied + math.max(0, touched - deleted_touched)
|
|
result.deleted = result.deleted + deleted_touched
|
|
for _, entry in ipairs(applied) do
|
|
if entry.status == "failed" then result.failed = result.failed + 1 end
|
|
if entry.status == "applied" and entry.verified == false then result.unverified = result.unverified + 1 end
|
|
end
|
|
|
|
local ack_body, ack_err = opts.client:exchangeAck({
|
|
{ hash = opts.digest, applied = applied, deleted = deleted },
|
|
})
|
|
if not ack_body then
|
|
logger.dbg("BookOrbit: annotation exchange ack failed:", ack_err)
|
|
result.had_errors = true
|
|
break
|
|
end
|
|
BookOrbitAnnotations.rememberApplied(book, stamps)
|
|
|
|
if not response.more then break end
|
|
local body, err = opts.client:exchangeAnnotations({
|
|
{ hash = opts.digest, keys = {}, keysComplete = false, changes = {} },
|
|
})
|
|
if not body then
|
|
logger.dbg("BookOrbit: annotation exchange follow-up failed:", err)
|
|
result.had_errors = true
|
|
break
|
|
end
|
|
response = body.results and body.results[1] or nil
|
|
end
|
|
|
|
if pull_complete then rememberComplete() end
|
|
-- Notes the server just applied are the device's notes now; without this
|
|
-- the next exchange would read them as local edits and send them back.
|
|
if not upload_failed and result.applied > 0 and opts.apply_mode == "live" and opts.ui and opts.ui.annotation then
|
|
BookOrbitAnnotations.rememberNotes(book, (BookOrbitSidecar.normalizeAnnotations(opts.ui.annotation.annotations)), os.date("%Y-%m-%d %H:%M:%S"))
|
|
end
|
|
return result
|
|
end
|
|
|
|
-- Convenience wrapper for the open book: collects live annotations, exchanges
|
|
-- and applies in place. Used by the reader-ready pull and the manual sync.
|
|
function BookOrbitAnnotations.exchangeOpenBook(opts)
|
|
local ui = opts.ui
|
|
if not ui or not ui.document then return nil, "no_document" end
|
|
local annotations, ann_max, signature =
|
|
BookOrbitSidecar.normalizeAnnotations(ui.annotation and ui.annotation.annotations)
|
|
return BookOrbitAnnotations.exchangeBook({
|
|
client = opts.client,
|
|
state = opts.state,
|
|
digest = opts.digest,
|
|
annotations = annotations,
|
|
ann_max_datetime = ann_max,
|
|
ann_signature = signature,
|
|
sidecar_state = opts.sidecar_state,
|
|
apply_mode = "live",
|
|
ui = ui,
|
|
file = ui.document.file,
|
|
})
|
|
end
|
|
|
|
return BookOrbitAnnotations
|